Remove canary requirement for CodeQL Methods Usage panel

This upgrades this package to fix tests on Windows. It seems like the
extraction of the VS Code archive is failing, which is fixed in the
newest version.

.github/workflows/cli-test.yml

@@ -60,7 +60,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'

.github/workflows/main.yml

@@ -20,7 +20,7 @@ jobs:
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
@@ -62,7 +62,7 @@ jobs:
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
@@ -110,7 +110,7 @@ jobs:
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
@@ -149,7 +149,7 @@ jobs:
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
@@ -183,7 +183,7 @@ jobs:
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
@@ -251,7 +251,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'

.github/workflows/release.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
 

docs/releasing.md

@@ -1,80 +1,62 @@
 # Releasing (write access required)
 
-1. Run the ["Run CLI tests" workflow](https://github.com/github/vscode-codeql/actions/workflows/cli-test.yml) and make sure the tests are green. If there were no merges between the time the workflow ran (it runs daily), and the release, you can skip this step.  
+1. Determine the new version number. We default to increasing the patch version number, but make our own judgement about whether a change is big enough to warrant a minor version bump. Common reasons for a minor bump could include:
+    * Making substantial new features available to all users. This can include lifting a feature flag.
+    * Breakage in compatibility with recent versions of the CLI.
+    * Minimum required version of VS Code is increased.
+    * New telemetry events are added.
+    * Deprecation or removal of commands.
+    * Accumulation of many changes, none of which are individually big enough to warrant a minor bump, but which together are. This does not include changes which are purely internal to the extension, such as refactoring, or which are only available behind a feature flag.
+1. Create a release branch named after the new version (e.g. `v1.3.6`):
+    * For a regular scheduled release this branch will be based on latest `main`.
+        * Make sure your local copy of `main` is up to date so you are including all changes.
+    * To do a minimal bug-fix release, base the release branch on the tag from the most recent release and then add only the changes you want to release.
+        * Choose this option if you want to release a specific set of changes (e.g. a bug fix) and don't want to incur extra risk by including other changes that have been merged to the `main` branch.
+
+        ```bash
+        git checkout -b <new_release_branch> <previous_release_tag>
+        ```
+
+1. Run the ["Run CLI tests" workflow](https://github.com/github/vscode-codeql/actions/workflows/cli-test.yml) and make sure the tests are green.
+    * You can skip this step if you are releasing from `main` and there were no merges since the most recent daily scheduled run of this workflow.
 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
-    * Go through all recent PRs and make sure they are properly accounted for.
+    * Go through PRs that have been merged since the previous release and make sure they are properly accounted for.
     * Make sure all changelog entries have links back to their PR(s) if appropriate.
-    * For picking the new version number, we default to increasing the patch version number, but make our own judgement about whether a change is big enough to warrant a minor version bump. Common reasons for a minor bump could include:
-      * Making substantial new features available to all users. This can include lifting a feature flag.
-      * Breakage in compatibility with recent versions of the CLI.
-      * Minimum required version of VS Code is increased.
-      * New telemetry events are added.
-      * Deprecation or removal of commands.
-      * Accumulation of many changes, none of which are individually big enough to warrant a minor bump, but which together are. This does not include changes which are purely internal to the extension, such as refactoring, or which are only available behind a feature flag.
-1. Double-check that the node version we're using matches the one used for VS Code. See the [Node.js version instructions](./node-version.md) for more information.
 1. Double-check that the extension `package.json` and `package-lock.json` have the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
-1. Create a PR for this release:
-    * This PR will contain any missing bits from steps 1, 2 and 3. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
-    * Create a new branch for the release named after the new version. For example: `v1.3.6`
-    * Create a new commit with a message the same as the branch name.
-    * Create a PR for this branch.
-    * Wait for the PR to be merged into `main`
-1. Switch to `main` branch and pull latest changes
-1. Lock the `main` branch.
-    * Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
-    * Select "Lock branch"
-    * Click "Save changes"
-1. Ensure that no PRs have been merged since the release PR that you merged. If there were, you might need to unlock `main` temporarily and update the CHANGELOG again.
-1. Build the extension `npm run build` and install it on your VS Code using "Install from VSIX".
+1. Commit any changes made during steps 4 and 5 with a commit message the same as the branch name (e.g. `v1.3.6`).
+1. Open a PR for this release.
+    * The PR diff should contain:
+        * Any missing bits from steps 4 and 5. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
+        * If releasing from a branch other than `main`, this PR will also contain the extension changes being released.
+1. Build the extension using `npm run build` and install it on your VS Code using "Install from VSIX".
 1. Go through [our test plan](./test-plan.md) to ensure that the extension is working as expected.
-1. Switch to `main` and add a new tag on the `main` branch with your new version (named after the release), e.g.
+1. Create a new tag on the release branch with your new version (named after the release), e.g.
 
     ```bash
-    git checkout main
     git tag v1.3.6
     ```
 
-   If you've accidentally created a badly named tag, you can delete it via
-
-   ```bash
-   git tag -d badly-named-tag
-   ```
-
-1. Unlock the main branch
-    * Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
-    * Deselect "Lock branch"
-    * Click "Save changes"
+1. Merge the release PR into `main`.
+    * If there are conflicts in the changelog, make sure to place any new changelog entries at the top, above the section for the current release, as these new entries are not part of the current release and should be placed in the "unreleased" section.
+    * The release PR must be merged before pushing the tag to ensure that we always release a commit that is present on the `main` branch. It's not required that the commit is the head of the `main` branch, but there should be no chance of a future release accidentally not including changes from this release.
 1. Push the new tag up:
 
-   a. If you're using a fork of the repo:
-
-    ```bash
-    git push upstream refs/tags/v1.3.6
-    ```
-
-   b. If you're working straight in this repo:
-
     ```bash
     git push origin refs/tags/v1.3.6
     ```
 
-   This will trigger [a release build](https://github.com/github/vscode-codeql/releases) on Actions.
-
-    * **IMPORTANT** Make sure you are on the `main` branch and your local checkout is fully updated when you add the tag.
-    * If you accidentally add the tag to the wrong ref, you can just force push it to the right one later.
-1. Monitor the status of the release build in the `Release` workflow in the Actions tab.
+1. Find the [Release](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease) workflow run that was just triggered by pushing the tag, and monitor the status of the release build.
     * DO NOT approve the "publish" stages of the workflow yet.
 1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
 1. Unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
    or look at the source if there's any doubt the right code is being shipped.
 1. Install the `.vsix` file into your vscode IDE and ensure the extension can load properly. Run a single command (like run query, or add database).
-1. Go to the actions tab of the vscode-codeql repository and select the [Release workflow](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease).
+1. Approve the deployments of the [Release](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease) workflow run. This will automatically publish to Open VSX and VS Code Marketplace.
     * If there is an authentication failure when publishing, be sure to check that the authentication keys haven't expired. See below.
-1. Approve the deployments of the correct Release workflow. This will automatically publish to Open VSX and VS Code Marketplace.
-1. Go to the draft GitHub release in [the releases tab of the repository](https://github.com/github/vscode-codeql/releases), click 'Edit', add some summary description, and publish it.
-1. Confirm the new release is marked as the latest release at <https://github.com/github/vscode-codeql/releases>.
+1. Go to the draft GitHub release in [the releases page](https://github.com/github/vscode-codeql/releases), click 'Edit', add some summary description, and publish it.
+1. Confirm the new release is marked as the latest release.
 1. If documentation changes need to be published, notify documentation team that release has been made.
-1. Review and merge the version bump PR that is automatically created by Actions.
+1. Review and merge the version bump PR that is automatically created by the Release workflow.
 
 ## Secrets and authentication for publishing
 

docs/test-plan.md

@@ -151,22 +151,27 @@ Run one of the above MRVAs, but cancel it from within VS Code:
 2. Open the Model Editor with the "CodeQL: Open CodeQL Model Editor" command from the command palette.
    - Check that the editor loads and shows methods to model.
    - Check that methods are grouped per library (e.g. `rocksdbjni@7.7.3` or `asm@6.0`)
-   - Check that the "Open database" link works.
+   - Check that the "Open source" link works.
+   - Check that the 'View' button works and the Method Usage panel highlight the correct method and usage
+   - Check that the Method Modeling panel shows the correct method and modeling state
 
 #### Test Case 2: Model methods
 
 1. Expand one of the libraries.
    - Change the model type and check that the other dropdowns change.
+   - Check that the method modeling panel updates accordingly
 2. Save the modeled methods.
 3. Click "Open extension pack"
    - Check that the file explorer opens a directory with a "models" directory
 4. Open the ".model.yml" file corresponding to the library that was changed.
-   - Check that the file contrains the entries that was modeled.
+   - Check that the file contains entries for the methods that were modeled.
 
 #### Test Case 3: Model with AI
 
 Note that this test requires the feature flag: `codeQL.model.llmGeneration`
 
+A package that the AI normally gives models for is `javax.servlet-api` from the `jhy/jsoup` repository.
+
 1. Click "Model with AI".
    - Check that rows change to "Thinking".
    - Check that results come back and rows get filled out.
@@ -189,9 +194,28 @@ Are there any components that are not showing up?
 
 ## Optional Test Cases
 
-These are mostly aimed at MRVA, but some of them are also applicable to non-MRVA queries.
+### Modeling Flow
 
-### Selecting repositories to run on
+1. Check that a method can have multiple models:
+   - Add a couple of new models for one method in the model editor
+   - Save and check that the modeling file (use the 'open extension pack' button to open it) shows multiple methods
+   - Check that the Method Modeling Panel shows the correct multiple models
+   - Check that you can browse through different models in the Method Modeling Panel
+   - Check that a 'duplicated classification' error appears in both model editor and modeling panel when a duplicate modeling occurs
+   - Check that a 'conflicting classification' error appears when a neutral model type is paired with a model of the same kind
+   - Check that clicking on the error highlights the correct modeling in both the editor and the modeling panel
+2. Check the Method Usage Panel
+   - Check that the Method Usage Panel opens and jumps to the correct usage when clicking on 'View' in the model editor
+   - Check that the first and following usages are opening when clicking on a usage
+   - Check that the usage icon color turns green when saving a newly modeled method
+   - Check that the usage icon color turns red when saving a newly unmodeld method
+3. Check the Method Modeling Panel
+   - Check that the 'Start modeling' button opens a new model editor
+   - Check that it refreshes the blank state when a model editor is opened/closed
+   - Check that when modeling in the editor the modeling panel updates accordingly
+   - Check that when modeling in the modeling panel the model editor updates accordingly
+
+### Selecting MRVA repositories to run on
 
 #### Test case 1: Running a query on a single repository
 
@@ -221,7 +245,7 @@ These are mostly aimed at MRVA, but some of them are also applicable to non-MRVA
    4. The org contains private repositories that are inaccessible
 2. The org does not exist
 
-### Using different types of controller repos
+### Using different types of controller repos for MRVA
 
 #### Test case 1: Running a query when the controller repository is public
 

docs/testing.md

@@ -27,7 +27,7 @@ Pre-requisites:
 Then, from the `extensions/ql-vscode` directory, use the appropriate command to run the tests:
 
 * Unit tests: `npm run test:unit`
-* View Tests: `npm test:view`
+* View Tests: `npm run test:view`
 * VSCode integration tests: `npm run test:vscode-integration`
 
 #### Running CLI integration tests from the terminal
@@ -48,8 +48,8 @@ Alternatively, you can run the tests inside of VSCode. There are several VSCode
 
 You will need to run tests using a task from inside of VS Code, under the "Run and Debug" view:
 
-* Unit tests: run the _Launch Unit Tests - React_ task
-* View Tests: run the _Launch Unit Tests_ task
+* Unit tests: run the _Launch Unit Tests_ task
+* View Tests: run the _Launch Unit Tests - React_ task
 * VSCode integration tests: run the _Launch Unit Tests - No Workspace_ and _Launch Unit Tests - Minimal Workspace_ tasks
 
 #### Running CLI integration tests from VSCode

extensions/ql-vscode/.eslintrc.js

@@ -50,6 +50,7 @@ const baseConfig = {
     "@typescript-eslint/no-throw-literal": "error",
     "no-useless-escape": 0,
     camelcase: "off",
+    curly: ["error", "all"],
     "escompat/no-regexp-lookbehind": "off",
     "etc/no-implicit-any-catch": "error",
     "filenames/match-regex": "off",
@@ -71,6 +72,7 @@ const baseConfig = {
     "no-shadow": "off",
     "github/array-foreach": "off",
     "github/no-then": "off",
+    "react/jsx-key": ["error", { checkFragmentShorthand: true }],
   },
 };
 

extensions/ql-vscode/.storybook/main.ts

@@ -6,6 +6,7 @@ const config: StorybookConfig = {
     "@storybook/addon-links",
     "@storybook/addon-essentials",
     "@storybook/addon-interactions",
+    "@storybook/addon-a11y",
     "./vscode-theme-addon/preset.ts",
   ],
   framework: {

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,43 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.11.0 - 13 December 2023
+
+- Add a new method modeling panel to classify methods as sources/sinks/summaries while in the context of the source code. [#3128](https://github.com/github/vscode-codeql/pull/3128)
+- Adds the ability to add multiple classifications per method in the CodeQL Model Editor. [#3128](https://github.com/github/vscode-codeql/pull/3128)
+- Switch add and delete button positions in the CodeQL Model Editor. [#3123](https://github.com/github/vscode-codeql/pull/3123)
+- Add a prompt to the "Quick query" command to encourage users in single-folder workspaces to use "Create query" instead. [#3082](https://github.com/github/vscode-codeql/pull/3082)
+- Remove support for CodeQL CLI versions older than 2.11.6. [#3087](https://github.com/github/vscode-codeql/pull/3087)
+- Preserve focus on results viewer when showing a location in a file. [#3088](https://github.com/github/vscode-codeql/pull/3088)
+- The `dataflowtracking` and `tainttracking` snippets expand to the new module-based interface. [#3091](https://github.com/github/vscode-codeql/pull/3091)
+- The compare view will now show a loading message while the results are loading. [#3107](https://github.com/github/vscode-codeql/pull/3107)
+- Make top-banner of the model editor sticky [#3120](https://github.com/github/vscode-codeql/pull/3120)
+
+## 1.10.0 - 16 November 2023
+
+- Add new CodeQL views for managing databases and queries:
+  1. A queries panel that shows all queries in your workspace. It allows you to view, create, and run queries in one place.
+  2. A language selector, which allows you to quickly filter databases and queries by language.
+  
+  For more information, see the [documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/analyzing-your-projects/#filtering-databases-and-queries-by-language).
+- When adding a CodeQL database, we no longer add the database source folder to the workspace by default (since this caused bugs in single-folder workspaces). [#3047](https://github.com/github/vscode-codeql/pull/3047)
+  - You can manually add individual database source folders to the workspace with the "Add Database Source to Workspace" right-click command in the databases view.
+  - To restore the old behavior of adding all database source folders by default, set the `codeQL.addingDatabases.addDatabaseSourceToWorkspace` setting to `true`.
+- Rename the `codeQL.databaseDownload.allowHttp` setting to `codeQL.addingDatabases.allowHttp`, so that database-related settings are grouped together in the Settings UI. [#3047](https://github.com/github/vscode-codeql/pull/3047) & [#3069](https://github.com/github/vscode-codeql/pull/3069)
+- The "Sort by Language" action in the databases view now sorts by name within each language. [#3055](https://github.com/github/vscode-codeql/pull/3055)
+
+## 1.9.4 - 6 November 2023
+
+No user facing changes.
+
+## 1.9.3 - 26 October 2023
+
+- Sorted result set filenames now include a hash of the result set name instead of the full name. [#2955](https://github.com/github/vscode-codeql/pull/2955)
+- The "Install Pack Dependencies" will now only list CodeQL packs located in the workspace. [#2960](https://github.com/github/vscode-codeql/pull/2960)
+- Fix a bug where the "View Query Log" action for a query history item was not working. [#2984](https://github.com/github/vscode-codeql/pull/2984)
+- Add a command to sort items in the databases view by language. [#2993](https://github.com/github/vscode-codeql/pull/2993)
+- Fix not being able to open the results directory or evaluator log for a cancelled local query run. [#2996](https://github.com/github/vscode-codeql/pull/2996)
+- Fix empty row in alert path when the SARIF location was empty. [#3018](https://github.com/github/vscode-codeql/pull/3018)
+
 ## 1.9.2 - 12 October 2023
 
 - Fix a bug where the query to Find Definitions in database source files would not be cancelled appropriately. [#2885](https://github.com/github/vscode-codeql/pull/2885)
@@ -7,6 +45,7 @@
 - Increase the required version of VS Code to 1.82.0. [#2877](https://github.com/github/vscode-codeql/pull/2877)
 - Fix a bug where the query server was restarted twice after configuration changes. [#2884](https://github.com/github/vscode-codeql/pull/2884).
 - Add support for the `telemetry.telemetryLevel` setting. For more information, see the [telemetry documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code). [#2824](https://github.com/github/vscode-codeql/pull/2824).
+- Add a "CodeQL: Trim Cache" command that clears the evaluation cache of a database except for predicates annotated with the `cached` keyword. Its purpose is to get accurate performance measurements when tuning the final stage of a query, like a data-flow configuration. This is equivalent to the `codeql database cleanup --mode=normal` CLI command. In contrast, the existing "CodeQL: Clear Cache" command clears the entire cache. CodeQL CLI v2.15.1 or later is required. [#2928](https://github.com/github/vscode-codeql/pull/2928)
 - Fix syntax highlighting directly after import statements with instantiation arguments. [#2792](https://github.com/github/vscode-codeql/pull/2792)
 - The `debug.saveBeforeStart` setting is now respected when running variant analyses. [#2950](https://github.com/github/vscode-codeql/pull/2950)
 - The 'open database' button of the model editor was renamed to 'open source'. Also, it's now only available if the source archive is available as a workspace folder. [#2945](https://github.com/github/vscode-codeql/pull/2945)
@@ -20,7 +59,7 @@
 
 ## 1.9.0 - 19 September 2023
 
-- Release the [CodeQL model editor](https://codeql.github.com/docs/codeql/codeql-for-visual-studio-code/using-the-codeql-model-editor) to create CodeQL model packs for Java frameworks. Open the editor using the "CodeQL: Open CodeQL Model Editor (Beta)" command. [#2823](https://github.com/github/vscode-codeql/pull/2823)
+- Release the [CodeQL model editor](https://codeql.github.com/docs/codeql-for-visual-studio-code/using-the-codeql-model-editor/) to create CodeQL model packs for Java frameworks. Open the editor using the "CodeQL: Open CodeQL Model Editor (Beta)" command. [#2823](https://github.com/github/vscode-codeql/pull/2823)
 
 ## 1.8.12 - 11 September 2023
 

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.9.2",
+  "version": "1.11.0",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -21,7 +21,8 @@
     "Programming Languages"
   ],
   "extensionDependencies": [
-    "hbenl.vscode-test-explorer"
+    "hbenl.vscode-test-explorer",
+    "vscode.git"
   ],
   "capabilities": {
     "untrustedWorkspaces": {
@@ -34,31 +35,12 @@
     }
   },
   "activationEvents": [
-    "onLanguage:ql",
-    "onLanguage:ql-summary",
-    "onView:codeQLQueries",
-    "onView:codeQLDatabases",
-    "onView:codeQLVariantAnalysisRepositories",
-    "onView:codeQLQueryHistory",
-    "onView:codeQLAstViewer",
-    "onView:codeQLEvalLogViewer",
     "onView:test-explorer",
-    "onCommand:codeQL.checkForUpdatesToCLI",
-    "onCommand:codeQL.authenticateToGitHub",
-    "onCommand:codeQL.viewAst",
-    "onCommand:codeQL.viewCfg",
-    "onCommand:codeQL.openReferencedFile",
-    "onCommand:codeQL.previewQueryHelp",
-    "onCommand:codeQL.chooseDatabaseFolder",
-    "onCommand:codeQL.chooseDatabaseArchive",
-    "onCommand:codeQL.chooseDatabaseInternet",
-    "onCommand:codeQL.chooseDatabaseGithub",
-    "onCommand:codeQL.quickQuery",
-    "onCommand:codeQL.restartQueryServer",
     "onWebviewPanel:resultsView",
     "onWebviewPanel:codeQL.variantAnalysis",
     "onWebviewPanel:codeQL.dataFlowPaths",
-    "onFileSystem:codeql-zip-archive"
+    "onFileSystem:codeql-zip-archive",
+    "workspaceContains:.git"
   ],
   "main": "./out/extension",
   "files": [
@@ -392,13 +374,23 @@
       },
       {
         "type": "object",
-        "title": "Downloading databases",
+        "title": "Adding databases",
         "order": 6,
         "properties": {
-          "codeQL.databaseDownload.allowHttp": {
+          "codeQL.addingDatabases.allowHttp": {
             "type": "boolean",
             "default": false,
-            "description": "Allow database to be downloaded via HTTP. Warning: enabling this option will allow downloading from insecure servers."
+            "description": "Allow databases to be downloaded via HTTP. Warning: enabling this option will allow downloading from insecure servers."
+          },
+          "codeQL.databaseDownload.allowHttp": {
+            "type": "boolean",
+            "markdownDeprecationMessage": "**Deprecated**: Please use `#codeQL.addingDatabases.allowHttp#` instead.",
+            "deprecationMessage": "Deprecated: Please use codeQL.addingDatabases.allowHttp instead."
+          },
+          "codeQL.addingDatabases.addDatabaseSourceToWorkspace": {
+            "type": "boolean",
+            "default": false,
+            "markdownDescription": "When adding a CodeQL database, automatically add the database's source folder as a workspace folder. Warning: enabling this option in a single-folder workspace will cause the workspace to reload as a [multi-root workspace](https://code.visualstudio.com/docs/editor/multi-root-workspaces). This may cause query history and database lists to be reset."
           }
         }
       },
@@ -429,8 +421,46 @@
       },
       {
         "type": "object",
-        "title": "Log insights",
+        "title": "GitHub Databases",
         "order": 8,
+        "properties": {
+          "codeQL.githubDatabase.enable": {
+            "type": "boolean",
+            "default": false,
+            "markdownDescription": "Enable automatic detection of GitHub databases."
+          },
+          "codeQL.githubDatabase.download": {
+            "type": "string",
+            "default": "ask",
+            "enum": [
+              "ask",
+              "never"
+            ],
+            "enumDescriptions": [
+              "Ask to download a GitHub database when a workspace is opened.",
+              "Never download a GitHub databases when a workspace is opened."
+            ],
+            "description": "Ask to download a GitHub database when a workspace is opened."
+          },
+          "codeQL.githubDatabase.update": {
+            "type": "string",
+            "default": "ask",
+            "enum": [
+              "ask",
+              "never"
+            ],
+            "enumDescriptions": [
+              "Ask to download an updated GitHub database when a new version is available.",
+              "Never download an updated GitHub database when a new version is available."
+            ],
+            "description": "Ask to download an updated GitHub database when a new version is available."
+          }
+        }
+      },
+      {
+        "type": "object",
+        "title": "Log insights",
+        "order": 9,
         "properties": {
           "codeQL.logInsights.joinOrderWarningThreshold": {
             "type": "number",
@@ -444,7 +474,7 @@
       {
         "type": "object",
         "title": "Telemetry",
-        "order": 9,
+        "order": 10,
         "properties": {
           "codeQL.telemetry.enableTelemetry": {
             "type": "boolean",
@@ -581,6 +611,10 @@
         "command": "codeQL.copyVersion",
         "title": "CodeQL: Copy Version Information"
       },
+      {
+        "command": "codeQLLanguageSelection.setSelectedItem",
+        "title": "Select"
+      },
       {
         "command": "codeQLQueries.runLocalQueryFromQueriesPanel",
         "title": "Run local query",
@@ -732,6 +766,10 @@
         "command": "codeQL.clearCache",
         "title": "CodeQL: Clear Cache"
       },
+      {
+        "command": "codeQL.trimCache",
+        "title": "CodeQL: Trim Cache"
+      },
       {
         "command": "codeQL.installPackDependencies",
         "title": "CodeQL: Install Pack Dependencies"
@@ -764,78 +802,6 @@
         "command": "codeQLDatabases.addDatabaseSource",
         "title": "Add Database Source to Workspace"
       },
-      {
-        "command": "codeQLDatabases.displayAllLanguages",
-        "title": "All languages"
-      },
-      {
-        "command": "codeQLDatabases.displayAllLanguagesSelected",
-        "title": "All languages (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displayCpp",
-        "title": "C/C++"
-      },
-      {
-        "command": "codeQLDatabases.displayCppSelected",
-        "title": "C/C++ (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displayCsharp",
-        "title": "C#"
-      },
-      {
-        "command": "codeQLDatabases.displayCsharpSelected",
-        "title": "C# (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displayGo",
-        "title": "Go"
-      },
-      {
-        "command": "codeQLDatabases.displayGoSelected",
-        "title": "Go (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displayJava",
-        "title": "Java/Kotlin"
-      },
-      {
-        "command": "codeQLDatabases.displayJavaSelected",
-        "title": "Java/Kotlin (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displayJavascript",
-        "title": "JavaScript/TypeScript"
-      },
-      {
-        "command": "codeQLDatabases.displayJavascriptSelected",
-        "title": "JavaScript/TypeScript (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displayPython",
-        "title": "Python"
-      },
-      {
-        "command": "codeQLDatabases.displayPythonSelected",
-        "title": "Python (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displayRuby",
-        "title": "Ruby"
-      },
-      {
-        "command": "codeQLDatabases.displayRubySelected",
-        "title": "Ruby (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displaySwift",
-        "title": "Swift"
-      },
-      {
-        "command": "codeQLDatabases.displaySwiftSelected",
-        "title": "Swift (selected)"
-      },
       {
         "command": "codeQL.chooseDatabaseFolder",
         "title": "CodeQL: Choose Database from Folder"
@@ -856,6 +822,10 @@
         "command": "codeQLDatabases.sortByName",
         "title": "Sort by Name"
       },
+      {
+        "command": "codeQLDatabases.sortByLanguage",
+        "title": "Sort by Language"
+      },
       {
         "command": "codeQLDatabases.sortByDateAdded",
         "title": "Sort by Date Added"
@@ -1082,14 +1052,14 @@
           "group": "1_databases@0"
         },
         {
-          "command": "codeQLDatabases.sortByDateAdded",
+          "command": "codeQLDatabases.sortByLanguage",
           "when": "view == codeQLDatabases",
           "group": "1_databases@1"
         },
         {
-          "submenu": "codeQLDatabases.languages",
-          "when": "view == codeQLDatabases && config.codeQL.canary && config.codeQL.showLanguageFilter",
-          "group": "2_databases@0"
+          "command": "codeQLDatabases.sortByDateAdded",
+          "when": "view == codeQLDatabases",
+          "group": "1_databases@2"
         },
         {
           "command": "codeQLQueries.createQuery",
@@ -1163,6 +1133,11 @@
           "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/",
           "group": "2_qlContextMenu@1"
         },
+        {
+          "command": "codeQLLanguageSelection.setSelectedItem",
+          "when": "view == codeQLLanguageSelection && viewItem =~ /canBeSelected/",
+          "group": "inline"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "group": "inline",
@@ -1511,6 +1486,10 @@
         {
           "command": "codeQL.openModelEditor"
         },
+        {
+          "command": "codeQLLanguageSelection.setSelectedItem",
+          "when": "false"
+        },
         {
           "command": "codeQLQueries.runLocalQueryContextMenu",
           "when": "false"
@@ -1579,6 +1558,10 @@
           "command": "codeQLDatabases.sortByName",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.sortByLanguage",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.sortByDateAdded",
           "when": "false"
@@ -1611,78 +1594,6 @@
           "command": "codeQLDatabases.upgradeDatabase",
           "when": "false"
         },
-        {
-          "command": "codeQLDatabases.displayAllLanguages",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayAllLanguagesSelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayCpp",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayCppSelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayCsharp",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayCsharpSelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayGo",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayGoSelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayJava",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayJavaSelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayJavascript",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayJavascriptSelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayPython",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayPythonSelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayRuby",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayRubySelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displaySwift",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displaySwiftSelected",
-          "when": "false"
-        },
         {
           "command": "codeQLQueryHistory.openQueryContextMenu",
           "when": "false"
@@ -1815,10 +1726,6 @@
           "command": "codeQL.mockGitHubApiServer.unloadScenario",
           "when": "config.codeQL.mockGitHubApiServer.enabled && codeQL.mockGitHubApiServer.scenarioLoaded"
         },
-        {
-          "command": "codeQL.createQuery",
-          "when": "config.codeQL.codespacesTemplate || config.codeQL.canary && config.codeQL.queriesPanel"
-        },
         {
           "command": "codeQLTests.acceptOutputContextTestItem",
           "when": "false"
@@ -1826,6 +1733,10 @@
         {
           "command": "codeQL.gotoQLContextEditor",
           "when": "false"
+        },
+        {
+          "command": "codeQL.trimCache",
+          "when": "codeql.supportsTrimCache"
         }
       ],
       "editor/context": [
@@ -1877,88 +1788,8 @@
           "command": "codeQL.gotoQLContextEditor",
           "when": "editorLangId == ql-summary && config.codeQL.canary"
         }
-      ],
-      "codeQLDatabases.languages": [
-        {
-          "command": "codeQLDatabases.displayAllLanguages",
-          "when": "codeQLDatabases.languageFilter"
-        },
-        {
-          "command": "codeQLDatabases.displayAllLanguagesSelected",
-          "when": "!codeQLDatabases.languageFilter"
-        },
-        {
-          "command": "codeQLDatabases.displayCpp",
-          "when": "codeQLDatabases.languageFilter != cpp"
-        },
-        {
-          "command": "codeQLDatabases.displayCppSelected",
-          "when": "codeQLDatabases.languageFilter == cpp"
-        },
-        {
-          "command": "codeQLDatabases.displayCsharp",
-          "when": "codeQLDatabases.languageFilter != csharp"
-        },
-        {
-          "command": "codeQLDatabases.displayCsharpSelected",
-          "when": "codeQLDatabases.languageFilter == csharp"
-        },
-        {
-          "command": "codeQLDatabases.displayGo",
-          "when": "codeQLDatabases.languageFilter != go"
-        },
-        {
-          "command": "codeQLDatabases.displayGoSelected",
-          "when": "codeQLDatabases.languageFilter == go"
-        },
-        {
-          "command": "codeQLDatabases.displayJava",
-          "when": "codeQLDatabases.languageFilter != java"
-        },
-        {
-          "command": "codeQLDatabases.displayJavaSelected",
-          "when": "codeQLDatabases.languageFilter == java"
-        },
-        {
-          "command": "codeQLDatabases.displayJavascript",
-          "when": "codeQLDatabases.languageFilter != javascript"
-        },
-        {
-          "command": "codeQLDatabases.displayJavascriptSelected",
-          "when": "codeQLDatabases.languageFilter == javascript"
-        },
-        {
-          "command": "codeQLDatabases.displayPython",
-          "when": "codeQLDatabases.languageFilter != python"
-        },
-        {
-          "command": "codeQLDatabases.displayPythonSelected",
-          "when": "codeQLDatabases.languageFilter == python"
-        },
-        {
-          "command": "codeQLDatabases.displayRuby",
-          "when": "codeQLDatabases.languageFilter != ruby"
-        },
-        {
-          "command": "codeQLDatabases.displayRubySelected",
-          "when": "codeQLDatabases.languageFilter == ruby"
-        },
-        {
-          "command": "codeQLDatabases.displaySwift",
-          "when": "codeQLDatabases.languageFilter != swift"
-        },
-        {
-          "command": "codeQLDatabases.displaySwiftSelected",
-          "when": "codeQLDatabases.languageFilter == swift"
-        }
       ]
     },
-    "submenus": [
-      {
-        "id": "codeQLDatabases.languages",
-        "label": "Languages"
-      }
-    ],
     "viewsContainers": {
       "activitybar": [
         {
@@ -1977,14 +1808,17 @@
     },
     "views": {
       "ql-container": [
+        {
+          "id": "codeQLLanguageSelection",
+          "name": "Language"
+        },
         {
           "id": "codeQLDatabases",
           "name": "Databases"
         },
         {
           "id": "codeQLQueries",
-          "name": "Queries",
-          "when": "config.codeQL.canary && config.codeQL.queriesPanel"
+          "name": "Queries"
         },
         {
           "id": "codeQLVariantAnalysisRepositories",
@@ -2006,15 +1840,14 @@
         {
           "id": "codeQLMethodModeling",
           "type": "webview",
-          "name": "CodeQL Method Modeling",
-          "when": "config.codeQL.canary"
+          "name": "CodeQL Method Modeling"
         }
       ],
       "codeql-methods-usage": [
         {
           "id": "codeQLMethodsUsage",
           "name": "CodeQL Methods Usage",
-          "when": "config.codeQL.canary && codeql.modelEditorOpen"
+          "when": "codeql.modelEditorOpen"
         }
       ]
     },
@@ -2094,7 +1927,7 @@
     "fs-extra": "^11.1.1",
     "immutable": "^4.0.0",
     "js-yaml": "^4.1.0",
-    "msw": "^0.0.0-fetch.rc-20",
+    "msw": "^2.0.0",
     "nanoid": "^5.0.1",
     "node-fetch": "^2.6.7",
     "p-queue": "^7.4.1",
@@ -2125,6 +1958,7 @@
     "@faker-js/faker": "^8.0.2",
     "@github/markdownlint-github": "^0.3.0",
     "@octokit/plugin-throttling": "^8.0.0",
+    "@storybook/addon-a11y": "^7.4.6",
     "@storybook/addon-actions": "^7.1.0",
     "@storybook/addon-essentials": "^7.1.0",
     "@storybook/addon-interactions": "^7.1.0",
@@ -2171,7 +2005,7 @@
     "@vscode/vsce": "^2.19.0",
     "ansi-colors": "^4.1.1",
     "applicationinsights": "^2.3.5",
-    "cosmiconfig": "^8.2.0",
+    "cosmiconfig": "^9.0.0",
     "cross-env": "^7.0.3",
     "css-loader": "^6.8.1",
     "del": "^6.0.0",
@@ -2195,7 +2029,7 @@
     "jest": "^29.0.3",
     "jest-environment-jsdom": "^29.0.3",
     "jest-runner-vscode": "^3.0.1",
-    "lint-staged": "^14.0.0",
+    "lint-staged": "^15.0.2",
     "markdownlint-cli2": "^0.6.0",
     "markdownlint-cli2-formatter-pretty": "^0.0.4",
     "mini-css-extract-plugin": "^2.6.1",

extensions/ql-vscode/scripts/source-map.ts

@@ -115,21 +115,35 @@ async function extractSourceMap() {
   }
 
   if (stacktrace.includes("at")) {
-    const rawSourceMaps = new Map<string, RawSourceMap>();
+    const rawSourceMaps = new Map<string, RawSourceMap | null>();
 
     const mappedStacktrace = await replaceAsync(
       stacktrace,
       stackLineRegex,
       async (match, name, file, line, column) => {
         if (!rawSourceMaps.has(file)) {
-          const rawSourceMap: RawSourceMap = await readJSON(
-            resolve(sourceMapsDirectory, `${basename(file)}.map`),
-          );
-          rawSourceMaps.set(file, rawSourceMap);
+          try {
+            const rawSourceMap: RawSourceMap = await readJSON(
+              resolve(sourceMapsDirectory, `${basename(file)}.map`),
+            );
+            rawSourceMaps.set(file, rawSourceMap);
+          } catch (e: unknown) {
+            // If the file is not found, we will not decode it and not try reading this source map again
+            if (e instanceof Error && "code" in e && e.code === "ENOENT") {
+              rawSourceMaps.set(file, null);
+            } else {
+              throw e;
+            }
+          }
+        }
+
+        const sourceMap = rawSourceMaps.get(file);
+        if (!sourceMap) {
+          return match;
         }
 
         const originalPosition = await SourceMapConsumer.with(
-          rawSourceMaps.get(file) as RawSourceMap,
+          sourceMap,
           null,
           async function (consumer) {
             return consumer.originalPositionFor({

extensions/ql-vscode/snippets.json

@@ -30,34 +30,34 @@
   "Dataflow Tracking Class": {
     "prefix": "dataflowtracking",
     "body": [
-      "class $1 extends DataFlow::Configuration {",
-      "\t$1() { this = \"$1\" }",
-      "\t",
-      "\toverride predicate isSource(DataFlow::Node node) {",
+      "module $1 implements DataFlow::ConfigSig {",
+      "\tpredicate isSource(DataFlow::Node node) {",
       "\t\t${2:none()}",
       "\t}",
-      "\t",
-      "\toverride predicate isSink(DataFlow::Node node) {",
+      "",
+      "\tpredicate isSink(DataFlow::Node node) {",
       "\t\t${3:none()}",
       "\t}",
-      "}"
+      "}",
+      "",
+      "module ${4:Flow} = DataFlow::Global<$1>;"
     ],
     "description": "Boilerplate for a dataflow tracking class"
   },
   "Taint Tracking Class": {
     "prefix": "tainttracking",
     "body": [
-      "class $1 extends TaintTracking::Configuration {",
-      "\t$1() { this = \"$1\" }",
-      "\t",
-      "\toverride predicate isSource(DataFlow::Node node) {",
+      "module $1 implements DataFlow::ConfigSig {",
+      "\tpredicate isSource(DataFlow::Node node) {",
       "\t\t${2:none()}",
       "\t}",
-      "\t",
-      "\toverride predicate isSink(DataFlow::Node node) {",
+      "",
+      "\tpredicate isSink(DataFlow::Node node) {",
       "\t\t${3:none()}",
       "\t}",
-      "}"
+      "}",
+      "",
+      "module ${4:Flow} = TaintTracking::Global<$1>;"
     ],
     "description": "Boilerplate for a taint tracking class"
   },

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -2,7 +2,7 @@ import { EOL } from "os";
 import { spawn } from "child-process-promise";
 import * as child_process from "child_process";
 import { readFile } from "fs-extra";
-import { dirname, join, delimiter } from "path";
+import { delimiter, dirname, join } from "path";
 import * as sarif from "sarif";
 import { SemVer } from "semver";
 import { Readable } from "stream";
@@ -10,8 +10,12 @@ import tk from "tree-kill";
 import { promisify } from "util";
 import { CancellationToken, Disposable, Uri } from "vscode";
 
-import { BQRSInfo, DecodedBqrsChunk } from "../common/bqrs-cli-types";
-import { allowCanaryQueryServer, CliConfig } from "../config";
+import {
+  BQRSInfo,
+  DecodedBqrs,
+  DecodedBqrsChunk,
+} from "../common/bqrs-cli-types";
+import { CliConfig } from "../config";
 import {
   DistributionProvider,
   FindDistributionResultKind,
@@ -237,15 +241,11 @@ export class CodeQLCliServer implements Disposable {
     if (this.distributionProvider.onDidChangeDistribution) {
       this.distributionProvider.onDidChangeDistribution(() => {
         this.restartCliServer();
-        this._version = undefined;
-        this._supportedLanguages = undefined;
       });
     }
     if (this.cliConfig.onDidChangeConfiguration) {
       this.cliConfig.onDidChangeConfiguration(() => {
         this.restartCliServer();
-        this._version = undefined;
-        this._supportedLanguages = undefined;
       });
     }
   }
@@ -286,6 +286,8 @@ export class CodeQLCliServer implements Disposable {
     const callback = (): void => {
       try {
         this.killProcessIfRunning();
+        this._version = undefined;
+        this._supportedLanguages = undefined;
       } finally {
         this.runNext();
       }
@@ -633,9 +635,10 @@ export class CodeQLCliServer implements Disposable {
     } = {},
   ): Promise<OutputType> {
     let args: string[] = [];
-    if (addFormat)
+    if (addFormat) {
       // Add format argument first, in case commandArgs contains positional parameters.
       args = args.concat(["--format", "json"]);
+    }
     args = args.concat(commandArgs);
     const result = await this.runCodeQlCliCommand(command, args, description, {
       progressReporter,
@@ -793,6 +796,11 @@ export class CodeQLCliServer implements Disposable {
       ["resolve", "tests", "--strict-test-discovery"],
       subcommandArgs,
       "Resolving tests",
+      {
+        // This happens as part of a background process, so we don't want to
+        // spam the log with messages.
+        silent: true,
+      },
     );
   }
 
@@ -877,10 +885,9 @@ export class CodeQLCliServer implements Disposable {
     additionalPacks: string[],
     queryPath: string,
   ): Promise<MlModelsInfo> {
-    const args = (await this.cliConstraints.supportsPreciseResolveMlModels())
-      ? // use the dirname of the path so that we can handle query libraries
-        [...this.getAdditionalPacksArg(additionalPacks), dirname(queryPath)]
-      : this.getAdditionalPacksArg(additionalPacks);
+    const args =
+      // use the dirname of the path so that we can handle query libraries
+      [...this.getAdditionalPacksArg(additionalPacks), dirname(queryPath)];
     return await this.runJsonCodeQlCliCommand<MlModelsInfo>(
       ["resolve", "ml-models"],
       args,
@@ -938,8 +945,12 @@ export class CodeQLCliServer implements Disposable {
     name?: string,
   ): Promise<string> {
     const subcommandArgs = [];
-    if (target) subcommandArgs.push("--target", target);
-    if (name) subcommandArgs.push("--name", name);
+    if (target) {
+      subcommandArgs.push("--target", target);
+    }
+    if (name) {
+      subcommandArgs.push("--name", name);
+    }
     subcommandArgs.push(archivePath);
 
     return await this.runCodeQlCliCommand(
@@ -960,7 +971,9 @@ export class CodeQLCliServer implements Disposable {
     outputDirectory?: string,
   ): Promise<string> {
     const subcommandArgs = ["--format=markdown"];
-    if (outputDirectory) subcommandArgs.push("--output", outputDirectory);
+    if (outputDirectory) {
+      subcommandArgs.push("--output", outputDirectory);
+    }
     subcommandArgs.push(pathToQhelp);
 
     return await this.runCodeQlCliCommand(
@@ -984,9 +997,7 @@ export class CodeQLCliServer implements Disposable {
     const subcommandArgs = [
       "--format=text",
       `--end-summary=${endSummaryPath}`,
-      ...((await this.cliConstraints.supportsSourceMap())
-        ? ["--sourcemap"]
-        : []),
+      "--sourcemap",
       inputPath,
       outputPath,
     ];
@@ -1040,6 +1051,18 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Gets all results from a bqrs.
+   * @param bqrsPath The path to the bqrs.
+   */
+  async bqrsDecodeAll(bqrsPath: string): Promise<DecodedBqrs> {
+    return await this.runJsonCodeQlCliCommand<DecodedBqrs>(
+      ["bqrs", "decode"],
+      [bqrsPath],
+      "Reading all bqrs data",
+    );
+  }
+
   async runInterpretCommand(
     format: string,
     additonalArgs: string[],
@@ -1244,11 +1267,13 @@ export class CodeQLCliServer implements Disposable {
    * @param additionalPacks A list of directories to search for qlpacks.
    * @param extensionPacksOnly Whether to only search for extension packs. If true, only extension packs will
    *    be returned. If false, all packs will be returned.
+   * @param kind Whether to only search for qlpacks with a certain kind.
    * @returns A dictionary mapping qlpack name to the directory it comes from
    */
   async resolveQlpacks(
     additionalPacks: string[],
     extensionPacksOnly = false,
+    kind?: "query" | "library" | "all",
   ): Promise<QlpacksInfo> {
     const args = this.getAdditionalPacksArg(additionalPacks);
     if (extensionPacksOnly) {
@@ -1259,6 +1284,8 @@ export class CodeQLCliServer implements Disposable {
         return {};
       }
       args.push("--kind", "extension", "--no-recursive");
+    } else if (kind) {
+      args.push("--kind", kind);
     }
 
     return this.runJsonCodeQlCliCommand<QlpacksInfo>(
@@ -1492,6 +1519,13 @@ export class CodeQLCliServer implements Disposable {
             CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
           ) >= 0,
         );
+        await this.app.commands.execute(
+          "setContext",
+          "codeql.supportsTrimCache",
+          newVersion.compare(
+            CliVersionConstraint.CLI_VERSION_WITH_TRIM_CACHE,
+          ) >= 0,
+        );
       } catch (e) {
         this._versionChangedListeners.forEach((listener) =>
           listener(undefined),
@@ -1587,16 +1621,19 @@ export function spawnServer(
   });
   // Set up event listeners.
   child.on("close", async (code, signal) => {
-    if (code !== null)
+    if (code !== null) {
       void logger.log(`Child process exited with code ${code}`);
-    if (signal)
+    }
+    if (signal) {
       void logger.log(
         `Child process exited due to receipt of signal ${signal}`,
       );
+    }
     // If the process exited abnormally, log the last stdout message,
     // It may be from the jvm.
-    if (code !== 0 && lastStdout !== undefined)
+    if (code !== 0 && lastStdout !== undefined) {
       void logger.log(`Last stdout was "${lastStdout.toString()}"`);
+    }
   });
   child.stderr!.on("data", stderrListener);
   if (stdoutListener !== undefined) {
@@ -1685,41 +1722,7 @@ export function shouldDebugCliServer() {
 export class CliVersionConstraint {
   // The oldest version of the CLI that we support. This is used to determine
   // whether to show a warning about the CLI being too old on startup.
-  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.9.4");
-
-  /**
-   * CLI version where building QLX packs for remote queries is supported.
-   * (The options were _accepted_ by a few earlier versions, but only from
-   * 2.11.3 will it actually use the existing compilation cache correctly).
-   */
-  public static CLI_VERSION_QLX_REMOTE = new SemVer("2.11.3");
-
-  /**
-   * CLI version where the `resolve ml-models` subcommand was enhanced to work with packaging.
-   */
-  public static CLI_VERSION_WITH_PRECISE_RESOLVE_ML_MODELS = new SemVer(
-    "2.10.0",
-  );
-
-  /**
-   * CLI version where the `resolve extensions` subcommand exists.
-   */
-  public static CLI_VERSION_WITH_RESOLVE_EXTENSIONS = new SemVer("2.10.2");
-
-  /**
-   * CLI version that supports the `--sourcemap` option for log generation.
-   */
-  public static CLI_VERSION_WITH_SOURCEMAP = new SemVer("2.10.3");
-
-  /**
-   * CLI version that supports the new query server.
-   */
-  public static CLI_VERSION_WITH_NEW_QUERY_SERVER = new SemVer("2.11.1");
-
-  /**
-   * CLI version that supports `${workspace}` references in qlpack.yml files.
-   */
-  public static CLI_VERSION_WITH_WORKSPACE_RFERENCES = new SemVer("2.11.3");
+  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.11.6");
 
   /**
    * CLI version that supports the `--kind` option for the `resolve qlpacks` command.
@@ -1755,6 +1758,12 @@ export class CliVersionConstraint {
     "2.14.0",
   );
 
+  /**
+   * CLI version where the query server supports the `evaluation/trimCache` method
+   * with `codeql database cleanup --mode=trim` semantics.
+   */
+  public static CLI_VERSION_WITH_TRIM_CACHE = new SemVer("2.15.1");
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1763,50 +1772,6 @@ export class CliVersionConstraint {
     return (await this.cli.getVersion()).compare(v) >= 0;
   }
 
-  async supportsQlxRemote() {
-    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_QLX_REMOTE);
-  }
-
-  async supportsPreciseResolveMlModels() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_PRECISE_RESOLVE_ML_MODELS,
-    );
-  }
-
-  async supportsResolveExtensions() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_RESOLVE_EXTENSIONS,
-    );
-  }
-
-  async supportsSourceMap() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_SOURCEMAP,
-    );
-  }
-
-  async supportsNewQueryServer() {
-    // This allows users to explicitly opt-out of the new query server.
-    return (
-      allowCanaryQueryServer() &&
-      this.isVersionAtLeast(
-        CliVersionConstraint.CLI_VERSION_WITH_NEW_QUERY_SERVER,
-      )
-    );
-  }
-
-  async supportsNewQueryServerForTests() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_NEW_QUERY_SERVER,
-    );
-  }
-
-  async supportsWorkspaceReferences() {
-    return this.isVersionAtLeast(
-      CliVersionConstraint.CLI_VERSION_WITH_WORKSPACE_RFERENCES,
-    );
-  }
-
   async supportsQlpacksKind() {
     return this.isVersionAtLeast(
       CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND,

extensions/ql-vscode/src/codeql-cli/query-language.ts

@@ -1,6 +1,10 @@
 import { CodeQLCliServer } from "./cli";
 import { Uri, window } from "vscode";
-import { isQueryLanguage, QueryLanguage } from "../common/query-language";
+import {
+  getLanguageDisplayName,
+  isQueryLanguage,
+  QueryLanguage,
+} from "../common/query-language";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { extLogger } from "../common/logging/vscode";
 import { UserCancellationException } from "../common/vscode/progress";
@@ -46,14 +50,22 @@ export async function askForLanguage(
   cliServer: CodeQLCliServer,
   throwOnEmpty = true,
 ): Promise<QueryLanguage | undefined> {
-  const language = await window.showQuickPick(
-    await cliServer.getSupportedLanguages(),
-    {
-      placeHolder: "Select target language for your query",
-      ignoreFocusOut: true,
-    },
-  );
-  if (!language) {
+  const supportedLanguages = await cliServer.getSupportedLanguages();
+
+  const items = supportedLanguages
+    .filter((language) => isQueryLanguage(language))
+    .map((language) => ({
+      label: getLanguageDisplayName(language),
+      description: language,
+      language,
+    }))
+    .sort((a, b) => a.label.localeCompare(b.label));
+
+  const selectedItem = await window.showQuickPick(items, {
+    placeHolder: "Select target language for your query",
+    ignoreFocusOut: true,
+  });
+  if (!selectedItem) {
     // This only happens if the user cancels the quick pick.
     if (throwOnEmpty) {
       throw new UserCancellationException("Cancelled.");
@@ -66,6 +78,8 @@ export async function askForLanguage(
     return undefined;
   }
 
+  const language = selectedItem.language;
+
   if (!isQueryLanguage(language)) {
     void showAndLogErrorMessage(
       extLogger,

extensions/ql-vscode/src/common/bqrs-cli-types.ts

@@ -21,7 +21,7 @@ type ColumnKind =
   | typeof ColumnKindCode.DATE
   | typeof ColumnKindCode.ENTITY;
 
-export interface Column {
+interface Column {
   name?: string;
   kind: ColumnKind;
 }
@@ -112,7 +112,7 @@ export type BqrsKind =
   | "Date"
   | "Entity";
 
-interface BqrsColumn {
+export interface BqrsColumn {
   name?: string;
   kind: BqrsKind;
 }
@@ -121,3 +121,5 @@ export interface DecodedBqrsChunk {
   next?: number;
   columns: BqrsColumn[];
 }
+
+export type DecodedBqrs = Record<string, DecodedBqrsChunk>;

extensions/ql-vscode/src/common/commands.ts

@@ -12,6 +12,8 @@ import type {
 } from "../variant-analysis/shared/variant-analysis";
 import type { QLDebugConfiguration } from "../debugger/debug-configuration";
 import type { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
+import type { LanguageSelectionTreeViewItem } from "../language-selection-panel/language-selection-data-provider";
+import type { Method, Usage } from "../model-editor/method";
 
 // A command function matching the signature that VS Code calls when
 // a command is invoked from a context menu on a TreeView with
@@ -144,6 +146,7 @@ export type LocalQueryCommands = {
   "codeQL.quickQuery": () => Promise<void>;
   "codeQL.getCurrentQuery": () => Promise<string>;
   "codeQL.createQuery": () => Promise<void>;
+  "codeQLQuickQuery.createQuery": () => Promise<void>;
 };
 
 // Debugger commands
@@ -198,6 +201,13 @@ export type QueryHistoryCommands = {
   "codeQL.exportSelectedVariantAnalysisResults": () => Promise<void>;
 };
 
+// Commands user for the language selector panel
+export type LanguageSelectionCommands = {
+  "codeQLLanguageSelection.setSelectedItem": (
+    item: LanguageSelectionTreeViewItem,
+  ) => Promise<void>;
+};
+
 // Commands used for the local databases panel
 export type LocalDatabasesCommands = {
   // Command palette commands
@@ -207,6 +217,7 @@ export type LocalDatabasesCommands = {
   "codeQL.chooseDatabaseGithub": () => Promise<void>;
   "codeQL.upgradeCurrentDatabase": () => Promise<void>;
   "codeQL.clearCache": () => Promise<void>;
+  "codeQL.trimCache": () => Promise<void>;
 
   // Explorer context menu
   "codeQL.setCurrentDatabase": (uri: Uri) => Promise<void>;
@@ -217,25 +228,8 @@ export type LocalDatabasesCommands = {
   "codeQLDatabases.chooseDatabaseInternet": () => Promise<void>;
   "codeQLDatabases.chooseDatabaseGithub": () => Promise<void>;
   "codeQLDatabases.sortByName": () => Promise<void>;
+  "codeQLDatabases.sortByLanguage": () => Promise<void>;
   "codeQLDatabases.sortByDateAdded": () => Promise<void>;
-  "codeQLDatabases.displayAllLanguages": () => Promise<void>;
-  "codeQLDatabases.displayCpp": () => Promise<void>;
-  "codeQLDatabases.displayCsharp": () => Promise<void>;
-  "codeQLDatabases.displayGo": () => Promise<void>;
-  "codeQLDatabases.displayJava": () => Promise<void>;
-  "codeQLDatabases.displayJavascript": () => Promise<void>;
-  "codeQLDatabases.displayPython": () => Promise<void>;
-  "codeQLDatabases.displayRuby": () => Promise<void>;
-  "codeQLDatabases.displaySwift": () => Promise<void>;
-  "codeQLDatabases.displayAllLanguagesSelected": () => Promise<void>;
-  "codeQLDatabases.displayCppSelected": () => Promise<void>;
-  "codeQLDatabases.displayCsharpSelected": () => Promise<void>;
-  "codeQLDatabases.displayGoSelected": () => Promise<void>;
-  "codeQLDatabases.displayJavaSelected": () => Promise<void>;
-  "codeQLDatabases.displayJavascriptSelected": () => Promise<void>;
-  "codeQLDatabases.displayPythonSelected": () => Promise<void>;
-  "codeQLDatabases.displayRubySelected": () => Promise<void>;
-  "codeQLDatabases.displaySwiftSelected": () => Promise<void>;
 
   // Database panel context menu
   "codeQLDatabases.setCurrentDatabase": (
@@ -324,7 +318,8 @@ export type ModelEditorCommands = {
   "codeQL.openModelEditor": () => Promise<void>;
   "codeQL.openModelEditorFromModelingPanel": () => Promise<void>;
   "codeQLModelEditor.jumpToMethod": (
-    methodSignature: string,
+    method: Method,
+    usage: Usage,
     databaseItem: DatabaseItem,
   ) => Promise<void>;
 };
@@ -359,6 +354,7 @@ export type AllExtensionCommands = BaseCommands &
   QueryEditorCommands &
   ResultsViewCommands &
   QueryHistoryCommands &
+  LanguageSelectionCommands &
   LocalDatabasesCommands &
   DebuggerCommands &
   VariantAnalysisCommands &

extensions/ql-vscode/src/common/errors.ts

@@ -22,6 +22,14 @@ export class RedactableError extends Error {
       .join("");
   }
 
+  public get fullMessageWithStack(): string {
+    if (!this.stack) {
+      return this.fullMessage;
+    }
+
+    return `${this.fullMessage}\n${this.stack}`;
+  }
+
   public get redactedMessage(): string {
     return this.strings
       .map((s, i) => s + (this.hasValue(i) ? this.getRedactedValue(i) : ""))

extensions/ql-vscode/src/common/interface-types.ts

@@ -3,8 +3,8 @@ import {
   RawResultSet,
   ResultRow,
   ResultSetSchema,
-  Column,
   ResolvableLocationValue,
+  BqrsColumn,
 } from "../common/bqrs-cli-types";
 import {
   VariantAnalysis,
@@ -334,13 +334,15 @@ interface ChangeCompareMessage {
   newResultSetName: string;
 }
 
-export type ToCompareViewMessage = SetComparisonsMessage;
+export type ToCompareViewMessage =
+  | SetComparisonQueryInfoMessage
+  | SetComparisonsMessage;
 
 /**
- * Message to the compare view that specifies the query results to compare.
+ * Message to the compare view that sets the metadata of the compared queries.
  */
-export interface SetComparisonsMessage {
-  readonly t: "setComparisons";
+export interface SetComparisonQueryInfoMessage {
+  readonly t: "setComparisonQueryInfo";
   readonly stats: {
     fromQuery?: {
       name: string;
@@ -353,28 +355,44 @@ export interface SetComparisonsMessage {
       time: string;
     };
   };
-  readonly columns: readonly Column[];
-  readonly commonResultSetNames: string[];
-  readonly currentResultSetName: string;
-  readonly rows: QueryCompareResult | undefined;
-  readonly message: string | undefined;
   readonly databaseUri: string;
+  readonly commonResultSetNames: string[];
 }
 
+/**
+ * Message to the compare view that specifies the query results to compare.
+ */
+export interface SetComparisonsMessage {
+  readonly t: "setComparisons";
+  readonly currentResultSetName: string;
+  readonly result: QueryCompareResult | undefined;
+  readonly message: string | undefined;
+}
+
+type QueryCompareResult = RawQueryCompareResult | InterpretedQueryCompareResult;
+
 /**
  * from is the set of rows that have changes in the "from" query.
  * to is the set of rows that have changes in the "to" query.
- * They are in the same order, so element 1 in "from" corresponds to
- * element 1 in "to".
- *
- * If an array element is null, that means that the element was removed
- * (or added) in the comparison.
  */
-export type QueryCompareResult = {
+export type RawQueryCompareResult = {
+  kind: "raw";
+  columns: readonly BqrsColumn[];
   from: ResultRow[];
   to: ResultRow[];
 };
 
+/**
+ * from is the set of results that have changes in the "from" query.
+ * to is the set of results that have changes in the "to" query.
+ */
+type InterpretedQueryCompareResult = {
+  kind: "interpreted";
+  sourceLocationPrefix: string;
+  from: sarif.Result[];
+  to: sarif.Result[];
+};
+
 /**
  * Extract the name of the default result. Prefer returning
  * 'alerts', or '#select'. Otherwise return the first in the list.
@@ -517,8 +535,7 @@ interface SetModifiedMethodsMessage {
 
 interface SetInProgressMethodsMessage {
   t: "setInProgressMethods";
-  packageName: string;
-  inProgressMethods: string[];
+  methods: string[];
 }
 
 interface SwitchModeMessage {
@@ -572,11 +589,6 @@ interface HideModeledMethodsMessage {
   hideModeledMethods: boolean;
 }
 
-interface SetModeledMethodMessage {
-  t: "setModeledMethod";
-  method: ModeledMethod;
-}
-
 interface SetMultipleModeledMethodsMessage {
   t: "setMultipleModeledMethods";
   methodSignature: string;
@@ -588,6 +600,11 @@ interface SetInModelingModeMessage {
   inModelingMode: boolean;
 }
 
+interface SetInProgressMessage {
+  t: "setInProgress";
+  inProgress: boolean;
+}
+
 interface RevealMethodMessage {
   t: "revealMethod";
   methodSignature: string;
@@ -614,7 +631,7 @@ export type FromModelEditorMessage =
   | StopGeneratingMethodsFromLlmMessage
   | ModelDependencyMessage
   | HideModeledMethodsMessage
-  | SetModeledMethodMessage;
+  | SetMultipleModeledMethodsMessage;
 
 interface RevealInEditorMessage {
   t: "revealInModelEditor";
@@ -627,7 +644,7 @@ interface StartModelingMessage {
 
 export type FromMethodModelingMessage =
   | CommonFromViewMessages
-  | SetModeledMethodMessage
+  | SetMultipleModeledMethodsMessage
   | RevealInEditorMessage
   | StartModelingMessage;
 
@@ -651,6 +668,7 @@ interface SetSelectedMethodMessage {
   method: Method;
   modeledMethods: ModeledMethod[];
   isModified: boolean;
+  isInProgress: boolean;
 }
 
 export type ToMethodModelingMessage =
@@ -659,4 +677,5 @@ export type ToMethodModelingMessage =
   | SetMultipleModeledMethodsMessage
   | SetMethodModifiedMessage
   | SetSelectedMethodMessage
-  | SetInModelingModeMessage;
+  | SetInModelingModeMessage
+  | SetInProgressMessage;

extensions/ql-vscode/src/common/logging/notifications.ts

@@ -112,5 +112,5 @@ export async function showAndLogExceptionWithTelemetry(
   options: ShowAndLogExceptionOptions = {},
 ): Promise<void> {
   telemetry?.sendError(error, options.extraTelemetryProperties);
-  return showAndLogErrorMessage(logger, error.fullMessage, options);
+  return showAndLogErrorMessage(logger, error.fullMessageWithStack, options);
 }

extensions/ql-vscode/src/common/mock-gh-api/recorder.ts

@@ -112,11 +112,14 @@ export class Recorder extends DisposableObject {
     return scenarioDirectory;
   }
 
-  private async onResponseBypass(
-    response: Response,
-    request: Request,
-    _requestId: string,
-  ): Promise<void> {
+  private async onResponseBypass({
+    response,
+    request,
+  }: {
+    response: Response;
+    request: Request;
+    requestId: string;
+  }): Promise<void> {
     if (request.headers.has("x-vscode-codeql-msw-bypass")) {
       return;
     }

extensions/ql-vscode/src/common/mock-gh-api/request-handlers.ts

@@ -1,6 +1,6 @@
 import { join } from "path";
 import { readdir, readJson, readFile } from "fs-extra";
-import { RequestHandler, rest } from "msw";
+import { http, RequestHandler } from "msw";
 import {
   GitHubApiRequest,
   isAutoModelRequest,
@@ -94,7 +94,7 @@ function createGetRepoRequestHandler(
 
   const getRepoRequest = getRepoRequests[0];
 
-  return rest.get(`${baseUrl}/repos/:owner/:name`, () => {
+  return http.get(`${baseUrl}/repos/:owner/:name`, () => {
     return jsonResponse(getRepoRequest.response.body, {
       status: getRepoRequest.response.status,
     });
@@ -114,7 +114,7 @@ function createSubmitVariantAnalysisRequestHandler(
 
   const getRepoRequest = submitVariantAnalysisRequests[0];
 
-  return rest.post(
+  return http.post(
     `${baseUrl}/repositories/:controllerRepoId/code-scanning/codeql/variant-analyses`,
     () => {
       return jsonResponse(getRepoRequest.response.body, {
@@ -135,7 +135,7 @@ function createGetVariantAnalysisRequestHandler(
   // During the lifetime of a variant analysis run, there are multiple requests
   // to get the variant analysis. We need to return different responses for each
   // request, so keep an index of the request and return the appropriate response.
-  return rest.get(
+  return http.get(
     `${baseUrl}/repositories/:controllerRepoId/code-scanning/codeql/variant-analyses/:variantAnalysisId`,
     () => {
       const request = getVariantAnalysisRequests[requestIndex];
@@ -159,7 +159,7 @@ function createGetVariantAnalysisRepoRequestHandler(
     isGetVariantAnalysisRepoRequest,
   );
 
-  return rest.get(
+  return http.get(
     `${baseUrl}/repositories/:controllerRepoId/code-scanning/codeql/variant-analyses/:variantAnalysisId/repositories/:repoId`,
     ({ request, params }) => {
       const scenarioRequest = getVariantAnalysisRepoRequests.find(
@@ -183,7 +183,7 @@ function createGetVariantAnalysisRepoResultRequestHandler(
     isGetVariantAnalysisRepoResultRequest,
   );
 
-  return rest.get(
+  return http.get(
     "https://objects-origin.githubusercontent.com/codeql-query-console/codeql-variant-analysis-repo-tasks/:variantAnalysisId/:repoId/*",
     ({ request, params }) => {
       const scenarioRequest = getVariantAnalysisRepoResultRequests.find(
@@ -216,7 +216,7 @@ function createCodeSearchRequestHandler(
   // During a code search, there are multiple request to get pages of results. We
   // need to return different responses for each request, so keep an index of the
   // request and return the appropriate response.
-  return rest.get(`${baseUrl}/search/code`, () => {
+  return http.get(`${baseUrl}/search/code`, () => {
     const request = codeSearchRequests[requestIndex];
 
     if (requestIndex < codeSearchRequests.length - 1) {
@@ -239,7 +239,7 @@ function createAutoModelRequestHandler(
   // During automodeling there can be multiple API requests for each batch
   // of candidates we want to model. We need to return different responses for each request,
   // so keep an index of the request and return the appropriate response.
-  return rest.post(
+  return http.post(
     `${baseUrl}/repos/github/codeql/code-scanning/codeql/auto-model`,
     () => {
       const request = autoModelRequests[requestIndex];

extensions/ql-vscode/src/common/mutable.ts

@@ -0,0 +1,6 @@
+/**
+ * Remove all readonly modifiers from a type.
+ */
+export type Mutable<T> = {
+  -readonly [P in keyof T]: T[P];
+};

extensions/ql-vscode/src/common/readonly.ts

@@ -0,0 +1,14 @@
+export type DeepReadonly<T> = T extends Array<infer R>
+  ? DeepReadonlyArray<R>
+  : // eslint-disable-next-line @typescript-eslint/ban-types
+  T extends Function
+  ? T
+  : T extends object
+  ? DeepReadonlyObject<T>
+  : T;
+
+interface DeepReadonlyArray<T> extends ReadonlyArray<DeepReadonly<T>> {}
+
+type DeepReadonlyObject<T> = {
+  readonly [P in keyof T]: DeepReadonly<T[P]>;
+};

extensions/ql-vscode/src/common/sarif-utils.ts

@@ -1,6 +1,7 @@
 import * as Sarif from "sarif";
 import type { HighlightedRegion } from "../variant-analysis/shared/analysis-result";
 import { ResolvableLocationValue } from "../common/bqrs-cli-types";
+import { isEmptyPath } from "./bqrs-utils";
 
 export interface SarifLink {
   dest: number;
@@ -106,11 +107,18 @@ export function parseSarifLocation(
   sourceLocationPrefix: string,
 ): ParsedSarifLocation {
   const physicalLocation = loc.physicalLocation;
-  if (physicalLocation === undefined) return { hint: "no physical location" };
-  if (physicalLocation.artifactLocation === undefined)
+  if (physicalLocation === undefined) {
+    return { hint: "no physical location" };
+  }
+  if (physicalLocation.artifactLocation === undefined) {
     return { hint: "no artifact location" };
-  if (physicalLocation.artifactLocation.uri === undefined)
+  }
+  if (physicalLocation.artifactLocation.uri === undefined) {
     return { hint: "artifact location has no uri" };
+  }
+  if (isEmptyPath(physicalLocation.artifactLocation.uri)) {
+    return { hint: "artifact location has empty uri" };
+  }
 
   // This is not necessarily really an absolute uri; it could either be a
   // file uri or a relative uri.

extensions/ql-vscode/src/common/vscode/abstract-webview-view-provider.ts

@@ -3,6 +3,7 @@ import { Uri, WebviewViewProvider } from "vscode";
 import { WebviewKind, WebviewMessage, getHtmlForWebview } from "./webview-html";
 import { Disposable } from "../disposable-object";
 import { App } from "../app";
+import { DeepReadonly } from "../readonly";
 
 export abstract class AbstractWebviewViewProvider<
   ToMessage extends WebviewMessage,
@@ -53,7 +54,7 @@ export abstract class AbstractWebviewViewProvider<
     return this.webviewView?.visible ?? false;
   }
 
-  protected async postMessage(msg: ToMessage): Promise<void> {
+  protected async postMessage(msg: DeepReadonly<ToMessage>): Promise<void> {
     await this.webviewView?.webview.postMessage(msg);
   }
 

extensions/ql-vscode/src/common/vscode/abstract-webview.ts

@@ -12,6 +12,7 @@ import { App } from "../app";
 import { Disposable } from "../disposable-object";
 import { tmpDir } from "../../tmp-dir";
 import { getHtmlForWebview, WebviewMessage, WebviewKind } from "./webview-html";
+import { DeepReadonly } from "../readonly";
 
 export type WebviewPanelConfig = {
   viewId: string;
@@ -146,7 +147,7 @@ export abstract class AbstractWebview<
     this.panelLoadedCallBacks = [];
   }
 
-  protected async postMessage(msg: ToMessage): Promise<boolean> {
+  protected async postMessage(msg: DeepReadonly<ToMessage>): Promise<boolean> {
     const panel = await this.getPanel();
     return panel.webview.postMessage(msg);
   }

extensions/ql-vscode/src/common/vscode/archive-filesystem-provider.ts

@@ -130,11 +130,14 @@ export function decodeSourceArchiveUri(uri: vscode.Uri): ZipFileReference {
     };
   }
   const match = sourceArchiveUriAuthorityPattern.exec(uri.authority);
-  if (match === null) throw new InvalidSourceArchiveUriError(uri);
+  if (match === null) {
+    throw new InvalidSourceArchiveUriError(uri);
+  }
   const zipPathStartIndex = parseInt(match[1]);
   const zipPathEndIndex = parseInt(match[2]);
-  if (isNaN(zipPathStartIndex) || isNaN(zipPathEndIndex))
+  if (isNaN(zipPathStartIndex) || isNaN(zipPathEndIndex)) {
     throw new InvalidSourceArchiveUriError(uri);
+  }
   return {
     pathWithinSourceArchive: uri.path.substring(zipPathEndIndex) || "/",
     sourceArchiveZipPath: uri.path.substring(
@@ -179,8 +182,9 @@ type Archive = {
 };
 
 async function parse_zip(zipPath: string): Promise<Archive> {
-  if (!(await pathExists(zipPath)))
+  if (!(await pathExists(zipPath))) {
     throw vscode.FileSystemError.FileNotFound(zipPath);
+  }
   const archive: Archive = {
     unzipped: await unzipper.Open.file(zipPath),
     dirMap: new Map(),

extensions/ql-vscode/src/common/vscode/commands.ts

@@ -6,11 +6,7 @@ import {
   showAndLogExceptionWithTelemetry,
 } from "../logging";
 import { extLogger } from "../logging/vscode";
-import {
-  asError,
-  getErrorMessage,
-  getErrorStack,
-} from "../../common/helpers-pure";
+import { asError, getErrorMessage } from "../../common/helpers-pure";
 import { redactableError } from "../../common/errors";
 import { UserCancellationException } from "./progress";
 import { telemetryListener } from "./telemetry";
@@ -66,10 +62,7 @@ export function registerCommandWithErrorHandling(
         }
       } else {
         // Include the full stack in the error log only.
-        const errorStack = getErrorStack(e);
-        const fullMessage = errorStack
-          ? `${errorMessage.fullMessage}\n${errorStack}`
-          : errorMessage.fullMessage;
+        const fullMessage = errorMessage.fullMessageWithStack;
         void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
           fullMessage,
           extraTelemetryProperties: {

extensions/ql-vscode/src/common/vscode/extension/git.ts

@@ -0,0 +1,436 @@
+// From https://github.com/microsoft/vscode/blob/5e27a2845a87be4b4bede3e51073f94609445e51/extensions/git/src/api/git.d.ts
+
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import {
+  Uri,
+  Event,
+  Disposable,
+  ProviderResult,
+  Command,
+  CancellationToken,
+  ThemeIcon,
+} from "vscode";
+
+interface Git {
+  readonly path: string;
+}
+
+interface InputBox {
+  value: string;
+}
+
+const enum ForcePushMode {
+  Force,
+  ForceWithLease,
+  ForceWithLeaseIfIncludes,
+}
+
+const enum RefType {
+  Head,
+  RemoteHead,
+  Tag,
+}
+
+interface Ref {
+  readonly type: RefType;
+  readonly name?: string;
+  readonly commit?: string;
+  readonly remote?: string;
+}
+
+interface UpstreamRef {
+  readonly remote: string;
+  readonly name: string;
+}
+
+interface Branch extends Ref {
+  readonly upstream?: UpstreamRef;
+  readonly ahead?: number;
+  readonly behind?: number;
+}
+
+interface Commit {
+  readonly hash: string;
+  readonly message: string;
+  readonly parents: string[];
+  readonly authorDate?: Date;
+  readonly authorName?: string;
+  readonly authorEmail?: string;
+  readonly commitDate?: Date;
+}
+
+interface Submodule {
+  readonly name: string;
+  readonly path: string;
+  readonly url: string;
+}
+
+interface Remote {
+  readonly name: string;
+  readonly fetchUrl?: string;
+  readonly pushUrl?: string;
+  readonly isReadOnly: boolean;
+}
+
+const enum Status {
+  INDEX_MODIFIED,
+  INDEX_ADDED,
+  INDEX_DELETED,
+  INDEX_RENAMED,
+  INDEX_COPIED,
+
+  MODIFIED,
+  DELETED,
+  UNTRACKED,
+  IGNORED,
+  INTENT_TO_ADD,
+  INTENT_TO_RENAME,
+  TYPE_CHANGED,
+
+  ADDED_BY_US,
+  ADDED_BY_THEM,
+  DELETED_BY_US,
+  DELETED_BY_THEM,
+  BOTH_ADDED,
+  BOTH_DELETED,
+  BOTH_MODIFIED,
+}
+
+interface Change {
+  /**
+   * Returns either `originalUri` or `renameUri`, depending
+   * on whether this change is a rename change. When
+   * in doubt always use `uri` over the other two alternatives.
+   */
+  readonly uri: Uri;
+  readonly originalUri: Uri;
+  readonly renameUri: Uri | undefined;
+  readonly status: Status;
+}
+
+interface RepositoryState {
+  readonly HEAD: Branch | undefined;
+  readonly refs: Ref[];
+  readonly remotes: Remote[];
+  readonly submodules: Submodule[];
+  readonly rebaseCommit: Commit | undefined;
+
+  readonly mergeChanges: Change[];
+  readonly indexChanges: Change[];
+  readonly workingTreeChanges: Change[];
+
+  readonly onDidChange: Event<void>;
+}
+
+interface RepositoryUIState {
+  readonly selected: boolean;
+  readonly onDidChange: Event<void>;
+}
+
+/**
+ * Log options.
+ */
+interface LogOptions {
+  /** Max number of log entries to retrieve. If not specified, the default is 32. */
+  readonly maxEntries?: number;
+  readonly path?: string;
+  /** A commit range, such as "0a47c67f0fb52dd11562af48658bc1dff1d75a38..0bb4bdea78e1db44d728fd6894720071e303304f" */
+  readonly range?: string;
+  readonly reverse?: boolean;
+  readonly sortByAuthorDate?: boolean;
+}
+
+interface CommitOptions {
+  all?: boolean | "tracked";
+  amend?: boolean;
+  signoff?: boolean;
+  signCommit?: boolean;
+  empty?: boolean;
+  noVerify?: boolean;
+  requireUserConfig?: boolean;
+  useEditor?: boolean;
+  verbose?: boolean;
+  /**
+   * string    - execute the specified command after the commit operation
+   * undefined - execute the command specified in git.postCommitCommand
+   *             after the commit operation
+   * null      - do not execute any command after the commit operation
+   */
+  postCommitCommand?: string | null;
+}
+
+interface FetchOptions {
+  remote?: string;
+  ref?: string;
+  all?: boolean;
+  prune?: boolean;
+  depth?: number;
+}
+
+interface InitOptions {
+  defaultBranch?: string;
+}
+
+interface RefQuery {
+  readonly contains?: string;
+  readonly count?: number;
+  readonly pattern?: string;
+  readonly sort?: "alphabetically" | "committerdate";
+}
+
+interface BranchQuery extends RefQuery {
+  readonly remote?: boolean;
+}
+
+export interface Repository {
+  readonly rootUri: Uri;
+  readonly inputBox: InputBox;
+  readonly state: RepositoryState;
+  readonly ui: RepositoryUIState;
+
+  getConfigs(): Promise<Array<{ key: string; value: string }>>;
+  getConfig(key: string): Promise<string>;
+  setConfig(key: string, value: string): Promise<string>;
+  getGlobalConfig(key: string): Promise<string>;
+
+  getObjectDetails(
+    treeish: string,
+    path: string,
+  ): Promise<{ mode: string; object: string; size: number }>;
+  detectObjectType(
+    object: string,
+  ): Promise<{ mimetype: string; encoding?: string }>;
+  buffer(ref: string, path: string): Promise<Buffer>;
+  show(ref: string, path: string): Promise<string>;
+  getCommit(ref: string): Promise<Commit>;
+
+  add(paths: string[]): Promise<void>;
+  revert(paths: string[]): Promise<void>;
+  clean(paths: string[]): Promise<void>;
+
+  apply(patch: string, reverse?: boolean): Promise<void>;
+  diff(cached?: boolean): Promise<string>;
+  diffWithHEAD(): Promise<Change[]>;
+  diffWithHEAD(path: string): Promise<string>;
+  diffWith(ref: string): Promise<Change[]>;
+  diffWith(ref: string, path: string): Promise<string>;
+  diffIndexWithHEAD(): Promise<Change[]>;
+  diffIndexWithHEAD(path: string): Promise<string>;
+  diffIndexWith(ref: string): Promise<Change[]>;
+  diffIndexWith(ref: string, path: string): Promise<string>;
+  diffBlobs(object1: string, object2: string): Promise<string>;
+  diffBetween(ref1: string, ref2: string): Promise<Change[]>;
+  diffBetween(ref1: string, ref2: string, path: string): Promise<string>;
+
+  hashObject(data: string): Promise<string>;
+
+  createBranch(name: string, checkout: boolean, ref?: string): Promise<void>;
+  deleteBranch(name: string, force?: boolean): Promise<void>;
+  getBranch(name: string): Promise<Branch>;
+  getBranches(
+    query: BranchQuery,
+    cancellationToken?: CancellationToken,
+  ): Promise<Ref[]>;
+  getBranchBase(name: string): Promise<Branch | undefined>;
+  setBranchUpstream(name: string, upstream: string): Promise<void>;
+
+  getRefs(
+    query: RefQuery,
+    cancellationToken?: CancellationToken,
+  ): Promise<Ref[]>;
+
+  getMergeBase(ref1: string, ref2: string): Promise<string>;
+
+  tag(name: string, upstream: string): Promise<void>;
+  deleteTag(name: string): Promise<void>;
+
+  status(): Promise<void>;
+  checkout(treeish: string): Promise<void>;
+
+  addRemote(name: string, url: string): Promise<void>;
+  removeRemote(name: string): Promise<void>;
+  renameRemote(name: string, newName: string): Promise<void>;
+
+  fetch(options?: FetchOptions): Promise<void>;
+  fetch(remote?: string, ref?: string, depth?: number): Promise<void>;
+  pull(unshallow?: boolean): Promise<void>;
+  push(
+    remoteName?: string,
+    branchName?: string,
+    setUpstream?: boolean,
+    force?: ForcePushMode,
+  ): Promise<void>;
+
+  blame(path: string): Promise<string>;
+  log(options?: LogOptions): Promise<Commit[]>;
+
+  commit(message: string, opts?: CommitOptions): Promise<void>;
+}
+
+interface RemoteSource {
+  readonly name: string;
+  readonly description?: string;
+  readonly url: string | string[];
+}
+
+interface RemoteSourceProvider {
+  readonly name: string;
+  readonly icon?: string; // codicon name
+  readonly supportsQuery?: boolean;
+  getRemoteSources(query?: string): ProviderResult<RemoteSource[]>;
+  getBranches?(url: string): ProviderResult<string[]>;
+  publishRepository?(repository: Repository): Promise<void>;
+}
+
+interface RemoteSourcePublisher {
+  readonly name: string;
+  readonly icon?: string; // codicon name
+  publishRepository(repository: Repository): Promise<void>;
+}
+
+interface Credentials {
+  readonly username: string;
+  readonly password: string;
+}
+
+interface CredentialsProvider {
+  getCredentials(host: Uri): ProviderResult<Credentials>;
+}
+
+interface PostCommitCommandsProvider {
+  getCommands(repository: Repository): Command[];
+}
+
+interface PushErrorHandler {
+  handlePushError(
+    repository: Repository,
+    remote: Remote,
+    refspec: string,
+    error: Error & { gitErrorCode: GitErrorCodes },
+  ): Promise<boolean>;
+}
+
+interface BranchProtection {
+  readonly remote: string;
+  readonly rules: BranchProtectionRule[];
+}
+
+interface BranchProtectionRule {
+  readonly include?: string[];
+  readonly exclude?: string[];
+}
+
+interface BranchProtectionProvider {
+  onDidChangeBranchProtection: Event<Uri>;
+  provideBranchProtection(): BranchProtection[];
+}
+
+interface CommitMessageProvider {
+  readonly title: string;
+  readonly icon?: Uri | { light: Uri; dark: Uri } | ThemeIcon;
+  provideCommitMessage(
+    repository: Repository,
+    changes: string[],
+    cancellationToken?: CancellationToken,
+  ): Promise<string | undefined>;
+}
+
+type APIState = "uninitialized" | "initialized";
+
+interface PublishEvent {
+  repository: Repository;
+  branch?: string;
+}
+
+export interface API {
+  readonly state: APIState;
+  readonly onDidChangeState: Event<APIState>;
+  readonly onDidPublish: Event<PublishEvent>;
+  readonly git: Git;
+  readonly repositories: Repository[];
+  readonly onDidOpenRepository: Event<Repository>;
+  readonly onDidCloseRepository: Event<Repository>;
+
+  toGitUri(uri: Uri, ref: string): Uri;
+  getRepository(uri: Uri): Repository | null;
+  init(root: Uri, options?: InitOptions): Promise<Repository | null>;
+  openRepository(root: Uri): Promise<Repository | null>;
+
+  registerRemoteSourcePublisher(publisher: RemoteSourcePublisher): Disposable;
+  registerRemoteSourceProvider(provider: RemoteSourceProvider): Disposable;
+  registerCredentialsProvider(provider: CredentialsProvider): Disposable;
+  registerPostCommitCommandsProvider(
+    provider: PostCommitCommandsProvider,
+  ): Disposable;
+  registerPushErrorHandler(handler: PushErrorHandler): Disposable;
+  registerBranchProtectionProvider(
+    root: Uri,
+    provider: BranchProtectionProvider,
+  ): Disposable;
+  registerCommitMessageProvider(provider: CommitMessageProvider): Disposable;
+}
+
+export interface GitExtension {
+  readonly enabled: boolean;
+  readonly onDidChangeEnablement: Event<boolean>;
+
+  /**
+   * Returns a specific API version.
+   *
+   * Throws error if git extension is disabled. You can listen to the
+   * [GitExtension.onDidChangeEnablement](#GitExtension.onDidChangeEnablement) event
+   * to know when the extension becomes enabled/disabled.
+   *
+   * @param version Version number.
+   * @returns API instance
+   */
+  getAPI(version: 1): API;
+}
+
+const enum GitErrorCodes {
+  BadConfigFile = "BadConfigFile",
+  AuthenticationFailed = "AuthenticationFailed",
+  NoUserNameConfigured = "NoUserNameConfigured",
+  NoUserEmailConfigured = "NoUserEmailConfigured",
+  NoRemoteRepositorySpecified = "NoRemoteRepositorySpecified",
+  NotAGitRepository = "NotAGitRepository",
+  NotAtRepositoryRoot = "NotAtRepositoryRoot",
+  Conflict = "Conflict",
+  StashConflict = "StashConflict",
+  UnmergedChanges = "UnmergedChanges",
+  PushRejected = "PushRejected",
+  ForcePushWithLeaseRejected = "ForcePushWithLeaseRejected",
+  ForcePushWithLeaseIfIncludesRejected = "ForcePushWithLeaseIfIncludesRejected",
+  RemoteConnectionError = "RemoteConnectionError",
+  DirtyWorkTree = "DirtyWorkTree",
+  CantOpenResource = "CantOpenResource",
+  GitNotFound = "GitNotFound",
+  CantCreatePipe = "CantCreatePipe",
+  PermissionDenied = "PermissionDenied",
+  CantAccessRemote = "CantAccessRemote",
+  RepositoryNotFound = "RepositoryNotFound",
+  RepositoryIsLocked = "RepositoryIsLocked",
+  BranchNotFullyMerged = "BranchNotFullyMerged",
+  NoRemoteReference = "NoRemoteReference",
+  InvalidBranchName = "InvalidBranchName",
+  BranchAlreadyExists = "BranchAlreadyExists",
+  NoLocalChanges = "NoLocalChanges",
+  NoStashFound = "NoStashFound",
+  LocalChangesOverwritten = "LocalChangesOverwritten",
+  NoUpstreamBranch = "NoUpstreamBranch",
+  IsInSubmodule = "IsInSubmodule",
+  WrongCase = "WrongCase",
+  CantLockRef = "CantLockRef",
+  CantRebaseMultipleBranches = "CantRebaseMultipleBranches",
+  PatchDoesNotApply = "PatchDoesNotApply",
+  NoPathFound = "NoPathFound",
+  UnknownPath = "UnknownPath",
+  EmptyCommitMessage = "EmptyCommitMessage",
+  BranchFastForwardRejected = "BranchFastForwardRejected",
+  BranchNotYetBorn = "BranchNotYetBorn",
+  TagConflict = "TagConflict",
+}

extensions/ql-vscode/src/common/vscode/workspace-folders.ts

@@ -43,7 +43,9 @@ export function getFirstWorkspaceFolder() {
   const workspaceFolders = getOnDiskWorkspaceFolders();
 
   if (!workspaceFolders || workspaceFolders.length === 0) {
-    throw new Error("No workspace folders found");
+    throw new Error(
+      "No workspace folders found. Please open a folder or workspace in VS Code.",
+    );
   }
 
   const firstFolderFsPath = workspaceFolders[0];

extensions/ql-vscode/src/compare/compare-view.ts

@@ -2,19 +2,15 @@ import { ViewColumn } from "vscode";
 
 import {
   FromCompareViewMessage,
+  RawQueryCompareResult,
   ToCompareViewMessage,
-  QueryCompareResult,
 } from "../common/interface-types";
 import { Logger, showAndLogExceptionWithTelemetry } from "../common/logging";
 import { extLogger } from "../common/logging/vscode";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { DatabaseManager } from "../databases/local-databases";
 import { jumpToLocation } from "../databases/local-databases/locations";
-import {
-  transformBqrsResultSet,
-  RawResultSet,
-  BQRSInfo,
-} from "../common/bqrs-cli-types";
+import { BQRSInfo, DecodedBqrsChunk } from "../common/bqrs-cli-types";
 import resultsDiff from "./resultsDiff";
 import { CompletedLocalQueryInfo } from "../query-results";
 import { assertNever, getErrorMessage } from "../common/helpers-pure";
@@ -26,10 +22,18 @@ import {
 import { telemetryListener } from "../common/vscode/telemetry";
 import { redactableError } from "../common/errors";
 import { App } from "../common/app";
+import {
+  findCommonResultSetNames,
+  findResultSetNames,
+} from "./result-set-names";
 
 interface ComparePair {
   from: CompletedLocalQueryInfo;
+  fromSchemas: BQRSInfo;
   to: CompletedLocalQueryInfo;
+  toSchemas: BQRSInfo;
+
+  commonResultSetNames: readonly string[];
 }
 
 export class CompareView extends AbstractWebview<
@@ -56,49 +60,82 @@ export class CompareView extends AbstractWebview<
     to: CompletedLocalQueryInfo,
     selectedResultSetName?: string,
   ) {
-    this.comparePair = { from, to };
+    const fromSchemas = await this.cliServer.bqrsInfo(
+      from.completedQuery.query.resultsPaths.resultsPath,
+    );
+    const toSchemas = await this.cliServer.bqrsInfo(
+      to.completedQuery.query.resultsPaths.resultsPath,
+    );
+
+    const commonResultSetNames = await findCommonResultSetNames(
+      fromSchemas,
+      toSchemas,
+    );
+
+    this.comparePair = {
+      from,
+      fromSchemas,
+      to,
+      toSchemas,
+      commonResultSetNames,
+    };
+
+    const panel = await this.getPanel();
+    panel.reveal(undefined, true);
+    await this.waitForPanelLoaded();
+
+    await this.postMessage({
+      t: "setComparisonQueryInfo",
+      stats: {
+        fromQuery: {
+          // since we split the description into several rows
+          // only run interpolation if the label is user-defined
+          // otherwise we will wind up with duplicated rows
+          name: this.labelProvider.getShortLabel(from),
+          status: from.completedQuery.statusString,
+          time: from.startTime,
+        },
+        toQuery: {
+          name: this.labelProvider.getShortLabel(to),
+          status: to.completedQuery.statusString,
+          time: to.startTime,
+        },
+      },
+      databaseUri: to.initialInfo.databaseInfo.databaseUri,
+      commonResultSetNames,
+    });
+
+    await this.showResultsInternal(selectedResultSetName);
+  }
+
+  private async showResultsInternal(selectedResultSetName?: string) {
+    if (!this.comparePair) {
+      return;
+    }
+
     const panel = await this.getPanel();
     panel.reveal(undefined, true);
 
     await this.waitForPanelLoaded();
-    const [
-      commonResultSetNames,
-      currentResultSetName,
-      fromResultSet,
-      toResultSet,
-    ] = await this.findCommonResultSetNames(from, to, selectedResultSetName);
-    if (currentResultSetName) {
-      let rows: QueryCompareResult | undefined;
+    const { currentResultSetDisplayName, fromResultSet, toResultSet } =
+      await this.findResultSetsToCompare(
+        this.comparePair,
+        selectedResultSetName,
+      );
+    if (currentResultSetDisplayName) {
+      let result: RawQueryCompareResult | undefined;
       let message: string | undefined;
       try {
-        rows = this.compareResults(fromResultSet, toResultSet);
+        result = this.compareResults(fromResultSet, toResultSet);
       } catch (e) {
         message = getErrorMessage(e);
       }
 
       await this.postMessage({
         t: "setComparisons",
-        stats: {
-          fromQuery: {
-            // since we split the description into several rows
-            // only run interpolation if the label is user-defined
-            // otherwise we will wind up with duplicated rows
-            name: this.labelProvider.getShortLabel(from),
-            status: from.completedQuery.statusString,
-            time: from.startTime,
-          },
-          toQuery: {
-            name: this.labelProvider.getShortLabel(to),
-            status: to.completedQuery.statusString,
-            time: to.startTime,
-          },
-        },
-        columns: fromResultSet.schema.columns,
-        commonResultSetNames,
-        currentResultSetName,
-        rows,
+        result,
+        currentResultSetName: currentResultSetDisplayName,
         message,
-        databaseUri: to.initialInfo.databaseInfo.databaseUri,
       });
     }
   }
@@ -165,93 +202,57 @@ export class CompareView extends AbstractWebview<
     }
   }
 
-  private async findCommonResultSetNames(
-    from: CompletedLocalQueryInfo,
-    to: CompletedLocalQueryInfo,
+  private async findResultSetsToCompare(
+    { from, fromSchemas, to, toSchemas, commonResultSetNames }: ComparePair,
     selectedResultSetName: string | undefined,
-  ): Promise<[string[], string, RawResultSet, RawResultSet]> {
-    const fromSchemas = await this.cliServer.bqrsInfo(
-      from.completedQuery.query.resultsPaths.resultsPath,
-    );
-    const toSchemas = await this.cliServer.bqrsInfo(
-      to.completedQuery.query.resultsPaths.resultsPath,
-    );
-    const fromSchemaNames = fromSchemas["result-sets"].map(
-      (schema) => schema.name,
-    );
-    const toSchemaNames = toSchemas["result-sets"].map((schema) => schema.name);
-    const commonResultSetNames = fromSchemaNames.filter((name) =>
-      toSchemaNames.includes(name),
-    );
-
-    // Fall back on the default result set names if there are no common ones.
-    const defaultFromResultSetName = fromSchemaNames.find((name) =>
-      name.startsWith("#"),
-    );
-    const defaultToResultSetName = toSchemaNames.find((name) =>
-      name.startsWith("#"),
-    );
-
-    if (
-      commonResultSetNames.length === 0 &&
-      !(defaultFromResultSetName || defaultToResultSetName)
-    ) {
-      throw new Error(
-        "No common result sets found between the two queries. Please check that the queries are compatible.",
+  ) {
+    const { currentResultSetDisplayName, fromResultSetName, toResultSetName } =
+      await findResultSetNames(
+        fromSchemas,
+        toSchemas,
+        commonResultSetNames,
+        selectedResultSetName,
       );
-    }
 
-    const currentResultSetName =
-      selectedResultSetName || commonResultSetNames[0];
     const fromResultSet = await this.getResultSet(
       fromSchemas,
-      currentResultSetName || defaultFromResultSetName!,
+      fromResultSetName,
       from.completedQuery.query.resultsPaths.resultsPath,
     );
     const toResultSet = await this.getResultSet(
       toSchemas,
-      currentResultSetName || defaultToResultSetName!,
+      toResultSetName,
       to.completedQuery.query.resultsPaths.resultsPath,
     );
-    return [
-      commonResultSetNames,
-      currentResultSetName ||
-        `${defaultFromResultSetName} <-> ${defaultToResultSetName}`,
+    return {
+      currentResultSetDisplayName,
       fromResultSet,
       toResultSet,
-    ];
+    };
   }
 
   private async changeTable(newResultSetName: string) {
-    if (!this.comparePair?.from || !this.comparePair.to) {
-      return;
-    }
-    await this.showResults(
-      this.comparePair.from,
-      this.comparePair.to,
-      newResultSetName,
-    );
+    await this.showResultsInternal(newResultSetName);
   }
 
   private async getResultSet(
     bqrsInfo: BQRSInfo,
     resultSetName: string,
     resultsPath: string,
-  ): Promise<RawResultSet> {
+  ): Promise<DecodedBqrsChunk> {
     const schema = bqrsInfo["result-sets"].find(
       (schema) => schema.name === resultSetName,
     );
     if (!schema) {
       throw new Error(`Schema ${resultSetName} not found.`);
     }
-    const chunk = await this.cliServer.bqrsDecode(resultsPath, resultSetName);
-    return transformBqrsResultSet(schema, chunk);
+    return await this.cliServer.bqrsDecode(resultsPath, resultSetName);
   }
 
   private compareResults(
-    fromResults: RawResultSet,
-    toResults: RawResultSet,
-  ): QueryCompareResult {
+    fromResults: DecodedBqrsChunk,
+    toResults: DecodedBqrsChunk,
+  ): RawQueryCompareResult {
     // Only compare columns that have the same name
     return resultsDiff(fromResults, toResults);
   }

extensions/ql-vscode/src/compare/result-set-names.ts

@@ -0,0 +1,56 @@
+import { BQRSInfo } from "../common/bqrs-cli-types";
+import { getDefaultResultSetName } from "../common/interface-types";
+
+export async function findCommonResultSetNames(
+  fromSchemas: BQRSInfo,
+  toSchemas: BQRSInfo,
+): Promise<string[]> {
+  const fromSchemaNames = fromSchemas["result-sets"].map(
+    (schema) => schema.name,
+  );
+  const toSchemaNames = toSchemas["result-sets"].map((schema) => schema.name);
+
+  return fromSchemaNames.filter((name) => toSchemaNames.includes(name));
+}
+
+export async function findResultSetNames(
+  fromSchemas: BQRSInfo,
+  toSchemas: BQRSInfo,
+  commonResultSetNames: readonly string[],
+  selectedResultSetName: string | undefined,
+) {
+  const fromSchemaNames = fromSchemas["result-sets"].map(
+    (schema) => schema.name,
+  );
+  const toSchemaNames = toSchemas["result-sets"].map((schema) => schema.name);
+
+  // Fall back on the default result set names if there are no common ones.
+  const defaultFromResultSetName = fromSchemaNames.find((name) =>
+    name.startsWith("#"),
+  );
+  const defaultToResultSetName = toSchemaNames.find((name) =>
+    name.startsWith("#"),
+  );
+
+  if (
+    commonResultSetNames.length === 0 &&
+    !(defaultFromResultSetName || defaultToResultSetName)
+  ) {
+    throw new Error(
+      "No common result sets found between the two queries. Please check that the queries are compatible.",
+    );
+  }
+
+  const currentResultSetName =
+    selectedResultSetName ?? getDefaultResultSetName(commonResultSetNames);
+  const fromResultSetName = currentResultSetName || defaultFromResultSetName!;
+  const toResultSetName = currentResultSetName || defaultToResultSetName!;
+
+  return {
+    currentResultSetDisplayName:
+      currentResultSetName ||
+      `${defaultFromResultSetName} <-> ${defaultToResultSetName}`,
+    fromResultSetName,
+    toResultSetName,
+  };
+}

extensions/ql-vscode/src/compare/resultsDiff.ts

@@ -1,5 +1,5 @@
-import { RawResultSet } from "../common/bqrs-cli-types";
-import { QueryCompareResult } from "../common/interface-types";
+import { DecodedBqrsChunk } from "../common/bqrs-cli-types";
+import { RawQueryCompareResult } from "../common/interface-types";
 
 /**
  * Compare the rows of two queries. Use deep equality to determine if
@@ -20,29 +20,31 @@ import { QueryCompareResult } from "../common/interface-types";
  *  3. If the queries are 100% disjoint
  */
 export default function resultsDiff(
-  fromResults: RawResultSet,
-  toResults: RawResultSet,
-): QueryCompareResult {
-  if (fromResults.schema.columns.length !== toResults.schema.columns.length) {
+  fromResults: DecodedBqrsChunk,
+  toResults: DecodedBqrsChunk,
+): RawQueryCompareResult {
+  if (fromResults.columns.length !== toResults.columns.length) {
     throw new Error("CodeQL Compare: Columns do not match.");
   }
 
-  if (!fromResults.rows.length) {
+  if (!fromResults.tuples.length) {
     throw new Error("CodeQL Compare: Source query has no results.");
   }
 
-  if (!toResults.rows.length) {
+  if (!toResults.tuples.length) {
     throw new Error("CodeQL Compare: Target query has no results.");
   }
 
-  const results = {
-    from: arrayDiff(fromResults.rows, toResults.rows),
-    to: arrayDiff(toResults.rows, fromResults.rows),
+  const results: RawQueryCompareResult = {
+    kind: "raw",
+    columns: fromResults.columns,
+    from: arrayDiff(fromResults.tuples, toResults.tuples),
+    to: arrayDiff(toResults.tuples, fromResults.tuples),
   };
 
   if (
-    fromResults.rows.length === results.from.length &&
-    toResults.rows.length === results.to.length
+    fromResults.tuples.length === results.from.length &&
+    toResults.tuples.length === results.to.length
   ) {
     throw new Error("CodeQL Compare: No overlap between the selected queries.");
   }

extensions/ql-vscode/src/config.ts

@@ -1,19 +1,19 @@
 import { DisposableObject } from "./common/disposable-object";
 import {
-  workspace,
+  ConfigurationChangeEvent,
+  ConfigurationScope,
+  ConfigurationTarget,
   Event,
   EventEmitter,
-  ConfigurationChangeEvent,
-  ConfigurationTarget,
-  ConfigurationScope,
+  workspace,
 } from "vscode";
 import { DistributionManager } from "./codeql-cli/distribution";
 import { extLogger } from "./common/logging/vscode";
 import { ONE_DAY_IN_MS } from "./common/time";
 import {
+  defaultFilterSortState,
   FilterKey,
   SortKey,
-  defaultFilterSortState,
 } from "./variant-analysis/shared/variant-analysis-filter-sort";
 
 export const ALL_SETTINGS: Setting[] = [];
@@ -119,6 +119,14 @@ export interface DistributionConfig {
   ownerName?: string;
   repositoryName?: string;
   onDidChangeConfiguration?: Event<void>;
+
+  /**
+   * This forces an update of the distribution, even if the settings haven't changed.
+   *
+   * This should only be used when the distribution has been updated outside of the extension
+   * and only in tests. It should not be called in production code.
+   */
+  forceUpdateConfiguration(): void;
 }
 
 // Query server configuration
@@ -275,6 +283,10 @@ export class DistributionConfigListener
     );
   }
 
+  public forceUpdateConfiguration() {
+    this._onDidChangeConfiguration.fire(undefined);
+  }
+
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
     this.handleDidChangeConfigurationForRelevantSettings(
       DISTRIBUTION_CHANGE_SETTINGS,
@@ -449,20 +461,6 @@ export function isCanary() {
   return !!CANARY_FEATURES.getValue<boolean>();
 }
 
-/**
- * Enables the experimental query server
- */
-export const CANARY_QUERY_SERVER = new Setting(
-  "canaryQueryServer",
-  ROOT_SETTING,
-);
-
-// The default value for this setting is now `true`
-export function allowCanaryQueryServer() {
-  const value = CANARY_QUERY_SERVER.getValue<boolean>();
-  return value === undefined ? true : !!value;
-}
-
 const LOG_INSIGHTS_SETTING = new Setting("logInsights", ROOT_SETTING);
 export const JOIN_ORDER_WARNING_THRESHOLD = new Setting(
   "joinOrderWarningThreshold",
@@ -641,12 +639,32 @@ export function isCodespacesTemplate() {
   return !!CODESPACES_TEMPLATE.getValue<boolean>();
 }
 
+// Deprecated after v1.9.4. Can be removed in a few versions.
 const DATABASE_DOWNLOAD_SETTING = new Setting("databaseDownload", ROOT_SETTING);
+const DEPRECATED_ALLOW_HTTP_SETTING = new Setting(
+  "allowHttp",
+  DATABASE_DOWNLOAD_SETTING,
+);
 
-const ALLOW_HTTP_SETTING = new Setting("allowHttp", DATABASE_DOWNLOAD_SETTING);
+const ADDING_DATABASES_SETTING = new Setting("addingDatabases", ROOT_SETTING);
+
+const ALLOW_HTTP_SETTING = new Setting("allowHttp", ADDING_DATABASES_SETTING);
 
 export function allowHttp(): boolean {
-  return ALLOW_HTTP_SETTING.getValue<boolean>() || false;
+  return (
+    ALLOW_HTTP_SETTING.getValue<boolean>() ||
+    DEPRECATED_ALLOW_HTTP_SETTING.getValue<boolean>() ||
+    false
+  );
+}
+
+const ADD_DATABASE_SOURCE_TO_WORKSPACE_SETTING = new Setting(
+  "addDatabaseSourceToWorkspace",
+  ADDING_DATABASES_SETTING,
+);
+
+export function addDatabaseSourceToWorkspace(): boolean {
+  return ADD_DATABASE_SOURCE_TO_WORKSPACE_SETTING.getValue<boolean>() || false;
 }
 
 /**
@@ -690,26 +708,25 @@ export async function setAutogenerateQlPacks(choice: AutogenerateQLPacks) {
   );
 }
 
-/**
- * A flag indicating whether to show the queries panel in the QL view container.
- */
-const QUERIES_PANEL = new Setting("queriesPanel", ROOT_SETTING);
-
-export function showQueriesPanel(): boolean {
-  return !!QUERIES_PANEL.getValue<boolean>();
-}
-
 const MODEL_SETTING = new Setting("model", ROOT_SETTING);
 const FLOW_GENERATION = new Setting("flowGeneration", MODEL_SETTING);
 const LLM_GENERATION = new Setting("llmGeneration", MODEL_SETTING);
+const LLM_GENERATION_BATCH_SIZE = new Setting(
+  "llmGenerationBatchSize",
+  MODEL_SETTING,
+);
+const LLM_GENERATION_DEV_ENDPOINT = new Setting(
+  "llmGenerationDevEndpoint",
+  MODEL_SETTING,
+);
 const EXTENSIONS_DIRECTORY = new Setting("extensionsDirectory", MODEL_SETTING);
-const SHOW_MULTIPLE_MODELS = new Setting("showMultipleModels", MODEL_SETTING);
+const ENABLE_RUBY = new Setting("enableRuby", MODEL_SETTING);
 
 export interface ModelConfig {
   flowGeneration: boolean;
   llmGeneration: boolean;
   getExtensionsDirectory(languageId: string): string | undefined;
-  showMultipleModels: boolean;
+  enableRuby: boolean;
 }
 
 export class ModelConfigListener extends ConfigListener implements ModelConfig {
@@ -725,13 +742,100 @@ export class ModelConfigListener extends ConfigListener implements ModelConfig {
     return !!LLM_GENERATION.getValue<boolean>();
   }
 
+  /**
+   * Limits the number of candidates we send to the model in each request to avoid long requests.
+   * Note that the model may return fewer than this number of candidates.
+   */
+  public get llmGenerationBatchSize(): number {
+    return LLM_GENERATION_BATCH_SIZE.getValue<number | null>() || 5;
+  }
+
+  /**
+   * The URL of the endpoint to use for LLM generation. This should only be set
+   * if you want to test against a dev server.
+   */
+  public get llmGenerationDevEndpoint(): string | undefined {
+    return LLM_GENERATION_DEV_ENDPOINT.getValue<string | undefined>();
+  }
+
   public getExtensionsDirectory(languageId: string): string | undefined {
     return EXTENSIONS_DIRECTORY.getValue<string>({
       languageId,
     });
   }
 
-  public get showMultipleModels(): boolean {
-    return !!SHOW_MULTIPLE_MODELS.getValue<boolean>();
+  public get enableRuby(): boolean {
+    return !!ENABLE_RUBY.getValue<boolean>();
+  }
+}
+
+const GITHUB_DATABASE_SETTING = new Setting("githubDatabase", ROOT_SETTING);
+
+// Feature flag for the GitHub database downnload.
+const GITHUB_DATABASE_ENABLE = new Setting("enable", GITHUB_DATABASE_SETTING);
+const GITHUB_DATABASE_DOWNLOAD = new Setting(
+  "download",
+  GITHUB_DATABASE_SETTING,
+);
+
+const GitHubDatabaseDownloadValues = ["ask", "never"] as const;
+type GitHubDatabaseDownload = (typeof GitHubDatabaseDownloadValues)[number];
+
+const GITHUB_DATABASE_UPDATE = new Setting("update", GITHUB_DATABASE_SETTING);
+
+const GitHubDatabaseUpdateValues = ["ask", "never"] as const;
+type GitHubDatabaseUpdate = (typeof GitHubDatabaseUpdateValues)[number];
+
+export interface GitHubDatabaseConfig {
+  enable: boolean;
+  download: GitHubDatabaseDownload;
+  update: GitHubDatabaseUpdate;
+  setDownload(
+    value: GitHubDatabaseDownload,
+    target?: ConfigurationTarget,
+  ): Promise<void>;
+  setUpdate(
+    value: GitHubDatabaseUpdate,
+    target?: ConfigurationTarget,
+  ): Promise<void>;
+}
+
+export class GitHubDatabaseConfigListener
+  extends ConfigListener
+  implements GitHubDatabaseConfig
+{
+  protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
+    this.handleDidChangeConfigurationForRelevantSettings(
+      [GITHUB_DATABASE_SETTING],
+      e,
+    );
+  }
+
+  public get enable() {
+    return !!GITHUB_DATABASE_ENABLE.getValue<boolean>();
+  }
+
+  public get download(): GitHubDatabaseDownload {
+    const value = GITHUB_DATABASE_DOWNLOAD.getValue<GitHubDatabaseDownload>();
+    return GitHubDatabaseDownloadValues.includes(value) ? value : "ask";
+  }
+
+  public get update(): GitHubDatabaseUpdate {
+    const value = GITHUB_DATABASE_UPDATE.getValue<GitHubDatabaseUpdate>();
+    return GitHubDatabaseUpdateValues.includes(value) ? value : "ask";
+  }
+
+  public async setDownload(
+    value: GitHubDatabaseDownload,
+    target: ConfigurationTarget = ConfigurationTarget.Workspace,
+  ): Promise<void> {
+    await GITHUB_DATABASE_DOWNLOAD.updateValue(value, target);
+  }
+
+  public async setUpdate(
+    value: GitHubDatabaseUpdate,
+    target: ConfigurationTarget = ConfigurationTarget.Workspace,
+  ): Promise<void> {
+    await GITHUB_DATABASE_UPDATE.updateValue(value, target);
   }
 }

extensions/ql-vscode/src/databases/code-search-api.ts

@@ -1,22 +1,24 @@
 import { throttling } from "@octokit/plugin-throttling";
 import { Octokit } from "@octokit/rest";
-import { Progress, CancellationToken } from "vscode";
+import { CancellationToken } from "vscode";
 import { Credentials } from "../common/authentication";
 import { BaseLogger } from "../common/logging";
 import { AppOctokit } from "../common/octokit";
+import {
+  ProgressCallback,
+  UserCancellationException,
+} from "../common/vscode/progress";
 
 export async function getCodeSearchRepositories(
   query: string,
-  progress: Progress<{
-    message?: string | undefined;
-    increment?: number | undefined;
-  }>,
+  progress: ProgressCallback,
   token: CancellationToken,
   credentials: Credentials,
   logger: BaseLogger,
 ): Promise<string[]> {
-  let nwos: string[] = [];
+  const nwos: string[] = [];
   const octokit = await provideOctokitWithThrottling(credentials, logger);
+  let i = 0;
 
   for await (const response of octokit.paginate.iterator(
     octokit.rest.search.code,
@@ -25,17 +27,19 @@ export async function getCodeSearchRepositories(
       per_page: 100,
     },
   )) {
+    i++;
     nwos.push(...response.data.map((item) => item.repository.full_name));
-    // calculate progress bar: 80% of the progress bar is used for the code search
-    const totalNumberOfRequests = Math.ceil(response.data.total_count / 100);
-    // Since we have a maximum of 1000 responses of the api, we can use a fixed increment whenever the totalNumberOfRequests would be greater than 10
-    const increment =
-      totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8;
-    progress.report({ increment });
+    const totalNumberOfResultPages = Math.ceil(response.data.total_count / 100);
+    const totalNumberOfRequests =
+      totalNumberOfResultPages > 10 ? 10 : totalNumberOfResultPages;
+    progress({
+      maxStep: totalNumberOfRequests,
+      step: i,
+      message: "Sending API requests to get Code Search results.",
+    });
 
     if (token.isCancellationRequested) {
-      nwos = [];
-      break;
+      throw new UserCancellationException("Code search cancelled.", true);
     }
   }
 

extensions/ql-vscode/src/databases/config/db-config-store.ts

@@ -1,17 +1,11 @@
 import { pathExists, outputJSON, readJSON, readJSONSync } from "fs-extra";
 import { join } from "path";
 import {
-  clearLocalDbConfig,
   cloneDbConfig,
   DbConfig,
-  initializeLocalDbConfig,
-  removeLocalDb,
-  removeLocalList,
   removeRemoteList,
   removeRemoteOwner,
   removeRemoteRepo,
-  renameLocalDb,
-  renameLocalList,
   renameRemoteList,
   SelectedDbItem,
   DB_CONFIG_VERSION,
@@ -30,13 +24,7 @@ import {
   DbConfigValidationErrorKind,
 } from "../db-validation-errors";
 import { ValueResult } from "../../common/value-result";
-import {
-  LocalDatabaseDbItem,
-  LocalListDbItem,
-  RemoteUserDefinedListDbItem,
-  DbItem,
-  DbItemKind,
-} from "../db-item";
+import { RemoteUserDefinedListDbItem, DbItem, DbItemKind } from "../db-item";
 
 export class DbConfigStore extends DisposableObject {
   public static readonly databaseConfigFileName = "databases.json";
@@ -119,20 +107,9 @@ export class DbConfigStore extends DisposableObject {
     let config: DbConfig;
 
     switch (dbItem.kind) {
-      case DbItemKind.LocalList:
-        config = removeLocalList(this.config, dbItem.listName);
-        break;
       case DbItemKind.RemoteUserDefinedList:
         config = removeRemoteList(this.config, dbItem.listName);
         break;
-      case DbItemKind.LocalDatabase:
-        // When we start using local databases these need to be removed from disk as well.
-        config = removeLocalDb(
-          this.config,
-          dbItem.databaseName,
-          dbItem.parentListName,
-        );
-        break;
       case DbItemKind.RemoteRepo:
         config = removeRemoteRepo(
           this.config,
@@ -229,22 +206,6 @@ export class DbConfigStore extends DisposableObject {
     await this.writeConfig(config);
   }
 
-  public async addLocalList(listName: string): Promise<void> {
-    if (!this.config) {
-      throw Error("Cannot add local list if config is not loaded");
-    }
-
-    this.validateLocalListName(listName);
-
-    const config = cloneDbConfig(this.config);
-    config.databases.local.lists.push({
-      name: listName,
-      databases: [],
-    });
-
-    await this.writeConfig(config);
-  }
-
   public async addRemoteList(listName: string): Promise<void> {
     if (!this.config) {
       throw Error("Cannot add variant analysis list if config is not loaded");
@@ -261,25 +222,6 @@ export class DbConfigStore extends DisposableObject {
     await this.writeConfig(config);
   }
 
-  public async renameLocalList(
-    currentDbItem: LocalListDbItem,
-    newName: string,
-  ) {
-    if (!this.config) {
-      throw Error("Cannot rename local list if config is not loaded");
-    }
-
-    this.validateLocalListName(newName);
-
-    const updatedConfig = renameLocalList(
-      this.config,
-      currentDbItem.listName,
-      newName,
-    );
-
-    await this.writeConfig(updatedConfig);
-  }
-
   public async renameRemoteList(
     currentDbItem: RemoteUserDefinedListDbItem,
     newName: string,
@@ -301,27 +243,6 @@ export class DbConfigStore extends DisposableObject {
     await this.writeConfig(updatedConfig);
   }
 
-  public async renameLocalDb(
-    currentDbItem: LocalDatabaseDbItem,
-    newName: string,
-    parentListName?: string,
-  ): Promise<void> {
-    if (!this.config) {
-      throw Error("Cannot rename local db if config is not loaded");
-    }
-
-    this.validateLocalDbName(newName);
-
-    const updatedConfig = renameLocalDb(
-      this.config,
-      currentDbItem.databaseName,
-      newName,
-      parentListName,
-    );
-
-    await this.writeConfig(updatedConfig);
-  }
-
   public doesRemoteListExist(listName: string): boolean {
     if (!this.config) {
       throw Error(
@@ -334,31 +255,6 @@ export class DbConfigStore extends DisposableObject {
     );
   }
 
-  public doesLocalListExist(listName: string): boolean {
-    if (!this.config) {
-      throw Error("Cannot check local list existence if config is not loaded");
-    }
-
-    return this.config.databases.local.lists.some((l) => l.name === listName);
-  }
-
-  public doesLocalDbExist(dbName: string, listName?: string): boolean {
-    if (!this.config) {
-      throw Error(
-        "Cannot check variant analysis repository existence if config is not loaded",
-      );
-    }
-
-    if (listName) {
-      return this.config.databases.local.lists.some(
-        (l) =>
-          l.name === listName && l.databases.some((d) => d.name === dbName),
-      );
-    }
-
-    return this.config.databases.local.databases.some((d) => d.name === dbName);
-  }
-
   public doesRemoteDbExist(dbName: string, listName?: string): boolean {
     if (!this.config) {
       throw Error(
@@ -384,7 +280,6 @@ export class DbConfigStore extends DisposableObject {
   }
 
   private async writeConfig(config: DbConfig): Promise<void> {
-    clearLocalDbConfig(config);
     await outputJSON(this.configPath, config, {
       spaces: 2,
     });
@@ -416,7 +311,6 @@ export class DbConfigStore extends DisposableObject {
     }
 
     if (newConfig) {
-      initializeLocalDbConfig(newConfig);
       this.configErrors = this.configValidator.validate(newConfig);
     }
 
@@ -451,7 +345,6 @@ export class DbConfigStore extends DisposableObject {
     }
 
     if (newConfig) {
-      initializeLocalDbConfig(newConfig);
       this.configErrors = this.configValidator.validate(newConfig);
     }
 
@@ -499,10 +392,6 @@ export class DbConfigStore extends DisposableObject {
           owners: [],
           repositories: [],
         },
-        local: {
-          lists: [],
-          databases: [],
-        },
       },
       selected: {
         kind: SelectedDbItemKind.VariantAnalysisSystemDefinedList,
@@ -511,16 +400,6 @@ export class DbConfigStore extends DisposableObject {
     };
   }
 
-  private validateLocalListName(listName: string): void {
-    if (listName === "") {
-      throw Error("List name cannot be empty");
-    }
-
-    if (this.doesLocalListExist(listName)) {
-      throw Error(`A local list with the name '${listName}' already exists`);
-    }
-  }
-
   private validateRemoteListName(listName: string): void {
     if (listName === "") {
       throw Error("List name cannot be empty");
@@ -532,14 +411,4 @@ export class DbConfigStore extends DisposableObject {
       );
     }
   }
-
-  private validateLocalDbName(dbName: string): void {
-    if (dbName === "") {
-      throw Error("Database name cannot be empty");
-    }
-
-    if (this.doesLocalDbExist(dbName)) {
-      throw Error(`A local database with the name '${dbName}' already exists`);
-    }
-  }
 }

extensions/ql-vscode/src/databases/config/db-config-validator.ts

@@ -1,7 +1,7 @@
 import { readJsonSync } from "fs-extra";
 import { resolve } from "path";
 import Ajv, { ValidateFunction } from "ajv";
-import { clearLocalDbConfig, DbConfig } from "./db-config";
+import { DbConfig } from "./db-config";
 import { findDuplicateStrings } from "../../common/text-utils";
 import {
   DbConfigValidationError,
@@ -19,8 +19,6 @@ export class DbConfigValidator {
   }
 
   public validate(dbConfig: DbConfig): DbConfigValidationError[] {
-    const localDbs = clearLocalDbConfig(dbConfig);
-
     this.validateSchemaFn(dbConfig);
 
     if (this.validateSchemaFn.errors) {
@@ -30,13 +28,6 @@ export class DbConfigValidator {
       }));
     }
 
-    // Add any local db config back so that we have a config
-    // object that respects its type and validation can happen
-    // as normal.
-    if (localDbs) {
-      dbConfig.databases.local = localDbs;
-    }
-
     return [
       ...this.validateDbListNames(dbConfig),
       ...this.validateDbNames(dbConfig),
@@ -55,14 +46,6 @@ export class DbConfigValidator {
       )}`,
     });
 
-    const duplicateLocalDbLists = findDuplicateStrings(
-      dbConfig.databases.local.lists.map((n) => n.name),
-    );
-
-    if (duplicateLocalDbLists.length > 0) {
-      errors.push(buildError(duplicateLocalDbLists));
-    }
-
     const duplicateRemoteDbLists = findDuplicateStrings(
       dbConfig.databases.variantAnalysis.repositoryLists.map((n) => n.name),
     );
@@ -81,14 +64,6 @@ export class DbConfigValidator {
       message: `There are databases with the same name: ${dups.join(", ")}`,
     });
 
-    const duplicateLocalDbs = findDuplicateStrings(
-      dbConfig.databases.local.databases.map((d) => d.name),
-    );
-
-    if (duplicateLocalDbs.length > 0) {
-      errors.push(buildError(duplicateLocalDbs));
-    }
-
     const duplicateRemoteDbs = findDuplicateStrings(
       dbConfig.databases.variantAnalysis.repositories,
     );
@@ -111,13 +86,6 @@ export class DbConfigValidator {
       )}`,
     });
 
-    for (const list of dbConfig.databases.local.lists) {
-      const dups = findDuplicateStrings(list.databases.map((d) => d.name));
-      if (dups.length > 0) {
-        errors.push(buildError(list.name, dups));
-      }
-    }
-
     for (const list of dbConfig.databases.variantAnalysis.repositoryLists) {
       const dups = findDuplicateStrings(list.repositories);
       if (dups.length > 0) {

extensions/ql-vscode/src/databases/config/db-config.ts

@@ -11,37 +11,21 @@ export interface DbConfig {
 
 interface DbConfigDatabases {
   variantAnalysis: RemoteDbConfig;
-  local: LocalDbConfig;
 }
 
 export type SelectedDbItem =
-  | SelectedLocalUserDefinedList
-  | SelectedLocalDatabase
   | SelectedRemoteSystemDefinedList
   | SelectedVariantAnalysisUserDefinedList
   | SelectedRemoteOwner
   | SelectedRemoteRepository;
 
 export enum SelectedDbItemKind {
-  LocalUserDefinedList = "localUserDefinedList",
-  LocalDatabase = "localDatabase",
   VariantAnalysisSystemDefinedList = "variantAnalysisSystemDefinedList",
   VariantAnalysisUserDefinedList = "variantAnalysisUserDefinedList",
   VariantAnalysisOwner = "variantAnalysisOwner",
   VariantAnalysisRepository = "variantAnalysisRepository",
 }
 
-interface SelectedLocalUserDefinedList {
-  kind: SelectedDbItemKind.LocalUserDefinedList;
-  listName: string;
-}
-
-interface SelectedLocalDatabase {
-  kind: SelectedDbItemKind.LocalDatabase;
-  databaseName: string;
-  listName?: string;
-}
-
 interface SelectedRemoteSystemDefinedList {
   kind: SelectedDbItemKind.VariantAnalysisSystemDefinedList;
   listName: string;
@@ -74,23 +58,6 @@ export interface RemoteRepositoryList {
   repositories: string[];
 }
 
-interface LocalDbConfig {
-  lists: LocalList[];
-  databases: LocalDatabase[];
-}
-
-export interface LocalList {
-  name: string;
-  databases: LocalDatabase[];
-}
-
-export interface LocalDatabase {
-  name: string;
-  dateAdded: number;
-  language: string;
-  storagePath: string;
-}
-
 export function cloneDbConfig(config: DbConfig): DbConfig {
   return {
     version: config.version,
@@ -105,13 +72,6 @@ export function cloneDbConfig(config: DbConfig): DbConfig {
         owners: [...config.databases.variantAnalysis.owners],
         repositories: [...config.databases.variantAnalysis.repositories],
       },
-      local: {
-        lists: config.databases.local.lists.map((list) => ({
-          name: list.name,
-          databases: list.databases.map((db) => ({ ...db })),
-        })),
-        databases: config.databases.local.databases.map((db) => ({ ...db })),
-      },
     },
     selected: config.selected
       ? cloneDbConfigSelectedItem(config.selected)
@@ -119,28 +79,6 @@ export function cloneDbConfig(config: DbConfig): DbConfig {
   };
 }
 
-export function renameLocalList(
-  originalConfig: DbConfig,
-  currentListName: string,
-  newListName: string,
-): DbConfig {
-  const config = cloneDbConfig(originalConfig);
-
-  const list = getLocalList(config, currentListName);
-  list.name = newListName;
-
-  if (
-    config.selected?.kind === SelectedDbItemKind.LocalUserDefinedList ||
-    config.selected?.kind === SelectedDbItemKind.LocalDatabase
-  ) {
-    if (config.selected.listName === currentListName) {
-      config.selected.listName = newListName;
-    }
-  }
-
-  return config;
-}
-
 export function renameRemoteList(
   originalConfig: DbConfig,
   currentListName: string,
@@ -164,67 +102,6 @@ export function renameRemoteList(
   return config;
 }
 
-export function renameLocalDb(
-  originalConfig: DbConfig,
-  currentDbName: string,
-  newDbName: string,
-  parentListName?: string,
-): DbConfig {
-  const config = cloneDbConfig(originalConfig);
-
-  if (parentListName) {
-    const list = getLocalList(config, parentListName);
-    const dbIndex = list.databases.findIndex((db) => db.name === currentDbName);
-    if (dbIndex === -1) {
-      throw Error(
-        `Cannot find database '${currentDbName}' in list '${parentListName}'`,
-      );
-    }
-    list.databases[dbIndex].name = newDbName;
-  } else {
-    const dbIndex = config.databases.local.databases.findIndex(
-      (db) => db.name === currentDbName,
-    );
-    if (dbIndex === -1) {
-      throw Error(`Cannot find database '${currentDbName}' in local databases`);
-    }
-    config.databases.local.databases[dbIndex].name = newDbName;
-  }
-
-  if (
-    config.selected?.kind === SelectedDbItemKind.LocalDatabase &&
-    config.selected.databaseName === currentDbName
-  ) {
-    config.selected.databaseName = newDbName;
-  }
-
-  return config;
-}
-
-export function removeLocalList(
-  originalConfig: DbConfig,
-  listName: string,
-): DbConfig {
-  const config = cloneDbConfig(originalConfig);
-
-  config.databases.local.lists = config.databases.local.lists.filter(
-    (list) => list.name !== listName,
-  );
-
-  if (config.selected?.kind === SelectedDbItemKind.LocalUserDefinedList) {
-    config.selected = undefined;
-  }
-
-  if (
-    config.selected?.kind === SelectedDbItemKind.LocalDatabase &&
-    config.selected?.listName === listName
-  ) {
-    config.selected = undefined;
-  }
-
-  return config;
-}
-
 export function removeRemoteList(
   originalConfig: DbConfig,
   listName: string,
@@ -252,35 +129,6 @@ export function removeRemoteList(
   return config;
 }
 
-export function removeLocalDb(
-  originalConfig: DbConfig,
-  databaseName: string,
-  parentListName?: string,
-): DbConfig {
-  const config = cloneDbConfig(originalConfig);
-
-  if (parentListName) {
-    const parentList = getLocalList(config, parentListName);
-    parentList.databases = parentList.databases.filter(
-      (db) => db.name !== databaseName,
-    );
-  } else {
-    config.databases.local.databases = config.databases.local.databases.filter(
-      (db) => db.name !== databaseName,
-    );
-  }
-
-  if (
-    config.selected?.kind === SelectedDbItemKind.LocalDatabase &&
-    config.selected?.databaseName === databaseName &&
-    config.selected?.listName === parentListName
-  ) {
-    config.selected = undefined;
-  }
-
-  return config;
-}
-
 export function removeRemoteRepo(
   originalConfig: DbConfig,
   repoFullName: string,
@@ -330,51 +178,8 @@ export function removeRemoteOwner(
   return config;
 }
 
-/**
- * Removes local db config from a db config object, if one is set.
- * We do this because we don't want to expose this feature to users
- * yet (since it's only partially implemented), but we also don't want
- * to remove all the code we've already implemented.
- * @param config The config object to change.
- * @returns Any removed local db config.
- */
-export function clearLocalDbConfig(
-  config: DbConfig,
-): LocalDbConfig | undefined {
-  let localDbs = undefined;
-
-  if (config && config.databases && config.databases.local) {
-    localDbs = config.databases.local;
-    delete (config.databases as any).local;
-  }
-
-  return localDbs;
-}
-
-/**
- * Initializes the local db config, if the config object contains
- * database configuration.
- * @param config The config object to change.
- */
-export function initializeLocalDbConfig(config: DbConfig): void {
-  if (config.databases) {
-    config.databases.local = { lists: [], databases: [] };
-  }
-}
-
 function cloneDbConfigSelectedItem(selected: SelectedDbItem): SelectedDbItem {
   switch (selected.kind) {
-    case SelectedDbItemKind.LocalUserDefinedList:
-      return {
-        kind: SelectedDbItemKind.LocalUserDefinedList,
-        listName: selected.listName,
-      };
-    case SelectedDbItemKind.LocalDatabase:
-      return {
-        kind: SelectedDbItemKind.LocalDatabase,
-        databaseName: selected.databaseName,
-        listName: selected.listName,
-      };
     case SelectedDbItemKind.VariantAnalysisSystemDefinedList:
       return {
         kind: SelectedDbItemKind.VariantAnalysisSystemDefinedList,
@@ -399,16 +204,6 @@ function cloneDbConfigSelectedItem(selected: SelectedDbItem): SelectedDbItem {
   }
 }
 
-function getLocalList(config: DbConfig, listName: string): LocalList {
-  const list = config.databases.local.lists.find((l) => l.name === listName);
-
-  if (!list) {
-    throw Error(`Cannot find local list '${listName}'`);
-  }
-
-  return list;
-}
-
 function getRemoteList(
   config: DbConfig,
   listName: string,

extensions/ql-vscode/src/databases/database-fetcher.ts

@@ -29,9 +29,11 @@ import {
 } from "../common/github-url-identifier-helper";
 import { Credentials } from "../common/authentication";
 import { AppCommandManager } from "../common/commands";
-import { allowHttp } from "../config";
+import { addDatabaseSourceToWorkspace, allowHttp } from "../config";
 import { showAndLogInformationMessage } from "../common/logging";
 import { AppOctokit } from "../common/octokit";
+import { getLanguageDisplayName } from "../common/query-language";
+import { DatabaseOrigin } from "./local-databases/database-origin";
 
 /**
  * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
@@ -61,6 +63,10 @@ export async function promptImportInternetDatabase(
     databaseManager,
     storagePath,
     undefined,
+    {
+      type: "url",
+      url: databaseUrl,
+    },
     progress,
     cli,
   );
@@ -98,7 +104,7 @@ export async function promptImportGithubDatabase(
   cli?: CodeQLCliServer,
   language?: string,
   makeSelected = true,
-  addSourceArchiveFolder = true,
+  addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
 ): Promise<DatabaseItem | undefined> {
   const githubRepo = await askForGitHubRepo(progress);
   if (!githubRepo) {
@@ -177,7 +183,7 @@ export async function downloadGitHubDatabase(
   cli?: CodeQLCliServer,
   language?: string,
   makeSelected = true,
-  addSourceArchiveFolder = true,
+  addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
 ): Promise<DatabaseItem | undefined> {
   const nwo = getNwoFromGitHubUrl(githubRepo) || githubRepo;
   if (!isValidGitHubNwo(nwo)) {
@@ -198,8 +204,41 @@ export async function downloadGitHubDatabase(
     return;
   }
 
-  const { databaseUrl, name, owner } = result;
+  const { databaseUrl, name, owner, databaseId, databaseCreatedAt, commitOid } =
+    result;
 
+  return downloadGitHubDatabaseFromUrl(
+    databaseUrl,
+    databaseId,
+    databaseCreatedAt,
+    commitOid,
+    owner,
+    name,
+    octokit,
+    progress,
+    databaseManager,
+    storagePath,
+    cli,
+    makeSelected,
+    addSourceArchiveFolder,
+  );
+}
+
+export async function downloadGitHubDatabaseFromUrl(
+  databaseUrl: string,
+  databaseId: number,
+  databaseCreatedAt: string,
+  commitOid: string | null,
+  owner: string,
+  name: string,
+  octokit: Octokit.Octokit,
+  progress: ProgressCallback,
+  databaseManager: DatabaseManager,
+  storagePath: string,
+  cli?: CodeQLCliServer,
+  makeSelected = true,
+  addSourceArchiveFolder = true,
+): Promise<DatabaseItem | undefined> {
   /**
    * The 'token' property of the token object returned by `octokit.auth()`.
    * The object is undocumented, but looks something like this:
@@ -220,6 +259,13 @@ export async function downloadGitHubDatabase(
     databaseManager,
     storagePath,
     `${owner}/${name}`,
+    {
+      type: "github",
+      repository: `${owner}/${name}`,
+      databaseId,
+      databaseCreatedAt,
+      commitOid,
+    },
     progress,
     cli,
     makeSelected,
@@ -249,6 +295,10 @@ export async function importArchiveDatabase(
       databaseManager,
       storagePath,
       undefined,
+      {
+        type: "archive",
+        path: databaseUrl,
+      },
       progress,
       cli,
     );
@@ -281,6 +331,7 @@ export async function importArchiveDatabase(
  * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  * @param nameOverride a name for the database that overrides the default
+ * @param origin the origin of the database
  * @param progress callback to send progress messages to
  * @param makeSelected make the new database selected in the databases panel (default: true)
  * @param addSourceArchiveFolder whether to add a workspace folder containing the source archive to the workspace
@@ -291,10 +342,11 @@ async function databaseArchiveFetcher(
   databaseManager: DatabaseManager,
   storagePath: string,
   nameOverride: string | undefined,
+  origin: DatabaseOrigin,
   progress: ProgressCallback,
   cli?: CodeQLCliServer,
   makeSelected = true,
-  addSourceArchiveFolder = true,
+  addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
 ): Promise<DatabaseItem> {
   progress({
     message: "Getting database",
@@ -335,6 +387,7 @@ async function databaseArchiveFetcher(
 
     const item = await databaseManager.openDatabase(
       Uri.file(dbPath),
+      origin,
       makeSelected,
       nameOverride,
       {
@@ -475,7 +528,7 @@ async function checkForFailingResponse(
     return response;
   }
 
-  // An error downloading the database. Attempt to extract the resaon behind it.
+  // An error downloading the database. Attempt to extract the reason behind it.
   const text = await response.text();
   let msg: string;
   try {
@@ -532,16 +585,19 @@ export async function convertGithubNwoToDatabaseUrl(
       databaseUrl: string;
       owner: string;
       name: string;
+      databaseId: number;
+      databaseCreatedAt: string;
+      commitOid: string | null;
     }
   | undefined
 > {
   try {
     const [owner, repo] = nwo.split("/");
 
-    const response = await octokit.request(
-      "GET /repos/:owner/:repo/code-scanning/codeql/databases",
-      { owner, repo },
-    );
+    const response = await octokit.rest.codeScanning.listCodeqlDatabases({
+      owner,
+      repo,
+    });
 
     const languages = response.data.map((db: any) => db.language);
 
@@ -552,10 +608,20 @@ export async function convertGithubNwoToDatabaseUrl(
       }
     }
 
+    const databaseForLanguage = response.data.find(
+      (db) => db.language === language,
+    );
+    if (!databaseForLanguage) {
+      throw new Error(`No database found for language '${language}'`);
+    }
+
     return {
-      databaseUrl: `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`,
+      databaseUrl: databaseForLanguage.url,
       owner,
       name: repo,
+      databaseId: databaseForLanguage.id,
+      databaseCreatedAt: databaseForLanguage.created_at,
+      commitOid: databaseForLanguage.commit_oid ?? null,
     };
   } catch (e) {
     void extLogger.log(`Error: ${getErrorMessage(e)}`);
@@ -565,9 +631,9 @@ export async function convertGithubNwoToDatabaseUrl(
 
 export async function promptForLanguage(
   languages: string[],
-  progress: ProgressCallback,
+  progress: ProgressCallback | undefined,
 ): Promise<string | undefined> {
-  progress({
+  progress?.({
     message: "Choose language",
     step: 2,
     maxStep: 2,
@@ -579,10 +645,23 @@ export async function promptForLanguage(
     return languages[0];
   }
 
-  return await window.showQuickPick(languages, {
+  const items = languages
+    .map((language) => ({
+      label: getLanguageDisplayName(language),
+      description: language,
+      language,
+    }))
+    .sort((a, b) => a.label.localeCompare(b.label));
+
+  const selectedItem = await window.showQuickPick(items, {
     placeHolder: "Select the database language to download:",
     ignoreFocusOut: true,
   });
+  if (!selectedItem) {
+    return undefined;
+  }
+
+  return selectedItem.language;
 }
 
 /**

extensions/ql-vscode/src/databases/db-item-expansion.ts

@@ -1,27 +1,14 @@
 import { DbItem, DbItemKind, flattenDbItems } from "./db-item";
 
 export type ExpandedDbItem =
-  | RootLocalExpandedDbItem
-  | LocalUserDefinedListExpandedDbItem
   | RootRemoteExpandedDbItem
   | RemoteUserDefinedListExpandedDbItem;
 
 export enum ExpandedDbItemKind {
-  RootLocal = "rootLocal",
-  LocalUserDefinedList = "localUserDefinedList",
   RootRemote = "rootRemote",
   RemoteUserDefinedList = "remoteUserDefinedList",
 }
 
-interface RootLocalExpandedDbItem {
-  kind: ExpandedDbItemKind.RootLocal;
-}
-
-interface LocalUserDefinedListExpandedDbItem {
-  kind: ExpandedDbItemKind.LocalUserDefinedList;
-  listName: string;
-}
-
 interface RootRemoteExpandedDbItem {
   kind: ExpandedDbItemKind.RootRemote;
 }
@@ -80,13 +67,6 @@ export function cleanNonExistentExpandedItems(
 
 function mapDbItemToExpandedDbItem(dbItem: DbItem): ExpandedDbItem {
   switch (dbItem.kind) {
-    case DbItemKind.RootLocal:
-      return { kind: ExpandedDbItemKind.RootLocal };
-    case DbItemKind.LocalList:
-      return {
-        kind: ExpandedDbItemKind.LocalUserDefinedList,
-        listName: dbItem.listName,
-      };
     case DbItemKind.RootRemote:
       return { kind: ExpandedDbItemKind.RootRemote };
     case DbItemKind.RemoteUserDefinedList:
@@ -104,13 +84,6 @@ function isDbItemEqualToExpandedDbItem(
   expandedDbItem: ExpandedDbItem,
 ) {
   switch (dbItem.kind) {
-    case DbItemKind.RootLocal:
-      return expandedDbItem.kind === ExpandedDbItemKind.RootLocal;
-    case DbItemKind.LocalList:
-      return (
-        expandedDbItem.kind === ExpandedDbItemKind.LocalUserDefinedList &&
-        expandedDbItem.listName === dbItem.listName
-      );
     case DbItemKind.RootRemote:
       return expandedDbItem.kind === ExpandedDbItemKind.RootRemote;
     case DbItemKind.RemoteUserDefinedList:
@@ -118,7 +91,6 @@ function isDbItemEqualToExpandedDbItem(
         expandedDbItem.kind === ExpandedDbItemKind.RemoteUserDefinedList &&
         expandedDbItem.listName === dbItem.listName
       );
-    case DbItemKind.LocalDatabase:
     case DbItemKind.RemoteSystemDefinedList:
     case DbItemKind.RemoteOwner:
     case DbItemKind.RemoteRepo:

extensions/ql-vscode/src/databases/db-item-naming.ts

@@ -2,17 +2,13 @@ import { DbItem, DbItemKind } from "./db-item";
 
 export function getDbItemName(dbItem: DbItem): string | undefined {
   switch (dbItem.kind) {
-    case DbItemKind.RootLocal:
     case DbItemKind.RootRemote:
       return undefined;
-    case DbItemKind.LocalList:
     case DbItemKind.RemoteUserDefinedList:
     case DbItemKind.RemoteSystemDefinedList:
       return dbItem.listName;
     case DbItemKind.RemoteOwner:
       return dbItem.ownerName;
-    case DbItemKind.LocalDatabase:
-      return dbItem.databaseName;
     case DbItemKind.RemoteRepo:
       return dbItem.repoFullName;
   }

extensions/ql-vscode/src/databases/db-item-selection.ts

@@ -1,38 +1,30 @@
-import { DbItem, DbItemKind, LocalDbItem, RemoteDbItem } from "./db-item";
+import { DbItem, DbItemKind, RemoteDbItem } from "./db-item";
 import { SelectedDbItem, SelectedDbItemKind } from "./config/db-config";
 
 export function getSelectedDbItem(dbItems: DbItem[]): DbItem | undefined {
   for (const dbItem of dbItems) {
-    if (
-      dbItem.kind === DbItemKind.RootRemote ||
-      dbItem.kind === DbItemKind.RootLocal
-    ) {
+    if (dbItem.kind === DbItemKind.RootRemote) {
       for (const child of dbItem.children) {
         const selectedItem = extractSelected(child);
-        if (selectedItem) return selectedItem;
+        if (selectedItem) {
+          return selectedItem;
+        }
       }
     } else {
       const selectedItem = extractSelected(dbItem);
-      if (selectedItem) return selectedItem;
+      if (selectedItem) {
+        return selectedItem;
+      }
     }
   }
   return undefined;
 }
 
-function extractSelected(
-  dbItem: RemoteDbItem | LocalDbItem,
-): DbItem | undefined {
+function extractSelected(dbItem: RemoteDbItem): DbItem | undefined {
   if (dbItem.selected) {
     return dbItem;
   }
   switch (dbItem.kind) {
-    case DbItemKind.LocalList:
-      for (const database of dbItem.databases) {
-        if (database.selected) {
-          return database;
-        }
-      }
-      break;
     case DbItemKind.RemoteUserDefinedList:
       for (const repo of dbItem.repos) {
         if (repo.selected) {
@@ -48,17 +40,10 @@ export function mapDbItemToSelectedDbItem(
   dbItem: DbItem,
 ): SelectedDbItem | undefined {
   switch (dbItem.kind) {
-    case DbItemKind.RootLocal:
     case DbItemKind.RootRemote:
       // Root items are not selectable.
       return undefined;
 
-    case DbItemKind.LocalList:
-      return {
-        kind: SelectedDbItemKind.LocalUserDefinedList,
-        listName: dbItem.listName,
-      };
-
     case DbItemKind.RemoteUserDefinedList:
       return {
         kind: SelectedDbItemKind.VariantAnalysisUserDefinedList,
@@ -77,13 +62,6 @@ export function mapDbItemToSelectedDbItem(
         ownerName: dbItem.ownerName,
       };
 
-    case DbItemKind.LocalDatabase:
-      return {
-        kind: SelectedDbItemKind.LocalDatabase,
-        databaseName: dbItem.databaseName,
-        listName: dbItem?.parentListName,
-      };
-
     case DbItemKind.RemoteRepo:
       return {
         kind: SelectedDbItemKind.VariantAnalysisRepository,

extensions/ql-vscode/src/databases/db-item.ts

@@ -1,9 +1,6 @@
 // This file contains models that are used to represent the databases.
 
 export enum DbItemKind {
-  RootLocal = "RootLocal",
-  LocalList = "LocalList",
-  LocalDatabase = "LocalDatabase",
   RootRemote = "RootRemote",
   RemoteSystemDefinedList = "RemoteSystemDefinedList",
   RemoteUserDefinedList = "RemoteUserDefinedList",
@@ -11,48 +8,13 @@ export enum DbItemKind {
   RemoteRepo = "RemoteRepo",
 }
 
-export enum DbListKind {
-  Local = "Local",
-  Remote = "Remote",
-}
-
-export interface RootLocalDbItem {
-  kind: DbItemKind.RootLocal;
-  expanded: boolean;
-  children: LocalDbItem[];
-}
-
-export type LocalDbItem = LocalListDbItem | LocalDatabaseDbItem;
-
-export interface LocalListDbItem {
-  kind: DbItemKind.LocalList;
-  expanded: boolean;
-  selected: boolean;
-  listName: string;
-  databases: LocalDatabaseDbItem[];
-}
-
-export interface LocalDatabaseDbItem {
-  kind: DbItemKind.LocalDatabase;
-  selected: boolean;
-  databaseName: string;
-  dateAdded: number;
-  language: string;
-  storagePath: string;
-  parentListName?: string;
-}
-
 export interface RootRemoteDbItem {
   kind: DbItemKind.RootRemote;
   expanded: boolean;
   children: RemoteDbItem[];
 }
 
-export type DbItem =
-  | RootLocalDbItem
-  | RootRemoteDbItem
-  | RemoteDbItem
-  | LocalDbItem;
+export type DbItem = RootRemoteDbItem | RemoteDbItem;
 
 export type RemoteDbItem =
   | RemoteSystemDefinedListDbItem
@@ -105,25 +67,13 @@ export function isRemoteRepoDbItem(dbItem: DbItem): dbItem is RemoteRepoDbItem {
   return dbItem.kind === DbItemKind.RemoteRepo;
 }
 
-export function isLocalListDbItem(dbItem: DbItem): dbItem is LocalListDbItem {
-  return dbItem.kind === DbItemKind.LocalList;
-}
-
-export function isLocalDatabaseDbItem(
-  dbItem: DbItem,
-): dbItem is LocalDatabaseDbItem {
-  return dbItem.kind === DbItemKind.LocalDatabase;
-}
-
-type SelectableDbItem = RemoteDbItem | LocalDbItem;
+type SelectableDbItem = RemoteDbItem;
 
 export function isSelectableDbItem(dbItem: DbItem): dbItem is SelectableDbItem {
   return SelectableDbItemKinds.includes(dbItem.kind);
 }
 
 const SelectableDbItemKinds = [
-  DbItemKind.LocalList,
-  DbItemKind.LocalDatabase,
   DbItemKind.RemoteSystemDefinedList,
   DbItemKind.RemoteUserDefinedList,
   DbItemKind.RemoteOwner,
@@ -136,19 +86,12 @@ export function flattenDbItems(dbItems: DbItem[]): DbItem[] {
   for (const dbItem of dbItems) {
     allItems.push(dbItem);
     switch (dbItem.kind) {
-      case DbItemKind.RootLocal:
-        allItems.push(...flattenDbItems(dbItem.children));
-        break;
-      case DbItemKind.LocalList:
-        allItems.push(...flattenDbItems(dbItem.databases));
-        break;
       case DbItemKind.RootRemote:
         allItems.push(...flattenDbItems(dbItem.children));
         break;
       case DbItemKind.RemoteUserDefinedList:
         allItems.push(...dbItem.repos);
         break;
-      case DbItemKind.LocalDatabase:
       case DbItemKind.RemoteSystemDefinedList:
       case DbItemKind.RemoteOwner:
       case DbItemKind.RemoteRepo:

extensions/ql-vscode/src/databases/db-manager.ts

@@ -3,14 +3,7 @@ import { AppEvent, AppEventEmitter } from "../common/events";
 import { ValueResult } from "../common/value-result";
 import { DisposableObject } from "../common/disposable-object";
 import { DbConfigStore } from "./config/db-config-store";
-import {
-  DbItem,
-  DbItemKind,
-  DbListKind,
-  LocalDatabaseDbItem,
-  LocalListDbItem,
-  RemoteUserDefinedListDbItem,
-} from "./db-item";
+import { DbItem, RemoteUserDefinedListDbItem } from "./db-item";
 import {
   updateExpandedItem,
   replaceExpandedItem,
@@ -116,31 +109,15 @@ export class DbManager extends DisposableObject {
     await this.dbConfigStore.addRemoteOwner(owner);
   }
 
-  public async addNewList(
-    listKind: DbListKind,
-    listName: string,
-  ): Promise<void> {
-    switch (listKind) {
-      case DbListKind.Local:
-        await this.dbConfigStore.addLocalList(listName);
-        break;
-      case DbListKind.Remote:
-        await this.dbConfigStore.addRemoteList(listName);
-        break;
-      default:
-        throw Error(`Unknown list kind '${listKind}'`);
-    }
+  public async addNewList(listName: string): Promise<void> {
+    await this.dbConfigStore.addRemoteList(listName);
   }
 
   public async renameList(
-    currentDbItem: LocalListDbItem | RemoteUserDefinedListDbItem,
+    currentDbItem: RemoteUserDefinedListDbItem,
     newName: string,
   ): Promise<void> {
-    if (currentDbItem.kind === DbItemKind.LocalList) {
-      await this.dbConfigStore.renameLocalList(currentDbItem, newName);
-    } else if (currentDbItem.kind === DbItemKind.RemoteUserDefinedList) {
-      await this.dbConfigStore.renameRemoteList(currentDbItem, newName);
-    }
+    await this.dbConfigStore.renameRemoteList(currentDbItem, newName);
 
     const newDbItem = { ...currentDbItem, listName: newName };
     const newExpandedItems = replaceExpandedItem(
@@ -152,26 +129,8 @@ export class DbManager extends DisposableObject {
     await this.setExpandedItems(newExpandedItems);
   }
 
-  public async renameLocalDb(
-    currentDbItem: LocalDatabaseDbItem,
-    newName: string,
-  ): Promise<void> {
-    await this.dbConfigStore.renameLocalDb(
-      currentDbItem,
-      newName,
-      currentDbItem.parentListName,
-    );
-  }
-
-  public doesListExist(listKind: DbListKind, listName: string): boolean {
-    switch (listKind) {
-      case DbListKind.Local:
-        return this.dbConfigStore.doesLocalListExist(listName);
-      case DbListKind.Remote:
-        return this.dbConfigStore.doesRemoteListExist(listName);
-      default:
-        throw Error(`Unknown list kind '${listKind}'`);
-    }
+  public doesListExist(listName: string): boolean {
+    return this.dbConfigStore.doesRemoteListExist(listName);
   }
 
   public doesRemoteOwnerExist(owner: string): boolean {
@@ -182,10 +141,6 @@ export class DbManager extends DisposableObject {
     return this.dbConfigStore.doesRemoteDbExist(nwo, listName);
   }
 
-  public doesLocalDbExist(dbName: string, listName?: string): boolean {
-    return this.dbConfigStore.doesLocalDbExist(dbName, listName);
-  }
-
   private getExpandedItems(): ExpandedDbItem[] {
     const items = this.app.workspaceState.get<ExpandedDbItem[]>(
       DbManager.DB_EXPANDED_STATE_KEY,

extensions/ql-vscode/src/databases/db-tree-creator.ts

@@ -1,19 +1,14 @@
 import {
   DbConfig,
-  LocalDatabase,
-  LocalList,
   RemoteRepositoryList,
   SelectedDbItemKind,
 } from "./config/db-config";
 import {
   DbItemKind,
-  LocalDatabaseDbItem,
-  LocalListDbItem,
   RemoteOwnerDbItem,
   RemoteRepoDbItem,
   RemoteSystemDefinedListDbItem,
   RemoteUserDefinedListDbItem,
-  RootLocalDbItem,
   RootRemoteDbItem,
 } from "./db-item";
 import { ExpandedDbItem, ExpandedDbItemKind } from "./db-item-expansion";
@@ -55,28 +50,6 @@ export function createRemoteTree(
   };
 }
 
-export function createLocalTree(
-  dbConfig: DbConfig,
-  expandedItems: ExpandedDbItem[],
-): RootLocalDbItem {
-  const localLists = dbConfig.databases.local.lists.map((l) =>
-    createLocalList(l, dbConfig, expandedItems),
-  );
-  const localDbs = dbConfig.databases.local.databases.map((l) =>
-    createLocalDb(l, dbConfig),
-  );
-
-  const expanded = expandedItems.some(
-    (e) => e.kind === ExpandedDbItemKind.RootLocal,
-  );
-
-  return {
-    kind: DbItemKind.RootLocal,
-    children: [...localLists, ...localDbs],
-    expanded: !!expanded,
-  };
-}
-
 function createSystemDefinedList(
   n: number,
   dbConfig: DbConfig,
@@ -155,50 +128,3 @@ function createRepoItem(
     parentListName: listName,
   };
 }
-
-function createLocalList(
-  list: LocalList,
-  dbConfig: DbConfig,
-  expandedItems: ExpandedDbItem[],
-): LocalListDbItem {
-  const selected =
-    dbConfig.selected &&
-    dbConfig.selected.kind === SelectedDbItemKind.LocalUserDefinedList &&
-    dbConfig.selected.listName === list.name;
-
-  const expanded = expandedItems.some(
-    (e) =>
-      e.kind === ExpandedDbItemKind.LocalUserDefinedList &&
-      e.listName === list.name,
-  );
-
-  return {
-    kind: DbItemKind.LocalList,
-    listName: list.name,
-    databases: list.databases.map((d) => createLocalDb(d, dbConfig, list.name)),
-    selected: !!selected,
-    expanded: !!expanded,
-  };
-}
-
-function createLocalDb(
-  db: LocalDatabase,
-  dbConfig: DbConfig,
-  listName?: string,
-): LocalDatabaseDbItem {
-  const selected =
-    dbConfig.selected &&
-    dbConfig.selected.kind === SelectedDbItemKind.LocalDatabase &&
-    dbConfig.selected.databaseName === db.name &&
-    dbConfig.selected.listName === listName;
-
-  return {
-    kind: DbItemKind.LocalDatabase,
-    databaseName: db.name,
-    dateAdded: db.dateAdded,
-    language: db.language,
-    storagePath: db.storagePath,
-    selected: !!selected,
-    parentListName: listName,
-  };
-}

extensions/ql-vscode/src/databases/github-databases/api.ts

@@ -0,0 +1,110 @@
+import { RequestError } from "@octokit/request-error";
+import { Octokit } from "@octokit/rest";
+import { RestEndpointMethodTypes } from "@octokit/plugin-rest-endpoint-methods";
+import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
+import { GitHubDatabaseConfig } from "../../config";
+import { Credentials } from "../../common/authentication";
+import { AppOctokit } from "../../common/octokit";
+
+export type CodeqlDatabase =
+  RestEndpointMethodTypes["codeScanning"]["listCodeqlDatabases"]["response"]["data"][number];
+
+/**
+ * Ask the user if they want to connect to GitHub to download CodeQL databases.
+ * This should be used when the user does not have an access token and should
+ * be followed by an access token prompt.
+ */
+async function askForGitHubConnect(
+  config: GitHubDatabaseConfig,
+): Promise<boolean> {
+  const answer = await showNeverAskAgainDialog(
+    "This repository has an origin (GitHub) that may have one or more CodeQL databases. Connect to GitHub and download any existing databases?",
+    false,
+    "Connect",
+    "Not now",
+    "Never",
+  );
+
+  if (answer === "Not now" || answer === undefined) {
+    return false;
+  }
+
+  if (answer === "Never") {
+    await config.setDownload("never");
+    return false;
+  }
+
+  return true;
+}
+
+export type ListDatabasesResult = {
+  /**
+   * Whether the user has been prompted for credentials. This can be used to determine
+   * follow-up actions based on whether the user has already had any feedback.
+   */
+  promptedForCredentials: boolean;
+  databases: CodeqlDatabase[];
+  octokit: Octokit;
+};
+
+/**
+ * List CodeQL databases for a GitHub repository.
+ *
+ * This will first try to fetch the CodeQL databases for the repository with
+ * existing credentials (or none if there are none). If that fails, it will
+ * prompt the user to connect to GitHub and try again.
+ *
+ * If the user does not want to connect to GitHub, this will return `undefined`.
+ */
+export async function listDatabases(
+  owner: string,
+  repo: string,
+  credentials: Credentials,
+  config: GitHubDatabaseConfig,
+): Promise<ListDatabasesResult | undefined> {
+  const hasAccessToken = !!(await credentials.getExistingAccessToken());
+
+  let octokit = hasAccessToken
+    ? await credentials.getOctokit()
+    : new AppOctokit();
+
+  let promptedForCredentials = false;
+
+  let databases: CodeqlDatabase[];
+  try {
+    const response = await octokit.rest.codeScanning.listCodeqlDatabases({
+      owner,
+      repo,
+    });
+    databases = response.data;
+  } catch (e) {
+    // If we get a 404 when we don't have an access token, it might be because
+    // the repository is private/internal. Therefore, we should ask the user
+    // whether they want to connect to GitHub and try again.
+    if (e instanceof RequestError && e.status === 404 && !hasAccessToken) {
+      // Check whether the user wants to connect to GitHub
+      if (!(await askForGitHubConnect(config))) {
+        return;
+      }
+
+      // Prompt for credentials
+      octokit = await credentials.getOctokit();
+
+      promptedForCredentials = true;
+
+      const response = await octokit.rest.codeScanning.listCodeqlDatabases({
+        owner,
+        repo,
+      });
+      databases = response.data;
+    } else {
+      throw e;
+    }
+  }
+
+  return {
+    promptedForCredentials,
+    databases,
+    octokit,
+  };
+}

extensions/ql-vscode/src/databases/github-databases/download.ts

@@ -0,0 +1,171 @@
+import { window } from "vscode";
+import { Octokit } from "@octokit/rest";
+import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
+import { getLanguageDisplayName } from "../../common/query-language";
+import { downloadGitHubDatabaseFromUrl } from "../database-fetcher";
+import { withProgress } from "../../common/vscode/progress";
+import { DatabaseManager } from "../local-databases";
+import { CodeQLCliServer } from "../../codeql-cli/cli";
+import { AppCommandManager } from "../../common/commands";
+import { GitHubDatabaseConfig } from "../../config";
+import type { CodeqlDatabase } from "./api";
+
+/**
+ * Ask whether the user wants to download a database from GitHub.
+ * @return true if the user wants to download a database, false otherwise.
+ */
+export async function askForGitHubDatabaseDownload(
+  databases: CodeqlDatabase[],
+  config: GitHubDatabaseConfig,
+): Promise<boolean> {
+  const languages = databases.map((database) => database.language);
+
+  const message =
+    databases.length === 1
+      ? `This repository has an origin (GitHub) that has a ${getLanguageDisplayName(
+          languages[0],
+        )} CodeQL database. Download the existing database from GitHub?`
+      : `This repository has an origin (GitHub) that has ${joinLanguages(
+          languages,
+        )} CodeQL databases. Download any existing databases from GitHub?`;
+
+  const answer = await showNeverAskAgainDialog(
+    message,
+    false,
+    "Download",
+    "Not now",
+    "Never",
+  );
+
+  if (answer === "Not now" || answer === undefined) {
+    return false;
+  }
+
+  if (answer === "Never") {
+    await config.setDownload("never");
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * Download a database from GitHub by asking the user for a language and then
+ * downloading the database for that language.
+ */
+export async function downloadDatabaseFromGitHub(
+  octokit: Octokit,
+  owner: string,
+  repo: string,
+  databases: CodeqlDatabase[],
+  databaseManager: DatabaseManager,
+  storagePath: string,
+  cliServer: CodeQLCliServer,
+  commandManager: AppCommandManager,
+): Promise<void> {
+  const selectedDatabases = await promptForDatabases(databases);
+  if (selectedDatabases.length === 0) {
+    return;
+  }
+
+  await Promise.all(
+    selectedDatabases.map((database) =>
+      withProgress(
+        async (progress) => {
+          await downloadGitHubDatabaseFromUrl(
+            database.url,
+            database.id,
+            database.created_at,
+            database.commit_oid ?? null,
+            owner,
+            repo,
+            octokit,
+            progress,
+            databaseManager,
+            storagePath,
+            cliServer,
+            true,
+            false,
+          );
+
+          await commandManager.execute("codeQLDatabases.focus");
+          void window.showInformationMessage(
+            `Downloaded ${getLanguageDisplayName(
+              database.language,
+            )} database from GitHub.`,
+          );
+        },
+        {
+          title: `Adding ${getLanguageDisplayName(
+            database.language,
+          )} database from GitHub`,
+        },
+      ),
+    ),
+  );
+}
+
+/**
+ * Join languages into a string for display. Will automatically add `,` and `and` as appropriate.
+ *
+ * @param languages The languages to join. These should be language identifiers, such as `csharp`.
+ */
+export function joinLanguages(languages: string[]): string {
+  const languageDisplayNames = languages
+    .map((language) => getLanguageDisplayName(language))
+    .sort();
+
+  let result = "";
+  for (let i = 0; i < languageDisplayNames.length; i++) {
+    if (i > 0) {
+      if (i === languageDisplayNames.length - 1) {
+        result += " and ";
+      } else {
+        result += ", ";
+      }
+    }
+    result += languageDisplayNames[i];
+  }
+
+  return result;
+}
+
+type PromptForDatabasesOptions = {
+  title?: string;
+  placeHolder?: string;
+};
+
+export async function promptForDatabases(
+  databases: CodeqlDatabase[],
+  {
+    title = "Select databases to download",
+    placeHolder = "Databases found in this repository",
+  }: PromptForDatabasesOptions = {},
+): Promise<CodeqlDatabase[]> {
+  if (databases.length === 1) {
+    return databases;
+  }
+
+  const items = databases
+    .map((database) => {
+      const bytesToDisplayMB = `${(database.size / (1024 * 1024)).toFixed(
+        1,
+      )} MB`;
+
+      return {
+        label: getLanguageDisplayName(database.language),
+        description: bytesToDisplayMB,
+        database,
+      };
+    })
+    .sort((a, b) => a.label.localeCompare(b.label));
+
+  const selectedItems = await window.showQuickPick(items, {
+    title,
+    placeHolder,
+    ignoreFocusOut: true,
+    canPickMany: true,
+  });
+
+  return selectedItems?.map((selectedItem) => selectedItem.database) ?? [];
+}

extensions/ql-vscode/src/databases/github-databases/github-databases-module.ts

@@ -0,0 +1,220 @@
+import { window } from "vscode";
+import { DisposableObject } from "../../common/disposable-object";
+import { App } from "../../common/app";
+import { findGitHubRepositoryForWorkspace } from "../github-repository-finder";
+import { redactableError } from "../../common/errors";
+import {
+  asError,
+  assertNever,
+  getErrorMessage,
+} from "../../common/helpers-pure";
+import {
+  askForGitHubDatabaseDownload,
+  downloadDatabaseFromGitHub,
+} from "./download";
+import {
+  GitHubDatabaseConfig,
+  GitHubDatabaseConfigListener,
+} from "../../config";
+import { DatabaseManager } from "../local-databases";
+import { CodeQLCliServer } from "../../codeql-cli/cli";
+import { CodeqlDatabase, listDatabases, ListDatabasesResult } from "./api";
+import {
+  askForGitHubDatabaseUpdate,
+  DatabaseUpdate,
+  downloadDatabaseUpdateFromGitHub,
+  isNewerDatabaseAvailable,
+} from "./updates";
+import { Octokit } from "@octokit/rest";
+
+export class GitHubDatabasesModule extends DisposableObject {
+  private readonly config: GitHubDatabaseConfig;
+
+  private constructor(
+    private readonly app: App,
+    private readonly databaseManager: DatabaseManager,
+    private readonly databaseStoragePath: string,
+    private readonly cliServer: CodeQLCliServer,
+  ) {
+    super();
+
+    this.config = this.push(new GitHubDatabaseConfigListener());
+  }
+
+  public static async initialize(
+    app: App,
+    databaseManager: DatabaseManager,
+    databaseStoragePath: string,
+    cliServer: CodeQLCliServer,
+  ): Promise<GitHubDatabasesModule> {
+    const githubDatabasesModule = new GitHubDatabasesModule(
+      app,
+      databaseManager,
+      databaseStoragePath,
+      cliServer,
+    );
+    app.subscriptions.push(githubDatabasesModule);
+
+    await githubDatabasesModule.initialize();
+    return githubDatabasesModule;
+  }
+
+  private async initialize(): Promise<void> {
+    if (!this.config.enable) {
+      return;
+    }
+
+    // Start the check and downloading the database asynchronously. We don't want to block on this
+    // in extension activation since this makes network requests and waits for user input.
+    void this.promptGitHubRepositoryDownload().catch((e: unknown) => {
+      const error = redactableError(
+        asError(e),
+      )`Failed to prompt for GitHub repository download`;
+
+      void this.app.logger.log(error.fullMessageWithStack);
+      this.app.telemetry?.sendError(error);
+    });
+  }
+
+  private async promptGitHubRepositoryDownload(): Promise<void> {
+    if (this.config.download === "never") {
+      return;
+    }
+
+    const githubRepositoryResult = await findGitHubRepositoryForWorkspace();
+    if (githubRepositoryResult.isFailure) {
+      void this.app.logger.log(
+        `Did not find a GitHub repository for workspace: ${githubRepositoryResult.errors.join(
+          ", ",
+        )}`,
+      );
+      return;
+    }
+
+    const githubRepository = githubRepositoryResult.value;
+
+    let result: ListDatabasesResult | undefined;
+    try {
+      result = await listDatabases(
+        githubRepository.owner,
+        githubRepository.name,
+        this.app.credentials,
+        this.config,
+      );
+    } catch (e) {
+      this.app.telemetry?.sendError(
+        redactableError(
+          asError(e),
+        )`Failed to prompt for GitHub database download`,
+      );
+
+      void this.app.logger.log(
+        `Failed to find GitHub databases for repository: ${getErrorMessage(e)}`,
+      );
+
+      return;
+    }
+
+    // This means the user didn't want to connect, so we can just return.
+    if (result === undefined) {
+      return;
+    }
+
+    const { databases, promptedForCredentials, octokit } = result;
+
+    if (databases.length === 0) {
+      // If the user didn't have an access token, they have already been prompted,
+      // so we should give feedback.
+      if (promptedForCredentials) {
+        void window.showInformationMessage(
+          "The GitHub repository does not have any CodeQL databases.",
+        );
+      }
+
+      return;
+    }
+
+    const updateStatus = isNewerDatabaseAvailable(
+      databases,
+      githubRepository.owner,
+      githubRepository.name,
+      this.databaseManager,
+    );
+
+    switch (updateStatus.type) {
+      case "upToDate":
+        return;
+      case "updateAvailable":
+        await this.updateGitHubDatabase(
+          octokit,
+          githubRepository.owner,
+          githubRepository.name,
+          updateStatus.databaseUpdates,
+        );
+        break;
+      case "noDatabase":
+        await this.downloadGitHubDatabase(
+          octokit,
+          githubRepository.owner,
+          githubRepository.name,
+          databases,
+          promptedForCredentials,
+        );
+        break;
+      default:
+        assertNever(updateStatus);
+    }
+  }
+
+  private async downloadGitHubDatabase(
+    octokit: Octokit,
+    owner: string,
+    repo: string,
+    databases: CodeqlDatabase[],
+    promptedForCredentials: boolean,
+  ) {
+    // If the user already had an access token, first ask if they even want to download the DB.
+    if (!promptedForCredentials) {
+      if (!(await askForGitHubDatabaseDownload(databases, this.config))) {
+        return;
+      }
+    }
+
+    await downloadDatabaseFromGitHub(
+      octokit,
+      owner,
+      repo,
+      databases,
+      this.databaseManager,
+      this.databaseStoragePath,
+      this.cliServer,
+      this.app.commands,
+    );
+  }
+
+  private async updateGitHubDatabase(
+    octokit: Octokit,
+    owner: string,
+    repo: string,
+    databaseUpdates: DatabaseUpdate[],
+  ): Promise<void> {
+    if (this.config.update === "never") {
+      return;
+    }
+
+    if (!(await askForGitHubDatabaseUpdate(databaseUpdates, this.config))) {
+      return;
+    }
+
+    await downloadDatabaseUpdateFromGitHub(
+      octokit,
+      owner,
+      repo,
+      databaseUpdates,
+      this.databaseManager,
+      this.databaseStoragePath,
+      this.cliServer,
+      this.app.commands,
+    );
+  }
+}

extensions/ql-vscode/src/databases/github-databases/index.ts

@@ -0,0 +1 @@
+export * from "./github-databases-module";

extensions/ql-vscode/src/databases/github-databases/updates.ts

@@ -0,0 +1,218 @@
+import { CodeqlDatabase } from "./api";
+import { DatabaseItem, DatabaseManager } from "../local-databases";
+import { Octokit } from "@octokit/rest";
+import { CodeQLCliServer } from "../../codeql-cli/cli";
+import { AppCommandManager } from "../../common/commands";
+import { getLanguageDisplayName } from "../../common/query-language";
+import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
+import { downloadGitHubDatabaseFromUrl } from "../database-fetcher";
+import { withProgress } from "../../common/vscode/progress";
+import { window } from "vscode";
+import { GitHubDatabaseConfig } from "../../config";
+import { joinLanguages, promptForDatabases } from "./download";
+
+export type DatabaseUpdate = {
+  database: CodeqlDatabase;
+  databaseItem: DatabaseItem;
+};
+
+type DatabaseUpdateStatusUpdateAvailable = {
+  type: "updateAvailable";
+  databaseUpdates: DatabaseUpdate[];
+};
+
+type DatabaseUpdateStatusUpToDate = {
+  type: "upToDate";
+};
+
+type DatabaseUpdateStatusNoDatabase = {
+  type: "noDatabase";
+};
+
+type DatabaseUpdateStatus =
+  | DatabaseUpdateStatusUpdateAvailable
+  | DatabaseUpdateStatusUpToDate
+  | DatabaseUpdateStatusNoDatabase;
+
+/**
+ * Check whether a newer database is available for the given repository. Databases are considered updated if:
+ * - They have a different commit OID, or
+ * - They have the same commit OID, but the remote database was created after the local database.
+ */
+export function isNewerDatabaseAvailable(
+  databases: CodeqlDatabase[],
+  owner: string,
+  name: string,
+  databaseManager: DatabaseManager,
+): DatabaseUpdateStatus {
+  // Sorted by date added ascending
+  const existingDatabasesForRepository = databaseManager.databaseItems
+    .filter(
+      (db) =>
+        db.origin?.type === "github" &&
+        db.origin.repository === `${owner}/${name}`,
+    )
+    .sort((a, b) => (a.dateAdded ?? 0) - (b.dateAdded ?? 0));
+
+  if (existingDatabasesForRepository.length === 0) {
+    return {
+      type: "noDatabase",
+    };
+  }
+
+  // Sort order is guaranteed by the sort call above. The newest database is the last one.
+  const newestExistingDatabasesByLanguage = new Map<string, DatabaseItem>();
+  for (const existingDatabase of existingDatabasesForRepository) {
+    newestExistingDatabasesByLanguage.set(
+      existingDatabase.language,
+      existingDatabase,
+    );
+  }
+
+  const databaseUpdates = Array.from(newestExistingDatabasesByLanguage.values())
+    .map((newestExistingDatabase): DatabaseUpdate | null => {
+      const origin = newestExistingDatabase.origin;
+      if (origin?.type !== "github") {
+        return null;
+      }
+
+      const matchingDatabase = databases.find(
+        (db) => db.language === newestExistingDatabase.language,
+      );
+      if (!matchingDatabase) {
+        return null;
+      }
+
+      // If they are not equal, we assume that the remote database is newer.
+      if (matchingDatabase.commit_oid === origin.commitOid) {
+        const remoteDatabaseCreatedAt = new Date(matchingDatabase.created_at);
+        const localDatabaseCreatedAt = new Date(origin.databaseCreatedAt);
+
+        // If the remote database was created before the local database,
+        // we assume that the local database is newer.
+        if (remoteDatabaseCreatedAt <= localDatabaseCreatedAt) {
+          return null;
+        }
+      }
+
+      return {
+        database: matchingDatabase,
+        databaseItem: newestExistingDatabase,
+      };
+    })
+    .filter((update): update is DatabaseUpdate => update !== null)
+    .sort((a, b) => a.database.language.localeCompare(b.database.language));
+
+  if (databaseUpdates.length === 0) {
+    return {
+      type: "upToDate",
+    };
+  }
+
+  return {
+    type: "updateAvailable",
+    databaseUpdates,
+  };
+}
+
+export async function askForGitHubDatabaseUpdate(
+  updates: DatabaseUpdate[],
+  config: GitHubDatabaseConfig,
+): Promise<boolean> {
+  const languages = updates.map((update) => update.database.language);
+
+  const message =
+    updates.length === 1
+      ? `There is a newer ${getLanguageDisplayName(
+          languages[0],
+        )} CodeQL database available for this repository. Download the database update from GitHub?`
+      : `There are newer ${joinLanguages(
+          languages,
+        )} CodeQL databases available for this repository. Download the database updates from GitHub?`;
+
+  const answer = await showNeverAskAgainDialog(
+    message,
+    false,
+    "Download",
+    "Not now",
+    "Never",
+  );
+
+  if (answer === "Not now" || answer === undefined) {
+    return false;
+  }
+
+  if (answer === "Never") {
+    await config.setUpdate("never");
+    return false;
+  }
+
+  return true;
+}
+
+export async function downloadDatabaseUpdateFromGitHub(
+  octokit: Octokit,
+  owner: string,
+  repo: string,
+  updates: DatabaseUpdate[],
+  databaseManager: DatabaseManager,
+  storagePath: string,
+  cliServer: CodeQLCliServer,
+  commandManager: AppCommandManager,
+): Promise<void> {
+  const selectedDatabases = await promptForDatabases(
+    updates.map((update) => update.database),
+    {
+      title: "Select databases to update",
+    },
+  );
+  if (selectedDatabases.length === 0) {
+    return;
+  }
+
+  await Promise.all(
+    selectedDatabases.map((database) => {
+      const update = updates.find((update) => update.database === database);
+      if (!update) {
+        return;
+      }
+
+      return withProgress(
+        async (progress) => {
+          const newDatabase = await downloadGitHubDatabaseFromUrl(
+            database.url,
+            database.id,
+            database.created_at,
+            database.commit_oid ?? null,
+            owner,
+            repo,
+            octokit,
+            progress,
+            databaseManager,
+            storagePath,
+            cliServer,
+            databaseManager.currentDatabaseItem === update.databaseItem,
+            update.databaseItem.hasSourceArchiveInExplorer(),
+          );
+          if (newDatabase === undefined) {
+            return;
+          }
+
+          await databaseManager.removeDatabaseItem(update.databaseItem);
+
+          await commandManager.execute("codeQLDatabases.focus");
+          void window.showInformationMessage(
+            `Updated ${getLanguageDisplayName(
+              database.language,
+            )} database from GitHub.`,
+          );
+        },
+        {
+          title: `Updating ${getLanguageDisplayName(
+            database.language,
+          )} database from GitHub`,
+        },
+      );
+    }),
+  );
+}

extensions/ql-vscode/src/databases/github-repository-finder.ts

@@ -0,0 +1,182 @@
+import {
+  API as GitExtensionAPI,
+  GitExtension,
+  Repository,
+} from "../common/vscode/extension/git";
+import { extensions, Uri } from "vscode";
+import { getOnDiskWorkspaceFoldersObjects } from "../common/vscode/workspace-folders";
+import { ValueResult } from "../common/value-result";
+
+// Based on https://github.com/microsoft/sarif-vscode-extension/blob/a1740e766122c1759d9f39d580c18b82d9e0dea4/src/extension/index.activateGithubAnalyses.ts
+
+async function getGitExtensionAPI(): Promise<
+  ValueResult<GitExtensionAPI, string>
+> {
+  const gitExtension = extensions.getExtension<GitExtension>("vscode.git");
+  if (!gitExtension) {
+    return ValueResult.fail(["Git extension not found"]);
+  }
+
+  if (!gitExtension.isActive) {
+    await gitExtension.activate();
+  }
+
+  const gitExtensionExports = gitExtension.exports;
+
+  if (!gitExtensionExports.enabled) {
+    return ValueResult.fail(["Git extension is not enabled"]);
+  }
+
+  const git = gitExtensionExports.getAPI(1);
+  if (git.state === "initialized") {
+    return ValueResult.ok(git);
+  }
+
+  return new Promise((resolve) => {
+    git.onDidChangeState((state) => {
+      if (state === "initialized") {
+        resolve(ValueResult.ok(git));
+      }
+    });
+  });
+}
+
+async function findRepositoryForWorkspaceFolder(
+  git: GitExtensionAPI,
+  workspaceFolderUri: Uri,
+): Promise<Repository | undefined> {
+  return git.repositories.find(
+    (repo) => repo.rootUri.toString() === workspaceFolderUri.toString(),
+  );
+}
+
+/**
+ * Finds the primary remote fetch URL for a repository.
+ *
+ * The priority is:
+ * 1. The remote associated with the current branch
+ * 2. The remote named "origin"
+ * 3. The first remote
+ *
+ * If none of these are found, undefined is returned.
+ *
+ * This is just a heuristic. We may not find the correct remote in all cases,
+ * for example when the user has defined an alias in their SSH or Git config.
+ *
+ * @param repository The repository to find the remote for.
+ */
+async function findRemote(repository: Repository): Promise<string | undefined> {
+  // Try to retrieve the remote 5 times with a 5 second delay between each attempt.
+  // This is to account for the case where the Git extension has not yet retrieved
+  // the state for all Git repositories.
+  // This can happen on Codespaces where the Git extension is initialized before the
+  // filesystem is ready.
+  for (let count = 0; count < 5; count++) {
+    const remoteName = repository.state.HEAD?.upstream?.remote ?? "origin";
+    const upstreamRemoteUrl = repository.state.remotes.find(
+      (remote) => remote.name === remoteName,
+    )?.fetchUrl;
+    if (upstreamRemoteUrl) {
+      return upstreamRemoteUrl;
+    }
+
+    if (remoteName !== "origin") {
+      const originRemoteUrl = repository.state.remotes.find(
+        (remote) => remote.name === "origin",
+      )?.fetchUrl;
+
+      if (originRemoteUrl) {
+        return originRemoteUrl;
+      }
+    }
+
+    // Maybe they have a different remote that is not named origin and is not the
+    // upstream of the current branch. If so, just select the first one.
+    const firstRemoteUrl = repository.state.remotes[0]?.fetchUrl;
+    if (firstRemoteUrl) {
+      return firstRemoteUrl;
+    }
+
+    // Wait for 5 seconds before trying again.
+    await new Promise((resolve) => setTimeout(resolve, 5000));
+  }
+
+  return undefined;
+}
+
+// Example: https://github.com/github/vscode-codeql.git
+const githubHTTPSRegex =
+  /https:\/\/github\.com\/(?<owner>[^/]+)\/(?<name>[^/]+)/;
+
+// Example: git@github.com:github/vscode-codeql.git
+const githubSSHRegex = /git@github\.com:(?<owner>[^/]+)\/(?<name>[^/]+)/;
+
+function findGitHubRepositoryForRemote(remoteUrl: string):
+  | {
+      owner: string;
+      name: string;
+    }
+  | undefined {
+  const match =
+    remoteUrl.match(githubHTTPSRegex) ?? remoteUrl.match(githubSSHRegex);
+  if (!match) {
+    return undefined;
+  }
+
+  const owner = match.groups?.owner;
+  let name = match.groups?.name;
+
+  if (!owner || !name) {
+    return undefined;
+  }
+
+  // If a repository ends with ".git", remove it.
+  if (name.endsWith(".git")) {
+    name = name.slice(0, -4);
+  }
+
+  return {
+    owner,
+    name,
+  };
+}
+
+export async function findGitHubRepositoryForWorkspace(): Promise<
+  ValueResult<{ owner: string; name: string }, string>
+> {
+  const gitResult = await getGitExtensionAPI();
+  if (gitResult.isFailure) {
+    return ValueResult.fail(gitResult.errors);
+  }
+
+  const git = gitResult.value;
+
+  const primaryWorkspaceFolder = getOnDiskWorkspaceFoldersObjects()[0]?.uri;
+  if (!primaryWorkspaceFolder) {
+    return ValueResult.fail(["No workspace folder found"]);
+  }
+
+  const primaryRepository = await findRepositoryForWorkspaceFolder(
+    git,
+    primaryWorkspaceFolder,
+  );
+  if (!primaryRepository) {
+    return ValueResult.fail([
+      "No Git repository found in primary workspace folder",
+    ]);
+  }
+
+  const remoteUrl = await findRemote(primaryRepository);
+  if (!remoteUrl) {
+    return ValueResult.fail(["No remote found"]);
+  }
+
+  const repoNwo = findGitHubRepositoryForRemote(remoteUrl);
+  if (!repoNwo) {
+    return ValueResult.fail(["Remote is not a GitHub repository"]);
+  }
+
+  const { owner, name } = repoNwo;
+
+  return ValueResult.ok({ owner, name });
+}

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -51,12 +51,14 @@ import {
   createMultiSelectionCommand,
   createSingleSelectionCommand,
 } from "../common/vscode/selection-commands";
-import { QueryLanguage, tryGetQueryLanguage } from "../common/query-language";
+import { tryGetQueryLanguage } from "../common/query-language";
 import { LanguageContextStore } from "../language-context-store";
 
 enum SortOrder {
   NameAsc = "NameAsc",
   NameDesc = "NameDesc",
+  LanguageAsc = "LanguageAsc",
+  LanguageDesc = "LanguageDesc",
   DateAddedAsc = "DateAddedAsc",
   DateAddedDesc = "DateAddedDesc",
 }
@@ -155,6 +157,18 @@ class DatabaseTreeDataProvider
             return db1.name.localeCompare(db2.name, env.language);
           case SortOrder.NameDesc:
             return db2.name.localeCompare(db1.name, env.language);
+          case SortOrder.LanguageAsc:
+            return (
+              db1.language.localeCompare(db2.language, env.language) ||
+              // If the languages are the same, sort by name
+              db1.name.localeCompare(db2.name, env.language)
+            );
+          case SortOrder.LanguageDesc:
+            return (
+              db2.language.localeCompare(db1.language, env.language) ||
+              // If the languages are the same, sort by name
+              db2.name.localeCompare(db1.name, env.language)
+            );
           case SortOrder.DateAddedAsc:
             return (db1.dateAdded || 0) - (db2.dateAdded || 0);
           case SortOrder.DateAddedDesc:
@@ -218,7 +232,7 @@ export class DatabaseUI extends DisposableObject {
   public constructor(
     private app: App,
     private databaseManager: DatabaseManager,
-    private languageContext: LanguageContextStore,
+    languageContext: LanguageContextStore,
     private readonly queryServer: QueryRunner | undefined,
     private readonly storagePath: string,
     readonly extensionPath: string,
@@ -252,6 +266,7 @@ export class DatabaseUI extends DisposableObject {
       "codeQL.upgradeCurrentDatabase":
         this.handleUpgradeCurrentDatabase.bind(this),
       "codeQL.clearCache": this.handleClearCache.bind(this),
+      "codeQL.trimCache": this.handleTrimCache.bind(this),
       "codeQLDatabases.chooseDatabaseFolder":
         this.handleChooseDatabaseFolder.bind(this),
       "codeQLDatabases.chooseDatabaseArchive":
@@ -263,61 +278,8 @@ export class DatabaseUI extends DisposableObject {
       "codeQLDatabases.setCurrentDatabase":
         this.handleMakeCurrentDatabase.bind(this),
       "codeQLDatabases.sortByName": this.handleSortByName.bind(this),
+      "codeQLDatabases.sortByLanguage": this.handleSortByLanguage.bind(this),
       "codeQLDatabases.sortByDateAdded": this.handleSortByDateAdded.bind(this),
-      "codeQLDatabases.displayAllLanguages":
-        this.handleClearLanguageFilter.bind(this),
-      "codeQLDatabases.displayCpp": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Cpp,
-      ),
-      "codeQLDatabases.displayCsharp": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.CSharp,
-      ),
-      "codeQLDatabases.displayGo": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Go,
-      ),
-      "codeQLDatabases.displayJava": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Java,
-      ),
-      "codeQLDatabases.displayJavascript": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Javascript,
-      ),
-      "codeQLDatabases.displayPython": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Python,
-      ),
-      "codeQLDatabases.displayRuby": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Ruby,
-      ),
-      "codeQLDatabases.displaySwift": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Swift,
-      ),
-      "codeQLDatabases.displayAllLanguagesSelected":
-        this.handleClearLanguageFilter.bind(this),
-      "codeQLDatabases.displayCppSelected":
-        this.handleChangeLanguageFilter.bind(this, QueryLanguage.Cpp),
-      "codeQLDatabases.displayCsharpSelected":
-        this.handleChangeLanguageFilter.bind(this, QueryLanguage.CSharp),
-      "codeQLDatabases.displayGoSelected": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Go,
-      ),
-      "codeQLDatabases.displayJavaSelected":
-        this.handleChangeLanguageFilter.bind(this, QueryLanguage.Java),
-      "codeQLDatabases.displayJavascriptSelected":
-        this.handleChangeLanguageFilter.bind(this, QueryLanguage.Javascript),
-      "codeQLDatabases.displayPythonSelected":
-        this.handleChangeLanguageFilter.bind(this, QueryLanguage.Python),
-      "codeQLDatabases.displayRubySelected":
-        this.handleChangeLanguageFilter.bind(this, QueryLanguage.Ruby),
-      "codeQLDatabases.displaySwiftSelected":
-        this.handleChangeLanguageFilter.bind(this, QueryLanguage.Swift),
       "codeQLDatabases.removeDatabase": createMultiSelectionCommand(
         this.handleRemoveDatabase.bind(this),
       ),
@@ -405,6 +367,9 @@ export class DatabaseUI extends DisposableObject {
 
               await this.databaseManager.openDatabase(
                 uri,
+                {
+                  type: "folder",
+                },
                 makeSelected,
                 nameOverride,
                 {
@@ -600,6 +565,14 @@ export class DatabaseUI extends DisposableObject {
     }
   }
 
+  private async handleSortByLanguage() {
+    if (this.treeDataProvider.sortOrder === SortOrder.LanguageAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.LanguageDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.LanguageAsc;
+    }
+  }
+
   private async handleSortByDateAdded() {
     if (this.treeDataProvider.sortOrder === SortOrder.DateAddedAsc) {
       this.treeDataProvider.sortOrder = SortOrder.DateAddedDesc;
@@ -608,14 +581,6 @@ export class DatabaseUI extends DisposableObject {
     }
   }
 
-  private async handleClearLanguageFilter() {
-    await this.languageContext.clearLanguageContext();
-  }
-
-  private async handleChangeLanguageFilter(languageFilter: QueryLanguage) {
-    await this.languageContext.setLanguageContext(languageFilter);
-  }
-
   private async handleUpgradeCurrentDatabase(): Promise<void> {
     return withProgress(
       async (progress, token) => {
@@ -703,6 +668,25 @@ export class DatabaseUI extends DisposableObject {
     );
   }
 
+  private async handleTrimCache(): Promise<void> {
+    return withProgress(
+      async (_progress, token) => {
+        if (
+          this.queryServer !== undefined &&
+          this.databaseManager.currentDatabaseItem !== undefined
+        ) {
+          await this.queryServer.trimCacheInDatabase(
+            this.databaseManager.currentDatabaseItem,
+            token,
+          );
+        }
+      },
+      {
+        title: "Trimming cache",
+      },
+    );
+  }
+
   private async handleGetCurrentDatabase(): Promise<string | undefined> {
     const dbItem = await this.getDatabaseItemInternal(undefined);
     return dbItem?.databaseUri.fsPath;
@@ -723,7 +707,9 @@ export class DatabaseUI extends DisposableObject {
               this.queryServer?.cliServer,
             );
           } else {
-            await this.databaseManager.openDatabase(uri);
+            await this.databaseManager.openDatabase(uri, {
+              type: "folder",
+            });
           }
         } catch (e) {
           // rethrow and let this be handled by default error handling.
@@ -838,7 +824,9 @@ export class DatabaseUI extends DisposableObject {
         if (byFolder) {
           const fixedUri = await this.fixDbUri(uri);
           // we are selecting a database folder
-          return await this.databaseManager.openDatabase(fixedUri);
+          return await this.databaseManager.openDatabase(fixedUri, {
+            type: "folder",
+          });
         } else {
           // we are selecting a database archive. Must unzip into a workspace-controlled area
           // before importing.

extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts

@@ -14,6 +14,7 @@ import { isLikelyDatabaseRoot } from "./db-contents-heuristics";
 import { stat } from "fs-extra";
 import { containsPath, pathsEqual } from "../../common/files";
 import { DatabaseContents } from "./database-contents";
+import { DatabaseOrigin } from "./database-origin";
 
 export class DatabaseItemImpl implements DatabaseItem {
   // These are only public in the implementation, they are readonly in the interface
@@ -61,6 +62,10 @@ export class DatabaseItemImpl implements DatabaseItem {
     return this.options.dateAdded;
   }
 
+  public get origin(): DatabaseOrigin | undefined {
+    return this.options.origin;
+  }
+
   public resolveSourceFile(uriStr: string | undefined): vscode.Uri {
     const sourceArchive = this.sourceArchive;
     const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined;
@@ -192,7 +197,9 @@ export class DatabaseItemImpl implements DatabaseItem {
    * Holds if `uri` belongs to this database's source archive.
    */
   public belongsToSourceArchiveExplorerUri(uri: vscode.Uri): boolean {
-    if (this.sourceArchive === undefined) return false;
+    if (this.sourceArchive === undefined) {
+      return false;
+    }
     return (
       uri.scheme === zipArchiveScheme &&
       decodeSourceArchiveUri(uri).sourceArchiveZipPath ===

extensions/ql-vscode/src/databases/local-databases/database-item.ts

@@ -2,6 +2,7 @@ import vscode from "vscode";
 import * as cli from "../../codeql-cli/cli";
 import { DatabaseContents } from "./database-contents";
 import { DatabaseOptions } from "./database-options";
+import { DatabaseOrigin } from "./database-origin";
 
 /** An item in the list of available databases */
 export interface DatabaseItem {
@@ -25,6 +26,11 @@ export interface DatabaseItem {
    */
   readonly dateAdded: number | undefined;
 
+  /**
+   * The origin this database item was retrieved from or undefined if unknown.
+   */
+  readonly origin: DatabaseOrigin | undefined;
+
   /** If the database is invalid, describes why. */
   readonly error: Error | undefined;
 

extensions/ql-vscode/src/databases/local-databases/database-manager.ts

@@ -7,6 +7,7 @@ import { QueryRunner } from "../../query-server";
 import * as cli from "../../codeql-cli/cli";
 import { ProgressCallback, withProgress } from "../../common/vscode/progress";
 import {
+  addDatabaseSourceToWorkspace,
   getAutogenerateQlPacks,
   isCodespacesTemplate,
   setAutogenerateQlPacks,
@@ -19,7 +20,10 @@ import {
   getFirstWorkspaceFolder,
   isFolderAlreadyInWorkspace,
 } from "../../common/vscode/workspace-folders";
-import { isQueryLanguage } from "../../common/query-language";
+import {
+  isQueryLanguage,
+  tryGetQueryLanguage,
+} from "../../common/query-language";
 import { existsSync } from "fs";
 import { QlPackGenerator } from "../../local-queries/qlpack-generator";
 import { asError, getErrorMessage } from "../../common/helpers-pure";
@@ -30,6 +34,8 @@ import { containsPath } from "../../common/files";
 import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
 import { DatabaseResolver } from "./database-resolver";
 import { telemetryListener } from "../../common/vscode/telemetry";
+import { LanguageContextStore } from "../../language-context-store";
+import { DatabaseOrigin } from "./database-origin";
 
 /**
  * The name of the key in the workspaceState dictionary in which we
@@ -100,11 +106,25 @@ export class DatabaseManager extends DisposableObject {
     private readonly app: App,
     private readonly qs: QueryRunner,
     private readonly cli: cli.CodeQLCliServer,
+    private readonly languageContext: LanguageContextStore,
     public logger: Logger,
   ) {
     super();
 
     qs.onStart(this.reregisterDatabases.bind(this));
+
+    this.push(
+      this.languageContext.onLanguageContextChanged(async () => {
+        if (
+          this.currentDatabaseItem !== undefined &&
+          !this.languageContext.shouldInclude(
+            tryGetQueryLanguage(this.currentDatabaseItem.language),
+          )
+        ) {
+          await this.setCurrentDatabaseItem(undefined);
+        }
+      }),
+    );
   }
 
   /**
@@ -113,14 +133,19 @@ export class DatabaseManager extends DisposableObject {
    */
   public async openDatabase(
     uri: vscode.Uri,
+    origin: DatabaseOrigin | undefined,
     makeSelected = true,
     displayName?: string,
     {
       isTutorialDatabase = false,
-      addSourceArchiveFolder = true,
+      addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
     }: OpenDatabaseOptions = {},
   ): Promise<DatabaseItem> {
-    const databaseItem = await this.createDatabaseItem(uri, displayName);
+    const databaseItem = await this.createDatabaseItem(
+      uri,
+      origin,
+      displayName,
+    );
 
     return await this.addExistingDatabaseItem(
       databaseItem,
@@ -140,7 +165,7 @@ export class DatabaseManager extends DisposableObject {
     databaseItem: DatabaseItemImpl,
     makeSelected: boolean,
     isTutorialDatabase?: boolean,
-    addSourceArchiveFolder = true,
+    addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
   ): Promise<DatabaseItem> {
     const existingItem = this.findDatabaseItem(databaseItem.databaseUri);
     if (existingItem !== undefined) {
@@ -171,6 +196,7 @@ export class DatabaseManager extends DisposableObject {
    */
   private async createDatabaseItem(
     uri: vscode.Uri,
+    origin: DatabaseOrigin | undefined,
     displayName: string | undefined,
   ): Promise<DatabaseItemImpl> {
     const contents = await DatabaseResolver.resolveDatabaseContents(uri);
@@ -179,6 +205,7 @@ export class DatabaseManager extends DisposableObject {
       displayName,
       dateAdded: Date.now(),
       language: await this.getPrimaryLanguage(uri.fsPath),
+      origin,
     };
     const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions);
 
@@ -194,6 +221,7 @@ export class DatabaseManager extends DisposableObject {
    */
   public async createOrOpenDatabaseItem(
     uri: vscode.Uri,
+    origin: DatabaseOrigin | undefined,
   ): Promise<DatabaseItem> {
     const existingItem = this.findDatabaseItem(uri);
     if (existingItem !== undefined) {
@@ -202,7 +230,7 @@ export class DatabaseManager extends DisposableObject {
     }
 
     // We don't add this to the list automatically, but the user can add it later.
-    return this.createDatabaseItem(uri, undefined);
+    return this.createDatabaseItem(uri, origin, undefined);
   }
 
   public async createSkeletonPacks(databaseItem: DatabaseItem) {
@@ -230,8 +258,10 @@ export class DatabaseManager extends DisposableObject {
     const firstWorkspaceFolder = getFirstWorkspaceFolder();
     const folderName = `codeql-custom-queries-${databaseItem.language}`;
 
+    const qlpackStoragePath = join(firstWorkspaceFolder, folderName);
+
     if (
-      existsSync(join(firstWorkspaceFolder, folderName)) ||
+      existsSync(qlpackStoragePath) ||
       isFolderAlreadyInWorkspace(folderName)
     ) {
       return;
@@ -245,7 +275,7 @@ export class DatabaseManager extends DisposableObject {
       `We've noticed you don't have a CodeQL pack available to analyze this database. Can we set up a query pack for you?`,
     );
 
-    if (answer === "No") {
+    if (answer === "No" || answer === undefined) {
       return;
     }
 
@@ -256,10 +286,10 @@ export class DatabaseManager extends DisposableObject {
 
     try {
       const qlPackGenerator = new QlPackGenerator(
-        folderName,
         databaseItem.language,
         this.cli,
-        firstWorkspaceFolder,
+        qlpackStoragePath,
+        qlpackStoragePath,
       );
       await qlPackGenerator.generate();
     } catch (e: unknown) {
@@ -335,6 +365,7 @@ export class DatabaseManager extends DisposableObject {
     let displayName: string | undefined = undefined;
     let dateAdded = undefined;
     let language = undefined;
+    let origin = undefined;
     if (state.options) {
       if (typeof state.options.displayName === "string") {
         displayName = state.options.displayName;
@@ -343,6 +374,7 @@ export class DatabaseManager extends DisposableObject {
         dateAdded = state.options.dateAdded;
       }
       language = state.options.language;
+      origin = state.options.origin;
     }
 
     const dbBaseUri = vscode.Uri.parse(state.uri, true);
@@ -355,6 +387,7 @@ export class DatabaseManager extends DisposableObject {
       displayName,
       dateAdded,
       language,
+      origin,
     };
     const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions);
 

extensions/ql-vscode/src/databases/local-databases/database-options.ts

@@ -1,10 +1,14 @@
+import { DatabaseOrigin } from "./database-origin";
+
 export interface DatabaseOptions {
   displayName?: string;
   dateAdded?: number | undefined;
   language?: string;
+  origin?: DatabaseOrigin;
 }
 
 export interface FullDatabaseOptions extends DatabaseOptions {
   dateAdded: number | undefined;
   language: string | undefined;
+  origin: DatabaseOrigin | undefined;
 }

extensions/ql-vscode/src/databases/local-databases/database-origin.ts

@@ -0,0 +1,32 @@
+interface DatabaseOriginFolder {
+  type: "folder";
+}
+
+interface DatabaseOriginArchive {
+  type: "archive";
+  path: string;
+}
+
+interface DatabaseOriginGitHub {
+  type: "github";
+  repository: string;
+  databaseId: number;
+  databaseCreatedAt: string;
+  commitOid: string | null;
+}
+
+interface DatabaseOriginInternet {
+  type: "url";
+  url: string;
+}
+
+interface DatabaseOriginDebugger {
+  type: "debugger";
+}
+
+export type DatabaseOrigin =
+  | DatabaseOriginFolder
+  | DatabaseOriginArchive
+  | DatabaseOriginGitHub
+  | DatabaseOriginInternet
+  | DatabaseOriginDebugger;

extensions/ql-vscode/src/databases/local-databases/locations.ts

@@ -128,6 +128,8 @@ export async function showLocation(location?: Location) {
           // avoid preview mode so editor is sticky and will be added to navigation and search histories.
           preview: false,
           viewColumn: ViewColumn.One,
+          // Keep the focus on the results view so that the user can easily navigate to the next result.
+          preserveFocus: true,
         });
 
   const range = location.range;

extensions/ql-vscode/src/databases/ui/db-item-mapper.ts

@@ -1,6 +1,5 @@
 import { DbItem, DbItemKind } from "../db-item";
 import {
-  createDbTreeViewItemLocalDatabase,
   createDbTreeViewItemOwner,
   createDbTreeViewItemRepo,
   createDbTreeViewItemRoot,
@@ -11,14 +10,6 @@ import {
 
 export function mapDbItemToTreeViewItem(dbItem: DbItem): DbTreeViewItem {
   switch (dbItem.kind) {
-    case DbItemKind.RootLocal:
-      return createDbTreeViewItemRoot(
-        dbItem,
-        "local",
-        "Local databases",
-        dbItem.children.map((c) => mapDbItemToTreeViewItem(c)),
-      );
-
     case DbItemKind.RootRemote:
       return createDbTreeViewItemRoot(
         dbItem,
@@ -46,19 +37,5 @@ export function mapDbItemToTreeViewItem(dbItem: DbItem): DbTreeViewItem {
 
     case DbItemKind.RemoteRepo:
       return createDbTreeViewItemRepo(dbItem, dbItem.repoFullName);
-
-    case DbItemKind.LocalList:
-      return createDbTreeViewItemUserDefinedList(
-        dbItem,
-        dbItem.listName,
-        dbItem.databases.map(mapDbItemToTreeViewItem),
-      );
-
-    case DbItemKind.LocalDatabase:
-      return createDbTreeViewItemLocalDatabase(
-        dbItem,
-        dbItem.databaseName,
-        dbItem.language,
-      );
   }
 }

extensions/ql-vscode/src/databases/ui/db-panel.ts

@@ -1,5 +1,4 @@
 import {
-  ProgressLocation,
   QuickPickItem,
   TreeView,
   TreeViewExpansionEvent,
@@ -7,7 +6,10 @@ import {
   window,
   workspace,
 } from "vscode";
-import { UserCancellationException } from "../../common/vscode/progress";
+import {
+  UserCancellationException,
+  withProgress,
+} from "../../common/vscode/progress";
 import {
   getNwoFromGitHubUrl,
   isValidGitHubNwo,
@@ -15,14 +17,7 @@ import {
   isValidGitHubOwner,
 } from "../../common/github-url-identifier-helper";
 import { DisposableObject } from "../../common/disposable-object";
-import {
-  DbItem,
-  DbItemKind,
-  DbListKind,
-  LocalDatabaseDbItem,
-  LocalListDbItem,
-  RemoteUserDefinedListDbItem,
-} from "../db-item";
+import { DbItem, DbItemKind, RemoteUserDefinedListDbItem } from "../db-item";
 import { getDbItemName } from "../db-item-naming";
 import { DbManager } from "../db-manager";
 import { DbTreeDataProvider } from "./db-tree-data-provider";
@@ -34,19 +29,12 @@ import { DatabasePanelCommands } from "../../common/commands";
 import { App } from "../../common/app";
 import { QueryLanguage } from "../../common/query-language";
 import { getCodeSearchRepositories } from "../code-search-api";
-import {
-  showAndLogErrorMessage,
-  showAndLogInformationMessage,
-} from "../../common/logging";
+import { showAndLogErrorMessage } from "../../common/logging";
 
 export interface RemoteDatabaseQuickPickItem extends QuickPickItem {
   remoteDatabaseKind: string;
 }
 
-export interface AddListQuickPickItem extends QuickPickItem {
-  databaseKind: DbListKind;
-}
-
 interface CodeSearchQuickPickItem extends QuickPickItem {
   language: string;
 }
@@ -224,8 +212,6 @@ export class DbPanel extends DisposableObject {
   }
 
   private async addNewList(): Promise<void> {
-    const listKind = DbListKind.Remote;
-
     const listName = await window.showInputBox({
       prompt: "Enter a name for the new list",
       placeHolder: "example-list",
@@ -234,7 +220,7 @@ export class DbPanel extends DisposableObject {
       return;
     }
 
-    if (this.dbManager.doesListExist(listKind, listName)) {
+    if (this.dbManager.doesListExist(listName)) {
       void showAndLogErrorMessage(
         this.app.logger,
         `The list '${listName}' already exists`,
@@ -242,7 +228,7 @@ export class DbPanel extends DisposableObject {
       return;
     }
 
-    await this.dbManager.addNewList(listKind, listName);
+    await this.dbManager.addNewList(listName);
   }
 
   private async setSelectedItem(treeViewItem: DbTreeViewItem): Promise<void> {
@@ -278,59 +264,13 @@ export class DbPanel extends DisposableObject {
       return;
     }
 
-    switch (dbItem.kind) {
-      case DbItemKind.LocalList:
-        await this.renameLocalListItem(dbItem, newName);
-        break;
-      case DbItemKind.LocalDatabase:
-        await this.renameLocalDatabaseItem(dbItem, newName);
-        break;
-      case DbItemKind.RemoteUserDefinedList:
-        await this.renameVariantAnalysisUserDefinedListItem(dbItem, newName);
-        break;
-      default:
-        throw Error(`Action not allowed for the '${dbItem.kind}' db item kind`);
+    if (dbItem.kind === DbItemKind.RemoteUserDefinedList) {
+      await this.renameVariantAnalysisUserDefinedListItem(dbItem, newName);
+    } else {
+      throw Error(`Action not allowed for the '${dbItem.kind}' db item kind`);
     }
   }
 
-  private async renameLocalListItem(
-    dbItem: LocalListDbItem,
-    newName: string,
-  ): Promise<void> {
-    if (dbItem.listName === newName) {
-      return;
-    }
-
-    if (this.dbManager.doesListExist(DbListKind.Local, newName)) {
-      void showAndLogErrorMessage(
-        this.app.logger,
-        `The list '${newName}' already exists`,
-      );
-      return;
-    }
-
-    await this.dbManager.renameList(dbItem, newName);
-  }
-
-  private async renameLocalDatabaseItem(
-    dbItem: LocalDatabaseDbItem,
-    newName: string,
-  ): Promise<void> {
-    if (dbItem.databaseName === newName) {
-      return;
-    }
-
-    if (this.dbManager.doesLocalDbExist(newName, dbItem.parentListName)) {
-      void showAndLogErrorMessage(
-        this.app.logger,
-        `The database '${newName}' already exists`,
-      );
-      return;
-    }
-
-    await this.dbManager.renameLocalDb(dbItem, newName);
-  }
-
   private async renameVariantAnalysisUserDefinedListItem(
     dbItem: RemoteUserDefinedListDbItem,
     newName: string,
@@ -339,7 +279,7 @@ export class DbPanel extends DisposableObject {
       return;
     }
 
-    if (this.dbManager.doesListExist(DbListKind.Remote, newName)) {
+    if (this.dbManager.doesListExist(newName)) {
       void showAndLogErrorMessage(
         this.app.logger,
         `The list '${newName}' already exists`,
@@ -409,15 +349,8 @@ export class DbPanel extends DisposableObject {
       return;
     }
 
-    await window.withProgress(
-      {
-        location: ProgressLocation.Notification,
-        title: "Searching for repositories... This might take a while",
-        cancellable: true,
-      },
+    await withProgress(
       async (progress, token) => {
-        progress.report({ increment: 10 });
-
         const repositories = await getCodeSearchRepositories(
           `${codeSearchQuery} ${languagePrompt}`,
           progress,
@@ -426,18 +359,22 @@ export class DbPanel extends DisposableObject {
           this.app.logger,
         );
 
-        token.onCancellationRequested(() => {
-          void showAndLogInformationMessage(
-            this.app.logger,
-            "Code search cancelled",
-          );
-          return;
+        if (token.isCancellationRequested) {
+          throw new UserCancellationException("Code search cancelled.", true);
+        }
+
+        progress({
+          maxStep: 12,
+          step: 12,
+          message: "Processing results...",
         });
 
-        progress.report({ increment: 10, message: "Processing results..." });
-
         await this.dbManager.addNewRemoteReposToList(repositories, listName);
       },
+      {
+        title: "Searching for repositories...",
+        cancellable: true,
+      },
     );
   }
 

extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts

@@ -29,18 +29,12 @@ export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] {
 }
 
 const dbItemKindsThatCanBeRemoved = [
-  DbItemKind.LocalList,
   DbItemKind.RemoteUserDefinedList,
-  DbItemKind.LocalDatabase,
   DbItemKind.RemoteRepo,
   DbItemKind.RemoteOwner,
 ];
 
-const dbItemKindsThatCanBeRenamed = [
-  DbItemKind.LocalList,
-  DbItemKind.RemoteUserDefinedList,
-  DbItemKind.LocalDatabase,
-];
+const dbItemKindsThatCanBeRenamed = [DbItemKind.RemoteUserDefinedList];
 
 const dbItemKindsThatCanBeOpenedOnGitHub = [
   DbItemKind.RemoteOwner,

extensions/ql-vscode/src/databases/ui/db-tree-view-item.ts

@@ -2,13 +2,10 @@ import * as vscode from "vscode";
 import {
   DbItem,
   isSelectableDbItem,
-  LocalDatabaseDbItem,
-  LocalListDbItem,
   RemoteOwnerDbItem,
   RemoteRepoDbItem,
   RemoteSystemDefinedListDbItem,
   RemoteUserDefinedListDbItem,
-  RootLocalDbItem,
   RootRemoteDbItem,
 } from "../db-item";
 import { getDbItemActions } from "./db-tree-view-item-action";
@@ -74,7 +71,7 @@ export function createDbTreeViewItemError(
 }
 
 export function createDbTreeViewItemRoot(
-  dbItem: RootLocalDbItem | RootRemoteDbItem,
+  dbItem: RootRemoteDbItem,
   label: string,
   tooltip: string,
   children: DbTreeViewItem[],
@@ -105,7 +102,7 @@ export function createDbTreeViewItemSystemDefinedList(
 }
 
 export function createDbTreeViewItemUserDefinedList(
-  dbItem: LocalListDbItem | RemoteUserDefinedListDbItem,
+  dbItem: RemoteUserDefinedListDbItem,
   listName: string,
   children: DbTreeViewItem[],
 ): DbTreeViewItem {
@@ -147,21 +144,6 @@ export function createDbTreeViewItemRepo(
   );
 }
 
-export function createDbTreeViewItemLocalDatabase(
-  dbItem: LocalDatabaseDbItem,
-  databaseName: string,
-  language: string,
-): DbTreeViewItem {
-  return new DbTreeViewItem(
-    dbItem,
-    new vscode.ThemeIcon("database"),
-    databaseName,
-    `Language: ${language}`,
-    vscode.TreeItemCollapsibleState.None,
-    [],
-  );
-}
-
 function getCollapsibleState(
   expanded: boolean,
 ): vscode.TreeItemCollapsibleState {

extensions/ql-vscode/src/debugger/debugger-ui.ts

@@ -105,7 +105,9 @@ class QLDebugAdapterTracker
     body: CodeQLProtocol.EvaluationStartedEvent["body"],
   ): Promise<void> {
     const dbUri = Uri.file(this.configuration.database);
-    const dbItem = await this.dbm.createOrOpenDatabaseItem(dbUri);
+    const dbItem = await this.dbm.createOrOpenDatabaseItem(dbUri, {
+      type: "debugger",
+    });
 
     // When cancellation is requested from the query history view, we just stop the debug session.
     const tokenSource = new CancellationTokenSource();

extensions/ql-vscode/src/extension.ts

@@ -13,7 +13,7 @@ import {
   workspace,
 } from "vscode";
 import { LanguageClient } from "vscode-languageclient/node";
-import { arch, platform, homedir } from "os";
+import { arch, homedir, platform } from "os";
 import { ensureDir } from "fs-extra";
 import { join } from "path";
 import { dirSync } from "tmp-promise";
@@ -35,13 +35,13 @@ import {
 } from "./config";
 import {
   AstViewer,
-  install,
+  createIDEServer,
   getQueryEditorCommands,
+  install,
   TemplatePrintAstProvider,
   TemplatePrintCfgProvider,
   TemplateQueryDefinitionProvider,
   TemplateQueryReferenceProvider,
-  createIDEServer,
 } from "./language-support";
 import { DatabaseManager } from "./databases/local-databases";
 import { DatabaseUI } from "./databases/local-databases-ui";
@@ -68,15 +68,15 @@ import {
   getErrorStack,
 } from "./common/helpers-pure";
 import {
-  ResultsView,
-  WebviewReveal,
   LocalQueries,
   QuickEvalCodeLensProvider,
+  ResultsView,
+  WebviewReveal,
 } from "./local-queries";
 import {
   BaseLogger,
-  showAndLogExceptionWithTelemetry,
   showAndLogErrorMessage,
+  showAndLogExceptionWithTelemetry,
   showAndLogInformationMessage,
   showAndLogWarningMessage,
 } from "./common/logging";
@@ -88,10 +88,6 @@ import {
 } from "./common/logging/vscode";
 import { QueryHistoryManager } from "./query-history/query-history-manager";
 import { CompletedLocalQueryInfo } from "./query-results";
-import {
-  LegacyQueryRunner,
-  QueryServerClient as LegacyQueryServerClient,
-} from "./query-server/legacy";
 import { QLTestAdapterFactory } from "./query-testing/test-adapter";
 import { TestUIService } from "./query-testing/test-ui";
 import { CompareView } from "./compare/compare-view";
@@ -136,6 +132,8 @@ import { NewQueryRunner, QueryRunner, QueryServerClient } from "./query-server";
 import { QueriesModule } from "./queries-panel/queries-module";
 import { OpenReferencedFileCodeLensProvider } from "./local-queries/open-referenced-file-code-lens-provider";
 import { LanguageContextStore } from "./language-context-store";
+import { LanguageSelectionPanel } from "./language-selection-panel/language-selection-panel";
+import { GitHubDatabasesModule } from "./databases/github-databases";
 
 /**
  * extension.ts
@@ -768,17 +766,28 @@ async function activateWithInstalledDistribution(
     fsWatcher.onDidDelete(clearPackCache);
   }
 
+  void extLogger.log("Initializing language context.");
+  const languageContext = new LanguageContextStore(app);
+
+  void extLogger.log("Initializing language selector.");
+  const languageSelectionPanel = new LanguageSelectionPanel(languageContext);
+  ctx.subscriptions.push(languageSelectionPanel);
+
   void extLogger.log("Initializing database manager.");
-  const dbm = new DatabaseManager(ctx, app, qs, cliServer, extLogger);
+  const dbm = new DatabaseManager(
+    ctx,
+    app,
+    qs,
+    cliServer,
+    languageContext,
+    extLogger,
+  );
 
   // Let this run async.
   void dbm.loadPersistedState();
 
   ctx.subscriptions.push(dbm);
 
-  void extLogger.log("Initializing language context.");
-  const languageContext = new LanguageContextStore(app);
-
   void extLogger.log("Initializing database panel.");
   const databaseUI = new DatabaseUI(
     app,
@@ -790,7 +799,11 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(databaseUI);
 
-  QueriesModule.initialize(app, languageContext, cliServer);
+  const queriesModule = QueriesModule.initialize(
+    app,
+    languageContext,
+    cliServer,
+  );
 
   void extLogger.log("Initializing evaluator log viewer.");
   const evalLogViewer = new EvalLogViewer();
@@ -854,6 +867,13 @@ async function activateWithInstalledDistribution(
     ),
   );
 
+  await GitHubDatabasesModule.initialize(
+    app,
+    dbm,
+    getContextStoragePath(ctx),
+    cliServer,
+  );
+
   void extLogger.log("Initializing query history.");
   const queryHistoryDirs: QueryHistoryDirs = {
     localQueriesDirPath: queryStorageDir,
@@ -925,9 +945,14 @@ async function activateWithInstalledDistribution(
     databaseUI,
     localQueryResultsView,
     queryStorageDir,
+    languageContext,
   );
   ctx.subscriptions.push(localQueries);
 
+  queriesModule.onDidChangeSelection((event) =>
+    localQueries.setSelectedQueryTreeViewItems(event.selection),
+  );
+
   void extLogger.log("Initializing debugger factory.");
   ctx.subscriptions.push(
     new QLDebugAdapterDescriptorFactory(queryStorageDir, qs, localQueries),
@@ -1016,6 +1041,7 @@ async function activateWithInstalledDistribution(
     ...getPackagingCommands({
       cliServer,
     }),
+    ...languageSelectionPanel.getCommands(),
     ...modelEditorModule.getCommands(),
     ...evalLogViewer.getCommands(),
     ...summaryLanguageSupport.getCommands(),
@@ -1164,10 +1190,7 @@ function addUnhandledRejectionListener() {
       const message = redactableError(
         asError(error),
       )`Unhandled error: ${getErrorMessage(error)}`;
-      const stack = getErrorStack(error);
-      const fullMessage = stack
-        ? `Unhandled error: ${stack}`
-        : message.fullMessage;
+      const fullMessage = message.fullMessageWithStack;
 
       // Add a catch so that showAndLogExceptionWithTelemetry fails, we avoid
       // triggering "unhandledRejection" and avoid an infinite loop
@@ -1215,29 +1238,17 @@ async function createQueryServer(
       { title: "CodeQL query server", location: ProgressLocation.Window },
       task,
     );
-  if (await cliServer.cliConstraints.supportsNewQueryServer()) {
-    const qs = new QueryServerClient(
-      app,
-      qlConfigurationListener,
-      cliServer,
-      qsOpts,
-      progressCallback,
-    );
-    ctx.subscriptions.push(qs);
-    await qs.startQueryServer();
-    return new NewQueryRunner(qs);
-  } else {
-    const qs = new LegacyQueryServerClient(
-      app,
-      qlConfigurationListener,
-      cliServer,
-      qsOpts,
-      progressCallback,
-    );
-    ctx.subscriptions.push(qs);
-    await qs.startQueryServer();
-    return new LegacyQueryRunner(qs);
-  }
+
+  const qs = new QueryServerClient(
+    app,
+    qlConfigurationListener,
+    cliServer,
+    qsOpts,
+    progressCallback,
+  );
+  ctx.subscriptions.push(qs);
+  await qs.startQueryServer();
+  return new NewQueryRunner(qs);
 }
 
 function getContextStoragePath(ctx: ExtensionContext) {

extensions/ql-vscode/src/language-context-store.ts

@@ -43,7 +43,36 @@ export class LanguageContextStore extends DisposableObject {
     );
   }
 
+  /**
+   * This returns true if the given language should be included.
+   *
+   * That means that either the given language is selected or the "All" option is selected.
+   *
+   * @param language a query language or undefined if the language is unknown.
+   */
   public shouldInclude(language: QueryLanguage | undefined): boolean {
     return this.languageFilter === "All" || this.languageFilter === language;
   }
+
+  /**
+   * This returns true if the given language is selected.
+   *
+   * If no language is given then it returns true if the "All" option is selected.
+   *
+   * @param language a query language or undefined.
+   */
+  public isSelectedLanguage(language: QueryLanguage | undefined): boolean {
+    return (
+      (this.languageFilter === "All" && language === undefined) ||
+      this.languageFilter === language
+    );
+  }
+
+  public get selectedLanguage(): QueryLanguage | undefined {
+    if (this.languageFilter === "All") {
+      return undefined;
+    }
+
+    return this.languageFilter;
+  }
 }

extensions/ql-vscode/src/language-selection-panel/language-selection-data-provider.ts

@@ -0,0 +1,91 @@
+import { DisposableObject } from "../common/disposable-object";
+import { LanguageContextStore } from "../language-context-store";
+import {
+  Event,
+  EventEmitter,
+  ThemeIcon,
+  TreeDataProvider,
+  TreeItem,
+} from "vscode";
+import {
+  QueryLanguage,
+  getLanguageDisplayName,
+} from "../common/query-language";
+
+const ALL_LANGUAGE_SELECTION_OPTIONS = [
+  undefined, // All languages
+  QueryLanguage.Cpp,
+  QueryLanguage.CSharp,
+  QueryLanguage.Go,
+  QueryLanguage.Java,
+  QueryLanguage.Javascript,
+  QueryLanguage.Python,
+  QueryLanguage.Ruby,
+  QueryLanguage.Swift,
+];
+
+// A tree view items consisting of of a language (or undefined for all languages)
+// and a boolean indicating whether it is selected or not.
+export class LanguageSelectionTreeViewItem extends TreeItem {
+  constructor(
+    public readonly language: QueryLanguage | undefined,
+    public readonly selected: boolean = false,
+  ) {
+    const label = language ? getLanguageDisplayName(language) : "All languages";
+    super(label);
+
+    this.iconPath = selected ? new ThemeIcon("check") : undefined;
+    this.contextValue = selected ? undefined : "canBeSelected";
+  }
+}
+
+export class LanguageSelectionTreeDataProvider
+  extends DisposableObject
+  implements TreeDataProvider<LanguageSelectionTreeViewItem>
+{
+  private treeItems: LanguageSelectionTreeViewItem[];
+  private readonly onDidChangeTreeDataEmitter = this.push(
+    new EventEmitter<void>(),
+  );
+
+  public constructor(private readonly languageContext: LanguageContextStore) {
+    super();
+
+    this.treeItems = this.createTree();
+
+    // If the language context changes, we need to update the tree.
+    this.push(
+      this.languageContext.onLanguageContextChanged(() => {
+        this.treeItems = this.createTree();
+        this.onDidChangeTreeDataEmitter.fire();
+      }),
+    );
+  }
+
+  public get onDidChangeTreeData(): Event<void> {
+    return this.onDidChangeTreeDataEmitter.event;
+  }
+
+  public getTreeItem(item: LanguageSelectionTreeViewItem): TreeItem {
+    return item;
+  }
+
+  public getChildren(
+    item?: LanguageSelectionTreeViewItem,
+  ): LanguageSelectionTreeViewItem[] {
+    if (!item) {
+      return this.treeItems;
+    } else {
+      return [];
+    }
+  }
+
+  private createTree(): LanguageSelectionTreeViewItem[] {
+    return ALL_LANGUAGE_SELECTION_OPTIONS.map((language) => {
+      return new LanguageSelectionTreeViewItem(
+        language,
+        this.languageContext.isSelectedLanguage(language),
+      );
+    });
+  }
+}

extensions/ql-vscode/src/language-selection-panel/language-selection-panel.ts

@@ -0,0 +1,41 @@
+import { DisposableObject } from "../common/disposable-object";
+import { window } from "vscode";
+import {
+  LanguageSelectionTreeDataProvider,
+  LanguageSelectionTreeViewItem,
+} from "./language-selection-data-provider";
+import { LanguageContextStore } from "../language-context-store";
+import { LanguageSelectionCommands } from "../common/commands";
+
+// This panel allows the selection of a single language, that will
+// then filter all other relevant views (e.g. db panel, query history).
+export class LanguageSelectionPanel extends DisposableObject {
+  constructor(private readonly languageContext: LanguageContextStore) {
+    super();
+
+    const dataProvider = new LanguageSelectionTreeDataProvider(languageContext);
+    this.push(dataProvider);
+
+    const treeView = window.createTreeView("codeQLLanguageSelection", {
+      treeDataProvider: dataProvider,
+    });
+    this.push(treeView);
+  }
+
+  public getCommands(): LanguageSelectionCommands {
+    return {
+      "codeQLLanguageSelection.setSelectedItem":
+        this.handleSetSelectedLanguage.bind(this),
+    };
+  }
+
+  private async handleSetSelectedLanguage(
+    item: LanguageSelectionTreeViewItem,
+  ): Promise<void> {
+    if (item.language) {
+      await this.languageContext.setLanguageContext(item.language);
+    } else {
+      await this.languageContext.clearLanguageContext();
+    }
+  }
+}

extensions/ql-vscode/src/language-support/contextual/location-finder.ts

@@ -9,16 +9,19 @@ import {
   ResultSetSchema,
 } from "../../common/bqrs-cli-types";
 import { CodeQLCliServer } from "../../codeql-cli/cli";
-import { DatabaseManager, DatabaseItem } from "../../databases/local-databases";
+import { DatabaseItem, DatabaseManager } from "../../databases/local-databases";
 import { ProgressCallback } from "../../common/vscode/progress";
 import { KeyType } from "./key-type";
-import { resolveQueries, runContextualQuery } from "./query-resolver";
+import {
+  resolveContextualQlPacksForDatabase,
+  resolveContextualQueries,
+  runContextualQuery,
+} from "./query-resolver";
 import { CancellationToken, LocationLink, Uri } from "vscode";
 import { QueryOutputDir } from "../../run-queries-shared";
 import { QueryRunner } from "../../query-server";
 import { QueryResultType } from "../../query-server/new-messages";
 import { fileRangeFromURI } from "./file-range-from-uri";
-import { qlpackOfDatabase } from "../../local-queries";
 
 export const SELECT_QUERY_NAME = "#select";
 export const SELECTED_SOURCE_FILE = "selectedSourceFile";
@@ -63,11 +66,11 @@ export async function getLocationsForUriString(
     return [];
   }
 
-  const qlpack = await qlpackOfDatabase(cli, db);
+  const qlpack = await resolveContextualQlPacksForDatabase(cli, db);
   const templates = createTemplates(uri.pathWithinSourceArchive);
 
   const links: FullLocationLink[] = [];
-  for (const query of await resolveQueries(cli, qlpack, keyType)) {
+  for (const query of await resolveContextualQueries(cli, qlpack, keyType)) {
     const results = await runContextualQuery(
       query,
       db,

extensions/ql-vscode/src/language-support/contextual/query-resolver.ts

@@ -8,7 +8,10 @@ import {
 } from "./key-type";
 import { CodeQLCliServer } from "../../codeql-cli/cli";
 import { DatabaseItem } from "../../databases/local-databases";
-import { resolveQueriesByLanguagePack as resolveLocalQueries } from "../../local-queries/query-resolver";
+import {
+  qlpackOfDatabase,
+  resolveQueriesByLanguagePack as resolveLocalQueriesByLanguagePack,
+} from "../../local-queries/query-resolver";
 import { extLogger } from "../../common/logging/vscode";
 import { TeeLogger } from "../../common/logging";
 import { CancellationToken } from "vscode";
@@ -16,15 +19,56 @@ import { ProgressCallback } from "../../common/vscode/progress";
 import { CoreCompletedQuery, QueryRunner } from "../../query-server";
 import { createLockFileForStandardQuery } from "../../local-queries/standard-queries";
 
-export async function resolveQueries(
+/**
+ * This wil try to determine the qlpacks for a given database. If it can't find a matching
+ * dbscheme with downloaded packs, it will download the default packs instead.
+ *
+ * @param cli The CLI server to use
+ * @param databaseItem The database item to find the qlpacks for
+ */
+export async function resolveContextualQlPacksForDatabase(
+  cli: CodeQLCliServer,
+  databaseItem: DatabaseItem,
+): Promise<QlPacksForLanguage> {
+  try {
+    return await qlpackOfDatabase(cli, databaseItem);
+  } catch (e) {
+    // If we can't find the qlpacks for the database, use the defaults instead
+  }
+
+  const dbInfo = await cli.resolveDatabase(databaseItem.databaseUri.fsPath);
+  const primaryLanguage = dbInfo.languages?.[0];
+  if (!primaryLanguage) {
+    throw new Error("Unable to determine primary language of database");
+  }
+
+  const libraryPack = `codeql/${primaryLanguage}-all`;
+  const queryPack = `codeql/${primaryLanguage}-queries`;
+
+  await cli.packDownload([libraryPack, queryPack]);
+
+  // Return the default packs. If these weren't valid packs, the download would have failed.
+  return {
+    dbschemePack: libraryPack,
+    dbschemePackIsLibraryPack: true,
+    queryPack,
+  };
+}
+
+export async function resolveContextualQueries(
   cli: CodeQLCliServer,
   qlpacks: QlPacksForLanguage,
   keyType: KeyType,
 ): Promise<string[]> {
-  return resolveLocalQueries(cli, qlpacks, nameOfKeyType(keyType), {
-    kind: kindOfKeyType(keyType),
-    "tags contain": [tagOfKeyType(keyType)],
-  });
+  return resolveLocalQueriesByLanguagePack(
+    cli,
+    qlpacks,
+    nameOfKeyType(keyType),
+    {
+      kind: kindOfKeyType(keyType),
+      "tags contain": [tagOfKeyType(keyType)],
+    },
+  );
 }
 
 export async function runContextualQuery(

extensions/ql-vscode/src/language-support/contextual/template-provider.ts

@@ -23,11 +23,15 @@ import { KeyType } from "./key-type";
 import {
   FullLocationLink,
   getLocationsForUriString,
+  SELECTED_SOURCE_COLUMN,
   SELECTED_SOURCE_FILE,
   SELECTED_SOURCE_LINE,
-  SELECTED_SOURCE_COLUMN,
 } from "./location-finder";
-import { resolveQueries, runContextualQuery } from "./query-resolver";
+import {
+  resolveContextualQlPacksForDatabase,
+  resolveContextualQueries,
+  runContextualQuery,
+} from "./query-resolver";
 import {
   isCanary,
   NO_CACHE_AST_VIEWER,
@@ -35,7 +39,6 @@ import {
 } from "../../config";
 import { CoreCompletedQuery, QueryRunner } from "../../query-server";
 import { AstBuilder } from "../ast-viewer/ast-builder";
-import { qlpackOfDatabase } from "../../local-queries";
 import { MultiCancellationToken } from "../../common/vscode/multi-cancellation-token";
 
 /**
@@ -248,8 +251,8 @@ export class TemplatePrintAstProvider {
       throw new Error("Can't infer database from the provided source.");
     }
 
-    const qlpacks = await qlpackOfDatabase(this.cli, db);
-    const queries = await resolveQueries(
+    const qlpacks = await resolveContextualQlPacksForDatabase(this.cli, db);
+    const queries = await resolveContextualQueries(
       this.cli,
       qlpacks,
       KeyType.PrintAstQuery,
@@ -336,11 +339,11 @@ export class TemplatePrintCfgProvider {
       throw new Error("Can't infer database from the provided source.");
     }
 
-    const qlpack = await qlpackOfDatabase(this.cli, db);
+    const qlpack = await resolveContextualQlPacksForDatabase(this.cli, db);
     if (!qlpack) {
       throw new Error("Can't infer qlpack from database source archive.");
     }
-    const queries = await resolveQueries(
+    const queries = await resolveContextualQueries(
       this.cli,
       qlpack,
       KeyType.PrintCfgQuery,

extensions/ql-vscode/src/local-queries/index.ts

@@ -1,5 +1,6 @@
 export * from "./local-queries";
 export * from "./local-query-run";
+export * from "./query-constraints";
 export * from "./query-resolver";
 export * from "./quick-eval-code-lens-provider";
 export * from "./quick-query";

extensions/ql-vscode/src/local-queries/local-queries.ts

@@ -42,7 +42,6 @@ import { WebviewReveal } from "./webview";
 import { asError, getErrorMessage } from "../common/helpers-pure";
 import { CliVersionConstraint, CodeQLCliServer } from "../codeql-cli/cli";
 import { LocalQueryCommands } from "../common/commands";
-import { App } from "../common/app";
 import { DisposableObject } from "../common/disposable-object";
 import { SkeletonQueryWizard } from "./skeleton-query-wizard";
 import { LocalQueryRun } from "./local-query-run";
@@ -50,6 +49,8 @@ import { createMultiSelectionCommand } from "../common/vscode/selection-commands
 import { findLanguage } from "../codeql-cli/query-language";
 import type { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
 import { tryGetQueryLanguage } from "../common/query-language";
+import { LanguageContextStore } from "../language-context-store";
+import { ExtensionApp } from "../common/vscode/vscode-app";
 
 interface DatabaseQuickPickItem extends QuickPickItem {
   databaseItem: DatabaseItem;
@@ -62,8 +63,10 @@ export enum QuickEvalType {
 }
 
 export class LocalQueries extends DisposableObject {
+  private selectedQueryTreeViewItems: readonly QueryTreeViewItem[] = [];
+
   public constructor(
-    private readonly app: App,
+    private readonly app: ExtensionApp,
     private readonly queryRunner: QueryRunner,
     private readonly queryHistoryManager: QueryHistoryManager,
     private readonly databaseManager: DatabaseManager,
@@ -71,10 +74,17 @@ export class LocalQueries extends DisposableObject {
     private readonly databaseUI: DatabaseUI,
     private readonly localQueryResultsView: ResultsView,
     private readonly queryStorageDir: string,
+    private readonly languageContextStore: LanguageContextStore,
   ) {
     super();
   }
 
+  public setSelectedQueryTreeViewItems(
+    selection: readonly QueryTreeViewItem[],
+  ) {
+    this.selectedQueryTreeViewItems = selection;
+  }
+
   public getCommands(): LocalQueryCommands {
     return {
       "codeQL.runQuery": this.runQuery.bind(this),
@@ -109,6 +119,7 @@ export class LocalQueries extends DisposableObject {
         return this.getCurrentQuery(true);
       },
       "codeQL.createQuery": this.createSkeletonQuery.bind(this),
+      "codeQLQuickQuery.createQuery": this.createSkeletonQuery.bind(this),
     };
   }
 
@@ -323,13 +334,16 @@ export class LocalQueries extends DisposableObject {
         const credentials = isCanary() ? this.app.credentials : undefined;
         const contextStoragePath =
           this.app.workspaceStoragePath || this.app.globalStoragePath;
+        const language = this.languageContextStore.selectedLanguage;
         const skeletonQueryWizard = new SkeletonQueryWizard(
           this.cliServer,
           progress,
           credentials,
-          this.app.logger,
+          this.app,
           this.databaseManager,
           contextStoragePath,
+          this.selectedQueryTreeViewItems,
+          language,
         );
         await skeletonQueryWizard.execute();
       },
@@ -362,11 +376,15 @@ export class LocalQueries extends DisposableObject {
       );
     }
 
-    const initialInfo = await createInitialQueryInfo(selectedQuery, {
-      databaseUri: dbItem.databaseUri.toString(),
-      name: dbItem.name,
-      language: tryGetQueryLanguage(dbItem.language),
-    });
+    const initialInfo = await createInitialQueryInfo(
+      selectedQuery,
+      {
+        databaseUri: dbItem.databaseUri.toString(),
+        name: dbItem.name,
+        language: tryGetQueryLanguage(dbItem.language),
+      },
+      outputDir,
+    );
 
     // When cancellation is requested from the query history view, we just stop the debug session.
     const queryInfo = new LocalQueryInfo(initialInfo, tokenSource);

extensions/ql-vscode/src/local-queries/local-query-run.ts

@@ -97,6 +97,15 @@ export class LocalQueryRun {
    * Updates the UI in the case where query evaluation throws an exception.
    */
   public async fail(err: Error): Promise<void> {
+    const evalLogPaths = await this.summarizeEvalLog(
+      QueryResultType.OTHER_ERROR,
+      this.outputDir,
+      this.logger,
+    );
+    if (evalLogPaths !== undefined) {
+      this.queryInfo.setEvaluatorLogPaths(evalLogPaths);
+    }
+
     err.message = `Error running query: ${err.message}`;
     this.queryInfo.failureReason = err.message;
     await this.queryHistoryManager.refreshTreeView();

extensions/ql-vscode/src/local-queries/qlpack-generator.ts

@@ -1,35 +1,36 @@
-import { mkdir, writeFile } from "fs-extra";
+import { ensureDir, writeFile } from "fs-extra";
 import { dump } from "js-yaml";
-import { join } from "path";
+import { dirname, join } from "path";
 import { Uri } from "vscode";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { QueryLanguage } from "../common/query-language";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { basename } from "../common/path";
 
 export class QlPackGenerator {
-  private readonly qlpackName: string;
+  private qlpackName: string | undefined;
   private readonly qlpackVersion: string;
   private readonly header: string;
   private readonly qlpackFileName: string;
   private readonly folderUri: Uri;
 
   constructor(
-    private readonly folderName: string,
     private readonly queryLanguage: QueryLanguage,
     private readonly cliServer: CodeQLCliServer,
-    private readonly storagePath: string | undefined,
+    private readonly storagePath: string,
+    private readonly queryStoragePath: string,
+    private readonly includeFolderNameInQlpackName: boolean = false,
   ) {
-    if (this.storagePath === undefined) {
-      throw new Error("Workspace storage path is undefined");
-    }
-    this.qlpackName = `getting-started/codeql-extra-queries-${this.queryLanguage}`;
     this.qlpackVersion = "1.0.0";
     this.header = "# This is an automatically generated file.\n\n";
 
     this.qlpackFileName = "codeql-pack.yml";
-    this.folderUri = Uri.file(join(this.storagePath, this.folderName));
+    this.folderUri = Uri.file(this.storagePath);
   }
 
   public async generate() {
+    this.qlpackName = await this.determineQlpackName();
+
     // create QL pack folder and add to workspace
     await this.createWorkspaceFolder();
 
@@ -43,8 +44,39 @@ export class QlPackGenerator {
     await this.createCodeqlPackLockYaml();
   }
 
+  private async determineQlpackName(): Promise<string> {
+    let qlpackBaseName = `getting-started/codeql-extra-queries-${this.queryLanguage}`;
+    if (this.includeFolderNameInQlpackName) {
+      const folderBasename = basename(dirname(this.folderUri.fsPath));
+      if (
+        folderBasename.includes("codeql") ||
+        folderBasename.includes("queries")
+      ) {
+        // If the user has already included "codeql" or "queries" in the folder name, don't include it twice
+        qlpackBaseName = `getting-started/${folderBasename}-${this.queryLanguage}`;
+      } else {
+        qlpackBaseName = `getting-started/codeql-extra-queries-${folderBasename}-${this.queryLanguage}`;
+      }
+    }
+
+    const existingQlPacks = await this.cliServer.resolveQlpacks(
+      getOnDiskWorkspaceFolders(),
+    );
+    const existingQlPackNames = Object.keys(existingQlPacks);
+
+    let qlpackName = qlpackBaseName;
+    let i = 0;
+    while (existingQlPackNames.includes(qlpackName)) {
+      i++;
+
+      qlpackName = `${qlpackBaseName}-${i}`;
+    }
+
+    return qlpackName;
+  }
+
   private async createWorkspaceFolder() {
-    await mkdir(this.folderUri.fsPath);
+    await ensureDir(this.folderUri.fsPath);
   }
 
   private async createQlPackYaml() {
@@ -60,7 +92,7 @@ export class QlPackGenerator {
   }
 
   public async createExampleQlFile(fileName = "example.ql") {
-    const exampleQlFilePath = join(this.folderUri.fsPath, fileName);
+    const exampleQlFilePath = join(this.queryStoragePath, fileName);
 
     const exampleQl = `
 /**

extensions/ql-vscode/src/local-queries/query-constraints.ts

@@ -0,0 +1,7 @@
+export interface QueryConstraints {
+  kind?: string;
+  "tags contain"?: string[];
+  "tags contain all"?: string[];
+  "query filename"?: string;
+  "query path"?: string;
+}

extensions/ql-vscode/src/local-queries/query-resolver.ts

@@ -14,7 +14,13 @@ import { showAndLogExceptionWithTelemetry } from "../common/logging";
 import { extLogger } from "../common/logging/vscode";
 import { telemetryListener } from "../common/vscode/telemetry";
 import { SuiteInstruction } from "../packaging/suite-instruction";
+import { QueryConstraints } from "./query-constraints";
 
+/**
+ * Consider using `resolveContextualQlPacksForDatabase` instead.
+ * @param cli The CLI server instance to use.
+ * @param db The database to find the QLPack for.
+ */
 export async function qlpackOfDatabase(
   cli: Pick<CodeQLCliServer, "resolveQlpacks">,
   db: Pick<DatabaseItem, "contents">,
@@ -27,12 +33,6 @@ export async function qlpackOfDatabase(
   return await getQlPackForDbscheme(cli, dbscheme);
 }
 
-export interface QueryConstraints {
-  kind?: string;
-  "tags contain"?: string[];
-  "tags contain all"?: string[];
-}
-
 /**
  * Finds the queries with the specified kind and tags in a list of CodeQL packs.
  *
@@ -132,6 +132,14 @@ export async function resolveQueries(
       `tagged all of "${constraints["tags contain all"].join(" ")}"`,
     );
   }
+  if (constraints["query filename"] !== undefined) {
+    humanConstraints.push(
+      `with query filename "${constraints["query filename"]}"`,
+    );
+  }
+  if (constraints["query path"] !== undefined) {
+    humanConstraints.push(`with query path "${constraints["query path"]}"`);
+  }
 
   const joinedPacksToSearch = packsToSearch.join(", ");
   const error = redactableError`No ${name} queries (${humanConstraints.join(

extensions/ql-vscode/src/local-queries/quick-query.ts

@@ -5,7 +5,6 @@ import { CancellationToken, window as Window, workspace, Uri } from "vscode";
 import { LSPErrorCodes, ResponseError } from "vscode-languageclient";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { DatabaseUI } from "../databases/local-databases-ui";
-import { showBinaryChoiceDialog } from "../common/vscode/dialog";
 import { getInitialQueryContents } from "./query-contents";
 import { getPrimaryDbscheme, getQlPackForDbscheme } from "../databases/qlpack";
 import {
@@ -15,6 +14,7 @@ import {
 import { getErrorMessage } from "../common/helpers-pure";
 import { FALLBACK_QLPACK_FILENAME, getQlPackPath } from "../common/ql";
 import { App } from "../common/app";
+import { ExtensionApp } from "../common/vscode/vscode-app";
 
 const QUICK_QUERIES_DIR_NAME = "quick-queries";
 const QUICK_QUERY_QUERY_NAME = "quick-query.ql";
@@ -52,7 +52,7 @@ function findExistingQuickQueryEditor() {
  * Show a buffer the user can enter a simple query into.
  */
 export async function displayQuickQuery(
-  app: App,
+  app: ExtensionApp,
   cliServer: CodeQLCliServer,
   databaseUI: DatabaseUI,
   progress: ProgressCallback,
@@ -80,12 +80,24 @@ export async function displayQuickQuery(
     // being undefined) just let the user know that they're in for a
     // restart.
     if (workspace.workspaceFile === undefined) {
-      const makeMultiRoot = await showBinaryChoiceDialog(
-        "Quick query requires multiple folders in the workspace. Reload workspace as multi-folder workspace?",
+      const createQueryOption = 'Run "Create query"';
+      const quickQueryOption = 'Run "Quick query" anyway';
+      const quickQueryPrompt = await Window.showWarningMessage(
+        '"Quick query" requires reloading your workspace as a multi-root workspace, which may cause query history and databases to be lost.',
+        {
+          modal: true,
+          detail:
+            'The "Create query" command does not require reloading the workspace.',
+        },
+        createQueryOption,
+        quickQueryOption,
       );
-      if (makeMultiRoot) {
+      if (quickQueryPrompt === quickQueryOption) {
         updateQuickQueryDir(queriesDir, workspaceFolders.length, 0);
       }
+      if (quickQueryPrompt === createQueryOption) {
+        await app.queryServerCommands.execute("codeQLQuickQuery.createQuery");
+      }
       return;
     }
 

extensions/ql-vscode/src/local-queries/results-view.ts

@@ -655,8 +655,9 @@ export class ResultsView extends AbstractWebview<
     const schema = resultSetSchemas.find(
       (resultSet) => resultSet.name === selectedTable,
     )!;
-    if (schema === undefined)
+    if (schema === undefined) {
       throw new Error(`Query result set '${selectedTable}' not found.`);
+    }
 
     const pageSize = PAGE_SIZE.getValue<number>();
     const chunk = await this.cliServer.bqrsDecode(
@@ -771,7 +772,9 @@ export class ResultsView extends AbstractWebview<
       );
     }
 
-    if (interp.data.t !== "SarifInterpretationData") return interp;
+    if (interp.data.t !== "SarifInterpretationData") {
+      return interp;
+    }
 
     if (interp.data.runs.length !== 1) {
       void this.logger.log(
@@ -887,7 +890,9 @@ export class ResultsView extends AbstractWebview<
   ): Promise<void> {
     const { data, sourceLocationPrefix } = interpretation;
 
-    if (data.t !== "SarifInterpretationData") return;
+    if (data.t !== "SarifInterpretationData") {
+      return;
+    }
 
     if (!data.runs || !data.runs[0].results) {
       void this.logger.log(

extensions/ql-vscode/src/local-queries/skeleton-query-wizard.ts

@@ -1,19 +1,23 @@
-import { join } from "path";
-import { Uri, workspace, window as Window } from "vscode";
+import { dirname, join } from "path";
+import { Uri, window, window as Window, workspace } from "vscode";
 import { CodeQLCliServer } from "../codeql-cli/cli";
-import { BaseLogger } from "../common/logging";
+import { showAndLogExceptionWithTelemetry } from "../common/logging";
 import { Credentials } from "../common/authentication";
-import { QueryLanguage } from "../common/query-language";
+import {
+  getLanguageDisplayName,
+  QueryLanguage,
+} from "../common/query-language";
 import {
   getFirstWorkspaceFolder,
-  isFolderAlreadyInWorkspace,
+  getOnDiskWorkspaceFolders,
 } from "../common/vscode/workspace-folders";
-import { getErrorMessage } from "../common/helpers-pure";
+import { asError, getErrorMessage } from "../common/helpers-pure";
 import { QlPackGenerator } from "./qlpack-generator";
 import { DatabaseItem, DatabaseManager } from "../databases/local-databases";
 import {
   ProgressCallback,
   UserCancellationException,
+  withProgress,
 } from "../common/vscode/progress";
 import {
   askForGitHubRepo,
@@ -24,8 +28,16 @@ import {
   isCodespacesTemplate,
   setQlPackLocation,
 } from "../config";
-import { existsSync } from "fs-extra";
+import { lstat, pathExists, readFile } from "fs-extra";
 import { askForLanguage } from "../codeql-cli/query-language";
+import { showInformationMessageWithAction } from "../common/vscode/dialog";
+import { redactableError } from "../common/errors";
+import { App } from "../common/app";
+import { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
+import { containsPath, pathsEqual } from "../common/files";
+import { getQlPackPath } from "../common/ql";
+import { load } from "js-yaml";
+import { QlPackFile } from "../packaging/qlpack-file";
 
 type QueryLanguagesToDatabaseMap = Record<string, string>;
 
@@ -41,73 +53,139 @@ export const QUERY_LANGUAGE_TO_DATABASE_REPO: QueryLanguagesToDatabaseMap = {
 };
 
 export class SkeletonQueryWizard {
-  private language: QueryLanguage | undefined;
   private fileName = "example.ql";
   private qlPackStoragePath: string | undefined;
+  private queryStoragePath: string | undefined;
+  private downloadPromise: Promise<void> | undefined;
 
   constructor(
     private readonly cliServer: CodeQLCliServer,
     private readonly progress: ProgressCallback,
     private readonly credentials: Credentials | undefined,
-    private readonly logger: BaseLogger,
+    private readonly app: App,
     private readonly databaseManager: DatabaseManager,
     private readonly databaseStoragePath: string | undefined,
+    private readonly selectedItems: readonly QueryTreeViewItem[],
+    private language: QueryLanguage | undefined = undefined,
   ) {}
 
-  private get folderName() {
-    return `codeql-custom-queries-${this.language}`;
+  /**
+   * Wait for the download process to complete by waiting for the user to select
+   * either "Download database" or closing the dialog. This is used for testing.
+   */
+  public async waitForDownload() {
+    if (this.downloadPromise) {
+      await this.downloadPromise;
+    }
   }
 
   public async execute() {
-    // show quick pick to choose language
-    this.language = await this.chooseLanguage();
+    // First try detecting the language based on the existing qlpacks.
+    // This will override the selected language if there is an existing query pack.
+    const detectedLanguage = await this.detectLanguage();
+    if (detectedLanguage) {
+      this.language = detectedLanguage;
+    }
+
+    // If no existing qlpack was found, we need to ask the user for the language
+    if (!this.language) {
+      // show quick pick to choose language
+      this.language = await this.chooseLanguage();
+    }
+
     if (!this.language) {
       return;
     }
 
-    this.qlPackStoragePath = await this.determineStoragePath();
+    let createSkeletonQueryPack: boolean = false;
 
-    const skeletonPackAlreadyExists =
-      existsSync(join(this.qlPackStoragePath, this.folderName)) ||
-      isFolderAlreadyInWorkspace(this.folderName);
+    if (!this.qlPackStoragePath) {
+      // This means no existing qlpack was detected in the selected folder, so we need
+      // to find a new location to store the qlpack. This new location could potentially
+      // already exist.
+      const storagePath = await this.determineStoragePath();
+      this.qlPackStoragePath = join(
+        storagePath,
+        `codeql-custom-queries-${this.language}`,
+      );
 
-    if (skeletonPackAlreadyExists) {
-      // just create a new example query file in skeleton QL pack
-      await this.createExampleFile();
+      // Try to detect if there is already a qlpack in this location. We will assume that
+      // the user hasn't changed the language of the qlpack.
+      const qlPackPath = await getQlPackPath(this.qlPackStoragePath);
+
+      // If we are creating or using a qlpack in the user's selected folder, we will also
+      // create the query in that folder
+      this.queryStoragePath = this.qlPackStoragePath;
+
+      createSkeletonQueryPack = qlPackPath === undefined;
     } else {
+      // A query pack was detected in the selected folder or one of its ancestors, so we
+      // directly use the selected folder as the storage path for the query.
+      this.queryStoragePath = await this.determineStoragePathFromSelection();
+    }
+
+    if (createSkeletonQueryPack) {
       // generate a new skeleton QL pack with query file
       await this.createQlPack();
+    } else {
+      // just create a new example query file in skeleton QL pack
+      await this.createExampleFile();
+    }
+
+    // open the query file
+    try {
+      await this.openExampleFile();
+    } catch (e: unknown) {
+      void this.app.logger.log(
+        `Could not open example query file: ${getErrorMessage(e)}`,
+      );
     }
 
     // select existing database for language or download a new one
     await this.selectOrDownloadDatabase();
-
-    // open a query file
-
-    try {
-      await this.openExampleFile();
-    } catch (e: unknown) {
-      void this.logger.log(
-        `Could not open example query file: ${getErrorMessage(e)}`,
-      );
-    }
   }
 
   private async openExampleFile() {
-    if (this.folderName === undefined || this.qlPackStoragePath === undefined) {
+    if (this.queryStoragePath === undefined) {
       throw new Error("Path to folder is undefined");
     }
 
-    const queryFileUri = Uri.file(
-      join(this.qlPackStoragePath, this.folderName, this.fileName),
-    );
+    const queryFileUri = Uri.file(join(this.queryStoragePath, this.fileName));
 
     void workspace.openTextDocument(queryFileUri).then((doc) => {
-      void Window.showTextDocument(doc);
+      void Window.showTextDocument(doc, {
+        preview: false,
+      });
     });
   }
 
-  public async determineStoragePath() {
+  public async determineStoragePath(): Promise<string> {
+    if (this.selectedItems.length === 0) {
+      return this.determineRootStoragePath();
+    }
+
+    return this.determineStoragePathFromSelection();
+  }
+
+  private async determineStoragePathFromSelection(): Promise<string> {
+    // Just like VS Code's "New File" command, if the user has selected multiple files/folders in the queries panel,
+    // we will create the new file in the same folder as the first selected item.
+    // See https://github.com/microsoft/vscode/blob/a8b7239d0311d4915b57c837972baf4b01394491/src/vs/workbench/contrib/files/browser/fileActions.ts#L893-L900
+    const selectedItem = this.selectedItems[0];
+
+    const path = selectedItem.path;
+
+    // We use stat to protect against outdated query tree items
+    const fileStat = await lstat(path);
+
+    if (fileStat.isDirectory()) {
+      return path;
+    }
+
+    return dirname(path);
+  }
+
+  public async determineRootStoragePath() {
     const firstStorageFolder = getFirstWorkspaceFolder();
 
     if (isCodespacesTemplate()) {
@@ -116,7 +194,7 @@ export class SkeletonQueryWizard {
 
     let storageFolder = getQlPackLocation();
 
-    if (storageFolder === undefined || !existsSync(storageFolder)) {
+    if (storageFolder === undefined || !(await pathExists(storageFolder))) {
       storageFolder = await Window.showInputBox({
         title:
           "Please choose a folder in which to create your new query pack. You can change this in the extension settings.",
@@ -129,7 +207,7 @@ export class SkeletonQueryWizard {
       throw new UserCancellationException("No storage folder entered.");
     }
 
-    if (!existsSync(storageFolder)) {
+    if (!(await pathExists(storageFolder))) {
       throw new UserCancellationException(
         "Invalid folder. Must be a folder that already exists.",
       );
@@ -139,6 +217,62 @@ export class SkeletonQueryWizard {
     return storageFolder;
   }
 
+  private async detectLanguage(): Promise<QueryLanguage | undefined> {
+    if (this.selectedItems.length < 1) {
+      return undefined;
+    }
+
+    this.progress({
+      message: "Resolving existing query packs",
+      step: 1,
+      maxStep: 3,
+    });
+
+    const storagePath = await this.determineStoragePathFromSelection();
+
+    const queryPacks = await this.cliServer.resolveQlpacks(
+      getOnDiskWorkspaceFolders(),
+      false,
+      "query",
+    );
+
+    const matchingQueryPacks = Object.values(queryPacks)
+      .map((paths) => paths.find((path) => containsPath(path, storagePath)))
+      .filter((path): path is string => path !== undefined)
+      // Find the longest matching path
+      .sort((a, b) => b.length - a.length);
+
+    if (matchingQueryPacks.length === 0) {
+      return undefined;
+    }
+
+    const matchingQueryPackPath = matchingQueryPacks[0];
+
+    const qlPackPath = await getQlPackPath(matchingQueryPackPath);
+    if (!qlPackPath) {
+      return undefined;
+    }
+
+    const qlPack = load(await readFile(qlPackPath, "utf8")) as
+      | QlPackFile
+      | undefined;
+    const dependencies = qlPack?.dependencies;
+    if (!dependencies || typeof dependencies !== "object") {
+      return;
+    }
+
+    const matchingLanguages = Object.values(QueryLanguage).filter(
+      (language) => `codeql/${language}-all` in dependencies,
+    );
+    if (matchingLanguages.length !== 1) {
+      return undefined;
+    }
+
+    this.qlPackStoragePath = matchingQueryPackPath;
+
+    return matchingLanguages[0];
+  }
+
   private async chooseLanguage() {
     this.progress({
       message: "Choose language",
@@ -150,13 +284,6 @@ export class SkeletonQueryWizard {
   }
 
   private async createQlPack() {
-    if (this.folderName === undefined) {
-      throw new Error("Folder name is undefined");
-    }
-    if (this.language === undefined) {
-      throw new Error("Language is undefined");
-    }
-
     this.progress({
       message: "Creating skeleton QL pack around query",
       step: 2,
@@ -164,29 +291,17 @@ export class SkeletonQueryWizard {
     });
 
     try {
-      const qlPackGenerator = new QlPackGenerator(
-        this.folderName,
-        this.language,
-        this.cliServer,
-        this.qlPackStoragePath,
-      );
+      const qlPackGenerator = this.createQlPackGenerator();
 
       await qlPackGenerator.generate();
     } catch (e: unknown) {
-      void this.logger.log(
+      void this.app.logger.log(
         `Could not create skeleton QL pack: ${getErrorMessage(e)}`,
       );
     }
   }
 
   private async createExampleFile() {
-    if (this.folderName === undefined) {
-      throw new Error("Folder name is undefined");
-    }
-    if (this.language === undefined) {
-      throw new Error("Language is undefined");
-    }
-
     this.progress({
       message:
         "Skeleton query pack already exists. Creating additional query example file.",
@@ -195,29 +310,29 @@ export class SkeletonQueryWizard {
     });
 
     try {
-      const qlPackGenerator = new QlPackGenerator(
-        this.folderName,
-        this.language,
-        this.cliServer,
-        this.qlPackStoragePath,
-      );
+      const qlPackGenerator = this.createQlPackGenerator();
 
-      this.fileName = await this.determineNextFileName(this.folderName);
+      this.fileName = await this.determineNextFileName();
       await qlPackGenerator.createExampleQlFile(this.fileName);
     } catch (e: unknown) {
-      void this.logger.log(
-        `Could not create skeleton QL pack: ${getErrorMessage(e)}`,
+      void this.app.logger.log(
+        `Could not create query example file: ${getErrorMessage(e)}`,
       );
     }
   }
 
-  private async determineNextFileName(folderName: string): Promise<string> {
-    if (this.qlPackStoragePath === undefined) {
-      throw new Error("QL Pack storage path is undefined");
+  private async determineNextFileName(): Promise<string> {
+    if (this.queryStoragePath === undefined) {
+      throw new Error("Query storage path is undefined");
     }
 
-    const folderUri = Uri.file(join(this.qlPackStoragePath, folderName));
+    const folderUri = Uri.file(this.queryStoragePath);
     const files = await workspace.fs.readDirectory(folderUri);
+    // If the example.ql file doesn't exist yet, use that name
+    if (!files.some(([filename, _fileType]) => filename === this.fileName)) {
+      return this.fileName;
+    }
+
     const qlFiles = files.filter(([filename, _fileType]) =>
       filename.match(/^example[0-9]*\.ql$/),
     );
@@ -225,11 +340,43 @@ export class SkeletonQueryWizard {
     return `example${qlFiles.length + 1}.ql`;
   }
 
-  private async downloadDatabase() {
-    if (this.qlPackStoragePath === undefined) {
-      throw new Error("QL Pack storage path is undefined");
+  private async promptDownloadDatabase() {
+    if (this.language === undefined) {
+      throw new Error("Language is undefined");
     }
 
+    const openFileLink = this.openFileMarkdownLink;
+
+    const displayLanguage = getLanguageDisplayName(this.language);
+    const action = await showInformationMessageWithAction(
+      `New CodeQL query for ${displayLanguage} ${openFileLink} created, but no CodeQL databases for ${displayLanguage} were detected in your workspace. Would you like to download a CodeQL database for ${displayLanguage} to analyze with ${openFileLink}?`,
+      "Download database",
+    );
+
+    if (action) {
+      void withProgress(async (progress) => {
+        try {
+          await this.downloadDatabase(progress);
+        } catch (e: unknown) {
+          if (e instanceof UserCancellationException) {
+            return;
+          }
+
+          void showAndLogExceptionWithTelemetry(
+            this.app.logger,
+            this.app.telemetry,
+            redactableError(
+              asError(e),
+            )`An error occurred while downloading the GitHub repository: ${getErrorMessage(
+              e,
+            )}`,
+          );
+        }
+      });
+    }
+  }
+
+  private async downloadDatabase(progress: ProgressCallback) {
     if (this.databaseStoragePath === undefined) {
       throw new Error("Database storage path is undefined");
     }
@@ -238,10 +385,10 @@ export class SkeletonQueryWizard {
       throw new Error("Language is undefined");
     }
 
-    this.progress({
+    progress({
       message: "Downloading database",
-      step: 3,
-      maxStep: 3,
+      step: 1,
+      maxStep: 2,
     });
 
     const githubRepoNwo = QUERY_LANGUAGE_TO_DATABASE_REPO[this.language];
@@ -256,7 +403,7 @@ export class SkeletonQueryWizard {
       this.databaseManager,
       this.databaseStoragePath,
       this.credentials,
-      this.progress,
+      progress,
       this.cliServer,
       this.language,
     );
@@ -267,10 +414,6 @@ export class SkeletonQueryWizard {
       throw new Error("Language is undefined");
     }
 
-    if (this.qlPackStoragePath === undefined) {
-      throw new Error("QL Pack storage path is undefined");
-    }
-
     const existingDatabaseItem =
       await SkeletonQueryWizard.findExistingDatabaseItem(
         this.language,
@@ -278,14 +421,65 @@ export class SkeletonQueryWizard {
       );
 
     if (existingDatabaseItem) {
-      // select the found database
-      await this.databaseManager.setCurrentDatabaseItem(existingDatabaseItem);
+      const openFileLink = this.openFileMarkdownLink;
+
+      if (this.databaseManager.currentDatabaseItem !== existingDatabaseItem) {
+        // select the found database
+        await this.databaseManager.setCurrentDatabaseItem(existingDatabaseItem);
+
+        const displayLanguage = getLanguageDisplayName(this.language);
+        void window.showInformationMessage(
+          `New CodeQL query for ${displayLanguage} ${openFileLink} created. We have automatically selected your existing CodeQL ${displayLanguage} database ${existingDatabaseItem.name} for you to analyze with ${openFileLink}.`,
+        );
+      }
     } else {
       // download new database and select it
-      await this.downloadDatabase();
+      this.downloadPromise = this.promptDownloadDatabase().finally(() => {
+        this.downloadPromise = undefined;
+      });
     }
   }
 
+  private get openFileMarkdownLink() {
+    if (this.queryStoragePath === undefined) {
+      throw new Error("QL Pack storage path is undefined");
+    }
+
+    const queryPath = join(this.queryStoragePath, this.fileName);
+    const queryPathUri = Uri.file(queryPath);
+
+    const openFileArgs = [queryPathUri.toString(true)];
+    const queryString = encodeURI(JSON.stringify(openFileArgs));
+    return `[${this.fileName}](command:vscode.open?${queryString})`;
+  }
+
+  private createQlPackGenerator() {
+    if (this.qlPackStoragePath === undefined) {
+      throw new Error("QL pack storage path is undefined");
+    }
+    if (this.queryStoragePath === undefined) {
+      throw new Error("Query storage path is undefined");
+    }
+    if (this.language === undefined) {
+      throw new Error("Language is undefined");
+    }
+
+    const parentFolder = dirname(this.qlPackStoragePath);
+
+    // Only include the folder name in the qlpack name if the qlpack is not in the root of the workspace.
+    const includeFolderNameInQlpackName = !getOnDiskWorkspaceFolders().some(
+      (workspaceFolder) => pathsEqual(workspaceFolder, parentFolder),
+    );
+
+    return new QlPackGenerator(
+      this.language,
+      this.cliServer,
+      this.qlPackStoragePath,
+      this.queryStoragePath,
+      includeFolderNameInQlpackName,
+    );
+  }
+
   public static async findDatabaseItemByNwo(
     language: string,
     databaseNwo: string,

extensions/ql-vscode/src/log-insights/join-order.ts

@@ -37,38 +37,49 @@ function makeKey(
   return `${queryCausingWork}:${predicate}${suffix ? ` ${suffix}` : ""}`;
 }
 
-const DEPENDENT_PREDICATES_REGEXP = (() => {
+function getDependentPredicates(operations: string[]): I.List<string> {
+  const id = String.raw`[0-9a-zA-Z:#_\./]+`;
+  const idWithAngleBrackets = String.raw`[0-9a-zA-Z:#_<>\./]+`;
+  const quotedId = String.raw`\`[^\`\r\n]*\``;
   const regexps = [
     // SCAN id
-    String.raw`SCAN\s+([0-9a-zA-Z:#_]+)\s`,
+    String.raw`SCAN\s+(${id}|${quotedId})\s`,
     // JOIN id WITH id
-    String.raw`JOIN\s+([0-9a-zA-Z:#_]+)\s+WITH\s+([0-9a-zA-Z:#_]+)\s`,
+    String.raw`JOIN\s+(${id}|${quotedId})\s+WITH\s+(${id}|${quotedId})\s`,
+    // JOIN WITH id
+    String.raw`JOIN\s+WITH\s+(${id}|${quotedId})\s`,
     // AGGREGATE id, id
-    String.raw`AGGREGATE\s+([0-9a-zA-Z:#_]+)\s*,\s+([0-9a-zA-Z:#_]+)`,
+    String.raw`AGGREGATE\s+(${id}|${quotedId})\s*,\s+(${id}|${quotedId})`,
     // id AND NOT id
-    String.raw`([0-9a-zA-Z:#_]+)\s+AND\s+NOT\s+([0-9a-zA-Z:#_]+)`,
+    String.raw`(${id}|${quotedId})\s+AND\s+NOT\s+(${id}|${quotedId})`,
+    // AND NOT id
+    String.raw`AND\s+NOT\s+(${id}|${quotedId})`,
     // INVOKE HIGHER-ORDER RELATION rel ON <id, ..., id>
-    String.raw`INVOKE\s+HIGHER-ORDER\s+RELATION\s[^\s]+\sON\s+<([0-9a-zA-Z:#_<>]+)((?:,[0-9a-zA-Z:#_<>]+)*)>`,
+    String.raw`INVOKE\s+HIGHER-ORDER\s+RELATION\s[^\s]+\sON\s+<(${idWithAngleBrackets}|${quotedId})((?:,${idWithAngleBrackets}|,${quotedId})*)>`,
     // SELECT id
-    String.raw`SELECT\s+([0-9a-zA-Z:#_]+)`,
+    String.raw`SELECT\s+(${id}|${quotedId})`,
+    // REWRITE id WITH
+    String.raw`REWRITE\s+(${id}|${quotedId})\s+WITH\s`,
+    // id UNION id UNION ... UNION id
+    String.raw`(${id}|${quotedId})((?:\s+UNION\s+${id}|${quotedId})+)`,
   ];
-  return new RegExp(
-    `${String.raw`\{[0-9]+\}\s+[0-9a-zA-Z]+\s=\s(?:` + regexps.join("|")})`,
+  const r = new RegExp(
+    `${
+      String.raw`\{[0-9]+\}\s+(?:[0-9a-zA-Z]+\s=|\|)\s(?:` + regexps.join("|")
+    })`,
   );
-})();
-
-function getDependentPredicates(operations: string[]): I.List<string> {
   return I.List(operations).flatMap((operation) => {
-    const matches = DEPENDENT_PREDICATES_REGEXP.exec(operation.trim());
-    if (matches !== null) {
-      return I.List(matches)
-        .rest() // Skip the first group as it's just the entire string
-        .filter((x) => !!x && !x.match("r[0-9]+|PRIMITIVE")) // Only keep the references to predicates.
-        .flatMap((x) => x.split(",")) // Group 2 in the INVOKE HIGHER_ORDER RELATION case is a comma-separated list of identifiers.
-        .filter((x) => !!x); // Remove empty strings
-    } else {
-      return I.List();
-    }
+    const matches = r.exec(operation.trim()) || [];
+    return I.List(matches)
+      .rest() // Skip the first group as it's just the entire string
+      .filter((x) => !!x)
+      .flatMap((x) => x.split(",")) // Group 2 in the INVOKE HIGHER_ORDER RELATION case is a comma-separated list of identifiers.
+      .flatMap((x) => x.split(" UNION ")) // Split n-ary unions into individual arguments.
+      .filter((x) => !x.match("r[0-9]+|PRIMITIVE")) // Only keep the references to predicates.
+      .filter((x) => !!x) // Remove empty strings
+      .map((x) =>
+        x.startsWith("`") && x.endsWith("`") ? x.substring(1, x.length - 1) : x,
+      ); // Remove quotes from quoted identifiers
   });
 }
 

extensions/ql-vscode/src/model-editor/auto-model-api.ts

@@ -1,5 +1,7 @@
 import { Credentials } from "../common/authentication";
 import { OctokitResponse } from "@octokit/types";
+import fetch from "node-fetch";
+import { ModelConfigListener } from "../config";
 
 export enum AutomodelMode {
   Unspecified = "AUTOMODEL_MODE_UNSPECIFIED",
@@ -20,15 +22,44 @@ export interface ModelResponse {
 export async function autoModel(
   credentials: Credentials,
   request: ModelRequest,
+  modelingConfig: ModelConfigListener,
 ): Promise<ModelResponse> {
-  const octokit = await credentials.getOctokit();
+  const devEndpoint = modelingConfig.llmGenerationDevEndpoint;
+  if (devEndpoint) {
+    return callAutoModelDevEndpoint(devEndpoint, request);
+  } else {
+    const octokit = await credentials.getOctokit();
 
-  const response: OctokitResponse<ModelResponse> = await octokit.request(
-    "POST /repos/github/codeql/code-scanning/codeql/auto-model",
-    {
-      data: request,
-    },
-  );
+    const response: OctokitResponse<ModelResponse> = await octokit.request(
+      "POST /repos/github/codeql/code-scanning/codeql/auto-model",
+      {
+        data: request,
+      },
+    );
 
-  return response.data;
+    return response.data;
+  }
+}
+
+async function callAutoModelDevEndpoint(
+  endpoint: string,
+  request: ModelRequest,
+): Promise<ModelResponse> {
+  const json = JSON.stringify(request);
+  const response = await fetch(endpoint, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: json,
+  });
+
+  if (!response.ok) {
+    throw new Error(
+      `Error calling auto-model API: ${response.status} ${response.statusText}`,
+    );
+  }
+
+  const data = await response.json();
+  return data as ModelResponse;
 }

extensions/ql-vscode/src/model-editor/auto-model.ts

@@ -19,8 +19,8 @@ import { groupMethods, sortGroupNames, sortMethods } from "./shared/sorting";
  */
 export function getCandidates(
   mode: Mode,
-  methods: Method[],
-  modeledMethodsBySignature: Record<string, ModeledMethod[]>,
+  methods: readonly Method[],
+  modeledMethodsBySignature: Record<string, readonly ModeledMethod[]>,
 ): MethodSignature[] {
   // Sort the same way as the UI so we send the first ones listed in the UI first
   const grouped = groupMethods(methods, mode);
@@ -32,8 +32,9 @@ export function getCandidates(
   const candidates: MethodSignature[] = [];
 
   for (const method of sortedMethods) {
-    const modeledMethods: ModeledMethod[] =
-      modeledMethodsBySignature[method.signature] ?? [];
+    const modeledMethods: ModeledMethod[] = [
+      ...(modeledMethodsBySignature[method.signature] ?? []),
+    ];
 
     // Anything that is modeled is not a candidate
     if (modeledMethods.some((m) => m.type !== "none")) {

extensions/ql-vscode/src/model-editor/auto-modeler.ts

@@ -1,6 +1,5 @@
 import { Method, MethodSignature } from "./method";
 import { ModeledMethod } from "./modeled-method";
-import { extLogger } from "../common/logging/vscode";
 import { load as loadYaml } from "js-yaml";
 import { ProgressCallback, withProgress } from "../common/vscode/progress";
 import { createAutoModelRequest, getCandidates } from "./auto-model";
@@ -16,11 +15,9 @@ import { QueryRunner } from "../query-server";
 import { DatabaseItem } from "../databases/local-databases";
 import { Mode } from "./shared/mode";
 import { CancellationTokenSource } from "vscode";
-
-// Limit the number of candidates we send to the model in each request
-// to avoid long requests.
-// Note that the model may return fewer than this number of candidates.
-const candidateBatchSize = 20;
+import { ModelingStore } from "./modeling-store";
+import { ModelConfigListener } from "../config";
+import { QueryLanguage } from "../common/query-language";
 
 /**
  * The auto-modeler holds state around auto-modeling jobs and allows
@@ -35,12 +32,11 @@ export class AutoModeler {
     private readonly app: App,
     private readonly cliServer: CodeQLCliServer,
     private readonly queryRunner: QueryRunner,
+    private readonly modelConfig: ModelConfigListener,
+    private readonly modelingStore: ModelingStore,
     private readonly queryStorageDir: string,
     private readonly databaseItem: DatabaseItem,
-    private readonly setInProgressMethods: (
-      packageName: string,
-      inProgressMethods: string[],
-    ) => Promise<void>,
+    private readonly language: QueryLanguage,
     private readonly addModeledMethods: (
       modeledMethods: Record<string, ModeledMethod[]>,
     ) => Promise<void>,
@@ -58,8 +54,8 @@ export class AutoModeler {
    */
   public async startModeling(
     packageName: string,
-    methods: Method[],
-    modeledMethods: Record<string, ModeledMethod[]>,
+    methods: readonly Method[],
+    modeledMethods: Record<string, readonly ModeledMethod[]>,
     mode: Mode,
   ): Promise<void> {
     if (this.jobs.has(packageName)) {
@@ -87,7 +83,7 @@ export class AutoModeler {
    * @param packageName The name of the package to stop modeling.
    */
   public async stopModeling(packageName: string): Promise<void> {
-    void extLogger.log(`Stopping modeling for package ${packageName}`);
+    void this.app.logger.log(`Stopping modeling for package ${packageName}`);
     const cancellationTokenSource = this.jobs.get(packageName);
     if (cancellationTokenSource) {
       cancellationTokenSource.cancel();
@@ -105,19 +101,22 @@ export class AutoModeler {
 
   private async modelPackage(
     packageName: string,
-    methods: Method[],
-    modeledMethods: Record<string, ModeledMethod[]>,
+    methods: readonly Method[],
+    modeledMethods: Record<string, readonly ModeledMethod[]>,
     mode: Mode,
     cancellationTokenSource: CancellationTokenSource,
   ): Promise<void> {
-    void extLogger.log(`Modeling package ${packageName}`);
+    void this.app.logger.log(`Modeling package ${packageName}`);
+
+    const candidateBatchSize = this.modelConfig.llmGenerationBatchSize;
+
     await withProgress(async (progress) => {
       // Fetch the candidates to send to the model
       const allCandidateMethods = getCandidates(mode, methods, modeledMethods);
 
       // If there are no candidates, there is nothing to model and we just return
       if (allCandidateMethods.length === 0) {
-        void extLogger.log("No candidates to model. Stopping.");
+        void this.app.logger.log("No candidates to model. Stopping.");
         return;
       }
 
@@ -135,11 +134,14 @@ export class AutoModeler {
           const start = i * candidateBatchSize;
           const end = start + candidateBatchSize;
           const candidatesToProcess = allCandidateMethods.slice(start, end);
+          const candidateSignatures = candidatesToProcess.map(
+            (c) => c.signature,
+          );
 
           // Let the UI know which candidates we are modeling
-          await this.setInProgressMethods(
-            packageName,
-            candidatesToProcess.map((c) => c.signature),
+          this.modelingStore.addInProgressMethods(
+            this.databaseItem,
+            candidateSignatures,
           );
 
           // Kick off the process to model the slice of candidates
@@ -149,10 +151,19 @@ export class AutoModeler {
             progress,
             cancellationTokenSource,
           );
+
+          // Let the UI know which candidates we are done modeling
+          this.modelingStore.removeInProgressMethods(
+            this.databaseItem,
+            candidateSignatures,
+          );
         }
       } finally {
-        // Clear out in progress methods
-        await this.setInProgressMethods(packageName, []);
+        // Clear out in progress methods in case anything went wrong
+        this.modelingStore.removeInProgressMethods(
+          this.databaseItem,
+          allCandidateMethods.map((c) => c.signature),
+        );
       }
     });
   }
@@ -163,7 +174,7 @@ export class AutoModeler {
     progress: ProgressCallback,
     cancellationTokenSource: CancellationTokenSource,
   ): Promise<void> {
-    void extLogger.log("Executing auto-model queries");
+    void this.app.logger.log("Executing auto-model queries");
 
     const usages = await runAutoModelQueries({
       mode,
@@ -181,7 +192,7 @@ export class AutoModeler {
 
     const request = await createAutoModelRequest(mode, usages);
 
-    void extLogger.log("Calling auto-model API");
+    void this.app.logger.log("Calling auto-model API");
 
     const response = await this.callAutoModelApi(request);
     if (!response) {
@@ -192,34 +203,11 @@ export class AutoModeler {
       filename: "auto-model.yml",
     });
 
-    const loadedMethods = loadDataExtensionYaml(models);
+    const loadedMethods = loadDataExtensionYaml(models, this.language);
     if (!loadedMethods) {
       return;
     }
 
-    // Any candidate that was part of the response is a negative result
-    // meaning that the canidate is not a sink for the kinds that the LLM is checking for.
-    // For now we model this as a sink neutral method, however this is subject
-    // to discussion.
-    for (const candidate of candidateMethods) {
-      if (!(candidate.signature in loadedMethods)) {
-        loadedMethods[candidate.signature] = [
-          {
-            type: "neutral",
-            kind: "sink",
-            input: "",
-            output: "",
-            provenance: "ai-generated",
-            signature: candidate.signature,
-            packageName: candidate.packageName,
-            typeName: candidate.typeName,
-            methodName: candidate.methodName,
-            methodParameters: candidate.methodParameters,
-          },
-        ];
-      }
-    }
-
     await this.addModeledMethods(loadedMethods);
   }
 
@@ -227,7 +215,7 @@ export class AutoModeler {
     request: ModelRequest,
   ): Promise<ModelResponse | null> {
     try {
-      return await autoModel(this.app.credentials, request);
+      return await autoModel(this.app.credentials, request, this.modelConfig);
     } catch (e) {
       if (e instanceof RequestError && e.status === 429) {
         void showAndLogExceptionWithTelemetry(

extensions/ql-vscode/src/model-editor/bqrs.ts

@@ -4,13 +4,18 @@ import { ModeledMethodType } from "./modeled-method";
 import { parseLibraryFilename } from "./library";
 import { Mode } from "./shared/mode";
 import { ApplicationModeTuple, FrameworkModeTuple } from "./queries/query";
+import { QueryLanguage } from "../common/query-language";
+import { getModelsAsDataLanguage } from "./languages";
 
 export function decodeBqrsToMethods(
   chunk: DecodedBqrsChunk,
   mode: Mode,
+  language: QueryLanguage,
 ): Method[] {
   const methodsByApiName = new Map<string, Method>();
 
+  const definition = getModelsAsDataLanguage(language);
+
   chunk?.tuples.forEach((tuple) => {
     let usage: Call;
     let packageName: string;
@@ -51,7 +56,12 @@ export function decodeBqrsToMethods(
       classification = CallClassification.Unknown;
     }
 
-    const signature = `${packageName}.${typeName}#${methodName}${methodParameters}`;
+    const signature = definition.createMethodSignature({
+      packageName,
+      typeName,
+      methodName,
+      methodParameters,
+    });
 
     // For Java, we'll always get back a .jar file, and the library version may be bad because not all library authors
     // properly specify the version. Therefore, we'll always try to parse the name and version from the library filename
@@ -88,9 +98,16 @@ export function decodeBqrsToMethods(
     }
 
     const method = methodsByApiName.get(signature)!;
-    method.usages.push({
-      ...usage,
-      classification,
+    const usages = [
+      ...method.usages,
+      {
+        ...usage,
+        classification,
+      },
+    ];
+    methodsByApiName.set(signature, {
+      ...method,
+      usages,
     });
   });
 

extensions/ql-vscode/src/model-editor/extension-pack-picker.ts

@@ -64,7 +64,7 @@ export async function pickExtensionPack(
   // If the setting is not set, automatically pick a suitable directory
   const extensionsDirectory = userExtensionsDirectory
     ? Uri.file(userExtensionsDirectory)
-    : await autoPickExtensionsDirectory();
+    : await autoPickExtensionsDirectory(logger);
 
   if (!extensionsDirectory) {
     return undefined;

extensions/ql-vscode/src/model-editor/extensions-workspace-folder.ts

@@ -1,7 +1,7 @@
 import { FileType, Uri, workspace, WorkspaceFolder } from "vscode";
 import { getOnDiskWorkspaceFoldersObjects } from "../common/vscode/workspace-folders";
-import { extLogger } from "../common/logging/vscode";
 import { tmpdir } from "../common/files";
+import { NotificationLogger, showAndLogErrorMessage } from "../common/logging";
 
 /**
  * Returns the ancestors of this path in order from furthest to closest (i.e. root of filesystem to parent directory)
@@ -143,9 +143,20 @@ async function findGitFolder(
  *    for which the .git directory is closest to a workspace folder
  * 6. If none of the above apply, return `undefined`
  */
-export async function autoPickExtensionsDirectory(): Promise<Uri | undefined> {
+export async function autoPickExtensionsDirectory(
+  logger: NotificationLogger,
+): Promise<Uri | undefined> {
   const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
 
+  // If there are no on-disk workspace folders, we can't do anything
+  if (workspaceFolders.length === 0) {
+    void showAndLogErrorMessage(
+      logger,
+      `Could not find any on-disk workspace folders. Please ensure that you have opened a folder or workspace.`,
+    );
+    return undefined;
+  }
+
   // If there's only 1 workspace folder, use the `.github/codeql/extensions` directory in that folder
   if (workspaceFolders.length === 1) {
     return Uri.joinPath(
@@ -168,7 +179,7 @@ export async function autoPickExtensionsDirectory(): Promise<Uri | undefined> {
   // Get the root workspace directory, i.e. the common root directory of all workspace folders
   const rootDirectory = await getRootWorkspaceDirectory();
   if (!rootDirectory) {
-    void extLogger.log("Unable to determine root workspace directory");
+    void logger.log("Unable to determine root workspace directory");
 
     return undefined;
   }
@@ -192,7 +203,7 @@ export async function autoPickExtensionsDirectory(): Promise<Uri | undefined> {
       },
     )
   ) {
-    void extLogger.log(
+    void logger.log(
       `Failed to add workspace folder for extensions at ${extensionsUri.fsPath}`,
     );
     return undefined;

extensions/ql-vscode/src/model-editor/external-api-usage-queries.ts

@@ -1,189 +0,0 @@
-import { QueryRunner } from "../query-server";
-import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
-import { extLogger } from "../common/logging/vscode";
-import { showAndLogExceptionWithTelemetry } from "../common/logging";
-import { CancellationToken } from "vscode";
-import { CodeQLCliServer } from "../codeql-cli/cli";
-import { DatabaseItem } from "../databases/local-databases";
-import { ProgressCallback } from "../common/vscode/progress";
-import { redactableError } from "../common/errors";
-import { telemetryListener } from "../common/vscode/telemetry";
-import { join } from "path";
-import { Mode } from "./shared/mode";
-import { writeFile } from "fs-extra";
-import { QueryLanguage } from "../common/query-language";
-import { fetchExternalApiQueries } from "./queries";
-import { Method } from "./method";
-import { runQuery } from "../local-queries/run-query";
-import { decodeBqrsToMethods } from "./bqrs";
-import {
-  resolveEndpointsQuery,
-  syntheticQueryPackName,
-} from "./model-editor-queries";
-
-type RunQueryOptions = {
-  cliServer: CodeQLCliServer;
-  queryRunner: QueryRunner;
-  databaseItem: DatabaseItem;
-  queryStorageDir: string;
-  queryDir: string;
-
-  progress: ProgressCallback;
-  token: CancellationToken;
-};
-
-export async function prepareExternalApiQuery(
-  queryDir: string,
-  language: QueryLanguage,
-): Promise<boolean> {
-  // Resolve the query that we want to run.
-  const query = fetchExternalApiQueries[language];
-  if (!query) {
-    void showAndLogExceptionWithTelemetry(
-      extLogger,
-      telemetryListener,
-      redactableError`No external API usage query found for language ${language}`,
-    );
-    return false;
-  }
-  // Create the query file.
-  Object.values(Mode).map(async (mode) => {
-    const queryFile = join(queryDir, queryNameFromMode(mode));
-    await writeFile(queryFile, query[`${mode}ModeQuery`], "utf8");
-  });
-
-  // Create any dependencies
-  if (query.dependencies) {
-    for (const [filename, contents] of Object.entries(query.dependencies)) {
-      const dependencyFile = join(queryDir, filename);
-      await writeFile(dependencyFile, contents, "utf8");
-    }
-  }
-  return true;
-}
-
-export const externalApiQueriesProgressMaxStep = 2000;
-
-export async function runExternalApiQueries(
-  mode: Mode,
-  {
-    cliServer,
-    queryRunner,
-    databaseItem,
-    queryStorageDir,
-    queryDir,
-    progress,
-    token,
-  }: RunQueryOptions,
-): Promise<Method[] | undefined> {
-  // The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
-  // move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
-  // This is intentionally not pretty code, as it will be removed soon.
-  // For a reference of what this should do in the future, see the previous implementation in
-  // https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72
-
-  progress({
-    message: "Resolving QL packs",
-    step: 1,
-    maxStep: externalApiQueriesProgressMaxStep,
-  });
-  const additionalPacks = getOnDiskWorkspaceFolders();
-  const extensionPacks = Object.keys(
-    await cliServer.resolveQlpacks(additionalPacks, true),
-  );
-
-  progress({
-    message: "Resolving query",
-    step: 2,
-    maxStep: externalApiQueriesProgressMaxStep,
-  });
-
-  // Resolve the queries from either codeql/java-queries or from the temporary queryDir
-  const queryPath = await resolveEndpointsQuery(
-    cliServer,
-    databaseItem.language,
-    mode,
-    [syntheticQueryPackName],
-    [queryDir],
-  );
-  if (!queryPath) {
-    void showAndLogExceptionWithTelemetry(
-      extLogger,
-      telemetryListener,
-      redactableError`The ${mode} model editor query could not be found. Try re-opening the model editor. If that doesn't work, try upgrading the CodeQL libraries.`,
-    );
-    return;
-  }
-
-  // Run the actual query
-  const completedQuery = await runQuery({
-    queryRunner,
-    databaseItem,
-    queryPath,
-    queryStorageDir,
-    additionalPacks,
-    extensionPacks,
-    progress: (update) =>
-      progress({
-        step: update.step + 500,
-        maxStep: externalApiQueriesProgressMaxStep,
-        message: update.message,
-      }),
-    token,
-  });
-
-  if (!completedQuery) {
-    return;
-  }
-
-  // Read the results and covert to internal representation
-  progress({
-    message: "Decoding results",
-    step: 1600,
-    maxStep: externalApiQueriesProgressMaxStep,
-  });
-
-  const bqrsChunk = await readQueryResults({
-    cliServer,
-    bqrsPath: completedQuery.outputDir.bqrsPath,
-  });
-  if (!bqrsChunk) {
-    return;
-  }
-
-  progress({
-    message: "Finalizing results",
-    step: 1950,
-    maxStep: externalApiQueriesProgressMaxStep,
-  });
-
-  return decodeBqrsToMethods(bqrsChunk, mode);
-}
-
-type GetResultsOptions = {
-  cliServer: Pick<CodeQLCliServer, "bqrsInfo" | "bqrsDecode">;
-  bqrsPath: string;
-};
-
-export async function readQueryResults({
-  cliServer,
-  bqrsPath,
-}: GetResultsOptions) {
-  const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
-  if (bqrsInfo["result-sets"].length !== 1) {
-    void showAndLogExceptionWithTelemetry(
-      extLogger,
-      telemetryListener,
-      redactableError`Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`,
-    );
-    return undefined;
-  }
-
-  const resultSet = bqrsInfo["result-sets"][0];
-
-  return cliServer.bqrsDecode(bqrsPath, resultSet.name);
-}
-
-function queryNameFromMode(mode: Mode): string {
-  return `${mode.charAt(0).toUpperCase() + mode.slice(1)}ModeEndpoints.ql`;
-}

extensions/ql-vscode/src/model-editor/flow-model-queries.ts

@@ -1,174 +0,0 @@
-import { CancellationToken } from "vscode";
-import { DatabaseItem } from "../databases/local-databases";
-import { basename } from "path";
-import { QueryRunner } from "../query-server";
-import { CodeQLCliServer } from "../codeql-cli/cli";
-import { showAndLogExceptionWithTelemetry } from "../common/logging";
-import { extLogger } from "../common/logging/vscode";
-import { extensiblePredicateDefinitions } from "./predicates";
-import { ProgressCallback } from "../common/vscode/progress";
-import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
-import { ModeledMethod, ModeledMethodType } from "./modeled-method";
-import { redactableError } from "../common/errors";
-import { telemetryListener } from "../common/vscode/telemetry";
-import { runQuery } from "../local-queries/run-query";
-import { resolveQueries } from "../local-queries";
-
-type FlowModelOptions = {
-  cliServer: CodeQLCliServer;
-  queryRunner: QueryRunner;
-  queryStorageDir: string;
-  databaseItem: DatabaseItem;
-  progress: ProgressCallback;
-  token: CancellationToken;
-  onResults: (results: ModeledMethod[]) => void | Promise<void>;
-};
-
-export async function runFlowModelQueries({
-  onResults,
-  ...options
-}: FlowModelOptions) {
-  const queries = await resolveFlowQueries(
-    options.cliServer,
-    options.databaseItem,
-  );
-
-  const queriesByBasename: Record<string, string> = {};
-  for (const query of queries) {
-    queriesByBasename[basename(query)] = query;
-  }
-
-  const summaryResults = await runSingleFlowQuery(
-    "summary",
-    queriesByBasename["CaptureSummaryModels.ql"],
-    0,
-    options,
-  );
-  if (summaryResults) {
-    await onResults(summaryResults);
-  }
-
-  const sinkResults = await runSingleFlowQuery(
-    "sink",
-    queriesByBasename["CaptureSinkModels.ql"],
-    1,
-    options,
-  );
-  if (sinkResults) {
-    await onResults(sinkResults);
-  }
-
-  const sourceResults = await runSingleFlowQuery(
-    "source",
-    queriesByBasename["CaptureSourceModels.ql"],
-    2,
-    options,
-  );
-  if (sourceResults) {
-    await onResults(sourceResults);
-  }
-
-  const neutralResults = await runSingleFlowQuery(
-    "neutral",
-    queriesByBasename["CaptureNeutralModels.ql"],
-    3,
-    options,
-  );
-  if (neutralResults) {
-    await onResults(neutralResults);
-  }
-}
-
-async function resolveFlowQueries(
-  cliServer: CodeQLCliServer,
-  databaseItem: DatabaseItem,
-): Promise<string[]> {
-  const packsToSearch = [`codeql/${databaseItem.language}-queries`];
-
-  return await resolveQueries(
-    cliServer,
-    packsToSearch,
-    "flow model generator",
-    {
-      "tags contain": ["modelgenerator"],
-    },
-  );
-}
-
-async function runSingleFlowQuery(
-  type: Exclude<ModeledMethodType, "none">,
-  queryPath: string | undefined,
-  queryStep: number,
-  {
-    cliServer,
-    queryRunner,
-    queryStorageDir,
-    databaseItem,
-    progress,
-    token,
-  }: Omit<FlowModelOptions, "onResults">,
-): Promise<ModeledMethod[]> {
-  // Check that the right query was found
-  if (queryPath === undefined) {
-    void showAndLogExceptionWithTelemetry(
-      extLogger,
-      telemetryListener,
-      redactableError`Failed to find ${type} query`,
-    );
-    return [];
-  }
-
-  // Run the query
-  const completedQuery = await runQuery({
-    queryRunner,
-    databaseItem,
-    queryPath,
-    queryStorageDir,
-    additionalPacks: getOnDiskWorkspaceFolders(),
-    extensionPacks: undefined,
-    progress: ({ step, message }) =>
-      progress({
-        message: `Generating ${type} model: ${message}`,
-        step: queryStep * 1000 + step,
-        maxStep: 4000,
-      }),
-    token,
-  });
-
-  if (!completedQuery) {
-    return [];
-  }
-
-  // Interpret the results
-  const definition = extensiblePredicateDefinitions[type];
-
-  const bqrsPath = completedQuery.outputDir.bqrsPath;
-
-  const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
-  if (bqrsInfo["result-sets"].length !== 1) {
-    void showAndLogExceptionWithTelemetry(
-      extLogger,
-      telemetryListener,
-      redactableError`Expected exactly one result set, got ${
-        bqrsInfo["result-sets"].length
-      } for ${basename(queryPath)}`,
-    );
-  }
-
-  const resultSet = bqrsInfo["result-sets"][0];
-
-  const decodedResults = await cliServer.bqrsDecode(bqrsPath, resultSet.name);
-
-  const results = decodedResults.tuples;
-
-  return (
-    results
-      // This is just a sanity check. The query should only return strings.
-      .filter((result) => typeof result[0] === "string")
-      .map((result) => {
-        const row = result[0] as string;
-
-        return definition.readModeledMethod(row.split(";"));
-      })
-  );
-}

extensions/ql-vscode/src/model-editor/generate.ts

@@ -0,0 +1,99 @@
+import { CancellationToken } from "vscode";
+import { DatabaseItem } from "../databases/local-databases";
+import { basename } from "path";
+import { QueryRunner } from "../query-server";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { ProgressCallback } from "../common/vscode/progress";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { ModeledMethod } from "./modeled-method";
+import { runQuery } from "../local-queries/run-query";
+import { QueryConstraints, resolveQueries } from "../local-queries";
+import { DecodedBqrs } from "../common/bqrs-cli-types";
+type GenerateQueriesOptions = {
+  queryConstraints: QueryConstraints;
+  filterQueries?: (queryPath: string) => boolean;
+  parseResults: (
+    queryPath: string,
+    results: DecodedBqrs,
+  ) => ModeledMethod[] | Promise<ModeledMethod[]>;
+  onResults: (results: ModeledMethod[]) => void | Promise<void>;
+
+  cliServer: CodeQLCliServer;
+  queryRunner: QueryRunner;
+  queryStorageDir: string;
+  databaseItem: DatabaseItem;
+  progress: ProgressCallback;
+  token: CancellationToken;
+};
+
+export async function runGenerateQueries(options: GenerateQueriesOptions) {
+  const { queryConstraints, filterQueries, parseResults, onResults } = options;
+
+  options.progress({
+    message: "Resolving queries",
+    step: 1,
+    maxStep: 5000,
+  });
+
+  const packsToSearch = [`codeql/${options.databaseItem.language}-queries`];
+  const queryPaths = await resolveQueries(
+    options.cliServer,
+    packsToSearch,
+    "generate model",
+    queryConstraints,
+  );
+
+  const filteredQueryPaths = filterQueries
+    ? queryPaths.filter(filterQueries)
+    : queryPaths;
+
+  const maxStep = filteredQueryPaths.length * 1000;
+
+  for (let i = 0; i < filteredQueryPaths.length; i++) {
+    const queryPath = filteredQueryPaths[i];
+
+    const bqrs = await runSingleGenerateQuery(queryPath, i, maxStep, options);
+    if (bqrs) {
+      await onResults(await parseResults(queryPath, bqrs));
+    }
+  }
+}
+
+async function runSingleGenerateQuery(
+  queryPath: string,
+  queryStep: number,
+  maxStep: number,
+  {
+    cliServer,
+    queryRunner,
+    queryStorageDir,
+    databaseItem,
+    progress,
+    token,
+  }: GenerateQueriesOptions,
+): Promise<DecodedBqrs | undefined> {
+  const queryBasename = basename(queryPath);
+
+  // Run the query
+  const completedQuery = await runQuery({
+    queryRunner,
+    databaseItem,
+    queryPath,
+    queryStorageDir,
+    additionalPacks: getOnDiskWorkspaceFolders(),
+    extensionPacks: undefined,
+    progress: ({ step, message }) =>
+      progress({
+        message: `Generating model from ${queryBasename}: ${message}`,
+        step: queryStep * 1000 + step,
+        maxStep,
+      }),
+    token,
+  });
+
+  if (!completedQuery) {
+    return undefined;
+  }
+
+  return cliServer.bqrsDecodeAll(completedQuery.outputDir.bqrsPath);
+}

extensions/ql-vscode/src/model-editor/languages/index.ts

@@ -0,0 +1,2 @@
+export * from "./languages";
+export * from "./models-as-data";

extensions/ql-vscode/src/model-editor/languages/languages.ts

@@ -0,0 +1,36 @@
+import { QueryLanguage } from "../../common/query-language";
+import {
+  ModelsAsDataLanguage,
+  ModelsAsDataLanguagePredicates,
+} from "./models-as-data";
+import { ruby } from "./ruby";
+import { staticLanguage } from "./static";
+
+const languages: Partial<Record<QueryLanguage, ModelsAsDataLanguage>> = {
+  [QueryLanguage.CSharp]: staticLanguage,
+  [QueryLanguage.Java]: staticLanguage,
+  [QueryLanguage.Ruby]: ruby,
+};
+
+export function getModelsAsDataLanguage(
+  language: QueryLanguage,
+): ModelsAsDataLanguage {
+  const definition = languages[language];
+  if (!definition) {
+    throw new Error(`No models-as-data definition for ${language}`);
+  }
+  return definition;
+}
+
+export function getModelsAsDataLanguageModel<
+  T extends keyof ModelsAsDataLanguagePredicates,
+>(
+  language: QueryLanguage,
+  model: T,
+): NonNullable<ModelsAsDataLanguagePredicates[T]> {
+  const definition = getModelsAsDataLanguage(language).predicates[model];
+  if (!definition) {
+    throw new Error(`No models-as-data predicate for ${model}`);
+  }
+  return definition;
+}

extensions/ql-vscode/src/model-editor/languages/models-as-data.ts

@@ -0,0 +1,69 @@
+import { MethodArgument, MethodDefinition } from "../method";
+import {
+  ModeledMethod,
+  NeutralModeledMethod,
+  SinkModeledMethod,
+  SourceModeledMethod,
+  SummaryModeledMethod,
+  TypeModeledMethod,
+} from "../modeled-method";
+import { DataTuple } from "../model-extension-file";
+import { Mode } from "../shared/mode";
+import type { QueryConstraints } from "../../local-queries/query-constraints";
+import { DecodedBqrs } from "../../common/bqrs-cli-types";
+import { BaseLogger } from "../../common/logging";
+
+type GenerateMethodDefinition<T> = (method: T) => DataTuple[];
+type ReadModeledMethod = (row: DataTuple[]) => ModeledMethod;
+
+export type ModelsAsDataLanguagePredicate<T> = {
+  extensiblePredicate: string;
+  supportedKinds?: string[];
+  generateMethodDefinition: GenerateMethodDefinition<T>;
+  readModeledMethod: ReadModeledMethod;
+};
+
+type ModelsAsDataLanguageModelGeneration = {
+  queryConstraints: QueryConstraints;
+  filterQueries?: (queryPath: string) => boolean;
+  parseResults: (
+    // The path to the query that generated the results.
+    queryPath: string,
+    // The results of the query.
+    bqrs: DecodedBqrs,
+    // The language-specific predicate that was used to generate the results. This is passed to allow
+    // sharing of code between different languages.
+    modelsAsDataLanguage: ModelsAsDataLanguage,
+    // The logger to use for logging.
+    logger: BaseLogger,
+  ) => ModeledMethod[];
+};
+
+export type ModelsAsDataLanguagePredicates = {
+  source?: ModelsAsDataLanguagePredicate<SourceModeledMethod>;
+  sink?: ModelsAsDataLanguagePredicate<SinkModeledMethod>;
+  summary?: ModelsAsDataLanguagePredicate<SummaryModeledMethod>;
+  neutral?: ModelsAsDataLanguagePredicate<NeutralModeledMethod>;
+  type?: ModelsAsDataLanguagePredicate<TypeModeledMethod>;
+};
+
+export type MethodArgumentOptions = {
+  options: MethodArgument[];
+  defaultArgumentPath: string;
+};
+
+export type ModelsAsDataLanguage = {
+  /**
+   * The modes that are available for this language. If not specified, all
+   * modes are available.
+   */
+  availableModes?: Mode[];
+  createMethodSignature: (method: MethodDefinition) => string;
+  predicates: ModelsAsDataLanguagePredicates;
+  modelGeneration?: ModelsAsDataLanguageModelGeneration;
+  /**
+   * Returns the list of valid arguments that can be selected for the given method.
+   * @param method The method to get the valid arguments for.
+   */
+  getArgumentOptions: (method: MethodDefinition) => MethodArgumentOptions;
+};

extensions/ql-vscode/src/model-editor/languages/ruby/generate.ts

@@ -0,0 +1,50 @@
+import { BaseLogger } from "../../../common/logging";
+import { DecodedBqrs } from "../../../common/bqrs-cli-types";
+import { ModelsAsDataLanguage } from "../models-as-data";
+import { ModeledMethod } from "../../modeled-method";
+import { DataTuple } from "../../model-extension-file";
+
+export function parseGenerateModelResults(
+  _queryPath: string,
+  bqrs: DecodedBqrs,
+  modelsAsDataLanguage: ModelsAsDataLanguage,
+  logger: BaseLogger,
+): ModeledMethod[] {
+  const modeledMethods: ModeledMethod[] = [];
+
+  for (const resultSetName in bqrs) {
+    const definition = Object.values(modelsAsDataLanguage.predicates).find(
+      (definition) => definition.extensiblePredicate === resultSetName,
+    );
+    if (definition === undefined) {
+      void logger.log(`No predicate found for ${resultSetName}`);
+
+      continue;
+    }
+
+    const resultSet = bqrs[resultSetName];
+
+    if (
+      resultSet.tuples.some((tuple) =>
+        tuple.some((value) => typeof value === "object"),
+      )
+    ) {
+      void logger.log(
+        `Skipping ${resultSetName} because it contains undefined values`,
+      );
+      continue;
+    }
+
+    modeledMethods.push(
+      ...resultSet.tuples.map((tuple) => {
+        const row = tuple.filter(
+          (value): value is DataTuple => typeof value !== "object",
+        );
+
+        return definition.readModeledMethod(row);
+      }),
+    );
+  }
+
+  return modeledMethods;
+}

extensions/ql-vscode/src/model-editor/languages/ruby/index.ts

@@ -0,0 +1,235 @@
+import { ModelsAsDataLanguage } from "../models-as-data";
+import { sharedExtensiblePredicates, sharedKinds } from "../shared";
+import { Mode } from "../../shared/mode";
+import { parseGenerateModelResults } from "./generate";
+import { getArgumentsList, MethodArgument } from "../../method";
+
+function parseRubyMethodFromPath(path: string): string {
+  const match = path.match(/Method\[([^\]]+)].*/);
+  if (match) {
+    return match[1];
+  } else {
+    return "";
+  }
+}
+
+function parseRubyAccessPath(path: string): {
+  methodName: string;
+  path: string;
+} {
+  const match = path.match(/Method\[([^\]]+)]\.(.*)/);
+  if (match) {
+    return { methodName: match[1], path: match[2] };
+  } else {
+    return { methodName: "", path: "" };
+  }
+}
+
+function rubyMethodSignature(typeName: string, methodName: string) {
+  return `${typeName}#${methodName}`;
+}
+
+function rubyMethodPath(methodName: string) {
+  if (methodName === "") {
+    return "";
+  }
+
+  return `Method[${methodName}]`;
+}
+
+function rubyPath(methodName: string, path: string) {
+  const methodPath = rubyMethodPath(methodName);
+  if (methodPath === "") {
+    return path;
+  }
+
+  return `${methodPath}.${path}`;
+}
+
+export const ruby: ModelsAsDataLanguage = {
+  availableModes: [Mode.Framework],
+  createMethodSignature: ({ typeName, methodName }) =>
+    `${typeName}#${methodName}`,
+  predicates: {
+    source: {
+      extensiblePredicate: sharedExtensiblePredicates.source,
+      supportedKinds: sharedKinds.source,
+      // extensible predicate sourceModel(
+      //   string type, string path, string kind
+      // );
+      generateMethodDefinition: (method) => [
+        method.typeName,
+        rubyPath(method.methodName, method.output),
+        method.kind,
+      ],
+      readModeledMethod: (row) => {
+        const typeName = row[0] as string;
+        const { methodName, path: output } = parseRubyAccessPath(
+          row[1] as string,
+        );
+        return {
+          type: "source",
+          input: "",
+          output,
+          kind: row[2] as string,
+          provenance: "manual",
+          signature: rubyMethodSignature(typeName, methodName),
+          packageName: "",
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+    sink: {
+      extensiblePredicate: sharedExtensiblePredicates.sink,
+      supportedKinds: sharedKinds.sink,
+      // extensible predicate sinkModel(
+      //   string type, string path, string kind
+      // );
+      generateMethodDefinition: (method) => {
+        return [
+          method.typeName,
+          rubyPath(method.methodName, method.input),
+          method.kind,
+        ];
+      },
+      readModeledMethod: (row) => {
+        const typeName = row[0] as string;
+        const { methodName, path: input } = parseRubyAccessPath(
+          row[1] as string,
+        );
+        return {
+          type: "sink",
+          input,
+          output: "",
+          kind: row[2] as string,
+          provenance: "manual",
+          signature: rubyMethodSignature(typeName, methodName),
+          packageName: "",
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+    summary: {
+      extensiblePredicate: sharedExtensiblePredicates.summary,
+      supportedKinds: sharedKinds.summary,
+      // extensible predicate summaryModel(
+      //   string type, string path, string input, string output, string kind
+      // );
+      generateMethodDefinition: (method) => [
+        method.typeName,
+        rubyMethodPath(method.methodName),
+        method.input,
+        method.output,
+        method.kind,
+      ],
+      readModeledMethod: (row) => {
+        const typeName = row[0] as string;
+        const methodName = parseRubyMethodFromPath(row[1] as string);
+        return {
+          type: "summary",
+          input: row[2] as string,
+          output: row[3] as string,
+          kind: row[4] as string,
+          provenance: "manual",
+          signature: rubyMethodSignature(typeName, methodName),
+          packageName: "",
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+    neutral: {
+      extensiblePredicate: sharedExtensiblePredicates.neutral,
+      supportedKinds: sharedKinds.neutral,
+      // extensible predicate neutralModel(
+      //   string type, string path, string kind
+      // );
+      generateMethodDefinition: (method) => [
+        method.typeName,
+        rubyMethodPath(method.methodName),
+        method.kind,
+      ],
+      readModeledMethod: (row) => {
+        const typeName = row[0] as string;
+        const methodName = parseRubyMethodFromPath(row[1] as string);
+        return {
+          type: "neutral",
+          input: "",
+          output: "",
+          kind: row[2] as string,
+          provenance: "manual",
+          signature: rubyMethodSignature(typeName, methodName),
+          packageName: "",
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+    type: {
+      extensiblePredicate: "typeModel",
+      // extensible predicate typeModel(string type1, string type2, string path);
+      generateMethodDefinition: (method) => [
+        method.relatedTypeName,
+        method.typeName,
+        rubyPath(method.methodName, method.path),
+      ],
+      readModeledMethod: (row) => {
+        const typeName = row[1] as string;
+        const { methodName, path } = parseRubyAccessPath(row[2] as string);
+
+        return {
+          type: "type",
+          relatedTypeName: row[0] as string,
+          path,
+          signature: rubyMethodSignature(typeName, methodName),
+          packageName: "",
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+  },
+  modelGeneration: {
+    queryConstraints: {
+      "query path": "queries/modeling/GenerateModel.ql",
+    },
+    parseResults: parseGenerateModelResults,
+  },
+  getArgumentOptions: (method) => {
+    const argumentsList = getArgumentsList(method.methodParameters).map(
+      (argument, index): MethodArgument => {
+        if (argument.endsWith(":")) {
+          return {
+            path: `Argument[${argument}]`,
+            label: `Argument[${argument}]`,
+          };
+        }
+
+        return {
+          path: `Argument[${index}]`,
+          label: `Argument[${index}]: ${argument}`,
+        };
+      },
+    );
+
+    return {
+      options: [
+        {
+          path: "Argument[self]",
+          label: "Argument[self]",
+        },
+        ...argumentsList,
+      ],
+      // If there are no arguments, we will default to "Argument[self]"
+      defaultArgumentPath:
+        argumentsList.length > 0 ? argumentsList[0].path : "Argument[self]",
+    };
+  },
+};

extensions/ql-vscode/src/model-editor/languages/shared.ts

@@ -0,0 +1,25 @@
+export const sharedExtensiblePredicates = {
+  source: "sourceModel",
+  sink: "sinkModel",
+  summary: "summaryModel",
+  neutral: "neutralModel",
+};
+
+export const sharedKinds = {
+  source: ["local", "remote"],
+  sink: [
+    "code-injection",
+    "command-injection",
+    "file-content-store",
+    "html-injection",
+    "js-injection",
+    "ldap-injection",
+    "log-injection",
+    "path-injection",
+    "request-forgery",
+    "sql-injection",
+    "url-redirection",
+  ],
+  summary: ["taint", "value"],
+  neutral: ["summary", "source", "sink"],
+};

extensions/ql-vscode/src/model-editor/languages/static/generate.ts

@@ -0,0 +1,60 @@
+import { BaseLogger } from "../../../common/logging";
+import {
+  ModelsAsDataLanguage,
+  ModelsAsDataLanguagePredicates,
+} from "../models-as-data";
+import { DecodedBqrs } from "../../../common/bqrs-cli-types";
+import { ModeledMethod } from "../../modeled-method";
+import { basename } from "../../../common/path";
+
+const queriesToModel: Record<string, keyof ModelsAsDataLanguagePredicates> = {
+  "CaptureSummaryModels.ql": "summary",
+  "CaptureSinkModels.ql": "sink",
+  "CaptureSourceModels.ql": "source",
+  "CaptureNeutralModels.ql": "neutral",
+};
+
+export function filterFlowModelQueries(queryPath: string): boolean {
+  return Object.keys(queriesToModel).includes(basename(queryPath));
+}
+
+export function parseFlowModelResults(
+  queryPath: string,
+  bqrs: DecodedBqrs,
+  modelsAsDataLanguage: ModelsAsDataLanguage,
+  logger: BaseLogger,
+): ModeledMethod[] {
+  if (Object.keys(bqrs).length !== 1) {
+    throw new Error(
+      `Expected exactly one result set from ${queryPath}, but got ${
+        Object.keys(bqrs).length
+      }`,
+    );
+  }
+
+  const modelType = queriesToModel[basename(queryPath)];
+  if (!modelType) {
+    void logger.log(`Unknown model type for ${queryPath}`);
+    return [];
+  }
+
+  const resultSet = bqrs[Object.keys(bqrs)[0]];
+
+  const results = resultSet.tuples;
+
+  const definition = modelsAsDataLanguage.predicates[modelType];
+  if (!definition) {
+    throw new Error(`No definition for ${modelType}`);
+  }
+
+  return (
+    results
+      // This is just a sanity check. The query should only return strings.
+      .filter((result) => typeof result[0] === "string")
+      .map((result) => {
+        const row = result[0] as string;
+
+        return definition.readModeledMethod(row.split(";"));
+      })
+  );
+}

extensions/ql-vscode/src/model-editor/languages/static/index.ts

@@ -0,0 +1,170 @@
+import { ModelsAsDataLanguage } from "../models-as-data";
+import { Provenance } from "../../modeled-method";
+import { DataTuple } from "../../model-extension-file";
+import { sharedExtensiblePredicates, sharedKinds } from "../shared";
+import { filterFlowModelQueries, parseFlowModelResults } from "./generate";
+import { getArgumentsList, MethodArgument } from "../../method";
+
+function readRowToMethod(row: DataTuple[]): string {
+  return `${row[0]}.${row[1]}#${row[3]}${row[4]}`;
+}
+
+export const staticLanguage: ModelsAsDataLanguage = {
+  createMethodSignature: ({
+    packageName,
+    typeName,
+    methodName,
+    methodParameters,
+  }) => `${packageName}.${typeName}#${methodName}${methodParameters}`,
+  predicates: {
+    source: {
+      extensiblePredicate: sharedExtensiblePredicates.source,
+      supportedKinds: sharedKinds.source,
+      // extensible predicate sourceModel(
+      //   string package, string type, boolean subtypes, string name, string signature, string ext,
+      //   string output, string kind, string provenance
+      // );
+      generateMethodDefinition: (method) => [
+        method.packageName,
+        method.typeName,
+        true,
+        method.methodName,
+        method.methodParameters,
+        "",
+        method.output,
+        method.kind,
+        method.provenance,
+      ],
+      readModeledMethod: (row) => ({
+        type: "source",
+        input: "",
+        output: row[6] as string,
+        kind: row[7] as string,
+        provenance: row[8] as Provenance,
+        signature: readRowToMethod(row),
+        packageName: row[0] as string,
+        typeName: row[1] as string,
+        methodName: row[3] as string,
+        methodParameters: row[4] as string,
+      }),
+    },
+    sink: {
+      extensiblePredicate: sharedExtensiblePredicates.sink,
+      supportedKinds: sharedKinds.sink,
+      // extensible predicate sinkModel(
+      //   string package, string type, boolean subtypes, string name, string signature, string ext,
+      //   string input, string kind, string provenance
+      // );
+      generateMethodDefinition: (method) => [
+        method.packageName,
+        method.typeName,
+        true,
+        method.methodName,
+        method.methodParameters,
+        "",
+        method.input,
+        method.kind,
+        method.provenance,
+      ],
+      readModeledMethod: (row) => ({
+        type: "sink",
+        input: row[6] as string,
+        output: "",
+        kind: row[7] as string,
+        provenance: row[8] as Provenance,
+        signature: readRowToMethod(row),
+        packageName: row[0] as string,
+        typeName: row[1] as string,
+        methodName: row[3] as string,
+        methodParameters: row[4] as string,
+      }),
+    },
+    summary: {
+      extensiblePredicate: sharedExtensiblePredicates.summary,
+      supportedKinds: sharedKinds.summary,
+      // extensible predicate summaryModel(
+      //   string package, string type, boolean subtypes, string name, string signature, string ext,
+      //   string input, string output, string kind, string provenance
+      // );
+      generateMethodDefinition: (method) => [
+        method.packageName,
+        method.typeName,
+        true,
+        method.methodName,
+        method.methodParameters,
+        "",
+        method.input,
+        method.output,
+        method.kind,
+        method.provenance,
+      ],
+      readModeledMethod: (row) => ({
+        type: "summary",
+        input: row[6] as string,
+        output: row[7] as string,
+        kind: row[8] as string,
+        provenance: row[9] as Provenance,
+        signature: readRowToMethod(row),
+        packageName: row[0] as string,
+        typeName: row[1] as string,
+        methodName: row[3] as string,
+        methodParameters: row[4] as string,
+      }),
+    },
+    neutral: {
+      extensiblePredicate: sharedExtensiblePredicates.neutral,
+      supportedKinds: sharedKinds.neutral,
+      // extensible predicate neutralModel(
+      //   string package, string type, string name, string signature, string kind, string provenance
+      // );
+      generateMethodDefinition: (method) => [
+        method.packageName,
+        method.typeName,
+        method.methodName,
+        method.methodParameters,
+        method.kind,
+        method.provenance,
+      ],
+      readModeledMethod: (row) => ({
+        type: "neutral",
+        input: "",
+        output: "",
+        kind: row[4] as string,
+        provenance: row[5] as Provenance,
+        signature: `${row[0]}.${row[1]}#${row[2]}${row[3]}`,
+        packageName: row[0] as string,
+        typeName: row[1] as string,
+        methodName: row[2] as string,
+        methodParameters: row[3] as string,
+      }),
+    },
+  },
+  modelGeneration: {
+    queryConstraints: {
+      "tags contain": ["modelgenerator"],
+    },
+    filterQueries: filterFlowModelQueries,
+    parseResults: parseFlowModelResults,
+  },
+  getArgumentOptions: (method) => {
+    const argumentsList = getArgumentsList(method.methodParameters).map(
+      (argument, index): MethodArgument => ({
+        path: `Argument[${index}]`,
+        label: `Argument[${index}]: ${argument}`,
+      }),
+    );
+
+    return {
+      options: [
+        {
+          path: "Argument[this]",
+          label: "Argument[this]",
+        },
+        ...argumentsList,
+      ],
+      // If there are no arguments, we will default to "Argument[this]"
+      defaultArgumentPath:
+        argumentsList.length > 0 ? argumentsList[0].path : "Argument[this]",
+    };
+  },
+};

extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-panel.ts

@@ -4,9 +4,9 @@ import { DisposableObject } from "../../common/disposable-object";
 import { MethodModelingViewProvider } from "./method-modeling-view-provider";
 import { Method } from "../method";
 import { ModelingStore } from "../modeling-store";
-import { ModelEditorViewTracker } from "../model-editor-view-tracker";
 import { ModelConfigListener } from "../../config";
 import { DatabaseItem } from "../../databases/local-databases";
+import { ModelingEvents } from "../modeling-events";
 
 export class MethodModelingPanel extends DisposableObject {
   private readonly provider: MethodModelingViewProvider;
@@ -14,7 +14,7 @@ export class MethodModelingPanel extends DisposableObject {
   constructor(
     app: App,
     modelingStore: ModelingStore,
-    editorViewTracker: ModelEditorViewTracker,
+    modelingEvents: ModelingEvents,
   ) {
     super();
 
@@ -26,7 +26,7 @@ export class MethodModelingPanel extends DisposableObject {
     this.provider = new MethodModelingViewProvider(
       app,
       modelingStore,
-      editorViewTracker,
+      modelingEvents,
       modelConfig,
     );
     this.push(

extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-view-provider.ts

@@ -4,17 +4,19 @@ import {
 } from "../../common/interface-types";
 import { telemetryListener } from "../../common/vscode/telemetry";
 import { showAndLogExceptionWithTelemetry } from "../../common/logging/notifications";
-import { extLogger } from "../../common/logging/vscode/loggers";
 import { App } from "../../common/app";
 import { redactableError } from "../../common/errors";
 import { Method } from "../method";
 import { ModelingStore } from "../modeling-store";
 import { AbstractWebviewViewProvider } from "../../common/vscode/abstract-webview-view-provider";
 import { assertNever } from "../../common/helpers-pure";
-import { ModelEditorViewTracker } from "../model-editor-view-tracker";
 import { ModelConfigListener } from "../../config";
 import { DatabaseItem } from "../../databases/local-databases";
-import { convertFromLegacyModeledMethod } from "../shared/modeled-methods-legacy";
+import { ModelingEvents } from "../modeling-events";
+import {
+  QueryLanguage,
+  tryGetQueryLanguage,
+} from "../../common/query-language";
 
 export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
   ToMethodModelingMessage,
@@ -24,11 +26,12 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
 
   private method: Method | undefined = undefined;
   private databaseItem: DatabaseItem | undefined = undefined;
+  private language: QueryLanguage | undefined = undefined;
 
   constructor(
     app: App,
     private readonly modelingStore: ModelingStore,
-    private readonly editorViewTracker: ModelEditorViewTracker,
+    private readonly modelingEvents: ModelingEvents,
     private readonly modelConfig: ModelConfigListener,
   ) {
     super(app, "method-modeling");
@@ -36,7 +39,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
 
   protected override async onWebViewLoaded(): Promise<void> {
     await Promise.all([this.setViewState(), this.setInitialState()]);
-    this.registerToModelingStoreEvents();
+    this.registerToModelingEvents();
     this.registerToModelConfigEvents();
   }
 
@@ -44,7 +47,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
     await this.postMessage({
       t: "setMethodModelingPanelViewState",
       viewState: {
-        showMultipleModels: this.modelConfig.showMultipleModels,
+        language: this.language,
       },
     });
   }
@@ -55,6 +58,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
   ): Promise<void> {
     this.method = method;
     this.databaseItem = databaseItem;
+    this.language = databaseItem && tryGetQueryLanguage(databaseItem.language);
 
     if (this.isShowingView) {
       await this.postMessage({
@@ -69,6 +73,9 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
       const selectedMethod = this.modelingStore.getSelectedMethodDetails();
       if (selectedMethod) {
         this.databaseItem = selectedMethod.databaseItem;
+        this.language = tryGetQueryLanguage(
+          selectedMethod.databaseItem.language,
+        );
         this.method = selectedMethod.method;
 
         await this.postMessage({
@@ -76,6 +83,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
           method: selectedMethod.method,
           modeledMethods: selectedMethod.modeledMethods,
           isModified: selectedMethod.isModified,
+          isInProgress: selectedMethod.isInProgress,
         });
       }
 
@@ -100,7 +108,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
 
       case "unhandledError":
         void showAndLogExceptionWithTelemetry(
-          extLogger,
+          this.app.logger,
           telemetryListener,
           redactableError(
             msg.error,
@@ -108,24 +116,27 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
         );
         break;
 
-      case "setModeledMethod": {
+      case "setMultipleModeledMethods": {
         if (!this.databaseItem) {
           return;
         }
 
         this.modelingStore.updateModeledMethods(
           this.databaseItem,
-          msg.method.signature,
-          convertFromLegacyModeledMethod(msg.method),
+          msg.methodSignature,
+          msg.modeledMethods,
         );
         this.modelingStore.addModifiedMethod(
           this.databaseItem,
-          msg.method.signature,
+          msg.methodSignature,
         );
         break;
       }
       case "revealInModelEditor":
         await this.revealInModelEditor(msg.method);
+        void telemetryListener?.sendUIInteraction(
+          "method-modeling-reveal-in-model-editor",
+        );
 
         break;
 
@@ -144,15 +155,15 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
       return;
     }
 
-    const view = this.editorViewTracker.getView(
+    this.modelingEvents.fireRevealInModelEditorEvent(
       this.databaseItem.databaseUri.toString(),
+      method,
     );
-    await view?.revealMethod(method);
   }
 
-  private registerToModelingStoreEvents(): void {
+  private registerToModelingEvents(): void {
     this.push(
-      this.modelingStore.onModeledMethodsChanged(async (e) => {
+      this.modelingEvents.onModeledMethodsChanged(async (e) => {
         if (this.webviewView && e.isActiveDb && this.method) {
           const modeledMethods = e.modeledMethods[this.method.signature];
           if (modeledMethods) {
@@ -167,7 +178,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
     );
 
     this.push(
-      this.modelingStore.onModifiedMethodsChanged(async (e) => {
+      this.modelingEvents.onModifiedMethodsChanged(async (e) => {
         if (this.webviewView && e.isActiveDb && this.method) {
           const isModified = e.modifiedMethods.has(this.method.signature);
           await this.postMessage({
@@ -179,32 +190,39 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
     );
 
     this.push(
-      this.modelingStore.onSelectedMethodChanged(async (e) => {
+      this.modelingEvents.onSelectedMethodChanged(async (e) => {
         if (this.webviewView) {
           this.method = e.method;
           this.databaseItem = e.databaseItem;
+          this.language = tryGetQueryLanguage(e.databaseItem.language);
 
           await this.postMessage({
             t: "setSelectedMethod",
             method: e.method,
             modeledMethods: e.modeledMethods,
             isModified: e.isModified,
+            isInProgress: e.isInProgress,
           });
         }
       }),
     );
 
     this.push(
-      this.modelingStore.onDbOpened(async () => {
+      this.modelingEvents.onDbOpened(async (databaseItem) => {
+        this.databaseItem = databaseItem;
+
         await this.postMessage({
           t: "setInModelingMode",
           inModelingMode: true,
         });
+
+        this.language = tryGetQueryLanguage(databaseItem.language);
+        await this.setViewState();
       }),
     );
 
     this.push(
-      this.modelingStore.onDbClosed(async (dbUri) => {
+      this.modelingEvents.onDbClosed(async (dbUri) => {
         if (!this.modelingStore.anyDbsBeingModeled()) {
           await this.postMessage({
             t: "setInModelingMode",
@@ -217,6 +235,21 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
         }
       }),
     );
+
+    this.push(
+      this.modelingEvents.onInProgressMethodsChanged(async (e) => {
+        if (this.method && this.databaseItem) {
+          const dbUri = this.databaseItem.databaseUri.toString();
+          if (e.dbUri === dbUri) {
+            const inProgress = e.methods.has(this.method.signature);
+            await this.postMessage({
+              t: "setInProgress",
+              inProgress,
+            });
+          }
+        }
+      }),
+    );
   }
 
   private registerToModelConfigEvents(): void {