v1.10.0

Use Octokit REST for listing CodeQL databases

.github/workflows/cli-test.yml

@@ -60,7 +60,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'

.github/workflows/main.yml

@@ -20,7 +20,7 @@ jobs:
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
@@ -62,7 +62,7 @@ jobs:
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
@@ -110,7 +110,7 @@ jobs:
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
@@ -149,7 +149,7 @@ jobs:
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
@@ -183,7 +183,7 @@ jobs:
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'
@@ -251,7 +251,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
           cache: 'npm'

.github/workflows/release.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version-file: extensions/ql-vscode/.nvmrc
 

docs/releasing.md

@@ -1,80 +1,62 @@
 # Releasing (write access required)
 
-1. Run the ["Run CLI tests" workflow](https://github.com/github/vscode-codeql/actions/workflows/cli-test.yml) and make sure the tests are green. If there were no merges between the time the workflow ran (it runs daily), and the release, you can skip this step.  
+1. Determine the new version number. We default to increasing the patch version number, but make our own judgement about whether a change is big enough to warrant a minor version bump. Common reasons for a minor bump could include:
+    * Making substantial new features available to all users. This can include lifting a feature flag.
+    * Breakage in compatibility with recent versions of the CLI.
+    * Minimum required version of VS Code is increased.
+    * New telemetry events are added.
+    * Deprecation or removal of commands.
+    * Accumulation of many changes, none of which are individually big enough to warrant a minor bump, but which together are. This does not include changes which are purely internal to the extension, such as refactoring, or which are only available behind a feature flag.
+1. Create a release branch named after the new version (e.g. `v1.3.6`):
+    * For a regular scheduled release this branch will be based on latest `main`.
+        * Make sure your local copy of `main` is up to date so you are including all changes.
+    * To do a minimal bug-fix release, base the release branch on the tag from the most recent release and then add only the changes you want to release.
+        * Choose this option if you want to release a specific set of changes (e.g. a bug fix) and don't want to incur extra risk by including other changes that have been merged to the `main` branch.
+
+        ```bash
+        git checkout -b <new_release_branch> <previous_release_tag>
+        ```
+
+1. Run the ["Run CLI tests" workflow](https://github.com/github/vscode-codeql/actions/workflows/cli-test.yml) and make sure the tests are green.
+    * You can skip this step if you are releasing from `main` and there were no merges since the most recent daily scheduled run of this workflow.
 1. Double-check the `CHANGELOG.md` contains all desired change comments and has the version to be released with date at the top.
-    * Go through all recent PRs and make sure they are properly accounted for.
+    * Go through PRs that have been merged since the previous release and make sure they are properly accounted for.
     * Make sure all changelog entries have links back to their PR(s) if appropriate.
-    * For picking the new version number, we default to increasing the patch version number, but make our own judgement about whether a change is big enough to warrant a minor version bump. Common reasons for a minor bump could include:
-      * Making substantial new features available to all users. This can include lifting a feature flag.
-      * Breakage in compatibility with recent versions of the CLI.
-      * Minimum required version of VS Code is increased.
-      * New telemetry events are added.
-      * Deprecation or removal of commands.
-      * Accumulation of many changes, none of which are individually big enough to warrant a minor bump, but which together are. This does not include changes which are purely internal to the extension, such as refactoring, or which are only available behind a feature flag.
-1. Double-check that the node version we're using matches the one used for VS Code. See the [Node.js version instructions](./node-version.md) for more information.
 1. Double-check that the extension `package.json` and `package-lock.json` have the version you intend to release. If you are doing a patch release (as opposed to minor or major version) this should already be correct.
-1. Create a PR for this release:
+1. Commit any changes made during steps 4 and 5 with a commit message the same as the branch name (e.g. `v1.3.6`).
-    * This PR will contain any missing bits from steps 1, 2 and 3. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
+1. Open a PR for this release.
-    * Create a new branch for the release named after the new version. For example: `v1.3.6`
+    * The PR diff should contain:
-    * Create a new commit with a message the same as the branch name.
+        * Any missing bits from steps 4 and 5. Most of the time, this will just be updating `CHANGELOG.md` with today's date.
-    * Create a PR for this branch.
+        * If releasing from a branch other than `main`, this PR will also contain the extension changes being released.
-    * Wait for the PR to be merged into `main`
+1. Build the extension using `npm run build` and install it on your VS Code using "Install from VSIX".
-1. Switch to `main` branch and pull latest changes
-1. Lock the `main` branch.
-    * Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
-    * Select "Lock branch"
-    * Click "Save changes"
-1. Ensure that no PRs have been merged since the release PR that you merged. If there were, you might need to unlock `main` temporarily and update the CHANGELOG again.
-1. Build the extension `npm run build` and install it on your VS Code using "Install from VSIX".
 1. Go through [our test plan](./test-plan.md) to ensure that the extension is working as expected.
-1. Switch to `main` and add a new tag on the `main` branch with your new version (named after the release), e.g.
+1. Create a new tag on the release branch with your new version (named after the release), e.g.
 
     ```bash
-    git checkout main
     git tag v1.3.6
     ```
 
-   If you've accidentally created a badly named tag, you can delete it via
+1. Merge the release PR into `main`.
-
+    * If there are conflicts in the changelog, make sure to place any new changelog entries at the top, above the section for the current release, as these new entries are not part of the current release and should be placed in the "unreleased" section.
-   ```bash
+    * The release PR must be merged before pushing the tag to ensure that we always release a commit that is present on the `main` branch. It's not required that the commit is the head of the `main` branch, but there should be no chance of a future release accidentally not including changes from this release.
-   git tag -d badly-named-tag
-   ```
-
-1. Unlock the main branch
-    * Go to the [branch protection rules for the `main` branch](https://github.com/github/vscode-codeql/settings/branch_protection_rules/16447115)
-    * Deselect "Lock branch"
-    * Click "Save changes"
 1. Push the new tag up:
 
-   a. If you're using a fork of the repo:
-
-    ```bash
-    git push upstream refs/tags/v1.3.6
-    ```
-
-   b. If you're working straight in this repo:
-
     ```bash
     git push origin refs/tags/v1.3.6
     ```
 
-   This will trigger [a release build](https://github.com/github/vscode-codeql/releases) on Actions.
+1. Find the [Release](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease) workflow run that was just triggered by pushing the tag, and monitor the status of the release build.
-
-    * **IMPORTANT** Make sure you are on the `main` branch and your local checkout is fully updated when you add the tag.
-    * If you accidentally add the tag to the wrong ref, you can just force push it to the right one later.
-1. Monitor the status of the release build in the `Release` workflow in the Actions tab.
     * DO NOT approve the "publish" stages of the workflow yet.
 1. Download the VSIX from the draft GitHub release at the top of [the releases page](https://github.com/github/vscode-codeql/releases) that is created when the release build finishes.
 1. Unzip the `.vsix` and inspect its `package.json` to make sure the version is what you expect,
    or look at the source if there's any doubt the right code is being shipped.
 1. Install the `.vsix` file into your vscode IDE and ensure the extension can load properly. Run a single command (like run query, or add database).
-1. Go to the actions tab of the vscode-codeql repository and select the [Release workflow](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease).
+1. Approve the deployments of the [Release](https://github.com/github/vscode-codeql/actions?query=workflow%3ARelease) workflow run. This will automatically publish to Open VSX and VS Code Marketplace.
     * If there is an authentication failure when publishing, be sure to check that the authentication keys haven't expired. See below.
-1. Approve the deployments of the correct Release workflow. This will automatically publish to Open VSX and VS Code Marketplace.
+1. Go to the draft GitHub release in [the releases page](https://github.com/github/vscode-codeql/releases), click 'Edit', add some summary description, and publish it.
-1. Go to the draft GitHub release in [the releases tab of the repository](https://github.com/github/vscode-codeql/releases), click 'Edit', add some summary description, and publish it.
+1. Confirm the new release is marked as the latest release.
-1. Confirm the new release is marked as the latest release at <https://github.com/github/vscode-codeql/releases>.
 1. If documentation changes need to be published, notify documentation team that release has been made.
-1. Review and merge the version bump PR that is automatically created by Actions.
+1. Review and merge the version bump PR that is automatically created by the Release workflow.
 
 ## Secrets and authentication for publishing
 

docs/test-plan.md

@@ -151,17 +151,20 @@ Run one of the above MRVAs, but cancel it from within VS Code:
 2. Open the Model Editor with the "CodeQL: Open CodeQL Model Editor" command from the command palette.
    - Check that the editor loads and shows methods to model.
    - Check that methods are grouped per library (e.g. `rocksdbjni@7.7.3` or `asm@6.0`)
-   - Check that the "Open database" link works.
+   - Check that the "Open source" link works.
+   - Check that the 'View' button works and the Method Usage panel highlight the correct method and usage
+   - Check that the Method Modeling panel shows the correct method and modeling state
 
 #### Test Case 2: Model methods
 
 1. Expand one of the libraries.
    - Change the model type and check that the other dropdowns change.
+   - Check that the method modeling panel updates accordingly
 2. Save the modeled methods.
 3. Click "Open extension pack"
    - Check that the file explorer opens a directory with a "models" directory
 4. Open the ".model.yml" file corresponding to the library that was changed.
-   - Check that the file contrains the entries that was modeled.
+   - Check that the file contains entries for the methods that were modeled.
 
 #### Test Case 3: Model with AI
 
@@ -189,9 +192,28 @@ Are there any components that are not showing up?
 
 ## Optional Test Cases
 
-These are mostly aimed at MRVA, but some of them are also applicable to non-MRVA queries.
+### Modeling Flow
 
-### Selecting repositories to run on
+1. Check that a method can have multiple models:
+   - Add a couple of new models for one method in the model editor
+   - Save and check that the modeling file (use the 'open extension pack' button to open it) shows multiple methods
+   - Check that the Method Modeling Panel shows the correct multiple models
+   - Check that you can browse through different models in the Method Modeling Panel
+   - Check that a 'duplicated classification' error appears in both model editor and modeling panel when a duplicate modeling occurs
+   - Check that a 'conflicting classification' error appears when a neutral model type is paired with a model of the same kind
+   - Check that clicking on the error highlights the correct modeling in both the editor and the modeling panel
+2. Check the Method Usage Panel
+   - Check that the Method Usage Panel opens and jumps to the correct usage when clicking on 'View' in the model editor
+   - Check that the first and following usages are opening when clicking on a usage
+   - Check that the usage icon color turns green when saving a newly modeled method
+   - Check that the usage icon color turns red when saving a newly unmodeld method
+3. Check the Method Modeling Panel
+   - Check that the 'Start modeling' button opens a new model editor
+   - Check that it refreshes the blank state when a model editor is opened/closed
+   - Check that when modeling in the editor the modeling panel updates accordingly
+   - Check that when modeling in the modeling panel the model editor updates accordingly
+
+### Selecting MRVA repositories to run on
 
 #### Test case 1: Running a query on a single repository
 
@@ -221,7 +243,7 @@ These are mostly aimed at MRVA, but some of them are also applicable to non-MRVA
    4. The org contains private repositories that are inaccessible
 2. The org does not exist
 
-### Using different types of controller repos
+### Using different types of controller repos for MRVA
 
 #### Test case 1: Running a query when the controller repository is public
 

docs/testing.md

@@ -27,7 +27,7 @@ Pre-requisites:
 Then, from the `extensions/ql-vscode` directory, use the appropriate command to run the tests:
 
 * Unit tests: `npm run test:unit`
-* View Tests: `npm test:view`
+* View Tests: `npm run test:view`
 * VSCode integration tests: `npm run test:vscode-integration`
 
 #### Running CLI integration tests from the terminal
@@ -48,8 +48,8 @@ Alternatively, you can run the tests inside of VSCode. There are several VSCode
 
 You will need to run tests using a task from inside of VS Code, under the "Run and Debug" view:
 
-* Unit tests: run the _Launch Unit Tests - React_ task
+* Unit tests: run the _Launch Unit Tests_ task
-* View Tests: run the _Launch Unit Tests_ task
+* View Tests: run the _Launch Unit Tests - React_ task
 * VSCode integration tests: run the _Launch Unit Tests - No Workspace_ and _Launch Unit Tests - Minimal Workspace_ tasks
 
 #### Running CLI integration tests from VSCode

extensions/ql-vscode/.eslintrc.js

@@ -71,6 +71,7 @@ const baseConfig = {
     "no-shadow": "off",
     "github/array-foreach": "off",
     "github/no-then": "off",
+    "react/jsx-key": ["error", { checkFragmentShorthand: true }],
   },
 };
 

extensions/ql-vscode/.storybook/main.ts

@@ -6,6 +6,7 @@ const config: StorybookConfig = {
     "@storybook/addon-links",
     "@storybook/addon-essentials",
     "@storybook/addon-interactions",
+    "@storybook/addon-a11y",
     "./vscode-theme-addon/preset.ts",
   ],
   framework: {

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,31 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.10.0 - 16 November 2023
+
+- Add new CodeQL views for managing databases and queries:
+  1. A queries panel that shows all queries in your workspace. It allows you to view, create, and run queries in one place.
+  2. A language selector, which allows you to quickly filter databases and queries by language.
+  
+  For more information, see the [documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/analyzing-your-projects/#filtering-databases-and-queries-by-language).
+- When adding a CodeQL database, we no longer add the database source folder to the workspace by default (since this caused bugs in single-folder workspaces). [#3047](https://github.com/github/vscode-codeql/pull/3047)
+  - You can manually add individual database source folders to the workspace with the "Add Database Source to Workspace" right-click command in the databases view.
+  - To restore the old behavior of adding all database source folders by default, set the `codeQL.addingDatabases.addDatabaseSourceToWorkspace` setting to `true`.
+- Rename the `codeQL.databaseDownload.allowHttp` setting to `codeQL.addingDatabases.allowHttp`, so that database-related settings are grouped together in the Settings UI. [#3047](https://github.com/github/vscode-codeql/pull/3047) & [#3069](https://github.com/github/vscode-codeql/pull/3069)
+- The "Sort by Language" action in the databases view now sorts by name within each language. [#3055](https://github.com/github/vscode-codeql/pull/3055)
+
+## 1.9.4 - 6 November 2023
+
+No user facing changes.
+
+## 1.9.3 - 26 October 2023
+
+- Sorted result set filenames now include a hash of the result set name instead of the full name. [#2955](https://github.com/github/vscode-codeql/pull/2955)
+- The "Install Pack Dependencies" will now only list CodeQL packs located in the workspace. [#2960](https://github.com/github/vscode-codeql/pull/2960)
+- Fix a bug where the "View Query Log" action for a query history item was not working. [#2984](https://github.com/github/vscode-codeql/pull/2984)
+- Add a command to sort items in the databases view by language. [#2993](https://github.com/github/vscode-codeql/pull/2993)
+- Fix not being able to open the results directory or evaluator log for a cancelled local query run. [#2996](https://github.com/github/vscode-codeql/pull/2996)
+- Fix empty row in alert path when the SARIF location was empty. [#3018](https://github.com/github/vscode-codeql/pull/3018)
+
 ## 1.9.2 - 12 October 2023
 
 - Fix a bug where the query to Find Definitions in database source files would not be cancelled appropriately. [#2885](https://github.com/github/vscode-codeql/pull/2885)
@@ -7,6 +33,7 @@
 - Increase the required version of VS Code to 1.82.0. [#2877](https://github.com/github/vscode-codeql/pull/2877)
 - Fix a bug where the query server was restarted twice after configuration changes. [#2884](https://github.com/github/vscode-codeql/pull/2884).
 - Add support for the `telemetry.telemetryLevel` setting. For more information, see the [telemetry documentation](https://codeql.github.com/docs/codeql-for-visual-studio-code/about-telemetry-in-codeql-for-visual-studio-code). [#2824](https://github.com/github/vscode-codeql/pull/2824).
+- Add a "CodeQL: Trim Cache" command that clears the evaluation cache of a database except for predicates annotated with the `cached` keyword. Its purpose is to get accurate performance measurements when tuning the final stage of a query, like a data-flow configuration. This is equivalent to the `codeql database cleanup --mode=normal` CLI command. In contrast, the existing "CodeQL: Clear Cache" command clears the entire cache. CodeQL CLI v2.15.1 or later is required. [#2928](https://github.com/github/vscode-codeql/pull/2928)
 - Fix syntax highlighting directly after import statements with instantiation arguments. [#2792](https://github.com/github/vscode-codeql/pull/2792)
 - The `debug.saveBeforeStart` setting is now respected when running variant analyses. [#2950](https://github.com/github/vscode-codeql/pull/2950)
 - The 'open database' button of the model editor was renamed to 'open source'. Also, it's now only available if the source archive is available as a workspace folder. [#2945](https://github.com/github/vscode-codeql/pull/2945)

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.9.2",
+  "version": "1.10.0",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -34,27 +34,7 @@
     }
   },
   "activationEvents": [
-    "onLanguage:ql",
-    "onLanguage:ql-summary",
-    "onView:codeQLQueries",
-    "onView:codeQLDatabases",
-    "onView:codeQLVariantAnalysisRepositories",
-    "onView:codeQLQueryHistory",
-    "onView:codeQLAstViewer",
-    "onView:codeQLEvalLogViewer",
     "onView:test-explorer",
-    "onCommand:codeQL.checkForUpdatesToCLI",
-    "onCommand:codeQL.authenticateToGitHub",
-    "onCommand:codeQL.viewAst",
-    "onCommand:codeQL.viewCfg",
-    "onCommand:codeQL.openReferencedFile",
-    "onCommand:codeQL.previewQueryHelp",
-    "onCommand:codeQL.chooseDatabaseFolder",
-    "onCommand:codeQL.chooseDatabaseArchive",
-    "onCommand:codeQL.chooseDatabaseInternet",
-    "onCommand:codeQL.chooseDatabaseGithub",
-    "onCommand:codeQL.quickQuery",
-    "onCommand:codeQL.restartQueryServer",
     "onWebviewPanel:resultsView",
     "onWebviewPanel:codeQL.variantAnalysis",
     "onWebviewPanel:codeQL.dataFlowPaths",
@@ -392,13 +372,23 @@
       },
       {
         "type": "object",
-        "title": "Downloading databases",
+        "title": "Adding databases",
         "order": 6,
         "properties": {
-          "codeQL.databaseDownload.allowHttp": {
+          "codeQL.addingDatabases.allowHttp": {
             "type": "boolean",
             "default": false,
-            "description": "Allow database to be downloaded via HTTP. Warning: enabling this option will allow downloading from insecure servers."
+            "description": "Allow databases to be downloaded via HTTP. Warning: enabling this option will allow downloading from insecure servers."
+          },
+          "codeQL.databaseDownload.allowHttp": {
+            "type": "boolean",
+            "markdownDeprecationMessage": "**Deprecated**: Please use `#codeQL.addingDatabases.allowHttp#` instead.",
+            "deprecationMessage": "Deprecated: Please use codeQL.addingDatabases.allowHttp instead."
+          },
+          "codeQL.addingDatabases.addDatabaseSourceToWorkspace": {
+            "type": "boolean",
+            "default": false,
+            "markdownDescription": "When adding a CodeQL database, automatically add the database's source folder as a workspace folder. Warning: enabling this option in a single-folder workspace will cause the workspace to reload as a [multi-root workspace](https://code.visualstudio.com/docs/editor/multi-root-workspaces). This may cause query history and database lists to be reset."
           }
         }
       },
@@ -581,6 +571,10 @@
         "command": "codeQL.copyVersion",
         "title": "CodeQL: Copy Version Information"
       },
+      {
+        "command": "codeQLLanguageSelection.setSelectedItem",
+        "title": "Select"
+      },
       {
         "command": "codeQLQueries.runLocalQueryFromQueriesPanel",
         "title": "Run local query",
@@ -732,6 +726,10 @@
         "command": "codeQL.clearCache",
         "title": "CodeQL: Clear Cache"
       },
+      {
+        "command": "codeQL.trimCache",
+        "title": "CodeQL: Trim Cache"
+      },
       {
         "command": "codeQL.installPackDependencies",
         "title": "CodeQL: Install Pack Dependencies"
@@ -764,78 +762,6 @@
         "command": "codeQLDatabases.addDatabaseSource",
         "title": "Add Database Source to Workspace"
       },
-      {
-        "command": "codeQLDatabases.displayAllLanguages",
-        "title": "All languages"
-      },
-      {
-        "command": "codeQLDatabases.displayAllLanguagesSelected",
-        "title": "All languages (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displayCpp",
-        "title": "C/C++"
-      },
-      {
-        "command": "codeQLDatabases.displayCppSelected",
-        "title": "C/C++ (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displayCsharp",
-        "title": "C#"
-      },
-      {
-        "command": "codeQLDatabases.displayCsharpSelected",
-        "title": "C# (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displayGo",
-        "title": "Go"
-      },
-      {
-        "command": "codeQLDatabases.displayGoSelected",
-        "title": "Go (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displayJava",
-        "title": "Java/Kotlin"
-      },
-      {
-        "command": "codeQLDatabases.displayJavaSelected",
-        "title": "Java/Kotlin (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displayJavascript",
-        "title": "JavaScript/TypeScript"
-      },
-      {
-        "command": "codeQLDatabases.displayJavascriptSelected",
-        "title": "JavaScript/TypeScript (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displayPython",
-        "title": "Python"
-      },
-      {
-        "command": "codeQLDatabases.displayPythonSelected",
-        "title": "Python (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displayRuby",
-        "title": "Ruby"
-      },
-      {
-        "command": "codeQLDatabases.displayRubySelected",
-        "title": "Ruby (selected)"
-      },
-      {
-        "command": "codeQLDatabases.displaySwift",
-        "title": "Swift"
-      },
-      {
-        "command": "codeQLDatabases.displaySwiftSelected",
-        "title": "Swift (selected)"
-      },
       {
         "command": "codeQL.chooseDatabaseFolder",
         "title": "CodeQL: Choose Database from Folder"
@@ -856,6 +782,10 @@
         "command": "codeQLDatabases.sortByName",
         "title": "Sort by Name"
       },
+      {
+        "command": "codeQLDatabases.sortByLanguage",
+        "title": "Sort by Language"
+      },
       {
         "command": "codeQLDatabases.sortByDateAdded",
         "title": "Sort by Date Added"
@@ -1082,14 +1012,14 @@
           "group": "1_databases@0"
         },
         {
-          "command": "codeQLDatabases.sortByDateAdded",
+          "command": "codeQLDatabases.sortByLanguage",
           "when": "view == codeQLDatabases",
           "group": "1_databases@1"
         },
         {
-          "submenu": "codeQLDatabases.languages",
+          "command": "codeQLDatabases.sortByDateAdded",
-          "when": "view == codeQLDatabases && config.codeQL.canary && config.codeQL.showLanguageFilter",
+          "when": "view == codeQLDatabases",
-          "group": "2_databases@0"
+          "group": "1_databases@2"
         },
         {
           "command": "codeQLQueries.createQuery",
@@ -1163,6 +1093,11 @@
           "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/",
           "group": "2_qlContextMenu@1"
         },
+        {
+          "command": "codeQLLanguageSelection.setSelectedItem",
+          "when": "view == codeQLLanguageSelection && viewItem =~ /canBeSelected/",
+          "group": "inline"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "group": "inline",
@@ -1511,6 +1446,10 @@
         {
           "command": "codeQL.openModelEditor"
         },
+        {
+          "command": "codeQLLanguageSelection.setSelectedItem",
+          "when": "false"
+        },
         {
           "command": "codeQLQueries.runLocalQueryContextMenu",
           "when": "false"
@@ -1579,6 +1518,10 @@
           "command": "codeQLDatabases.sortByName",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.sortByLanguage",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.sortByDateAdded",
           "when": "false"
@@ -1611,78 +1554,6 @@
           "command": "codeQLDatabases.upgradeDatabase",
           "when": "false"
         },
-        {
-          "command": "codeQLDatabases.displayAllLanguages",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayAllLanguagesSelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayCpp",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayCppSelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayCsharp",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayCsharpSelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayGo",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayGoSelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayJava",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayJavaSelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayJavascript",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayJavascriptSelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayPython",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayPythonSelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayRuby",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displayRubySelected",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displaySwift",
-          "when": "false"
-        },
-        {
-          "command": "codeQLDatabases.displaySwiftSelected",
-          "when": "false"
-        },
         {
           "command": "codeQLQueryHistory.openQueryContextMenu",
           "when": "false"
@@ -1815,10 +1686,6 @@
           "command": "codeQL.mockGitHubApiServer.unloadScenario",
           "when": "config.codeQL.mockGitHubApiServer.enabled && codeQL.mockGitHubApiServer.scenarioLoaded"
         },
-        {
-          "command": "codeQL.createQuery",
-          "when": "config.codeQL.codespacesTemplate || config.codeQL.canary && config.codeQL.queriesPanel"
-        },
         {
           "command": "codeQLTests.acceptOutputContextTestItem",
           "when": "false"
@@ -1826,6 +1693,10 @@
         {
           "command": "codeQL.gotoQLContextEditor",
           "when": "false"
+        },
+        {
+          "command": "codeQL.trimCache",
+          "when": "codeql.supportsTrimCache"
         }
       ],
       "editor/context": [
@@ -1877,88 +1748,8 @@
           "command": "codeQL.gotoQLContextEditor",
           "when": "editorLangId == ql-summary && config.codeQL.canary"
         }
-      ],
-      "codeQLDatabases.languages": [
-        {
-          "command": "codeQLDatabases.displayAllLanguages",
-          "when": "codeQLDatabases.languageFilter"
-        },
-        {
-          "command": "codeQLDatabases.displayAllLanguagesSelected",
-          "when": "!codeQLDatabases.languageFilter"
-        },
-        {
-          "command": "codeQLDatabases.displayCpp",
-          "when": "codeQLDatabases.languageFilter != cpp"
-        },
-        {
-          "command": "codeQLDatabases.displayCppSelected",
-          "when": "codeQLDatabases.languageFilter == cpp"
-        },
-        {
-          "command": "codeQLDatabases.displayCsharp",
-          "when": "codeQLDatabases.languageFilter != csharp"
-        },
-        {
-          "command": "codeQLDatabases.displayCsharpSelected",
-          "when": "codeQLDatabases.languageFilter == csharp"
-        },
-        {
-          "command": "codeQLDatabases.displayGo",
-          "when": "codeQLDatabases.languageFilter != go"
-        },
-        {
-          "command": "codeQLDatabases.displayGoSelected",
-          "when": "codeQLDatabases.languageFilter == go"
-        },
-        {
-          "command": "codeQLDatabases.displayJava",
-          "when": "codeQLDatabases.languageFilter != java"
-        },
-        {
-          "command": "codeQLDatabases.displayJavaSelected",
-          "when": "codeQLDatabases.languageFilter == java"
-        },
-        {
-          "command": "codeQLDatabases.displayJavascript",
-          "when": "codeQLDatabases.languageFilter != javascript"
-        },
-        {
-          "command": "codeQLDatabases.displayJavascriptSelected",
-          "when": "codeQLDatabases.languageFilter == javascript"
-        },
-        {
-          "command": "codeQLDatabases.displayPython",
-          "when": "codeQLDatabases.languageFilter != python"
-        },
-        {
-          "command": "codeQLDatabases.displayPythonSelected",
-          "when": "codeQLDatabases.languageFilter == python"
-        },
-        {
-          "command": "codeQLDatabases.displayRuby",
-          "when": "codeQLDatabases.languageFilter != ruby"
-        },
-        {
-          "command": "codeQLDatabases.displayRubySelected",
-          "when": "codeQLDatabases.languageFilter == ruby"
-        },
-        {
-          "command": "codeQLDatabases.displaySwift",
-          "when": "codeQLDatabases.languageFilter != swift"
-        },
-        {
-          "command": "codeQLDatabases.displaySwiftSelected",
-          "when": "codeQLDatabases.languageFilter == swift"
-        }
       ]
     },
-    "submenus": [
-      {
-        "id": "codeQLDatabases.languages",
-        "label": "Languages"
-      }
-    ],
     "viewsContainers": {
       "activitybar": [
         {
@@ -1977,14 +1768,17 @@
     },
     "views": {
       "ql-container": [
+        {
+          "id": "codeQLLanguageSelection",
+          "name": "Language"
+        },
         {
           "id": "codeQLDatabases",
           "name": "Databases"
         },
         {
           "id": "codeQLQueries",
-          "name": "Queries",
+          "name": "Queries"
-          "when": "config.codeQL.canary && config.codeQL.queriesPanel"
         },
         {
           "id": "codeQLVariantAnalysisRepositories",
@@ -2094,7 +1888,7 @@
     "fs-extra": "^11.1.1",
     "immutable": "^4.0.0",
     "js-yaml": "^4.1.0",
-    "msw": "^0.0.0-fetch.rc-20",
+    "msw": "^2.0.0",
     "nanoid": "^5.0.1",
     "node-fetch": "^2.6.7",
     "p-queue": "^7.4.1",
@@ -2125,6 +1919,7 @@
     "@faker-js/faker": "^8.0.2",
     "@github/markdownlint-github": "^0.3.0",
     "@octokit/plugin-throttling": "^8.0.0",
+    "@storybook/addon-a11y": "^7.4.6",
     "@storybook/addon-actions": "^7.1.0",
     "@storybook/addon-essentials": "^7.1.0",
     "@storybook/addon-interactions": "^7.1.0",
@@ -2195,7 +1990,7 @@
     "jest": "^29.0.3",
     "jest-environment-jsdom": "^29.0.3",
     "jest-runner-vscode": "^3.0.1",
-    "lint-staged": "^14.0.0",
+    "lint-staged": "^15.0.2",
     "markdownlint-cli2": "^0.6.0",
     "markdownlint-cli2-formatter-pretty": "^0.0.4",
     "mini-css-extract-plugin": "^2.6.1",

extensions/ql-vscode/scripts/source-map.ts

@@ -115,21 +115,35 @@ async function extractSourceMap() {
   }
 
   if (stacktrace.includes("at")) {
-    const rawSourceMaps = new Map<string, RawSourceMap>();
+    const rawSourceMaps = new Map<string, RawSourceMap | null>();
 
     const mappedStacktrace = await replaceAsync(
       stacktrace,
       stackLineRegex,
       async (match, name, file, line, column) => {
         if (!rawSourceMaps.has(file)) {
-          const rawSourceMap: RawSourceMap = await readJSON(
+          try {
-            resolve(sourceMapsDirectory, `${basename(file)}.map`),
+            const rawSourceMap: RawSourceMap = await readJSON(
-          );
+              resolve(sourceMapsDirectory, `${basename(file)}.map`),
-          rawSourceMaps.set(file, rawSourceMap);
+            );
+            rawSourceMaps.set(file, rawSourceMap);
+          } catch (e: unknown) {
+            // If the file is not found, we will not decode it and not try reading this source map again
+            if (e instanceof Error && "code" in e && e.code === "ENOENT") {
+              rawSourceMaps.set(file, null);
+            } else {
+              throw e;
+            }
+          }
+        }
+
+        const sourceMap = rawSourceMaps.get(file);
+        if (!sourceMap) {
+          return match;
         }
 
         const originalPosition = await SourceMapConsumer.with(
-          rawSourceMaps.get(file) as RawSourceMap,
+          sourceMap,
           null,
           async function (consumer) {
             return consumer.originalPositionFor({

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -10,7 +10,11 @@ import tk from "tree-kill";
 import { promisify } from "util";
 import { CancellationToken, Disposable, Uri } from "vscode";
 
-import { BQRSInfo, DecodedBqrsChunk } from "../common/bqrs-cli-types";
+import {
+  BQRSInfo,
+  DecodedBqrs,
+  DecodedBqrsChunk,
+} from "../common/bqrs-cli-types";
 import { allowCanaryQueryServer, CliConfig } from "../config";
 import {
   DistributionProvider,
@@ -1040,6 +1044,18 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Gets all results from a bqrs.
+   * @param bqrsPath The path to the bqrs.
+   */
+  async bqrsDecodeAll(bqrsPath: string): Promise<DecodedBqrs> {
+    return await this.runJsonCodeQlCliCommand<DecodedBqrs>(
+      ["bqrs", "decode"],
+      [bqrsPath],
+      "Reading all bqrs data",
+    );
+  }
+
   async runInterpretCommand(
     format: string,
     additonalArgs: string[],
@@ -1244,11 +1260,13 @@ export class CodeQLCliServer implements Disposable {
    * @param additionalPacks A list of directories to search for qlpacks.
    * @param extensionPacksOnly Whether to only search for extension packs. If true, only extension packs will
    *    be returned. If false, all packs will be returned.
+   * @param kind Whether to only search for qlpacks with a certain kind.
    * @returns A dictionary mapping qlpack name to the directory it comes from
    */
   async resolveQlpacks(
     additionalPacks: string[],
     extensionPacksOnly = false,
+    kind?: "query" | "library" | "all",
   ): Promise<QlpacksInfo> {
     const args = this.getAdditionalPacksArg(additionalPacks);
     if (extensionPacksOnly) {
@@ -1259,6 +1277,8 @@ export class CodeQLCliServer implements Disposable {
         return {};
       }
       args.push("--kind", "extension", "--no-recursive");
+    } else if (kind) {
+      args.push("--kind", kind);
     }
 
     return this.runJsonCodeQlCliCommand<QlpacksInfo>(
@@ -1492,6 +1512,13 @@ export class CodeQLCliServer implements Disposable {
             CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
           ) >= 0,
         );
+        await this.app.commands.execute(
+          "setContext",
+          "codeql.supportsTrimCache",
+          newVersion.compare(
+            CliVersionConstraint.CLI_VERSION_WITH_TRIM_CACHE,
+          ) >= 0,
+        );
       } catch (e) {
         this._versionChangedListeners.forEach((listener) =>
           listener(undefined),
@@ -1755,6 +1782,12 @@ export class CliVersionConstraint {
     "2.14.0",
   );
 
+  /**
+   * CLI version where the query server supports the `evaluation/trimCache` method
+   * with `codeql database cleanup --mode=trim` semantics.
+   */
+  public static CLI_VERSION_WITH_TRIM_CACHE = new SemVer("2.15.1");
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }

extensions/ql-vscode/src/codeql-cli/query-language.ts

@@ -1,6 +1,10 @@
 import { CodeQLCliServer } from "./cli";
 import { Uri, window } from "vscode";
-import { isQueryLanguage, QueryLanguage } from "../common/query-language";
+import {
+  getLanguageDisplayName,
+  isQueryLanguage,
+  QueryLanguage,
+} from "../common/query-language";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { extLogger } from "../common/logging/vscode";
 import { UserCancellationException } from "../common/vscode/progress";
@@ -46,14 +50,22 @@ export async function askForLanguage(
   cliServer: CodeQLCliServer,
   throwOnEmpty = true,
 ): Promise<QueryLanguage | undefined> {
-  const language = await window.showQuickPick(
+  const supportedLanguages = await cliServer.getSupportedLanguages();
-    await cliServer.getSupportedLanguages(),
+
-    {
+  const items = supportedLanguages
-      placeHolder: "Select target language for your query",
+    .filter((language) => isQueryLanguage(language))
-      ignoreFocusOut: true,
+    .map((language) => ({
-    },
+      label: getLanguageDisplayName(language),
-  );
+      description: language,
-  if (!language) {
+      language,
+    }))
+    .sort((a, b) => a.label.localeCompare(b.label));
+
+  const selectedItem = await window.showQuickPick(items, {
+    placeHolder: "Select target language for your query",
+    ignoreFocusOut: true,
+  });
+  if (!selectedItem) {
     // This only happens if the user cancels the quick pick.
     if (throwOnEmpty) {
       throw new UserCancellationException("Cancelled.");
@@ -66,6 +78,8 @@ export async function askForLanguage(
     return undefined;
   }
 
+  const language = selectedItem.language;
+
   if (!isQueryLanguage(language)) {
     void showAndLogErrorMessage(
       extLogger,

extensions/ql-vscode/src/common/bqrs-cli-types.ts

@@ -121,3 +121,5 @@ export interface DecodedBqrsChunk {
   next?: number;
   columns: BqrsColumn[];
 }
+
+export type DecodedBqrs = Record<string, DecodedBqrsChunk>;

extensions/ql-vscode/src/common/commands.ts

@@ -12,6 +12,8 @@ import type {
 } from "../variant-analysis/shared/variant-analysis";
 import type { QLDebugConfiguration } from "../debugger/debug-configuration";
 import type { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
+import type { LanguageSelectionTreeViewItem } from "../language-selection-panel/language-selection-data-provider";
+import type { Method, Usage } from "../model-editor/method";
 
 // A command function matching the signature that VS Code calls when
 // a command is invoked from a context menu on a TreeView with
@@ -198,6 +200,13 @@ export type QueryHistoryCommands = {
   "codeQL.exportSelectedVariantAnalysisResults": () => Promise<void>;
 };
 
+// Commands user for the language selector panel
+export type LanguageSelectionCommands = {
+  "codeQLLanguageSelection.setSelectedItem": (
+    item: LanguageSelectionTreeViewItem,
+  ) => Promise<void>;
+};
+
 // Commands used for the local databases panel
 export type LocalDatabasesCommands = {
   // Command palette commands
@@ -207,6 +216,7 @@ export type LocalDatabasesCommands = {
   "codeQL.chooseDatabaseGithub": () => Promise<void>;
   "codeQL.upgradeCurrentDatabase": () => Promise<void>;
   "codeQL.clearCache": () => Promise<void>;
+  "codeQL.trimCache": () => Promise<void>;
 
   // Explorer context menu
   "codeQL.setCurrentDatabase": (uri: Uri) => Promise<void>;
@@ -217,25 +227,8 @@ export type LocalDatabasesCommands = {
   "codeQLDatabases.chooseDatabaseInternet": () => Promise<void>;
   "codeQLDatabases.chooseDatabaseGithub": () => Promise<void>;
   "codeQLDatabases.sortByName": () => Promise<void>;
+  "codeQLDatabases.sortByLanguage": () => Promise<void>;
   "codeQLDatabases.sortByDateAdded": () => Promise<void>;
-  "codeQLDatabases.displayAllLanguages": () => Promise<void>;
-  "codeQLDatabases.displayCpp": () => Promise<void>;
-  "codeQLDatabases.displayCsharp": () => Promise<void>;
-  "codeQLDatabases.displayGo": () => Promise<void>;
-  "codeQLDatabases.displayJava": () => Promise<void>;
-  "codeQLDatabases.displayJavascript": () => Promise<void>;
-  "codeQLDatabases.displayPython": () => Promise<void>;
-  "codeQLDatabases.displayRuby": () => Promise<void>;
-  "codeQLDatabases.displaySwift": () => Promise<void>;
-  "codeQLDatabases.displayAllLanguagesSelected": () => Promise<void>;
-  "codeQLDatabases.displayCppSelected": () => Promise<void>;
-  "codeQLDatabases.displayCsharpSelected": () => Promise<void>;
-  "codeQLDatabases.displayGoSelected": () => Promise<void>;
-  "codeQLDatabases.displayJavaSelected": () => Promise<void>;
-  "codeQLDatabases.displayJavascriptSelected": () => Promise<void>;
-  "codeQLDatabases.displayPythonSelected": () => Promise<void>;
-  "codeQLDatabases.displayRubySelected": () => Promise<void>;
-  "codeQLDatabases.displaySwiftSelected": () => Promise<void>;
 
   // Database panel context menu
   "codeQLDatabases.setCurrentDatabase": (
@@ -324,7 +317,8 @@ export type ModelEditorCommands = {
   "codeQL.openModelEditor": () => Promise<void>;
   "codeQL.openModelEditorFromModelingPanel": () => Promise<void>;
   "codeQLModelEditor.jumpToMethod": (
-    methodSignature: string,
+    method: Method,
+    usage: Usage,
     databaseItem: DatabaseItem,
   ) => Promise<void>;
 };
@@ -359,6 +353,7 @@ export type AllExtensionCommands = BaseCommands &
   QueryEditorCommands &
   ResultsViewCommands &
   QueryHistoryCommands &
+  LanguageSelectionCommands &
   LocalDatabasesCommands &
   DebuggerCommands &
   VariantAnalysisCommands &

extensions/ql-vscode/src/common/errors.ts

@@ -22,6 +22,14 @@ export class RedactableError extends Error {
       .join("");
   }
 
+  public get fullMessageWithStack(): string {
+    if (!this.stack) {
+      return this.fullMessage;
+    }
+
+    return `${this.fullMessage}\n${this.stack}`;
+  }
+
   public get redactedMessage(): string {
     return this.strings
       .map((s, i) => s + (this.hasValue(i) ? this.getRedactedValue(i) : ""))

extensions/ql-vscode/src/common/interface-types.ts

@@ -517,8 +517,7 @@ interface SetModifiedMethodsMessage {
 
 interface SetInProgressMethodsMessage {
   t: "setInProgressMethods";
-  packageName: string;
+  methods: string[];
-  inProgressMethods: string[];
 }
 
 interface SwitchModeMessage {
@@ -572,11 +571,6 @@ interface HideModeledMethodsMessage {
   hideModeledMethods: boolean;
 }
 
-interface SetModeledMethodMessage {
-  t: "setModeledMethod";
-  method: ModeledMethod;
-}
-
 interface SetMultipleModeledMethodsMessage {
   t: "setMultipleModeledMethods";
   methodSignature: string;
@@ -588,6 +582,11 @@ interface SetInModelingModeMessage {
   inModelingMode: boolean;
 }
 
+interface SetInProgressMessage {
+  t: "setInProgress";
+  inProgress: boolean;
+}
+
 interface RevealMethodMessage {
   t: "revealMethod";
   methodSignature: string;
@@ -614,7 +613,7 @@ export type FromModelEditorMessage =
   | StopGeneratingMethodsFromLlmMessage
   | ModelDependencyMessage
   | HideModeledMethodsMessage
-  | SetModeledMethodMessage;
+  | SetMultipleModeledMethodsMessage;
 
 interface RevealInEditorMessage {
   t: "revealInModelEditor";
@@ -627,7 +626,7 @@ interface StartModelingMessage {
 
 export type FromMethodModelingMessage =
   | CommonFromViewMessages
-  | SetModeledMethodMessage
+  | SetMultipleModeledMethodsMessage
   | RevealInEditorMessage
   | StartModelingMessage;
 
@@ -651,6 +650,7 @@ interface SetSelectedMethodMessage {
   method: Method;
   modeledMethods: ModeledMethod[];
   isModified: boolean;
+  isInProgress: boolean;
 }
 
 export type ToMethodModelingMessage =
@@ -659,4 +659,5 @@ export type ToMethodModelingMessage =
   | SetMultipleModeledMethodsMessage
   | SetMethodModifiedMessage
   | SetSelectedMethodMessage
-  | SetInModelingModeMessage;
+  | SetInModelingModeMessage
+  | SetInProgressMessage;

extensions/ql-vscode/src/common/logging/notifications.ts

@@ -112,5 +112,5 @@ export async function showAndLogExceptionWithTelemetry(
   options: ShowAndLogExceptionOptions = {},
 ): Promise<void> {
   telemetry?.sendError(error, options.extraTelemetryProperties);
-  return showAndLogErrorMessage(logger, error.fullMessage, options);
+  return showAndLogErrorMessage(logger, error.fullMessageWithStack, options);
 }

extensions/ql-vscode/src/common/mock-gh-api/recorder.ts

@@ -112,11 +112,14 @@ export class Recorder extends DisposableObject {
     return scenarioDirectory;
   }
 
-  private async onResponseBypass(
+  private async onResponseBypass({
-    response: Response,
+    response,
-    request: Request,
+    request,
-    _requestId: string,
+  }: {
-  ): Promise<void> {
+    response: Response;
+    request: Request;
+    requestId: string;
+  }): Promise<void> {
     if (request.headers.has("x-vscode-codeql-msw-bypass")) {
       return;
     }

extensions/ql-vscode/src/common/mock-gh-api/request-handlers.ts

@@ -1,6 +1,6 @@
 import { join } from "path";
 import { readdir, readJson, readFile } from "fs-extra";
-import { RequestHandler, rest } from "msw";
+import { http, RequestHandler } from "msw";
 import {
   GitHubApiRequest,
   isAutoModelRequest,
@@ -94,7 +94,7 @@ function createGetRepoRequestHandler(
 
   const getRepoRequest = getRepoRequests[0];
 
-  return rest.get(`${baseUrl}/repos/:owner/:name`, () => {
+  return http.get(`${baseUrl}/repos/:owner/:name`, () => {
     return jsonResponse(getRepoRequest.response.body, {
       status: getRepoRequest.response.status,
     });
@@ -114,7 +114,7 @@ function createSubmitVariantAnalysisRequestHandler(
 
   const getRepoRequest = submitVariantAnalysisRequests[0];
 
-  return rest.post(
+  return http.post(
     `${baseUrl}/repositories/:controllerRepoId/code-scanning/codeql/variant-analyses`,
     () => {
       return jsonResponse(getRepoRequest.response.body, {
@@ -135,7 +135,7 @@ function createGetVariantAnalysisRequestHandler(
   // During the lifetime of a variant analysis run, there are multiple requests
   // to get the variant analysis. We need to return different responses for each
   // request, so keep an index of the request and return the appropriate response.
-  return rest.get(
+  return http.get(
     `${baseUrl}/repositories/:controllerRepoId/code-scanning/codeql/variant-analyses/:variantAnalysisId`,
     () => {
       const request = getVariantAnalysisRequests[requestIndex];
@@ -159,7 +159,7 @@ function createGetVariantAnalysisRepoRequestHandler(
     isGetVariantAnalysisRepoRequest,
   );
 
-  return rest.get(
+  return http.get(
     `${baseUrl}/repositories/:controllerRepoId/code-scanning/codeql/variant-analyses/:variantAnalysisId/repositories/:repoId`,
     ({ request, params }) => {
       const scenarioRequest = getVariantAnalysisRepoRequests.find(
@@ -183,7 +183,7 @@ function createGetVariantAnalysisRepoResultRequestHandler(
     isGetVariantAnalysisRepoResultRequest,
   );
 
-  return rest.get(
+  return http.get(
     "https://objects-origin.githubusercontent.com/codeql-query-console/codeql-variant-analysis-repo-tasks/:variantAnalysisId/:repoId/*",
     ({ request, params }) => {
       const scenarioRequest = getVariantAnalysisRepoResultRequests.find(
@@ -216,7 +216,7 @@ function createCodeSearchRequestHandler(
   // During a code search, there are multiple request to get pages of results. We
   // need to return different responses for each request, so keep an index of the
   // request and return the appropriate response.
-  return rest.get(`${baseUrl}/search/code`, () => {
+  return http.get(`${baseUrl}/search/code`, () => {
     const request = codeSearchRequests[requestIndex];
 
     if (requestIndex < codeSearchRequests.length - 1) {
@@ -239,7 +239,7 @@ function createAutoModelRequestHandler(
   // During automodeling there can be multiple API requests for each batch
   // of candidates we want to model. We need to return different responses for each request,
   // so keep an index of the request and return the appropriate response.
-  return rest.post(
+  return http.post(
     `${baseUrl}/repos/github/codeql/code-scanning/codeql/auto-model`,
     () => {
       const request = autoModelRequests[requestIndex];

extensions/ql-vscode/src/common/mutable.ts

@@ -0,0 +1,6 @@
+/**
+ * Remove all readonly modifiers from a type.
+ */
+export type Mutable<T> = {
+  -readonly [P in keyof T]: T[P];
+};

extensions/ql-vscode/src/common/readonly.ts

@@ -0,0 +1,14 @@
+export type DeepReadonly<T> = T extends Array<infer R>
+  ? DeepReadonlyArray<R>
+  : // eslint-disable-next-line @typescript-eslint/ban-types
+  T extends Function
+  ? T
+  : T extends object
+  ? DeepReadonlyObject<T>
+  : T;
+
+interface DeepReadonlyArray<T> extends ReadonlyArray<DeepReadonly<T>> {}
+
+type DeepReadonlyObject<T> = {
+  readonly [P in keyof T]: DeepReadonly<T[P]>;
+};

extensions/ql-vscode/src/common/sarif-utils.ts

@@ -1,6 +1,7 @@
 import * as Sarif from "sarif";
 import type { HighlightedRegion } from "../variant-analysis/shared/analysis-result";
 import { ResolvableLocationValue } from "../common/bqrs-cli-types";
+import { isEmptyPath } from "./bqrs-utils";
 
 export interface SarifLink {
   dest: number;
@@ -111,6 +112,9 @@ export function parseSarifLocation(
     return { hint: "no artifact location" };
   if (physicalLocation.artifactLocation.uri === undefined)
     return { hint: "artifact location has no uri" };
+  if (isEmptyPath(physicalLocation.artifactLocation.uri)) {
+    return { hint: "artifact location has empty uri" };
+  }
 
   // This is not necessarily really an absolute uri; it could either be a
   // file uri or a relative uri.

extensions/ql-vscode/src/common/vscode/abstract-webview-view-provider.ts

@@ -3,6 +3,7 @@ import { Uri, WebviewViewProvider } from "vscode";
 import { WebviewKind, WebviewMessage, getHtmlForWebview } from "./webview-html";
 import { Disposable } from "../disposable-object";
 import { App } from "../app";
+import { DeepReadonly } from "../readonly";
 
 export abstract class AbstractWebviewViewProvider<
   ToMessage extends WebviewMessage,
@@ -53,7 +54,7 @@ export abstract class AbstractWebviewViewProvider<
     return this.webviewView?.visible ?? false;
   }
 
-  protected async postMessage(msg: ToMessage): Promise<void> {
+  protected async postMessage(msg: DeepReadonly<ToMessage>): Promise<void> {
     await this.webviewView?.webview.postMessage(msg);
   }
 

extensions/ql-vscode/src/common/vscode/abstract-webview.ts

@@ -12,6 +12,7 @@ import { App } from "../app";
 import { Disposable } from "../disposable-object";
 import { tmpDir } from "../../tmp-dir";
 import { getHtmlForWebview, WebviewMessage, WebviewKind } from "./webview-html";
+import { DeepReadonly } from "../readonly";
 
 export type WebviewPanelConfig = {
   viewId: string;
@@ -146,7 +147,7 @@ export abstract class AbstractWebview<
     this.panelLoadedCallBacks = [];
   }
 
-  protected async postMessage(msg: ToMessage): Promise<boolean> {
+  protected async postMessage(msg: DeepReadonly<ToMessage>): Promise<boolean> {
     const panel = await this.getPanel();
     return panel.webview.postMessage(msg);
   }

extensions/ql-vscode/src/common/vscode/commands.ts

@@ -6,11 +6,7 @@ import {
   showAndLogExceptionWithTelemetry,
 } from "../logging";
 import { extLogger } from "../logging/vscode";
-import {
+import { asError, getErrorMessage } from "../../common/helpers-pure";
-  asError,
-  getErrorMessage,
-  getErrorStack,
-} from "../../common/helpers-pure";
 import { redactableError } from "../../common/errors";
 import { UserCancellationException } from "./progress";
 import { telemetryListener } from "./telemetry";
@@ -66,10 +62,7 @@ export function registerCommandWithErrorHandling(
         }
       } else {
         // Include the full stack in the error log only.
-        const errorStack = getErrorStack(e);
+        const fullMessage = errorMessage.fullMessageWithStack;
-        const fullMessage = errorStack
-          ? `${errorMessage.fullMessage}\n${errorStack}`
-          : errorMessage.fullMessage;
         void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
           fullMessage,
           extraTelemetryProperties: {

extensions/ql-vscode/src/common/vscode/workspace-folders.ts

@@ -43,7 +43,9 @@ export function getFirstWorkspaceFolder() {
   const workspaceFolders = getOnDiskWorkspaceFolders();
 
   if (!workspaceFolders || workspaceFolders.length === 0) {
-    throw new Error("No workspace folders found");
+    throw new Error(
+      "No workspace folders found. Please open a folder or workspace in VS Code.",
+    );
   }
 
   const firstFolderFsPath = workspaceFolders[0];

extensions/ql-vscode/src/config.ts

@@ -641,12 +641,32 @@ export function isCodespacesTemplate() {
   return !!CODESPACES_TEMPLATE.getValue<boolean>();
 }
 
+// Deprecated after v1.9.4. Can be removed in a few versions.
 const DATABASE_DOWNLOAD_SETTING = new Setting("databaseDownload", ROOT_SETTING);
+const DEPRECATED_ALLOW_HTTP_SETTING = new Setting(
+  "allowHttp",
+  DATABASE_DOWNLOAD_SETTING,
+);
 
-const ALLOW_HTTP_SETTING = new Setting("allowHttp", DATABASE_DOWNLOAD_SETTING);
+const ADDING_DATABASES_SETTING = new Setting("addingDatabases", ROOT_SETTING);
+
+const ALLOW_HTTP_SETTING = new Setting("allowHttp", ADDING_DATABASES_SETTING);
 
 export function allowHttp(): boolean {
-  return ALLOW_HTTP_SETTING.getValue<boolean>() || false;
+  return (
+    ALLOW_HTTP_SETTING.getValue<boolean>() ||
+    DEPRECATED_ALLOW_HTTP_SETTING.getValue<boolean>() ||
+    false
+  );
+}
+
+const ADD_DATABASE_SOURCE_TO_WORKSPACE_SETTING = new Setting(
+  "addDatabaseSourceToWorkspace",
+  ADDING_DATABASES_SETTING,
+);
+
+export function addDatabaseSourceToWorkspace(): boolean {
+  return ADD_DATABASE_SOURCE_TO_WORKSPACE_SETTING.getValue<boolean>() || false;
 }
 
 /**
@@ -690,26 +710,26 @@ export async function setAutogenerateQlPacks(choice: AutogenerateQLPacks) {
   );
 }
 
-/**
- * A flag indicating whether to show the queries panel in the QL view container.
- */
-const QUERIES_PANEL = new Setting("queriesPanel", ROOT_SETTING);
-
-export function showQueriesPanel(): boolean {
-  return !!QUERIES_PANEL.getValue<boolean>();
-}
-
 const MODEL_SETTING = new Setting("model", ROOT_SETTING);
 const FLOW_GENERATION = new Setting("flowGeneration", MODEL_SETTING);
 const LLM_GENERATION = new Setting("llmGeneration", MODEL_SETTING);
+const LLM_GENERATION_BATCH_SIZE = new Setting(
+  "llmGenerationBatchSize",
+  MODEL_SETTING,
+);
+const LLM_GENERATION_DEV_ENDPOINT = new Setting(
+  "llmGenerationDevEndpoint",
+  MODEL_SETTING,
+);
 const EXTENSIONS_DIRECTORY = new Setting("extensionsDirectory", MODEL_SETTING);
-const SHOW_MULTIPLE_MODELS = new Setting("showMultipleModels", MODEL_SETTING);
+const ENABLE_RUBY = new Setting("enableRuby", MODEL_SETTING);
 
 export interface ModelConfig {
   flowGeneration: boolean;
   llmGeneration: boolean;
   getExtensionsDirectory(languageId: string): string | undefined;
   showMultipleModels: boolean;
+  enableRuby: boolean;
 }
 
 export class ModelConfigListener extends ConfigListener implements ModelConfig {
@@ -725,6 +745,22 @@ export class ModelConfigListener extends ConfigListener implements ModelConfig {
     return !!LLM_GENERATION.getValue<boolean>();
   }
 
+  /**
+   * Limits the number of candidates we send to the model in each request to avoid long requests.
+   * Note that the model may return fewer than this number of candidates.
+   */
+  public get llmGenerationBatchSize(): number {
+    return LLM_GENERATION_BATCH_SIZE.getValue<number | null>() || 5;
+  }
+
+  /**
+   * The URL of the endpoint to use for LLM generation. This should only be set
+   * if you want to test against a dev server.
+   */
+  public get llmGenerationDevEndpoint(): string | undefined {
+    return LLM_GENERATION_DEV_ENDPOINT.getValue<string | undefined>();
+  }
+
   public getExtensionsDirectory(languageId: string): string | undefined {
     return EXTENSIONS_DIRECTORY.getValue<string>({
       languageId,
@@ -732,6 +768,10 @@ export class ModelConfigListener extends ConfigListener implements ModelConfig {
   }
 
   public get showMultipleModels(): boolean {
-    return !!SHOW_MULTIPLE_MODELS.getValue<boolean>();
+    return isCanary();
+  }
+
+  public get enableRuby(): boolean {
+    return !!ENABLE_RUBY.getValue<boolean>();
   }
 }

extensions/ql-vscode/src/databases/code-search-api.ts

@@ -1,22 +1,24 @@
 import { throttling } from "@octokit/plugin-throttling";
 import { Octokit } from "@octokit/rest";
-import { Progress, CancellationToken } from "vscode";
+import { CancellationToken } from "vscode";
 import { Credentials } from "../common/authentication";
 import { BaseLogger } from "../common/logging";
 import { AppOctokit } from "../common/octokit";
+import {
+  ProgressCallback,
+  UserCancellationException,
+} from "../common/vscode/progress";
 
 export async function getCodeSearchRepositories(
   query: string,
-  progress: Progress<{
+  progress: ProgressCallback,
-    message?: string | undefined;
-    increment?: number | undefined;
-  }>,
   token: CancellationToken,
   credentials: Credentials,
   logger: BaseLogger,
 ): Promise<string[]> {
-  let nwos: string[] = [];
+  const nwos: string[] = [];
   const octokit = await provideOctokitWithThrottling(credentials, logger);
+  let i = 0;
 
   for await (const response of octokit.paginate.iterator(
     octokit.rest.search.code,
@@ -25,17 +27,19 @@ export async function getCodeSearchRepositories(
       per_page: 100,
     },
   )) {
+    i++;
     nwos.push(...response.data.map((item) => item.repository.full_name));
-    // calculate progress bar: 80% of the progress bar is used for the code search
+    const totalNumberOfResultPages = Math.ceil(response.data.total_count / 100);
-    const totalNumberOfRequests = Math.ceil(response.data.total_count / 100);
+    const totalNumberOfRequests =
-    // Since we have a maximum of 1000 responses of the api, we can use a fixed increment whenever the totalNumberOfRequests would be greater than 10
+      totalNumberOfResultPages > 10 ? 10 : totalNumberOfResultPages;
-    const increment =
+    progress({
-      totalNumberOfRequests < 10 ? 80 / totalNumberOfRequests : 8;
+      maxStep: totalNumberOfRequests,
-    progress.report({ increment });
+      step: i,
+      message: "Sending API requests to get Code Search results.",
+    });
 
     if (token.isCancellationRequested) {
-      nwos = [];
+      throw new UserCancellationException("Code search cancelled.", true);
-      break;
     }
   }
 

extensions/ql-vscode/src/databases/config/db-config.ts

@@ -1,6 +1,8 @@
 // Contains models and consts for the data we want to store in the database config.
 // Changes to these models should be done carefully and account for backwards compatibility of data.
 
+import { DatabaseOrigin } from "../local-databases/database-origin";
+
 export const DB_CONFIG_VERSION = 1;
 
 export interface DbConfig {
@@ -88,6 +90,7 @@ export interface LocalDatabase {
   name: string;
   dateAdded: number;
   language: string;
+  origin: DatabaseOrigin;
   storagePath: string;
 }
 

extensions/ql-vscode/src/databases/database-fetcher.ts

@@ -29,9 +29,11 @@ import {
 } from "../common/github-url-identifier-helper";
 import { Credentials } from "../common/authentication";
 import { AppCommandManager } from "../common/commands";
-import { allowHttp } from "../config";
+import { addDatabaseSourceToWorkspace, allowHttp } from "../config";
 import { showAndLogInformationMessage } from "../common/logging";
 import { AppOctokit } from "../common/octokit";
+import { getLanguageDisplayName } from "../common/query-language";
+import { DatabaseOrigin } from "./local-databases/database-origin";
 
 /**
  * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
@@ -61,6 +63,10 @@ export async function promptImportInternetDatabase(
     databaseManager,
     storagePath,
     undefined,
+    {
+      type: "url",
+      url: databaseUrl,
+    },
     progress,
     cli,
   );
@@ -98,7 +104,7 @@ export async function promptImportGithubDatabase(
   cli?: CodeQLCliServer,
   language?: string,
   makeSelected = true,
-  addSourceArchiveFolder = true,
+  addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
 ): Promise<DatabaseItem | undefined> {
   const githubRepo = await askForGitHubRepo(progress);
   if (!githubRepo) {
@@ -177,7 +183,7 @@ export async function downloadGitHubDatabase(
   cli?: CodeQLCliServer,
   language?: string,
   makeSelected = true,
-  addSourceArchiveFolder = true,
+  addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
 ): Promise<DatabaseItem | undefined> {
   const nwo = getNwoFromGitHubUrl(githubRepo) || githubRepo;
   if (!isValidGitHubNwo(nwo)) {
@@ -198,7 +204,8 @@ export async function downloadGitHubDatabase(
     return;
   }
 
-  const { databaseUrl, name, owner } = result;
+  const { databaseUrl, name, owner, databaseId, databaseCreatedAt, commitOid } =
+    result;
 
   /**
    * The 'token' property of the token object returned by `octokit.auth()`.
@@ -220,6 +227,13 @@ export async function downloadGitHubDatabase(
     databaseManager,
     storagePath,
     `${owner}/${name}`,
+    {
+      type: "github",
+      repository: nwo,
+      databaseId,
+      databaseCreatedAt,
+      commitOid,
+    },
     progress,
     cli,
     makeSelected,
@@ -249,6 +263,10 @@ export async function importArchiveDatabase(
       databaseManager,
       storagePath,
       undefined,
+      {
+        type: "archive",
+        path: databaseUrl,
+      },
       progress,
       cli,
     );
@@ -281,6 +299,7 @@ export async function importArchiveDatabase(
  * @param databaseManager the DatabaseManager
  * @param storagePath where to store the unzipped database.
  * @param nameOverride a name for the database that overrides the default
+ * @param origin the origin of the database
  * @param progress callback to send progress messages to
  * @param makeSelected make the new database selected in the databases panel (default: true)
  * @param addSourceArchiveFolder whether to add a workspace folder containing the source archive to the workspace
@@ -291,10 +310,11 @@ async function databaseArchiveFetcher(
   databaseManager: DatabaseManager,
   storagePath: string,
   nameOverride: string | undefined,
+  origin: DatabaseOrigin,
   progress: ProgressCallback,
   cli?: CodeQLCliServer,
   makeSelected = true,
-  addSourceArchiveFolder = true,
+  addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
 ): Promise<DatabaseItem> {
   progress({
     message: "Getting database",
@@ -335,6 +355,7 @@ async function databaseArchiveFetcher(
 
     const item = await databaseManager.openDatabase(
       Uri.file(dbPath),
+      origin,
       makeSelected,
       nameOverride,
       {
@@ -475,7 +496,7 @@ async function checkForFailingResponse(
     return response;
   }
 
-  // An error downloading the database. Attempt to extract the resaon behind it.
+  // An error downloading the database. Attempt to extract the reason behind it.
   const text = await response.text();
   let msg: string;
   try {
@@ -532,16 +553,19 @@ export async function convertGithubNwoToDatabaseUrl(
       databaseUrl: string;
       owner: string;
       name: string;
+      databaseId: number;
+      databaseCreatedAt: string;
+      commitOid: string | null;
     }
   | undefined
 > {
   try {
     const [owner, repo] = nwo.split("/");
 
-    const response = await octokit.request(
+    const response = await octokit.rest.codeScanning.listCodeqlDatabases({
-      "GET /repos/:owner/:repo/code-scanning/codeql/databases",
+      owner,
-      { owner, repo },
+      repo,
-    );
+    });
 
     const languages = response.data.map((db: any) => db.language);
 
@@ -552,10 +576,20 @@ export async function convertGithubNwoToDatabaseUrl(
       }
     }
 
+    const databaseForLanguage = response.data.find(
+      (db: any) => db.language === language,
+    );
+    if (!databaseForLanguage) {
+      throw new Error(`No database found for language '${language}'`);
+    }
+
     return {
-      databaseUrl: `https://api.github.com/repos/${owner}/${repo}/code-scanning/codeql/databases/${language}`,
+      databaseUrl: databaseForLanguage.url,
       owner,
       name: repo,
+      databaseId: databaseForLanguage.id,
+      databaseCreatedAt: databaseForLanguage.created_at,
+      commitOid: databaseForLanguage.commit_oid ?? null,
     };
   } catch (e) {
     void extLogger.log(`Error: ${getErrorMessage(e)}`);
@@ -579,10 +613,23 @@ export async function promptForLanguage(
     return languages[0];
   }
 
-  return await window.showQuickPick(languages, {
+  const items = languages
+    .map((language) => ({
+      label: getLanguageDisplayName(language),
+      description: language,
+      language,
+    }))
+    .sort((a, b) => a.label.localeCompare(b.label));
+
+  const selectedItem = await window.showQuickPick(items, {
     placeHolder: "Select the database language to download:",
     ignoreFocusOut: true,
   });
+  if (!selectedItem) {
+    return undefined;
+  }
+
+  return selectedItem.language;
 }
 
 /**

extensions/ql-vscode/src/databases/db-item.ts

@@ -1,5 +1,7 @@
 // This file contains models that are used to represent the databases.
 
+import { DatabaseOrigin } from "./local-databases/database-origin";
+
 export enum DbItemKind {
   RootLocal = "RootLocal",
   LocalList = "LocalList",
@@ -38,6 +40,7 @@ export interface LocalDatabaseDbItem {
   databaseName: string;
   dateAdded: number;
   language: string;
+  origin: DatabaseOrigin;
   storagePath: string;
   parentListName?: string;
 }

extensions/ql-vscode/src/databases/db-tree-creator.ts

@@ -197,6 +197,7 @@ function createLocalDb(
     databaseName: db.name,
     dateAdded: db.dateAdded,
     language: db.language,
+    origin: db.origin,
     storagePath: db.storagePath,
     selected: !!selected,
     parentListName: listName,

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -51,12 +51,14 @@ import {
   createMultiSelectionCommand,
   createSingleSelectionCommand,
 } from "../common/vscode/selection-commands";
-import { QueryLanguage, tryGetQueryLanguage } from "../common/query-language";
+import { tryGetQueryLanguage } from "../common/query-language";
 import { LanguageContextStore } from "../language-context-store";
 
 enum SortOrder {
   NameAsc = "NameAsc",
   NameDesc = "NameDesc",
+  LanguageAsc = "LanguageAsc",
+  LanguageDesc = "LanguageDesc",
   DateAddedAsc = "DateAddedAsc",
   DateAddedDesc = "DateAddedDesc",
 }
@@ -155,6 +157,18 @@ class DatabaseTreeDataProvider
             return db1.name.localeCompare(db2.name, env.language);
           case SortOrder.NameDesc:
             return db2.name.localeCompare(db1.name, env.language);
+          case SortOrder.LanguageAsc:
+            return (
+              db1.language.localeCompare(db2.language, env.language) ||
+              // If the languages are the same, sort by name
+              db1.name.localeCompare(db2.name, env.language)
+            );
+          case SortOrder.LanguageDesc:
+            return (
+              db2.language.localeCompare(db1.language, env.language) ||
+              // If the languages are the same, sort by name
+              db2.name.localeCompare(db1.name, env.language)
+            );
           case SortOrder.DateAddedAsc:
             return (db1.dateAdded || 0) - (db2.dateAdded || 0);
           case SortOrder.DateAddedDesc:
@@ -218,7 +232,7 @@ export class DatabaseUI extends DisposableObject {
   public constructor(
     private app: App,
     private databaseManager: DatabaseManager,
-    private languageContext: LanguageContextStore,
+    languageContext: LanguageContextStore,
     private readonly queryServer: QueryRunner | undefined,
     private readonly storagePath: string,
     readonly extensionPath: string,
@@ -252,6 +266,7 @@ export class DatabaseUI extends DisposableObject {
       "codeQL.upgradeCurrentDatabase":
         this.handleUpgradeCurrentDatabase.bind(this),
       "codeQL.clearCache": this.handleClearCache.bind(this),
+      "codeQL.trimCache": this.handleTrimCache.bind(this),
       "codeQLDatabases.chooseDatabaseFolder":
         this.handleChooseDatabaseFolder.bind(this),
       "codeQLDatabases.chooseDatabaseArchive":
@@ -263,61 +278,8 @@ export class DatabaseUI extends DisposableObject {
       "codeQLDatabases.setCurrentDatabase":
         this.handleMakeCurrentDatabase.bind(this),
       "codeQLDatabases.sortByName": this.handleSortByName.bind(this),
+      "codeQLDatabases.sortByLanguage": this.handleSortByLanguage.bind(this),
       "codeQLDatabases.sortByDateAdded": this.handleSortByDateAdded.bind(this),
-      "codeQLDatabases.displayAllLanguages":
-        this.handleClearLanguageFilter.bind(this),
-      "codeQLDatabases.displayCpp": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Cpp,
-      ),
-      "codeQLDatabases.displayCsharp": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.CSharp,
-      ),
-      "codeQLDatabases.displayGo": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Go,
-      ),
-      "codeQLDatabases.displayJava": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Java,
-      ),
-      "codeQLDatabases.displayJavascript": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Javascript,
-      ),
-      "codeQLDatabases.displayPython": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Python,
-      ),
-      "codeQLDatabases.displayRuby": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Ruby,
-      ),
-      "codeQLDatabases.displaySwift": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Swift,
-      ),
-      "codeQLDatabases.displayAllLanguagesSelected":
-        this.handleClearLanguageFilter.bind(this),
-      "codeQLDatabases.displayCppSelected":
-        this.handleChangeLanguageFilter.bind(this, QueryLanguage.Cpp),
-      "codeQLDatabases.displayCsharpSelected":
-        this.handleChangeLanguageFilter.bind(this, QueryLanguage.CSharp),
-      "codeQLDatabases.displayGoSelected": this.handleChangeLanguageFilter.bind(
-        this,
-        QueryLanguage.Go,
-      ),
-      "codeQLDatabases.displayJavaSelected":
-        this.handleChangeLanguageFilter.bind(this, QueryLanguage.Java),
-      "codeQLDatabases.displayJavascriptSelected":
-        this.handleChangeLanguageFilter.bind(this, QueryLanguage.Javascript),
-      "codeQLDatabases.displayPythonSelected":
-        this.handleChangeLanguageFilter.bind(this, QueryLanguage.Python),
-      "codeQLDatabases.displayRubySelected":
-        this.handleChangeLanguageFilter.bind(this, QueryLanguage.Ruby),
-      "codeQLDatabases.displaySwiftSelected":
-        this.handleChangeLanguageFilter.bind(this, QueryLanguage.Swift),
       "codeQLDatabases.removeDatabase": createMultiSelectionCommand(
         this.handleRemoveDatabase.bind(this),
       ),
@@ -405,6 +367,9 @@ export class DatabaseUI extends DisposableObject {
 
               await this.databaseManager.openDatabase(
                 uri,
+                {
+                  type: "folder",
+                },
                 makeSelected,
                 nameOverride,
                 {
@@ -600,6 +565,14 @@ export class DatabaseUI extends DisposableObject {
     }
   }
 
+  private async handleSortByLanguage() {
+    if (this.treeDataProvider.sortOrder === SortOrder.LanguageAsc) {
+      this.treeDataProvider.sortOrder = SortOrder.LanguageDesc;
+    } else {
+      this.treeDataProvider.sortOrder = SortOrder.LanguageAsc;
+    }
+  }
+
   private async handleSortByDateAdded() {
     if (this.treeDataProvider.sortOrder === SortOrder.DateAddedAsc) {
       this.treeDataProvider.sortOrder = SortOrder.DateAddedDesc;
@@ -608,14 +581,6 @@ export class DatabaseUI extends DisposableObject {
     }
   }
 
-  private async handleClearLanguageFilter() {
-    await this.languageContext.clearLanguageContext();
-  }
-
-  private async handleChangeLanguageFilter(languageFilter: QueryLanguage) {
-    await this.languageContext.setLanguageContext(languageFilter);
-  }
-
   private async handleUpgradeCurrentDatabase(): Promise<void> {
     return withProgress(
       async (progress, token) => {
@@ -703,6 +668,25 @@ export class DatabaseUI extends DisposableObject {
     );
   }
 
+  private async handleTrimCache(): Promise<void> {
+    return withProgress(
+      async (_progress, token) => {
+        if (
+          this.queryServer !== undefined &&
+          this.databaseManager.currentDatabaseItem !== undefined
+        ) {
+          await this.queryServer.trimCacheInDatabase(
+            this.databaseManager.currentDatabaseItem,
+            token,
+          );
+        }
+      },
+      {
+        title: "Trimming cache",
+      },
+    );
+  }
+
   private async handleGetCurrentDatabase(): Promise<string | undefined> {
     const dbItem = await this.getDatabaseItemInternal(undefined);
     return dbItem?.databaseUri.fsPath;
@@ -723,7 +707,9 @@ export class DatabaseUI extends DisposableObject {
               this.queryServer?.cliServer,
             );
           } else {
-            await this.databaseManager.openDatabase(uri);
+            await this.databaseManager.openDatabase(uri, {
+              type: "folder",
+            });
           }
         } catch (e) {
           // rethrow and let this be handled by default error handling.
@@ -838,7 +824,9 @@ export class DatabaseUI extends DisposableObject {
         if (byFolder) {
           const fixedUri = await this.fixDbUri(uri);
           // we are selecting a database folder
-          return await this.databaseManager.openDatabase(fixedUri);
+          return await this.databaseManager.openDatabase(fixedUri, {
+            type: "folder",
+          });
         } else {
           // we are selecting a database archive. Must unzip into a workspace-controlled area
           // before importing.

extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts

@@ -14,6 +14,7 @@ import { isLikelyDatabaseRoot } from "./db-contents-heuristics";
 import { stat } from "fs-extra";
 import { containsPath, pathsEqual } from "../../common/files";
 import { DatabaseContents } from "./database-contents";
+import { DatabaseOrigin } from "./database-origin";
 
 export class DatabaseItemImpl implements DatabaseItem {
   // These are only public in the implementation, they are readonly in the interface
@@ -61,6 +62,10 @@ export class DatabaseItemImpl implements DatabaseItem {
     return this.options.dateAdded;
   }
 
+  public get origin(): DatabaseOrigin | undefined {
+    return this.options.origin;
+  }
+
   public resolveSourceFile(uriStr: string | undefined): vscode.Uri {
     const sourceArchive = this.sourceArchive;
     const uri = uriStr ? vscode.Uri.parse(uriStr, true) : undefined;

extensions/ql-vscode/src/databases/local-databases/database-item.ts

@@ -2,6 +2,7 @@ import vscode from "vscode";
 import * as cli from "../../codeql-cli/cli";
 import { DatabaseContents } from "./database-contents";
 import { DatabaseOptions } from "./database-options";
+import { DatabaseOrigin } from "./database-origin";
 
 /** An item in the list of available databases */
 export interface DatabaseItem {
@@ -25,6 +26,11 @@ export interface DatabaseItem {
    */
   readonly dateAdded: number | undefined;
 
+  /**
+   * The origin this database item was retrieved from or undefined if unknown.
+   */
+  readonly origin: DatabaseOrigin | undefined;
+
   /** If the database is invalid, describes why. */
   readonly error: Error | undefined;
 

extensions/ql-vscode/src/databases/local-databases/database-manager.ts

@@ -7,6 +7,7 @@ import { QueryRunner } from "../../query-server";
 import * as cli from "../../codeql-cli/cli";
 import { ProgressCallback, withProgress } from "../../common/vscode/progress";
 import {
+  addDatabaseSourceToWorkspace,
   getAutogenerateQlPacks,
   isCodespacesTemplate,
   setAutogenerateQlPacks,
@@ -19,7 +20,10 @@ import {
   getFirstWorkspaceFolder,
   isFolderAlreadyInWorkspace,
 } from "../../common/vscode/workspace-folders";
-import { isQueryLanguage } from "../../common/query-language";
+import {
+  isQueryLanguage,
+  tryGetQueryLanguage,
+} from "../../common/query-language";
 import { existsSync } from "fs";
 import { QlPackGenerator } from "../../local-queries/qlpack-generator";
 import { asError, getErrorMessage } from "../../common/helpers-pure";
@@ -30,6 +34,8 @@ import { containsPath } from "../../common/files";
 import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
 import { DatabaseResolver } from "./database-resolver";
 import { telemetryListener } from "../../common/vscode/telemetry";
+import { LanguageContextStore } from "../../language-context-store";
+import { DatabaseOrigin } from "./database-origin";
 
 /**
  * The name of the key in the workspaceState dictionary in which we
@@ -100,11 +106,25 @@ export class DatabaseManager extends DisposableObject {
     private readonly app: App,
     private readonly qs: QueryRunner,
     private readonly cli: cli.CodeQLCliServer,
+    private readonly languageContext: LanguageContextStore,
     public logger: Logger,
   ) {
     super();
 
     qs.onStart(this.reregisterDatabases.bind(this));
+
+    this.push(
+      this.languageContext.onLanguageContextChanged(async () => {
+        if (
+          this.currentDatabaseItem !== undefined &&
+          !this.languageContext.shouldInclude(
+            tryGetQueryLanguage(this.currentDatabaseItem.language),
+          )
+        ) {
+          await this.setCurrentDatabaseItem(undefined);
+        }
+      }),
+    );
   }
 
   /**
@@ -113,14 +133,19 @@ export class DatabaseManager extends DisposableObject {
    */
   public async openDatabase(
     uri: vscode.Uri,
+    origin: DatabaseOrigin | undefined,
     makeSelected = true,
     displayName?: string,
     {
       isTutorialDatabase = false,
-      addSourceArchiveFolder = true,
+      addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
     }: OpenDatabaseOptions = {},
   ): Promise<DatabaseItem> {
-    const databaseItem = await this.createDatabaseItem(uri, displayName);
+    const databaseItem = await this.createDatabaseItem(
+      uri,
+      origin,
+      displayName,
+    );
 
     return await this.addExistingDatabaseItem(
       databaseItem,
@@ -140,7 +165,7 @@ export class DatabaseManager extends DisposableObject {
     databaseItem: DatabaseItemImpl,
     makeSelected: boolean,
     isTutorialDatabase?: boolean,
-    addSourceArchiveFolder = true,
+    addSourceArchiveFolder = addDatabaseSourceToWorkspace(),
   ): Promise<DatabaseItem> {
     const existingItem = this.findDatabaseItem(databaseItem.databaseUri);
     if (existingItem !== undefined) {
@@ -171,6 +196,7 @@ export class DatabaseManager extends DisposableObject {
    */
   private async createDatabaseItem(
     uri: vscode.Uri,
+    origin: DatabaseOrigin | undefined,
     displayName: string | undefined,
   ): Promise<DatabaseItemImpl> {
     const contents = await DatabaseResolver.resolveDatabaseContents(uri);
@@ -179,6 +205,7 @@ export class DatabaseManager extends DisposableObject {
       displayName,
       dateAdded: Date.now(),
       language: await this.getPrimaryLanguage(uri.fsPath),
+      origin,
     };
     const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions);
 
@@ -194,6 +221,7 @@ export class DatabaseManager extends DisposableObject {
    */
   public async createOrOpenDatabaseItem(
     uri: vscode.Uri,
+    origin: DatabaseOrigin | undefined,
   ): Promise<DatabaseItem> {
     const existingItem = this.findDatabaseItem(uri);
     if (existingItem !== undefined) {
@@ -202,7 +230,7 @@ export class DatabaseManager extends DisposableObject {
     }
 
     // We don't add this to the list automatically, but the user can add it later.
-    return this.createDatabaseItem(uri, undefined);
+    return this.createDatabaseItem(uri, origin, undefined);
   }
 
   public async createSkeletonPacks(databaseItem: DatabaseItem) {
@@ -230,8 +258,10 @@ export class DatabaseManager extends DisposableObject {
     const firstWorkspaceFolder = getFirstWorkspaceFolder();
     const folderName = `codeql-custom-queries-${databaseItem.language}`;
 
+    const qlpackStoragePath = join(firstWorkspaceFolder, folderName);
+
     if (
-      existsSync(join(firstWorkspaceFolder, folderName)) ||
+      existsSync(qlpackStoragePath) ||
       isFolderAlreadyInWorkspace(folderName)
     ) {
       return;
@@ -245,7 +275,7 @@ export class DatabaseManager extends DisposableObject {
       `We've noticed you don't have a CodeQL pack available to analyze this database. Can we set up a query pack for you?`,
     );
 
-    if (answer === "No") {
+    if (answer === "No" || answer === undefined) {
       return;
     }
 
@@ -256,10 +286,10 @@ export class DatabaseManager extends DisposableObject {
 
     try {
       const qlPackGenerator = new QlPackGenerator(
-        folderName,
         databaseItem.language,
         this.cli,
-        firstWorkspaceFolder,
+        qlpackStoragePath,
+        qlpackStoragePath,
       );
       await qlPackGenerator.generate();
     } catch (e: unknown) {
@@ -335,6 +365,7 @@ export class DatabaseManager extends DisposableObject {
     let displayName: string | undefined = undefined;
     let dateAdded = undefined;
     let language = undefined;
+    let origin = undefined;
     if (state.options) {
       if (typeof state.options.displayName === "string") {
         displayName = state.options.displayName;
@@ -343,6 +374,7 @@ export class DatabaseManager extends DisposableObject {
         dateAdded = state.options.dateAdded;
       }
       language = state.options.language;
+      origin = state.options.origin;
     }
 
     const dbBaseUri = vscode.Uri.parse(state.uri, true);
@@ -355,6 +387,7 @@ export class DatabaseManager extends DisposableObject {
       displayName,
       dateAdded,
       language,
+      origin,
     };
     const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions);
 

extensions/ql-vscode/src/databases/local-databases/database-options.ts

@@ -1,10 +1,14 @@
+import { DatabaseOrigin } from "./database-origin";
+
 export interface DatabaseOptions {
   displayName?: string;
   dateAdded?: number | undefined;
   language?: string;
+  origin?: DatabaseOrigin;
 }
 
 export interface FullDatabaseOptions extends DatabaseOptions {
   dateAdded: number | undefined;
   language: string | undefined;
+  origin: DatabaseOrigin | undefined;
 }

extensions/ql-vscode/src/databases/local-databases/database-origin.ts

@@ -0,0 +1,32 @@
+interface DatabaseOriginFolder {
+  type: "folder";
+}
+
+interface DatabaseOriginArchive {
+  type: "archive";
+  path: string;
+}
+
+interface DatabaseOriginGitHub {
+  type: "github";
+  repository: string;
+  databaseId: number;
+  databaseCreatedAt: string;
+  commitOid: string | null;
+}
+
+interface DatabaseOriginInternet {
+  type: "url";
+  url: string;
+}
+
+interface DatabaseOriginDebugger {
+  type: "debugger";
+}
+
+export type DatabaseOrigin =
+  | DatabaseOriginFolder
+  | DatabaseOriginArchive
+  | DatabaseOriginGitHub
+  | DatabaseOriginInternet
+  | DatabaseOriginDebugger;

extensions/ql-vscode/src/databases/ui/db-panel.ts

@@ -1,5 +1,4 @@
 import {
-  ProgressLocation,
   QuickPickItem,
   TreeView,
   TreeViewExpansionEvent,
@@ -7,7 +6,10 @@ import {
   window,
   workspace,
 } from "vscode";
-import { UserCancellationException } from "../../common/vscode/progress";
+import {
+  UserCancellationException,
+  withProgress,
+} from "../../common/vscode/progress";
 import {
   getNwoFromGitHubUrl,
   isValidGitHubNwo,
@@ -34,10 +36,7 @@ import { DatabasePanelCommands } from "../../common/commands";
 import { App } from "../../common/app";
 import { QueryLanguage } from "../../common/query-language";
 import { getCodeSearchRepositories } from "../code-search-api";
-import {
+import { showAndLogErrorMessage } from "../../common/logging";
-  showAndLogErrorMessage,
-  showAndLogInformationMessage,
-} from "../../common/logging";
 
 export interface RemoteDatabaseQuickPickItem extends QuickPickItem {
   remoteDatabaseKind: string;
@@ -409,15 +408,8 @@ export class DbPanel extends DisposableObject {
       return;
     }
 
-    await window.withProgress(
+    await withProgress(
-      {
-        location: ProgressLocation.Notification,
-        title: "Searching for repositories... This might take a while",
-        cancellable: true,
-      },
       async (progress, token) => {
-        progress.report({ increment: 10 });
-
         const repositories = await getCodeSearchRepositories(
           `${codeSearchQuery} ${languagePrompt}`,
           progress,
@@ -426,18 +418,22 @@ export class DbPanel extends DisposableObject {
           this.app.logger,
         );
 
-        token.onCancellationRequested(() => {
+        if (token.isCancellationRequested) {
-          void showAndLogInformationMessage(
+          throw new UserCancellationException("Code search cancelled.", true);
-            this.app.logger,
+        }
-            "Code search cancelled",
+
-          );
+        progress({
-          return;
+          maxStep: 12,
+          step: 12,
+          message: "Processing results...",
         });
 
-        progress.report({ increment: 10, message: "Processing results..." });
-
         await this.dbManager.addNewRemoteReposToList(repositories, listName);
       },
+      {
+        title: "Searching for repositories...",
+        cancellable: true,
+      },
     );
   }
 

extensions/ql-vscode/src/debugger/debugger-ui.ts

@@ -105,7 +105,9 @@ class QLDebugAdapterTracker
     body: CodeQLProtocol.EvaluationStartedEvent["body"],
   ): Promise<void> {
     const dbUri = Uri.file(this.configuration.database);
-    const dbItem = await this.dbm.createOrOpenDatabaseItem(dbUri);
+    const dbItem = await this.dbm.createOrOpenDatabaseItem(dbUri, {
+      type: "debugger",
+    });
 
     // When cancellation is requested from the query history view, we just stop the debug session.
     const tokenSource = new CancellationTokenSource();

extensions/ql-vscode/src/extension.ts

@@ -136,6 +136,7 @@ import { NewQueryRunner, QueryRunner, QueryServerClient } from "./query-server";
 import { QueriesModule } from "./queries-panel/queries-module";
 import { OpenReferencedFileCodeLensProvider } from "./local-queries/open-referenced-file-code-lens-provider";
 import { LanguageContextStore } from "./language-context-store";
+import { LanguageSelectionPanel } from "./language-selection-panel/language-selection-panel";
 
 /**
  * extension.ts
@@ -768,17 +769,28 @@ async function activateWithInstalledDistribution(
     fsWatcher.onDidDelete(clearPackCache);
   }
 
+  void extLogger.log("Initializing language context.");
+  const languageContext = new LanguageContextStore(app);
+
+  void extLogger.log("Initializing language selector.");
+  const languageSelectionPanel = new LanguageSelectionPanel(languageContext);
+  ctx.subscriptions.push(languageSelectionPanel);
+
   void extLogger.log("Initializing database manager.");
-  const dbm = new DatabaseManager(ctx, app, qs, cliServer, extLogger);
+  const dbm = new DatabaseManager(
+    ctx,
+    app,
+    qs,
+    cliServer,
+    languageContext,
+    extLogger,
+  );
 
   // Let this run async.
   void dbm.loadPersistedState();
 
   ctx.subscriptions.push(dbm);
 
-  void extLogger.log("Initializing language context.");
-  const languageContext = new LanguageContextStore(app);
-
   void extLogger.log("Initializing database panel.");
   const databaseUI = new DatabaseUI(
     app,
@@ -790,7 +802,11 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(databaseUI);
 
-  QueriesModule.initialize(app, languageContext, cliServer);
+  const queriesModule = QueriesModule.initialize(
+    app,
+    languageContext,
+    cliServer,
+  );
 
   void extLogger.log("Initializing evaluator log viewer.");
   const evalLogViewer = new EvalLogViewer();
@@ -925,9 +941,14 @@ async function activateWithInstalledDistribution(
     databaseUI,
     localQueryResultsView,
     queryStorageDir,
+    languageContext,
   );
   ctx.subscriptions.push(localQueries);
 
+  queriesModule.onDidChangeSelection((event) =>
+    localQueries.setSelectedQueryTreeViewItems(event.selection),
+  );
+
   void extLogger.log("Initializing debugger factory.");
   ctx.subscriptions.push(
     new QLDebugAdapterDescriptorFactory(queryStorageDir, qs, localQueries),
@@ -1016,6 +1037,7 @@ async function activateWithInstalledDistribution(
     ...getPackagingCommands({
       cliServer,
     }),
+    ...languageSelectionPanel.getCommands(),
     ...modelEditorModule.getCommands(),
     ...evalLogViewer.getCommands(),
     ...summaryLanguageSupport.getCommands(),
@@ -1164,10 +1186,7 @@ function addUnhandledRejectionListener() {
       const message = redactableError(
         asError(error),
       )`Unhandled error: ${getErrorMessage(error)}`;
-      const stack = getErrorStack(error);
+      const fullMessage = message.fullMessageWithStack;
-      const fullMessage = stack
-        ? `Unhandled error: ${stack}`
-        : message.fullMessage;
 
       // Add a catch so that showAndLogExceptionWithTelemetry fails, we avoid
       // triggering "unhandledRejection" and avoid an infinite loop

extensions/ql-vscode/src/language-context-store.ts

@@ -43,7 +43,36 @@ export class LanguageContextStore extends DisposableObject {
     );
   }
 
+  /**
+   * This returns true if the given language should be included.
+   *
+   * That means that either the given language is selected or the "All" option is selected.
+   *
+   * @param language a query language or undefined if the language is unknown.
+   */
   public shouldInclude(language: QueryLanguage | undefined): boolean {
     return this.languageFilter === "All" || this.languageFilter === language;
   }
+
+  /**
+   * This returns true if the given language is selected.
+   *
+   * If no language is given then it returns true if the "All" option is selected.
+   *
+   * @param language a query language or undefined.
+   */
+  public isSelectedLanguage(language: QueryLanguage | undefined): boolean {
+    return (
+      (this.languageFilter === "All" && language === undefined) ||
+      this.languageFilter === language
+    );
+  }
+
+  public get selectedLanguage(): QueryLanguage | undefined {
+    if (this.languageFilter === "All") {
+      return undefined;
+    }
+
+    return this.languageFilter;
+  }
 }

extensions/ql-vscode/src/language-selection-panel/language-selection-data-provider.ts

@@ -0,0 +1,91 @@
+import { DisposableObject } from "../common/disposable-object";
+import { LanguageContextStore } from "../language-context-store";
+import {
+  Event,
+  EventEmitter,
+  ThemeIcon,
+  TreeDataProvider,
+  TreeItem,
+} from "vscode";
+import {
+  QueryLanguage,
+  getLanguageDisplayName,
+} from "../common/query-language";
+
+const ALL_LANGUAGE_SELECTION_OPTIONS = [
+  undefined, // All languages
+  QueryLanguage.Cpp,
+  QueryLanguage.CSharp,
+  QueryLanguage.Go,
+  QueryLanguage.Java,
+  QueryLanguage.Javascript,
+  QueryLanguage.Python,
+  QueryLanguage.Ruby,
+  QueryLanguage.Swift,
+];
+
+// A tree view items consisting of of a language (or undefined for all languages)
+// and a boolean indicating whether it is selected or not.
+export class LanguageSelectionTreeViewItem extends TreeItem {
+  constructor(
+    public readonly language: QueryLanguage | undefined,
+    public readonly selected: boolean = false,
+  ) {
+    const label = language ? getLanguageDisplayName(language) : "All languages";
+    super(label);
+
+    this.iconPath = selected ? new ThemeIcon("check") : undefined;
+    this.contextValue = selected ? undefined : "canBeSelected";
+  }
+}
+
+export class LanguageSelectionTreeDataProvider
+  extends DisposableObject
+  implements TreeDataProvider<LanguageSelectionTreeViewItem>
+{
+  private treeItems: LanguageSelectionTreeViewItem[];
+  private readonly onDidChangeTreeDataEmitter = this.push(
+    new EventEmitter<void>(),
+  );
+
+  public constructor(private readonly languageContext: LanguageContextStore) {
+    super();
+
+    this.treeItems = this.createTree();
+
+    // If the language context changes, we need to update the tree.
+    this.push(
+      this.languageContext.onLanguageContextChanged(() => {
+        this.treeItems = this.createTree();
+        this.onDidChangeTreeDataEmitter.fire();
+      }),
+    );
+  }
+
+  public get onDidChangeTreeData(): Event<void> {
+    return this.onDidChangeTreeDataEmitter.event;
+  }
+
+  public getTreeItem(item: LanguageSelectionTreeViewItem): TreeItem {
+    return item;
+  }
+
+  public getChildren(
+    item?: LanguageSelectionTreeViewItem,
+  ): LanguageSelectionTreeViewItem[] {
+    if (!item) {
+      return this.treeItems;
+    } else {
+      return [];
+    }
+  }
+
+  private createTree(): LanguageSelectionTreeViewItem[] {
+    return ALL_LANGUAGE_SELECTION_OPTIONS.map((language) => {
+      return new LanguageSelectionTreeViewItem(
+        language,
+        this.languageContext.isSelectedLanguage(language),
+      );
+    });
+  }
+}

extensions/ql-vscode/src/language-selection-panel/language-selection-panel.ts

@@ -0,0 +1,41 @@
+import { DisposableObject } from "../common/disposable-object";
+import { window } from "vscode";
+import {
+  LanguageSelectionTreeDataProvider,
+  LanguageSelectionTreeViewItem,
+} from "./language-selection-data-provider";
+import { LanguageContextStore } from "../language-context-store";
+import { LanguageSelectionCommands } from "../common/commands";
+
+// This panel allows the selection of a single language, that will
+// then filter all other relevant views (e.g. db panel, query history).
+export class LanguageSelectionPanel extends DisposableObject {
+  constructor(private readonly languageContext: LanguageContextStore) {
+    super();
+
+    const dataProvider = new LanguageSelectionTreeDataProvider(languageContext);
+    this.push(dataProvider);
+
+    const treeView = window.createTreeView("codeQLLanguageSelection", {
+      treeDataProvider: dataProvider,
+    });
+    this.push(treeView);
+  }
+
+  public getCommands(): LanguageSelectionCommands {
+    return {
+      "codeQLLanguageSelection.setSelectedItem":
+        this.handleSetSelectedLanguage.bind(this),
+    };
+  }
+
+  private async handleSetSelectedLanguage(
+    item: LanguageSelectionTreeViewItem,
+  ): Promise<void> {
+    if (item.language) {
+      await this.languageContext.setLanguageContext(item.language);
+    } else {
+      await this.languageContext.clearLanguageContext();
+    }
+  }
+}

extensions/ql-vscode/src/language-support/contextual/location-finder.ts

@@ -9,16 +9,19 @@ import {
   ResultSetSchema,
 } from "../../common/bqrs-cli-types";
 import { CodeQLCliServer } from "../../codeql-cli/cli";
-import { DatabaseManager, DatabaseItem } from "../../databases/local-databases";
+import { DatabaseItem, DatabaseManager } from "../../databases/local-databases";
 import { ProgressCallback } from "../../common/vscode/progress";
 import { KeyType } from "./key-type";
-import { resolveQueries, runContextualQuery } from "./query-resolver";
+import {
+  resolveContextualQlPacksForDatabase,
+  resolveContextualQueries,
+  runContextualQuery,
+} from "./query-resolver";
 import { CancellationToken, LocationLink, Uri } from "vscode";
 import { QueryOutputDir } from "../../run-queries-shared";
 import { QueryRunner } from "../../query-server";
 import { QueryResultType } from "../../query-server/new-messages";
 import { fileRangeFromURI } from "./file-range-from-uri";
-import { qlpackOfDatabase } from "../../local-queries";
 
 export const SELECT_QUERY_NAME = "#select";
 export const SELECTED_SOURCE_FILE = "selectedSourceFile";
@@ -63,11 +66,11 @@ export async function getLocationsForUriString(
     return [];
   }
 
-  const qlpack = await qlpackOfDatabase(cli, db);
+  const qlpack = await resolveContextualQlPacksForDatabase(cli, db);
   const templates = createTemplates(uri.pathWithinSourceArchive);
 
   const links: FullLocationLink[] = [];
-  for (const query of await resolveQueries(cli, qlpack, keyType)) {
+  for (const query of await resolveContextualQueries(cli, qlpack, keyType)) {
     const results = await runContextualQuery(
       query,
       db,

extensions/ql-vscode/src/language-support/contextual/query-resolver.ts

@@ -8,7 +8,10 @@ import {
 } from "./key-type";
 import { CodeQLCliServer } from "../../codeql-cli/cli";
 import { DatabaseItem } from "../../databases/local-databases";
-import { resolveQueriesByLanguagePack as resolveLocalQueries } from "../../local-queries/query-resolver";
+import {
+  qlpackOfDatabase,
+  resolveQueriesByLanguagePack as resolveLocalQueriesByLanguagePack,
+} from "../../local-queries/query-resolver";
 import { extLogger } from "../../common/logging/vscode";
 import { TeeLogger } from "../../common/logging";
 import { CancellationToken } from "vscode";
@@ -16,15 +19,56 @@ import { ProgressCallback } from "../../common/vscode/progress";
 import { CoreCompletedQuery, QueryRunner } from "../../query-server";
 import { createLockFileForStandardQuery } from "../../local-queries/standard-queries";
 
-export async function resolveQueries(
+/**
+ * This wil try to determine the qlpacks for a given database. If it can't find a matching
+ * dbscheme with downloaded packs, it will download the default packs instead.
+ *
+ * @param cli The CLI server to use
+ * @param databaseItem The database item to find the qlpacks for
+ */
+export async function resolveContextualQlPacksForDatabase(
+  cli: CodeQLCliServer,
+  databaseItem: DatabaseItem,
+): Promise<QlPacksForLanguage> {
+  try {
+    return await qlpackOfDatabase(cli, databaseItem);
+  } catch (e) {
+    // If we can't find the qlpacks for the database, use the defaults instead
+  }
+
+  const dbInfo = await cli.resolveDatabase(databaseItem.databaseUri.fsPath);
+  const primaryLanguage = dbInfo.languages?.[0];
+  if (!primaryLanguage) {
+    throw new Error("Unable to determine primary language of database");
+  }
+
+  const libraryPack = `codeql/${primaryLanguage}-all`;
+  const queryPack = `codeql/${primaryLanguage}-queries`;
+
+  await cli.packDownload([libraryPack, queryPack]);
+
+  // Return the default packs. If these weren't valid packs, the download would have failed.
+  return {
+    dbschemePack: libraryPack,
+    dbschemePackIsLibraryPack: true,
+    queryPack,
+  };
+}
+
+export async function resolveContextualQueries(
   cli: CodeQLCliServer,
   qlpacks: QlPacksForLanguage,
   keyType: KeyType,
 ): Promise<string[]> {
-  return resolveLocalQueries(cli, qlpacks, nameOfKeyType(keyType), {
+  return resolveLocalQueriesByLanguagePack(
-    kind: kindOfKeyType(keyType),
+    cli,
-    "tags contain": [tagOfKeyType(keyType)],
+    qlpacks,
-  });
+    nameOfKeyType(keyType),
+    {
+      kind: kindOfKeyType(keyType),
+      "tags contain": [tagOfKeyType(keyType)],
+    },
+  );
 }
 
 export async function runContextualQuery(

extensions/ql-vscode/src/language-support/contextual/template-provider.ts

@@ -23,11 +23,15 @@ import { KeyType } from "./key-type";
 import {
   FullLocationLink,
   getLocationsForUriString,
+  SELECTED_SOURCE_COLUMN,
   SELECTED_SOURCE_FILE,
   SELECTED_SOURCE_LINE,
-  SELECTED_SOURCE_COLUMN,
 } from "./location-finder";
-import { resolveQueries, runContextualQuery } from "./query-resolver";
+import {
+  resolveContextualQlPacksForDatabase,
+  resolveContextualQueries,
+  runContextualQuery,
+} from "./query-resolver";
 import {
   isCanary,
   NO_CACHE_AST_VIEWER,
@@ -35,7 +39,6 @@ import {
 } from "../../config";
 import { CoreCompletedQuery, QueryRunner } from "../../query-server";
 import { AstBuilder } from "../ast-viewer/ast-builder";
-import { qlpackOfDatabase } from "../../local-queries";
 import { MultiCancellationToken } from "../../common/vscode/multi-cancellation-token";
 
 /**
@@ -248,8 +251,8 @@ export class TemplatePrintAstProvider {
       throw new Error("Can't infer database from the provided source.");
     }
 
-    const qlpacks = await qlpackOfDatabase(this.cli, db);
+    const qlpacks = await resolveContextualQlPacksForDatabase(this.cli, db);
-    const queries = await resolveQueries(
+    const queries = await resolveContextualQueries(
       this.cli,
       qlpacks,
       KeyType.PrintAstQuery,
@@ -336,11 +339,11 @@ export class TemplatePrintCfgProvider {
       throw new Error("Can't infer database from the provided source.");
     }
 
-    const qlpack = await qlpackOfDatabase(this.cli, db);
+    const qlpack = await resolveContextualQlPacksForDatabase(this.cli, db);
     if (!qlpack) {
       throw new Error("Can't infer qlpack from database source archive.");
     }
-    const queries = await resolveQueries(
+    const queries = await resolveContextualQueries(
       this.cli,
       qlpack,
       KeyType.PrintCfgQuery,

extensions/ql-vscode/src/local-queries/index.ts

@@ -1,5 +1,6 @@
 export * from "./local-queries";
 export * from "./local-query-run";
+export * from "./query-constraints";
 export * from "./query-resolver";
 export * from "./quick-eval-code-lens-provider";
 export * from "./quick-query";

extensions/ql-vscode/src/local-queries/local-queries.ts

@@ -50,6 +50,7 @@ import { createMultiSelectionCommand } from "../common/vscode/selection-commands
 import { findLanguage } from "../codeql-cli/query-language";
 import type { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
 import { tryGetQueryLanguage } from "../common/query-language";
+import { LanguageContextStore } from "../language-context-store";
 
 interface DatabaseQuickPickItem extends QuickPickItem {
   databaseItem: DatabaseItem;
@@ -62,6 +63,8 @@ export enum QuickEvalType {
 }
 
 export class LocalQueries extends DisposableObject {
+  private selectedQueryTreeViewItems: readonly QueryTreeViewItem[] = [];
+
   public constructor(
     private readonly app: App,
     private readonly queryRunner: QueryRunner,
@@ -71,10 +74,17 @@ export class LocalQueries extends DisposableObject {
     private readonly databaseUI: DatabaseUI,
     private readonly localQueryResultsView: ResultsView,
     private readonly queryStorageDir: string,
+    private readonly languageContextStore: LanguageContextStore,
   ) {
     super();
   }
 
+  public setSelectedQueryTreeViewItems(
+    selection: readonly QueryTreeViewItem[],
+  ) {
+    this.selectedQueryTreeViewItems = selection;
+  }
+
   public getCommands(): LocalQueryCommands {
     return {
       "codeQL.runQuery": this.runQuery.bind(this),
@@ -323,13 +333,16 @@ export class LocalQueries extends DisposableObject {
         const credentials = isCanary() ? this.app.credentials : undefined;
         const contextStoragePath =
           this.app.workspaceStoragePath || this.app.globalStoragePath;
+        const language = this.languageContextStore.selectedLanguage;
         const skeletonQueryWizard = new SkeletonQueryWizard(
           this.cliServer,
           progress,
           credentials,
-          this.app.logger,
+          this.app,
           this.databaseManager,
           contextStoragePath,
+          this.selectedQueryTreeViewItems,
+          language,
         );
         await skeletonQueryWizard.execute();
       },
@@ -362,11 +375,15 @@ export class LocalQueries extends DisposableObject {
       );
     }
 
-    const initialInfo = await createInitialQueryInfo(selectedQuery, {
+    const initialInfo = await createInitialQueryInfo(
-      databaseUri: dbItem.databaseUri.toString(),
+      selectedQuery,
-      name: dbItem.name,
+      {
-      language: tryGetQueryLanguage(dbItem.language),
+        databaseUri: dbItem.databaseUri.toString(),
-    });
+        name: dbItem.name,
+        language: tryGetQueryLanguage(dbItem.language),
+      },
+      outputDir,
+    );
 
     // When cancellation is requested from the query history view, we just stop the debug session.
     const queryInfo = new LocalQueryInfo(initialInfo, tokenSource);

extensions/ql-vscode/src/local-queries/local-query-run.ts

@@ -97,6 +97,15 @@ export class LocalQueryRun {
    * Updates the UI in the case where query evaluation throws an exception.
    */
   public async fail(err: Error): Promise<void> {
+    const evalLogPaths = await this.summarizeEvalLog(
+      QueryResultType.OTHER_ERROR,
+      this.outputDir,
+      this.logger,
+    );
+    if (evalLogPaths !== undefined) {
+      this.queryInfo.setEvaluatorLogPaths(evalLogPaths);
+    }
+
     err.message = `Error running query: ${err.message}`;
     this.queryInfo.failureReason = err.message;
     await this.queryHistoryManager.refreshTreeView();

extensions/ql-vscode/src/local-queries/qlpack-generator.ts

@@ -1,35 +1,36 @@
-import { mkdir, writeFile } from "fs-extra";
+import { ensureDir, writeFile } from "fs-extra";
 import { dump } from "js-yaml";
-import { join } from "path";
+import { dirname, join } from "path";
 import { Uri } from "vscode";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { QueryLanguage } from "../common/query-language";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { basename } from "../common/path";
 
 export class QlPackGenerator {
-  private readonly qlpackName: string;
+  private qlpackName: string | undefined;
   private readonly qlpackVersion: string;
   private readonly header: string;
   private readonly qlpackFileName: string;
   private readonly folderUri: Uri;
 
   constructor(
-    private readonly folderName: string,
     private readonly queryLanguage: QueryLanguage,
     private readonly cliServer: CodeQLCliServer,
-    private readonly storagePath: string | undefined,
+    private readonly storagePath: string,
+    private readonly queryStoragePath: string,
+    private readonly includeFolderNameInQlpackName: boolean = false,
   ) {
-    if (this.storagePath === undefined) {
-      throw new Error("Workspace storage path is undefined");
-    }
-    this.qlpackName = `getting-started/codeql-extra-queries-${this.queryLanguage}`;
     this.qlpackVersion = "1.0.0";
     this.header = "# This is an automatically generated file.\n\n";
 
     this.qlpackFileName = "codeql-pack.yml";
-    this.folderUri = Uri.file(join(this.storagePath, this.folderName));
+    this.folderUri = Uri.file(this.storagePath);
   }
 
   public async generate() {
+    this.qlpackName = await this.determineQlpackName();
+
     // create QL pack folder and add to workspace
     await this.createWorkspaceFolder();
 
@@ -43,8 +44,39 @@ export class QlPackGenerator {
     await this.createCodeqlPackLockYaml();
   }
 
+  private async determineQlpackName(): Promise<string> {
+    let qlpackBaseName = `getting-started/codeql-extra-queries-${this.queryLanguage}`;
+    if (this.includeFolderNameInQlpackName) {
+      const folderBasename = basename(dirname(this.folderUri.fsPath));
+      if (
+        folderBasename.includes("codeql") ||
+        folderBasename.includes("queries")
+      ) {
+        // If the user has already included "codeql" or "queries" in the folder name, don't include it twice
+        qlpackBaseName = `getting-started/${folderBasename}-${this.queryLanguage}`;
+      } else {
+        qlpackBaseName = `getting-started/codeql-extra-queries-${folderBasename}-${this.queryLanguage}`;
+      }
+    }
+
+    const existingQlPacks = await this.cliServer.resolveQlpacks(
+      getOnDiskWorkspaceFolders(),
+    );
+    const existingQlPackNames = Object.keys(existingQlPacks);
+
+    let qlpackName = qlpackBaseName;
+    let i = 0;
+    while (existingQlPackNames.includes(qlpackName)) {
+      i++;
+
+      qlpackName = `${qlpackBaseName}-${i}`;
+    }
+
+    return qlpackName;
+  }
+
   private async createWorkspaceFolder() {
-    await mkdir(this.folderUri.fsPath);
+    await ensureDir(this.folderUri.fsPath);
   }
 
   private async createQlPackYaml() {
@@ -60,7 +92,7 @@ export class QlPackGenerator {
   }
 
   public async createExampleQlFile(fileName = "example.ql") {
-    const exampleQlFilePath = join(this.folderUri.fsPath, fileName);
+    const exampleQlFilePath = join(this.queryStoragePath, fileName);
 
     const exampleQl = `
 /**

extensions/ql-vscode/src/local-queries/query-constraints.ts

@@ -0,0 +1,7 @@
+export interface QueryConstraints {
+  kind?: string;
+  "tags contain"?: string[];
+  "tags contain all"?: string[];
+  "query filename"?: string;
+  "query path"?: string;
+}

extensions/ql-vscode/src/local-queries/query-resolver.ts

@@ -14,7 +14,13 @@ import { showAndLogExceptionWithTelemetry } from "../common/logging";
 import { extLogger } from "../common/logging/vscode";
 import { telemetryListener } from "../common/vscode/telemetry";
 import { SuiteInstruction } from "../packaging/suite-instruction";
+import { QueryConstraints } from "./query-constraints";
 
+/**
+ * Consider using `resolveContextualQlPacksForDatabase` instead.
+ * @param cli The CLI server instance to use.
+ * @param db The database to find the QLPack for.
+ */
 export async function qlpackOfDatabase(
   cli: Pick<CodeQLCliServer, "resolveQlpacks">,
   db: Pick<DatabaseItem, "contents">,
@@ -27,12 +33,6 @@ export async function qlpackOfDatabase(
   return await getQlPackForDbscheme(cli, dbscheme);
 }
 
-export interface QueryConstraints {
-  kind?: string;
-  "tags contain"?: string[];
-  "tags contain all"?: string[];
-}
-
 /**
  * Finds the queries with the specified kind and tags in a list of CodeQL packs.
  *
@@ -132,6 +132,14 @@ export async function resolveQueries(
       `tagged all of "${constraints["tags contain all"].join(" ")}"`,
     );
   }
+  if (constraints["query filename"] !== undefined) {
+    humanConstraints.push(
+      `with query filename "${constraints["query filename"]}"`,
+    );
+  }
+  if (constraints["query path"] !== undefined) {
+    humanConstraints.push(`with query path "${constraints["query path"]}"`);
+  }
 
   const joinedPacksToSearch = packsToSearch.join(", ");
   const error = redactableError`No ${name} queries (${humanConstraints.join(

extensions/ql-vscode/src/local-queries/skeleton-query-wizard.ts

@@ -1,19 +1,23 @@
-import { join } from "path";
+import { dirname, join } from "path";
-import { Uri, workspace, window as Window } from "vscode";
+import { Uri, window, window as Window, workspace } from "vscode";
 import { CodeQLCliServer } from "../codeql-cli/cli";
-import { BaseLogger } from "../common/logging";
+import { showAndLogExceptionWithTelemetry } from "../common/logging";
 import { Credentials } from "../common/authentication";
-import { QueryLanguage } from "../common/query-language";
+import {
+  getLanguageDisplayName,
+  QueryLanguage,
+} from "../common/query-language";
 import {
   getFirstWorkspaceFolder,
-  isFolderAlreadyInWorkspace,
+  getOnDiskWorkspaceFolders,
 } from "../common/vscode/workspace-folders";
-import { getErrorMessage } from "../common/helpers-pure";
+import { asError, getErrorMessage } from "../common/helpers-pure";
 import { QlPackGenerator } from "./qlpack-generator";
 import { DatabaseItem, DatabaseManager } from "../databases/local-databases";
 import {
   ProgressCallback,
   UserCancellationException,
+  withProgress,
 } from "../common/vscode/progress";
 import {
   askForGitHubRepo,
@@ -24,8 +28,16 @@ import {
   isCodespacesTemplate,
   setQlPackLocation,
 } from "../config";
-import { existsSync } from "fs-extra";
+import { lstat, pathExists, readFile } from "fs-extra";
 import { askForLanguage } from "../codeql-cli/query-language";
+import { showInformationMessageWithAction } from "../common/vscode/dialog";
+import { redactableError } from "../common/errors";
+import { App } from "../common/app";
+import { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
+import { containsPath, pathsEqual } from "../common/files";
+import { getQlPackPath } from "../common/ql";
+import { load } from "js-yaml";
+import { QlPackFile } from "../packaging/qlpack-file";
 
 type QueryLanguagesToDatabaseMap = Record<string, string>;
 
@@ -41,73 +53,139 @@ export const QUERY_LANGUAGE_TO_DATABASE_REPO: QueryLanguagesToDatabaseMap = {
 };
 
 export class SkeletonQueryWizard {
-  private language: QueryLanguage | undefined;
   private fileName = "example.ql";
   private qlPackStoragePath: string | undefined;
+  private queryStoragePath: string | undefined;
+  private downloadPromise: Promise<void> | undefined;
 
   constructor(
     private readonly cliServer: CodeQLCliServer,
     private readonly progress: ProgressCallback,
     private readonly credentials: Credentials | undefined,
-    private readonly logger: BaseLogger,
+    private readonly app: App,
     private readonly databaseManager: DatabaseManager,
     private readonly databaseStoragePath: string | undefined,
+    private readonly selectedItems: readonly QueryTreeViewItem[],
+    private language: QueryLanguage | undefined = undefined,
   ) {}
 
-  private get folderName() {
+  /**
-    return `codeql-custom-queries-${this.language}`;
+   * Wait for the download process to complete by waiting for the user to select
+   * either "Download database" or closing the dialog. This is used for testing.
+   */
+  public async waitForDownload() {
+    if (this.downloadPromise) {
+      await this.downloadPromise;
+    }
   }
 
   public async execute() {
-    // show quick pick to choose language
+    // First try detecting the language based on the existing qlpacks.
-    this.language = await this.chooseLanguage();
+    // This will override the selected language if there is an existing query pack.
+    const detectedLanguage = await this.detectLanguage();
+    if (detectedLanguage) {
+      this.language = detectedLanguage;
+    }
+
+    // If no existing qlpack was found, we need to ask the user for the language
+    if (!this.language) {
+      // show quick pick to choose language
+      this.language = await this.chooseLanguage();
+    }
+
     if (!this.language) {
       return;
     }
 
-    this.qlPackStoragePath = await this.determineStoragePath();
+    let createSkeletonQueryPack: boolean = false;
 
-    const skeletonPackAlreadyExists =
+    if (!this.qlPackStoragePath) {
-      existsSync(join(this.qlPackStoragePath, this.folderName)) ||
+      // This means no existing qlpack was detected in the selected folder, so we need
-      isFolderAlreadyInWorkspace(this.folderName);
+      // to find a new location to store the qlpack. This new location could potentially
+      // already exist.
+      const storagePath = await this.determineStoragePath();
+      this.qlPackStoragePath = join(
+        storagePath,
+        `codeql-custom-queries-${this.language}`,
+      );
 
-    if (skeletonPackAlreadyExists) {
+      // Try to detect if there is already a qlpack in this location. We will assume that
-      // just create a new example query file in skeleton QL pack
+      // the user hasn't changed the language of the qlpack.
-      await this.createExampleFile();
+      const qlPackPath = await getQlPackPath(this.qlPackStoragePath);
+
+      // If we are creating or using a qlpack in the user's selected folder, we will also
+      // create the query in that folder
+      this.queryStoragePath = this.qlPackStoragePath;
+
+      createSkeletonQueryPack = qlPackPath === undefined;
     } else {
+      // A query pack was detected in the selected folder or one of its ancestors, so we
+      // directly use the selected folder as the storage path for the query.
+      this.queryStoragePath = await this.determineStoragePathFromSelection();
+    }
+
+    if (createSkeletonQueryPack) {
       // generate a new skeleton QL pack with query file
       await this.createQlPack();
+    } else {
+      // just create a new example query file in skeleton QL pack
+      await this.createExampleFile();
+    }
+
+    // open the query file
+    try {
+      await this.openExampleFile();
+    } catch (e: unknown) {
+      void this.app.logger.log(
+        `Could not open example query file: ${getErrorMessage(e)}`,
+      );
     }
 
     // select existing database for language or download a new one
     await this.selectOrDownloadDatabase();
-
-    // open a query file
-
-    try {
-      await this.openExampleFile();
-    } catch (e: unknown) {
-      void this.logger.log(
-        `Could not open example query file: ${getErrorMessage(e)}`,
-      );
-    }
   }
 
   private async openExampleFile() {
-    if (this.folderName === undefined || this.qlPackStoragePath === undefined) {
+    if (this.queryStoragePath === undefined) {
       throw new Error("Path to folder is undefined");
     }
 
-    const queryFileUri = Uri.file(
+    const queryFileUri = Uri.file(join(this.queryStoragePath, this.fileName));
-      join(this.qlPackStoragePath, this.folderName, this.fileName),
-    );
 
     void workspace.openTextDocument(queryFileUri).then((doc) => {
-      void Window.showTextDocument(doc);
+      void Window.showTextDocument(doc, {
+        preview: false,
+      });
     });
   }
 
-  public async determineStoragePath() {
+  public async determineStoragePath(): Promise<string> {
+    if (this.selectedItems.length === 0) {
+      return this.determineRootStoragePath();
+    }
+
+    return this.determineStoragePathFromSelection();
+  }
+
+  private async determineStoragePathFromSelection(): Promise<string> {
+    // Just like VS Code's "New File" command, if the user has selected multiple files/folders in the queries panel,
+    // we will create the new file in the same folder as the first selected item.
+    // See https://github.com/microsoft/vscode/blob/a8b7239d0311d4915b57c837972baf4b01394491/src/vs/workbench/contrib/files/browser/fileActions.ts#L893-L900
+    const selectedItem = this.selectedItems[0];
+
+    const path = selectedItem.path;
+
+    // We use stat to protect against outdated query tree items
+    const fileStat = await lstat(path);
+
+    if (fileStat.isDirectory()) {
+      return path;
+    }
+
+    return dirname(path);
+  }
+
+  public async determineRootStoragePath() {
     const firstStorageFolder = getFirstWorkspaceFolder();
 
     if (isCodespacesTemplate()) {
@@ -116,7 +194,7 @@ export class SkeletonQueryWizard {
 
     let storageFolder = getQlPackLocation();
 
-    if (storageFolder === undefined || !existsSync(storageFolder)) {
+    if (storageFolder === undefined || !(await pathExists(storageFolder))) {
       storageFolder = await Window.showInputBox({
         title:
           "Please choose a folder in which to create your new query pack. You can change this in the extension settings.",
@@ -129,7 +207,7 @@ export class SkeletonQueryWizard {
       throw new UserCancellationException("No storage folder entered.");
     }
 
-    if (!existsSync(storageFolder)) {
+    if (!(await pathExists(storageFolder))) {
       throw new UserCancellationException(
         "Invalid folder. Must be a folder that already exists.",
       );
@@ -139,6 +217,62 @@ export class SkeletonQueryWizard {
     return storageFolder;
   }
 
+  private async detectLanguage(): Promise<QueryLanguage | undefined> {
+    if (this.selectedItems.length < 1) {
+      return undefined;
+    }
+
+    this.progress({
+      message: "Resolving existing query packs",
+      step: 1,
+      maxStep: 3,
+    });
+
+    const storagePath = await this.determineStoragePathFromSelection();
+
+    const queryPacks = await this.cliServer.resolveQlpacks(
+      getOnDiskWorkspaceFolders(),
+      false,
+      "query",
+    );
+
+    const matchingQueryPacks = Object.values(queryPacks)
+      .map((paths) => paths.find((path) => containsPath(path, storagePath)))
+      .filter((path): path is string => path !== undefined)
+      // Find the longest matching path
+      .sort((a, b) => b.length - a.length);
+
+    if (matchingQueryPacks.length === 0) {
+      return undefined;
+    }
+
+    const matchingQueryPackPath = matchingQueryPacks[0];
+
+    const qlPackPath = await getQlPackPath(matchingQueryPackPath);
+    if (!qlPackPath) {
+      return undefined;
+    }
+
+    const qlPack = load(await readFile(qlPackPath, "utf8")) as
+      | QlPackFile
+      | undefined;
+    const dependencies = qlPack?.dependencies;
+    if (!dependencies || typeof dependencies !== "object") {
+      return;
+    }
+
+    const matchingLanguages = Object.values(QueryLanguage).filter(
+      (language) => `codeql/${language}-all` in dependencies,
+    );
+    if (matchingLanguages.length !== 1) {
+      return undefined;
+    }
+
+    this.qlPackStoragePath = matchingQueryPackPath;
+
+    return matchingLanguages[0];
+  }
+
   private async chooseLanguage() {
     this.progress({
       message: "Choose language",
@@ -150,13 +284,6 @@ export class SkeletonQueryWizard {
   }
 
   private async createQlPack() {
-    if (this.folderName === undefined) {
-      throw new Error("Folder name is undefined");
-    }
-    if (this.language === undefined) {
-      throw new Error("Language is undefined");
-    }
-
     this.progress({
       message: "Creating skeleton QL pack around query",
       step: 2,
@@ -164,29 +291,17 @@ export class SkeletonQueryWizard {
     });
 
     try {
-      const qlPackGenerator = new QlPackGenerator(
+      const qlPackGenerator = this.createQlPackGenerator();
-        this.folderName,
-        this.language,
-        this.cliServer,
-        this.qlPackStoragePath,
-      );
 
       await qlPackGenerator.generate();
     } catch (e: unknown) {
-      void this.logger.log(
+      void this.app.logger.log(
         `Could not create skeleton QL pack: ${getErrorMessage(e)}`,
       );
     }
   }
 
   private async createExampleFile() {
-    if (this.folderName === undefined) {
-      throw new Error("Folder name is undefined");
-    }
-    if (this.language === undefined) {
-      throw new Error("Language is undefined");
-    }
-
     this.progress({
       message:
         "Skeleton query pack already exists. Creating additional query example file.",
@@ -195,29 +310,29 @@ export class SkeletonQueryWizard {
     });
 
     try {
-      const qlPackGenerator = new QlPackGenerator(
+      const qlPackGenerator = this.createQlPackGenerator();
-        this.folderName,
-        this.language,
-        this.cliServer,
-        this.qlPackStoragePath,
-      );
 
-      this.fileName = await this.determineNextFileName(this.folderName);
+      this.fileName = await this.determineNextFileName();
       await qlPackGenerator.createExampleQlFile(this.fileName);
     } catch (e: unknown) {
-      void this.logger.log(
+      void this.app.logger.log(
-        `Could not create skeleton QL pack: ${getErrorMessage(e)}`,
+        `Could not create query example file: ${getErrorMessage(e)}`,
       );
     }
   }
 
-  private async determineNextFileName(folderName: string): Promise<string> {
+  private async determineNextFileName(): Promise<string> {
-    if (this.qlPackStoragePath === undefined) {
+    if (this.queryStoragePath === undefined) {
-      throw new Error("QL Pack storage path is undefined");
+      throw new Error("Query storage path is undefined");
     }
 
-    const folderUri = Uri.file(join(this.qlPackStoragePath, folderName));
+    const folderUri = Uri.file(this.queryStoragePath);
     const files = await workspace.fs.readDirectory(folderUri);
+    // If the example.ql file doesn't exist yet, use that name
+    if (!files.some(([filename, _fileType]) => filename === this.fileName)) {
+      return this.fileName;
+    }
+
     const qlFiles = files.filter(([filename, _fileType]) =>
       filename.match(/^example[0-9]*\.ql$/),
     );
@@ -225,11 +340,43 @@ export class SkeletonQueryWizard {
     return `example${qlFiles.length + 1}.ql`;
   }
 
-  private async downloadDatabase() {
+  private async promptDownloadDatabase() {
-    if (this.qlPackStoragePath === undefined) {
+    if (this.language === undefined) {
-      throw new Error("QL Pack storage path is undefined");
+      throw new Error("Language is undefined");
     }
 
+    const openFileLink = this.openFileMarkdownLink;
+
+    const displayLanguage = getLanguageDisplayName(this.language);
+    const action = await showInformationMessageWithAction(
+      `New CodeQL query for ${displayLanguage} ${openFileLink} created, but no CodeQL databases for ${displayLanguage} were detected in your workspace. Would you like to download a CodeQL database for ${displayLanguage} to analyze with ${openFileLink}?`,
+      "Download database",
+    );
+
+    if (action) {
+      void withProgress(async (progress) => {
+        try {
+          await this.downloadDatabase(progress);
+        } catch (e: unknown) {
+          if (e instanceof UserCancellationException) {
+            return;
+          }
+
+          void showAndLogExceptionWithTelemetry(
+            this.app.logger,
+            this.app.telemetry,
+            redactableError(
+              asError(e),
+            )`An error occurred while downloading the GitHub repository: ${getErrorMessage(
+              e,
+            )}`,
+          );
+        }
+      });
+    }
+  }
+
+  private async downloadDatabase(progress: ProgressCallback) {
     if (this.databaseStoragePath === undefined) {
       throw new Error("Database storage path is undefined");
     }
@@ -238,10 +385,10 @@ export class SkeletonQueryWizard {
       throw new Error("Language is undefined");
     }
 
-    this.progress({
+    progress({
       message: "Downloading database",
-      step: 3,
+      step: 1,
-      maxStep: 3,
+      maxStep: 2,
     });
 
     const githubRepoNwo = QUERY_LANGUAGE_TO_DATABASE_REPO[this.language];
@@ -256,7 +403,7 @@ export class SkeletonQueryWizard {
       this.databaseManager,
       this.databaseStoragePath,
       this.credentials,
-      this.progress,
+      progress,
       this.cliServer,
       this.language,
     );
@@ -267,10 +414,6 @@ export class SkeletonQueryWizard {
       throw new Error("Language is undefined");
     }
 
-    if (this.qlPackStoragePath === undefined) {
-      throw new Error("QL Pack storage path is undefined");
-    }
-
     const existingDatabaseItem =
       await SkeletonQueryWizard.findExistingDatabaseItem(
         this.language,
@@ -278,14 +421,65 @@ export class SkeletonQueryWizard {
       );
 
     if (existingDatabaseItem) {
-      // select the found database
+      const openFileLink = this.openFileMarkdownLink;
-      await this.databaseManager.setCurrentDatabaseItem(existingDatabaseItem);
+
+      if (this.databaseManager.currentDatabaseItem !== existingDatabaseItem) {
+        // select the found database
+        await this.databaseManager.setCurrentDatabaseItem(existingDatabaseItem);
+
+        const displayLanguage = getLanguageDisplayName(this.language);
+        void window.showInformationMessage(
+          `New CodeQL query for ${displayLanguage} ${openFileLink} created. We have automatically selected your existing CodeQL ${displayLanguage} database ${existingDatabaseItem.name} for you to analyze with ${openFileLink}.`,
+        );
+      }
     } else {
       // download new database and select it
-      await this.downloadDatabase();
+      this.downloadPromise = this.promptDownloadDatabase().finally(() => {
+        this.downloadPromise = undefined;
+      });
     }
   }
 
+  private get openFileMarkdownLink() {
+    if (this.queryStoragePath === undefined) {
+      throw new Error("QL Pack storage path is undefined");
+    }
+
+    const queryPath = join(this.queryStoragePath, this.fileName);
+    const queryPathUri = Uri.file(queryPath);
+
+    const openFileArgs = [queryPathUri.toString(true)];
+    const queryString = encodeURI(JSON.stringify(openFileArgs));
+    return `[${this.fileName}](command:vscode.open?${queryString})`;
+  }
+
+  private createQlPackGenerator() {
+    if (this.qlPackStoragePath === undefined) {
+      throw new Error("QL pack storage path is undefined");
+    }
+    if (this.queryStoragePath === undefined) {
+      throw new Error("Query storage path is undefined");
+    }
+    if (this.language === undefined) {
+      throw new Error("Language is undefined");
+    }
+
+    const parentFolder = dirname(this.qlPackStoragePath);
+
+    // Only include the folder name in the qlpack name if the qlpack is not in the root of the workspace.
+    const includeFolderNameInQlpackName = !getOnDiskWorkspaceFolders().some(
+      (workspaceFolder) => pathsEqual(workspaceFolder, parentFolder),
+    );
+
+    return new QlPackGenerator(
+      this.language,
+      this.cliServer,
+      this.qlPackStoragePath,
+      this.queryStoragePath,
+      includeFolderNameInQlpackName,
+    );
+  }
+
   public static async findDatabaseItemByNwo(
     language: string,
     databaseNwo: string,

extensions/ql-vscode/src/log-insights/join-order.ts

@@ -40,17 +40,19 @@ function makeKey(
 const DEPENDENT_PREDICATES_REGEXP = (() => {
   const regexps = [
     // SCAN id
-    String.raw`SCAN\s+([0-9a-zA-Z:#_]+)\s`,
+    String.raw`SCAN\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)\s`,
     // JOIN id WITH id
-    String.raw`JOIN\s+([0-9a-zA-Z:#_]+)\s+WITH\s+([0-9a-zA-Z:#_]+)\s`,
+    String.raw`JOIN\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)\s+WITH\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)\s`,
     // AGGREGATE id, id
-    String.raw`AGGREGATE\s+([0-9a-zA-Z:#_]+)\s*,\s+([0-9a-zA-Z:#_]+)`,
+    String.raw`AGGREGATE\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)\s*,\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)`,
     // id AND NOT id
-    String.raw`([0-9a-zA-Z:#_]+)\s+AND\s+NOT\s+([0-9a-zA-Z:#_]+)`,
+    String.raw`([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)\s+AND\s+NOT\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)`,
     // INVOKE HIGHER-ORDER RELATION rel ON <id, ..., id>
-    String.raw`INVOKE\s+HIGHER-ORDER\s+RELATION\s[^\s]+\sON\s+<([0-9a-zA-Z:#_<>]+)((?:,[0-9a-zA-Z:#_<>]+)*)>`,
+    String.raw`INVOKE\s+HIGHER-ORDER\s+RELATION\s[^\s]+\sON\s+<([0-9a-zA-Z:#_<>]+|\`[^\`\r\n]*\`)((?:,[0-9a-zA-Z:#_<>]+|,\`[^\`\r\n]*\`)*)>`,
     // SELECT id
-    String.raw`SELECT\s+([0-9a-zA-Z:#_]+)`,
+    String.raw`SELECT\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)`,
+    // REWRITE id WITH
+    String.raw`REWRITE\s+([0-9a-zA-Z:#_]+|\`[^\`\r\n]*\`)\s+WITH\s`,
   ];
   return new RegExp(
     `${String.raw`\{[0-9]+\}\s+[0-9a-zA-Z]+\s=\s(?:` + regexps.join("|")})`,
@@ -65,7 +67,12 @@ function getDependentPredicates(operations: string[]): I.List<string> {
         .rest() // Skip the first group as it's just the entire string
         .filter((x) => !!x && !x.match("r[0-9]+|PRIMITIVE")) // Only keep the references to predicates.
         .flatMap((x) => x.split(",")) // Group 2 in the INVOKE HIGHER_ORDER RELATION case is a comma-separated list of identifiers.
-        .filter((x) => !!x); // Remove empty strings
+        .filter((x) => !!x) // Remove empty strings
+        .map((x) =>
+          x.startsWith("`") && x.endsWith("`")
+            ? x.substring(1, x.length - 1)
+            : x,
+        ); // Remove quotes from quoted identifiers
     } else {
       return I.List();
     }

extensions/ql-vscode/src/model-editor/auto-model-api.ts

@@ -1,5 +1,7 @@
 import { Credentials } from "../common/authentication";
 import { OctokitResponse } from "@octokit/types";
+import fetch from "node-fetch";
+import { ModelConfigListener } from "../config";
 
 export enum AutomodelMode {
   Unspecified = "AUTOMODEL_MODE_UNSPECIFIED",
@@ -20,15 +22,44 @@ export interface ModelResponse {
 export async function autoModel(
   credentials: Credentials,
   request: ModelRequest,
+  modelingConfig: ModelConfigListener,
 ): Promise<ModelResponse> {
-  const octokit = await credentials.getOctokit();
+  const devEndpoint = modelingConfig.llmGenerationDevEndpoint;
+  if (devEndpoint) {
+    return callAutoModelDevEndpoint(devEndpoint, request);
+  } else {
+    const octokit = await credentials.getOctokit();
 
-  const response: OctokitResponse<ModelResponse> = await octokit.request(
+    const response: OctokitResponse<ModelResponse> = await octokit.request(
-    "POST /repos/github/codeql/code-scanning/codeql/auto-model",
+      "POST /repos/github/codeql/code-scanning/codeql/auto-model",
-    {
+      {
-      data: request,
+        data: request,
-    },
+      },
-  );
+    );
 
-  return response.data;
+    return response.data;
+  }
+}
+
+async function callAutoModelDevEndpoint(
+  endpoint: string,
+  request: ModelRequest,
+): Promise<ModelResponse> {
+  const json = JSON.stringify(request);
+  const response = await fetch(endpoint, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: json,
+  });
+
+  if (!response.ok) {
+    throw new Error(
+      `Error calling auto-model API: ${response.status} ${response.statusText}`,
+    );
+  }
+
+  const data = await response.json();
+  return data as ModelResponse;
 }

extensions/ql-vscode/src/model-editor/auto-model.ts

@@ -19,8 +19,8 @@ import { groupMethods, sortGroupNames, sortMethods } from "./shared/sorting";
  */
 export function getCandidates(
   mode: Mode,
-  methods: Method[],
+  methods: readonly Method[],
-  modeledMethodsBySignature: Record<string, ModeledMethod[]>,
+  modeledMethodsBySignature: Record<string, readonly ModeledMethod[]>,
 ): MethodSignature[] {
   // Sort the same way as the UI so we send the first ones listed in the UI first
   const grouped = groupMethods(methods, mode);
@@ -32,8 +32,9 @@ export function getCandidates(
   const candidates: MethodSignature[] = [];
 
   for (const method of sortedMethods) {
-    const modeledMethods: ModeledMethod[] =
+    const modeledMethods: ModeledMethod[] = [
-      modeledMethodsBySignature[method.signature] ?? [];
+      ...(modeledMethodsBySignature[method.signature] ?? []),
+    ];
 
     // Anything that is modeled is not a candidate
     if (modeledMethods.some((m) => m.type !== "none")) {

extensions/ql-vscode/src/model-editor/auto-modeler.ts

@@ -1,6 +1,5 @@
 import { Method, MethodSignature } from "./method";
 import { ModeledMethod } from "./modeled-method";
-import { extLogger } from "../common/logging/vscode";
 import { load as loadYaml } from "js-yaml";
 import { ProgressCallback, withProgress } from "../common/vscode/progress";
 import { createAutoModelRequest, getCandidates } from "./auto-model";
@@ -16,11 +15,9 @@ import { QueryRunner } from "../query-server";
 import { DatabaseItem } from "../databases/local-databases";
 import { Mode } from "./shared/mode";
 import { CancellationTokenSource } from "vscode";
-
+import { ModelingStore } from "./modeling-store";
-// Limit the number of candidates we send to the model in each request
+import { ModelConfigListener } from "../config";
-// to avoid long requests.
+import { QueryLanguage } from "../common/query-language";
-// Note that the model may return fewer than this number of candidates.
-const candidateBatchSize = 20;
 
 /**
  * The auto-modeler holds state around auto-modeling jobs and allows
@@ -35,12 +32,11 @@ export class AutoModeler {
     private readonly app: App,
     private readonly cliServer: CodeQLCliServer,
     private readonly queryRunner: QueryRunner,
+    private readonly modelConfig: ModelConfigListener,
+    private readonly modelingStore: ModelingStore,
     private readonly queryStorageDir: string,
     private readonly databaseItem: DatabaseItem,
-    private readonly setInProgressMethods: (
+    private readonly language: QueryLanguage,
-      packageName: string,
-      inProgressMethods: string[],
-    ) => Promise<void>,
     private readonly addModeledMethods: (
       modeledMethods: Record<string, ModeledMethod[]>,
     ) => Promise<void>,
@@ -58,8 +54,8 @@ export class AutoModeler {
    */
   public async startModeling(
     packageName: string,
-    methods: Method[],
+    methods: readonly Method[],
-    modeledMethods: Record<string, ModeledMethod[]>,
+    modeledMethods: Record<string, readonly ModeledMethod[]>,
     mode: Mode,
   ): Promise<void> {
     if (this.jobs.has(packageName)) {
@@ -87,7 +83,7 @@ export class AutoModeler {
    * @param packageName The name of the package to stop modeling.
    */
   public async stopModeling(packageName: string): Promise<void> {
-    void extLogger.log(`Stopping modeling for package ${packageName}`);
+    void this.app.logger.log(`Stopping modeling for package ${packageName}`);
     const cancellationTokenSource = this.jobs.get(packageName);
     if (cancellationTokenSource) {
       cancellationTokenSource.cancel();
@@ -105,19 +101,22 @@ export class AutoModeler {
 
   private async modelPackage(
     packageName: string,
-    methods: Method[],
+    methods: readonly Method[],
-    modeledMethods: Record<string, ModeledMethod[]>,
+    modeledMethods: Record<string, readonly ModeledMethod[]>,
     mode: Mode,
     cancellationTokenSource: CancellationTokenSource,
   ): Promise<void> {
-    void extLogger.log(`Modeling package ${packageName}`);
+    void this.app.logger.log(`Modeling package ${packageName}`);
+
+    const candidateBatchSize = this.modelConfig.llmGenerationBatchSize;
+
     await withProgress(async (progress) => {
       // Fetch the candidates to send to the model
       const allCandidateMethods = getCandidates(mode, methods, modeledMethods);
 
       // If there are no candidates, there is nothing to model and we just return
       if (allCandidateMethods.length === 0) {
-        void extLogger.log("No candidates to model. Stopping.");
+        void this.app.logger.log("No candidates to model. Stopping.");
         return;
       }
 
@@ -135,11 +134,14 @@ export class AutoModeler {
           const start = i * candidateBatchSize;
           const end = start + candidateBatchSize;
           const candidatesToProcess = allCandidateMethods.slice(start, end);
+          const candidateSignatures = candidatesToProcess.map(
+            (c) => c.signature,
+          );
 
           // Let the UI know which candidates we are modeling
-          await this.setInProgressMethods(
+          this.modelingStore.addInProgressMethods(
-            packageName,
+            this.databaseItem,
-            candidatesToProcess.map((c) => c.signature),
+            candidateSignatures,
           );
 
           // Kick off the process to model the slice of candidates
@@ -149,10 +151,19 @@ export class AutoModeler {
             progress,
             cancellationTokenSource,
           );
+
+          // Let the UI know which candidates we are done modeling
+          this.modelingStore.removeInProgressMethods(
+            this.databaseItem,
+            candidateSignatures,
+          );
         }
       } finally {
-        // Clear out in progress methods
+        // Clear out in progress methods in case anything went wrong
-        await this.setInProgressMethods(packageName, []);
+        this.modelingStore.removeInProgressMethods(
+          this.databaseItem,
+          allCandidateMethods.map((c) => c.signature),
+        );
       }
     });
   }
@@ -163,7 +174,7 @@ export class AutoModeler {
     progress: ProgressCallback,
     cancellationTokenSource: CancellationTokenSource,
   ): Promise<void> {
-    void extLogger.log("Executing auto-model queries");
+    void this.app.logger.log("Executing auto-model queries");
 
     const usages = await runAutoModelQueries({
       mode,
@@ -181,7 +192,7 @@ export class AutoModeler {
 
     const request = await createAutoModelRequest(mode, usages);
 
-    void extLogger.log("Calling auto-model API");
+    void this.app.logger.log("Calling auto-model API");
 
     const response = await this.callAutoModelApi(request);
     if (!response) {
@@ -192,34 +203,11 @@ export class AutoModeler {
       filename: "auto-model.yml",
     });
 
-    const loadedMethods = loadDataExtensionYaml(models);
+    const loadedMethods = loadDataExtensionYaml(models, this.language);
     if (!loadedMethods) {
       return;
     }
 
-    // Any candidate that was part of the response is a negative result
-    // meaning that the canidate is not a sink for the kinds that the LLM is checking for.
-    // For now we model this as a sink neutral method, however this is subject
-    // to discussion.
-    for (const candidate of candidateMethods) {
-      if (!(candidate.signature in loadedMethods)) {
-        loadedMethods[candidate.signature] = [
-          {
-            type: "neutral",
-            kind: "sink",
-            input: "",
-            output: "",
-            provenance: "ai-generated",
-            signature: candidate.signature,
-            packageName: candidate.packageName,
-            typeName: candidate.typeName,
-            methodName: candidate.methodName,
-            methodParameters: candidate.methodParameters,
-          },
-        ];
-      }
-    }
-
     await this.addModeledMethods(loadedMethods);
   }
 
@@ -227,7 +215,7 @@ export class AutoModeler {
     request: ModelRequest,
   ): Promise<ModelResponse | null> {
     try {
-      return await autoModel(this.app.credentials, request);
+      return await autoModel(this.app.credentials, request, this.modelConfig);
     } catch (e) {
       if (e instanceof RequestError && e.status === 429) {
         void showAndLogExceptionWithTelemetry(

extensions/ql-vscode/src/model-editor/bqrs.ts

@@ -4,13 +4,18 @@ import { ModeledMethodType } from "./modeled-method";
 import { parseLibraryFilename } from "./library";
 import { Mode } from "./shared/mode";
 import { ApplicationModeTuple, FrameworkModeTuple } from "./queries/query";
+import { QueryLanguage } from "../common/query-language";
+import { getModelsAsDataLanguage } from "./languages";
 
 export function decodeBqrsToMethods(
   chunk: DecodedBqrsChunk,
   mode: Mode,
+  language: QueryLanguage,
 ): Method[] {
   const methodsByApiName = new Map<string, Method>();
 
+  const definition = getModelsAsDataLanguage(language);
+
   chunk?.tuples.forEach((tuple) => {
     let usage: Call;
     let packageName: string;
@@ -51,7 +56,12 @@ export function decodeBqrsToMethods(
       classification = CallClassification.Unknown;
     }
 
-    const signature = `${packageName}.${typeName}#${methodName}${methodParameters}`;
+    const signature = definition.createMethodSignature({
+      packageName,
+      typeName,
+      methodName,
+      methodParameters,
+    });
 
     // For Java, we'll always get back a .jar file, and the library version may be bad because not all library authors
     // properly specify the version. Therefore, we'll always try to parse the name and version from the library filename
@@ -88,9 +98,16 @@ export function decodeBqrsToMethods(
     }
 
     const method = methodsByApiName.get(signature)!;
-    method.usages.push({
+    const usages = [
-      ...usage,
+      ...method.usages,
-      classification,
+      {
+        ...usage,
+        classification,
+      },
+    ];
+    methodsByApiName.set(signature, {
+      ...method,
+      usages,
     });
   });
 

extensions/ql-vscode/src/model-editor/extension-pack-picker.ts

@@ -64,7 +64,7 @@ export async function pickExtensionPack(
   // If the setting is not set, automatically pick a suitable directory
   const extensionsDirectory = userExtensionsDirectory
     ? Uri.file(userExtensionsDirectory)
-    : await autoPickExtensionsDirectory();
+    : await autoPickExtensionsDirectory(logger);
 
   if (!extensionsDirectory) {
     return undefined;

extensions/ql-vscode/src/model-editor/extensions-workspace-folder.ts

@@ -1,7 +1,7 @@
 import { FileType, Uri, workspace, WorkspaceFolder } from "vscode";
 import { getOnDiskWorkspaceFoldersObjects } from "../common/vscode/workspace-folders";
-import { extLogger } from "../common/logging/vscode";
 import { tmpdir } from "../common/files";
+import { NotificationLogger, showAndLogErrorMessage } from "../common/logging";
 
 /**
  * Returns the ancestors of this path in order from furthest to closest (i.e. root of filesystem to parent directory)
@@ -143,9 +143,20 @@ async function findGitFolder(
  *    for which the .git directory is closest to a workspace folder
  * 6. If none of the above apply, return `undefined`
  */
-export async function autoPickExtensionsDirectory(): Promise<Uri | undefined> {
+export async function autoPickExtensionsDirectory(
+  logger: NotificationLogger,
+): Promise<Uri | undefined> {
   const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
 
+  // If there are no on-disk workspace folders, we can't do anything
+  if (workspaceFolders.length === 0) {
+    void showAndLogErrorMessage(
+      logger,
+      `Could not find any on-disk workspace folders. Please ensure that you have opened a folder or workspace.`,
+    );
+    return undefined;
+  }
+
   // If there's only 1 workspace folder, use the `.github/codeql/extensions` directory in that folder
   if (workspaceFolders.length === 1) {
     return Uri.joinPath(
@@ -168,7 +179,7 @@ export async function autoPickExtensionsDirectory(): Promise<Uri | undefined> {
   // Get the root workspace directory, i.e. the common root directory of all workspace folders
   const rootDirectory = await getRootWorkspaceDirectory();
   if (!rootDirectory) {
-    void extLogger.log("Unable to determine root workspace directory");
+    void logger.log("Unable to determine root workspace directory");
 
     return undefined;
   }
@@ -192,7 +203,7 @@ export async function autoPickExtensionsDirectory(): Promise<Uri | undefined> {
       },
     )
   ) {
-    void extLogger.log(
+    void logger.log(
       `Failed to add workspace folder for extensions at ${extensionsUri.fsPath}`,
     );
     return undefined;

extensions/ql-vscode/src/model-editor/external-api-usage-queries.ts

@@ -1,189 +0,0 @@
-import { QueryRunner } from "../query-server";
-import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
-import { extLogger } from "../common/logging/vscode";
-import { showAndLogExceptionWithTelemetry } from "../common/logging";
-import { CancellationToken } from "vscode";
-import { CodeQLCliServer } from "../codeql-cli/cli";
-import { DatabaseItem } from "../databases/local-databases";
-import { ProgressCallback } from "../common/vscode/progress";
-import { redactableError } from "../common/errors";
-import { telemetryListener } from "../common/vscode/telemetry";
-import { join } from "path";
-import { Mode } from "./shared/mode";
-import { writeFile } from "fs-extra";
-import { QueryLanguage } from "../common/query-language";
-import { fetchExternalApiQueries } from "./queries";
-import { Method } from "./method";
-import { runQuery } from "../local-queries/run-query";
-import { decodeBqrsToMethods } from "./bqrs";
-import {
-  resolveEndpointsQuery,
-  syntheticQueryPackName,
-} from "./model-editor-queries";
-
-type RunQueryOptions = {
-  cliServer: CodeQLCliServer;
-  queryRunner: QueryRunner;
-  databaseItem: DatabaseItem;
-  queryStorageDir: string;
-  queryDir: string;
-
-  progress: ProgressCallback;
-  token: CancellationToken;
-};
-
-export async function prepareExternalApiQuery(
-  queryDir: string,
-  language: QueryLanguage,
-): Promise<boolean> {
-  // Resolve the query that we want to run.
-  const query = fetchExternalApiQueries[language];
-  if (!query) {
-    void showAndLogExceptionWithTelemetry(
-      extLogger,
-      telemetryListener,
-      redactableError`No external API usage query found for language ${language}`,
-    );
-    return false;
-  }
-  // Create the query file.
-  Object.values(Mode).map(async (mode) => {
-    const queryFile = join(queryDir, queryNameFromMode(mode));
-    await writeFile(queryFile, query[`${mode}ModeQuery`], "utf8");
-  });
-
-  // Create any dependencies
-  if (query.dependencies) {
-    for (const [filename, contents] of Object.entries(query.dependencies)) {
-      const dependencyFile = join(queryDir, filename);
-      await writeFile(dependencyFile, contents, "utf8");
-    }
-  }
-  return true;
-}
-
-export const externalApiQueriesProgressMaxStep = 2000;
-
-export async function runExternalApiQueries(
-  mode: Mode,
-  {
-    cliServer,
-    queryRunner,
-    databaseItem,
-    queryStorageDir,
-    queryDir,
-    progress,
-    token,
-  }: RunQueryOptions,
-): Promise<Method[] | undefined> {
-  // The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
-  // move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
-  // This is intentionally not pretty code, as it will be removed soon.
-  // For a reference of what this should do in the future, see the previous implementation in
-  // https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72
-
-  progress({
-    message: "Resolving QL packs",
-    step: 1,
-    maxStep: externalApiQueriesProgressMaxStep,
-  });
-  const additionalPacks = getOnDiskWorkspaceFolders();
-  const extensionPacks = Object.keys(
-    await cliServer.resolveQlpacks(additionalPacks, true),
-  );
-
-  progress({
-    message: "Resolving query",
-    step: 2,
-    maxStep: externalApiQueriesProgressMaxStep,
-  });
-
-  // Resolve the queries from either codeql/java-queries or from the temporary queryDir
-  const queryPath = await resolveEndpointsQuery(
-    cliServer,
-    databaseItem.language,
-    mode,
-    [syntheticQueryPackName],
-    [queryDir],
-  );
-  if (!queryPath) {
-    void showAndLogExceptionWithTelemetry(
-      extLogger,
-      telemetryListener,
-      redactableError`The ${mode} model editor query could not be found. Try re-opening the model editor. If that doesn't work, try upgrading the CodeQL libraries.`,
-    );
-    return;
-  }
-
-  // Run the actual query
-  const completedQuery = await runQuery({
-    queryRunner,
-    databaseItem,
-    queryPath,
-    queryStorageDir,
-    additionalPacks,
-    extensionPacks,
-    progress: (update) =>
-      progress({
-        step: update.step + 500,
-        maxStep: externalApiQueriesProgressMaxStep,
-        message: update.message,
-      }),
-    token,
-  });
-
-  if (!completedQuery) {
-    return;
-  }
-
-  // Read the results and covert to internal representation
-  progress({
-    message: "Decoding results",
-    step: 1600,
-    maxStep: externalApiQueriesProgressMaxStep,
-  });
-
-  const bqrsChunk = await readQueryResults({
-    cliServer,
-    bqrsPath: completedQuery.outputDir.bqrsPath,
-  });
-  if (!bqrsChunk) {
-    return;
-  }
-
-  progress({
-    message: "Finalizing results",
-    step: 1950,
-    maxStep: externalApiQueriesProgressMaxStep,
-  });
-
-  return decodeBqrsToMethods(bqrsChunk, mode);
-}
-
-type GetResultsOptions = {
-  cliServer: Pick<CodeQLCliServer, "bqrsInfo" | "bqrsDecode">;
-  bqrsPath: string;
-};
-
-export async function readQueryResults({
-  cliServer,
-  bqrsPath,
-}: GetResultsOptions) {
-  const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
-  if (bqrsInfo["result-sets"].length !== 1) {
-    void showAndLogExceptionWithTelemetry(
-      extLogger,
-      telemetryListener,
-      redactableError`Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`,
-    );
-    return undefined;
-  }
-
-  const resultSet = bqrsInfo["result-sets"][0];
-
-  return cliServer.bqrsDecode(bqrsPath, resultSet.name);
-}
-
-function queryNameFromMode(mode: Mode): string {
-  return `${mode.charAt(0).toUpperCase() + mode.slice(1)}ModeEndpoints.ql`;
-}

extensions/ql-vscode/src/model-editor/flow-model-queries.ts

@@ -1,174 +0,0 @@
-import { CancellationToken } from "vscode";
-import { DatabaseItem } from "../databases/local-databases";
-import { basename } from "path";
-import { QueryRunner } from "../query-server";
-import { CodeQLCliServer } from "../codeql-cli/cli";
-import { showAndLogExceptionWithTelemetry } from "../common/logging";
-import { extLogger } from "../common/logging/vscode";
-import { extensiblePredicateDefinitions } from "./predicates";
-import { ProgressCallback } from "../common/vscode/progress";
-import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
-import { ModeledMethod, ModeledMethodType } from "./modeled-method";
-import { redactableError } from "../common/errors";
-import { telemetryListener } from "../common/vscode/telemetry";
-import { runQuery } from "../local-queries/run-query";
-import { resolveQueries } from "../local-queries";
-
-type FlowModelOptions = {
-  cliServer: CodeQLCliServer;
-  queryRunner: QueryRunner;
-  queryStorageDir: string;
-  databaseItem: DatabaseItem;
-  progress: ProgressCallback;
-  token: CancellationToken;
-  onResults: (results: ModeledMethod[]) => void | Promise<void>;
-};
-
-export async function runFlowModelQueries({
-  onResults,
-  ...options
-}: FlowModelOptions) {
-  const queries = await resolveFlowQueries(
-    options.cliServer,
-    options.databaseItem,
-  );
-
-  const queriesByBasename: Record<string, string> = {};
-  for (const query of queries) {
-    queriesByBasename[basename(query)] = query;
-  }
-
-  const summaryResults = await runSingleFlowQuery(
-    "summary",
-    queriesByBasename["CaptureSummaryModels.ql"],
-    0,
-    options,
-  );
-  if (summaryResults) {
-    await onResults(summaryResults);
-  }
-
-  const sinkResults = await runSingleFlowQuery(
-    "sink",
-    queriesByBasename["CaptureSinkModels.ql"],
-    1,
-    options,
-  );
-  if (sinkResults) {
-    await onResults(sinkResults);
-  }
-
-  const sourceResults = await runSingleFlowQuery(
-    "source",
-    queriesByBasename["CaptureSourceModels.ql"],
-    2,
-    options,
-  );
-  if (sourceResults) {
-    await onResults(sourceResults);
-  }
-
-  const neutralResults = await runSingleFlowQuery(
-    "neutral",
-    queriesByBasename["CaptureNeutralModels.ql"],
-    3,
-    options,
-  );
-  if (neutralResults) {
-    await onResults(neutralResults);
-  }
-}
-
-async function resolveFlowQueries(
-  cliServer: CodeQLCliServer,
-  databaseItem: DatabaseItem,
-): Promise<string[]> {
-  const packsToSearch = [`codeql/${databaseItem.language}-queries`];
-
-  return await resolveQueries(
-    cliServer,
-    packsToSearch,
-    "flow model generator",
-    {
-      "tags contain": ["modelgenerator"],
-    },
-  );
-}
-
-async function runSingleFlowQuery(
-  type: Exclude<ModeledMethodType, "none">,
-  queryPath: string | undefined,
-  queryStep: number,
-  {
-    cliServer,
-    queryRunner,
-    queryStorageDir,
-    databaseItem,
-    progress,
-    token,
-  }: Omit<FlowModelOptions, "onResults">,
-): Promise<ModeledMethod[]> {
-  // Check that the right query was found
-  if (queryPath === undefined) {
-    void showAndLogExceptionWithTelemetry(
-      extLogger,
-      telemetryListener,
-      redactableError`Failed to find ${type} query`,
-    );
-    return [];
-  }
-
-  // Run the query
-  const completedQuery = await runQuery({
-    queryRunner,
-    databaseItem,
-    queryPath,
-    queryStorageDir,
-    additionalPacks: getOnDiskWorkspaceFolders(),
-    extensionPacks: undefined,
-    progress: ({ step, message }) =>
-      progress({
-        message: `Generating ${type} model: ${message}`,
-        step: queryStep * 1000 + step,
-        maxStep: 4000,
-      }),
-    token,
-  });
-
-  if (!completedQuery) {
-    return [];
-  }
-
-  // Interpret the results
-  const definition = extensiblePredicateDefinitions[type];
-
-  const bqrsPath = completedQuery.outputDir.bqrsPath;
-
-  const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
-  if (bqrsInfo["result-sets"].length !== 1) {
-    void showAndLogExceptionWithTelemetry(
-      extLogger,
-      telemetryListener,
-      redactableError`Expected exactly one result set, got ${
-        bqrsInfo["result-sets"].length
-      } for ${basename(queryPath)}`,
-    );
-  }
-
-  const resultSet = bqrsInfo["result-sets"][0];
-
-  const decodedResults = await cliServer.bqrsDecode(bqrsPath, resultSet.name);
-
-  const results = decodedResults.tuples;
-
-  return (
-    results
-      // This is just a sanity check. The query should only return strings.
-      .filter((result) => typeof result[0] === "string")
-      .map((result) => {
-        const row = result[0] as string;
-
-        return definition.readModeledMethod(row.split(";"));
-      })
-  );
-}

extensions/ql-vscode/src/model-editor/generate.ts

@@ -0,0 +1,99 @@
+import { CancellationToken } from "vscode";
+import { DatabaseItem } from "../databases/local-databases";
+import { basename } from "path";
+import { QueryRunner } from "../query-server";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { ProgressCallback } from "../common/vscode/progress";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { ModeledMethod } from "./modeled-method";
+import { runQuery } from "../local-queries/run-query";
+import { QueryConstraints, resolveQueries } from "../local-queries";
+import { DecodedBqrs } from "../common/bqrs-cli-types";
+type GenerateQueriesOptions = {
+  queryConstraints: QueryConstraints;
+  filterQueries?: (queryPath: string) => boolean;
+  parseResults: (
+    queryPath: string,
+    results: DecodedBqrs,
+  ) => ModeledMethod[] | Promise<ModeledMethod[]>;
+  onResults: (results: ModeledMethod[]) => void | Promise<void>;
+
+  cliServer: CodeQLCliServer;
+  queryRunner: QueryRunner;
+  queryStorageDir: string;
+  databaseItem: DatabaseItem;
+  progress: ProgressCallback;
+  token: CancellationToken;
+};
+
+export async function runGenerateQueries(options: GenerateQueriesOptions) {
+  const { queryConstraints, filterQueries, parseResults, onResults } = options;
+
+  options.progress({
+    message: "Resolving queries",
+    step: 1,
+    maxStep: 5000,
+  });
+
+  const packsToSearch = [`codeql/${options.databaseItem.language}-queries`];
+  const queryPaths = await resolveQueries(
+    options.cliServer,
+    packsToSearch,
+    "generate model",
+    queryConstraints,
+  );
+
+  const filteredQueryPaths = filterQueries
+    ? queryPaths.filter(filterQueries)
+    : queryPaths;
+
+  const maxStep = filteredQueryPaths.length * 1000;
+
+  for (let i = 0; i < filteredQueryPaths.length; i++) {
+    const queryPath = filteredQueryPaths[i];
+
+    const bqrs = await runSingleGenerateQuery(queryPath, i, maxStep, options);
+    if (bqrs) {
+      await onResults(await parseResults(queryPath, bqrs));
+    }
+  }
+}
+
+async function runSingleGenerateQuery(
+  queryPath: string,
+  queryStep: number,
+  maxStep: number,
+  {
+    cliServer,
+    queryRunner,
+    queryStorageDir,
+    databaseItem,
+    progress,
+    token,
+  }: GenerateQueriesOptions,
+): Promise<DecodedBqrs | undefined> {
+  const queryBasename = basename(queryPath);
+
+  // Run the query
+  const completedQuery = await runQuery({
+    queryRunner,
+    databaseItem,
+    queryPath,
+    queryStorageDir,
+    additionalPacks: getOnDiskWorkspaceFolders(),
+    extensionPacks: undefined,
+    progress: ({ step, message }) =>
+      progress({
+        message: `Generating model from ${queryBasename}: ${message}`,
+        step: queryStep * 1000 + step,
+        maxStep,
+      }),
+    token,
+  });
+
+  if (!completedQuery) {
+    return undefined;
+  }
+
+  return cliServer.bqrsDecodeAll(completedQuery.outputDir.bqrsPath);
+}

extensions/ql-vscode/src/model-editor/languages/index.ts

@@ -0,0 +1,2 @@
+export * from "./languages";
+export * from "./models-as-data";

extensions/ql-vscode/src/model-editor/languages/languages.ts

@@ -0,0 +1,36 @@
+import { QueryLanguage } from "../../common/query-language";
+import {
+  ModelsAsDataLanguage,
+  ModelsAsDataLanguagePredicates,
+} from "./models-as-data";
+import { ruby } from "./ruby";
+import { staticLanguage } from "./static";
+
+const languages: Partial<Record<QueryLanguage, ModelsAsDataLanguage>> = {
+  [QueryLanguage.CSharp]: staticLanguage,
+  [QueryLanguage.Java]: staticLanguage,
+  [QueryLanguage.Ruby]: ruby,
+};
+
+export function getModelsAsDataLanguage(
+  language: QueryLanguage,
+): ModelsAsDataLanguage {
+  const definition = languages[language];
+  if (!definition) {
+    throw new Error(`No models-as-data definition for ${language}`);
+  }
+  return definition;
+}
+
+export function getModelsAsDataLanguageModel<
+  T extends keyof ModelsAsDataLanguagePredicates,
+>(
+  language: QueryLanguage,
+  model: T,
+): NonNullable<ModelsAsDataLanguagePredicates[T]> {
+  const definition = getModelsAsDataLanguage(language).predicates[model];
+  if (!definition) {
+    throw new Error(`No models-as-data predicate for ${model}`);
+  }
+  return definition;
+}

extensions/ql-vscode/src/model-editor/languages/models-as-data.ts

@@ -0,0 +1,69 @@
+import { MethodArgument, MethodDefinition } from "../method";
+import {
+  ModeledMethod,
+  NeutralModeledMethod,
+  SinkModeledMethod,
+  SourceModeledMethod,
+  SummaryModeledMethod,
+  TypeModeledMethod,
+} from "../modeled-method";
+import { DataTuple } from "../model-extension-file";
+import { Mode } from "../shared/mode";
+import type { QueryConstraints } from "../../local-queries/query-constraints";
+import { DecodedBqrs } from "../../common/bqrs-cli-types";
+import { BaseLogger } from "../../common/logging";
+
+type GenerateMethodDefinition<T> = (method: T) => DataTuple[];
+type ReadModeledMethod = (row: DataTuple[]) => ModeledMethod;
+
+export type ModelsAsDataLanguagePredicate<T> = {
+  extensiblePredicate: string;
+  supportedKinds?: string[];
+  generateMethodDefinition: GenerateMethodDefinition<T>;
+  readModeledMethod: ReadModeledMethod;
+};
+
+type ModelsAsDataLanguageModelGeneration = {
+  queryConstraints: QueryConstraints;
+  filterQueries?: (queryPath: string) => boolean;
+  parseResults: (
+    // The path to the query that generated the results.
+    queryPath: string,
+    // The results of the query.
+    bqrs: DecodedBqrs,
+    // The language-specific predicate that was used to generate the results. This is passed to allow
+    // sharing of code between different languages.
+    modelsAsDataLanguage: ModelsAsDataLanguage,
+    // The logger to use for logging.
+    logger: BaseLogger,
+  ) => ModeledMethod[];
+};
+
+export type ModelsAsDataLanguagePredicates = {
+  source?: ModelsAsDataLanguagePredicate<SourceModeledMethod>;
+  sink?: ModelsAsDataLanguagePredicate<SinkModeledMethod>;
+  summary?: ModelsAsDataLanguagePredicate<SummaryModeledMethod>;
+  neutral?: ModelsAsDataLanguagePredicate<NeutralModeledMethod>;
+  type?: ModelsAsDataLanguagePredicate<TypeModeledMethod>;
+};
+
+export type MethodArgumentOptions = {
+  options: MethodArgument[];
+  defaultArgumentPath: string;
+};
+
+export type ModelsAsDataLanguage = {
+  /**
+   * The modes that are available for this language. If not specified, all
+   * modes are available.
+   */
+  availableModes?: Mode[];
+  createMethodSignature: (method: MethodDefinition) => string;
+  predicates: ModelsAsDataLanguagePredicates;
+  modelGeneration?: ModelsAsDataLanguageModelGeneration;
+  /**
+   * Returns the list of valid arguments that can be selected for the given method.
+   * @param method The method to get the valid arguments for.
+   */
+  getArgumentOptions: (method: MethodDefinition) => MethodArgumentOptions;
+};

extensions/ql-vscode/src/model-editor/languages/ruby/generate.ts

@@ -0,0 +1,50 @@
+import { BaseLogger } from "../../../common/logging";
+import { DecodedBqrs } from "../../../common/bqrs-cli-types";
+import { ModelsAsDataLanguage } from "../models-as-data";
+import { ModeledMethod } from "../../modeled-method";
+import { DataTuple } from "../../model-extension-file";
+
+export function parseGenerateModelResults(
+  _queryPath: string,
+  bqrs: DecodedBqrs,
+  modelsAsDataLanguage: ModelsAsDataLanguage,
+  logger: BaseLogger,
+): ModeledMethod[] {
+  const modeledMethods: ModeledMethod[] = [];
+
+  for (const resultSetName in bqrs) {
+    const definition = Object.values(modelsAsDataLanguage.predicates).find(
+      (definition) => definition.extensiblePredicate === resultSetName,
+    );
+    if (definition === undefined) {
+      void logger.log(`No predicate found for ${resultSetName}`);
+
+      continue;
+    }
+
+    const resultSet = bqrs[resultSetName];
+
+    if (
+      resultSet.tuples.some((tuple) =>
+        tuple.some((value) => typeof value === "object"),
+      )
+    ) {
+      void logger.log(
+        `Skipping ${resultSetName} because it contains undefined values`,
+      );
+      continue;
+    }
+
+    modeledMethods.push(
+      ...resultSet.tuples.map((tuple) => {
+        const row = tuple.filter(
+          (value): value is DataTuple => typeof value !== "object",
+        );
+
+        return definition.readModeledMethod(row);
+      }),
+    );
+  }
+
+  return modeledMethods;
+}

extensions/ql-vscode/src/model-editor/languages/ruby/index.ts

@@ -0,0 +1,215 @@
+import { ModelsAsDataLanguage } from "../models-as-data";
+import { sharedExtensiblePredicates, sharedKinds } from "../shared";
+import { Mode } from "../../shared/mode";
+import { parseGenerateModelResults } from "./generate";
+import { getArgumentsList, MethodArgument } from "../../method";
+
+function parseRubyMethodFromPath(path: string): string {
+  const match = path.match(/Method\[([^\]]+)].*/);
+  if (match) {
+    return match[1];
+  } else {
+    return "";
+  }
+}
+
+function parseRubyAccessPath(path: string): {
+  methodName: string;
+  path: string;
+} {
+  const match = path.match(/Method\[([^\]]+)]\.(.*)/);
+  if (match) {
+    return { methodName: match[1], path: match[2] };
+  } else {
+    return { methodName: "", path: "" };
+  }
+}
+
+function rubyMethodSignature(typeName: string, methodName: string) {
+  return `${typeName}#${methodName}`;
+}
+
+export const ruby: ModelsAsDataLanguage = {
+  availableModes: [Mode.Framework],
+  createMethodSignature: ({ typeName, methodName }) =>
+    `${typeName}#${methodName}`,
+  predicates: {
+    source: {
+      extensiblePredicate: sharedExtensiblePredicates.source,
+      supportedKinds: sharedKinds.source,
+      // extensible predicate sourceModel(
+      //   string type, string path, string kind
+      // );
+      generateMethodDefinition: (method) => [
+        method.typeName,
+        `Method[${method.methodName}].${method.output}`,
+        method.kind,
+      ],
+      readModeledMethod: (row) => {
+        const typeName = row[0] as string;
+        const { methodName, path: output } = parseRubyAccessPath(
+          row[1] as string,
+        );
+        return {
+          type: "source",
+          input: "",
+          output,
+          kind: row[2] as string,
+          provenance: "manual",
+          signature: rubyMethodSignature(typeName, methodName),
+          packageName: "",
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+    sink: {
+      extensiblePredicate: sharedExtensiblePredicates.sink,
+      supportedKinds: sharedKinds.sink,
+      // extensible predicate sinkModel(
+      //   string type, string path, string kind
+      // );
+      generateMethodDefinition: (method) => {
+        const path = `Method[${method.methodName}].${method.input}`;
+        return [method.typeName, path, method.kind];
+      },
+      readModeledMethod: (row) => {
+        const typeName = row[0] as string;
+        const { methodName, path: input } = parseRubyAccessPath(
+          row[1] as string,
+        );
+        return {
+          type: "sink",
+          input,
+          output: "",
+          kind: row[2] as string,
+          provenance: "manual",
+          signature: rubyMethodSignature(typeName, methodName),
+          packageName: "",
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+    summary: {
+      extensiblePredicate: sharedExtensiblePredicates.summary,
+      supportedKinds: sharedKinds.summary,
+      // extensible predicate summaryModel(
+      //   string type, string path, string input, string output, string kind
+      // );
+      generateMethodDefinition: (method) => [
+        method.typeName,
+        `Method[${method.methodName}]`,
+        method.input,
+        method.output,
+        method.kind,
+      ],
+      readModeledMethod: (row) => {
+        const typeName = row[0] as string;
+        const methodName = parseRubyMethodFromPath(row[1] as string);
+        return {
+          type: "summary",
+          input: row[2] as string,
+          output: row[3] as string,
+          kind: row[4] as string,
+          provenance: "manual",
+          signature: rubyMethodSignature(typeName, methodName),
+          packageName: "",
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+    neutral: {
+      extensiblePredicate: sharedExtensiblePredicates.neutral,
+      supportedKinds: sharedKinds.neutral,
+      // extensible predicate neutralModel(
+      //   string type, string path, string kind
+      // );
+      generateMethodDefinition: (method) => [
+        method.typeName,
+        `Method[${method.methodName}]`,
+        method.kind,
+      ],
+      readModeledMethod: (row) => {
+        const typeName = row[0] as string;
+        const methodName = parseRubyMethodFromPath(row[1] as string);
+        return {
+          type: "neutral",
+          input: "",
+          output: "",
+          kind: row[2] as string,
+          provenance: "manual",
+          signature: rubyMethodSignature(typeName, methodName),
+          packageName: "",
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+    type: {
+      extensiblePredicate: "typeModel",
+      // extensible predicate typeModel(string type1, string type2, string path);
+      generateMethodDefinition: (method) => [
+        method.relatedTypeName,
+        method.typeName,
+        `Method[${method.methodName}].${method.path}`,
+      ],
+      readModeledMethod: (row) => {
+        const typeName = row[1] as string;
+        const { methodName, path } = parseRubyAccessPath(row[2] as string);
+
+        return {
+          type: "type",
+          relatedTypeName: row[0] as string,
+          path,
+          signature: rubyMethodSignature(typeName, methodName),
+          packageName: "",
+          typeName,
+          methodName,
+          methodParameters: "",
+        };
+      },
+    },
+  },
+  modelGeneration: {
+    queryConstraints: {
+      "query path": "queries/modeling/GenerateModel.ql",
+    },
+    parseResults: parseGenerateModelResults,
+  },
+  getArgumentOptions: (method) => {
+    const argumentsList = getArgumentsList(method.methodParameters).map(
+      (argument, index): MethodArgument => {
+        if (argument.endsWith(":")) {
+          return {
+            path: `Argument[${argument}]`,
+            label: `Argument[${argument}]`,
+          };
+        }
+
+        return {
+          path: `Argument[${index}]`,
+          label: `Argument[${index}]: ${argument}`,
+        };
+      },
+    );
+
+    return {
+      options: [
+        {
+          path: "Argument[self]",
+          label: "Argument[self]",
+        },
+        ...argumentsList,
+      ],
+      // If there are no arguments, we will default to "Argument[self]"
+      defaultArgumentPath:
+        argumentsList.length > 0 ? argumentsList[0].path : "Argument[self]",
+    };
+  },
+};

extensions/ql-vscode/src/model-editor/languages/shared.ts

@@ -0,0 +1,25 @@
+export const sharedExtensiblePredicates = {
+  source: "sourceModel",
+  sink: "sinkModel",
+  summary: "summaryModel",
+  neutral: "neutralModel",
+};
+
+export const sharedKinds = {
+  source: ["local", "remote"],
+  sink: [
+    "code-injection",
+    "command-injection",
+    "file-content-store",
+    "html-injection",
+    "js-injection",
+    "ldap-injection",
+    "log-injection",
+    "path-injection",
+    "request-forgery",
+    "sql-injection",
+    "url-redirection",
+  ],
+  summary: ["taint", "value"],
+  neutral: ["summary", "source", "sink"],
+};

extensions/ql-vscode/src/model-editor/languages/static/generate.ts

@@ -0,0 +1,60 @@
+import { BaseLogger } from "../../../common/logging";
+import {
+  ModelsAsDataLanguage,
+  ModelsAsDataLanguagePredicates,
+} from "../models-as-data";
+import { DecodedBqrs } from "../../../common/bqrs-cli-types";
+import { ModeledMethod } from "../../modeled-method";
+import { basename } from "../../../common/path";
+
+const queriesToModel: Record<string, keyof ModelsAsDataLanguagePredicates> = {
+  "CaptureSummaryModels.ql": "summary",
+  "CaptureSinkModels.ql": "sink",
+  "CaptureSourceModels.ql": "source",
+  "CaptureNeutralModels.ql": "neutral",
+};
+
+export function filterFlowModelQueries(queryPath: string): boolean {
+  return Object.keys(queriesToModel).includes(basename(queryPath));
+}
+
+export function parseFlowModelResults(
+  queryPath: string,
+  bqrs: DecodedBqrs,
+  modelsAsDataLanguage: ModelsAsDataLanguage,
+  logger: BaseLogger,
+): ModeledMethod[] {
+  if (Object.keys(bqrs).length !== 1) {
+    throw new Error(
+      `Expected exactly one result set from ${queryPath}, but got ${
+        Object.keys(bqrs).length
+      }`,
+    );
+  }
+
+  const modelType = queriesToModel[basename(queryPath)];
+  if (!modelType) {
+    void logger.log(`Unknown model type for ${queryPath}`);
+    return [];
+  }
+
+  const resultSet = bqrs[Object.keys(bqrs)[0]];
+
+  const results = resultSet.tuples;
+
+  const definition = modelsAsDataLanguage.predicates[modelType];
+  if (!definition) {
+    throw new Error(`No definition for ${modelType}`);
+  }
+
+  return (
+    results
+      // This is just a sanity check. The query should only return strings.
+      .filter((result) => typeof result[0] === "string")
+      .map((result) => {
+        const row = result[0] as string;
+
+        return definition.readModeledMethod(row.split(";"));
+      })
+  );
+}

extensions/ql-vscode/src/model-editor/languages/static/index.ts

@@ -0,0 +1,170 @@
+import { ModelsAsDataLanguage } from "../models-as-data";
+import { Provenance } from "../../modeled-method";
+import { DataTuple } from "../../model-extension-file";
+import { sharedExtensiblePredicates, sharedKinds } from "../shared";
+import { filterFlowModelQueries, parseFlowModelResults } from "./generate";
+import { getArgumentsList, MethodArgument } from "../../method";
+
+function readRowToMethod(row: DataTuple[]): string {
+  return `${row[0]}.${row[1]}#${row[3]}${row[4]}`;
+}
+
+export const staticLanguage: ModelsAsDataLanguage = {
+  createMethodSignature: ({
+    packageName,
+    typeName,
+    methodName,
+    methodParameters,
+  }) => `${packageName}.${typeName}#${methodName}${methodParameters}`,
+  predicates: {
+    source: {
+      extensiblePredicate: sharedExtensiblePredicates.source,
+      supportedKinds: sharedKinds.source,
+      // extensible predicate sourceModel(
+      //   string package, string type, boolean subtypes, string name, string signature, string ext,
+      //   string output, string kind, string provenance
+      // );
+      generateMethodDefinition: (method) => [
+        method.packageName,
+        method.typeName,
+        true,
+        method.methodName,
+        method.methodParameters,
+        "",
+        method.output,
+        method.kind,
+        method.provenance,
+      ],
+      readModeledMethod: (row) => ({
+        type: "source",
+        input: "",
+        output: row[6] as string,
+        kind: row[7] as string,
+        provenance: row[8] as Provenance,
+        signature: readRowToMethod(row),
+        packageName: row[0] as string,
+        typeName: row[1] as string,
+        methodName: row[3] as string,
+        methodParameters: row[4] as string,
+      }),
+    },
+    sink: {
+      extensiblePredicate: sharedExtensiblePredicates.sink,
+      supportedKinds: sharedKinds.sink,
+      // extensible predicate sinkModel(
+      //   string package, string type, boolean subtypes, string name, string signature, string ext,
+      //   string input, string kind, string provenance
+      // );
+      generateMethodDefinition: (method) => [
+        method.packageName,
+        method.typeName,
+        true,
+        method.methodName,
+        method.methodParameters,
+        "",
+        method.input,
+        method.kind,
+        method.provenance,
+      ],
+      readModeledMethod: (row) => ({
+        type: "sink",
+        input: row[6] as string,
+        output: "",
+        kind: row[7] as string,
+        provenance: row[8] as Provenance,
+        signature: readRowToMethod(row),
+        packageName: row[0] as string,
+        typeName: row[1] as string,
+        methodName: row[3] as string,
+        methodParameters: row[4] as string,
+      }),
+    },
+    summary: {
+      extensiblePredicate: sharedExtensiblePredicates.summary,
+      supportedKinds: sharedKinds.summary,
+      // extensible predicate summaryModel(
+      //   string package, string type, boolean subtypes, string name, string signature, string ext,
+      //   string input, string output, string kind, string provenance
+      // );
+      generateMethodDefinition: (method) => [
+        method.packageName,
+        method.typeName,
+        true,
+        method.methodName,
+        method.methodParameters,
+        "",
+        method.input,
+        method.output,
+        method.kind,
+        method.provenance,
+      ],
+      readModeledMethod: (row) => ({
+        type: "summary",
+        input: row[6] as string,
+        output: row[7] as string,
+        kind: row[8] as string,
+        provenance: row[9] as Provenance,
+        signature: readRowToMethod(row),
+        packageName: row[0] as string,
+        typeName: row[1] as string,
+        methodName: row[3] as string,
+        methodParameters: row[4] as string,
+      }),
+    },
+    neutral: {
+      extensiblePredicate: sharedExtensiblePredicates.neutral,
+      supportedKinds: sharedKinds.neutral,
+      // extensible predicate neutralModel(
+      //   string package, string type, string name, string signature, string kind, string provenance
+      // );
+      generateMethodDefinition: (method) => [
+        method.packageName,
+        method.typeName,
+        method.methodName,
+        method.methodParameters,
+        method.kind,
+        method.provenance,
+      ],
+      readModeledMethod: (row) => ({
+        type: "neutral",
+        input: "",
+        output: "",
+        kind: row[4] as string,
+        provenance: row[5] as Provenance,
+        signature: `${row[0]}.${row[1]}#${row[2]}${row[3]}`,
+        packageName: row[0] as string,
+        typeName: row[1] as string,
+        methodName: row[2] as string,
+        methodParameters: row[3] as string,
+      }),
+    },
+  },
+  modelGeneration: {
+    queryConstraints: {
+      "tags contain": ["modelgenerator"],
+    },
+    filterQueries: filterFlowModelQueries,
+    parseResults: parseFlowModelResults,
+  },
+  getArgumentOptions: (method) => {
+    const argumentsList = getArgumentsList(method.methodParameters).map(
+      (argument, index): MethodArgument => ({
+        path: `Argument[${index}]`,
+        label: `Argument[${index}]: ${argument}`,
+      }),
+    );
+
+    return {
+      options: [
+        {
+          path: "Argument[this]",
+          label: "Argument[this]",
+        },
+        ...argumentsList,
+      ],
+      // If there are no arguments, we will default to "Argument[this]"
+      defaultArgumentPath:
+        argumentsList.length > 0 ? argumentsList[0].path : "Argument[this]",
+    };
+  },
+};

extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-panel.ts

@@ -4,9 +4,9 @@ import { DisposableObject } from "../../common/disposable-object";
 import { MethodModelingViewProvider } from "./method-modeling-view-provider";
 import { Method } from "../method";
 import { ModelingStore } from "../modeling-store";
-import { ModelEditorViewTracker } from "../model-editor-view-tracker";
 import { ModelConfigListener } from "../../config";
 import { DatabaseItem } from "../../databases/local-databases";
+import { ModelingEvents } from "../modeling-events";
 
 export class MethodModelingPanel extends DisposableObject {
   private readonly provider: MethodModelingViewProvider;
@@ -14,7 +14,7 @@ export class MethodModelingPanel extends DisposableObject {
   constructor(
     app: App,
     modelingStore: ModelingStore,
-    editorViewTracker: ModelEditorViewTracker,
+    modelingEvents: ModelingEvents,
   ) {
     super();
 
@@ -26,7 +26,7 @@ export class MethodModelingPanel extends DisposableObject {
     this.provider = new MethodModelingViewProvider(
       app,
       modelingStore,
-      editorViewTracker,
+      modelingEvents,
       modelConfig,
     );
     this.push(

extensions/ql-vscode/src/model-editor/method-modeling/method-modeling-view-provider.ts

@@ -4,17 +4,19 @@ import {
 } from "../../common/interface-types";
 import { telemetryListener } from "../../common/vscode/telemetry";
 import { showAndLogExceptionWithTelemetry } from "../../common/logging/notifications";
-import { extLogger } from "../../common/logging/vscode/loggers";
 import { App } from "../../common/app";
 import { redactableError } from "../../common/errors";
 import { Method } from "../method";
 import { ModelingStore } from "../modeling-store";
 import { AbstractWebviewViewProvider } from "../../common/vscode/abstract-webview-view-provider";
 import { assertNever } from "../../common/helpers-pure";
-import { ModelEditorViewTracker } from "../model-editor-view-tracker";
 import { ModelConfigListener } from "../../config";
 import { DatabaseItem } from "../../databases/local-databases";
-import { convertFromLegacyModeledMethod } from "../shared/modeled-methods-legacy";
+import { ModelingEvents } from "../modeling-events";
+import {
+  QueryLanguage,
+  tryGetQueryLanguage,
+} from "../../common/query-language";
 
 export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
   ToMethodModelingMessage,
@@ -24,11 +26,12 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
 
   private method: Method | undefined = undefined;
   private databaseItem: DatabaseItem | undefined = undefined;
+  private language: QueryLanguage | undefined = undefined;
 
   constructor(
     app: App,
     private readonly modelingStore: ModelingStore,
-    private readonly editorViewTracker: ModelEditorViewTracker,
+    private readonly modelingEvents: ModelingEvents,
     private readonly modelConfig: ModelConfigListener,
   ) {
     super(app, "method-modeling");
@@ -36,7 +39,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
 
   protected override async onWebViewLoaded(): Promise<void> {
     await Promise.all([this.setViewState(), this.setInitialState()]);
-    this.registerToModelingStoreEvents();
+    this.registerToModelingEvents();
     this.registerToModelConfigEvents();
   }
 
@@ -44,6 +47,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
     await this.postMessage({
       t: "setMethodModelingPanelViewState",
       viewState: {
+        language: this.language,
         showMultipleModels: this.modelConfig.showMultipleModels,
       },
     });
@@ -55,6 +59,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
   ): Promise<void> {
     this.method = method;
     this.databaseItem = databaseItem;
+    this.language = databaseItem && tryGetQueryLanguage(databaseItem.language);
 
     if (this.isShowingView) {
       await this.postMessage({
@@ -69,6 +74,9 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
       const selectedMethod = this.modelingStore.getSelectedMethodDetails();
       if (selectedMethod) {
         this.databaseItem = selectedMethod.databaseItem;
+        this.language = tryGetQueryLanguage(
+          selectedMethod.databaseItem.language,
+        );
         this.method = selectedMethod.method;
 
         await this.postMessage({
@@ -76,6 +84,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
           method: selectedMethod.method,
           modeledMethods: selectedMethod.modeledMethods,
           isModified: selectedMethod.isModified,
+          isInProgress: selectedMethod.isInProgress,
         });
       }
 
@@ -100,7 +109,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
 
       case "unhandledError":
         void showAndLogExceptionWithTelemetry(
-          extLogger,
+          this.app.logger,
           telemetryListener,
           redactableError(
             msg.error,
@@ -108,24 +117,27 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
         );
         break;
 
-      case "setModeledMethod": {
+      case "setMultipleModeledMethods": {
         if (!this.databaseItem) {
           return;
         }
 
         this.modelingStore.updateModeledMethods(
           this.databaseItem,
-          msg.method.signature,
+          msg.methodSignature,
-          convertFromLegacyModeledMethod(msg.method),
+          msg.modeledMethods,
         );
         this.modelingStore.addModifiedMethod(
           this.databaseItem,
-          msg.method.signature,
+          msg.methodSignature,
         );
         break;
       }
       case "revealInModelEditor":
         await this.revealInModelEditor(msg.method);
+        void telemetryListener?.sendUIInteraction(
+          "method-modeling-reveal-in-model-editor",
+        );
 
         break;
 
@@ -144,15 +156,15 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
       return;
     }
 
-    const view = this.editorViewTracker.getView(
+    this.modelingEvents.fireRevealInModelEditorEvent(
       this.databaseItem.databaseUri.toString(),
+      method,
     );
-    await view?.revealMethod(method);
   }
 
-  private registerToModelingStoreEvents(): void {
+  private registerToModelingEvents(): void {
     this.push(
-      this.modelingStore.onModeledMethodsChanged(async (e) => {
+      this.modelingEvents.onModeledMethodsChanged(async (e) => {
         if (this.webviewView && e.isActiveDb && this.method) {
           const modeledMethods = e.modeledMethods[this.method.signature];
           if (modeledMethods) {
@@ -167,7 +179,7 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
     );
 
     this.push(
-      this.modelingStore.onModifiedMethodsChanged(async (e) => {
+      this.modelingEvents.onModifiedMethodsChanged(async (e) => {
         if (this.webviewView && e.isActiveDb && this.method) {
           const isModified = e.modifiedMethods.has(this.method.signature);
           await this.postMessage({
@@ -179,32 +191,39 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
     );
 
     this.push(
-      this.modelingStore.onSelectedMethodChanged(async (e) => {
+      this.modelingEvents.onSelectedMethodChanged(async (e) => {
         if (this.webviewView) {
           this.method = e.method;
           this.databaseItem = e.databaseItem;
+          this.language = tryGetQueryLanguage(e.databaseItem.language);
 
           await this.postMessage({
             t: "setSelectedMethod",
             method: e.method,
             modeledMethods: e.modeledMethods,
             isModified: e.isModified,
+            isInProgress: e.isInProgress,
           });
         }
       }),
     );
 
     this.push(
-      this.modelingStore.onDbOpened(async () => {
+      this.modelingEvents.onDbOpened(async (databaseItem) => {
+        this.databaseItem = databaseItem;
+
         await this.postMessage({
           t: "setInModelingMode",
           inModelingMode: true,
         });
+
+        this.language = tryGetQueryLanguage(databaseItem.language);
+        await this.setViewState();
       }),
     );
 
     this.push(
-      this.modelingStore.onDbClosed(async (dbUri) => {
+      this.modelingEvents.onDbClosed(async (dbUri) => {
         if (!this.modelingStore.anyDbsBeingModeled()) {
           await this.postMessage({
             t: "setInModelingMode",
@@ -217,6 +236,21 @@ export class MethodModelingViewProvider extends AbstractWebviewViewProvider<
         }
       }),
     );
+
+    this.push(
+      this.modelingEvents.onInProgressMethodsChanged(async (e) => {
+        if (this.method && this.databaseItem) {
+          const dbUri = this.databaseItem.databaseUri.toString();
+          if (e.dbUri === dbUri) {
+            const inProgress = e.methods.has(this.method.signature);
+            await this.postMessage({
+              t: "setInProgress",
+              inProgress,
+            });
+          }
+        }
+      }),
+    );
   }
 
   private registerToModelConfigEvents(): void {

extensions/ql-vscode/src/model-editor/method.ts

@@ -2,8 +2,8 @@ import { ResolvableLocationValue } from "../common/bqrs-cli-types";
 import { ModeledMethod, ModeledMethodType } from "./modeled-method";
 
 export type Call = {
-  label: string;
+  readonly label: string;
-  url: ResolvableLocationValue;
+  readonly url: Readonly<ResolvableLocationValue>;
 };
 
 export enum CallClassification {
@@ -14,14 +14,29 @@ export enum CallClassification {
 }
 
 export type Usage = Call & {
-  classification: CallClassification;
+  readonly classification: CallClassification;
 };
 
-export interface MethodSignature {
+export interface MethodDefinition {
+  /**
+   * The package name in Java, or the namespace in C#, e.g. `org.sql2o` or `System.Net.Http.Headers`.
+   *
+   * If the class is not in a package, the value should be an empty string.
+   */
+  readonly packageName: string;
+  readonly typeName: string;
+  readonly methodName: string;
+  /**
+   * The method parameters, including enclosing parentheses, e.g. `(String, String)`
+   */
+  readonly methodParameters: string;
+}
+
+export interface MethodSignature extends MethodDefinition {
   /**
    * Contains the version of the library if it can be determined by CodeQL, e.g. `4.2.2.2`
    */
-  libraryVersion?: string;
+  readonly libraryVersion?: string;
   /**
    * A unique signature that can be used to identify this external API usage.
    *
@@ -29,33 +44,26 @@ export interface MethodSignature {
    * in the form "packageName.typeName#methodName(methodParameters)".
    * e.g. `org.sql2o.Connection#createQuery(String)`
    */
-  signature: string;
+  readonly signature: string;
-  /**
-   * The package name in Java, or the namespace in C#, e.g. `org.sql2o` or `System.Net.Http.Headers`.
-   *
-   * If the class is not in a package, the value should be an empty string.
-   */
-  packageName: string;
-  typeName: string;
-  methodName: string;
-  /**
-   * The method parameters, including enclosing parentheses, e.g. `(String, String)`
-   */
-  methodParameters: string;
 }
 
 export interface Method extends MethodSignature {
   /**
    * Contains the name of the library containing the method declaration, e.g. `sql2o-1.6.0.jar` or `System.Runtime.dll`
    */
-  library: string;
+  readonly library: string;
   /**
    * Is this method already supported by CodeQL standard libraries.
    * If so, there is no need for the user to model it themselves.
    */
-  supported: boolean;
+  readonly supported: boolean;
-  supportedType: ModeledMethodType;
+  readonly supportedType: ModeledMethodType;
-  usages: Usage[];
+  readonly usages: readonly Usage[];
+}
+
+export interface MethodArgument {
+  path: string;
+  label: string;
 }
 
 export function getArgumentsList(methodParameters: string): string[] {
@@ -66,9 +74,15 @@ export function getArgumentsList(methodParameters: string): string[] {
   return methodParameters.substring(1, methodParameters.length - 1).split(",");
 }
 
+/**
+ * Should we present the user with the ability to edit to modelings for this method.
+ *
+ * A method may be unmodelable if it is already modeled by CodeQL or by an extension
+ * pack other than the one currently being edited.
+ */
 export function canMethodBeModeled(
   method: Method,
-  modeledMethods: ModeledMethod[],
+  modeledMethods: readonly ModeledMethod[],
   methodIsUnsaved: boolean,
 ): boolean {
   return (

extensions/ql-vscode/src/model-editor/methods-usage/methods-usage-data-provider.ts

@@ -9,7 +9,7 @@ import {
   Uri,
 } from "vscode";
 import { DisposableObject } from "../../common/disposable-object";
-import { Method, Usage } from "../method";
+import { Method, Usage, canMethodBeModeled } from "../method";
 import { DatabaseItem } from "../../databases/local-databases";
 import { relative } from "path";
 import { CodeQLCliServer } from "../../codeql-cli/cli";
@@ -24,16 +24,17 @@ export class MethodsUsageDataProvider
   extends DisposableObject
   implements TreeDataProvider<MethodsUsageTreeViewItem>
 {
-  private methods: Method[] = [];
+  private methods: readonly Method[] = [];
   // sortedMethods is a separate field so we can check if the methods have changed
   // by reference, which is faster than checking if the methods have changed by value.
-  private sortedMethods: Method[] = [];
+  private sortedTreeItems: readonly MethodTreeViewItem[] = [];
   private databaseItem: DatabaseItem | undefined = undefined;
   private sourceLocationPrefix: string | undefined = undefined;
   private hideModeledMethods: boolean = INITIAL_HIDE_MODELED_METHODS_VALUE;
   private mode: Mode = INITIAL_MODE;
-  private modeledMethods: Record<string, ModeledMethod[]> = {};
+  private modeledMethods: Readonly<Record<string, readonly ModeledMethod[]>> =
-  private modifiedMethodSignatures: Set<string> = new Set();
+    {};
+  private modifiedMethodSignatures: ReadonlySet<string> = new Set();
 
   private readonly onDidChangeTreeDataEmitter = this.push(
     new EventEmitter<void>(),
@@ -55,12 +56,12 @@ export class MethodsUsageDataProvider
    * method and instead always pass new objects/arrays.
    */
   public async setState(
-    methods: Method[],
+    methods: readonly Method[],
     databaseItem: DatabaseItem,
     hideModeledMethods: boolean,
     mode: Mode,
-    modeledMethods: Record<string, ModeledMethod[]>,
+    modeledMethods: Readonly<Record<string, readonly ModeledMethod[]>>,
-    modifiedMethodSignatures: Set<string>,
+    modifiedMethodSignatures: ReadonlySet<string>,
   ): Promise<void> {
     if (
       this.methods !== methods ||
@@ -71,7 +72,9 @@ export class MethodsUsageDataProvider
       this.modifiedMethodSignatures !== modifiedMethodSignatures
     ) {
       this.methods = methods;
-      this.sortedMethods = sortMethodsInGroups(methods, mode);
+      this.sortedTreeItems = createTreeItems(
+        sortMethodsInGroups(methods, mode),
+      );
       this.databaseItem = databaseItem;
       this.sourceLocationPrefix =
         await this.databaseItem.getSourceLocationPrefix(this.cliServer);
@@ -85,27 +88,27 @@ export class MethodsUsageDataProvider
   }
 
   getTreeItem(item: MethodsUsageTreeViewItem): TreeItem {
-    if (isExternalApiUsage(item)) {
+    if (isMethodTreeViewItem(item)) {
+      const { method } = item;
+
       return {
-        label: `${item.packageName}.${item.typeName}.${item.methodName}${item.methodParameters}`,
+        label: `${method.packageName}.${method.typeName}.${method.methodName}${method.methodParameters}`,
         collapsibleState: TreeItemCollapsibleState.Collapsed,
-        iconPath: this.getModelingStatusIcon(item),
+        iconPath: this.getModelingStatusIcon(method),
       };
     } else {
-      const method = this.getParent(item);
+      const { method, usage } = item;
-      if (!method || !isExternalApiUsage(method)) {
+
-        throw new Error("Parent not found for tree item");
-      }
       return {
-        label: item.label,
+        label: usage.label,
-        description: `${this.relativePathWithinDatabase(item.url.uri)} [${
+        description: `${this.relativePathWithinDatabase(usage.url.uri)} [${
-          item.url.startLine
+          usage.url.startLine
-        }, ${item.url.endLine}]`,
+        }, ${usage.url.endLine}]`,
         collapsibleState: TreeItemCollapsibleState.None,
         command: {
           title: "Show usage",
           command: "codeQLModelEditor.jumpToMethod",
-          arguments: [method.signature, this.databaseItem],
+          arguments: [method, usage, this.databaseItem],
         },
       };
     }
@@ -143,12 +146,18 @@ export class MethodsUsageDataProvider
   getChildren(item?: MethodsUsageTreeViewItem): MethodsUsageTreeViewItem[] {
     if (item === undefined) {
       if (this.hideModeledMethods) {
-        return this.sortedMethods.filter((api) => !api.supported);
+        return this.sortedTreeItems.filter((api) =>
+          canMethodBeModeled(
+            api.method,
+            this.modeledMethods[api.method.signature] ?? [],
+            this.modifiedMethodSignatures.has(api.method.signature),
+          ),
+        );
       } else {
-        return this.sortedMethods;
+        return [...this.sortedTreeItems];
       }
-    } else if (isExternalApiUsage(item)) {
+    } else if (isMethodTreeViewItem(item)) {
-      return item.usages;
+      return item.children;
     } else {
       return [];
     }
@@ -157,29 +166,45 @@ export class MethodsUsageDataProvider
   getParent(
     item: MethodsUsageTreeViewItem,
   ): MethodsUsageTreeViewItem | undefined {
-    if (isExternalApiUsage(item)) {
+    if (isMethodTreeViewItem(item)) {
       return undefined;
     } else {
-      return this.methods.find((e) => e.usages.includes(item));
+      return item.parent;
     }
   }
 
-  public resolveCanonicalUsage(usage: Usage): Usage | undefined {
+  public resolveUsageTreeViewItem(
-    for (const method of this.methods) {
+    methodSignature: string,
-      for (const u of method.usages) {
+    usage: Usage,
-        if (usagesAreEqual(u, usage)) {
+  ): UsageTreeViewItem | undefined {
-          return u;
+    const method = this.sortedTreeItems.find(
-        }
+      (m) => m.method.signature === methodSignature,
-      }
+    );
+    if (!method) {
+      return undefined;
     }
-    return undefined;
+
+    return method.children.find((u) => usagesAreEqual(u.usage, usage));
   }
 }
 
-export type MethodsUsageTreeViewItem = Method | Usage;
+type MethodTreeViewItem = {
+  method: Method;
+  children: UsageTreeViewItem[];
+};
 
-function isExternalApiUsage(item: MethodsUsageTreeViewItem): item is Method {
+type UsageTreeViewItem = {
-  return (item as any).usages !== undefined;
+  method: Method;
+  usage: Usage;
+  parent: MethodTreeViewItem;
+};
+
+export type MethodsUsageTreeViewItem = MethodTreeViewItem | UsageTreeViewItem;
+
+function isMethodTreeViewItem(
+  item: MethodsUsageTreeViewItem,
+): item is MethodTreeViewItem {
+  return "children" in item && "method" in item;
 }
 
 function usagesAreEqual(u1: Usage, u2: Usage): boolean {
@@ -194,7 +219,7 @@ function usagesAreEqual(u1: Usage, u2: Usage): boolean {
   );
 }
 
-function sortMethodsInGroups(methods: Method[], mode: Mode): Method[] {
+function sortMethodsInGroups(methods: readonly Method[], mode: Mode): Method[] {
   const grouped = groupMethods(methods, mode);
 
   const sortedGroupNames = sortGroupNames(grouped);
@@ -205,3 +230,21 @@ function sortMethodsInGroups(methods: Method[], mode: Mode): Method[] {
     return sortMethods(group);
   });
 }
+
+function createTreeItems(methods: readonly Method[]): MethodTreeViewItem[] {
+  return methods.map((method) => {
+    const newMethod: MethodTreeViewItem = {
+      method,
+      children: [],
+    };
+
+    newMethod.children = method.usages.map((usage) => ({
+      method,
+      usage,
+      // This needs to be a reference to the parent method, not a copy of it.
+      parent: newMethod,
+    }));
+
+    return newMethod;
+  });
+}

extensions/ql-vscode/src/model-editor/methods-usage/methods-usage-panel.ts

@@ -10,6 +10,7 @@ import { CodeQLCliServer } from "../../codeql-cli/cli";
 import { ModelingStore } from "../modeling-store";
 import { ModeledMethod } from "../modeled-method";
 import { Mode } from "../shared/mode";
+import { ModelingEvents } from "../modeling-events";
 
 export class MethodsUsagePanel extends DisposableObject {
   private readonly dataProvider: MethodsUsageDataProvider;
@@ -17,6 +18,7 @@ export class MethodsUsagePanel extends DisposableObject {
 
   public constructor(
     private readonly modelingStore: ModelingStore,
+    private readonly modelingEvents: ModelingEvents,
     cliServer: CodeQLCliServer,
   ) {
     super();
@@ -28,16 +30,16 @@ export class MethodsUsagePanel extends DisposableObject {
     });
     this.push(this.treeView);
 
-    this.registerToModelingStoreEvents();
+    this.registerToModelingEvents();
   }
 
   public async setState(
-    methods: Method[],
+    methods: readonly Method[],
     databaseItem: DatabaseItem,
     hideModeledMethods: boolean,
     mode: Mode,
-    modeledMethods: Record<string, ModeledMethod[]>,
+    modeledMethods: Readonly<Record<string, readonly ModeledMethod[]>>,
-    modifiedMethodSignatures: Set<string>,
+    modifiedMethodSignatures: ReadonlySet<string>,
   ): Promise<void> {
     await this.dataProvider.setState(
       methods,
@@ -56,22 +58,28 @@ export class MethodsUsagePanel extends DisposableObject {
     };
   }
 
-  public async revealItem(usage: Usage): Promise<void> {
+  public async revealItem(
-    const canonicalUsage = this.dataProvider.resolveCanonicalUsage(usage);
+    methodSignature: string,
-    if (canonicalUsage !== undefined) {
+    usage: Usage,
-      await this.treeView.reveal(canonicalUsage);
+  ): Promise<void> {
+    const usageTreeViewItem = this.dataProvider.resolveUsageTreeViewItem(
+      methodSignature,
+      usage,
+    );
+    if (usageTreeViewItem !== undefined) {
+      await this.treeView.reveal(usageTreeViewItem);
     }
   }
 
-  private registerToModelingStoreEvents(): void {
+  private registerToModelingEvents(): void {
     this.push(
-      this.modelingStore.onActiveDbChanged(async () => {
+      this.modelingEvents.onActiveDbChanged(async () => {
         await this.handleStateChangeEvent();
       }),
     );
 
     this.push(
-      this.modelingStore.onMethodsChanged(async (event) => {
+      this.modelingEvents.onMethodsChanged(async (event) => {
         if (event.isActiveDb) {
           await this.handleStateChangeEvent();
         }
@@ -79,7 +87,7 @@ export class MethodsUsagePanel extends DisposableObject {
     );
 
     this.push(
-      this.modelingStore.onHideModeledMethodsChanged(async (event) => {
+      this.modelingEvents.onHideModeledMethodsChanged(async (event) => {
         if (event.isActiveDb) {
           await this.handleStateChangeEvent();
         }
@@ -87,7 +95,7 @@ export class MethodsUsagePanel extends DisposableObject {
     );
 
     this.push(
-      this.modelingStore.onModeChanged(async (event) => {
+      this.modelingEvents.onModeChanged(async (event) => {
         if (event.isActiveDb) {
           await this.handleStateChangeEvent();
         }
@@ -95,7 +103,7 @@ export class MethodsUsagePanel extends DisposableObject {
     );
 
     this.push(
-      this.modelingStore.onModifiedMethodsChanged(async (event) => {
+      this.modelingEvents.onModifiedMethodsChanged(async (event) => {
         if (event.isActiveDb) {
           await this.handleStateChangeEvent();
         }

extensions/ql-vscode/src/model-editor/model-editor-module.ts

@@ -15,21 +15,23 @@ import { isQueryLanguage } from "../common/query-language";
 import { DisposableObject } from "../common/disposable-object";
 import { MethodsUsagePanel } from "./methods-usage/methods-usage-panel";
 import { Method, Usage } from "./method";
-import { setUpPack } from "./model-editor-queries";
+import { setUpPack } from "./model-editor-queries-setup";
 import { MethodModelingPanel } from "./method-modeling/method-modeling-panel";
 import { ModelingStore } from "./modeling-store";
 import { showResolvableLocation } from "../databases/local-databases/locations";
-import { ModelEditorViewTracker } from "./model-editor-view-tracker";
 import { ModelConfigListener } from "../config";
-
+import { ModelingEvents } from "./modeling-events";
-const SUPPORTED_LANGUAGES: string[] = ["java", "csharp"];
+import { getModelsAsDataLanguage } from "./languages";
+import { INITIAL_MODE } from "./shared/mode";
+import { isSupportedLanguage } from "./supported-languages";
 
 export class ModelEditorModule extends DisposableObject {
   private readonly queryStorageDir: string;
   private readonly modelingStore: ModelingStore;
-  private readonly editorViewTracker: ModelEditorViewTracker<ModelEditorView>;
+  private readonly modelingEvents: ModelingEvents;
   private readonly methodsUsagePanel: MethodsUsagePanel;
   private readonly methodModelingPanel: MethodModelingPanel;
+  private readonly modelConfig: ModelConfigListener;
 
   private constructor(
     private readonly app: App,
@@ -40,16 +42,17 @@ export class ModelEditorModule extends DisposableObject {
   ) {
     super();
     this.queryStorageDir = join(baseQueryStorageDir, "model-editor-results");
-    this.modelingStore = new ModelingStore(app);
+    this.modelingEvents = new ModelingEvents(app);
-    this.editorViewTracker = new ModelEditorViewTracker();
+    this.modelingStore = new ModelingStore(this.modelingEvents);
     this.methodsUsagePanel = this.push(
-      new MethodsUsagePanel(this.modelingStore, cliServer),
+      new MethodsUsagePanel(this.modelingStore, this.modelingEvents, cliServer),
     );
     this.methodModelingPanel = this.push(
-      new MethodModelingPanel(app, this.modelingStore, this.editorViewTracker),
+      new MethodModelingPanel(app, this.modelingStore, this.modelingEvents),
     );
+    this.modelConfig = this.push(new ModelConfigListener());
 
-    this.registerToModelingStoreEvents();
+    this.registerToModelingEvents();
   }
 
   public static async initialize(
@@ -77,10 +80,11 @@ export class ModelEditorModule extends DisposableObject {
       "codeQL.openModelEditorFromModelingPanel":
         this.openModelEditor.bind(this),
       "codeQLModelEditor.jumpToMethod": async (
-        methodSignature: string,
+        method: Method,
+        usage: Usage,
         databaseItem: DatabaseItem,
       ) => {
-        this.modelingStore.setSelectedMethod(databaseItem, methodSignature);
+        this.modelingStore.setSelectedMethod(databaseItem, method, usage);
       },
     };
   }
@@ -89,9 +93,9 @@ export class ModelEditorModule extends DisposableObject {
     await ensureDir(this.queryStorageDir);
   }
 
-  private registerToModelingStoreEvents(): void {
+  private registerToModelingEvents(): void {
     this.push(
-      this.modelingStore.onSelectedMethodChanged(async (event) => {
+      this.modelingEvents.onSelectedMethodChanged(async (event) => {
         await this.showMethod(event.databaseItem, event.method, event.usage);
       }),
     );
@@ -102,7 +106,7 @@ export class ModelEditorModule extends DisposableObject {
     method: Method,
     usage: Usage,
   ): Promise<void> {
-    await this.methodsUsagePanel.revealItem(usage);
+    await this.methodsUsagePanel.revealItem(method.signature, usage);
     await this.methodModelingPanel.setMethod(databaseItem, method);
     await showResolvableLocation(usage.url, databaseItem, this.app.logger);
   }
@@ -116,9 +120,10 @@ export class ModelEditorModule extends DisposableObject {
       }
 
       const language = db.language;
+
       if (
-        !SUPPORTED_LANGUAGES.includes(language) ||
+        !isQueryLanguage(language) ||
-        !isQueryLanguage(language)
+        !isSupportedLanguage(language, this.modelConfig)
       ) {
         void showAndLogErrorMessage(
           this.app.logger,
@@ -127,12 +132,14 @@ export class ModelEditorModule extends DisposableObject {
         return;
       }
 
-      const existingView = this.editorViewTracker.getView(
+      const definition = getModelsAsDataLanguage(language);
-        db.databaseUri.toString(),
-      );
-      if (existingView) {
-        await existingView.focusView();
 
+      const initialMode = definition.availableModes?.[0] ?? INITIAL_MODE;
+
+      if (this.modelingStore.isDbOpen(db.databaseUri.toString())) {
+        this.modelingEvents.fireFocusModelEditorEvent(
+          db.databaseUri.toString(),
+        );
         return;
       }
 
@@ -158,12 +165,10 @@ export class ModelEditorModule extends DisposableObject {
             return;
           }
 
-          const modelConfig = this.push(new ModelConfigListener());
-
           const modelFile = await pickExtensionPack(
             this.cliServer,
             db,
-            modelConfig,
+            this.modelConfig,
             this.app.logger,
             progress,
             maxStep,
@@ -185,9 +190,10 @@ export class ModelEditorModule extends DisposableObject {
 
           const success = await setUpPack(
             this.cliServer,
+            this.app.logger,
             queryDir,
             language,
-            modelConfig,
+            this.modelConfig,
           );
           if (!success) {
             await cleanupQueryDir();
@@ -202,20 +208,18 @@ export class ModelEditorModule extends DisposableObject {
 
           // Check again just before opening the editor to ensure no model editor has been opened between
           // our first check and now.
-          const existingView = this.editorViewTracker.getView(
+          if (this.modelingStore.isDbOpen(db.databaseUri.toString())) {
-            db.databaseUri.toString(),
+            this.modelingEvents.fireFocusModelEditorEvent(
-          );
+              db.databaseUri.toString(),
-          if (existingView) {
+            );
-            await existingView.focusView();
-
             return;
           }
 
           const view = new ModelEditorView(
             this.app,
             this.modelingStore,
-            this.editorViewTracker,
+            this.modelingEvents,
-            modelConfig,
+            this.modelConfig,
             this.databaseManager,
             this.cliServer,
             this.queryRunner,
@@ -223,9 +227,11 @@ export class ModelEditorModule extends DisposableObject {
             queryDir,
             db,
             modelFile,
+            language,
+            initialMode,
           );
 
-          this.modelingStore.onDbClosed(async (dbUri) => {
+          this.modelingEvents.onDbClosed(async (dbUri) => {
             if (dbUri === db.databaseUri.toString()) {
               await cleanupQueryDir();
             }

extensions/ql-vscode/src/model-editor/model-editor-queries-setup.ts

@@ -0,0 +1,144 @@
+import { join } from "path";
+import { QueryLanguage } from "../common/query-language";
+import { writeFile } from "fs-extra";
+import { dump } from "js-yaml";
+import { prepareModelEditorQueries } from "./model-editor-queries";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { ModelConfig } from "../config";
+import { Mode } from "./shared/mode";
+import { resolveQueriesFromPacks } from "../local-queries";
+import { modeTag } from "./mode-tag";
+import { NotificationLogger } from "../common/logging";
+
+export const syntheticQueryPackName = "codeql/model-editor-queries";
+
+/**
+ * setUpPack sets up a directory to use for the data extension editor queries if required.
+ *
+ * There are two cases (example language is Java):
+ * - In case the queries are present in the codeql/java-queries, we don't need to write our own queries
+ *   to disk. We still need to create a synthetic query pack so we can pass the queryDir to the query
+ *   resolver without caring about whether the queries are present in the pack or not.
+ * - In case the queries are not present in the codeql/java-queries, we need to write our own queries
+ *   to disk. We will create a synthetic query pack and install its dependencies so it is fully independent
+ *   and we can simply pass it through when resolving the queries.
+ *
+ * These steps together ensure that later steps of the process don't need to keep track of whether the queries
+ * are present in codeql/java-queries or in our own query pack. They just need to resolve the query.
+ *
+ * @param cliServer The CodeQL CLI server to use.
+ * @param logger The logger to use.
+ * @param queryDir The directory to set up.
+ * @param language The language to use for the queries.
+ * @param modelConfig The model config to use.
+ * @returns true if the setup was successful, false otherwise.
+ */
+export async function setUpPack(
+  cliServer: CodeQLCliServer,
+  logger: NotificationLogger,
+  queryDir: string,
+  language: QueryLanguage,
+  modelConfig: ModelConfig,
+): Promise<boolean> {
+  // Download the required query packs
+  await cliServer.packDownload([`codeql/${language}-queries`]);
+
+  // We'll only check if the application mode query exists in the pack and assume that if it does,
+  // the framework mode query will also exist.
+  const applicationModeQuery = await resolveEndpointsQuery(
+    cliServer,
+    language,
+    Mode.Application,
+    [],
+    [],
+  );
+
+  if (applicationModeQuery) {
+    // Set up a synthetic pack so CodeQL doesn't crash later when we try
+    // to resolve a query within this directory
+    const syntheticQueryPack = {
+      name: syntheticQueryPackName,
+      version: "0.0.0",
+      dependencies: {},
+    };
+
+    const qlpackFile = join(queryDir, "codeql-pack.yml");
+    await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8");
+  } else {
+    // If we can't resolve the query, we need to write them to desk ourselves.
+    const externalApiQuerySuccess = await prepareModelEditorQueries(
+      logger,
+      queryDir,
+      language,
+    );
+    if (!externalApiQuerySuccess) {
+      return false;
+    }
+
+    // Set up a synthetic pack so that the query can be resolved later.
+    const syntheticQueryPack = {
+      name: syntheticQueryPackName,
+      version: "0.0.0",
+      dependencies: {
+        [`codeql/${language}-all`]: "*",
+      },
+    };
+
+    const qlpackFile = join(queryDir, "codeql-pack.yml");
+    await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8");
+    await cliServer.packInstall(queryDir);
+  }
+
+  // Download any other required packs
+  if (language === "java" && modelConfig.llmGeneration) {
+    await cliServer.packDownload([`codeql/${language}-automodel-queries`]);
+  }
+
+  return true;
+}
+
+/**
+ * Resolve the query path to the model editor endpoints query. All queries are tagged like this:
+ * modeleditor endpoints <mode>
+ * Example: modeleditor endpoints framework-mode
+ *
+ * @param cliServer The CodeQL CLI server to use.
+ * @param language The language of the query pack to use.
+ * @param mode The mode to resolve the query for.
+ * @param additionalPackNames Additional pack names to search.
+ * @param additionalPackPaths Additional pack paths to search.
+ */
+export async function resolveEndpointsQuery(
+  cliServer: CodeQLCliServer,
+  language: string,
+  mode: Mode,
+  additionalPackNames: string[] = [],
+  additionalPackPaths: string[] = [],
+): Promise<string | undefined> {
+  const packsToSearch = [`codeql/${language}-queries`, ...additionalPackNames];
+
+  // First, resolve the query that we want to run.
+  // All queries are tagged like this:
+  // internal extract automodel <mode> <queryTag>
+  // Example: internal extract automodel framework-mode candidates
+  const queries = await resolveQueriesFromPacks(
+    cliServer,
+    packsToSearch,
+    {
+      kind: "table",
+      "tags contain all": ["modeleditor", "endpoints", modeTag(mode)],
+    },
+    additionalPackPaths,
+  );
+  if (queries.length > 1) {
+    throw new Error(
+      `Found multiple endpoints queries for ${mode}. Can't continue`,
+    );
+  }
+
+  if (queries.length === 0) {
+    return undefined;
+  }
+
+  return queries[0];
+}

extensions/ql-vscode/src/model-editor/model-editor-queries.ts

@@ -1,140 +1,199 @@
-import { join } from "path";
+import { QueryRunner } from "../query-server";
-import { QueryLanguage } from "../common/query-language";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
-import { writeFile } from "fs-extra";
+import {
-import { dump } from "js-yaml";
+  NotificationLogger,
-import { prepareExternalApiQuery } from "./external-api-usage-queries";
+  showAndLogExceptionWithTelemetry,
+} from "../common/logging";
+import { CancellationToken } from "vscode";
 import { CodeQLCliServer } from "../codeql-cli/cli";
-import { ModelConfig } from "../config";
+import { DatabaseItem } from "../databases/local-databases";
+import { ProgressCallback } from "../common/vscode/progress";
+import { redactableError } from "../common/errors";
+import { telemetryListener } from "../common/vscode/telemetry";
+import { join } from "path";
 import { Mode } from "./shared/mode";
-import { resolveQueriesFromPacks } from "../local-queries";
+import { outputFile, writeFile } from "fs-extra";
-import { modeTag } from "./mode-tag";
+import { QueryLanguage } from "../common/query-language";
+import { fetchExternalApiQueries } from "./queries";
+import { Method } from "./method";
+import { runQuery } from "../local-queries/run-query";
+import { decodeBqrsToMethods } from "./bqrs";
+import {
+  resolveEndpointsQuery,
+  syntheticQueryPackName,
+} from "./model-editor-queries-setup";
 
-export const syntheticQueryPackName = "codeql/external-api-usage";
+type RunQueryOptions = {
+  cliServer: CodeQLCliServer;
+  queryRunner: QueryRunner;
+  logger: NotificationLogger;
+  databaseItem: DatabaseItem;
+  language: QueryLanguage;
+  queryStorageDir: string;
+  queryDir: string;
 
-/**
+  progress: ProgressCallback;
- * setUpPack sets up a directory to use for the data extension editor queries if required.
+  token: CancellationToken;
- *
+};
- * There are two cases (example language is Java):
+
- * - In case the queries are present in the codeql/java-queries, we don't need to write our own queries
+export async function prepareModelEditorQueries(
- *   to disk. We still need to create a synthetic query pack so we can pass the queryDir to the query
+  logger: NotificationLogger,
- *   resolver without caring about whether the queries are present in the pack or not.
- * - In case the queries are not present in the codeql/java-queries, we need to write our own queries
- *   to disk. We will create a synthetic query pack and install its dependencies so it is fully independent
- *   and we can simply pass it through when resolving the queries.
- *
- * These steps together ensure that later steps of the process don't need to keep track of whether the queries
- * are present in codeql/java-queries or in our own query pack. They just need to resolve the query.
- *
- * @param cliServer The CodeQL CLI server to use.
- * @param queryDir The directory to set up.
- * @param language The language to use for the queries.
- * @param modelConfig The model config to use.
- * @returns true if the setup was successful, false otherwise.
- */
-export async function setUpPack(
-  cliServer: CodeQLCliServer,
   queryDir: string,
   language: QueryLanguage,
-  modelConfig: ModelConfig,
 ): Promise<boolean> {
-  // Download the required query packs
+  // Resolve the query that we want to run.
-  await cliServer.packDownload([`codeql/${language}-queries`]);
+  const query = fetchExternalApiQueries[language];
-
+  if (!query) {
-  // We'll only check if the application mode query exists in the pack and assume that if it does,
+    void showAndLogExceptionWithTelemetry(
-  // the framework mode query will also exist.
+      logger,
-  const applicationModeQuery = await resolveEndpointsQuery(
+      telemetryListener,
-    cliServer,
+      redactableError`No bundled model editor query found for language ${language}`,
-    language,
-    Mode.Application,
-    [],
-    [],
-  );
-
-  if (applicationModeQuery) {
-    // Set up a synthetic pack so CodeQL doesn't crash later when we try
-    // to resolve a query within this directory
-    const syntheticQueryPack = {
-      name: syntheticQueryPackName,
-      version: "0.0.0",
-      dependencies: {},
-    };
-
-    const qlpackFile = join(queryDir, "codeql-pack.yml");
-    await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8");
-  } else {
-    // If we can't resolve the query, we need to write them to desk ourselves.
-    const externalApiQuerySuccess = await prepareExternalApiQuery(
-      queryDir,
-      language,
     );
-    if (!externalApiQuerySuccess) {
+    return false;
-      return false;
+  }
+  // Create the query file.
+  Object.values(Mode).map(async (mode) => {
+    const queryFile = join(queryDir, queryNameFromMode(mode));
+    await writeFile(queryFile, query[`${mode}ModeQuery`], "utf8");
+  });
+
+  // Create any dependencies
+  if (query.dependencies) {
+    for (const [filename, contents] of Object.entries(query.dependencies)) {
+      const dependencyFile = join(queryDir, filename);
+      await outputFile(dependencyFile, contents, "utf8");
     }
-
-    // Set up a synthetic pack so that the query can be resolved later.
-    const syntheticQueryPack = {
-      name: syntheticQueryPackName,
-      version: "0.0.0",
-      dependencies: {
-        [`codeql/${language}-all`]: "*",
-      },
-    };
-
-    const qlpackFile = join(queryDir, "codeql-pack.yml");
-    await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8");
-    await cliServer.packInstall(queryDir);
   }
-
-  // Download any other required packs
-  if (language === "java" && modelConfig.llmGeneration) {
-    await cliServer.packDownload([`codeql/${language}-automodel-queries`]);
-  }
-
   return true;
 }
 
-/**
+export const externalApiQueriesProgressMaxStep = 2000;
- * Resolve the query path to the model editor endpoints query. All queries are tagged like this:
- * modeleditor endpoints <mode>
- * Example: modeleditor endpoints framework-mode
- *
- * @param cliServer The CodeQL CLI server to use.
- * @param language The language of the query pack to use.
- * @param mode The mode to resolve the query for.
- * @param additionalPackNames Additional pack names to search.
- * @param additionalPackPaths Additional pack paths to search.
- */
-export async function resolveEndpointsQuery(
-  cliServer: CodeQLCliServer,
-  language: string,
-  mode: Mode,
-  additionalPackNames: string[] = [],
-  additionalPackPaths: string[] = [],
-): Promise<string | undefined> {
-  const packsToSearch = [`codeql/${language}-queries`, ...additionalPackNames];
 
-  // First, resolve the query that we want to run.
+export async function runModelEditorQueries(
-  // All queries are tagged like this:
+  mode: Mode,
-  // internal extract automodel <mode> <queryTag>
+  {
-  // Example: internal extract automodel framework-mode candidates
-  const queries = await resolveQueriesFromPacks(
     cliServer,
-    packsToSearch,
+    queryRunner,
-    {
+    logger,
-      kind: "table",
+    databaseItem,
-      "tags contain all": ["modeleditor", "endpoints", modeTag(mode)],
+    language,
-    },
+    queryStorageDir,
-    additionalPackPaths,
+    queryDir,
+    progress,
+    token,
+  }: RunQueryOptions,
+): Promise<Method[] | undefined> {
+  // The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
+  // move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
+  // This is intentionally not pretty code, as it will be removed soon.
+  // For a reference of what this should do in the future, see the previous implementation in
+  // https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72
+
+  progress({
+    message: "Resolving QL packs",
+    step: 1,
+    maxStep: externalApiQueriesProgressMaxStep,
+  });
+  const additionalPacks = getOnDiskWorkspaceFolders();
+  const extensionPacks = Object.keys(
+    await cliServer.resolveQlpacks(additionalPacks, true),
   );
-  if (queries.length > 1) {
+
-    throw new Error(
+  progress({
-      `Found multiple endpoints queries for ${mode}. Can't continue`,
+    message: "Resolving query",
+    step: 2,
+    maxStep: externalApiQueriesProgressMaxStep,
+  });
+
+  // Resolve the queries from either codeql/java-queries or from the temporary queryDir
+  const queryPath = await resolveEndpointsQuery(
+    cliServer,
+    databaseItem.language,
+    mode,
+    [syntheticQueryPackName],
+    [queryDir],
+  );
+  if (!queryPath) {
+    void showAndLogExceptionWithTelemetry(
+      logger,
+      telemetryListener,
+      redactableError`The ${mode} model editor query could not be found. Try re-opening the model editor. If that doesn't work, try upgrading the CodeQL libraries.`,
     );
+    return;
   }
 
-  if (queries.length === 0) {
+  // Run the actual query
+  const completedQuery = await runQuery({
+    queryRunner,
+    databaseItem,
+    queryPath,
+    queryStorageDir,
+    additionalPacks,
+    extensionPacks,
+    progress: (update) =>
+      progress({
+        step: update.step + 500,
+        maxStep: externalApiQueriesProgressMaxStep,
+        message: update.message,
+      }),
+    token,
+  });
+
+  if (!completedQuery) {
+    return;
+  }
+
+  // Read the results and covert to internal representation
+  progress({
+    message: "Decoding results",
+    step: 1600,
+    maxStep: externalApiQueriesProgressMaxStep,
+  });
+
+  const bqrsChunk = await readQueryResults({
+    cliServer,
+    logger,
+    bqrsPath: completedQuery.outputDir.bqrsPath,
+  });
+  if (!bqrsChunk) {
+    return;
+  }
+
+  progress({
+    message: "Finalizing results",
+    step: 1950,
+    maxStep: externalApiQueriesProgressMaxStep,
+  });
+
+  return decodeBqrsToMethods(bqrsChunk, mode, language);
+}
+
+type GetResultsOptions = {
+  cliServer: Pick<CodeQLCliServer, "bqrsInfo" | "bqrsDecode">;
+  logger: NotificationLogger;
+  bqrsPath: string;
+};
+
+export async function readQueryResults({
+  cliServer,
+  logger,
+  bqrsPath,
+}: GetResultsOptions) {
+  const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
+  if (bqrsInfo["result-sets"].length !== 1) {
+    void showAndLogExceptionWithTelemetry(
+      logger,
+      telemetryListener,
+      redactableError`Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`,
+    );
     return undefined;
   }
 
-  return queries[0];
+  const resultSet = bqrsInfo["result-sets"][0];
+
+  return cliServer.bqrsDecode(bqrsPath, resultSet.name);
+}
+
+function queryNameFromMode(mode: Mode): string {
+  return `${mode.charAt(0).toUpperCase() + mode.slice(1)}ModeEndpoints.ql`;
 }

extensions/ql-vscode/src/model-editor/model-editor-view-tracker.ts

@@ -1,33 +0,0 @@
-import { Method } from "./method";
-
-interface ModelEditorViewInterface {
-  databaseUri: string;
-
-  revealMethod(method: Method): Promise<void>;
-}
-
-export class ModelEditorViewTracker<
-  T extends ModelEditorViewInterface = ModelEditorViewInterface,
-> {
-  private readonly views = new Map<string, T>();
-
-  constructor() {}
-
-  public registerView(view: T): void {
-    const databaseUri = view.databaseUri;
-
-    if (this.views.has(databaseUri)) {
-      throw new Error(`View for database ${databaseUri} already registered`);
-    }
-
-    this.views.set(databaseUri, view);
-  }
-
-  public unregisterView(view: T): void {
-    this.views.delete(view.databaseUri);
-  }
-
-  public getView(databaseUri: string): T | undefined {
-    return this.views.get(databaseUri);
-  }
-}

extensions/ql-vscode/src/model-editor/model-editor-view.ts

@@ -23,38 +23,42 @@ import {
 import { DatabaseItem, DatabaseManager } from "../databases/local-databases";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { asError, assertNever, getErrorMessage } from "../common/helpers-pure";
-import { runFlowModelQueries } from "./flow-model-queries";
 import { promptImportGithubDatabase } from "../databases/database-fetcher";
 import { App } from "../common/app";
 import { redactableError } from "../common/errors";
 import {
   externalApiQueriesProgressMaxStep,
-  runExternalApiQueries,
+  runModelEditorQueries,
-} from "./external-api-usage-queries";
+} from "./model-editor-queries";
 import { Method } from "./method";
 import { ModeledMethod } from "./modeled-method";
 import { ExtensionPack } from "./shared/extension-pack";
 import { ModelConfigListener } from "../config";
-import { INITIAL_MODE, Mode } from "./shared/mode";
+import { Mode } from "./shared/mode";
 import { loadModeledMethods, saveModeledMethods } from "./modeled-method-fs";
 import { pickExtensionPack } from "./extension-pack-picker";
-import { getLanguageDisplayName } from "../common/query-language";
+import {
+  getLanguageDisplayName,
+  QueryLanguage,
+} from "../common/query-language";
 import { AutoModeler } from "./auto-modeler";
 import { telemetryListener } from "../common/vscode/telemetry";
 import { ModelingStore } from "./modeling-store";
-import { ModelEditorViewTracker } from "./model-editor-view-tracker";
+import { ModelingEvents } from "./modeling-events";
-import { convertFromLegacyModeledMethod } from "./shared/modeled-methods-legacy";
+import { getModelsAsDataLanguage, ModelsAsDataLanguage } from "./languages";
+import { runGenerateQueries } from "./generate";
 
 export class ModelEditorView extends AbstractWebview<
   ToModelEditorMessage,
   FromModelEditorMessage
 > {
   private readonly autoModeler: AutoModeler;
+  private readonly languageDefinition: ModelsAsDataLanguage;
 
   public constructor(
     protected readonly app: App,
     private readonly modelingStore: ModelingStore,
-    private readonly viewTracker: ModelEditorViewTracker<ModelEditorView>,
+    private readonly modelingEvents: ModelingEvents,
     private readonly modelConfig: ModelConfigListener,
     private readonly databaseManager: DatabaseManager,
     private readonly cliServer: CodeQLCliServer,
@@ -63,33 +67,30 @@ export class ModelEditorView extends AbstractWebview<
     private readonly queryDir: string,
     private readonly databaseItem: DatabaseItem,
     private readonly extensionPack: ExtensionPack,
-    initialMode: Mode = INITIAL_MODE,
+    // The language is equal to databaseItem.language but is properly typed as QueryLanguage
+    private readonly language: QueryLanguage,
+    initialMode: Mode,
   ) {
     super(app);
 
     this.modelingStore.initializeStateForDb(databaseItem, initialMode);
-    this.registerToModelingStoreEvents();
+    this.registerToModelingEvents();
     this.registerToModelConfigEvents();
 
-    this.viewTracker.registerView(this);
-
     this.autoModeler = new AutoModeler(
       app,
       cliServer,
       queryRunner,
+      this.modelConfig,
+      modelingStore,
       queryStorageDir,
       databaseItem,
-      async (packageName, inProgressMethods) => {
+      language,
-        await this.postMessage({
-          t: "setInProgressMethods",
-          packageName,
-          inProgressMethods,
-        });
-      },
       async (modeledMethods) => {
         this.addModeledMethods(modeledMethods);
       },
     );
+    this.languageDefinition = getModelsAsDataLanguage(language);
   }
 
   public async openView() {
@@ -161,7 +162,7 @@ export class ModelEditorView extends AbstractWebview<
   }
 
   protected onPanelDispose(): void {
-    this.viewTracker.unregisterView(this);
+    // Nothing to do
   }
 
   protected async onMessage(msg: FromModelEditorMessage): Promise<void> {
@@ -222,7 +223,7 @@ export class ModelEditorView extends AbstractWebview<
               });
               await saveModeledMethods(
                 this.extensionPack,
-                this.databaseItem.language,
+                this.language,
                 methods,
                 modeledMethods,
                 mode,
@@ -259,6 +260,7 @@ export class ModelEditorView extends AbstractWebview<
         break;
       case "generateMethod":
         await this.generateModeledMethods();
+
         void telemetryListener?.sendUIInteraction(
           "model-editor-generate-modeled-methods",
         );
@@ -310,11 +312,8 @@ export class ModelEditorView extends AbstractWebview<
           "model-editor-hide-modeled-methods",
         );
         break;
-      case "setModeledMethod": {
+      case "setMultipleModeledMethods": {
-        this.setModeledMethods(
+        this.setModeledMethods(msg.methodSignature, msg.modeledMethods);
-          msg.method.signature,
-          convertFromLegacyModeledMethod(msg.method),
-        );
         break;
       }
       case "telemetry":
@@ -364,33 +363,55 @@ export class ModelEditorView extends AbstractWebview<
   }
 
   private async setViewState(): Promise<void> {
+    const modelsAsDataLanguage = getModelsAsDataLanguage(this.language);
+
+    const showGenerateButton =
+      this.modelConfig.flowGeneration && !!modelsAsDataLanguage.modelGeneration;
+
     const showLlmButton =
       this.databaseItem.language === "java" && this.modelConfig.llmGeneration;
 
     const sourceArchiveAvailable =
       this.databaseItem.hasSourceArchiveInExplorer();
 
+    const showModeSwitchButton =
+      this.languageDefinition.availableModes === undefined ||
+      this.languageDefinition.availableModes.length > 1;
+
     await this.postMessage({
       t: "setModelEditorViewState",
       viewState: {
         extensionPack: this.extensionPack,
-        showFlowGeneration: this.modelConfig.flowGeneration,
+        language: this.language,
+        showGenerateButton,
         showLlmButton,
         showMultipleModels: this.modelConfig.showMultipleModels,
         mode: this.modelingStore.getMode(this.databaseItem),
+        showModeSwitchButton,
         sourceArchiveAvailable,
       },
     });
   }
 
   protected async handleJumpToMethod(methodSignature: string) {
-    this.modelingStore.setSelectedMethod(this.databaseItem, methodSignature);
+    const method = this.modelingStore.getMethod(
+      this.databaseItem,
+      methodSignature,
+    );
+    if (method) {
+      this.modelingStore.setSelectedMethod(
+        this.databaseItem,
+        method,
+        method.usages[0],
+      );
+    }
   }
 
   protected async loadExistingModeledMethods(): Promise<void> {
     try {
       const modeledMethods = await loadModeledMethods(
         this.extensionPack,
+        this.language,
         this.cliServer,
         this.app.logger,
       );
@@ -408,10 +429,12 @@ export class ModelEditorView extends AbstractWebview<
 
     try {
       const cancellationTokenSource = new CancellationTokenSource();
-      const queryResult = await runExternalApiQueries(mode, {
+      const queryResult = await runModelEditorQueries(mode, {
         cliServer: this.cliServer,
         queryRunner: this.queryRunner,
+        logger: this.app.logger,
         databaseItem: this.databaseItem,
+        language: this.language,
         queryStorageDir: this.queryStorageDir,
         queryDir: this.queryDir,
         progress: (update) =>
@@ -430,9 +453,9 @@ export class ModelEditorView extends AbstractWebview<
       void showAndLogExceptionWithTelemetry(
         this.app.logger,
         this.app.telemetry,
-        redactableError(
+        redactableError(asError(err))`Failed to load results: ${getErrorMessage(
-          asError(err),
+          err,
-        )`Failed to load external API usages: ${getErrorMessage(err)}`,
+        )}`,
       );
     }
   }
@@ -444,6 +467,16 @@ export class ModelEditorView extends AbstractWebview<
 
         const mode = this.modelingStore.getMode(this.databaseItem);
 
+        const modelsAsDataLanguage = getModelsAsDataLanguage(this.language);
+        const modelGeneration = modelsAsDataLanguage.modelGeneration;
+        if (!modelGeneration) {
+          void showAndLogErrorMessage(
+            this.app.logger,
+            `Model generation is not supported for ${this.language}.`,
+          );
+          return;
+        }
+
         let addedDatabase: DatabaseItem | undefined;
 
         // In application mode, we need the database of a specific library to generate
@@ -455,6 +488,14 @@ export class ModelEditorView extends AbstractWebview<
           if (!addedDatabase) {
             return;
           }
+
+          if (addedDatabase.language !== this.language) {
+            void showAndLogErrorMessage(
+              this.app.logger,
+              `The selected database is for ${addedDatabase.language}, but the current database is for ${this.language}.`,
+            );
+            return;
+          }
         }
 
         progress({
@@ -464,26 +505,23 @@ export class ModelEditorView extends AbstractWebview<
         });
 
         try {
-          await runFlowModelQueries({
+          await runGenerateQueries({
+            queryConstraints: modelGeneration.queryConstraints,
+            filterQueries: modelGeneration.filterQueries,
+            parseResults: (queryPath, results) =>
+              modelGeneration.parseResults(
+                queryPath,
+                results,
+                modelsAsDataLanguage,
+                this.app.logger,
+              ),
+            onResults: async (modeledMethods) => {
+              this.addModeledMethodsFromArray(modeledMethods);
+            },
             cliServer: this.cliServer,
             queryRunner: this.queryRunner,
             queryStorageDir: this.queryStorageDir,
             databaseItem: addedDatabase ?? this.databaseItem,
-            onResults: async (modeledMethods) => {
-              const modeledMethodsByName: Record<string, ModeledMethod[]> = {};
-
-              for (const modeledMethod of modeledMethods) {
-                if (!(modeledMethod.signature in modeledMethodsByName)) {
-                  modeledMethodsByName[modeledMethod.signature] = [];
-                }
-
-                modeledMethodsByName[modeledMethod.signature].push(
-                  modeledMethod,
-                );
-              }
-
-              this.addModeledMethods(modeledMethodsByName);
-            },
             progress,
             token: tokenSource.token,
           });
@@ -493,7 +531,7 @@ export class ModelEditorView extends AbstractWebview<
             this.app.telemetry,
             redactableError(
               asError(e),
-            )`Failed to generate flow model: ${getErrorMessage(e)}`,
+            )`Failed to generate models: ${getErrorMessage(e)}`,
           );
         }
       },
@@ -531,12 +569,9 @@ export class ModelEditorView extends AbstractWebview<
         return;
       }
 
-      let existingView = this.viewTracker.getView(
+      const addedDbUri = addedDatabase.databaseUri.toString();
-        addedDatabase.databaseUri.toString(),
+      if (this.modelingStore.isDbOpen(addedDbUri)) {
-      );
+        this.modelingEvents.fireFocusModelEditorEvent(addedDbUri);
-      if (existingView) {
-        await existingView.focusView();
-
         return;
       }
 
@@ -554,19 +589,15 @@ export class ModelEditorView extends AbstractWebview<
 
       // Check again just before opening the editor to ensure no model editor has been opened between
       // our first check and now.
-      existingView = this.viewTracker.getView(
+      if (this.modelingStore.isDbOpen(addedDbUri)) {
-        addedDatabase.databaseUri.toString(),
+        this.modelingEvents.fireFocusModelEditorEvent(addedDbUri);
-      );
-      if (existingView) {
-        await existingView.focusView();
-
         return;
       }
 
       const view = new ModelEditorView(
         this.app,
         this.modelingStore,
-        this.viewTracker,
+        this.modelingEvents,
         this.modelConfig,
         this.databaseManager,
         this.cliServer,
@@ -575,6 +606,7 @@ export class ModelEditorView extends AbstractWebview<
         this.queryDir,
         addedDatabase,
         modelFile,
+        this.language,
         Mode.Framework,
       );
       await view.openView();
@@ -654,9 +686,9 @@ export class ModelEditorView extends AbstractWebview<
     return addedDatabase;
   }
 
-  private registerToModelingStoreEvents() {
+  private registerToModelingEvents() {
     this.push(
-      this.modelingStore.onMethodsChanged(async (event) => {
+      this.modelingEvents.onMethodsChanged(async (event) => {
         if (event.dbUri === this.databaseItem.databaseUri.toString()) {
           await this.postMessage({
             t: "setMethods",
@@ -667,7 +699,7 @@ export class ModelEditorView extends AbstractWebview<
     );
 
     this.push(
-      this.modelingStore.onModeledMethodsChanged(async (event) => {
+      this.modelingEvents.onModeledMethodsChanged(async (event) => {
         if (event.dbUri === this.databaseItem.databaseUri.toString()) {
           await this.postMessage({
             t: "setModeledMethods",
@@ -678,7 +710,7 @@ export class ModelEditorView extends AbstractWebview<
     );
 
     this.push(
-      this.modelingStore.onModifiedMethodsChanged(async (event) => {
+      this.modelingEvents.onModifiedMethodsChanged(async (event) => {
         if (event.dbUri === this.databaseItem.databaseUri.toString()) {
           await this.postMessage({
             t: "setModifiedMethods",
@@ -687,6 +719,33 @@ export class ModelEditorView extends AbstractWebview<
         }
       }),
     );
+
+    this.push(
+      this.modelingEvents.onInProgressMethodsChanged(async (event) => {
+        if (event.dbUri === this.databaseItem.databaseUri.toString()) {
+          await this.postMessage({
+            t: "setInProgressMethods",
+            methods: Array.from(event.methods),
+          });
+        }
+      }),
+    );
+
+    this.push(
+      this.modelingEvents.onRevealInModelEditor(async (event) => {
+        if (event.dbUri === this.databaseItem.databaseUri.toString()) {
+          await this.revealMethod(event.method);
+        }
+      }),
+    );
+
+    this.push(
+      this.modelingEvents.onFocusModelEditor(async (event) => {
+        if (event.dbUri === this.databaseItem.databaseUri.toString()) {
+          await this.focusView();
+        }
+      }),
+    );
   }
 
   private registerToModelConfigEvents() {
@@ -706,6 +765,20 @@ export class ModelEditorView extends AbstractWebview<
     );
   }
 
+  private addModeledMethodsFromArray(modeledMethods: ModeledMethod[]) {
+    const modeledMethodsByName: Record<string, ModeledMethod[]> = {};
+
+    for (const modeledMethod of modeledMethods) {
+      if (!(modeledMethod.signature in modeledMethodsByName)) {
+        modeledMethodsByName[modeledMethod.signature] = [];
+      }
+
+      modeledMethodsByName[modeledMethod.signature].push(modeledMethod);
+    }
+
+    this.addModeledMethods(modeledMethodsByName);
+  }
+
   private setModeledMethods(signature: string, methods: ModeledMethod[]) {
     this.modelingStore.updateModeledMethods(
       this.databaseItem,

extensions/ql-vscode/src/model-editor/modeled-method-empty.ts

@@ -0,0 +1,101 @@
+import { ModeledMethod, SinkModeledMethod } from "./modeled-method";
+import { MethodSignature } from "./method";
+import { assertNever } from "../common/helpers-pure";
+
+export function createEmptyModeledMethod(
+  type: ModeledMethod["type"],
+  methodSignature: MethodSignature,
+) {
+  const canonicalMethodSignature: MethodSignature = {
+    packageName: methodSignature.packageName,
+    typeName: methodSignature.typeName,
+    methodName: methodSignature.methodName,
+    methodParameters: methodSignature.methodParameters,
+    signature: methodSignature.signature,
+  };
+
+  switch (type) {
+    case "none":
+      return createEmptyNoneModeledMethod(canonicalMethodSignature);
+    case "source":
+      return createEmptySourceModeledMethod(canonicalMethodSignature);
+    case "sink":
+      return createEmptySinkModeledMethod(canonicalMethodSignature);
+    case "summary":
+      return createEmptySummaryModeledMethod(canonicalMethodSignature);
+    case "neutral":
+      return createEmptyNeutralModeledMethod(canonicalMethodSignature);
+    case "type":
+      return createEmptyTypeModeledMethod(canonicalMethodSignature);
+    default:
+      assertNever(type);
+  }
+}
+
+function createEmptyNoneModeledMethod(
+  methodSignature: MethodSignature,
+): ModeledMethod {
+  return {
+    ...methodSignature,
+    type: "none",
+  };
+}
+
+function createEmptySourceModeledMethod(
+  methodSignature: MethodSignature,
+): ModeledMethod {
+  return {
+    ...methodSignature,
+    type: "source",
+    output: "",
+    kind: "",
+    provenance: "manual",
+  };
+}
+
+function createEmptySinkModeledMethod(
+  methodSignature: MethodSignature,
+): SinkModeledMethod {
+  return {
+    ...methodSignature,
+    type: "sink",
+    input: "",
+    kind: "",
+    provenance: "manual",
+  };
+}
+
+function createEmptySummaryModeledMethod(
+  methodSignature: MethodSignature,
+): ModeledMethod {
+  return {
+    ...methodSignature,
+    type: "summary",
+    input: "",
+    output: "",
+    kind: "",
+    provenance: "manual",
+  };
+}
+
+function createEmptyNeutralModeledMethod(
+  methodSignature: MethodSignature,
+): ModeledMethod {
+  return {
+    ...methodSignature,
+    type: "neutral",
+    kind: "",
+    provenance: "manual",
+  };
+}
+
+function createEmptyTypeModeledMethod(
+  methodSignature: MethodSignature,
+): ModeledMethod {
+  return {
+    ...methodSignature,
+    type: "type",
+    relatedTypeName: "",
+    path: "",
+  };
+}

extensions/ql-vscode/src/model-editor/modeled-method-fs.ts

@@ -10,18 +10,20 @@ import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { load as loadYaml } from "js-yaml";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { pathsEqual } from "../common/files";
+import { QueryLanguage } from "../common/query-language";
 
 export async function saveModeledMethods(
   extensionPack: ExtensionPack,
-  language: string,
+  language: QueryLanguage,
-  methods: Method[],
+  methods: readonly Method[],
-  modeledMethods: Record<string, ModeledMethod[]>,
+  modeledMethods: Readonly<Record<string, readonly ModeledMethod[]>>,
   mode: Mode,
   cliServer: CodeQLCliServer,
   logger: NotificationLogger,
 ): Promise<void> {
   const existingModeledMethods = await loadModeledMethodFiles(
     extensionPack,
+    language,
     cliServer,
     logger,
   );
@@ -43,6 +45,7 @@ export async function saveModeledMethods(
 
 async function loadModeledMethodFiles(
   extensionPack: ExtensionPack,
+  language: QueryLanguage,
   cliServer: CodeQLCliServer,
   logger: NotificationLogger,
 ): Promise<Record<string, Record<string, ModeledMethod[]>>> {
@@ -60,7 +63,7 @@ async function loadModeledMethodFiles(
       filename: modelFile,
     });
 
-    const modeledMethods = loadDataExtensionYaml(data);
+    const modeledMethods = loadDataExtensionYaml(data, language);
     if (!modeledMethods) {
       void showAndLogErrorMessage(
         logger,
@@ -76,6 +79,7 @@ async function loadModeledMethodFiles(
 
 export async function loadModeledMethods(
   extensionPack: ExtensionPack,
+  language: QueryLanguage,
   cliServer: CodeQLCliServer,
   logger: NotificationLogger,
 ): Promise<Record<string, ModeledMethod[]>> {
@@ -83,6 +87,7 @@ export async function loadModeledMethods(
 
   const modeledMethodsByFile = await loadModeledMethodFiles(
     extensionPack,
+    language,
     cliServer,
     logger,
   );

extensions/ql-vscode/src/model-editor/modeled-method.ts

@@ -1,10 +1,12 @@
 import { MethodSignature } from "./method";
+import { ModelingStatus } from "./shared/modeling-status";
 
 export type ModeledMethodType =
   | "none"
   | "source"
   | "sink"
   | "summary"
+  | "type"
   | "neutral";
 
 export type Provenance =
@@ -19,12 +21,143 @@ export type Provenance =
   // Entered by the user in the editor manually
   | "manual";
 
-export interface ModeledMethod extends MethodSignature {
+export interface NoneModeledMethod extends MethodSignature {
-  type: ModeledMethodType;
+  readonly type: "none";
-  input: string;
-  output: string;
-  kind: ModeledMethodKind;
-  provenance: Provenance;
 }
 
+export interface SourceModeledMethod extends MethodSignature {
+  readonly type: "source";
+  readonly output: string;
+  readonly kind: ModeledMethodKind;
+  readonly provenance: Provenance;
+}
+
+export interface SinkModeledMethod extends MethodSignature {
+  readonly type: "sink";
+  readonly input: string;
+  readonly kind: ModeledMethodKind;
+  readonly provenance: Provenance;
+}
+
+export interface SummaryModeledMethod extends MethodSignature {
+  readonly type: "summary";
+  readonly input: string;
+  readonly output: string;
+  readonly kind: ModeledMethodKind;
+  readonly provenance: Provenance;
+}
+
+export interface NeutralModeledMethod extends MethodSignature {
+  readonly type: "neutral";
+  readonly kind: ModeledMethodKind;
+  readonly provenance: Provenance;
+}
+
+export interface TypeModeledMethod extends MethodSignature {
+  readonly type: "type";
+  readonly relatedTypeName: string;
+  readonly path: string;
+}
+
+export type ModeledMethod =
+  | NoneModeledMethod
+  | SourceModeledMethod
+  | SinkModeledMethod
+  | SummaryModeledMethod
+  | NeutralModeledMethod
+  | TypeModeledMethod;
+
 export type ModeledMethodKind = string;
+
+export function modeledMethodSupportsKind(
+  modeledMethod: ModeledMethod,
+): modeledMethod is
+  | SourceModeledMethod
+  | SinkModeledMethod
+  | SummaryModeledMethod
+  | NeutralModeledMethod {
+  return (
+    modeledMethod.type === "source" ||
+    modeledMethod.type === "sink" ||
+    modeledMethod.type === "summary" ||
+    modeledMethod.type === "neutral"
+  );
+}
+
+export function modeledMethodSupportsInput(
+  modeledMethod: ModeledMethod,
+): modeledMethod is SinkModeledMethod | SummaryModeledMethod {
+  return modeledMethod.type === "sink" || modeledMethod.type === "summary";
+}
+
+export function modeledMethodSupportsOutput(
+  modeledMethod: ModeledMethod,
+): modeledMethod is SourceModeledMethod | SummaryModeledMethod {
+  return modeledMethod.type === "source" || modeledMethod.type === "summary";
+}
+
+export function modeledMethodSupportsProvenance(
+  modeledMethod: ModeledMethod,
+): modeledMethod is
+  | SourceModeledMethod
+  | SinkModeledMethod
+  | SummaryModeledMethod
+  | NeutralModeledMethod {
+  return (
+    modeledMethod.type === "source" ||
+    modeledMethod.type === "sink" ||
+    modeledMethod.type === "summary" ||
+    modeledMethod.type === "neutral"
+  );
+}
+
+export function isModelAccepted(
+  modeledMethod: ModeledMethod | undefined,
+  modelingStatus: ModelingStatus,
+): boolean {
+  if (!modeledMethod) {
+    return true;
+  }
+
+  return (
+    modelingStatus !== "unsaved" ||
+    modeledMethod.type === "none" ||
+    !modeledMethodSupportsProvenance(modeledMethod) ||
+    modeledMethod.provenance !== "ai-generated"
+  );
+}
+
+/**
+ * Calculates the new provenance for a modeled method based on the current provenance.
+ * @param modeledMethod The modeled method if there is one.
+ * @returns The new provenance.
+ */
+export function calculateNewProvenance(
+  modeledMethod: ModeledMethod | undefined,
+) {
+  if (!modeledMethod || !modeledMethodSupportsProvenance(modeledMethod)) {
+    // If nothing has been modeled or the modeled method does not support
+    // provenance, we assume that the user has entered it manually.
+    return "manual";
+  }
+
+  switch (modeledMethod.provenance) {
+    case "df-generated":
+      // If the method has been generated and there has been a change, we assume
+      // that the user has manually edited it.
+      return "df-manual";
+    case "df-manual":
+      // If the method has had manual edits, we want the provenance to stay the same.
+      return "df-manual";
+    case "ai-generated":
+      // If the method has been generated and there has been a change, we assume
+      // that the user has manually edited it.
+      return "ai-manual";
+    case "ai-manual":
+      // If the method has had manual edits, we want the provenance to stay the same.
+      return "ai-manual";
+    default:
+      // The method has been modeled manually.
+      return "manual";
+  }
+}

extensions/ql-vscode/src/model-editor/modeling-events.ts

@@ -0,0 +1,259 @@
+import { App } from "../common/app";
+import { DisposableObject } from "../common/disposable-object";
+import { AppEvent, AppEventEmitter } from "../common/events";
+import { DatabaseItem } from "../databases/local-databases";
+import { Method, Usage } from "./method";
+import { ModeledMethod } from "./modeled-method";
+import { Mode } from "./shared/mode";
+
+interface MethodsChangedEvent {
+  readonly methods: readonly Method[];
+  readonly dbUri: string;
+  readonly isActiveDb: boolean;
+}
+
+interface HideModeledMethodsChangedEvent {
+  readonly hideModeledMethods: boolean;
+  readonly isActiveDb: boolean;
+}
+
+interface ModeChangedEvent {
+  readonly mode: Mode;
+  readonly isActiveDb: boolean;
+}
+
+interface ModeledMethodsChangedEvent {
+  readonly modeledMethods: Readonly<Record<string, ModeledMethod[]>>;
+  readonly dbUri: string;
+  readonly isActiveDb: boolean;
+}
+
+interface ModifiedMethodsChangedEvent {
+  readonly modifiedMethods: ReadonlySet<string>;
+  readonly dbUri: string;
+  readonly isActiveDb: boolean;
+}
+
+interface SelectedMethodChangedEvent {
+  readonly databaseItem: DatabaseItem;
+  readonly method: Method;
+  readonly usage: Usage;
+  readonly modeledMethods: readonly ModeledMethod[];
+  readonly isModified: boolean;
+  readonly isInProgress: boolean;
+}
+
+interface InProgressMethodsChangedEvent {
+  readonly dbUri: string;
+  readonly methods: ReadonlySet<string>;
+}
+
+interface RevealInModelEditorEvent {
+  dbUri: string;
+  method: Method;
+}
+
+interface FocusModelEditorEvent {
+  dbUri: string;
+}
+
+export class ModelingEvents extends DisposableObject {
+  public readonly onActiveDbChanged: AppEvent<void>;
+  public readonly onDbOpened: AppEvent<DatabaseItem>;
+  public readonly onDbClosed: AppEvent<string>;
+  public readonly onMethodsChanged: AppEvent<MethodsChangedEvent>;
+  public readonly onHideModeledMethodsChanged: AppEvent<HideModeledMethodsChangedEvent>;
+  public readonly onModeChanged: AppEvent<ModeChangedEvent>;
+  public readonly onModeledMethodsChanged: AppEvent<ModeledMethodsChangedEvent>;
+  public readonly onModifiedMethodsChanged: AppEvent<ModifiedMethodsChangedEvent>;
+  public readonly onSelectedMethodChanged: AppEvent<SelectedMethodChangedEvent>;
+  public readonly onInProgressMethodsChanged: AppEvent<InProgressMethodsChangedEvent>;
+  public readonly onRevealInModelEditor: AppEvent<RevealInModelEditorEvent>;
+  public readonly onFocusModelEditor: AppEvent<FocusModelEditorEvent>;
+
+  private readonly onActiveDbChangedEventEmitter: AppEventEmitter<void>;
+  private readonly onDbOpenedEventEmitter: AppEventEmitter<DatabaseItem>;
+  private readonly onDbClosedEventEmitter: AppEventEmitter<string>;
+  private readonly onMethodsChangedEventEmitter: AppEventEmitter<MethodsChangedEvent>;
+  private readonly onHideModeledMethodsChangedEventEmitter: AppEventEmitter<HideModeledMethodsChangedEvent>;
+  private readonly onModeChangedEventEmitter: AppEventEmitter<ModeChangedEvent>;
+  private readonly onModeledMethodsChangedEventEmitter: AppEventEmitter<ModeledMethodsChangedEvent>;
+  private readonly onModifiedMethodsChangedEventEmitter: AppEventEmitter<ModifiedMethodsChangedEvent>;
+  private readonly onSelectedMethodChangedEventEmitter: AppEventEmitter<SelectedMethodChangedEvent>;
+  private readonly onInProgressMethodsChangedEventEmitter: AppEventEmitter<InProgressMethodsChangedEvent>;
+  private readonly onRevealInModelEditorEventEmitter: AppEventEmitter<RevealInModelEditorEvent>;
+  private readonly onFocusModelEditorEventEmitter: AppEventEmitter<FocusModelEditorEvent>;
+
+  constructor(app: App) {
+    super();
+
+    this.onActiveDbChangedEventEmitter = this.push(
+      app.createEventEmitter<void>(),
+    );
+    this.onActiveDbChanged = this.onActiveDbChangedEventEmitter.event;
+
+    this.onDbOpenedEventEmitter = this.push(
+      app.createEventEmitter<DatabaseItem>(),
+    );
+    this.onDbOpened = this.onDbOpenedEventEmitter.event;
+
+    this.onDbClosedEventEmitter = this.push(app.createEventEmitter<string>());
+    this.onDbClosed = this.onDbClosedEventEmitter.event;
+
+    this.onMethodsChangedEventEmitter = this.push(
+      app.createEventEmitter<MethodsChangedEvent>(),
+    );
+    this.onMethodsChanged = this.onMethodsChangedEventEmitter.event;
+
+    this.onHideModeledMethodsChangedEventEmitter = this.push(
+      app.createEventEmitter<HideModeledMethodsChangedEvent>(),
+    );
+    this.onHideModeledMethodsChanged =
+      this.onHideModeledMethodsChangedEventEmitter.event;
+
+    this.onModeChangedEventEmitter = this.push(
+      app.createEventEmitter<ModeChangedEvent>(),
+    );
+    this.onModeChanged = this.onModeChangedEventEmitter.event;
+
+    this.onModeledMethodsChangedEventEmitter = this.push(
+      app.createEventEmitter<ModeledMethodsChangedEvent>(),
+    );
+    this.onModeledMethodsChanged =
+      this.onModeledMethodsChangedEventEmitter.event;
+
+    this.onModifiedMethodsChangedEventEmitter = this.push(
+      app.createEventEmitter<ModifiedMethodsChangedEvent>(),
+    );
+    this.onModifiedMethodsChanged =
+      this.onModifiedMethodsChangedEventEmitter.event;
+
+    this.onSelectedMethodChangedEventEmitter = this.push(
+      app.createEventEmitter<SelectedMethodChangedEvent>(),
+    );
+    this.onSelectedMethodChanged =
+      this.onSelectedMethodChangedEventEmitter.event;
+
+    this.onInProgressMethodsChangedEventEmitter = this.push(
+      app.createEventEmitter<InProgressMethodsChangedEvent>(),
+    );
+    this.onInProgressMethodsChanged =
+      this.onInProgressMethodsChangedEventEmitter.event;
+
+    this.onRevealInModelEditorEventEmitter = this.push(
+      app.createEventEmitter<RevealInModelEditorEvent>(),
+    );
+    this.onRevealInModelEditor = this.onRevealInModelEditorEventEmitter.event;
+
+    this.onFocusModelEditorEventEmitter = this.push(
+      app.createEventEmitter<FocusModelEditorEvent>(),
+    );
+    this.onFocusModelEditor = this.onFocusModelEditorEventEmitter.event;
+  }
+
+  public fireActiveDbChangedEvent() {
+    this.onActiveDbChangedEventEmitter.fire();
+  }
+
+  public fireDbOpenedEvent(databaseItem: DatabaseItem) {
+    this.onDbOpenedEventEmitter.fire(databaseItem);
+  }
+
+  public fireDbClosedEvent(dbUri: string) {
+    this.onDbClosedEventEmitter.fire(dbUri);
+  }
+
+  public fireMethodsChangedEvent(
+    methods: Method[],
+    dbUri: string,
+    isActiveDb: boolean,
+  ) {
+    this.onMethodsChangedEventEmitter.fire({
+      methods,
+      dbUri,
+      isActiveDb,
+    });
+  }
+
+  public fireHideModeledMethodsChangedEvent(
+    hideModeledMethods: boolean,
+    isActiveDb: boolean,
+  ) {
+    this.onHideModeledMethodsChangedEventEmitter.fire({
+      hideModeledMethods,
+      isActiveDb,
+    });
+  }
+
+  public fireModeChangedEvent(mode: Mode, isActiveDb: boolean) {
+    this.onModeChangedEventEmitter.fire({
+      mode,
+      isActiveDb,
+    });
+  }
+
+  public fireModeledMethodsChangedEvent(
+    modeledMethods: Record<string, ModeledMethod[]>,
+    dbUri: string,
+    isActiveDb: boolean,
+  ) {
+    this.onModeledMethodsChangedEventEmitter.fire({
+      modeledMethods,
+      dbUri,
+      isActiveDb,
+    });
+  }
+
+  public fireModifiedMethodsChangedEvent(
+    modifiedMethods: ReadonlySet<string>,
+    dbUri: string,
+    isActiveDb: boolean,
+  ) {
+    this.onModifiedMethodsChangedEventEmitter.fire({
+      modifiedMethods,
+      dbUri,
+      isActiveDb,
+    });
+  }
+
+  public fireSelectedMethodChangedEvent(
+    databaseItem: DatabaseItem,
+    method: Method,
+    usage: Usage,
+    modeledMethods: ModeledMethod[],
+    isModified: boolean,
+    isInProgress: boolean,
+  ) {
+    this.onSelectedMethodChangedEventEmitter.fire({
+      databaseItem,
+      method,
+      usage,
+      modeledMethods,
+      isModified,
+      isInProgress,
+    });
+  }
+
+  public fireInProgressMethodsChangedEvent(
+    dbUri: string,
+    methods: ReadonlySet<string>,
+  ) {
+    this.onInProgressMethodsChangedEventEmitter.fire({
+      dbUri,
+      methods,
+    });
+  }
+
+  public fireRevealInModelEditorEvent(dbUri: string, method: Method) {
+    this.onRevealInModelEditorEventEmitter.fire({
+      dbUri,
+      method,
+    });
+  }
+
+  public fireFocusModelEditorEvent(dbUri: string) {
+    this.onFocusModelEditorEventEmitter.fire({
+      dbUri,
+    });
+  }
+}

extensions/ql-vscode/src/model-editor/modeling-store.ts

@@ -1,140 +1,56 @@
-import { App } from "../common/app";
 import { DisposableObject } from "../common/disposable-object";
-import { AppEvent, AppEventEmitter } from "../common/events";
 import { DatabaseItem } from "../databases/local-databases";
 import { Method, Usage } from "./method";
 import { ModeledMethod } from "./modeled-method";
+import { ModelingEvents } from "./modeling-events";
 import { INITIAL_HIDE_MODELED_METHODS_VALUE } from "./shared/hide-modeled-methods";
-import { INITIAL_MODE, Mode } from "./shared/mode";
+import { Mode } from "./shared/mode";
 
-interface DbModelingState {
+interface InternalDbModelingState {
   databaseItem: DatabaseItem;
   methods: Method[];
   hideModeledMethods: boolean;
   mode: Mode;
   modeledMethods: Record<string, ModeledMethod[]>;
   modifiedMethodSignatures: Set<string>;
+  inProgressMethods: Set<string>;
   selectedMethod: Method | undefined;
   selectedUsage: Usage | undefined;
 }
 
-interface MethodsChangedEvent {
+interface DbModelingState {
-  methods: Method[];
+  readonly databaseItem: DatabaseItem;
-  dbUri: string;
+  readonly methods: readonly Method[];
-  isActiveDb: boolean;
+  readonly hideModeledMethods: boolean;
+  readonly mode: Mode;
+  readonly modeledMethods: Readonly<Record<string, readonly ModeledMethod[]>>;
+  readonly modifiedMethodSignatures: ReadonlySet<string>;
+  readonly inProgressMethods: ReadonlySet<string>;
+  readonly selectedMethod: Method | undefined;
+  readonly selectedUsage: Usage | undefined;
 }
 
-interface HideModeledMethodsChangedEvent {
+interface SelectedMethodDetails {
-  hideModeledMethods: boolean;
+  readonly databaseItem: DatabaseItem;
-  isActiveDb: boolean;
+  readonly method: Method;
-}
+  readonly usage: Usage | undefined;
-
+  readonly modeledMethods: readonly ModeledMethod[];
-interface ModeChangedEvent {
+  readonly isModified: boolean;
-  mode: Mode;
+  readonly isInProgress: boolean;
-  isActiveDb: boolean;
-}
-
-interface ModeledMethodsChangedEvent {
-  modeledMethods: Record<string, ModeledMethod[]>;
-  dbUri: string;
-  isActiveDb: boolean;
-}
-
-interface ModifiedMethodsChangedEvent {
-  modifiedMethods: Set<string>;
-  dbUri: string;
-  isActiveDb: boolean;
-}
-
-interface SelectedMethodChangedEvent {
-  databaseItem: DatabaseItem;
-  method: Method;
-  usage: Usage;
-  modeledMethods: ModeledMethod[];
-  isModified: boolean;
 }
 
 export class ModelingStore extends DisposableObject {
-  public readonly onActiveDbChanged: AppEvent<void>;
+  private readonly state: Map<string, InternalDbModelingState>;
-  public readonly onDbOpened: AppEvent<string>;
-  public readonly onDbClosed: AppEvent<string>;
-  public readonly onMethodsChanged: AppEvent<MethodsChangedEvent>;
-  public readonly onHideModeledMethodsChanged: AppEvent<HideModeledMethodsChangedEvent>;
-  public readonly onModeChanged: AppEvent<ModeChangedEvent>;
-  public readonly onModeledMethodsChanged: AppEvent<ModeledMethodsChangedEvent>;
-  public readonly onModifiedMethodsChanged: AppEvent<ModifiedMethodsChangedEvent>;
-  public readonly onSelectedMethodChanged: AppEvent<SelectedMethodChangedEvent>;
-
-  private readonly state: Map<string, DbModelingState>;
   private activeDb: string | undefined;
 
-  private readonly onActiveDbChangedEventEmitter: AppEventEmitter<void>;
+  constructor(private readonly modelingEvents: ModelingEvents) {
-  private readonly onDbOpenedEventEmitter: AppEventEmitter<string>;
-  private readonly onDbClosedEventEmitter: AppEventEmitter<string>;
-  private readonly onMethodsChangedEventEmitter: AppEventEmitter<MethodsChangedEvent>;
-  private readonly onHideModeledMethodsChangedEventEmitter: AppEventEmitter<HideModeledMethodsChangedEvent>;
-  private readonly onModeChangedEventEmitter: AppEventEmitter<ModeChangedEvent>;
-  private readonly onModeledMethodsChangedEventEmitter: AppEventEmitter<ModeledMethodsChangedEvent>;
-  private readonly onModifiedMethodsChangedEventEmitter: AppEventEmitter<ModifiedMethodsChangedEvent>;
-  private readonly onSelectedMethodChangedEventEmitter: AppEventEmitter<SelectedMethodChangedEvent>;
-
-  constructor(app: App) {
     super();
 
     // State initialization
-    this.state = new Map<string, DbModelingState>();
+    this.state = new Map<string, InternalDbModelingState>();
-
-    // Event initialization
-    this.onActiveDbChangedEventEmitter = this.push(
-      app.createEventEmitter<void>(),
-    );
-    this.onActiveDbChanged = this.onActiveDbChangedEventEmitter.event;
-
-    this.onDbOpenedEventEmitter = this.push(app.createEventEmitter<string>());
-    this.onDbOpened = this.onDbOpenedEventEmitter.event;
-
-    this.onDbClosedEventEmitter = this.push(app.createEventEmitter<string>());
-    this.onDbClosed = this.onDbClosedEventEmitter.event;
-
-    this.onMethodsChangedEventEmitter = this.push(
-      app.createEventEmitter<MethodsChangedEvent>(),
-    );
-    this.onMethodsChanged = this.onMethodsChangedEventEmitter.event;
-
-    this.onHideModeledMethodsChangedEventEmitter = this.push(
-      app.createEventEmitter<HideModeledMethodsChangedEvent>(),
-    );
-    this.onHideModeledMethodsChanged =
-      this.onHideModeledMethodsChangedEventEmitter.event;
-
-    this.onModeChangedEventEmitter = this.push(
-      app.createEventEmitter<ModeChangedEvent>(),
-    );
-    this.onModeChanged = this.onModeChangedEventEmitter.event;
-
-    this.onModeledMethodsChangedEventEmitter = this.push(
-      app.createEventEmitter<ModeledMethodsChangedEvent>(),
-    );
-    this.onModeledMethodsChanged =
-      this.onModeledMethodsChangedEventEmitter.event;
-
-    this.onModifiedMethodsChangedEventEmitter = this.push(
-      app.createEventEmitter<ModifiedMethodsChangedEvent>(),
-    );
-    this.onModifiedMethodsChanged =
-      this.onModifiedMethodsChangedEventEmitter.event;
-
-    this.onSelectedMethodChangedEventEmitter = this.push(
-      app.createEventEmitter<SelectedMethodChangedEvent>(),
-    );
-    this.onSelectedMethodChanged =
-      this.onSelectedMethodChangedEventEmitter.event;
   }
 
-  public initializeStateForDb(
+  public initializeStateForDb(databaseItem: DatabaseItem, mode: Mode) {
-    databaseItem: DatabaseItem,
-    mode: Mode = INITIAL_MODE,
-  ) {
     const dbUri = databaseItem.databaseUri.toString();
     this.state.set(dbUri, {
       databaseItem,
@@ -145,14 +61,15 @@ export class ModelingStore extends DisposableObject {
       modifiedMethodSignatures: new Set(),
       selectedMethod: undefined,
       selectedUsage: undefined,
+      inProgressMethods: new Set(),
     });
 
-    this.onDbOpenedEventEmitter.fire(dbUri);
+    this.modelingEvents.fireDbOpenedEvent(databaseItem);
   }
 
   public setActiveDb(databaseItem: DatabaseItem) {
     this.activeDb = databaseItem.databaseUri.toString();
-    this.onActiveDbChangedEventEmitter.fire();
+    this.modelingEvents.fireActiveDbChangedEvent();
   }
 
   public removeDb(databaseItem: DatabaseItem) {
@@ -164,11 +81,11 @@ export class ModelingStore extends DisposableObject {
 
     if (this.activeDb === dbUri) {
       this.activeDb = undefined;
-      this.onActiveDbChangedEventEmitter.fire();
+      this.modelingEvents.fireActiveDbChangedEvent();
     }
 
     this.state.delete(dbUri);
-    this.onDbClosedEventEmitter.fire(dbUri);
+    this.modelingEvents.fireDbClosedEvent(dbUri);
   }
 
   public getStateForActiveDb(): DbModelingState | undefined {
@@ -179,6 +96,14 @@ export class ModelingStore extends DisposableObject {
     return this.state.get(this.activeDb);
   }
 
+  private getInternalStateForActiveDb(): InternalDbModelingState | undefined {
+    if (!this.activeDb) {
+      return undefined;
+    }
+
+    return this.state.get(this.activeDb);
+  }
+
   public hasStateForActiveDb(): boolean {
     return !!this.getStateForActiveDb();
   }
@@ -187,6 +112,23 @@ export class ModelingStore extends DisposableObject {
     return this.state.size > 0;
   }
 
+  public isDbOpen(dbUri: string): boolean {
+    return this.state.has(dbUri);
+  }
+
+  /**
+   * Returns the method for the given database item and method signature.
+   * Returns undefined if no method exists with that signature.
+   */
+  public getMethod(
+    dbItem: DatabaseItem,
+    methodSignature: string,
+  ): Method | undefined {
+    return this.getState(dbItem).methods.find(
+      (m) => m.signature === methodSignature,
+    );
+  }
+
   /**
    * Returns the methods for the given database item and method signatures.
    * If the `methodSignatures` argument is not provided or is undefined, returns all methods.
@@ -194,7 +136,7 @@ export class ModelingStore extends DisposableObject {
   public getMethods(
     dbItem: DatabaseItem,
     methodSignatures?: string[],
-  ): Method[] {
+  ): readonly Method[] {
     const methods = this.getState(dbItem).methods;
     if (!methodSignatures) {
       return methods;
@@ -210,11 +152,11 @@ export class ModelingStore extends DisposableObject {
 
     dbState.methods = [...methods];
 
-    this.onMethodsChangedEventEmitter.fire({
+    this.modelingEvents.fireMethodsChangedEvent(
       methods,
       dbUri,
-      isActiveDb: dbUri === this.activeDb,
+      dbUri === this.activeDb,
-    });
+    );
   }
 
   public setHideModeledMethods(
@@ -226,10 +168,10 @@ export class ModelingStore extends DisposableObject {
 
     dbState.hideModeledMethods = hideModeledMethods;
 
-    this.onHideModeledMethodsChangedEventEmitter.fire({
+    this.modelingEvents.fireHideModeledMethodsChangedEvent(
       hideModeledMethods,
-      isActiveDb: dbUri === this.activeDb,
+      dbUri === this.activeDb,
-    });
+    );
   }
 
   public setMode(dbItem: DatabaseItem, mode: Mode) {
@@ -238,10 +180,7 @@ export class ModelingStore extends DisposableObject {
 
     dbState.mode = mode;
 
-    this.onModeChangedEventEmitter.fire({
+    this.modelingEvents.fireModeChangedEvent(mode, dbUri === this.activeDb);
-      mode,
-      isActiveDb: dbUri === this.activeDb,
-    });
   }
 
   public getMode(dbItem: DatabaseItem) {
@@ -255,7 +194,7 @@ export class ModelingStore extends DisposableObject {
   public getModeledMethods(
     dbItem: DatabaseItem,
     methodSignatures?: string[],
-  ): Record<string, ModeledMethod[]> {
+  ): Readonly<Record<string, readonly ModeledMethod[]>> {
     const modeledMethods = this.getState(dbItem).modeledMethods;
     if (!methodSignatures) {
       return modeledMethods;
@@ -345,32 +284,59 @@ export class ModelingStore extends DisposableObject {
     });
   }
 
-  public setSelectedMethod(dbItem: DatabaseItem, methodSignature: string) {
+  /**
+   * Sets which method is considered to be selected. This method will be shown in the method modeling panel.
+   *
+   * The `Method` and `Usage` objects must have been retrieved from the modeling store, and not from
+   * a webview. This is because we rely on object referential identity so it must be the same object
+   * that is held internally by the modeling store.
+   */
+  public setSelectedMethod(dbItem: DatabaseItem, method: Method, usage: Usage) {
     const dbState = this.getState(dbItem);
 
-    const method = dbState.methods.find((m) => m.signature === methodSignature);
-    if (method === undefined) {
-      throw new Error(
-        `No method with signature "${methodSignature}" found in modeling store`,
-      );
-    }
-
-    const usage = method.usages[0];
-
     dbState.selectedMethod = method;
     dbState.selectedUsage = usage;
 
-    this.onSelectedMethodChangedEventEmitter.fire({
+    const modeledMethods = dbState.modeledMethods[method.signature] ?? [];
-      databaseItem: dbItem,
+    const isModified = dbState.modifiedMethodSignatures.has(method.signature);
+    const isInProgress = dbState.inProgressMethods.has(method.signature);
+    this.modelingEvents.fireSelectedMethodChangedEvent(
+      dbItem,
       method,
       usage,
-      modeledMethods: dbState.modeledMethods[method.signature] ?? [],
+      modeledMethods,
-      isModified: dbState.modifiedMethodSignatures.has(method.signature),
+      isModified,
+      isInProgress,
+    );
+  }
+
+  public addInProgressMethods(
+    dbItem: DatabaseItem,
+    inProgressMethods: string[],
+  ) {
+    this.changeInProgressMethods(dbItem, (state) => {
+      state.inProgressMethods = new Set([
+        ...state.inProgressMethods,
+        ...inProgressMethods,
+      ]);
     });
   }
 
-  public getSelectedMethodDetails() {
+  public removeInProgressMethods(
-    const dbState = this.getStateForActiveDb();
+    dbItem: DatabaseItem,
+    methodSignatures: string[],
+  ) {
+    this.changeInProgressMethods(dbItem, (state) => {
+      state.inProgressMethods = new Set(
+        Array.from(state.inProgressMethods).filter(
+          (s) => !methodSignatures.includes(s),
+        ),
+      );
+    });
+  }
+
+  public getSelectedMethodDetails(): SelectedMethodDetails | undefined {
+    const dbState = this.getInternalStateForActiveDb();
     if (!dbState) {
       throw new Error("No active state found in modeling store");
     }
@@ -388,10 +354,11 @@ export class ModelingStore extends DisposableObject {
       isModified: dbState.modifiedMethodSignatures.has(
         selectedMethod.signature,
       ),
+      isInProgress: dbState.inProgressMethods.has(selectedMethod.signature),
     };
   }
 
-  private getState(databaseItem: DatabaseItem): DbModelingState {
+  private getState(databaseItem: DatabaseItem): InternalDbModelingState {
     if (!this.state.has(databaseItem.databaseUri.toString())) {
       throw Error(
         "Cannot get state for a database that has not been initialized",
@@ -403,31 +370,45 @@ export class ModelingStore extends DisposableObject {
 
   private changeModifiedMethods(
     dbItem: DatabaseItem,
-    updateState: (state: DbModelingState) => void,
+    updateState: (state: InternalDbModelingState) => void,
   ) {
     const state = this.getState(dbItem);
 
     updateState(state);
 
-    this.onModifiedMethodsChangedEventEmitter.fire({
+    this.modelingEvents.fireModifiedMethodsChangedEvent(
-      modifiedMethods: state.modifiedMethodSignatures,
+      state.modifiedMethodSignatures,
-      dbUri: dbItem.databaseUri.toString(),
+      dbItem.databaseUri.toString(),
-      isActiveDb: dbItem.databaseUri.toString() === this.activeDb,
+      dbItem.databaseUri.toString() === this.activeDb,
-    });
+    );
   }
 
   private changeModeledMethods(
     dbItem: DatabaseItem,
-    updateState: (state: DbModelingState) => void,
+    updateState: (state: InternalDbModelingState) => void,
   ) {
     const state = this.getState(dbItem);
 
     updateState(state);
 
-    this.onModeledMethodsChangedEventEmitter.fire({
+    this.modelingEvents.fireModeledMethodsChangedEvent(
-      modeledMethods: state.modeledMethods,
+      state.modeledMethods,
-      dbUri: dbItem.databaseUri.toString(),
+      dbItem.databaseUri.toString(),
-      isActiveDb: dbItem.databaseUri.toString() === this.activeDb,
+      dbItem.databaseUri.toString() === this.activeDb,
-    });
+    );
+  }
+
+  private changeInProgressMethods(
+    dbItem: DatabaseItem,
+    updateState: (state: InternalDbModelingState) => void,
+  ) {
+    const state = this.getState(dbItem);
+
+    updateState(state);
+
+    this.modelingEvents.fireInProgressMethodsChangedEvent(
+      dbItem.databaseUri.toString(),
+      state.inProgressMethods,
+    );
   }
 }

extensions/ql-vscode/src/model-editor/predicates.ts

@@ -1,153 +0,0 @@
-import { ModeledMethod, ModeledMethodType, Provenance } from "./modeled-method";
-import { DataTuple } from "./model-extension-file";
-
-export type ExtensiblePredicateDefinition = {
-  extensiblePredicate: string;
-  generateMethodDefinition: (method: ModeledMethod) => DataTuple[];
-  readModeledMethod: (row: DataTuple[]) => ModeledMethod;
-
-  supportedKinds?: string[];
-};
-
-function readRowToMethod(row: DataTuple[]): string {
-  return `${row[0]}.${row[1]}#${row[3]}${row[4]}`;
-}
-
-export const extensiblePredicateDefinitions: Record<
-  Exclude<ModeledMethodType, "none">,
-  ExtensiblePredicateDefinition
-> = {
-  source: {
-    extensiblePredicate: "sourceModel",
-    // extensible predicate sourceModel(
-    //   string package, string type, boolean subtypes, string name, string signature, string ext,
-    //   string output, string kind, string provenance
-    // );
-    generateMethodDefinition: (method) => [
-      method.packageName,
-      method.typeName,
-      true,
-      method.methodName,
-      method.methodParameters,
-      "",
-      method.output,
-      method.kind,
-      method.provenance,
-    ],
-    readModeledMethod: (row) => ({
-      type: "source",
-      input: "",
-      output: row[6] as string,
-      kind: row[7] as string,
-      provenance: row[8] as Provenance,
-      signature: readRowToMethod(row),
-      packageName: row[0] as string,
-      typeName: row[1] as string,
-      methodName: row[3] as string,
-      methodParameters: row[4] as string,
-    }),
-    supportedKinds: ["local", "remote"],
-  },
-  sink: {
-    extensiblePredicate: "sinkModel",
-    // extensible predicate sinkModel(
-    //   string package, string type, boolean subtypes, string name, string signature, string ext,
-    //   string input, string kind, string provenance
-    // );
-    generateMethodDefinition: (method) => [
-      method.packageName,
-      method.typeName,
-      true,
-      method.methodName,
-      method.methodParameters,
-      "",
-      method.input,
-      method.kind,
-      method.provenance,
-    ],
-    readModeledMethod: (row) => ({
-      type: "sink",
-      input: row[6] as string,
-      output: "",
-      kind: row[7] as string,
-      provenance: row[8] as Provenance,
-      signature: readRowToMethod(row),
-      packageName: row[0] as string,
-      typeName: row[1] as string,
-      methodName: row[3] as string,
-      methodParameters: row[4] as string,
-    }),
-    supportedKinds: [
-      "code-injection",
-      "command-injection",
-      "file-content-store",
-      "html-injection",
-      "js-injection",
-      "ldap-injection",
-      "log-injection",
-      "path-injection",
-      "request-forgery",
-      "sql-injection",
-      "url-redirection",
-    ],
-  },
-  summary: {
-    extensiblePredicate: "summaryModel",
-    // extensible predicate summaryModel(
-    //   string package, string type, boolean subtypes, string name, string signature, string ext,
-    //   string input, string output, string kind, string provenance
-    // );
-    generateMethodDefinition: (method) => [
-      method.packageName,
-      method.typeName,
-      true,
-      method.methodName,
-      method.methodParameters,
-      "",
-      method.input,
-      method.output,
-      method.kind,
-      method.provenance,
-    ],
-    readModeledMethod: (row) => ({
-      type: "summary",
-      input: row[6] as string,
-      output: row[7] as string,
-      kind: row[8] as string,
-      provenance: row[9] as Provenance,
-      signature: readRowToMethod(row),
-      packageName: row[0] as string,
-      typeName: row[1] as string,
-      methodName: row[3] as string,
-      methodParameters: row[4] as string,
-    }),
-    supportedKinds: ["taint", "value"],
-  },
-  neutral: {
-    extensiblePredicate: "neutralModel",
-    // extensible predicate neutralModel(
-    //   string package, string type, string name, string signature, string kind, string provenance
-    // );
-    generateMethodDefinition: (method) => [
-      method.packageName,
-      method.typeName,
-      method.methodName,
-      method.methodParameters,
-      method.kind,
-      method.provenance,
-    ],
-    readModeledMethod: (row) => ({
-      type: "neutral",
-      input: "",
-      output: "",
-      kind: row[4] as string,
-      provenance: row[5] as Provenance,
-      signature: `${row[0]}.${row[1]}#${row[2]}${row[3]}`,
-      packageName: row[0] as string,
-      typeName: row[1] as string,
-      methodName: row[2] as string,
-      methodParameters: row[3] as string,
-    }),
-    supportedKinds: ["summary", "source", "sink"],
-  },
-};

extensions/ql-vscode/src/model-editor/queries/index.ts

@@ -1,9 +1,11 @@
 import { fetchExternalApisQuery as csharpFetchExternalApisQuery } from "./csharp";
 import { fetchExternalApisQuery as javaFetchExternalApisQuery } from "./java";
+import { fetchExternalApisQuery as rubyFetchExternalApisQuery } from "./ruby";
 import { Query } from "./query";
 import { QueryLanguage } from "../../common/query-language";
 
 export const fetchExternalApiQueries: Partial<Record<QueryLanguage, Query>> = {
   [QueryLanguage.CSharp]: csharpFetchExternalApisQuery,
   [QueryLanguage.Java]: javaFetchExternalApisQuery,
+  [QueryLanguage.Ruby]: rubyFetchExternalApisQuery,
 };

extensions/ql-vscode/src/model-editor/queries/ruby.ts

@@ -0,0 +1,404 @@
+import { Query } from "./query";
+
+export const fetchExternalApisQuery: Query = {
+  applicationModeQuery: `/**
+ * @name Fetch endpoints for use in the model editor (application mode)
+ * @description A list of 3rd party endpoints (methods and attributes) used in the codebase. Excludes test and generated code.
+ * @kind table
+ * @id rb/utils/modeleditor/application-mode-endpoints
+ * @tags modeleditor endpoints application-mode
+ */
+
+import ruby
+
+select "todo", "todo", "todo", "todo", "todo", false, "todo", "todo", "todo", "todo"
+`,
+  frameworkModeQuery: `/**
+ * @name Fetch endpoints for use in the model editor (framework mode)
+ * @description A list of endpoints accessible (methods and attributes) for consumers of the library. Excludes test and generated code.
+ * @kind table
+ * @id rb/utils/modeleditor/framework-mode-endpoints
+ * @tags modeleditor endpoints framework-mode
+ */
+
+import ruby
+import FrameworkModeEndpointsQuery
+import ModelEditor
+
+from PublicEndpointFromSource endpoint, boolean supported, string type
+where
+  supported = isSupported(endpoint) and
+  type = supportedType(endpoint)
+select endpoint, endpoint.getNamespace(), endpoint.getTypeName(), endpoint.getName(),
+  endpoint.getParameterTypes(), supported, endpoint.getFile().getBaseName(), type
+`,
+  dependencies: {
+    "FrameworkModeEndpointsQuery.qll": `private import ruby
+private import ModelEditor
+private import modeling.internal.Util as Util
+
+/**
+ * A class of effectively public callables from source code.
+ */
+class PublicEndpointFromSource extends Endpoint {
+  PublicEndpointFromSource() {
+    this.getFile() instanceof Util::RelevantFile
+  }
+
+  override predicate isSource() { this instanceof SourceCallable }
+
+  override predicate isSink() { this instanceof SinkCallable }
+}
+`,
+    "ModelEditor.qll": `/** Provides classes and predicates related to handling APIs for the VS Code extension. */
+
+private import ruby
+private import codeql.ruby.dataflow.FlowSummary
+private import codeql.ruby.dataflow.internal.DataFlowPrivate
+private import codeql.ruby.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
+private import codeql.ruby.dataflow.internal.FlowSummaryImplSpecific
+private import modeling.internal.Util as Util
+private import modeling.internal.Types
+private import codeql.ruby.frameworks.core.Gem
+
+/** Holds if the given callable is not worth supporting. */
+private predicate isUninteresting(DataFlow::MethodNode c) {
+  c.getLocation().getFile().getRelativePath().regexpMatch(".*(test|spec).*")
+}
+
+/**
+ * A callable method or accessor from either the Ruby Standard Library, a 3rd party library, or from the source.
+ */
+class Endpoint extends DataFlow::MethodNode {
+  Endpoint() {
+    this.isPublic() and not isUninteresting(this)
+  }
+
+  File getFile() { result = this.getLocation().getFile() }
+
+  string getName() { result = this.getMethodName() }
+
+  /**
+   * Gets the namespace of this endpoint.
+   */
+  bindingset[this]
+  string getNamespace() {
+    // Return the name of any gemspec file in the database.
+    // TODO: make this work for projects with multiple gems (and hence multiple gemspec files)
+    result = any(Gem::GemSpec g).getName()
+  }
+
+  /**
+   * Gets the unbound type name of this endpoint.
+   */
+  bindingset[this]
+  string getTypeName() {
+    // result = nestedName(this.getDeclaringType().getUnboundDeclaration())
+    // result = any(DataFlow::ClassNode c | Types::methodReturnsType(this, c) | c).getQualifiedName()
+    result = Util::getAnAccessPathPrefixWithoutSuffix(this)
+  }
+
+  /**
+   * Gets the parameter types of this endpoint.
+   */
+  bindingset[this]
+  string getParameterTypes() {
+    // For now, return the names of postional parameters. We don't always have type information, so we can't return type names.
+    // We don't yet handle keyword params, splat params or block params.
+    // result = "(" + parameterQualifiedTypeNamesToString(this) + ")"
+    result =
+      "(" +
+        concat(DataFlow::ParameterNode p, int i |
+          p = this.asCallable().getParameter(i)
+        |
+          p.getName(), "," order by i
+        ) + ")"
+  }
+
+  /** Holds if this API has a supported summary. */
+  pragma[nomagic]
+  predicate hasSummary() {
+    // this instanceof SummarizedCallable
+    none()
+  }
+
+  /** Holds if this API is a known source. */
+  pragma[nomagic]
+  abstract predicate isSource();
+
+  /** Holds if this API is a known sink. */
+  pragma[nomagic]
+  abstract predicate isSink();
+
+  /** Holds if this API is a known neutral. */
+  pragma[nomagic]
+  predicate isNeutral() {
+    // this instanceof FlowSummaryImpl::Public::NeutralCallable
+    none()
+  }
+
+  /**
+   * Holds if this API is supported by existing CodeQL libraries, that is, it is either a
+   * recognized source, sink or neutral or it has a flow summary.
+   */
+  predicate isSupported() {
+    this.hasSummary() or this.isSource() or this.isSink() or this.isNeutral()
+  }
+}
+
+boolean isSupported(Endpoint endpoint) {
+  if endpoint.isSupported() then result = true else result = false
+}
+
+string supportedType(Endpoint endpoint) {
+  endpoint.isSink() and result = "sink"
+  or
+  endpoint.isSource() and result = "source"
+  or
+  endpoint.hasSummary() and result = "summary"
+  or
+  endpoint.isNeutral() and result = "neutral"
+  or
+  not endpoint.isSupported() and result = ""
+}
+
+string methodClassification(Call method) {
+  result = "source"
+}
+
+/**
+ * A callable where there exists a MaD sink model that applies to it.
+ */
+class SinkCallable extends DataFlow::CallableNode {
+  SinkCallable() { sinkElement(this.asExpr().getExpr(), _, _, _) }
+}
+
+/**
+ * A callable where there exists a MaD source model that applies to it.
+ */
+class SourceCallable extends DataFlow::CallableNode {
+  SourceCallable() { sourceElement(this.asExpr().getExpr(), _, _, _) }
+}`,
+    "modeling/internal/Util.qll": `private import ruby
+
+// \`SomeClass#initialize\` methods are usually called indirectly via
+// \`SomeClass.new\`, so we need to account for this when generating access paths
+private string getNormalizedMethodName(DataFlow::MethodNode methodNode) {
+  exists(string actualMethodName | actualMethodName = methodNode.getMethodName() |
+    if actualMethodName = "initialize" then result = "new" else result = actualMethodName
+  )
+}
+
+private string getAccessPathSuffix(Ast::MethodBase method) {
+  if method instanceof Ast::SingletonMethod or method.getName() = "initialize"
+  then result = "!"
+  else result = ""
+}
+
+string getAnAccessPathPrefix(DataFlow::MethodNode methodNode) {
+  result =
+    getAnAccessPathPrefixWithoutSuffix(methodNode) +
+      getAccessPathSuffix(methodNode.asExpr().getExpr())
+}
+
+string getAnAccessPathPrefixWithoutSuffix(DataFlow::MethodNode methodNode) {
+  result =
+    methodNode
+        .asExpr()
+        .getExpr()
+        .getEnclosingModule()
+        .(Ast::ConstantWriteAccess)
+        .getAQualifiedName()
+}
+
+class RelevantFile extends File {
+  RelevantFile() { not this.getRelativePath().regexpMatch(".*/?test(case)?s?/.*") }
+}
+
+string getMethodPath(DataFlow::MethodNode methodNode) {
+  result = "Method[" + getNormalizedMethodName(methodNode) + "]"
+}
+
+private string getParameterPath(DataFlow::ParameterNode paramNode) {
+  exists(Ast::Parameter param, string paramSpec |
+    param = paramNode.asParameter() and
+    (
+      paramSpec = param.getPosition().toString()
+      or
+      paramSpec = param.(Ast::KeywordParameter).getName() + ":"
+      or
+      param instanceof Ast::BlockParameter and
+      paramSpec = "block"
+    )
+  |
+    result = "Parameter[" + paramSpec + "]"
+  )
+}
+
+string getMethodParameterPath(DataFlow::MethodNode methodNode, DataFlow::ParameterNode paramNode) {
+  result = getMethodPath(methodNode) + "." + getParameterPath(paramNode)
+}
+`,
+    "modeling/internal/Types.qll": `private import ruby
+private import codeql.ruby.ApiGraphs
+private import Util as Util
+
+module Types {
+  private module Config implements DataFlow::ConfigSig {
+    predicate isSource(DataFlow::Node source) {
+      // TODO: construction of type values not using a "new" call
+      source.(DataFlow::CallNode).getMethodName() = "new"
+    }
+
+    predicate isSink(DataFlow::Node sink) { sink = any(DataFlow::MethodNode m).getAReturnNode() }
+  }
+
+  private import DataFlow::Global<Config>
+
+  predicate methodReturnsType(DataFlow::MethodNode methodNode, DataFlow::ClassNode classNode) {
+    // ignore cases of initializing instance of self
+    not methodNode.getMethodName() = "initialize" and
+    exists(DataFlow::CallNode initCall |
+      flow(initCall, methodNode.getAReturnNode()) and
+      classNode.getAnImmediateReference().getAMethodCall() = initCall and
+      // constructed object does not have a type declared in test code
+      /*
+       * TODO: this may be too restrictive, e.g.
+       * - if a type is declared in both production and test code
+       * - if a built-in type is extended in test code
+       */
+
+      forall(Ast::ModuleBase classDecl | classDecl = classNode.getADeclaration() |
+        classDecl.getLocation().getFile() instanceof Util::RelevantFile
+      )
+    )
+  }
+
+  // \`exprNode\` is an instance of \`classNode\`
+  private predicate exprHasType(DataFlow::ExprNode exprNode, DataFlow::ClassNode classNode) {
+    exists(DataFlow::MethodNode methodNode, DataFlow::CallNode callNode |
+      methodReturnsType(methodNode, classNode) and
+      callNode.getATarget() = methodNode
+    |
+      exprNode.getALocalSource() = callNode
+    )
+    or
+    exists(DataFlow::MethodNode containingMethod |
+      classNode.getInstanceMethod(containingMethod.getMethodName()) = containingMethod
+    |
+      exprNode.getALocalSource() = containingMethod.getSelfParameter()
+    )
+  }
+
+  // extensible predicate typeModel(string type1, string type2, string path);
+  // the method node in type2 constructs an instance of classNode
+  private predicate typeModelReturns(string type1, string type2, string path) {
+    exists(DataFlow::MethodNode methodNode, DataFlow::ClassNode classNode |
+      methodNode.getLocation().getFile() instanceof Util::RelevantFile and
+      methodReturnsType(methodNode, classNode)
+    |
+      type1 = classNode.getQualifiedName() and
+      type2 = Util::getAnAccessPathPrefix(methodNode) and
+      path = Util::getMethodPath(methodNode) + ".ReturnValue"
+    )
+  }
+
+  predicate methodTakesParameterOfType(
+    DataFlow::MethodNode methodNode, DataFlow::ClassNode classNode,
+    DataFlow::ParameterNode parameterNode
+  ) {
+    exists(DataFlow::CallNode callToMethodNode, DataFlow::LocalSourceNode argumentNode |
+      callToMethodNode.getATarget() = methodNode and
+      // positional parameter
+      exists(int paramIndex |
+        argumentNode.flowsTo(callToMethodNode.getArgument(paramIndex)) and
+        parameterNode = methodNode.getParameter(paramIndex)
+      )
+      or
+      // keyword parameter
+      exists(string kwName |
+        argumentNode.flowsTo(callToMethodNode.getKeywordArgument(kwName)) and
+        parameterNode = methodNode.getKeywordParameter(kwName)
+      )
+      or
+      // block parameter
+      argumentNode.flowsTo(callToMethodNode.getBlock()) and
+      parameterNode = methodNode.getBlockParameter()
+    |
+      // parameter directly from new call
+      argumentNode.(DataFlow::CallNode).getMethodName() = "new" and
+      classNode.getAnImmediateReference().getAMethodCall() = argumentNode
+      or
+      // parameter from indirect new call
+      exists(DataFlow::ExprNode argExpr |
+        exprHasType(argExpr, classNode) and
+        argumentNode.(DataFlow::CallNode).getATarget() = argExpr
+      )
+    )
+  }
+
+  private predicate typeModelParameters(string type1, string type2, string path) {
+    exists(
+      DataFlow::MethodNode methodNode, DataFlow::ClassNode classNode,
+      DataFlow::ParameterNode parameterNode
+    |
+      methodNode.getLocation().getFile() instanceof Util::RelevantFile and
+      methodTakesParameterOfType(methodNode, classNode, parameterNode)
+    |
+      type1 = classNode.getQualifiedName() and
+      type2 = Util::getAnAccessPathPrefix(methodNode) and
+      path = Util::getMethodParameterPath(methodNode, parameterNode)
+    )
+  }
+
+  // TODO: non-positional params for block arg parameters
+  private predicate methodYieldsType(
+    DataFlow::CallableNode callableNode, int argIdx, DataFlow::ClassNode classNode
+  ) {
+    exprHasType(callableNode.getABlockCall().getArgument(argIdx), classNode)
+  }
+
+  /*
+   * e.g. for
+   * \`\`\`rb
+   * class Foo
+   *  def initialize
+   *    // do some stuff...
+   *    if block_given?
+   *      yield self
+   *    end
+   *  end
+   *
+   *  def do_something
+   *    // do something else
+   *  end
+   * end
+   *
+   * Foo.new do |foo| foo.do_something end
+   * \`\`\`
+   *
+   * the parameter foo to the block is an instance of Foo.
+   */
+
+  private predicate typeModelBlockArgumentParameters(string type1, string type2, string path) {
+    exists(DataFlow::MethodNode methodNode, DataFlow::ClassNode classNode, int argIdx |
+      methodNode.getLocation().getFile() instanceof Util::RelevantFile and
+      methodYieldsType(methodNode, argIdx, classNode)
+    |
+      type1 = classNode.getQualifiedName() and
+      type2 = Util::getAnAccessPathPrefix(methodNode) and
+      path = Util::getMethodPath(methodNode) + ".Argument[block].Parameter[" + argIdx + "]"
+    )
+  }
+
+  predicate typeModel(string type1, string type2, string path) {
+    typeModelReturns(type1, type2, path)
+    or
+    typeModelParameters(type1, type2, path)
+    or
+    typeModelBlockArgumentParameters(type1, type2, path)
+  }
+}
+`,
+  },
+};

extensions/ql-vscode/src/model-editor/shared/in-progress-methods.ts

@@ -1,34 +0,0 @@
-/**
- * A class that keeps track of which methods are in progress for each package.
- *
- * This class is immutable and therefore is safe to be used in a React useState hook.
- */
-export class InProgressMethods {
-  // A map of in-progress method signatures for each package.
-  private readonly methodMap: ReadonlyMap<string, Set<string>>;
-
-  constructor(methodMap?: ReadonlyMap<string, Set<string>>) {
-    this.methodMap = methodMap ?? new Map<string, Set<string>>();
-  }
-
-  /**
-   * Sets the in-progress methods for the given package.
-   * Returns a new InProgressMethods instance.
-   */
-  public setPackageMethods(
-    packageName: string,
-    methods: Set<string>,
-  ): InProgressMethods {
-    const newMethodMap = new Map<string, Set<string>>(this.methodMap);
-    newMethodMap.set(packageName, methods);
-    return new InProgressMethods(newMethodMap);
-  }
-
-  public hasMethod(packageName: string, method: string): boolean {
-    const methods = this.methodMap.get(packageName);
-    if (methods) {
-      return methods.has(method);
-    }
-    return false;
-  }
-}

extensions/ql-vscode/src/model-editor/shared/modeled-methods-legacy.ts

@@ -1,21 +1,5 @@
 import { ModeledMethod } from "../modeled-method";
 
-/**
- * Converts a single ModeledMethod to a ModeledMethod[] for legacy usage. This function should always be used instead
- * of the trivial conversion to track usages of this conversion.
- *
- * This method should only be called inside a `onMessage` function (or its equivalent). If it's used anywhere else,
- * consider whether the boundary is correct: the boundary should as close as possible to the webview -> extension host
- * boundary.
- *
- * @param modeledMethod The single ModeledMethod
- */
-export function convertFromLegacyModeledMethod(
-  modeledMethod: ModeledMethod | undefined,
-): ModeledMethod[] {
-  return modeledMethod ? [modeledMethod] : [];
-}
-
 /**
  * Converts a ModeledMethod[] to a single ModeledMethod for legacy usage. This function should always be used instead
  * of the trivial conversion to track usages of this conversion.

extensions/ql-vscode/src/model-editor/shared/modeled-percentage.ts

@@ -1,6 +1,6 @@
 import { Method } from "../method";
 
-export function calculateModeledPercentage(methods: Method[]): number {
+export function calculateModeledPercentage(methods: readonly Method[]): number {
   if (methods.length === 0) {
     return 0;
   }

extensions/ql-vscode/src/model-editor/shared/modeling-status.ts

@@ -3,7 +3,7 @@ import { ModeledMethod } from "../modeled-method";
 export type ModelingStatus = "unmodeled" | "unsaved" | "saved";
 
 export function getModelingStatus(
-  modeledMethods: ModeledMethod[],
+  modeledMethods: readonly ModeledMethod[],
   methodIsUnsaved: boolean,
 ): ModelingStatus {
   if (modeledMethods.length > 0) {

extensions/ql-vscode/src/model-editor/shared/multiple-modeled-methods.ts

@@ -0,0 +1,20 @@
+import { ModeledMethod } from "../modeled-method";
+
+export function canAddNewModeledMethod(
+  modeledMethods: ModeledMethod[],
+): boolean {
+  // Disallow adding methods when there are no modeled methods or where there is a single unmodeled method.
+  // In both of these cases the UI will already be showing the user inputs they can use for modeling.
+  return (
+    modeledMethods.length > 1 ||
+    (modeledMethods.length === 1 && modeledMethods[0].type !== "none")
+  );
+}
+
+export function canRemoveModeledMethod(
+  modeledMethods: ModeledMethod[],
+): boolean {
+  // Don't allow removing the last modeled method. In this case the user is intended to
+  // set the type to "none" instead.
+  return modeledMethods.length > 1;
+}

extensions/ql-vscode/src/model-editor/shared/sorting.ts

@@ -3,7 +3,7 @@ import { Mode } from "./mode";
 import { calculateModeledPercentage } from "./modeled-percentage";
 
 export function groupMethods(
-  methods: Method[],
+  methods: readonly Method[],
   mode: Mode,
 ): Record<string, Method[]> {
   const groupedByLibrary: Record<string, Method[]> = {};
@@ -19,22 +19,24 @@ export function groupMethods(
   return groupedByLibrary;
 }
 
-export function sortGroupNames(methods: Record<string, Method[]>): string[] {
+export function sortGroupNames(
+  methods: Record<string, readonly Method[]>,
+): string[] {
   return Object.keys(methods).sort((a, b) =>
     compareGroups(methods[a], a, methods[b], b),
   );
 }
 
-export function sortMethods(methods: Method[]): Method[] {
+export function sortMethods(methods: readonly Method[]): Method[] {
   const sortedMethods = [...methods];
   sortedMethods.sort((a, b) => compareMethod(a, b));
   return sortedMethods;
 }
 
 function compareGroups(
-  a: Method[],
+  a: readonly Method[],
   aName: string,
-  b: Method[],
+  b: readonly Method[],
   bName: string,
 ): number {
   const supportedPercentageA = calculateModeledPercentage(a);

extensions/ql-vscode/src/model-editor/shared/validation.ts

@@ -0,0 +1,161 @@
+import { ModeledMethod, NeutralModeledMethod } from "../modeled-method";
+import { MethodSignature } from "../method";
+import { assertNever } from "../../common/helpers-pure";
+
+export type ModeledMethodValidationError = {
+  title: string;
+  message: string;
+  actionText: string;
+  index: number;
+};
+
+/**
+ * This method will reset any properties which are not used for the specific type of modeled method.
+ *
+ * It will also set the `provenance` to `manual` since multiple modelings of the same method with a
+ * different provenance are not actually different.
+ *
+ * The returned canonical modeled method should only be used for comparisons. It should not be used
+ * for display purposes, saving the model, or any other purpose which requires the original modeled
+ * method to be preserved.
+ *
+ * @param modeledMethod The modeled method to canonicalize
+ */
+function canonicalizeModeledMethod(
+  modeledMethod: ModeledMethod,
+): ModeledMethod {
+  const methodSignature: MethodSignature = {
+    signature: modeledMethod.signature,
+    packageName: modeledMethod.packageName,
+    typeName: modeledMethod.typeName,
+    methodName: modeledMethod.methodName,
+    methodParameters: modeledMethod.methodParameters,
+  };
+
+  switch (modeledMethod.type) {
+    case "none":
+      return {
+        ...methodSignature,
+        type: "none",
+      };
+    case "source":
+      return {
+        ...methodSignature,
+        type: "source",
+        output: modeledMethod.output,
+        kind: modeledMethod.kind,
+        provenance: "manual",
+      };
+    case "sink":
+      return {
+        ...methodSignature,
+        type: "sink",
+        input: modeledMethod.input,
+        kind: modeledMethod.kind,
+        provenance: "manual",
+      };
+    case "summary":
+      return {
+        ...methodSignature,
+        type: "summary",
+        input: modeledMethod.input,
+        output: modeledMethod.output,
+        kind: modeledMethod.kind,
+        provenance: "manual",
+      };
+    case "neutral":
+      return {
+        ...methodSignature,
+        type: "neutral",
+        kind: modeledMethod.kind,
+        provenance: "manual",
+      };
+    case "type":
+      return {
+        ...methodSignature,
+        type: "type",
+        relatedTypeName: modeledMethod.relatedTypeName,
+        path: modeledMethod.path,
+      };
+    default:
+      assertNever(modeledMethod);
+  }
+}
+
+export function validateModeledMethods(
+  modeledMethods: ModeledMethod[],
+): ModeledMethodValidationError[] {
+  // Anything that is not modeled will not be saved, so we don't need to validate it
+  const consideredModeledMethods = modeledMethods.filter(
+    (modeledMethod) => modeledMethod.type !== "none",
+  );
+
+  const result: ModeledMethodValidationError[] = [];
+
+  // If the same model is present multiple times, only the first one makes sense, so we should give
+  // an error for any duplicates.
+  const seenModeledMethods = new Set<string>();
+  for (const modeledMethod of consideredModeledMethods) {
+    const canonicalModeledMethod = canonicalizeModeledMethod(modeledMethod);
+    const key = JSON.stringify(
+      canonicalModeledMethod,
+      // This ensures the keys are always in the same order
+      Object.keys(canonicalModeledMethod).sort(),
+    );
+
+    if (seenModeledMethods.has(key)) {
+      result.push({
+        title: "Duplicated classification",
+        message:
+          "This method has two identical or conflicting classifications.",
+        actionText: "Modify or remove the duplicated classification.",
+        index: modeledMethods.indexOf(modeledMethod),
+      });
+    } else {
+      seenModeledMethods.add(key);
+    }
+  }
+
+  const neutralModeledMethods = consideredModeledMethods.filter(
+    (modeledMethod): modeledMethod is NeutralModeledMethod =>
+      modeledMethod.type === "neutral",
+  );
+
+  const neutralModeledMethodsByKind = new Map<string, ModeledMethod[]>();
+  for (const neutralModeledMethod of neutralModeledMethods) {
+    if (!neutralModeledMethodsByKind.has(neutralModeledMethod.kind)) {
+      neutralModeledMethodsByKind.set(neutralModeledMethod.kind, []);
+    }
+
+    neutralModeledMethodsByKind
+      .get(neutralModeledMethod.kind)
+      ?.push(neutralModeledMethod);
+  }
+
+  for (const [
+    neutralModeledMethodKind,
+    neutralModeledMethods,
+  ] of neutralModeledMethodsByKind) {
+    const conflictingMethods = consideredModeledMethods.filter(
+      (method) => neutralModeledMethodKind === method.type,
+    );
+
+    if (conflictingMethods.length < 1) {
+      continue;
+    }
+
+    result.push({
+      title: "Conflicting classification",
+      message: `This method has a neutral ${neutralModeledMethodKind} classification, which conflicts with other ${neutralModeledMethodKind} classifications.`,
+      actionText: "Modify or remove the neutral classification.",
+      // Another validation will validate that only one neutral method is present, so we only need
+      // to return an error for the first one
+      index: modeledMethods.indexOf(neutralModeledMethods[0]),
+    });
+  }
+
+  // Sort by index so that the errors are always in the same order
+  result.sort((a, b) => a.index - b.index);
+
+  return result;
+}

extensions/ql-vscode/src/model-editor/shared/view-state.ts

@@ -1,15 +1,19 @@
 import { ExtensionPack } from "./extension-pack";
 import { Mode } from "./mode";
+import { QueryLanguage } from "../../common/query-language";
 
 export interface ModelEditorViewState {
   extensionPack: ExtensionPack;
-  showFlowGeneration: boolean;
+  language: QueryLanguage;
+  showGenerateButton: boolean;
   showLlmButton: boolean;
   showMultipleModels: boolean;
   mode: Mode;
+  showModeSwitchButton: boolean;
   sourceArchiveAvailable: boolean;
 }
 
 export interface MethodModelingPanelViewState {
+  language: QueryLanguage | undefined;
   showMultipleModels: boolean;
 }