Merge pull request #2565 from github/v1.8.7

v1.8.7

.github/codeql/queries/assert-no-vscode-dependency.ql

@@ -0,0 +1,37 @@
+/**
+ * @name Unwanted dependency on vscode API
+ * @kind path-problem
+ * @problem.severity error
+ * @id vscode-codeql/assert-no-vscode-dependency
+ * @description The modules stored under `common` should not have dependencies on the VS Code API
+ */
+
+import javascript
+
+class VSCodeImport extends ImportDeclaration {
+  VSCodeImport() { this.getImportedPath().getValue() = "vscode" }
+}
+
+class CommonFile extends File {
+  CommonFile() {
+    this.getRelativePath().regexpMatch(".*/src/common/.*") and
+    not this.getRelativePath().regexpMatch(".*/vscode/.*")
+  }
+}
+
+Import getANonTypeOnlyImport(Module m) {
+  result = m.getAnImport() and not result.(ImportDeclaration).isTypeOnly()
+}
+
+query predicate edges(AstNode a, AstNode b) {
+  getANonTypeOnlyImport(a) = b or
+  a.(Import).getImportedModule() = b
+}
+
+from Module m, VSCodeImport v
+where
+  m.getFile() instanceof CommonFile and
+  edges+(m, v)
+select m, m, v,
+  "This module is in the 'common' directory but has a transitive dependency on the vscode API imported $@",
+  v, "here"

.github/codeql/queries/assert-pure.ql

@@ -1,21 +0,0 @@
-/**
- * @name Unwanted dependency on vscode API
- * @kind problem
- * @problem.severity error
- * @id vscode-codeql/assert-pure
- * @description The modules stored under `pure` and tested in the `pure-tests`
- * are intended to be "pure".
- */
-import javascript
-
-class VSCodeImport extends ASTNode {
-  VSCodeImport() {
-    this.(Import).getImportedPath().getValue() = "vscode"
-  }
-}
-
-from Module m, VSCodeImport v
-where
-  m.getFile().getRelativePath().regexpMatch(".*src/pure/.*")  and
-  m.getAnImportedModule*().getAnImport() = v
-select m, "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here"

.github/codeql/queries/qlpack.yml

@@ -1,3 +1,4 @@
 name: vscode-codeql-custom-queries-javascript
 version: 0.0.0
-libraryPathDependencies: codeql-javascript
+dependencies:
+  codeql/javascript-queries: "*"

.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -80,7 +80,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -124,7 +124,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -158,7 +158,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 
@@ -209,6 +209,7 @@ jobs:
     name: CLI Test
     runs-on: ${{ matrix.os }}
     needs: [find-nightly, set-matrix]
+    timeout-minutes: 30
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
@@ -225,7 +226,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
           cache: 'npm'
           cache-dependency-path: extensions/ql-vscode/package-lock.json
 

.github/workflows/release.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '16.14.2'
+          node-version: '16.17.1'
 
       - name: Install dependencies
         run: |

CODEOWNERS

@@ -2,3 +2,4 @@
 **/variant-analysis/ @github/code-scanning-secexp-reviewers
 **/databases/ @github/code-scanning-secexp-reviewers
 **/data-extensions-editor/ @github/code-scanning-secexp-reviewers
+**/queries-panel/ @github/code-scanning-secexp-reviewers

docs/releasing.md

@@ -10,7 +10,7 @@
       * New telemetry events are added.
       * Deprecation or removal of commands.
       * Accumulation of many changes, none of which are individually big enough to warrant a minor bump, but which together are. This does not include changes which are purely internal to the extension, such as refactoring, or which are only available behind a feature flag.
-1. Double-check that the node version we're using matches the one used for VS Code. If it doesn't, you will then need to update the node version in the following files:
+1. Double-check that the node version we're using matches the one used for VS Code. You can find this info by seleting "About Visual Studio Code" from the top menu. If it doesn't match, you will then need to update the node version in the following files:
     * `.nvmrc` - this will enable `nvm` to automatically switch to the correct node version when you're in the project folder
     * `.github/workflows/main.yml` - all the "node-version: '[VERSION]'" settings
     * `.github/workflows/release.yml` - the "node-version: '[VERSION]'" setting

docs/test-plan.md

@@ -11,16 +11,67 @@ We don't need to test features (and permutations of features) that are covered b
 
 ## Before releasing the VS Code extension
 
-- Go through the required test cases listed below
+- Run at least one local query and MRVA using the existing version of the extension (to generate "old" query history items).
+- Go through the required test cases listed below.
 - Check major PRs since the previous release for specific one-off things to test. Based on that, you might want to
 choose to go through some of the Optional Test Cases.
-- Run a query using the existing version of the extension (to generate an "old" query history item)
 
 ## Required Test Cases
 
-### Test Case 1: MRVA - Running a problem path query and viewing results
+### Local databases
 
-1. Open the [UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
+#### Test case 1: Download a database from GitHub
+
+1. Click "Download Database from GitHub" and enter `angular-cn/ng-nice` and select the javascript language if prompted
+
+#### Test case 2: Import a database from an archive
+
+1. Obtain a javascript database for `babel/babel`
+   - You can do `gh api "/repos/babel/babel/code-scanning/codeql/databases/javascript" -H "Accept: application/zip" > babel.zip` to fetch a database from GitHub.
+2. Click "Choose Database from Archive" and select the file you just downloaded above.
+
+### Local queries
+
+#### Test case 1: Running a path problem query and viewing results
+
+1. Open the [javascript UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
+2. Select the `angular-cn/ng-nice` database (or download it if you don't have one already)
+3. Run a local query.
+4. Once the query completes:
+   - Check that the result messages are rendered
+   - Check that the paths can be opened and paths are rendered correctly
+   - Check that alert locations can be clicked on
+
+#### Test case 2: Running a problem query and viewing results
+
+1. Open the [javascript UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
+2. Select the `babel/babel` database (or download it if you don't have one already)
+3. Run a local query.
+4. Once the query completes:
+   - Check that the result messages are rendered
+   - Check that alert locations can be clicked on
+
+#### Test case 3: Running a non-probem query and viewing results
+
+1. Open the [cpp FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
+2. Select the `google/brotli` database (or download it if you don't have one already)
+3. Run a local query.
+4. Once the query completes:
+   - Check that the results table is rendered
+   - Check that alert locations can be clicked on
+
+#### Test case 3: Can use AST viewer
+
+1. Click on any code location from a previous query to open a source file from a database
+2. Open the AST viewing panel and click "View AST"
+3. Once the AST is computed:
+   - Check that it can be navigated
+
+### MRVA
+
+#### Test Case 1: Running a path problem query and viewing results
+
+1. Open the [javascript UnsafeJQueryPlugin query](https://github.com/github/codeql/blob/main/javascript/ql/src/Security/CWE-079/UnsafeJQueryPlugin.ql).
 2. Run a MRVA against the following repo list:
 
    ```json
@@ -47,40 +98,44 @@ choose to go through some of the Optional Test Cases.
 6. Once the query completes:
    - Check that the query history item is updated to show the query status as "complete"
 
-### Test Case 2: MRVA - Running a problem query and viewing results
+#### Test Case 2: Running a problem query and viewing results
 
-1. Open the [ReDoS query](https://github.com/github/codeql/blob/main/javascript/ql/src/Performance/ReDoS.ql).
+1. Open the [javascript ReDoS query](https://github.com/github/codeql/blob/main/javascript/ql/src/Performance/ReDoS.ql).
 2. Run a MRVA against the "Top 10" repositories.
-3. Check that there is a notification message.
+3. Check that a notification message pops up and the results view is opened.
 4. Check the query history. It should:
    - Show that an item has been added to the query history
    - The item should be marked as "in progress".
-5. Once the query starts:
-   - Check that a notification is shown with a link to the results view
+5. Once the query completes:
    - Check that the results are rendered with an alert message and a highlighted code snippet:
-![highlighted-code-snippet](images/highlighted-code-snippet.png)
 
-### Test Case 3: MRVA - Running a non-problem query and viewing results
+   ![highlighted-code-snippet](images/highlighted-code-snippet.png)
 
-1. Open the [FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
+#### Test Case 3: Running a non-problem query and viewing results
+
+1. Open the [cpp FunLinesOfCode query](https://github.com/github/codeql/blob/main/cpp/ql/src/Metrics/Functions/FunLinesOfCode.ql).
 2. Run a MRVA against a single repository (e.g. `google/brotli`).
-3. Once the query starts:
-   - Open the query results
+3. Check that a notification message pops up and the results view is opened.
+4. Check the query history. It should:
+   - Show that an item has been added to the query history
+   - The item should be marked as "in progress".
+5. Once the query completes:
    - Check that the results show up in a table:
-![results-table](images/results-table.png)
 
-### Test Case 4: MRVA - Interacting with query history
+   ![results-table](images/results-table.png)
+
+#### Test Case 4: Interacting with query history
 
 1. Click a history item (for MRVA):
    - Check that exporting results works
    - Check that sorting results works
    - Check that copying repo lists works
-2. Open the query results directory:
+2. Click "Open Results Directory":
    - Check that the correct directory is opened and there are results in it
-3. View logs
+3. Click "View Logs":
    - Check that the correct workflow is opened
 
-### Test Case 5: MRVA - Canceling a variant analysis run
+#### Test Case 5: Canceling a variant analysis run
 
 Run one of the above MRVAs, but cancel it from within VS Code:
 
@@ -88,9 +143,11 @@ Run one of the above MRVAs, but cancel it from within VS Code:
 - Check that the workflow run is also canceled.
 - Check that any available results are visible in VS Code.
 
-### Test Case 6: MRVA - Change to a different colour theme
+### General
 
-Open one of the above MRVAs, try changing to a different colour theme and check that everything looks sensible.
+#### Test case 1: Change to a different colour theme
+
+Open at least one of the above MRVAs and at least one local query, then try changing to a different colour theme and check that everything looks sensible.
 Are there any components that are not showing up?
 
 ## Optional Test Cases
@@ -258,10 +315,10 @@ This requires running a MRVA query and seeing the results view.
    2. When the file does not exist
 7. Can open query text
 8. Can sort repos
-   1. By name
-   2. By results
-   3. By stars
-   4. By last updated
+   1. Alphabetically
+   2. By number of results
+   3. By popularity
+   4. By most recent commit
 9. Can filter repos
 10. Shows correct statistics
     1. Total number of results

docs/testing.md

@@ -133,4 +133,4 @@ Once the scenario has been recorded, it's often useful to remove some of the req
 
 ### Scenario data location
 
-Pre-recorded scenarios are stored in `./src/mocks/scenarios`. However, it's possible to configure the location, by setting the `codeQL.mockGitHubApiServer.scenariosPath` configuration property in the VS Code user settings.
+Pre-recorded scenarios are stored in `./src/variant-analysis/github-api/mocks/scenarios`. However, it's possible to configure the location, by setting the `codeQL.mockGitHubApiServer.scenariosPath` configuration property in the VS Code user settings.

extensions/ql-vscode/.eslintrc.js

@@ -65,10 +65,6 @@ const baseConfig = {
     "import/no-namespace": "off",
     "import/no-unresolved": "off",
     "import/no-webpack-loader-syntax": "off",
-    "jsx-a11y/anchor-is-valid": "off",
-    "jsx-a11y/no-noninteractive-element-interactions": "off",
-    "jsx-a11y/no-static-element-interactions": "off",
-    "jsx-a11y/click-events-have-key-events": "off",
     "no-invalid-this": "off",
     "no-fallthrough": "off",
     "no-console": "off",

extensions/ql-vscode/.nvmrc

@@ -1 +1 @@
-v16.14.2
+v16.17.1

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,31 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.8.7 - 29 June 2023
+
+- Show a run button on the file tab for query files, that will start a local query. This button will only show when a local database is selected in the extension. [#2544](https://github.com/github/vscode-codeql/pull/2544)
+- Add `CodeQL: Quick Evaluation Count` command to generate the count summary statistics of the results set
+  without speding the time to compute locations and strings.
+
+## 1.8.6 - 14 June 2023
+
+- Add repositories to a variant analysis list with GitHub Code Search. [#2439](https://github.com/github/vscode-codeql/pull/2439) and [#2476](https://github.com/github/vscode-codeql/pull/2476)
+
+## 1.8.5 - 6 June 2023
+
+- Add settings `codeQL.variantAnalysis.defaultResultsFilter` and `codeQL.variantAnalysis.defaultResultsSort` for configuring how variant analysis results are filtered and sorted in the results view. The default is to show all repositories, and to sort by the number of results. [#2392](https://github.com/github/vscode-codeql/pull/2392)
+- Fix bug to ensure error messages have complete stack trace in message logs. [#2425](https://github.com/github/vscode-codeql/pull/2425)
+- Fix bug where the `CodeQL: Compare Query` command did not work for comparing quick-eval queries. [#2422](https://github.com/github/vscode-codeql/pull/2422)
+- Update text of copy and export buttons in variant analysis results view to clarify that they only copy/export the selected/filtered results. [#2427](https://github.com/github/vscode-codeql/pull/2427)
+- Add warning when using unsupported CodeQL CLI version. [#2428](https://github.com/github/vscode-codeql/pull/2428)
+- Retry variant analysis results download if connection times out. [#2440](https://github.com/github/vscode-codeql/pull/2440)
+
+## 1.8.4 - 3 May 2023
+
+- Avoid repeated error messages when unable to monitor a variant analysis. [#2396](https://github.com/github/vscode-codeql/pull/2396)
+- Fix bug where a variant analysis didn't display the `#select` results set correctly when the [query metadata](https://codeql.github.com/docs/writing-codeql-queries/about-codeql-queries/#query-metadata) didn't exactly match the query results. [#2395](https://github.com/github/vscode-codeql/pull/2395)
+- On the variant analysis results page, show the count of successful analyses instead of completed analyses, and indicate the reason why analyses were not successful. [#2349](https://github.com/github/vscode-codeql/pull/2349)
+- Fix bug where the "CodeQL: Set Current Database" command didn't always select the database. [#2384](https://github.com/github/vscode-codeql/pull/2384)
+
 ## 1.8.3 - 26 April 2023
 
 - Added ability to filter repositories for a variant analysis to only those that have results [#2343](https://github.com/github/vscode-codeql/pull/2343)

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -62,18 +62,10 @@ export const config: webpack.Configuration = {
       },
       {
         test: /\.(woff(2)?|ttf|eot)$/,
-        use: [
-          {
-            loader: "file-loader",
-            options: {
-              name: "[name].[ext]",
-              outputPath: "fonts/",
-              // We need this to make Webpack use the correct path for the fonts.
-              // Without this, the CSS file will use `url([object Module])`
-              esModule: false,
-            },
-          },
-        ],
+        type: "asset/resource",
+        generator: {
+          filename: "fonts/[hash][ext][query]",
+        },
       },
     ],
   },

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.8.3",
+  "version": "1.8.7",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -36,6 +36,7 @@
   "activationEvents": [
     "onLanguage:ql",
     "onLanguage:ql-summary",
+    "onView:codeQLQueries",
     "onView:codeQLDatabases",
     "onView:codeQLVariantAnalysisRepositories",
     "onView:codeQLQueryHistory",
@@ -188,7 +189,7 @@
           "scope": "machine-overridable",
           "type": "string",
           "default": "",
-          "markdownDescription": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable."
+          "markdownDescription": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable (note: if you later introduce CodeQL on your PATH, the extension will prefer a CodeQL executable it has downloaded itself)."
         },
         "codeQL.runningQueries.numberOfThreads": {
           "type": "integer",
@@ -329,6 +330,36 @@
           "patternErrorMessage": "Please enter a valid GitHub repository",
           "markdownDescription": "[For internal use only] The name of the GitHub repository in which the GitHub Actions workflow is run when using the \"Run Variant Analysis\" command. The repository should be of the form `<owner>/<repo>`)."
         },
+        "codeQL.variantAnalysis.defaultResultsFilter": {
+          "type": "string",
+          "default": "all",
+          "enum": [
+            "all",
+            "withResults"
+          ],
+          "enumDescriptions": [
+            "Show all repositories in the results view.",
+            "Show only repositories with results in the results view."
+          ],
+          "description": "The default filter to apply to the variant analysis results view."
+        },
+        "codeQL.variantAnalysis.defaultResultsSort": {
+          "type": "string",
+          "default": "numberOfResults",
+          "enum": [
+            "alphabetically",
+            "popularity",
+            "mostRecentCommit",
+            "numberOfResults"
+          ],
+          "enumDescriptions": [
+            "Sort repositories alphabetically in the results view.",
+            "Sort repositories by popularity in the results view.",
+            "Sort repositories by most recent commit in the results view.",
+            "Sort repositories by number of results in the results view."
+          ],
+          "description": "The default sorting order for repositories in the variant analysis results view."
+        },
         "codeQL.logInsights.joinOrderWarningThreshold": {
           "type": "number",
           "default": 50,
@@ -341,11 +372,23 @@
           "default": false,
           "description": "Allow database to be downloaded via HTTP. Warning: enabling this option will allow downloading from insecure servers."
         },
-        "codeQL.createQuery.folder": {
+        "codeQL.createQuery.qlPackLocation": {
           "type": "string",
-          "default": "",
           "patternErrorMessage": "Please enter a valid folder",
-          "markdownDescription": "The name of the folder where we want to create queries and query packs via the \"CodeQL: Create Query\" command. The folder should exist."
+          "markdownDescription": "The name of the folder where we want to create queries and QL packs via the \"CodeQL: Create Query\" command. The folder should exist."
+        },
+        "codeQL.createQuery.autogenerateQlPacks": {
+          "type": "string",
+          "default": "ask",
+          "enum": [
+            "ask",
+            "never"
+          ],
+          "enumDescriptions": [
+            "Ask to create a QL pack when a new CodeQL database is added.",
+            "Never create a QL pack when a new CodeQL database is added."
+          ],
+          "description": "Ask the user to generate a QL pack when a new CodeQL database is downloaded."
         }
       }
     },
@@ -414,6 +457,10 @@
         "command": "codeQL.quickEval",
         "title": "CodeQL: Quick Evaluation"
       },
+      {
+        "command": "codeQL.quickEvalCount",
+        "title": "CodeQL: Quick Evaluation Count"
+      },
       {
         "command": "codeQL.quickEvalContextEditor",
         "title": "CodeQL: Quick Evaluation"
@@ -434,6 +481,14 @@
         "command": "codeQL.previewQueryHelp",
         "title": "CodeQL: Preview Query Help"
       },
+      {
+        "command": "codeQL.previewQueryHelpContextExplorer",
+        "title": "CodeQL: Preview Query Help"
+      },
+      {
+        "command": "codeQL.previewQueryHelpContextEditor",
+        "title": "CodeQL: Preview Query Help"
+      },
       {
         "command": "codeQL.quickQuery",
         "title": "CodeQL: Quick Query"
@@ -450,6 +505,33 @@
         "command": "codeQL.copyVersion",
         "title": "CodeQL: Copy Version Information"
       },
+      {
+        "command": "codeQLQueries.runLocalQueryFromQueriesPanel",
+        "title": "Run local query",
+        "icon": "$(run)"
+      },
+      {
+        "command": "codeQLQueries.runLocalQueriesFromPanel",
+        "title": "Run local queries",
+        "icon": "$(run-all)"
+      },
+      {
+        "command": "codeQL.runLocalQueryFromFileTab",
+        "title": "CodeQL: Run local query",
+        "icon": "$(run)"
+      },
+      {
+        "command": "codeQLQueries.runLocalQueryContextMenu",
+        "title": "Run against local database"
+      },
+      {
+        "command": "codeQLQueries.runLocalQueriesContextMenu",
+        "title": "Run against local database"
+      },
+      {
+        "command": "codeQLQueries.runVariantAnalysisContextMenu",
+        "title": "Run against variant analysis repositories"
+      },
       {
         "command": "codeQLVariantAnalysisRepositories.openConfigFile",
         "title": "Open database configuration file",
@@ -465,6 +547,10 @@
         "title": "Add new list",
         "icon": "$(new-folder)"
       },
+      {
+        "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+        "title": "Add repositories with GitHub Code Search"
+      },
       {
         "command": "codeQLVariantAnalysisRepositories.setSelectedItem",
         "title": "Select"
@@ -786,6 +872,11 @@
         "title": "CodeQL: Go to QL Code",
         "enablement": "codeql.hasQLSource"
       },
+      {
+        "command": "codeQL.gotoQLContextEditor",
+        "title": "CodeQL: Go to QL Code",
+        "enablement": "codeql.hasQLSource"
+      },
       {
         "command": "codeQL.openDataExtensionsEditor",
         "title": "CodeQL: Open Data Extensions Editor"
@@ -812,6 +903,13 @@
       }
     ],
     "menus": {
+      "editor/title": [
+        {
+          "command": "codeQL.runLocalQueryFromFileTab",
+          "group": "navigation",
+          "when": "resourceExtname == .ql && codeQL.currentDatabaseItem"
+        }
+      ],
       "view/title": [
         {
           "command": "codeQLDatabases.sortByName",
@@ -905,6 +1003,11 @@
           "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canBeOpenedOnGitHub/",
           "group": "2_qlContextMenu@1"
         },
+        {
+          "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+          "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/",
+          "group": "2_qlContextMenu@1"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "group": "inline",
@@ -1030,6 +1133,31 @@
           "group": "1_queryHistory@1",
           "when": "viewItem == remoteResultsItem"
         },
+        {
+          "command": "codeQLQueries.runLocalQueryFromQueriesPanel",
+          "group": "inline",
+          "when": "view == codeQLQueries && viewItem == queryFile && codeQL.currentDatabaseItem"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueryContextMenu",
+          "group": "queriesPanel@1",
+          "when": "view == codeQLQueries && viewItem == queryFile && codeQL.currentDatabaseItem"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesContextMenu",
+          "group": "queriesPanel@1",
+          "when": "view == codeQLQueries && viewItem == queryFolder && codeQL.currentDatabaseItem"
+        },
+        {
+          "command": "codeQLQueries.runVariantAnalysisContextMenu",
+          "group": "queriesPanel@1",
+          "when": "view == codeQLQueries && viewItem == queryFile"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesFromPanel",
+          "group": "inline",
+          "when": "view == codeQLQueries && viewItem == queryFolder && codeQL.currentDatabaseItem"
+        },
         {
           "command": "codeQLTests.showOutputDifferences",
           "group": "qltest@1",
@@ -1075,7 +1203,7 @@
           "when": "resourceExtname == .qlref"
         },
         {
-          "command": "codeQL.previewQueryHelp",
+          "command": "codeQL.previewQueryHelpContextExplorer",
           "group": "9_qlCommands",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         }
@@ -1089,6 +1217,18 @@
           "command": "codeQL.runQuery",
           "when": "resourceLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQLQueries.runLocalQueryFromQueriesPanel",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesFromPanel",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.runLocalQueryFromFileTab",
+          "when": "false"
+        },
         {
           "command": "codeQL.runQueryContextEditor",
           "when": "false"
@@ -1141,6 +1281,10 @@
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
         },
+        {
+          "command": "codeQL.quickEvalCount",
+          "when": "editorLangId == ql && codeql.supportsQuickEvalCount"
+        },
         {
           "command": "codeQL.quickEvalContextEditor",
           "when": "false"
@@ -1161,6 +1305,14 @@
           "command": "codeQL.previewQueryHelp",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         },
+        {
+          "command": "codeQL.previewQueryHelpContextEditor",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.previewQueryHelpContextExplorer",
+          "when": "false"
+        },
         {
           "command": "codeQL.setCurrentDatabase",
           "when": "false"
@@ -1201,6 +1353,18 @@
           "command": "codeQL.openDataExtensionsEditor",
           "when": "config.codeQL.canary && config.codeQL.dataExtensions.editor"
         },
+        {
+          "command": "codeQLQueries.runLocalQueryContextMenu",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesContextMenu",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueries.runVariantAnalysisContextMenu",
+          "when": "false"
+        },
         {
           "command": "codeQLVariantAnalysisRepositories.openConfigFile",
           "when": "false"
@@ -1233,6 +1397,10 @@
           "command": "codeQLVariantAnalysisRepositories.removeItemContextMenu",
           "when": "false"
         },
+        {
+          "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "when": "false"
@@ -1424,6 +1592,10 @@
         {
           "command": "codeQLTests.acceptOutputContextTestItem",
           "when": "false"
+        },
+        {
+          "command": "codeQL.gotoQLContextEditor",
+          "when": "false"
         }
       ],
       "editor/context": [
@@ -1468,11 +1640,11 @@
           "when": "resourceExtname == .qlref"
         },
         {
-          "command": "codeQL.previewQueryHelp",
+          "command": "codeQL.previewQueryHelpContextEditor",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         },
         {
-          "command": "codeQL.gotoQL",
+          "command": "codeQL.gotoQLContextEditor",
           "when": "editorLangId == ql-summary && config.codeQL.canary"
         }
       ]
@@ -1488,6 +1660,11 @@
     },
     "views": {
       "ql-container": [
+        {
+          "id": "codeQLQueries",
+          "name": "Queries",
+          "when": "config.codeQL.canary && config.codeQL.queriesPanel"
+        },
         {
           "id": "codeQLDatabases",
           "name": "Databases"
@@ -1520,6 +1697,10 @@
         "view": "codeQLQueryHistory",
         "contents": "You have no query history items at the moment.\n\nSelect a database to run a CodeQL query and get your first results."
       },
+      {
+        "view": "codeQLQueries",
+        "contents": "This workspace doesn't contain any CodeQL queries at the moment."
+      },
       {
         "view": "codeQLDatabases",
         "contents": "Add a CodeQL database:\n[From a folder](command:codeQLDatabases.chooseDatabaseFolder)\n[From an archive](command:codeQLDatabases.chooseDatabaseArchive)\n[From a URL (as a zip file)](command:codeQLDatabases.chooseDatabaseInternet)\n[From GitHub](command:codeQLDatabases.chooseDatabaseGithub)"
@@ -1583,7 +1764,7 @@
     "path-browserify": "^1.0.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
-    "semver": "~7.3.2",
+    "semver": "~7.5.2",
     "source-map": "^0.7.4",
     "source-map-support": "^0.5.21",
     "stream": "^0.0.2",
@@ -1604,7 +1785,7 @@
   "devDependencies": {
     "@babel/core": "^7.18.13",
     "@babel/plugin-transform-modules-commonjs": "^7.18.6",
-    "@faker-js/faker": "^7.5.0",
+    "@faker-js/faker": "^8.0.2",
     "@github/markdownlint-github": "^0.3.0",
     "@octokit/plugin-throttling": "^5.0.1",
     "@storybook/addon-actions": "^6.5.17-alpha.0",
@@ -1615,6 +1796,7 @@
     "@storybook/manager-webpack5": "^6.5.17-alpha.0",
     "@storybook/react": "^6.5.17-alpha.0",
     "@storybook/testing-library": "^0.0.13",
+    "@testing-library/dom": "^9.3.0",
     "@testing-library/jest-dom": "^5.16.5",
     "@testing-library/react": "^14.0.0",
     "@testing-library/user-event": "^14.4.3",
@@ -1648,22 +1830,21 @@
     "@types/vscode": "^1.67.0",
     "@types/webpack": "^5.28.0",
     "@types/webpack-env": "^1.18.0",
-    "@types/xml2js": "~0.4.4",
     "@typescript-eslint/eslint-plugin": "^5.38.0",
     "@typescript-eslint/parser": "^5.38.0",
     "@vscode/test-electron": "^2.2.0",
-    "@vscode/vsce": "^2.15.0",
+    "@vscode/vsce": "^2.19.0",
     "ansi-colors": "^4.1.1",
     "applicationinsights": "^2.3.5",
     "cross-env": "^7.0.3",
-    "css-loader": "~3.1.0",
+    "css-loader": "~6.8.1",
     "del": "^6.0.0",
     "esbuild": "^0.15.15",
     "eslint": "^8.23.1",
     "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-etc": "^2.0.2",
     "eslint-plugin-github": "^4.4.1",
-    "eslint-plugin-jest-dom": "^4.0.2",
+    "eslint-plugin-jest-dom": "^5.0.1",
     "eslint-plugin-prettier": "^4.2.1",
     "eslint-plugin-react": "^7.31.8",
     "eslint-plugin-react-hooks": "^4.6.0",
@@ -1684,7 +1865,7 @@
     "markdownlint-cli2-formatter-pretty": "^0.0.4",
     "mini-css-extract-plugin": "^2.6.1",
     "npm-run-all": "^4.1.5",
-    "patch-package": "^6.5.0",
+    "patch-package": "^7.0.0",
     "prettier": "^2.7.1",
     "tar-stream": "^3.0.0",
     "through2": "^4.0.2",

extensions/ql-vscode/scripts/add-fields-to-scenarios.ts

@@ -18,13 +18,16 @@ import { Octokit, type RestEndpointMethodTypes } from "@octokit/rest";
 import { throttling } from "@octokit/plugin-throttling";
 
 import { getFiles } from "./util/files";
-import type { GitHubApiRequest } from "../src/mocks/gh-api-request";
-import { isGetVariantAnalysisRequest } from "../src/mocks/gh-api-request";
+import type { GitHubApiRequest } from "../src/variant-analysis/gh-api/mocks/gh-api-request";
+import { isGetVariantAnalysisRequest } from "../src/variant-analysis/gh-api/mocks/gh-api-request";
 import { VariantAnalysis } from "../src/variant-analysis/gh-api/variant-analysis";
 import { RepositoryWithMetadata } from "../src/variant-analysis/gh-api/repository";
 
 const extensionDirectory = resolve(__dirname, "..");
-const scenariosDirectory = resolve(extensionDirectory, "src/mocks/scenarios");
+const scenariosDirectory = resolve(
+  extensionDirectory,
+  "src/variant-analysis/gh-api/mocks/scenarios",
+);
 
 // Make sure we don't run into rate limits by automatically waiting until we can
 // make another request.

extensions/ql-vscode/scripts/fix-scenario-file-numbering.ts

@@ -20,7 +20,10 @@ if (process.argv.length !== 3) {
 const scenarioName = process.argv[2];
 
 const extensionDirectory = resolve(__dirname, "..");
-const scenariosDirectory = resolve(extensionDirectory, "src/mocks/scenarios");
+const scenariosDirectory = resolve(
+  extensionDirectory,
+  "src/variant-analysis/gh-api/mocks/scenarios",
+);
 const scenarioDirectory = resolve(scenariosDirectory, scenarioName);
 
 async function fixScenarioFiles() {

extensions/ql-vscode/scripts/lint-scenarios.ts

@@ -8,13 +8,19 @@ import { getFiles } from "./util/files";
 
 const extensionDirectory = resolve(__dirname, "..");
 const rootDirectory = resolve(extensionDirectory, "../..");
-const scenariosDirectory = resolve(extensionDirectory, "src/mocks/scenarios");
+const scenariosDirectory = resolve(
+  extensionDirectory,
+  "src/variant-analysis/gh-api/mocks/scenarios",
+);
 
 const debug = process.env.RUNNER_DEBUG || process.argv.includes("--debug");
 
 async function lintScenarios() {
   const schema = createGenerator({
-    path: resolve(extensionDirectory, "src/mocks/gh-api-request.ts"),
+    path: resolve(
+      extensionDirectory,
+      "src/variant-analysis/gh-api/mocks/gh-api-request.ts",
+    ),
     tsconfig: resolve(extensionDirectory, "tsconfig.json"),
     type: "GitHubApiRequest",
     skipTypeCheck: true,

extensions/ql-vscode/src/code-tour.ts

@@ -0,0 +1,55 @@
+import { AppCommandManager } from "./common/commands";
+import { Uri, workspace } from "vscode";
+import { join } from "path";
+import { pathExists } from "fs-extra";
+import { isCodespacesTemplate } from "./config";
+import { showBinaryChoiceDialog } from "./common/vscode/dialog";
+import { extLogger } from "./common/logging/vscode";
+
+/**
+ * Check if the current workspace is the CodeTour and open the workspace folder.
+ * Without this, we can't run the code tour correctly.
+ **/
+export async function prepareCodeTour(
+  commandManager: AppCommandManager,
+): Promise<void> {
+  if (workspace.workspaceFolders?.length) {
+    const currentFolder = workspace.workspaceFolders[0].uri.fsPath;
+
+    const tutorialWorkspacePath = join(
+      currentFolder,
+      "tutorial.code-workspace",
+    );
+    const toursFolderPath = join(currentFolder, ".tours");
+
+    /** We're opening the tutorial workspace, if we detect it.
+     * This will only happen if the following three conditions are met:
+     * - the .tours folder exists
+     * - the tutorial.code-workspace file exists
+     * - the CODESPACES_TEMPLATE setting doesn't exist (it's only set if the user has already opened
+     * the tutorial workspace so it's a good indicator that the user is in the folder but has ignored
+     * the prompt to open the workspace)
+     */
+    if (
+      (await pathExists(tutorialWorkspacePath)) &&
+      (await pathExists(toursFolderPath)) &&
+      !isCodespacesTemplate()
+    ) {
+      const answer = await showBinaryChoiceDialog(
+        "We've detected you're in the CodeQL Tour repo. We will need to open the workspace file to continue. Reload?",
+      );
+
+      if (!answer) {
+        return;
+      }
+
+      const tutorialWorkspaceUri = Uri.file(tutorialWorkspacePath);
+
+      void extLogger.log(
+        `In prepareCodeTour() method, going to open the tutorial workspace file: ${tutorialWorkspacePath}`,
+      );
+
+      await commandManager.execute("vscode.openFolder", tutorialWorkspaceUri);
+    }
+  }
+}

extensions/ql-vscode/src/codeql-cli/cli-version.ts

@@ -1,7 +1,7 @@
 import * as semver from "semver";
 import { runCodeQlCliCommand } from "./cli";
-import { Logger } from "./common";
-import { getErrorMessage } from "./pure/helpers-pure";
+import { Logger } from "../common/logging";
+import { getErrorMessage } from "../common/helpers-pure";
 
 /**
  * Get the version of a CodeQL CLI.

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -11,8 +11,8 @@ import tk from "tree-kill";
 import { promisify } from "util";
 import { CancellationToken, Disposable, Uri } from "vscode";
 
-import { BQRSInfo, DecodedBqrsChunk } from "./pure/bqrs-cli-types";
-import { allowCanaryQueryServer, CliConfig } from "./config";
+import { BQRSInfo, DecodedBqrsChunk } from "../common/bqrs-cli-types";
+import { allowCanaryQueryServer, CliConfig } from "../config";
 import {
   DistributionProvider,
   FindDistributionResultKind,
@@ -21,14 +21,15 @@ import {
   assertNever,
   getErrorMessage,
   getErrorStack,
-} from "./pure/helpers-pure";
-import { QueryMetadata, SortDirection } from "./pure/interface-types";
-import { BaseLogger, Logger, ProgressReporter } from "./common";
-import { CompilationMessage } from "./pure/legacy-messages";
-import { sarifParser } from "./sarif-parser";
-import { walkDirectory } from "./helpers";
-import { App } from "./common/app";
-import { QueryLanguage } from "./common/query-language";
+} from "../common/helpers-pure";
+import { walkDirectory } from "../common/files";
+import { QueryMetadata, SortDirection } from "../common/interface-types";
+import { BaseLogger, Logger } from "../common/logging";
+import { ProgressReporter } from "../common/logging/vscode";
+import { CompilationMessage } from "../query-server/legacy-messages";
+import { sarifParser } from "../common/sarif-parser";
+import { App } from "../common/app";
+import { QueryLanguage } from "../common/query-language";
 
 /**
  * The version of the SARIF format that we are using.
@@ -134,6 +135,11 @@ export interface SourceInfo {
   sourceLocationPrefix: string;
 }
 
+/**
+ * The expected output of `codeql resolve queries`.
+ */
+export type ResolvedQueries = string[];
+
 /**
  * The expected output of `codeql resolve tests`.
  */
@@ -213,7 +219,7 @@ export class CodeQLCliServer implements Disposable {
     private readonly app: App,
     private distributionProvider: DistributionProvider,
     private cliConfig: CliConfig,
-    private logger: Logger,
+    public readonly logger: Logger,
   ) {
     this.commandQueue = [];
     this.commandInProcess = false;
@@ -325,6 +331,7 @@ export class CodeQLCliServer implements Disposable {
     commandArgs: string[],
     description: string,
     onLine?: OnLineCallback,
+    silent?: boolean,
   ): Promise<string> {
     const stderrBuffers: Buffer[] = [];
     if (this.commandInProcess) {
@@ -344,7 +351,12 @@ export class CodeQLCliServer implements Disposable {
       // Compute the full args array
       const args = command.concat(LOGGING_FLAGS).concat(commandArgs);
       const argsString = args.join(" ");
-      void this.logger.log(`${description} using CodeQL CLI: ${argsString}...`);
+      // If we are running silently, we don't want to print anything to the console.
+      if (!silent) {
+        void this.logger.log(
+          `${description} using CodeQL CLI: ${argsString}...`,
+        );
+      }
       try {
         await new Promise<void>((resolve, reject) => {
           // Start listening to stdout
@@ -390,24 +402,30 @@ export class CodeQLCliServer implements Disposable {
         const fullBuffer = Buffer.concat(stdoutBuffers);
         // Make sure we remove the terminator;
         const data = fullBuffer.toString("utf8", 0, fullBuffer.length - 1);
-        void this.logger.log("CLI command succeeded.");
+        if (!silent) {
+          void this.logger.log("CLI command succeeded.");
+        }
         return data;
       } catch (err) {
         // Kill the process if it isn't already dead.
         this.killProcessIfRunning();
-        // Report the error (if there is a stderr then use that otherwise just report the error cod or nodejs error)
+        // Report the error (if there is a stderr then use that otherwise just report the error code or nodejs error)
         const newError =
           stderrBuffers.length === 0
-            ? new Error(`${description} failed: ${err}`)
+            ? new Error(
+                `${description} failed with args:${EOL}    ${argsString}${EOL}${err}`,
+              )
             : new Error(
-                `${description} failed: ${Buffer.concat(stderrBuffers).toString(
-                  "utf8",
-                )}`,
+                `${description} failed with args:${EOL}    ${argsString}${EOL}${Buffer.concat(
+                  stderrBuffers,
+                ).toString("utf8")}`,
               );
         newError.stack += getErrorStack(err);
         throw newError;
       } finally {
-        void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        if (!silent) {
+          void this.logger.log(Buffer.concat(stderrBuffers).toString("utf8"));
+        }
         // Remove the listeners we set up.
         process.stdout.removeAllListeners("data");
         process.stderr.removeAllListeners("data");
@@ -544,9 +562,11 @@ export class CodeQLCliServer implements Disposable {
     {
       progressReporter,
       onLine,
+      silent = false,
     }: {
       progressReporter?: ProgressReporter;
       onLine?: OnLineCallback;
+      silent?: boolean;
     } = {},
   ): Promise<string> {
     if (progressReporter) {
@@ -562,6 +582,7 @@ export class CodeQLCliServer implements Disposable {
             commandArgs,
             description,
             onLine,
+            silent,
           ).then(resolve, reject);
         } catch (err) {
           reject(err);
@@ -595,10 +616,12 @@ export class CodeQLCliServer implements Disposable {
       addFormat = true,
       progressReporter,
       onLine,
+      silent = false,
     }: {
       addFormat?: boolean;
       progressReporter?: ProgressReporter;
       onLine?: OnLineCallback;
+      silent?: boolean;
     } = {},
   ): Promise<OutputType> {
     let args: string[] = [];
@@ -609,6 +632,7 @@ export class CodeQLCliServer implements Disposable {
     const result = await this.runCodeQlCliCommand(command, args, description, {
       progressReporter,
       onLine,
+      silent,
     });
     try {
       return JSON.parse(result) as OutputType;
@@ -695,6 +719,7 @@ export class CodeQLCliServer implements Disposable {
   async resolveLibraryPath(
     workspaces: string[],
     queryPath: string,
+    silent = false,
   ): Promise<QuerySetup> {
     const subcommandArgs = [
       "--query",
@@ -705,6 +730,7 @@ export class CodeQLCliServer implements Disposable {
       ["resolve", "library-path"],
       subcommandArgs,
       "Resolving library paths",
+      { silent },
     );
   }
 
@@ -731,6 +757,25 @@ export class CodeQLCliServer implements Disposable {
     );
   }
 
+  /**
+   * Finds all available queries in a given directory.
+   * @param queryDir Root of directory tree to search for queries.
+   * @param silent If true, don't print logs to the CodeQL extension log.
+   * @returns The list of queries that were found.
+   */
+  public async resolveQueries(
+    queryDir: string,
+    silent?: boolean,
+  ): Promise<ResolvedQueries> {
+    const subcommandArgs = [queryDir];
+    return await this.runJsonCodeQlCliCommand<ResolvedQueries>(
+      ["resolve", "queries"],
+      subcommandArgs,
+      "Resolving queries",
+      { silent },
+    );
+  }
+
   /**
    * Finds all available QL tests in a given directory.
    * @param testPath Root of directory tree to search for tests.
@@ -1031,6 +1076,7 @@ export class CodeQLCliServer implements Disposable {
     resultsPath: string,
     interpretedResultsPath: string,
     sourceInfo?: SourceInfo,
+    args?: string[],
   ): Promise<sarif.Log> {
     const additionalArgs = [
       // TODO: This flag means that we don't group interpreted results
@@ -1038,6 +1084,7 @@ export class CodeQLCliServer implements Disposable {
       // interpretation with and without this flag, or do some
       // grouping client-side.
       "--no-group-results",
+      ...(args ?? []),
     ];
 
     await this.runInterpretCommand(
@@ -1436,6 +1483,13 @@ export class CodeQLCliServer implements Disposable {
             CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG,
           ) >= 0,
         );
+        await this.app.commands.execute(
+          "setContext",
+          "codeql.supportsQuickEvalCount",
+          newVersion.compare(
+            CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
+          ) >= 0,
+        );
       } catch (e) {
         this._versionChangedListeners.forEach((listener) =>
           listener(undefined),
@@ -1525,10 +1579,23 @@ export function spawnServer(
     );
   }
 
+  let lastStdout: any = undefined;
+  child.stdout!.on("data", (data) => {
+    lastStdout = data;
+  });
   // Set up event listeners.
-  child.on("close", (code) =>
-    logger.log(`Child process exited with code ${code}`),
-  );
+  child.on("close", async (code, signal) => {
+    if (code !== null)
+      void logger.log(`Child process exited with code ${code}`);
+    if (signal)
+      void logger.log(
+        `Child process exited due to receipt of signal ${signal}`,
+      );
+    // If the process exited abnormally, log the last stdout message,
+    // It may be from the jvm.
+    if (code !== 0 && lastStdout !== undefined)
+      void logger.log(`Last stdout was "${lastStdout.toString()}"`);
+  });
   child.stderr!.on("data", stderrListener);
   if (stdoutListener !== undefined) {
     child.stdout!.on("data", stdoutListener);
@@ -1699,31 +1766,32 @@ async function logStream(stream: Readable, logger: BaseLogger): Promise<void> {
   }
 }
 
-export function shouldDebugIdeServer() {
+function isEnvTrue(name: string): boolean {
   return (
-    "IDE_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.IDE_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.IDE_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
+    name in process.env &&
+    process.env[name] !== "0" &&
+    // Use en-US since we expect the value to be either "false" or "FALSE", not a localized version.
+    process.env[name]?.toLocaleLowerCase("en-US") !== "false"
   );
 }
 
+export function shouldDebugIdeServer() {
+  return isEnvTrue("IDE_SERVER_JAVA_DEBUG");
+}
+
 export function shouldDebugQueryServer() {
-  return (
-    "QUERY_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.QUERY_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.QUERY_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
-  );
+  return isEnvTrue("QUERY_SERVER_JAVA_DEBUG");
 }
 
 export function shouldDebugCliServer() {
-  return (
-    "CLI_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.CLI_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.CLI_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
-  );
+  return isEnvTrue("CLI_SERVER_JAVA_DEBUG");
 }
 
 export class CliVersionConstraint {
+  // The oldest version of the CLI that we support. This is used to determine
+  // whether to show a warning about the CLI being too old on startup.
+  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.7.6");
+
   /**
    * CLI version where building QLX packs for remote queries is supported.
    * (The options were _accepted_ by a few earlier versions, but only from
@@ -1782,6 +1850,20 @@ export class CliVersionConstraint {
     "2.12.4",
   );
 
+  public static CLI_VERSION_GLOBAL_CACHE = new SemVer("2.12.4");
+
+  /**
+   * CLI version where the query server supports quick-eval count mode.
+   */
+  public static CLI_VERSION_WITH_QUICK_EVAL_COUNT = new SemVer("2.13.3");
+
+  /**
+   * CLI version where the langauge server supports visisbility change notifications.
+   */
+  public static CLI_VERSION_WITH_VISIBILITY_NOTIFICATIONS = new SemVer(
+    "2.14.0",
+  );
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1851,4 +1933,20 @@ export class CliVersionConstraint {
       CliVersionConstraint.CLI_VERSION_WITH_ADDITIONAL_PACKS_INSTALL,
     );
   }
+
+  async usesGlobalCompilationCache() {
+    return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_GLOBAL_CACHE);
+  }
+
+  async supportsVisibilityNotifications() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_VISIBILITY_NOTIFICATIONS,
+    );
+  }
+
+  async supportsQuickEvalCount() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
+    );
+  }
 }

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -3,24 +3,29 @@ import { pathExists, mkdtemp, createWriteStream, remove } from "fs-extra";
 import { tmpdir } from "os";
 import { delimiter, dirname, join } from "path";
 import * as semver from "semver";
-import { parse } from "url";
+import { URL } from "url";
 import { ExtensionContext, Event } from "vscode";
-import { DistributionConfig } from "./config";
-import {
-  InvocationRateLimiter,
-  InvocationRateLimiterResultKind,
-  showAndLogErrorMessage,
-  showAndLogWarningMessage,
-} from "./helpers";
-import { extLogger } from "./common";
+import { DistributionConfig } from "../config";
+import { extLogger } from "../common/logging/vscode";
 import { getCodeQlCliVersion } from "./cli-version";
-import { ProgressCallback, reportStreamProgress } from "./progress";
+import {
+  ProgressCallback,
+  reportStreamProgress,
+} from "../common/vscode/progress";
 import {
   codeQlLauncherName,
   deprecatedCodeQlLauncherName,
   extractZipArchive,
   getRequiredAssetName,
-} from "./pure/distribution";
+} from "../common/distribution";
+import {
+  InvocationRateLimiter,
+  InvocationRateLimiterResultKind,
+} from "../common/invocation-rate-limiter";
+import {
+  showAndLogErrorMessage,
+  showAndLogWarningMessage,
+} from "../common/logging";
 
 /**
  * distribution.ts
@@ -73,7 +78,7 @@ export class DistributionManager implements DistributionProvider {
         extensionContext,
       );
     this.updateCheckRateLimiter = new InvocationRateLimiter(
-      extensionContext,
+      extensionContext.globalState,
       "extensionSpecificDistributionUpdateCheck",
       () =>
         this.extensionSpecificDistributionManager.checkForUpdatesToDistribution(),
@@ -155,6 +160,7 @@ export class DistributionManager implements DistributionProvider {
     if (this.config.customCodeQlPath) {
       if (!(await pathExists(this.config.customCodeQlPath))) {
         void showAndLogErrorMessage(
+          extLogger,
           `The CodeQL executable path is specified as "${this.config.customCodeQlPath}" ` +
             "by a configuration setting, but a CodeQL executable could not be found at that path. Please check " +
             "that a CodeQL executable exists at the specified path or remove the setting.",
@@ -502,7 +508,7 @@ class ExtensionSpecificDistributionManager {
         0,
       ) || "";
     return join(
-      this.extensionContext.globalStoragePath,
+      this.extensionContext.globalStorageUri.fsPath,
       ExtensionSpecificDistributionManager._currentDistributionFolderBaseName +
         distributionFolderIndex,
     );
@@ -667,7 +673,7 @@ export class ReleasesApiConsumer {
       redirectUrl &&
       redirectCount < ReleasesApiConsumer._maxRedirects
     ) {
-      const parsedRedirectUrl = parse(redirectUrl);
+      const parsedRedirectUrl = new URL(redirectUrl);
       if (parsedRedirectUrl.protocol !== "https:") {
         throw new Error("Encountered a non-https redirect, rejecting");
       }
@@ -847,6 +853,7 @@ export async function getExecutableFromDirectory(
 
 function warnDeprecatedLauncher() {
   void showAndLogWarningMessage(
+    extLogger,
     `The "${deprecatedCodeQlLauncherName()!}" launcher has been deprecated and will be removed in a future version. ` +
       `Please use "${codeQlLauncherName()}" instead. It is recommended to update to the latest CodeQL binaries.`,
   );

extensions/ql-vscode/src/codeql-cli/query-language.ts

@@ -0,0 +1,80 @@
+import { CodeQLCliServer } from "./cli";
+import { Uri, window } from "vscode";
+import { isQueryLanguage, QueryLanguage } from "../common/query-language";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { extLogger } from "../common/logging/vscode";
+import { UserCancellationException } from "../common/vscode/progress";
+import { showAndLogErrorMessage } from "../common/logging";
+
+/**
+ * Finds the language that a query targets.
+ * If it can't be autodetected, prompt the user to specify the language manually.
+ */
+export async function findLanguage(
+  cliServer: CodeQLCliServer,
+  queryUri: Uri | undefined,
+): Promise<QueryLanguage | undefined> {
+  const uri = queryUri || window.activeTextEditor?.document.uri;
+  if (uri !== undefined) {
+    try {
+      const queryInfo = await cliServer.resolveQueryByLanguage(
+        getOnDiskWorkspaceFolders(),
+        uri,
+      );
+      const language = Object.keys(queryInfo.byLanguage)[0];
+      void extLogger.log(`Detected query language: ${language}`);
+
+      if (isQueryLanguage(language)) {
+        return language;
+      }
+
+      void extLogger.log(
+        "Query language is unsupported. Select language manually.",
+      );
+    } catch (e) {
+      void extLogger.log(
+        "Could not autodetect query language. Select language manually.",
+      );
+    }
+  }
+
+  // will be undefined if user cancels the quick pick.
+  return await askForLanguage(cliServer, false);
+}
+
+export async function askForLanguage(
+  cliServer: CodeQLCliServer,
+  throwOnEmpty = true,
+): Promise<QueryLanguage | undefined> {
+  const language = await window.showQuickPick(
+    await cliServer.getSupportedLanguages(),
+    {
+      placeHolder: "Select target language for your query",
+      ignoreFocusOut: true,
+    },
+  );
+  if (!language) {
+    // This only happens if the user cancels the quick pick.
+    if (throwOnEmpty) {
+      throw new UserCancellationException("Cancelled.");
+    } else {
+      void showAndLogErrorMessage(
+        extLogger,
+        "Language not found. Language must be specified manually.",
+      );
+    }
+    return undefined;
+  }
+
+  if (!isQueryLanguage(language)) {
+    void showAndLogErrorMessage(
+      extLogger,
+      `Language '${language}' is not supported. Only languages ${Object.values(
+        QueryLanguage,
+      ).join(", ")} are supported.`,
+    );
+    return undefined;
+  }
+
+  return language;
+}

extensions/ql-vscode/src/codeql-cli/query-metadata.ts

@@ -0,0 +1,22 @@
+import { CodeQLCliServer } from "./cli";
+import { QueryMetadata } from "../common/interface-types";
+import { extLogger } from "../common/logging/vscode";
+
+/**
+ * Gets metadata for a query, if it exists.
+ * @param cliServer The CLI server.
+ * @param queryPath The path to the query.
+ * @returns A promise that resolves to the query metadata, if available.
+ */
+export async function tryGetQueryMetadata(
+  cliServer: CodeQLCliServer,
+  queryPath: string,
+): Promise<QueryMetadata | undefined> {
+  try {
+    return await cliServer.resolveMetadata(queryPath);
+  } catch (e) {
+    // Ignore errors and provide no metadata.
+    void extLogger.log(`Couldn't resolve metadata for ${queryPath}: ${e}`);
+    return;
+  }
+}

extensions/ql-vscode/src/common/app.ts

@@ -1,14 +1,16 @@
 import { Credentials } from "./authentication";
-import { Disposable } from "../pure/disposable-object";
+import { Disposable } from "./disposable-object";
 import { AppEventEmitter } from "./events";
-import { Logger } from "./logging";
+import { NotificationLogger } from "./logging";
 import { Memento } from "./memento";
 import { AppCommandManager } from "./commands";
+import { AppTelemetry } from "./telemetry";
 
 export interface App {
   createEventEmitter<T>(): AppEventEmitter<T>;
   readonly mode: AppMode;
-  readonly logger: Logger;
+  readonly logger: NotificationLogger;
+  readonly telemetry?: AppTelemetry;
   readonly subscriptions: Disposable[];
   readonly extensionPath: string;
   readonly globalStoragePath: string;
@@ -16,6 +18,7 @@ export interface App {
   readonly workspaceState: Memento;
   readonly credentials: Credentials;
   readonly commands: AppCommandManager;
+  readonly environment: EnvironmentContext;
 }
 
 export enum AppMode {
@@ -23,3 +26,7 @@ export enum AppMode {
   Development = 2,
   Test = 3,
 }
+
+export interface EnvironmentContext {
+  language: string;
+}

extensions/ql-vscode/src/common/bqrs-cli-types.ts

@@ -115,7 +115,7 @@ export type BqrsKind =
   | "Entity";
 
 interface BqrsColumn {
-  name: string;
+  name?: string;
   kind: BqrsKind;
 }
 export interface DecodedBqrsChunk {

extensions/ql-vscode/src/common/bqrs-utils.ts

@@ -4,7 +4,7 @@ import {
   LineColumnLocation,
   WholeFileLocation,
 } from "./bqrs-cli-types";
-import { createRemoteFileRef } from "./location-link-utils";
+import { createRemoteFileRef } from "../common/location-link-utils";
 
 /**
  * The CodeQL filesystem libraries use this pattern in `getURL()` predicates

extensions/ql-vscode/src/common/commands.ts

@@ -1,17 +1,18 @@
 import type { CommandManager } from "../packages/commands";
 import type { Uri, Range, TextDocumentShowOptions } from "vscode";
-import type { AstItem } from "../astViewer";
+import type { AstItem } from "../language-support";
 import type { DbTreeViewItem } from "../databases/ui/db-tree-view-item";
-import type { DatabaseItem } from "../local-databases";
+import type { DatabaseItem } from "../databases/local-databases";
 import type { QueryHistoryInfo } from "../query-history/query-history-info";
-import type { RepositoriesFilterSortStateWithIds } from "../pure/variant-analysis-filter-sort";
-import type { TestTreeNode } from "../test-tree-node";
+import type { RepositoriesFilterSortStateWithIds } from "../variant-analysis/shared/variant-analysis-filter-sort";
+import type { TestTreeNode } from "../query-testing/test-tree-node";
 import type {
   VariantAnalysis,
   VariantAnalysisScannedRepository,
   VariantAnalysisScannedRepositoryResult,
 } from "../variant-analysis/shared/variant-analysis";
 import type { QLDebugConfiguration } from "../debugger/debug-configuration";
+import type { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
 
 // A command function matching the signature that VS Code calls when
 // a command is invoked from a context menu on a TreeView with
@@ -115,6 +116,10 @@ export type QueryEditorCommands = {
     selectedQuery: Uri,
   ) => Promise<void>;
   "codeQL.previewQueryHelp": (selectedQuery: Uri) => Promise<void>;
+  "codeQL.previewQueryHelpContextEditor": (selectedQuery: Uri) => Promise<void>;
+  "codeQL.previewQueryHelpContextExplorer": (
+    selectedQuery: Uri,
+  ) => Promise<void>;
 };
 
 // Commands used for running local queries
@@ -125,8 +130,14 @@ export type LocalQueryCommands = {
   "codeQL.runQueryOnMultipleDatabasesContextEditor": (
     uri?: Uri,
   ) => Promise<void>;
+  "codeQLQueries.runLocalQueryFromQueriesPanel": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueryContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueriesContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueriesFromPanel": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQL.runLocalQueryFromFileTab": (uri: Uri) => Promise<void>;
   "codeQL.runQueries": ExplorerSelectionCommandFunction<Uri>;
   "codeQL.quickEval": (uri: Uri) => Promise<void>;
+  "codeQL.quickEvalCount": (uri: Uri) => Promise<void>;
   "codeQL.quickEvalContextEditor": (uri: Uri) => Promise<void>;
   "codeQL.codeLensQuickEval": (uri: Uri, range: Range) => Promise<void>;
   "codeQL.quickQuery": () => Promise<void>;
@@ -247,6 +258,9 @@ export type VariantAnalysisCommands = {
   "codeQL.monitorRehydratedVariantAnalysis": (
     variantAnalysis: VariantAnalysis,
   ) => Promise<void>;
+  "codeQL.monitorReauthenticatedVariantAnalysis": (
+    variantAnalysis: VariantAnalysis,
+  ) => Promise<void>;
   "codeQL.openVariantAnalysisLogs": (
     variantAnalysisId: number,
   ) => Promise<void>;
@@ -255,6 +269,7 @@ export type VariantAnalysisCommands = {
   ) => Promise<void>;
   "codeQL.runVariantAnalysis": (uri?: Uri) => Promise<void>;
   "codeQL.runVariantAnalysisContextEditor": (uri?: Uri) => Promise<void>;
+  "codeQLQueries.runVariantAnalysisContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
 };
 
 export type DatabasePanelCommands = {
@@ -268,6 +283,7 @@ export type DatabasePanelCommands = {
   "codeQLVariantAnalysisRepositories.openOnGitHubContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.renameItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.removeItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.importFromCodeSearch": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
 };
 
 export type AstCfgCommands = {
@@ -299,6 +315,7 @@ export type EvalLogViewerCommands = {
 
 export type SummaryLanguageSupportCommands = {
   "codeQL.gotoQL": () => Promise<void>;
+  "codeQL.gotoQLContextEditor": () => Promise<void>;
 };
 
 export type TestUICommands = {

extensions/ql-vscode/src/common/discovery.ts

@@ -1,25 +1,38 @@
-import { DisposableObject } from "./pure/disposable-object";
-import { extLogger } from "./common";
-import { getErrorMessage } from "./pure/helpers-pure";
+import { DisposableObject } from "./disposable-object";
+import { getErrorMessage } from "./helpers-pure";
+import { Logger } from "./logging";
 
 /**
  * Base class for "discovery" operations, which scan the file system to find specific kinds of
  * files. This class automatically prevents more than one discovery operation from running at the
  * same time.
  */
-export abstract class Discovery<T> extends DisposableObject {
-  private retry = false;
-  private discoveryInProgress = false;
+export abstract class Discovery extends DisposableObject {
+  private restartWhenFinished = false;
+  private currentDiscoveryPromise: Promise<void> | undefined;
 
-  constructor(private readonly name: string) {
+  constructor(
+    protected readonly name: string,
+    private readonly logger: Logger,
+  ) {
     super();
   }
 
+  /**
+   * Returns the promise of the currently running refresh operation, if one is in progress.
+   * Otherwise returns a promise that resolves immediately.
+   */
+  public waitForCurrentRefresh(): Promise<void> {
+    return this.currentDiscoveryPromise ?? Promise.resolve();
+  }
+
   /**
    * Force the discovery process to run. Normally invoked by the derived class when a relevant file
    * system change is detected.
+   *
+   * Returns a promise that resolves when the refresh is complete, including any retries.
    */
-  public refresh(): void {
+  public refresh(): Promise<void> {
     // We avoid having multiple discovery operations in progress at the same time. Otherwise, if we
     // got a storm of refresh requests due to, say, the copying or deletion of a large directory
     // tree, we could potentially spawn a separate simultaneous discovery operation for each
@@ -36,14 +49,16 @@ export abstract class Discovery<T> extends DisposableObject {
     // other change notifications that might be coming along. However, this would create more
     // latency in the common case, in order to save a bit of latency in the uncommon case.
 
-    if (this.discoveryInProgress) {
+    if (this.currentDiscoveryPromise !== undefined) {
       // There's already a discovery operation in progress. Tell it to restart when it's done.
-      this.retry = true;
+      this.restartWhenFinished = true;
     } else {
       // No discovery in progress, so start one now.
-      this.discoveryInProgress = true;
-      this.launchDiscovery();
+      this.currentDiscoveryPromise = this.launchDiscovery().finally(() => {
+        this.currentDiscoveryPromise = undefined;
+      });
     }
+    return this.currentDiscoveryPromise;
   }
 
   /**
@@ -51,46 +66,28 @@ export abstract class Discovery<T> extends DisposableObject {
    * discovery operation completes, the `update` function will be invoked with the results of the
    * discovery.
    */
-  private launchDiscovery(): void {
-    const discoveryPromise = this.discover();
-    discoveryPromise
-      .then((results) => {
-        if (!this.retry) {
-          // Update any listeners with the results of the discovery.
-          this.discoveryInProgress = false;
-          this.update(results);
-        }
-      })
+  private async launchDiscovery(): Promise<void> {
+    try {
+      await this.discover();
+    } catch (err) {
+      void this.logger.log(
+        `${this.name} failed. Reason: ${getErrorMessage(err)}`,
+      );
+    }
 
-      .catch((err: unknown) => {
-        void extLogger.log(
-          `${this.name} failed. Reason: ${getErrorMessage(err)}`,
-        );
-      })
-
-      .finally(() => {
-        if (this.retry) {
-          // Another refresh request came in while we were still running a previous discovery
-          // operation. Since the discovery results we just computed are now stale, we'll launch
-          // another discovery operation instead of updating.
-          // Note that by doing this inside of `finally`, we will relaunch discovery even if the
-          // initial discovery operation failed.
-          this.retry = false;
-          this.launchDiscovery();
-        }
-      });
+    if (this.restartWhenFinished) {
+      // Another refresh request came in while we were still running a previous discovery
+      // operation. Since the discovery results we just computed are now stale, we'll launch
+      // another discovery operation instead of updating.
+      // We want to relaunch discovery regardless of if the initial discovery operation
+      // succeeded or failed.
+      this.restartWhenFinished = false;
+      await this.launchDiscovery();
+    }
   }
 
   /**
    * Overridden by the derived class to spawn the actual discovery operation, returning the results.
    */
-  protected abstract discover(): Promise<T>;
-
-  /**
-   * Overridden by the derived class to atomically update the `Discovery` object with the results of
-   * the discovery operation, and to notify any listeners that the discovery results may have
-   * changed.
-   * @param results The discovery results returned by the `discover` function.
-   */
-  protected abstract update(results: T): void;
+  protected abstract discover(): Promise<void>;
 }

extensions/ql-vscode/src/common/events.ts

@@ -1,10 +1,10 @@
-import { Disposable } from "../pure/disposable-object";
+import { Disposable } from "./disposable-object";
 
 export interface AppEvent<T> {
   (listener: (event: T) => void): Disposable;
 }
 
-export interface AppEventEmitter<T> {
+export interface AppEventEmitter<T> extends Disposable {
   event: AppEvent<T>;
   fire(data: T): void;
 }

extensions/ql-vscode/src/common/file-tree-nodes.ts

@@ -0,0 +1,122 @@
+import { basename, dirname, join } from "path";
+import { EnvironmentContext } from "./app";
+
+/**
+ * A node in the tree of files. This will be either a `FileTreeDirectory` or a `FileTreeLeaf`.
+ */
+export abstract class FileTreeNode<T = undefined> {
+  constructor(
+    private _path: string,
+    private _name: string,
+    private _data?: T,
+  ) {}
+
+  public get path(): string {
+    return this._path;
+  }
+
+  public get name(): string {
+    return this._name;
+  }
+
+  public get data(): T | undefined {
+    return this._data;
+  }
+
+  public abstract get children(): ReadonlyArray<FileTreeNode<T>>;
+
+  public abstract finish(): void;
+}
+
+/**
+ * A directory containing one or more files or other directories.
+ */
+export class FileTreeDirectory<T = undefined> extends FileTreeNode<T> {
+  constructor(
+    _path: string,
+    _name: string,
+    protected readonly env: EnvironmentContext,
+    private _children: Array<FileTreeNode<T>> = [],
+  ) {
+    super(_path, _name);
+  }
+
+  public get children(): ReadonlyArray<FileTreeNode<T>> {
+    return this._children;
+  }
+
+  public addChild(child: FileTreeNode<T>): void {
+    this._children.push(child);
+  }
+
+  public createDirectory(relativePath: string): FileTreeDirectory<T> {
+    if (relativePath === ".") {
+      return this;
+    }
+    const dirName = dirname(relativePath);
+    if (dirName === ".") {
+      return this.createChildDirectory(relativePath);
+    } else {
+      const parent = this.createDirectory(dirName);
+      return parent.createDirectory(basename(relativePath));
+    }
+  }
+
+  public finish(): void {
+    // remove empty directories
+    this._children.filter(
+      (child) => child instanceof FileTreeLeaf || child.children.length > 0,
+    );
+    this._children.sort((a, b) =>
+      a.name.localeCompare(b.name, this.env.language),
+    );
+    this._children.forEach((child, i) => {
+      child.finish();
+      if (
+        child.children?.length === 1 &&
+        child.children[0] instanceof FileTreeDirectory
+      ) {
+        // collapse children
+        const replacement = new FileTreeDirectory<T>(
+          child.children[0].path,
+          `${child.name} / ${child.children[0].name}`,
+          this.env,
+          Array.from(child.children[0].children),
+        );
+        this._children[i] = replacement;
+      }
+    });
+  }
+
+  private createChildDirectory(name: string): FileTreeDirectory<T> {
+    const existingChild = this._children.find((child) => child.name === name);
+    if (existingChild !== undefined) {
+      return existingChild as FileTreeDirectory<T>;
+    } else {
+      const newChild = new FileTreeDirectory<T>(
+        join(this.path, name),
+        name,
+        this.env,
+      );
+      this.addChild(newChild);
+      return newChild;
+    }
+  }
+}
+
+/**
+ * A single file.
+ */
+export class FileTreeLeaf<T = undefined> extends FileTreeNode<T> {
+  constructor(_path: string, _name: string, _data?: T) {
+    super(_path, _name, _data);
+  }
+
+  public get children(): ReadonlyArray<FileTreeNode<T>> {
+    return [];
+  }
+
+  public finish(): void {
+    /**/
+  }
+}

extensions/ql-vscode/src/common/files.ts

@@ -0,0 +1,129 @@
+import { pathExists, stat, readdir, opendir } from "fs-extra";
+import { isAbsolute, join, relative, resolve } from "path";
+import { tmpdir as osTmpdir } from "os";
+
+/**
+ * Recursively finds all .ql files in this set of Uris.
+ *
+ * @param paths The list of Uris to search through
+ *
+ * @returns list of ql files and a boolean describing whether or not a directory was found/
+ */
+export async function gatherQlFiles(
+  paths: string[],
+): Promise<[string[], boolean]> {
+  const gatheredUris: Set<string> = new Set();
+  let dirFound = false;
+  for (const nextPath of paths) {
+    if ((await pathExists(nextPath)) && (await stat(nextPath)).isDirectory()) {
+      dirFound = true;
+      const subPaths = await readdir(nextPath);
+      const fullPaths = subPaths.map((p) => join(nextPath, p));
+      const nestedFiles = (await gatherQlFiles(fullPaths))[0];
+      nestedFiles.forEach((nested) => gatheredUris.add(nested));
+    } else if (nextPath.endsWith(".ql")) {
+      gatheredUris.add(nextPath);
+    }
+  }
+  return [Array.from(gatheredUris), dirFound];
+}
+
+/**
+ * Lists the names of directories inside the given path.
+ * @param path The path to the directory to read.
+ * @returns the names of the directories inside the given path.
+ */
+export async function getDirectoryNamesInsidePath(
+  path: string,
+): Promise<string[]> {
+  if (!(await pathExists(path))) {
+    throw Error(`Path does not exist: ${path}`);
+  }
+  if (!(await stat(path)).isDirectory()) {
+    throw Error(`Path is not a directory: ${path}`);
+  }
+
+  const dirItems = await readdir(path, { withFileTypes: true });
+
+  const dirNames = dirItems
+    .filter((dirent) => dirent.isDirectory())
+    .map((dirent) => dirent.name);
+
+  return dirNames;
+}
+
+export function normalizePath(path: string): string {
+  // On Windows, "C:/", "C:\", and "c:/" are all equivalent. We need
+  // to normalize the paths to ensure they all get resolved to the
+  // same format. On Windows, we also need to do the comparison
+  // case-insensitively.
+  path = resolve(path);
+  if (process.platform === "win32") {
+    path = path.toLowerCase();
+  }
+  return path;
+}
+
+export function pathsEqual(path1: string, path2: string): boolean {
+  return normalizePath(path1) === normalizePath(path2);
+}
+
+/**
+ * Returns true if `parent` contains `child`, or if they are equal.
+ */
+export function containsPath(parent: string, child: string): boolean {
+  const relativePath = relative(parent, child);
+  return (
+    !relativePath.startsWith("..") &&
+    // On windows, if the two paths are in different drives, then the
+    // relative path will be an absolute path to the other drive.
+    !isAbsolute(relativePath)
+  );
+}
+
+export async function readDirFullPaths(path: string): Promise<string[]> {
+  const baseNames = await readdir(path);
+  return baseNames.map((baseName) => join(path, baseName));
+}
+
+/**
+ * Recursively walk a directory and return the full path to all files found.
+ * Symbolic links are ignored.
+ *
+ * @param dir the directory to walk
+ *
+ * @return An iterator of the full path to all files recursively found in the directory.
+ */
+export async function* walkDirectory(
+  dir: string,
+): AsyncIterableIterator<string> {
+  const seenFiles = new Set<string>();
+  for await (const d of await opendir(dir)) {
+    const entry = join(dir, d.name);
+    seenFiles.add(entry);
+    if (d.isDirectory()) {
+      yield* walkDirectory(entry);
+    } else if (d.isFile()) {
+      yield entry;
+    }
+  }
+}
+
+/**
+ * Error thrown from methods from the `fs` module.
+ *
+ * In practice, any error matching this is likely an instance of `NodeJS.ErrnoException`.
+ * If desired in the future, we could model more fields or use `NodeJS.ErrnoException` directly.
+ */
+export interface IOError {
+  readonly code: string;
+}
+
+export function isIOError(e: any): e is IOError {
+  return e.code !== undefined && typeof e.code === "string";
+}
+
+// This function is a wrapper around `os.tmpdir()` to make it easier to mock in tests.
+export function tmpdir(): string {
+  return osTmpdir();
+}

extensions/ql-vscode/src/common/github-url-identifier-helper.ts

@@ -1,4 +1,4 @@
-import { OWNER_REGEX, REPO_REGEX } from "../pure/helpers-pure";
+import { OWNER_REGEX, REPO_REGEX } from "./helpers-pure";
 
 /**
  * Checks if a string is a valid GitHub NWO.

extensions/ql-vscode/src/common/index.ts

@@ -1 +0,0 @@
-export * from "./logging";

extensions/ql-vscode/src/common/interface-types.ts

@@ -5,18 +5,22 @@ import {
   ResultSetSchema,
   Column,
   ResolvableLocationValue,
-} from "./bqrs-cli-types";
+} from "../common/bqrs-cli-types";
 import {
   VariantAnalysis,
   VariantAnalysisScannedRepositoryResult,
   VariantAnalysisScannedRepositoryState,
 } from "../variant-analysis/shared/variant-analysis";
-import { RepositoriesFilterSortStateWithIds } from "./variant-analysis-filter-sort";
-import { ErrorLike } from "./errors";
+import {
+  RepositoriesFilterSortState,
+  RepositoriesFilterSortStateWithIds,
+} from "../variant-analysis/shared/variant-analysis-filter-sort";
+import { ErrorLike } from "../common/errors";
 import { DataFlowPaths } from "../variant-analysis/shared/data-flow-paths";
 import { ExternalApiUsage } from "../data-extensions-editor/external-api-usage";
 import { ModeledMethod } from "../data-extensions-editor/modeled-method";
 import { DataExtensionEditorViewState } from "../data-extensions-editor/shared/view-state";
+import { Mode } from "../data-extensions-editor/shared/mode";
 
 /**
  * This module contains types and code that are shared between
@@ -407,6 +411,11 @@ export interface SetVariantAnalysisMessage {
   variantAnalysis: VariantAnalysis;
 }
 
+export interface SetFilterSortStateMessage {
+  t: "setFilterSortState";
+  filterSortState: RepositoriesFilterSortState;
+}
+
 export type VariantAnalysisState = {
   variantAnalysisId: number;
 };
@@ -459,6 +468,7 @@ export interface ShowDataFlowPathsMessage {
 
 export type ToVariantAnalysisMessage =
   | SetVariantAnalysisMessage
+  | SetFilterSortStateMessage
   | SetRepoResultsMessage
   | SetRepoStatesMessage;
 
@@ -512,6 +522,11 @@ export interface AddModeledMethodsMessage {
   overrideNone?: boolean;
 }
 
+export interface SwitchModeMessage {
+  t: "switchMode";
+  mode: Mode;
+}
+
 export interface JumpToUsageMessage {
   t: "jumpToUsage";
   location: ResolvableLocationValue;
@@ -521,8 +536,8 @@ export interface OpenExtensionPackMessage {
   t: "openExtensionPack";
 }
 
-export interface OpenModelFileMessage {
-  t: "openModelFile";
+export interface RefreshExternalApiUsages {
+  t: "refreshExternalApiUsages";
 }
 
 export interface SaveModeledMethods {
@@ -535,6 +550,12 @@ export interface GenerateExternalApiMessage {
   t: "generateExternalApi";
 }
 
+export interface GenerateExternalApiFromLlmMessage {
+  t: "generateExternalApiFromLlm";
+  externalApiUsages: ExternalApiUsage[];
+  modeledMethods: Record<string, ModeledMethod>;
+}
+
 export type ToDataExtensionsEditorMessage =
   | SetExtensionPackStateMessage
   | SetExternalApiUsagesMessage
@@ -543,8 +564,10 @@ export type ToDataExtensionsEditorMessage =
 
 export type FromDataExtensionsEditorMessage =
   | ViewLoadedMsg
-  | OpenModelFileMessage
+  | SwitchModeMessage
+  | RefreshExternalApiUsages
   | OpenExtensionPackMessage
   | JumpToUsageMessage
   | SaveModeledMethods
-  | GenerateExternalApiMessage;
+  | GenerateExternalApiMessage
+  | GenerateExternalApiFromLlmMessage;

extensions/ql-vscode/src/common/invocation-rate-limiter.ts

@@ -0,0 +1,89 @@
+import { Memento } from "./memento";
+
+/**
+ * Provides a utility method to invoke a function only if a minimum time interval has elapsed since
+ * the last invocation of that function.
+ */
+export class InvocationRateLimiter<T> {
+  constructor(
+    private readonly globalState: Memento,
+    private readonly funcIdentifier: string,
+    private readonly func: () => Promise<T>,
+    private readonly createDate: (dateString?: string) => Date = (s) =>
+      s ? new Date(s) : new Date(),
+  ) {}
+
+  /**
+   * Invoke the function if `minSecondsSinceLastInvocation` seconds have elapsed since the last invocation.
+   */
+  public async invokeFunctionIfIntervalElapsed(
+    minSecondsSinceLastInvocation: number,
+  ): Promise<InvocationRateLimiterResult<T>> {
+    const updateCheckStartDate = this.createDate();
+    const lastInvocationDate = this.getLastInvocationDate();
+    if (
+      minSecondsSinceLastInvocation &&
+      lastInvocationDate &&
+      lastInvocationDate <= updateCheckStartDate &&
+      lastInvocationDate.getTime() + minSecondsSinceLastInvocation * 1000 >
+        updateCheckStartDate.getTime()
+    ) {
+      return createRateLimitedResult();
+    }
+    const result = await this.func();
+    await this.setLastInvocationDate(updateCheckStartDate);
+    return createInvokedResult(result);
+  }
+
+  private getLastInvocationDate(): Date | undefined {
+    const maybeDateString: string | undefined = this.globalState.get(
+      InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier,
+    );
+    return maybeDateString ? this.createDate(maybeDateString) : undefined;
+  }
+
+  private async setLastInvocationDate(date: Date): Promise<void> {
+    return await this.globalState.update(
+      InvocationRateLimiter._invocationRateLimiterPrefix + this.funcIdentifier,
+      date,
+    );
+  }
+
+  private static readonly _invocationRateLimiterPrefix =
+    "invocationRateLimiter_lastInvocationDate_";
+}
+
+export enum InvocationRateLimiterResultKind {
+  Invoked,
+  RateLimited,
+}
+
+/**
+ * The function was invoked and returned the value `result`.
+ */
+interface InvokedResult<T> {
+  kind: InvocationRateLimiterResultKind.Invoked;
+  result: T;
+}
+
+/**
+ * The function was not invoked as the minimum interval since the last invocation had not elapsed.
+ */
+interface RateLimitedResult {
+  kind: InvocationRateLimiterResultKind.RateLimited;
+}
+
+type InvocationRateLimiterResult<T> = InvokedResult<T> | RateLimitedResult;
+
+function createInvokedResult<T>(result: T): InvokedResult<T> {
+  return {
+    kind: InvocationRateLimiterResultKind.Invoked,
+    result,
+  };
+}
+
+function createRateLimitedResult(): RateLimitedResult {
+  return {
+    kind: InvocationRateLimiterResultKind.RateLimited,
+  };
+}

extensions/ql-vscode/src/common/location-link-utils.ts

@@ -5,7 +5,7 @@ export function createRemoteFileRef(
   startLine?: number,
   endLine?: number,
 ): string {
-  if (startLine && endLine) {
+  if (startLine && endLine && startLine !== endLine) {
     return `${fileLink.fileLinkPrefix}/${fileLink.filePath}#L${startLine}-L${endLine}`;
   } else if (startLine) {
     return `${fileLink.fileLinkPrefix}/${fileLink.filePath}#L${startLine}`;

extensions/ql-vscode/src/common/logging/index.ts

@@ -1,4 +1,4 @@
 export * from "./logger";
+export * from "./notification-logger";
+export * from "./notifications";
 export * from "./tee-logger";
-export * from "./vscode/loggers";
-export * from "./vscode/output-channel-logger";

extensions/ql-vscode/src/common/logging/notification-logger.ts

@@ -0,0 +1,7 @@
+import { Logger } from "./logger";
+
+export interface NotificationLogger extends Logger {
+  showErrorMessage(message: string): Promise<void>;
+  showWarningMessage(message: string): Promise<void>;
+  showInformationMessage(message: string): Promise<void>;
+}

extensions/ql-vscode/src/common/logging/notifications.ts

@@ -0,0 +1,116 @@
+import { NotificationLogger } from "./notification-logger";
+import { AppTelemetry } from "../telemetry";
+import { RedactableError } from "../errors";
+
+export interface ShowAndLogOptions {
+  /**
+   * An alternate message that is added to the log, but not displayed in the popup.
+   * This is useful for adding extra detail to the logs that would be too noisy for the popup.
+   */
+  fullMessage?: string;
+}
+
+/**
+ * Show an error message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogErrorMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    dropLinesExceptInitial(message),
+    logger.showErrorMessage,
+    { fullMessage: message, ...options },
+  );
+}
+
+function dropLinesExceptInitial(message: string, n = 2) {
+  return message.toString().split(/\r?\n/).slice(0, n).join("\n");
+}
+
+/**
+ * Show a warning message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogWarningMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    message,
+    logger.showWarningMessage,
+    options,
+  );
+}
+
+/**
+ * Show an information message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogInformationMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    message,
+    logger.showInformationMessage,
+    options,
+  );
+}
+
+async function internalShowAndLog(
+  logger: NotificationLogger,
+  message: string,
+  fn: (message: string) => Promise<void>,
+  { fullMessage }: ShowAndLogOptions = {},
+): Promise<void> {
+  void logger.log(fullMessage || message);
+  await fn.bind(logger)(message);
+}
+
+interface ShowAndLogExceptionOptions extends ShowAndLogOptions {
+  /** Custom properties to include in the telemetry report. */
+  extraTelemetryProperties?: { [key: string]: string };
+}
+
+/**
+ * Show an error message, log it to the console, and emit redacted information as telemetry
+ *
+ * @param logger The logger that will receive the message.
+ * @param telemetry The telemetry instance to use for reporting.
+ * @param error The error to show. Only redacted information will be included in the telemetry.
+ * @param options See individual fields on `ShowAndLogExceptionOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogExceptionWithTelemetry(
+  logger: NotificationLogger,
+  telemetry: AppTelemetry | undefined,
+  error: RedactableError,
+  options: ShowAndLogExceptionOptions = {},
+): Promise<void> {
+  telemetry?.sendError(error, options.extraTelemetryProperties);
+  return showAndLogErrorMessage(logger, error.fullMessage, options);
+}

extensions/ql-vscode/src/common/logging/tee-logger.ts

@@ -1,6 +1,6 @@
 import { appendFile, ensureFile } from "fs-extra";
 import { isAbsolute } from "path";
-import { getErrorMessage } from "../../pure/helpers-pure";
+import { getErrorMessage } from "../helpers-pure";
 import { Logger, LogOptions } from "./logger";
 
 /**

extensions/ql-vscode/src/common/logging/vscode/index.ts

@@ -0,0 +1,2 @@
+export * from "./loggers";
+export * from "./output-channel-logger";

extensions/ql-vscode/src/common/logging/vscode/output-channel-logger.ts

@@ -1,11 +1,15 @@
 import { window as Window, OutputChannel, Progress } from "vscode";
 import { Logger, LogOptions } from "../logger";
-import { DisposableObject } from "../../../pure/disposable-object";
+import { DisposableObject } from "../../disposable-object";
+import { NotificationLogger } from "../notification-logger";
 
 /**
  * A logger that writes messages to an output channel in the VS Code Output tab.
  */
-export class OutputChannelLogger extends DisposableObject implements Logger {
+export class OutputChannelLogger
+  extends DisposableObject
+  implements Logger, NotificationLogger
+{
   public readonly outputChannel: OutputChannel;
   isCustomLogDirectory: boolean;
 
@@ -42,6 +46,30 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
   show(preserveFocus?: boolean): void {
     this.outputChannel.show(preserveFocus);
   }
+
+  async showErrorMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showErrorMessage);
+  }
+
+  async showInformationMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showInformationMessage);
+  }
+
+  async showWarningMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showWarningMessage);
+  }
+
+  private async showMessage(
+    message: string,
+    show: (message: string, ...items: string[]) => Thenable<string | undefined>,
+  ): Promise<void> {
+    const label = "Show Log";
+    const result = await show(message, label);
+
+    if (result === label) {
+      this.show();
+    }
+  }
 }
 
 export type ProgressReporter = Progress<{ message: string }>;

extensions/ql-vscode/src/common/path.ts

@@ -19,3 +19,11 @@ export const basename = (path: string): string => {
   const index = path.lastIndexOf("\\");
   return index === -1 ? path : path.slice(index + 1);
 };
+
+// Returns the extension of a path, including the leading dot.
+export const extname = (path: string): string => {
+  const name = basename(path);
+
+  const index = name.lastIndexOf(".");
+  return index === -1 ? "" : name.slice(index);
+};

extensions/ql-vscode/src/common/query-language.ts

@@ -25,13 +25,17 @@ export const PACKS_BY_QUERY_LANGUAGE = {
   [QueryLanguage.Ruby]: ["codeql/ruby-queries"],
 };
 
-export const dbSchemeToLanguage = {
-  "semmlecode.javascript.dbscheme": "javascript",
-  "semmlecode.cpp.dbscheme": "cpp",
-  "semmlecode.dbscheme": "java",
-  "semmlecode.python.dbscheme": "python",
-  "semmlecode.csharp.dbscheme": "csharp",
-  "go.dbscheme": "go",
-  "ruby.dbscheme": "ruby",
-  "swift.dbscheme": "swift",
+export const dbSchemeToLanguage: Record<string, QueryLanguage> = {
+  "semmlecode.javascript.dbscheme": QueryLanguage.Javascript,
+  "semmlecode.cpp.dbscheme": QueryLanguage.Cpp,
+  "semmlecode.dbscheme": QueryLanguage.Java,
+  "semmlecode.python.dbscheme": QueryLanguage.Python,
+  "semmlecode.csharp.dbscheme": QueryLanguage.CSharp,
+  "go.dbscheme": QueryLanguage.Go,
+  "ruby.dbscheme": QueryLanguage.Ruby,
+  "swift.dbscheme": QueryLanguage.Swift,
 };
+
+export function isQueryLanguage(language: string): language is QueryLanguage {
+  return Object.values(QueryLanguage).includes(language as QueryLanguage);
+}

extensions/ql-vscode/src/common/sarif-parser.ts

@@ -1,7 +1,7 @@
 import * as Sarif from "sarif";
 import { createReadStream } from "fs-extra";
 import { connectTo } from "stream-json/Assembler";
-import { getErrorMessage } from "./pure/helpers-pure";
+import { getErrorMessage } from "./helpers-pure";
 import { withParser } from "stream-json/filters/Pick";
 
 const DUMMY_TOOL: Sarif.Tool = { driver: { name: "" } };

extensions/ql-vscode/src/common/sarif-utils.ts

@@ -1,6 +1,6 @@
 import * as Sarif from "sarif";
 import type { HighlightedRegion } from "../variant-analysis/shared/analysis-result";
-import { ResolvableLocationValue } from "./bqrs-cli-types";
+import { ResolvableLocationValue } from "../common/bqrs-cli-types";
 
 export interface SarifLink {
   dest: number;

extensions/ql-vscode/src/common/telemetry.ts

@@ -0,0 +1,10 @@
+import { RedactableError } from "./errors";
+
+export interface AppTelemetry {
+  sendCommandUsage(name: string, executionTime: number, error?: Error): void;
+  sendUIInteraction(name: string): void;
+  sendError(
+    error: RedactableError,
+    extraProperties?: { [key: string]: string },
+  ): void;
+}

extensions/ql-vscode/src/common/vscode/abstract-webview.ts

@@ -9,13 +9,9 @@ import {
 } from "vscode";
 import { join } from "path";
 
-import { DisposableObject, DisposeHandler } from "./pure/disposable-object";
-import { tmpDir } from "./helpers";
-import {
-  getHtmlForWebview,
-  WebviewMessage,
-  WebviewView,
-} from "./interface-utils";
+import { DisposableObject, DisposeHandler } from "../disposable-object";
+import { tmpDir } from "../../tmp-dir";
+import { getHtmlForWebview, WebviewMessage, WebviewView } from "./webview-html";
 
 export type WebviewPanelConfig = {
   viewId: string;

extensions/ql-vscode/src/common/vscode/archive-filesystem-provider.ts

@@ -1,7 +1,7 @@
 import { pathExists } from "fs-extra";
 import * as unzipper from "unzipper";
 import * as vscode from "vscode";
-import { extLogger } from "./common";
+import { extLogger } from "../logging/vscode";
 
 // All path operations in this file must be on paths *within* the zip
 // archive.

extensions/ql-vscode/src/common/vscode/authentication.ts

@@ -1,9 +1,9 @@
 import * as vscode from "vscode";
 import * as Octokit from "@octokit/rest";
 import { retry } from "@octokit/plugin-retry";
-import { Credentials } from "./common/authentication";
+import { Credentials } from "../authentication";
 
-const GITHUB_AUTH_PROVIDER_ID = "github";
+export const GITHUB_AUTH_PROVIDER_ID = "github";
 
 // We need 'repo' scope for triggering workflows, 'gist' scope for exporting results to Gist,
 // and 'read:packages' for reading private CodeQL packages.

extensions/ql-vscode/src/common/vscode/commands.ts

@@ -1,18 +1,20 @@
 import { commands, Disposable } from "vscode";
 import { CommandFunction, CommandManager } from "../../packages/commands";
-import { extLogger, OutputChannelLogger } from "../logging";
+import {
+  NotificationLogger,
+  showAndLogWarningMessage,
+  showAndLogExceptionWithTelemetry,
+} from "../logging";
+import { extLogger } from "../logging/vscode";
 import {
   asError,
   getErrorMessage,
   getErrorStack,
-} from "../../pure/helpers-pure";
-import { redactableError } from "../../pure/errors";
-import { UserCancellationException } from "../../progress";
-import {
-  showAndLogExceptionWithTelemetry,
-  showAndLogWarningMessage,
-} from "../../helpers";
-import { telemetryListener } from "../../telemetry";
+} from "../../common/helpers-pure";
+import { redactableError } from "../../common/errors";
+import { UserCancellationException } from "./progress";
+import { telemetryListener } from "./telemetry";
+import { AppTelemetry } from "../telemetry";
 
 /**
  * Create a command manager for VSCode, wrapping registerCommandWithErrorHandling
@@ -20,9 +22,12 @@ import { telemetryListener } from "../../telemetry";
  */
 export function createVSCodeCommandManager<
   Commands extends Record<string, CommandFunction>,
->(outputLogger?: OutputChannelLogger): CommandManager<Commands> {
+>(
+  logger?: NotificationLogger,
+  telemetry?: AppTelemetry,
+): CommandManager<Commands> {
   return new CommandManager((commandId, task) => {
-    return registerCommandWithErrorHandling(commandId, task, outputLogger);
+    return registerCommandWithErrorHandling(commandId, task, logger, telemetry);
   }, wrapExecuteCommand);
 }
 
@@ -32,11 +37,14 @@ export function createVSCodeCommandManager<
  * @param commandId The ID of the command to register.
  * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
  * arguments to the command handler are passed on to the task.
+ * @param logger The logger to use for error reporting.
+ * @param telemetry The telemetry listener to use for error reporting.
  */
 export function registerCommandWithErrorHandling(
   commandId: string,
   task: (...args: any[]) => Promise<any>,
-  outputLogger = extLogger,
+  logger: NotificationLogger = extLogger,
+  telemetry: AppTelemetry | undefined = telemetryListener,
 ): Disposable {
   return commands.registerCommand(commandId, async (...args: any[]) => {
     const startTime = Date.now();
@@ -49,23 +57,20 @@ export function registerCommandWithErrorHandling(
       const errorMessage = redactableError(error)`${
         getErrorMessage(e) || e
       } (${commandId})`;
-      const errorStack = getErrorStack(e);
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
         if (e.silent) {
-          void outputLogger.log(errorMessage.fullMessage);
+          void logger.log(errorMessage.fullMessage);
         } else {
-          void showAndLogWarningMessage(errorMessage.fullMessage, {
-            outputLogger,
-          });
+          void showAndLogWarningMessage(logger, errorMessage.fullMessage);
         }
       } else {
         // Include the full stack in the error log only.
+        const errorStack = getErrorStack(e);
         const fullMessage = errorStack
           ? `${errorMessage.fullMessage}\n${errorStack}`
           : errorMessage.fullMessage;
-        void showAndLogExceptionWithTelemetry(errorMessage, {
-          outputLogger,
+        void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
           fullMessage,
           extraTelemetryProperties: {
             command: commandId,

extensions/ql-vscode/src/common/vscode/dialog.ts

@@ -0,0 +1,135 @@
+import { env, Uri, window } from "vscode";
+
+/**
+ * Opens a modal dialog for the user to make a yes/no choice.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ * @param yesTitle The text in the box indicating the affirmative choice.
+ * @param noTitle The text in the box indicating the negative choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showBinaryChoiceDialog(
+  message: string,
+  modal = true,
+  yesTitle = "Yes",
+  noTitle = "No",
+): Promise<boolean | undefined> {
+  const yesItem = { title: yesTitle, isCloseAffordance: false };
+  const noItem = { title: noTitle, isCloseAffordance: true };
+  const chosenItem = await window.showInformationMessage(
+    message,
+    { modal },
+    yesItem,
+    noItem,
+  );
+  if (!chosenItem) {
+    return undefined;
+  }
+  return chosenItem?.title === yesItem.title;
+}
+
+/**
+ * Opens a modal dialog for the user to make a yes/no choice.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showBinaryChoiceWithUrlDialog(
+  message: string,
+  url: string,
+): Promise<boolean | undefined> {
+  const urlItem = { title: "More Information", isCloseAffordance: false };
+  const yesItem = { title: "Yes", isCloseAffordance: false };
+  const noItem = { title: "No", isCloseAffordance: true };
+  let chosenItem;
+
+  // Keep the dialog open as long as the user is clicking the 'more information' option.
+  // To prevent an infinite loop, if the user clicks 'more information' 5 times, close the dialog and return cancelled
+  let count = 0;
+  do {
+    chosenItem = await window.showInformationMessage(
+      message,
+      { modal: true },
+      urlItem,
+      yesItem,
+      noItem,
+    );
+    if (chosenItem === urlItem) {
+      await env.openExternal(Uri.parse(url, true));
+    }
+    count++;
+  } while (chosenItem === urlItem && count < 5);
+
+  if (!chosenItem || chosenItem.title === urlItem.title) {
+    return undefined;
+  }
+  return chosenItem.title === yesItem.title;
+}
+
+/**
+ * Show an information message with a customisable action.
+ * @param message The message to show.
+ * @param actionMessage The call to action message.
+ *
+ * @return `true` if the user clicks the action, `false` if the user cancels the dialog.
+ */
+export async function showInformationMessageWithAction(
+  message: string,
+  actionMessage: string,
+): Promise<boolean> {
+  const actionItem = { title: actionMessage, isCloseAffordance: false };
+  const chosenItem = await window.showInformationMessage(message, actionItem);
+  return chosenItem === actionItem;
+}
+
+/**
+ * Opens a modal dialog for the user to make a choice between yes/no/never be asked again.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ * @param yesTitle The text in the box indicating the affirmative choice.
+ * @param noTitle The text in the box indicating the negative choice.
+ * @param neverTitle The text in the box indicating the opt out choice.
+ *
+ * @return
+ *  `Yes` if the user clicks 'Yes',
+ *  `No` if the user clicks 'No' or cancels the dialog,
+ *  `No, and never ask me again` if the user clicks 'No, and never ask me again',
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showNeverAskAgainDialog(
+  message: string,
+  modal = true,
+  yesTitle = "Yes",
+  noTitle = "No",
+  neverAskAgainTitle = "No, and never ask me again",
+): Promise<string | undefined> {
+  const yesItem = { title: yesTitle, isCloseAffordance: true };
+  const noItem = { title: noTitle, isCloseAffordance: false };
+  const neverAskAgainItem = {
+    title: neverAskAgainTitle,
+    isCloseAffordance: false,
+  };
+  const chosenItem = await window.showInformationMessage(
+    message,
+    { modal },
+    yesItem,
+    noItem,
+    neverAskAgainItem,
+  );
+
+  return chosenItem?.title;
+}

extensions/ql-vscode/src/common/vscode/environment-context.ts

@@ -0,0 +1,8 @@
+import { env } from "vscode";
+import { EnvironmentContext } from "../app";
+
+export class AppEnvironmentContext implements EnvironmentContext {
+  public get language(): string {
+    return env.language;
+  }
+}

extensions/ql-vscode/src/common/vscode/external-files.ts

@@ -1,11 +1,15 @@
 import { Uri, window } from "vscode";
-import { AppCommandManager } from "../common/commands";
+import { AppCommandManager } from "../commands";
+import { showBinaryChoiceDialog } from "./dialog";
+import { redactableError } from "../../common/errors";
 import {
-  showAndLogExceptionWithTelemetry,
-  showBinaryChoiceDialog,
-} from "../helpers";
-import { redactableError } from "../pure/errors";
-import { asError, getErrorMessage, getErrorStack } from "../pure/helpers-pure";
+  asError,
+  getErrorMessage,
+  getErrorStack,
+} from "../../common/helpers-pure";
+import { showAndLogExceptionWithTelemetry } from "../logging";
+import { extLogger } from "../logging/vscode";
+import { telemetryListener } from "./telemetry";
 
 export async function tryOpenExternalFile(
   commandManager: AppCommandManager,
@@ -32,6 +36,8 @@ the file in the file explorer and dragging it into the workspace.`,
           await commandManager.execute("revealFileInOS", uri);
         } catch (e) {
           void showAndLogExceptionWithTelemetry(
+            extLogger,
+            telemetryListener,
             redactableError(
               asError(e),
             )`Failed to reveal file in OS: ${getErrorMessage(e)}`,
@@ -40,6 +46,8 @@ the file in the file explorer and dragging it into the workspace.`,
       }
     } else {
       void showAndLogExceptionWithTelemetry(
+        extLogger,
+        telemetryListener,
         redactableError(asError(e))`Could not open file ${fileLocation}`,
         {
           fullMessage: `${getErrorMessage(e)}\n${getErrorStack(e)}`,

extensions/ql-vscode/src/common/vscode/file-path-discovery.ts

@@ -0,0 +1,258 @@
+import { Discovery } from "../discovery";
+import {
+  Event,
+  EventEmitter,
+  RelativePattern,
+  Uri,
+  WorkspaceFoldersChangeEvent,
+  workspace,
+} from "vscode";
+import { MultiFileSystemWatcher } from "./multi-file-system-watcher";
+import { AppEventEmitter } from "../events";
+import { extLogger } from "../logging/vscode";
+import { lstat } from "fs-extra";
+import { containsPath, isIOError } from "../files";
+import {
+  getOnDiskWorkspaceFolders,
+  getOnDiskWorkspaceFoldersObjects,
+} from "./workspace-folders";
+import { getErrorMessage } from "../../common/helpers-pure";
+
+interface PathData {
+  path: string;
+}
+
+/**
+ * Discovers and watches for changes to all files matching a given filter
+ * contained in the workspace. Also allows computing extra data about each
+ * file path, and only recomputing the data when the file changes.
+ *
+ * Scans the whole workspace on startup, and then watches for changes to files
+ * to do the minimum work to keep up with changes.
+ *
+ * Can configure which changes it watches for, which files are considered
+ * relevant, and what extra data to compute for each file.
+ */
+export abstract class FilePathDiscovery<T extends PathData> extends Discovery {
+  /** The set of known paths and associated data that we are tracking */
+  private pathData: T[] = [];
+
+  /** Event that fires whenever the contents of `pathData` changes */
+  private readonly onDidChangePathDataEmitter: AppEventEmitter<void>;
+
+  /**
+   * The set of file paths that may have changed on disk since the last time
+   * refresh was run. Whenever a watcher reports some change to a file we add
+   * it to this set, and then during the next refresh we will process all
+   * file paths from this set and update our internal state to match whatever
+   * we find on disk (i.e. the file exists, doesn't exist, computed data has
+   * changed).
+   */
+  private readonly changedFilePaths = new Set<string>();
+
+  /**
+   * Watches for changes to files and directories in all workspace folders.
+   */
+  private readonly watcher: MultiFileSystemWatcher = this.push(
+    new MultiFileSystemWatcher(),
+  );
+
+  /**
+   * @param name Name of the discovery operation, for logging purposes.
+   * @param fileWatchPattern Passed to `vscode.RelativePattern` to determine the files to watch for changes to.
+   */
+  constructor(name: string, private readonly fileWatchPattern: string) {
+    super(name, extLogger);
+
+    this.onDidChangePathDataEmitter = this.push(new EventEmitter<void>());
+    this.push(
+      workspace.onDidChangeWorkspaceFolders(
+        this.workspaceFoldersChanged.bind(this),
+      ),
+    );
+    this.push(this.watcher.onDidChange(this.fileChanged.bind(this)));
+  }
+
+  protected getPathData(): ReadonlyArray<Readonly<T>> {
+    return this.pathData;
+  }
+
+  protected get onDidChangePathData(): Event<void> {
+    return this.onDidChangePathDataEmitter.event;
+  }
+
+  /**
+   * Compute any extra data to be stored regarding the given path.
+   */
+  protected abstract getDataForPath(path: string): Promise<T>;
+
+  /**
+   * Is the given path relevant to this discovery operation?
+   */
+  protected abstract pathIsRelevant(path: string): boolean;
+
+  /**
+   * Should the given new data overwrite the existing data we have stored?
+   */
+  protected abstract shouldOverwriteExistingData(
+    newData: T,
+    existingData: T,
+  ): boolean;
+
+  /**
+   * Update the data for every path by calling `getDataForPath`.
+   */
+  protected async recomputeAllData() {
+    this.pathData = await Promise.all(
+      this.pathData.map((p) => this.getDataForPath(p.path)),
+    );
+    this.onDidChangePathDataEmitter.fire();
+  }
+
+  /**
+   * Do the initial scan of the entire workspace and set up watchers for future changes.
+   */
+  public async initialRefresh() {
+    getOnDiskWorkspaceFolders().forEach((workspaceFolder) => {
+      this.changedFilePaths.add(workspaceFolder);
+    });
+
+    this.updateWatchers();
+    return this.refresh();
+  }
+
+  private workspaceFoldersChanged(event: WorkspaceFoldersChangeEvent) {
+    event.added.forEach((workspaceFolder) => {
+      this.changedFilePaths.add(workspaceFolder.uri.fsPath);
+    });
+    event.removed.forEach((workspaceFolder) => {
+      this.changedFilePaths.add(workspaceFolder.uri.fsPath);
+    });
+
+    this.updateWatchers();
+    void this.refresh();
+  }
+
+  private updateWatchers() {
+    this.watcher.clear();
+    for (const workspaceFolder of getOnDiskWorkspaceFoldersObjects()) {
+      // Watch for changes to individual files
+      this.watcher.addWatch(
+        new RelativePattern(workspaceFolder, this.fileWatchPattern),
+      );
+      // need to explicitly watch for changes to directories themselves.
+      this.watcher.addWatch(new RelativePattern(workspaceFolder, "**/"));
+    }
+  }
+
+  private fileChanged(uri: Uri) {
+    this.changedFilePaths.add(uri.fsPath);
+    void this.refresh();
+  }
+
+  protected async discover() {
+    let pathsUpdated = false;
+    for (const path of this.changedFilePaths) {
+      try {
+        this.changedFilePaths.delete(path);
+        if (await this.handleChangedPath(path)) {
+          pathsUpdated = true;
+        }
+      } catch (e) {
+        // If we get an error while processing a path, just log it and continue.
+        // There aren't any network operations happening here or anything else
+        // that's likely to succeed on a retry, so don't bother adding it back
+        // to the changedFilePaths set.
+        void extLogger.log(
+          `${
+            this.name
+          } failed while processing path "${path}": ${getErrorMessage(e)}`,
+        );
+      }
+    }
+
+    if (pathsUpdated) {
+      this.onDidChangePathDataEmitter.fire();
+    }
+  }
+
+  private async handleChangedPath(path: string): Promise<boolean> {
+    try {
+      // If the path is not in the workspace then we don't want to be
+      // tracking or displaying it, so treat it as if it doesn't exist.
+      if (!this.pathIsInWorkspace(path)) {
+        return this.handleRemovedPath(path);
+      }
+
+      if ((await lstat(path)).isDirectory()) {
+        return await this.handleChangedDirectory(path);
+      } else {
+        return this.handleChangedFile(path);
+      }
+    } catch (e) {
+      if (isIOError(e) && e.code === "ENOENT") {
+        return this.handleRemovedPath(path);
+      }
+      throw e;
+    }
+  }
+
+  private pathIsInWorkspace(path: string): boolean {
+    return getOnDiskWorkspaceFolders().some((workspaceFolder) =>
+      containsPath(workspaceFolder, path),
+    );
+  }
+
+  private handleRemovedPath(path: string): boolean {
+    const oldLength = this.pathData.length;
+    this.pathData = this.pathData.filter(
+      (existingPathData) => !containsPath(path, existingPathData.path),
+    );
+    return this.pathData.length !== oldLength;
+  }
+
+  private async handleChangedDirectory(path: string): Promise<boolean> {
+    const newPaths = await workspace.findFiles(
+      new RelativePattern(path, this.fileWatchPattern),
+    );
+
+    let pathsUpdated = false;
+    for (const path of newPaths) {
+      if (await this.addOrUpdatePath(path.fsPath)) {
+        pathsUpdated = true;
+      }
+    }
+    return pathsUpdated;
+  }
+
+  private async handleChangedFile(path: string): Promise<boolean> {
+    if (this.pathIsRelevant(path)) {
+      return await this.addOrUpdatePath(path);
+    } else {
+      return false;
+    }
+  }
+
+  private async addOrUpdatePath(path: string): Promise<boolean> {
+    const data = await this.getDataForPath(path);
+    const existingPathDataIndex = this.pathData.findIndex(
+      (existingPathData) => existingPathData.path === path,
+    );
+    if (existingPathDataIndex !== -1) {
+      if (
+        this.shouldOverwriteExistingData(
+          data,
+          this.pathData[existingPathDataIndex],
+        )
+      ) {
+        this.pathData.splice(existingPathDataIndex, 1, data);
+        return true;
+      } else {
+        return false;
+      }
+    } else {
+      this.pathData.push(data);
+      return true;
+    }
+  }
+}

extensions/ql-vscode/src/common/vscode/multi-file-system-watcher.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from "../pure/disposable-object";
+import { DisposableObject } from "../disposable-object";
 import { EventEmitter, Event, Uri, GlobPattern, workspace } from "vscode";
 
 /**

extensions/ql-vscode/src/common/vscode/selection-commands.ts

@@ -0,0 +1,55 @@
+import {
+  ExplorerSelectionCommandFunction,
+  TreeViewContextMultiSelectionCommandFunction,
+  TreeViewContextSingleSelectionCommandFunction,
+} from "../commands";
+import { showAndLogErrorMessage, NotificationLogger } from "../logging";
+
+// A hack to match types that are not an array, which is useful to help avoid
+// misusing createSingleSelectionCommand, e.g. where T accidentally gets instantiated
+// as DatabaseItem[] instead of DatabaseItem.
+type NotArray = object & { length?: never };
+
+// A way to get the type system to help assert that one type is a supertype of another.
+type CreateSupertypeOf<Super, Sub extends Super> = Sub;
+
+// This asserts that SelectionCommand is assignable to all of the different types of
+// SelectionCommand defined in commands.ts. The intention is the output from the helpers
+// in this file can be used with any of the select command types and can handle any of
+// the inputs.
+type SelectionCommand<T extends NotArray> = CreateSupertypeOf<
+  TreeViewContextMultiSelectionCommandFunction<T> &
+    TreeViewContextSingleSelectionCommandFunction<T> &
+    ExplorerSelectionCommandFunction<T>,
+  (singleItem: T, multiSelect?: T[] | undefined) => Promise<void>
+>;
+
+export function createSingleSelectionCommand<T extends NotArray>(
+  logger: NotificationLogger,
+  f: (argument: T) => Promise<void>,
+  itemName: string,
+): SelectionCommand<T> {
+  return async (singleItem, multiSelect) => {
+    if (multiSelect === undefined || multiSelect.length === 1) {
+      return f(singleItem);
+    } else {
+      void showAndLogErrorMessage(
+        logger,
+        `Please select a single ${itemName}.`,
+      );
+      return;
+    }
+  };
+}
+
+export function createMultiSelectionCommand<T extends NotArray>(
+  f: (argument: T[]) => Promise<void>,
+): SelectionCommand<T> {
+  return async (singleItem, multiSelect) => {
+    if (multiSelect !== undefined && multiSelect.length > 0) {
+      return f(multiSelect);
+    } else {
+      return f([singleItem]);
+    }
+  };
+}

extensions/ql-vscode/src/common/vscode/telemetry.ts

@@ -13,13 +13,14 @@ import {
   LOG_TELEMETRY,
   isIntegrationTestMode,
   isCanary,
-} from "./config";
+} from "../../config";
 import * as appInsights from "applicationinsights";
-import { extLogger } from "./common";
+import { extLogger } from "../logging/vscode";
 import { UserCancellationException } from "./progress";
-import { showBinaryChoiceWithUrlDialog } from "./helpers";
-import { RedactableError } from "./pure/errors";
+import { showBinaryChoiceWithUrlDialog } from "./dialog";
+import { RedactableError } from "../errors";
 import { SemVer } from "semver";
+import { AppTelemetry } from "../telemetry";
 
 // Key is injected at build time through the APP_INSIGHTS_KEY environment variable.
 const key = "REPLACE-APP-INSIGHTS-KEY";
@@ -54,7 +55,10 @@ const baseDataPropertiesToRemove = [
 
 const NOT_SET_CLI_VERSION = "not-set";
 
-export class TelemetryListener extends ConfigListener {
+export class ExtensionTelemetryListener
+  extends ConfigListener
+  implements AppTelemetry
+{
   static relevantSettings = [ENABLE_TELEMETRY, CANARY_FEATURES];
 
   private reporter?: TelemetryReporter;
@@ -152,7 +156,7 @@ export class TelemetryListener extends ConfigListener {
     void this.reporter?.dispose();
   }
 
-  sendCommandUsage(name: string, executionTime: number, error?: Error) {
+  sendCommandUsage(name: string, executionTime: number, error?: Error): void {
     if (!this.reporter) {
       return;
     }
@@ -174,7 +178,7 @@ export class TelemetryListener extends ConfigListener {
     );
   }
 
-  sendUIInteraction(name: string) {
+  sendUIInteraction(name: string): void {
     if (!this.reporter) {
       return;
     }
@@ -193,7 +197,7 @@ export class TelemetryListener extends ConfigListener {
   sendError(
     error: RedactableError,
     extraProperties?: { [key: string]: string },
-  ) {
+  ): void {
     if (!this.reporter) {
       return;
     }
@@ -272,16 +276,16 @@ export class TelemetryListener extends ConfigListener {
 /**
  * The global Telemetry instance
  */
-export let telemetryListener: TelemetryListener | undefined;
+export let telemetryListener: ExtensionTelemetryListener | undefined;
 
 export async function initializeTelemetry(
   extension: Extension<any>,
   ctx: ExtensionContext,
-): Promise<TelemetryListener> {
+): Promise<ExtensionTelemetryListener> {
   if (telemetryListener !== undefined) {
     throw new Error("Telemetry is already initialized");
   }
-  telemetryListener = new TelemetryListener(
+  telemetryListener = new ExtensionTelemetryListener(
     extension.id,
     extension.packageJSON.version,
     key,

extensions/ql-vscode/src/common/vscode/vscode-app.ts

@@ -1,13 +1,17 @@
 import * as vscode from "vscode";
-import { VSCodeCredentials } from "../../authentication";
-import { Disposable } from "../../pure/disposable-object";
-import { App, AppMode } from "../app";
+import { VSCodeCredentials } from "./authentication";
+import { Disposable } from "../disposable-object";
+import { App, AppMode, EnvironmentContext } from "../app";
 import { AppEventEmitter } from "../events";
-import { extLogger, Logger, queryServerLogger } from "../logging";
+import { NotificationLogger } from "../logging";
+import { extLogger, queryServerLogger } from "../logging/vscode";
 import { Memento } from "../memento";
 import { VSCodeAppEventEmitter } from "./events";
 import { AppCommandManager, QueryServerCommandManager } from "../commands";
 import { createVSCodeCommandManager } from "./commands";
+import { AppEnvironmentContext } from "./environment-context";
+import { AppTelemetry } from "../telemetry";
+import { telemetryListener } from "./telemetry";
 
 export class ExtensionApp implements App {
   public readonly credentials: VSCodeCredentials;
@@ -54,11 +58,19 @@ export class ExtensionApp implements App {
     }
   }
 
-  public get logger(): Logger {
+  public get logger(): NotificationLogger {
     return extLogger;
   }
 
+  public get telemetry(): AppTelemetry | undefined {
+    return telemetryListener;
+  }
+
   public createEventEmitter<T>(): AppEventEmitter<T> {
     return new VSCodeAppEventEmitter<T>();
   }
+
+  public get environment(): EnvironmentContext {
+    return new AppEnvironmentContext();
+  }
 }

extensions/ql-vscode/src/common/vscode/webview-html.ts

@@ -0,0 +1,101 @@
+import { ExtensionContext, Uri, Webview } from "vscode";
+import { randomBytes } from "crypto";
+import { EOL } from "os";
+
+export type WebviewView =
+  | "results"
+  | "compare"
+  | "variant-analysis"
+  | "data-flow-paths"
+  | "data-extensions-editor";
+
+export interface WebviewMessage {
+  t: string;
+}
+
+/**
+ * Returns HTML to populate the given webview.
+ * Uses a content security policy that only loads the given script.
+ */
+export function getHtmlForWebview(
+  ctx: ExtensionContext,
+  webview: Webview,
+  view: WebviewView,
+  {
+    allowInlineStyles,
+    allowWasmEval,
+  }: {
+    allowInlineStyles?: boolean;
+    allowWasmEval?: boolean;
+  } = {
+    allowInlineStyles: false,
+    allowWasmEval: false,
+  },
+): string {
+  const scriptUriOnDisk = Uri.file(ctx.asAbsolutePath("out/webview.js"));
+
+  const stylesheetUrisOnDisk = [
+    Uri.file(ctx.asAbsolutePath("out/webview.css")),
+  ];
+
+  // Convert the on-disk URIs into webview URIs.
+  const scriptWebviewUri = webview.asWebviewUri(scriptUriOnDisk);
+  const stylesheetWebviewUris = stylesheetUrisOnDisk.map(
+    (stylesheetUriOnDisk) => webview.asWebviewUri(stylesheetUriOnDisk),
+  );
+
+  // Use a nonce in the content security policy to uniquely identify the above resources.
+  const nonce = getNonce();
+
+  const stylesheetsHtmlLines = allowInlineStyles
+    ? stylesheetWebviewUris.map((uri) => createStylesLinkWithoutNonce(uri))
+    : stylesheetWebviewUris.map((uri) => createStylesLinkWithNonce(nonce, uri));
+
+  const styleSrc = allowInlineStyles
+    ? `${webview.cspSource} vscode-file: 'unsafe-inline'`
+    : `'nonce-${nonce}'`;
+
+  const fontSrc = webview.cspSource;
+
+  /*
+   * Content security policy:
+   * default-src: allow nothing by default.
+   * script-src:
+   *   - allow the given script, using the nonce.
+   *   - 'wasm-unsafe-eval: allow loading WebAssembly modules if necessary.
+   * style-src: allow only the given stylesheet, using the nonce.
+   * connect-src: only allow fetch calls to webview resource URIs
+   * (this is used to load BQRS result files).
+   */
+  return `
+<html>
+  <head>
+    <meta http-equiv="Content-Security-Policy"
+          content="default-src 'none'; script-src 'nonce-${nonce}'${
+    allowWasmEval ? " 'wasm-unsafe-eval'" : ""
+  }; font-src ${fontSrc}; style-src ${styleSrc}; connect-src ${
+    webview.cspSource
+  };">
+        ${stylesheetsHtmlLines.join(`    ${EOL}`)}
+  </head>
+  <body>
+    <div id=root data-view="${view}">
+    </div>
+      <script nonce="${nonce}" src="${scriptWebviewUri}">
+    </script>
+  </body>
+</html>`;
+}
+
+/** Gets a nonce string created with 128 bits of entropy. */
+function getNonce(): string {
+  return randomBytes(16).toString("base64");
+}
+
+function createStylesLinkWithNonce(nonce: string, uri: Uri): string {
+  return `<link nonce="${nonce}" rel="stylesheet" href="${uri}">`;
+}
+
+function createStylesLinkWithoutNonce(uri: Uri): string {
+  return `<link rel="stylesheet" href="${uri}">`;
+}

extensions/ql-vscode/src/common/vscode/workspace-folders.ts

@@ -0,0 +1,64 @@
+import { dirname, join } from "path";
+import { workspace, WorkspaceFolder } from "vscode";
+
+/** Returns true if the specified workspace folder is on the file system. */
+export function isWorkspaceFolderOnDisk(
+  workspaceFolder: WorkspaceFolder,
+): boolean {
+  return workspaceFolder.uri.scheme === "file";
+}
+
+/** Gets all active workspace folders that are on the filesystem. */
+export function getOnDiskWorkspaceFoldersObjects() {
+  const workspaceFolders = workspace.workspaceFolders ?? [];
+  return workspaceFolders.filter(isWorkspaceFolderOnDisk);
+}
+
+/** Gets all active workspace folders that are on the filesystem. */
+export function getOnDiskWorkspaceFolders() {
+  return getOnDiskWorkspaceFoldersObjects().map((folder) => folder.uri.fsPath);
+}
+
+/** Check if folder is already present in workspace */
+export function isFolderAlreadyInWorkspace(folderName: string) {
+  const workspaceFolders = workspace.workspaceFolders || [];
+
+  return !!workspaceFolders.find(
+    (workspaceFolder) => workspaceFolder.name === folderName,
+  );
+}
+
+/**
+ * Returns the path of the first folder in the workspace.
+ * This is used to decide where to create skeleton QL packs.
+ *
+ * If the first folder is a QL pack, then the parent folder is returned.
+ * This is because the vscode-codeql-starter repo contains a ql pack in
+ * the first folder.
+ *
+ * This is a temporary workaround until we can retire the
+ * vscode-codeql-starter repo.
+ */
+export function getFirstWorkspaceFolder() {
+  const workspaceFolders = getOnDiskWorkspaceFolders();
+
+  if (!workspaceFolders || workspaceFolders.length === 0) {
+    throw new Error("No workspace folders found");
+  }
+
+  const firstFolderFsPath = workspaceFolders[0];
+
+  // For the vscode-codeql-starter repo, the first folder will be a ql pack
+  // so we need to get the parent folder
+  if (
+    firstFolderFsPath.includes(
+      join("vscode-codeql-starter", "codeql-custom-queries"),
+    )
+  ) {
+    // return the parent folder
+    return dirname(firstFolderFsPath);
+  } else {
+    // if the first folder is not a ql pack, then we are in a normal workspace
+    return firstFolderFsPath;
+  }
+}

extensions/ql-vscode/src/common/word.ts

@@ -7,8 +7,9 @@ export function pluralize(
   numItems: number | undefined,
   singular: string,
   plural: string,
+  numberFormatter: (value: number) => string = (value) => value.toString(),
 ): string {
   return numItems !== undefined
-    ? `${numItems} ${numItems === 1 ? singular : plural}`
+    ? `${numberFormatter(numItems)} ${numItems === 1 ? singular : plural}`
     : "";
 }

extensions/ql-vscode/src/compare/compare-view.ts

@@ -4,24 +4,27 @@ import {
   FromCompareViewMessage,
   ToCompareViewMessage,
   QueryCompareResult,
-} from "../pure/interface-types";
-import { Logger } from "../common";
-import { CodeQLCliServer } from "../cli";
-import { DatabaseManager } from "../local-databases";
-import { jumpToLocation } from "../interface-utils";
+} from "../common/interface-types";
+import { Logger, showAndLogExceptionWithTelemetry } from "../common/logging";
+import { extLogger } from "../common/logging/vscode";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { DatabaseManager } from "../databases/local-databases";
+import { jumpToLocation } from "../databases/local-databases/locations";
 import {
   transformBqrsResultSet,
   RawResultSet,
   BQRSInfo,
-} from "../pure/bqrs-cli-types";
+} from "../common/bqrs-cli-types";
 import resultsDiff from "./resultsDiff";
 import { CompletedLocalQueryInfo } from "../query-results";
-import { assertNever, getErrorMessage } from "../pure/helpers-pure";
+import { assertNever, getErrorMessage } from "../common/helpers-pure";
 import { HistoryItemLabelProvider } from "../query-history/history-item-label-provider";
-import { AbstractWebview, WebviewPanelConfig } from "../abstract-webview";
-import { telemetryListener } from "../telemetry";
-import { redactableError } from "../pure/errors";
-import { showAndLogExceptionWithTelemetry } from "../helpers";
+import {
+  AbstractWebview,
+  WebviewPanelConfig,
+} from "../common/vscode/abstract-webview";
+import { telemetryListener } from "../common/vscode/telemetry";
+import { redactableError } from "../common/errors";
 
 interface ComparePair {
   from: CompletedLocalQueryInfo;
@@ -127,7 +130,12 @@ export class CompareView extends AbstractWebview<
         break;
 
       case "viewSourceFile":
-        await jumpToLocation(msg, this.databaseManager, this.logger);
+        await jumpToLocation(
+          msg.databaseUri,
+          msg.loc,
+          this.databaseManager,
+          this.logger,
+        );
         break;
 
       case "openQuery":
@@ -143,6 +151,8 @@ export class CompareView extends AbstractWebview<
 
       case "unhandledError":
         void showAndLogExceptionWithTelemetry(
+          extLogger,
+          telemetryListener,
           redactableError(
             msg.error,
           )`Unhandled error in result comparison view: ${msg.error.message}`,
@@ -172,21 +182,40 @@ export class CompareView extends AbstractWebview<
     const commonResultSetNames = fromSchemaNames.filter((name) =>
       toSchemaNames.includes(name),
     );
+
+    // Fall back on the default result set names if there are no common ones.
+    const defaultFromResultSetName = fromSchemaNames.find((name) =>
+      name.startsWith("#"),
+    );
+    const defaultToResultSetName = toSchemaNames.find((name) =>
+      name.startsWith("#"),
+    );
+
+    if (
+      commonResultSetNames.length === 0 &&
+      !(defaultFromResultSetName || defaultToResultSetName)
+    ) {
+      throw new Error(
+        "No common result sets found between the two queries. Please check that the queries are compatible.",
+      );
+    }
+
     const currentResultSetName =
       selectedResultSetName || commonResultSetNames[0];
     const fromResultSet = await this.getResultSet(
       fromSchemas,
-      currentResultSetName,
+      currentResultSetName || defaultFromResultSetName!,
       from.completedQuery.query.resultsPaths.resultsPath,
     );
     const toResultSet = await this.getResultSet(
       toSchemas,
-      currentResultSetName,
+      currentResultSetName || defaultToResultSetName!,
       to.completedQuery.query.resultsPaths.resultsPath,
     );
     return [
       commonResultSetNames,
-      currentResultSetName,
+      currentResultSetName ||
+        `${defaultFromResultSetName} <-> ${defaultToResultSetName}`,
       fromResultSet,
       toResultSet,
     ];

extensions/ql-vscode/src/compare/resultsDiff.ts

@@ -1,5 +1,5 @@
-import { RawResultSet } from "../pure/bqrs-cli-types";
-import { QueryCompareResult } from "../pure/interface-types";
+import { RawResultSet } from "../common/bqrs-cli-types";
+import { QueryCompareResult } from "../common/interface-types";
 
 /**
  * Compare the rows of two queries. Use deep equality to determine if

extensions/ql-vscode/src/config.ts

@@ -1,14 +1,20 @@
-import { DisposableObject } from "./pure/disposable-object";
+import { DisposableObject } from "./common/disposable-object";
 import {
   workspace,
   Event,
   EventEmitter,
   ConfigurationChangeEvent,
   ConfigurationTarget,
+  ConfigurationScope,
 } from "vscode";
-import { DistributionManager } from "./distribution";
-import { extLogger } from "./common";
-import { ONE_DAY_IN_MS } from "./pure/time";
+import { DistributionManager } from "./codeql-cli/distribution";
+import { extLogger } from "./common/logging/vscode";
+import { ONE_DAY_IN_MS } from "./common/time";
+import {
+  FilterKey,
+  SortKey,
+  defaultFilterSortState,
+} from "./variant-analysis/shared/variant-analysis-filter-sort";
 
 export const ALL_SETTINGS: Setting[] = [];
 
@@ -39,12 +45,12 @@ export class Setting {
     }
   }
 
-  getValue<T>(): T {
+  getValue<T>(scope?: ConfigurationScope | null): T {
     if (this.parent === undefined) {
       throw new Error("Cannot get the value of a root setting.");
     }
     return workspace
-      .getConfiguration(this.parent.qualifiedName)
+      .getConfiguration(this.parent.qualifiedName, scope)
       .get<T>(this.name)!;
   }
 
@@ -69,6 +75,10 @@ const ROOT_SETTING = new Setting("codeQL");
 // Global configuration
 const TELEMETRY_SETTING = new Setting("telemetry", ROOT_SETTING);
 const AST_VIEWER_SETTING = new Setting("astViewer", ROOT_SETTING);
+const CONTEXTUAL_QUERIES_SETTINGS = new Setting(
+  "contextualQueries",
+  ROOT_SETTING,
+);
 const GLOBAL_TELEMETRY_SETTING = new Setting("telemetry");
 const LOG_INSIGHTS_SETTING = new Setting("logInsights", ROOT_SETTING);
 
@@ -479,13 +489,21 @@ export function joinOrderWarningThreshold(): number {
 }
 
 /**
- * Avoids caching in the AST viewer if the user is also a canary user.
+ * Hidden setting: Avoids caching in the AST viewer if the user is also a canary user.
  */
 export const NO_CACHE_AST_VIEWER = new Setting(
   "disableCache",
   AST_VIEWER_SETTING,
 );
 
+/**
+ * Hidden setting: Avoids caching in jump to def and find refs contextual queries if the user is also a canary user.
+ */
+export const NO_CACHE_CONTEXTUAL_QUERIES = new Setting(
+  "disableCache",
+  CONTEXTUAL_QUERIES_SETTINGS,
+);
+
 // Settings for variant analysis
 const VARIANT_ANALYSIS_SETTING = new Setting("variantAnalysis", ROOT_SETTING);
 
@@ -529,6 +547,34 @@ export class VariantAnalysisConfigListener
   }
 }
 
+const VARIANT_ANALYSIS_FILTER_RESULTS = new Setting(
+  "defaultResultsFilter",
+  VARIANT_ANALYSIS_SETTING,
+);
+
+export function getVariantAnalysisDefaultResultsFilter(): FilterKey {
+  const value = VARIANT_ANALYSIS_FILTER_RESULTS.getValue<string>();
+  if (Object.values(FilterKey).includes(value as FilterKey)) {
+    return value as FilterKey;
+  } else {
+    return defaultFilterSortState.filterKey;
+  }
+}
+
+const VARIANT_ANALYSIS_SORT_RESULTS = new Setting(
+  "defaultResultsSort",
+  VARIANT_ANALYSIS_SETTING,
+);
+
+export function getVariantAnalysisDefaultResultsSort(): SortKey {
+  const value = VARIANT_ANALYSIS_SORT_RESULTS.getValue<string>();
+  if (Object.values(SortKey).includes(value as SortKey)) {
+    return value as SortKey;
+  } else {
+    return defaultFilterSortState.sortKey;
+  }
+}
+
 /**
  * The branch of "github/codeql-variant-analysis-action" to use with the "Run Variant Analysis" command.
  * Default value is "main".
@@ -621,17 +667,78 @@ export function allowHttp(): boolean {
 }
 
 /**
- * The name of the folder where we want to create skeleton wizard QL packs.
+ * Parent setting for all settings related to the "Create Query" command.
+ */
+const CREATE_QUERY_COMMAND = new Setting("createQuery", ROOT_SETTING);
+
+/**
+ * The name of the folder where we want to create QL packs.
  **/
-const SKELETON_WIZARD_FOLDER = new Setting(
-  "folder",
-  new Setting("createQuery", ROOT_SETTING),
+const QL_PACK_LOCATION = new Setting("qlPackLocation", CREATE_QUERY_COMMAND);
+
+export function getQlPackLocation(): string | undefined {
+  return QL_PACK_LOCATION.getValue<string>() || undefined;
+}
+
+export async function setQlPackLocation(folder: string | undefined) {
+  await QL_PACK_LOCATION.updateValue(folder, ConfigurationTarget.Global);
+}
+
+/**
+ * Whether to ask the user to autogenerate a QL pack. The options are "ask" and "never".
+ **/
+const AUTOGENERATE_QL_PACKS = new Setting(
+  "autogenerateQlPacks",
+  CREATE_QUERY_COMMAND,
 );
 
-export function getSkeletonWizardFolder(): string | undefined {
-  return SKELETON_WIZARD_FOLDER.getValue<string>() || undefined;
+const AutogenerateQLPacksValues = ["ask", "never"] as const;
+type AutogenerateQLPacks = typeof AutogenerateQLPacksValues[number];
+
+export function getAutogenerateQlPacks(): AutogenerateQLPacks {
+  const value = AUTOGENERATE_QL_PACKS.getValue<AutogenerateQLPacks>();
+  return AutogenerateQLPacksValues.includes(value) ? value : "ask";
 }
 
-export async function setSkeletonWizardFolder(folder: string | undefined) {
-  await SKELETON_WIZARD_FOLDER.updateValue(folder, ConfigurationTarget.Global);
+export async function setAutogenerateQlPacks(choice: AutogenerateQLPacks) {
+  await AUTOGENERATE_QL_PACKS.updateValue(choice, ConfigurationTarget.Global);
+}
+
+/**
+ * A flag indicating whether to show the queries panel in the QL view container.
+ */
+const QUERIES_PANEL = new Setting("queriesPanel", ROOT_SETTING);
+
+export function showQueriesPanel(): boolean {
+  return !!QUERIES_PANEL.getValue<boolean>();
+}
+
+const DATA_EXTENSIONS = new Setting("dataExtensions", ROOT_SETTING);
+const LLM_GENERATION = new Setting("llmGeneration", DATA_EXTENSIONS);
+const FRAMEWORK_MODE = new Setting("frameworkMode", DATA_EXTENSIONS);
+const DISABLE_AUTO_NAME_EXTENSION_PACK = new Setting(
+  "disableAutoNameExtensionPack",
+  DATA_EXTENSIONS,
+);
+const EXTENSIONS_DIRECTORY = new Setting(
+  "extensionsDirectory",
+  DATA_EXTENSIONS,
+);
+
+export function showLlmGeneration(): boolean {
+  return !!LLM_GENERATION.getValue<boolean>();
+}
+
+export function enableFrameworkMode(): boolean {
+  return !!FRAMEWORK_MODE.getValue<boolean>();
+}
+
+export function disableAutoNameExtensionPack(): boolean {
+  return !!DISABLE_AUTO_NAME_EXTENSION_PACK.getValue<boolean>();
+}
+
+export function getExtensionsDirectory(languageId: string): string | undefined {
+  return EXTENSIONS_DIRECTORY.getValue<string>({
+    languageId,
+  });
 }

extensions/ql-vscode/src/data-extensions-editor/auto-model-api.ts

@@ -0,0 +1,54 @@
+import { Credentials } from "../common/authentication";
+import { OctokitResponse } from "@octokit/types";
+
+export enum ClassificationType {
+  Unknown = "CLASSIFICATION_TYPE_UNKNOWN",
+  Neutral = "CLASSIFICATION_TYPE_NEUTRAL",
+  Source = "CLASSIFICATION_TYPE_SOURCE",
+  Sink = "CLASSIFICATION_TYPE_SINK",
+  Summary = "CLASSIFICATION_TYPE_SUMMARY",
+}
+
+export interface Classification {
+  type: ClassificationType;
+  kind: string;
+  explanation: string;
+}
+
+export interface Method {
+  package: string;
+  type: string;
+  name: string;
+  signature: string;
+  usages: string[];
+  classification?: Classification;
+  input?: string;
+  output?: string;
+}
+
+export interface ModelRequest {
+  language: string;
+  candidates: Method[];
+  samples: Method[];
+}
+
+export interface ModelResponse {
+  language: string;
+  predicted?: Method[];
+}
+
+export async function autoModel(
+  credentials: Credentials,
+  request: ModelRequest,
+): Promise<ModelResponse> {
+  const octokit = await credentials.getOctokit();
+
+  const response: OctokitResponse<ModelResponse> = await octokit.request(
+    "POST /repos/github/codeql/code-scanning/codeql/auto-model",
+    {
+      data: request,
+    },
+  );
+
+  return response.data;
+}

extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts

@@ -0,0 +1,136 @@
+import { CancellationTokenSource } from "vscode";
+import { join } from "path";
+import { runQuery } from "./external-api-usage-query";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { QueryRunner } from "../query-server";
+import { DatabaseItem } from "../databases/local-databases";
+import { interpretResultsSarif } from "../query-results";
+import { ProgressCallback } from "../common/vscode/progress";
+
+type Options = {
+  cliServer: CodeQLCliServer;
+  queryRunner: QueryRunner;
+  databaseItem: DatabaseItem;
+  queryStorageDir: string;
+
+  progress: ProgressCallback;
+};
+
+export type UsageSnippetsBySignature = Record<string, string[]>;
+
+export async function getAutoModelUsages({
+  cliServer,
+  queryRunner,
+  databaseItem,
+  queryStorageDir,
+  progress,
+}: Options): Promise<UsageSnippetsBySignature> {
+  const maxStep = 1500;
+
+  const cancellationTokenSource = new CancellationTokenSource();
+
+  // This will re-run the query that was already run when opening the data extensions editor. This
+  // might be unnecessary, but this makes it really easy to get the path to the BQRS file which we
+  // need to interpret the results.
+  const queryResult = await runQuery("applicationModeQuery", {
+    cliServer,
+    queryRunner,
+    queryStorageDir,
+    databaseItem,
+    progress: (update) =>
+      progress({
+        maxStep,
+        step: update.step,
+        message: update.message,
+      }),
+    token: cancellationTokenSource.token,
+  });
+  if (!queryResult) {
+    throw new Error("Query failed");
+  }
+
+  progress({
+    maxStep,
+    step: 1100,
+    message: "Retrieving source location prefix",
+  });
+
+  // CodeQL needs to have access to the database to be able to retrieve the
+  // snippets from it. The source location prefix is used to determine the
+  // base path of the database.
+  const sourceLocationPrefix = await databaseItem.getSourceLocationPrefix(
+    cliServer,
+  );
+  const sourceArchiveUri = databaseItem.sourceArchive;
+  const sourceInfo =
+    sourceArchiveUri === undefined
+      ? undefined
+      : {
+          sourceArchive: sourceArchiveUri.fsPath,
+          sourceLocationPrefix,
+        };
+
+  progress({
+    maxStep,
+    step: 1200,
+    message: "Interpreting results",
+  });
+
+  // Convert the results to SARIF so that Codeql will retrieve the snippets
+  // from the datababe. This means we don't need to do that in the extension
+  // and everything is handled by the CodeQL CLI.
+  const sarif = await interpretResultsSarif(
+    cliServer,
+    {
+      // To interpret the results we need to provide metadata about the query. We could do this using
+      // `resolveMetadata` but that would be an extra call to the CodeQL CLI server and would require
+      // us to know the path to the query on the filesystem. Since we know what the metadata should
+      // look like and the only metadata that the CodeQL CLI requires is an ID and the kind, we can
+      // simply use constants here.
+      kind: "problem",
+      id: "usage",
+    },
+    {
+      resultsPath: queryResult.outputDir.bqrsPath,
+      interpretedResultsPath: join(
+        queryStorageDir,
+        "interpreted-results.sarif",
+      ),
+    },
+    sourceInfo,
+    ["--sarif-add-snippets"],
+  );
+
+  progress({
+    maxStep,
+    step: 1400,
+    message: "Parsing results",
+  });
+
+  const snippets: UsageSnippetsBySignature = {};
+
+  const results = sarif.runs[0]?.results;
+  if (!results) {
+    throw new Error("No results");
+  }
+
+  // This will group the snippets by the method signature.
+  for (const result of results) {
+    const signature = result.message.text;
+
+    const snippet =
+      result.locations?.[0]?.physicalLocation?.contextRegion?.snippet?.text;
+
+    if (!signature || !snippet) {
+      continue;
+    }
+
+    if (!(signature in snippets)) {
+      snippets[signature] = [];
+    }
+
+    snippets[signature].push(snippet);
+  }
+
+  return snippets;
+}

extensions/ql-vscode/src/data-extensions-editor/auto-model.ts

@@ -0,0 +1,251 @@
+import { ExternalApiUsage } from "./external-api-usage";
+import { ModeledMethod, ModeledMethodType } from "./modeled-method";
+import {
+  Classification,
+  ClassificationType,
+  Method,
+  ModelRequest,
+} from "./auto-model-api";
+import type { UsageSnippetsBySignature } from "./auto-model-usages-query";
+import { groupMethods, sortGroupNames, sortMethods } from "./shared/sorting";
+import { Mode } from "./shared/mode";
+
+// Soft limit on the number of candidates to send to the model.
+// Note that the model may return fewer than this number of candidates.
+const candidateLimit = 20;
+// Soft limit on the number of samples to send to the model.
+const sampleLimit = 100;
+
+export function createAutoModelRequest(
+  language: string,
+  externalApiUsages: ExternalApiUsage[],
+  modeledMethods: Record<string, ModeledMethod>,
+  usages: UsageSnippetsBySignature,
+  mode: Mode,
+): ModelRequest {
+  const request: ModelRequest = {
+    language,
+    samples: [],
+    candidates: [],
+  };
+
+  // Sort the same way as the UI so we send the first ones listed in the UI first
+  const grouped = groupMethods(externalApiUsages, mode);
+  const sortedGroupNames = sortGroupNames(grouped);
+  const sortedExternalApiUsages = sortedGroupNames.flatMap((name) =>
+    sortMethods(grouped[name]),
+  );
+
+  for (const externalApiUsage of sortedExternalApiUsages) {
+    const modeledMethod: ModeledMethod = modeledMethods[
+      externalApiUsage.signature
+    ] ?? {
+      type: "none",
+    };
+
+    const usagesForMethod =
+      usages[externalApiUsage.signature] ??
+      externalApiUsage.usages.map((usage) => usage.label);
+
+    const numberOfArguments =
+      externalApiUsage.methodParameters === "()"
+        ? 0
+        : externalApiUsage.methodParameters.split(",").length;
+
+    const candidates: Method[] = [];
+    const samples: Method[] = [];
+    for (
+      let argumentIndex = -1; // Start at -1 which means `this` as in `this.method()`
+      argumentIndex < numberOfArguments;
+      argumentIndex++
+    ) {
+      const argumentInput: string =
+        argumentIndex === -1 ? "Argument[this]" : `Argument[${argumentIndex}]`;
+      const method: Method = {
+        package: externalApiUsage.packageName,
+        type: externalApiUsage.typeName,
+        name: externalApiUsage.methodName,
+        signature: externalApiUsage.methodParameters,
+        classification:
+          modeledMethod.type === "none"
+            ? undefined
+            : toMethodClassification(modeledMethod),
+        usages: usagesForMethod.slice(0, 6), // At most 6 usages per argument
+        input: argumentInput,
+      };
+
+      // A method that is supported is modeled outside of the model file, so it is not a candidate.
+      // We also do not want it as a sample because we do not know the classification.
+      if (modeledMethod.type === "none" && externalApiUsage.supported) {
+        continue;
+      }
+
+      // Candidates are methods that are not currently modeled
+      if (modeledMethod.type === "none") {
+        candidates.push(method);
+      } else {
+        samples.push(method);
+      }
+    }
+    // If there is room for at least one candidate, add all candidates.
+    // This ensures that we send all arguments for a method together.
+    // NOTE: this might go above the candidate limit, but that's okay.
+    if (request.candidates.length < candidateLimit) {
+      request.candidates.push(...candidates);
+    }
+    // Same for samples
+    if (request.samples.length < sampleLimit) {
+      request.samples.push(...samples);
+    }
+  }
+
+  return request;
+}
+
+/**
+ * For now, we have a simplified model that only models methods as sinks. It does not model methods as neutral,
+ * so we aren't actually able to correctly determine that a method is neutral; it could still be a source or summary.
+ * However, to keep this method simple and give output to the user, we will model any method for which none of its
+ * arguments are modeled as sinks as neutral.
+ *
+ * If there are multiple arguments which are modeled as sinks, we will only model the first one.
+ */
+export function parsePredictedClassifications(
+  predicted: Method[],
+): Record<string, ModeledMethod> {
+  const predictedBySignature: Record<string, Method[]> = {};
+  for (const method of predicted) {
+    const signature = toFullMethodSignature(method);
+
+    if (!(signature in predictedBySignature)) {
+      predictedBySignature[signature] = [];
+    }
+
+    predictedBySignature[signature].push(method);
+  }
+
+  const modeledMethods: Record<string, ModeledMethod> = {};
+
+  for (const signature in predictedBySignature) {
+    const predictedMethods = predictedBySignature[signature];
+
+    const sinks = predictedMethods.filter(
+      (method) => method.classification?.type === ClassificationType.Sink,
+    );
+    if (sinks.length === 0) {
+      // For now, model any method for which none of its arguments are modeled as sinks as neutral
+      modeledMethods[signature] = {
+        type: "neutral",
+        kind: "summary",
+        input: "",
+        output: "",
+        provenance: "ai-generated",
+      };
+      continue;
+    }
+
+    // Order the sinks by the input alphabetically. This will ensure that the first argument is always
+    // first in the list of sinks, the second argument is always second, etc.
+    // If we get back "Argument[1]" and "Argument[3]", "Argument[1]" should always be first
+    sinks.sort((a, b) => compareInputOutput(a.input ?? "", b.input ?? ""));
+
+    const sink = sinks[0];
+
+    modeledMethods[signature] = {
+      type: "sink",
+      kind: sink.classification?.kind ?? "",
+      input: sink.input ?? "",
+      output: sink.output ?? "",
+      provenance: "ai-generated",
+    };
+  }
+
+  return modeledMethods;
+}
+
+function toMethodClassificationType(
+  type: ModeledMethodType,
+): ClassificationType {
+  switch (type) {
+    case "source":
+      return ClassificationType.Source;
+    case "sink":
+      return ClassificationType.Sink;
+    case "summary":
+      return ClassificationType.Summary;
+    case "neutral":
+      return ClassificationType.Neutral;
+    default:
+      return ClassificationType.Unknown;
+  }
+}
+
+function toMethodClassification(modeledMethod: ModeledMethod): Classification {
+  return {
+    type: toMethodClassificationType(modeledMethod.type),
+    kind: modeledMethod.kind,
+    explanation: "",
+  };
+}
+
+function toFullMethodSignature(method: Method): string {
+  return `${method.package}.${method.type}#${method.name}${method.signature}`;
+}
+
+const argumentRegex = /^Argument\[(\d+)]$/;
+
+// Argument[this] is before ReturnValue
+const nonNumericArgumentOrder = ["Argument[this]", "ReturnValue"];
+
+/**
+ * Compare two inputs or outputs matching `Argument[<number>]`, `Argument[this]`, or `ReturnValue`.
+ * If they are the same, return 0. If a is less than b, returns a negative number.
+ * If a is greater than b, returns a positive number.
+ */
+export function compareInputOutput(a: string, b: string): number {
+  if (a === b) {
+    return 0;
+  }
+
+  const aMatch = a.match(argumentRegex);
+  const bMatch = b.match(argumentRegex);
+
+  // Numeric arguments are always first
+  if (aMatch && !bMatch) {
+    return -1;
+  }
+  if (!aMatch && bMatch) {
+    return 1;
+  }
+
+  // Neither is an argument
+  if (!aMatch && !bMatch) {
+    const aIndex = nonNumericArgumentOrder.indexOf(a);
+    const bIndex = nonNumericArgumentOrder.indexOf(b);
+
+    // If either one is unknown, it is sorted last
+    if (aIndex === -1 && bIndex === -1) {
+      // Use en-US because these are well-known strings that are not localized
+      return a.localeCompare(b, "en-US");
+    }
+    if (aIndex === -1) {
+      return 1;
+    }
+    if (bIndex === -1) {
+      return -1;
+    }
+
+    return aIndex - bIndex;
+  }
+
+  // This case shouldn't happen, but makes TypeScript happy
+  if (!aMatch || !bMatch) {
+    return 0;
+  }
+
+  // Both are arguments
+  const aIndex = parseInt(aMatch[1]);
+  const bIndex = parseInt(bMatch[1]);
+
+  return aIndex - bIndex;
+}

extensions/ql-vscode/src/data-extensions-editor/bqrs.ts

@@ -1,4 +1,4 @@
-import { DecodedBqrsChunk } from "../pure/bqrs-cli-types";
+import { DecodedBqrsChunk } from "../common/bqrs-cli-types";
 import { Call, ExternalApiUsage } from "./external-api-usage";
 
 export function decodeBqrsToExternalApiUsages(
@@ -7,9 +7,10 @@ export function decodeBqrsToExternalApiUsages(
   const methodsByApiName = new Map<string, ExternalApiUsage>();
 
   chunk?.tuples.forEach((tuple) => {
-    const signature = tuple[0] as string;
-    const supported = tuple[1] as boolean;
-    const usage = tuple[2] as Call;
+    const usage = tuple[0] as Call;
+    const signature = tuple[1] as string;
+    const supported = (tuple[2] as string) === "true";
+    const library = tuple[4] as string;
 
     const [packageWithType, methodDeclaration] = signature.split("#");
 
@@ -31,6 +32,7 @@ export function decodeBqrsToExternalApiUsages(
 
     if (!methodsByApiName.has(signature)) {
       methodsByApiName.set(signature, {
+        library,
         signature,
         packageName,
         typeName,
@@ -45,17 +47,5 @@ export function decodeBqrsToExternalApiUsages(
     method.usages.push(usage);
   });
 
-  const externalApiUsages = Array.from(methodsByApiName.values());
-  externalApiUsages.sort((a, b) => {
-    // Sort first by supported, putting unmodeled methods first.
-    if (a.supported && !b.supported) {
-      return 1;
-    }
-    if (!a.supported && b.supported) {
-      return -1;
-    }
-    // Then sort by number of usages descending
-    return b.usages.length - a.usages.length;
-  });
-  return externalApiUsages;
+  return Array.from(methodsByApiName.values());
 }

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts

@@ -1,15 +1,15 @@
 import { ExtensionContext } from "vscode";
 import { DataExtensionsEditorView } from "./data-extensions-editor-view";
 import { DataExtensionsEditorCommands } from "../common/commands";
-import { CliVersionConstraint, CodeQLCliServer } from "../cli";
+import { CliVersionConstraint, CodeQLCliServer } from "../codeql-cli/cli";
 import { QueryRunner } from "../query-server";
-import { DatabaseManager } from "../local-databases";
+import { DatabaseManager } from "../databases/local-databases";
 import { ensureDir } from "fs-extra";
 import { join } from "path";
 import { App } from "../common/app";
-import { showAndLogErrorMessage } from "../helpers";
-import { withProgress } from "../progress";
-import { pickExtensionPackModelFile } from "./extension-pack-picker";
+import { withProgress } from "../common/vscode/progress";
+import { pickExtensionPack } from "./extension-pack-picker";
+import { showAndLogErrorMessage } from "../common/logging";
 
 const SUPPORTED_LANGUAGES: string[] = ["java", "csharp"];
 
@@ -56,12 +56,13 @@ export class DataExtensionsEditorModule {
       "codeQL.openDataExtensionsEditor": async () => {
         const db = this.databaseManager.currentDatabaseItem;
         if (!db) {
-          void showAndLogErrorMessage("No database selected");
+          void showAndLogErrorMessage(this.app.logger, "No database selected");
           return;
         }
 
         if (!SUPPORTED_LANGUAGES.includes(db.language)) {
           void showAndLogErrorMessage(
+            this.app.logger,
             `The data extensions editor is not supported for ${db.language} databases.`,
           );
           return;
@@ -71,14 +72,16 @@ export class DataExtensionsEditorModule {
           async (progress, token) => {
             if (!(await this.cliServer.cliConstraints.supportsQlpacksKind())) {
               void showAndLogErrorMessage(
+                this.app.logger,
                 `This feature requires CodeQL CLI version ${CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND.format()} or later.`,
               );
               return;
             }
 
-            const modelFile = await pickExtensionPackModelFile(
+            const modelFile = await pickExtensionPack(
               this.cliServer,
               db,
+              this.app.logger,
               progress,
               token,
             );

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts

@@ -4,51 +4,53 @@ import {
   Uri,
   ViewColumn,
   window,
-  workspace,
-  WorkspaceFolder,
 } from "vscode";
-import { AbstractWebview, WebviewPanelConfig } from "../abstract-webview";
+import { join } from "path";
+import { RequestError } from "@octokit/request-error";
+import {
+  AbstractWebview,
+  WebviewPanelConfig,
+} from "../common/vscode/abstract-webview";
 import {
   FromDataExtensionsEditorMessage,
   ToDataExtensionsEditorMessage,
-} from "../pure/interface-types";
-import { ProgressUpdate } from "../progress";
+} from "../common/interface-types";
+import { ProgressUpdate } from "../common/vscode/progress";
 import { QueryRunner } from "../query-server";
 import {
-  showAndLogErrorMessage,
   showAndLogExceptionWithTelemetry,
-} from "../helpers";
-import { extLogger } from "../common";
-import { outputFile, pathExists, readFile } from "fs-extra";
+  showAndLogErrorMessage,
+} from "../common/logging";
+import { outputFile, readFile } from "fs-extra";
 import { load as loadYaml } from "js-yaml";
-import { DatabaseItem, DatabaseManager } from "../local-databases";
-import { CodeQLCliServer } from "../cli";
-import { asError, assertNever, getErrorMessage } from "../pure/helpers-pure";
+import { DatabaseItem, DatabaseManager } from "../databases/local-databases";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { asError, assertNever, getErrorMessage } from "../common/helpers-pure";
 import { generateFlowModel } from "./generate-flow-model";
-import { promptImportGithubDatabase } from "../databaseFetcher";
+import { promptImportGithubDatabase } from "../databases/database-fetcher";
 import { App } from "../common/app";
-import { ResolvableLocationValue } from "../pure/bqrs-cli-types";
-import { showResolvableLocation } from "../interface-utils";
+import { ResolvableLocationValue } from "../common/bqrs-cli-types";
+import { showResolvableLocation } from "../databases/local-databases/locations";
 import { decodeBqrsToExternalApiUsages } from "./bqrs";
-import { redactableError } from "../pure/errors";
+import { redactableError } from "../common/errors";
 import { readQueryResults, runQuery } from "./external-api-usage-query";
-import { createDataExtensionYaml, loadDataExtensionYaml } from "./yaml";
+import {
+  createDataExtensionYamlsForApplicationMode,
+  createDataExtensionYamlsForFrameworkMode,
+  loadDataExtensionYaml,
+} from "./yaml";
 import { ExternalApiUsage } from "./external-api-usage";
 import { ModeledMethod } from "./modeled-method";
-import { ExtensionPackModelFile } from "./shared/extension-pack";
-
-function getQlSubmoduleFolder(): WorkspaceFolder | undefined {
-  const workspaceFolder = workspace.workspaceFolders?.find(
-    (folder) => folder.name === "ql",
-  );
-  if (!workspaceFolder) {
-    void extLogger.log("No workspace folder 'ql' found");
-
-    return;
-  }
-
-  return workspaceFolder;
-}
+import { ExtensionPack } from "./shared/extension-pack";
+import { autoModel, ModelRequest, ModelResponse } from "./auto-model-api";
+import {
+  createAutoModelRequest,
+  parsePredictedClassifications,
+} from "./auto-model";
+import { enableFrameworkMode, showLlmGeneration } from "../config";
+import { getAutoModelUsages } from "./auto-model-usages-query";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { Mode } from "./shared/mode";
 
 export class DataExtensionsEditorView extends AbstractWebview<
   ToDataExtensionsEditorMessage,
@@ -62,7 +64,8 @@ export class DataExtensionsEditorView extends AbstractWebview<
     private readonly queryRunner: QueryRunner,
     private readonly queryStorageDir: string,
     private readonly databaseItem: DatabaseItem,
-    private readonly modelFile: ExtensionPackModelFile,
+    private readonly extensionPack: ExtensionPack,
+    private mode: Mode = Mode.Application,
   ) {
     super(ctx);
   }
@@ -99,14 +102,12 @@ export class DataExtensionsEditorView extends AbstractWebview<
       case "openExtensionPack":
         await this.app.commands.execute(
           "revealInExplorer",
-          Uri.file(this.modelFile.extensionPack.path),
+          Uri.file(this.extensionPack.path),
         );
 
         break;
-      case "openModelFile":
-        await window.showTextDocument(
-          await workspace.openTextDocument(this.modelFile.filename),
-        );
+      case "refreshExternalApiUsages":
+        await this.loadExternalApiUsages();
 
         break;
       case "jumpToUsage":
@@ -124,6 +125,19 @@ export class DataExtensionsEditorView extends AbstractWebview<
       case "generateExternalApi":
         await this.generateModeledMethods();
 
+        break;
+      case "generateExternalApiFromLlm":
+        await this.generateModeledMethodsFromLlm(
+          msg.externalApiUsages,
+          msg.modeledMethods,
+        );
+
+        break;
+      case "switchMode":
+        this.mode = msg.mode;
+
+        await Promise.all([this.setViewState(), this.loadExternalApiUsages()]);
+
         break;
       default:
         assertNever(msg);
@@ -144,8 +158,10 @@ export class DataExtensionsEditorView extends AbstractWebview<
     await this.postMessage({
       t: "setDataExtensionEditorViewState",
       viewState: {
-        extensionPackModelFile: this.modelFile,
-        modelFileExists: await pathExists(this.modelFile.filename),
+        extensionPack: this.extensionPack,
+        enableFrameworkMode: enableFrameworkMode(),
+        showLlmButton: showLlmGeneration(),
+        mode: this.mode,
       },
     });
   }
@@ -162,10 +178,10 @@ export class DataExtensionsEditorView extends AbstractWebview<
             "Original file of this result is not in the database's source archive.",
           );
         } else {
-          void extLogger.log(`Unable to handleMsgFromView: ${e.message}`);
+          void this.app.logger.log(`Unable to handleMsgFromView: ${e.message}`);
         }
       } else {
-        void extLogger.log(`Unable to handleMsgFromView: ${e}`);
+        void this.app.logger.log(`Unable to handleMsgFromView: ${e}`);
       }
     }
   }
@@ -174,38 +190,70 @@ export class DataExtensionsEditorView extends AbstractWebview<
     externalApiUsages: ExternalApiUsage[],
     modeledMethods: Record<string, ModeledMethod>,
   ): Promise<void> {
-    const yaml = createDataExtensionYaml(
-      this.databaseItem.language,
-      externalApiUsages,
-      modeledMethods,
-    );
+    let yamls: Record<string, string>;
+    switch (this.mode) {
+      case Mode.Application:
+        yamls = createDataExtensionYamlsForApplicationMode(
+          this.databaseItem.language,
+          externalApiUsages,
+          modeledMethods,
+        );
+        break;
+      case Mode.Framework:
+        yamls = createDataExtensionYamlsForFrameworkMode(
+          this.databaseItem.name,
+          this.databaseItem.language,
+          externalApiUsages,
+          modeledMethods,
+        );
+        break;
+      default:
+        assertNever(this.mode);
+    }
 
-    await outputFile(this.modelFile.filename, yaml);
+    for (const [filename, yaml] of Object.entries(yamls)) {
+      await outputFile(join(this.extensionPack.path, filename), yaml);
+    }
 
-    void extLogger.log(
-      `Saved data extension YAML to ${this.modelFile.filename}`,
-    );
+    void this.app.logger.log(`Saved data extension YAML`);
   }
 
   protected async loadExistingModeledMethods(): Promise<void> {
     try {
-      if (!(await pathExists(this.modelFile.filename))) {
-        return;
+      const extensions = await this.cliServer.resolveExtensions(
+        this.extensionPack.path,
+        getOnDiskWorkspaceFolders(),
+      );
+
+      const modelFiles = new Set<string>();
+
+      if (this.extensionPack.path in extensions.data) {
+        for (const extension of extensions.data[this.extensionPack.path]) {
+          modelFiles.add(extension.file);
+        }
       }
 
-      const yaml = await readFile(this.modelFile.filename, "utf8");
+      const existingModeledMethods: Record<string, ModeledMethod> = {};
 
-      const data = loadYaml(yaml, {
-        filename: this.modelFile.filename,
-      });
+      for (const modelFile of modelFiles) {
+        const yaml = await readFile(modelFile, "utf8");
 
-      const existingModeledMethods = loadDataExtensionYaml(data);
+        const data = loadYaml(yaml, {
+          filename: modelFile,
+        });
 
-      if (!existingModeledMethods) {
-        void showAndLogErrorMessage(
-          `Failed to parse data extension YAML ${this.modelFile.filename}.`,
-        );
-        return;
+        const modeledMethods = loadDataExtensionYaml(data);
+        if (!modeledMethods) {
+          void showAndLogErrorMessage(
+            this.app.logger,
+            `Failed to parse data extension YAML ${modelFile}.`,
+          );
+          continue;
+        }
+
+        for (const [key, value] of Object.entries(modeledMethods)) {
+          existingModeledMethods[key] = value;
+        }
       }
 
       await this.postMessage({
@@ -214,9 +262,8 @@ export class DataExtensionsEditorView extends AbstractWebview<
       });
     } catch (e: unknown) {
       void showAndLogErrorMessage(
-        `Unable to read data extension YAML ${
-          this.modelFile.filename
-        }: ${getErrorMessage(e)}`,
+        this.app.logger,
+        `Unable to read data extension YAML: ${getErrorMessage(e)}`,
       );
     }
   }
@@ -225,16 +272,21 @@ export class DataExtensionsEditorView extends AbstractWebview<
     const cancellationTokenSource = new CancellationTokenSource();
 
     try {
-      const queryResult = await runQuery({
-        cliServer: this.cliServer,
-        queryRunner: this.queryRunner,
-        databaseItem: this.databaseItem,
-        queryStorageDir: this.queryStorageDir,
-        progress: (progressUpdate: ProgressUpdate) => {
-          void this.showProgress(progressUpdate, 1500);
+      const queryResult = await runQuery(
+        this.mode === Mode.Framework
+          ? "frameworkModeQuery"
+          : "applicationModeQuery",
+        {
+          cliServer: this.cliServer,
+          queryRunner: this.queryRunner,
+          databaseItem: this.databaseItem,
+          queryStorageDir: this.queryStorageDir,
+          progress: (progressUpdate: ProgressUpdate) => {
+            void this.showProgress(progressUpdate, 1500);
+          },
+          token: cancellationTokenSource.token,
         },
-        token: cancellationTokenSource.token,
-      });
+      );
       if (!queryResult) {
         await this.clearProgress();
         return;
@@ -271,6 +323,8 @@ export class DataExtensionsEditorView extends AbstractWebview<
       await this.clearProgress();
     } catch (err) {
       void showAndLogExceptionWithTelemetry(
+        this.app.logger,
+        this.app.telemetry,
         redactableError(
           asError(err),
         )`Failed to load external API usages: ${getErrorMessage(err)}`,
@@ -281,34 +335,34 @@ export class DataExtensionsEditorView extends AbstractWebview<
   protected async generateModeledMethods(): Promise<void> {
     const tokenSource = new CancellationTokenSource();
 
-    const selectedDatabase = this.databaseManager.currentDatabaseItem;
+    let addedDatabase: DatabaseItem | undefined;
 
-    // The external API methods are in the library source code, so we need to ask
-    // the user to import the library database. We need to have the database
-    // imported to the query server, so we need to register it to our workspace.
-    const database = await promptImportGithubDatabase(
-      this.app.commands,
-      this.databaseManager,
-      this.app.workspaceStoragePath ?? this.app.globalStoragePath,
-      this.app.credentials,
-      (update) => this.showProgress(update),
-      tokenSource.token,
-      this.cliServer,
-    );
-    if (!database) {
-      await this.clearProgress();
-      void extLogger.log("No database chosen");
+    // In application mode, we need the database of a specific library to generate
+    // the modeled methods. In framework mode, we'll use the current database.
+    if (this.mode === Mode.Application) {
+      const selectedDatabase = this.databaseManager.currentDatabaseItem;
 
-      return;
-    }
+      // The external API methods are in the library source code, so we need to ask
+      // the user to import the library database. We need to have the database
+      // imported to the query server, so we need to register it to our workspace.
+      addedDatabase = await promptImportGithubDatabase(
+        this.app.commands,
+        this.databaseManager,
+        this.app.workspaceStoragePath ?? this.app.globalStoragePath,
+        this.app.credentials,
+        (update) => this.showProgress(update),
+        this.cliServer,
+      );
+      if (!addedDatabase) {
+        await this.clearProgress();
+        void this.app.logger.log("No database chosen");
 
-    // The library database was set as the current database by importing it,
-    // but we need to set it back to the originally selected database.
-    await this.databaseManager.setCurrentDatabaseItem(selectedDatabase);
+        return;
+      }
 
-    const workspaceFolder = getQlSubmoduleFolder();
-    if (!workspaceFolder) {
-      return;
+      // The library database was set as the current database by importing it,
+      // but we need to set it back to the originally selected database.
+      await this.databaseManager.setCurrentDatabaseItem(selectedDatabase);
     }
 
     await this.showProgress({
@@ -322,8 +376,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
         cliServer: this.cliServer,
         queryRunner: this.queryRunner,
         queryStorageDir: this.queryStorageDir,
-        qlDir: workspaceFolder.uri.fsPath,
-        databaseItem: database,
+        databaseItem: addedDatabase ?? this.databaseItem,
         onResults: async (results) => {
           const modeledMethodsByName: Record<string, ModeledMethod> = {};
 
@@ -342,25 +395,95 @@ export class DataExtensionsEditorView extends AbstractWebview<
       });
     } catch (e: unknown) {
       void showAndLogExceptionWithTelemetry(
+        this.app.logger,
+        this.app.telemetry,
         redactableError(
           asError(e),
         )`Failed to generate flow model: ${getErrorMessage(e)}`,
       );
     }
 
-    // After the flow model has been generated, we can remove the temporary database
-    // which we used for generating the flow model.
-    await this.databaseManager.removeDatabaseItem(
-      () =>
-        this.showProgress({
-          step: 3900,
-          maxStep: 4000,
-          message: "Removing temporary database",
-        }),
-      tokenSource.token,
-      database,
+    if (addedDatabase) {
+      // After the flow model has been generated, we can remove the temporary database
+      // which we used for generating the flow model.
+      await this.showProgress({
+        step: 3900,
+        maxStep: 4000,
+        message: "Removing temporary database",
+      });
+      await this.databaseManager.removeDatabaseItem(addedDatabase);
+    }
+
+    await this.clearProgress();
+  }
+
+  private async generateModeledMethodsFromLlm(
+    externalApiUsages: ExternalApiUsage[],
+    modeledMethods: Record<string, ModeledMethod>,
+  ): Promise<void> {
+    const maxStep = 3000;
+
+    await this.showProgress({
+      step: 0,
+      maxStep,
+      message: "Retrieving usages",
+    });
+
+    const usages = await getAutoModelUsages({
+      cliServer: this.cliServer,
+      queryRunner: this.queryRunner,
+      queryStorageDir: this.queryStorageDir,
+      databaseItem: this.databaseItem,
+      progress: (update) => this.showProgress(update, maxStep),
+    });
+
+    await this.showProgress({
+      step: 1800,
+      maxStep,
+      message: "Creating request",
+    });
+
+    const request = createAutoModelRequest(
+      this.databaseItem.language,
+      externalApiUsages,
+      modeledMethods,
+      usages,
+      this.mode,
     );
 
+    await this.showProgress({
+      step: 2000,
+      maxStep,
+      message: "Sending request",
+    });
+
+    const response = await this.callAutoModelApi(request);
+    if (!response) {
+      return;
+    }
+
+    await this.showProgress({
+      step: 2500,
+      maxStep,
+      message: "Parsing response",
+    });
+
+    const predictedModeledMethods = parsePredictedClassifications(
+      response.predicted || [],
+    );
+
+    await this.showProgress({
+      step: 2800,
+      maxStep,
+      message: "Applying results",
+    });
+
+    await this.postMessage({
+      t: "addModeledMethods",
+      modeledMethods: predictedModeledMethods,
+      overrideNone: true,
+    });
+
     await this.clearProgress();
   }
 
@@ -395,4 +518,25 @@ export class DataExtensionsEditorView extends AbstractWebview<
       message: "",
     });
   }
+
+  private async callAutoModelApi(
+    request: ModelRequest,
+  ): Promise<ModelResponse | null> {
+    try {
+      return await autoModel(this.app.credentials, request);
+    } catch (e) {
+      await this.clearProgress();
+
+      if (e instanceof RequestError && e.status === 429) {
+        void showAndLogExceptionWithTelemetry(
+          this.app.logger,
+          this.app.telemetry,
+          redactableError(e)`Rate limit hit, please try again soon.`,
+        );
+        return null;
+      } else {
+        throw e;
+      }
+    }
+  }
 }

extensions/ql-vscode/src/data-extensions-editor/extension-pack-name.ts

@@ -0,0 +1,93 @@
+const packNamePartRegex = /[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/;
+const packNameRegex = new RegExp(
+  `^(?<scope>${packNamePartRegex.source})/(?<name>${packNamePartRegex.source})$`,
+);
+const packNameLength = 128;
+
+export interface ExtensionPackName {
+  scope: string;
+  name: string;
+}
+
+export function formatPackName(packName: ExtensionPackName): string {
+  return `${packName.scope}/${packName.name}`;
+}
+
+export function autoNameExtensionPack(
+  name: string,
+  language: string,
+): ExtensionPackName | undefined {
+  let packName = `${name}-${language}`;
+  if (!packName.includes("/")) {
+    packName = `pack/${packName}`;
+  }
+
+  const parts = packName.split("/");
+  const sanitizedParts = parts.map((part) => sanitizeExtensionPackName(part));
+
+  // If the scope is empty (e.g. if the given name is "-/b"), then we need to still set a scope
+  if (sanitizedParts[0].length === 0) {
+    sanitizedParts[0] = "pack";
+  }
+
+  return {
+    scope: sanitizedParts[0],
+    // This will ensure there's only 1 slash
+    name: sanitizedParts.slice(1).join("-"),
+  };
+}
+
+export function sanitizeExtensionPackName(name: string) {
+  // Lowercase everything
+  name = name.toLowerCase();
+
+  // Replace all spaces, dots, and underscores with hyphens
+  name = name.replaceAll(/[\s._]+/g, "-");
+
+  // Replace all characters which are not allowed by empty strings
+  name = name.replaceAll(/[^a-z0-9-]/g, "");
+
+  // Remove any leading or trailing hyphens
+  name = name.replaceAll(/^-|-$/g, "");
+
+  // Remove any duplicate hyphens
+  name = name.replaceAll(/-{2,}/g, "-");
+
+  return name;
+}
+
+export function parsePackName(packName: string): ExtensionPackName | undefined {
+  const matches = packNameRegex.exec(packName);
+  if (!matches?.groups) {
+    return;
+  }
+
+  const scope = matches.groups.scope;
+  const name = matches.groups.name;
+
+  return {
+    scope,
+    name,
+  };
+}
+
+export function validatePackName(name: string): string | undefined {
+  if (!name) {
+    return "Pack name must not be empty";
+  }
+
+  if (name.length > packNameLength) {
+    return `Pack name must be no longer than ${packNameLength} characters`;
+  }
+
+  const matches = packNameRegex.exec(name);
+  if (!matches?.groups) {
+    if (!name.includes("/")) {
+      return "Invalid package name: a pack name must contain a slash to separate the scope from the pack name";
+    }
+
+    return "Invalid package name: a pack name must contain only lowercase ASCII letters, ASCII digits, and hyphens";
+  }
+
+  return undefined;
+}

extensions/ql-vscode/src/data-extensions-editor/extension-pack-picker.ts

@@ -1,64 +1,37 @@
-import { join, relative, resolve, sep } from "path";
+import { join } from "path";
 import { outputFile, pathExists, readFile } from "fs-extra";
 import { dump as dumpYaml, load as loadYaml } from "js-yaml";
-import { minimatch } from "minimatch";
-import { CancellationToken, window } from "vscode";
-import { CodeQLCliServer } from "../cli";
+import { CancellationToken, Uri, window } from "vscode";
+import { CodeQLCliServer, QlpacksInfo } from "../codeql-cli/cli";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { ProgressCallback } from "../common/vscode/progress";
+import { DatabaseItem } from "../databases/local-databases";
+import { getQlPackPath, QLPACK_FILENAMES } from "../common/ql";
+import { getErrorMessage } from "../common/helpers-pure";
+import { ExtensionPack } from "./shared/extension-pack";
+import { NotificationLogger, showAndLogErrorMessage } from "../common/logging";
 import {
-  getOnDiskWorkspaceFolders,
-  getOnDiskWorkspaceFoldersObjects,
-  showAndLogErrorMessage,
-} from "../helpers";
-import { ProgressCallback } from "../progress";
-import { DatabaseItem } from "../local-databases";
-import { getQlPackPath, QLPACK_FILENAMES } from "../pure/ql";
-import { getErrorMessage } from "../pure/helpers-pure";
-import { ExtensionPack, ExtensionPackModelFile } from "./shared/extension-pack";
+  disableAutoNameExtensionPack,
+  getExtensionsDirectory,
+} from "../config";
+import {
+  autoNameExtensionPack,
+  ExtensionPackName,
+  formatPackName,
+  parsePackName,
+  validatePackName,
+} from "./extension-pack-name";
+import {
+  askForWorkspaceFolder,
+  autoPickExtensionsDirectory,
+} from "./extensions-workspace-folder";
 
 const maxStep = 3;
 
-const packNamePartRegex = /[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/;
-const packNameRegex = new RegExp(
-  `^(?:(?<scope>${packNamePartRegex.source})/)?(?<name>${packNamePartRegex.source})$`,
-);
-const packNameLength = 128;
-
-export async function pickExtensionPackModelFile(
-  cliServer: Pick<CodeQLCliServer, "resolveQlpacks" | "resolveExtensions">,
-  databaseItem: Pick<DatabaseItem, "name" | "language">,
-  progress: ProgressCallback,
-  token: CancellationToken,
-): Promise<ExtensionPackModelFile | undefined> {
-  const extensionPack = await pickExtensionPack(
-    cliServer,
-    databaseItem,
-    progress,
-    token,
-  );
-  if (!extensionPack) {
-    return undefined;
-  }
-
-  const modelFile = await pickModelFile(
-    cliServer,
-    databaseItem,
-    extensionPack,
-    progress,
-    token,
-  );
-  if (!modelFile) {
-    return;
-  }
-
-  return {
-    filename: modelFile,
-    extensionPack,
-  };
-}
-
-async function pickExtensionPack(
+export async function pickExtensionPack(
   cliServer: Pick<CodeQLCliServer, "resolveQlpacks">,
   databaseItem: Pick<DatabaseItem, "name" | "language">,
+  logger: NotificationLogger,
   progress: ProgressCallback,
   token: CancellationToken,
 ): Promise<ExtensionPack | undefined> {
@@ -75,6 +48,21 @@ async function pickExtensionPack(
     true,
   );
 
+  if (!disableAutoNameExtensionPack()) {
+    progress({
+      message: "Creating extension pack...",
+      step: 2,
+      maxStep,
+    });
+
+    return autoCreateExtensionPack(
+      databaseItem.name,
+      databaseItem.language,
+      extensionPacksInfo,
+      logger,
+    );
+  }
+
   if (Object.keys(extensionPacksInfo).length === 0) {
     return pickNewExtensionPack(databaseItem, token);
   }
@@ -84,6 +72,7 @@ async function pickExtensionPack(
       Object.entries(extensionPacksInfo).map(async ([name, paths]) => {
         if (paths.length !== 1) {
           void showAndLogErrorMessage(
+            logger,
             `Extension pack ${name} resolves to multiple paths`,
             {
               fullMessage: `Extension pack ${name} resolves to multiple paths: ${paths.join(
@@ -101,11 +90,15 @@ async function pickExtensionPack(
         try {
           extensionPack = await readExtensionPack(path);
         } catch (e: unknown) {
-          void showAndLogErrorMessage(`Could not read extension pack ${name}`, {
-            fullMessage: `Could not read extension pack ${name} at ${path}: ${getErrorMessage(
-              e,
-            )}`,
-          });
+          void showAndLogErrorMessage(
+            logger,
+            `Could not read extension pack ${name}`,
+            {
+              fullMessage: `Could not read extension pack ${name} at ${path}: ${getErrorMessage(
+                e,
+              )}`,
+            },
+          );
 
           return undefined;
         }
@@ -163,109 +156,39 @@ async function pickExtensionPack(
   return extensionPackOption.extensionPack;
 }
 
-async function pickModelFile(
-  cliServer: Pick<CodeQLCliServer, "resolveExtensions">,
-  databaseItem: Pick<DatabaseItem, "name">,
-  extensionPack: ExtensionPack,
-  progress: ProgressCallback,
-  token: CancellationToken,
-): Promise<string | undefined> {
-  // Find the existing model files in the extension pack
-  const additionalPacks = getOnDiskWorkspaceFolders();
-  const extensions = await cliServer.resolveExtensions(
-    extensionPack.path,
-    additionalPacks,
-  );
-
-  const modelFiles = new Set<string>();
-
-  if (extensionPack.path in extensions.data) {
-    for (const extension of extensions.data[extensionPack.path]) {
-      modelFiles.add(extension.file);
-    }
-  }
-
-  if (modelFiles.size === 0) {
-    return pickNewModelFile(databaseItem, extensionPack, token);
-  }
-
-  const fileOptions: Array<{ label: string; file: string | null }> = [];
-  for (const file of modelFiles) {
-    fileOptions.push({
-      label: relative(extensionPack.path, file).replaceAll(sep, "/"),
-      file,
-    });
-  }
-  fileOptions.push({
-    label: "Create new model file",
-    file: null,
-  });
-
-  progress({
-    message: "Choosing model file...",
-    step: 3,
-    maxStep,
-  });
-
-  const fileOption = await window.showQuickPick(
-    fileOptions,
-    {
-      title: "Select model file to use",
-    },
-    token,
-  );
-
-  if (!fileOption) {
-    return undefined;
-  }
-
-  if (fileOption.file) {
-    return fileOption.file;
-  }
-
-  return pickNewModelFile(databaseItem, extensionPack, token);
-}
-
 async function pickNewExtensionPack(
   databaseItem: Pick<DatabaseItem, "name" | "language">,
   token: CancellationToken,
 ): Promise<ExtensionPack | undefined> {
-  const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
-  const workspaceFolderOptions = workspaceFolders.map((folder) => ({
-    label: folder.name,
-    detail: folder.uri.fsPath,
-    path: folder.uri.fsPath,
-  }));
-
-  // We're not using window.showWorkspaceFolderPick because that also includes the database source folders while
-  // we only want to include on-disk workspace folders.
-  const workspaceFolder = await window.showQuickPick(workspaceFolderOptions, {
-    title: "Select workspace folder to create extension pack in",
-  });
+  const workspaceFolder = await askForWorkspaceFolder();
   if (!workspaceFolder) {
     return undefined;
   }
 
-  const packName = await window.showInputBox(
+  const examplePackName = autoNameExtensionPack(
+    databaseItem.name,
+    databaseItem.language,
+  );
+
+  const name = await window.showInputBox(
     {
       title: "Create new extension pack",
       prompt: "Enter name of extension pack",
-      placeHolder: `e.g. ${databaseItem.name}-extensions`,
+      placeHolder: examplePackName
+        ? `e.g. ${formatPackName(examplePackName)}`
+        : "",
       validateInput: async (value: string): Promise<string | undefined> => {
-        if (!value) {
-          return "Pack name must not be empty";
+        const message = validatePackName(value);
+        if (message) {
+          return message;
         }
 
-        if (value.length > packNameLength) {
-          return `Pack name must be no longer than ${packNameLength} characters`;
+        const packName = parsePackName(value);
+        if (!packName) {
+          return "Invalid pack name";
         }
 
-        const matches = packNameRegex.exec(value);
-        if (!matches?.groups) {
-          return "Invalid package name: a pack name must contain only lowercase ASCII letters, ASCII digits, and hyphens";
-        }
-
-        const packPath = join(workspaceFolder.path, matches.groups.name);
+        const packPath = join(workspaceFolder.uri.fsPath, packName.name);
         if (await pathExists(packPath)) {
           return `A pack already exists at ${packPath}`;
         }
@@ -275,31 +198,127 @@ async function pickNewExtensionPack(
     },
     token,
   );
+  if (!name) {
+    return undefined;
+  }
+
+  const packName = parsePackName(name);
   if (!packName) {
     return undefined;
   }
 
-  const matches = packNameRegex.exec(packName);
-  if (!matches?.groups) {
-    return;
-  }
-
-  const name = matches.groups.name;
-  const packPath = join(workspaceFolder.path, name);
+  const packPath = join(workspaceFolder.uri.fsPath, packName.name);
 
   if (await pathExists(packPath)) {
     return undefined;
   }
 
+  return writeExtensionPack(packPath, packName, databaseItem.language);
+}
+
+async function autoCreateExtensionPack(
+  name: string,
+  language: string,
+  extensionPacksInfo: QlpacksInfo,
+  logger: NotificationLogger,
+): Promise<ExtensionPack | undefined> {
+  // Get the `codeQL.dataExtensions.extensionsDirectory` setting for the language
+  const userExtensionsDirectory = getExtensionsDirectory(language);
+
+  // If the setting is not set, automatically pick a suitable directory
+  const extensionsDirectory = userExtensionsDirectory
+    ? Uri.file(userExtensionsDirectory)
+    : await autoPickExtensionsDirectory();
+
+  if (!extensionsDirectory) {
+    return undefined;
+  }
+
+  // Generate the name of the extension pack
+  const packName = autoNameExtensionPack(name, language);
+  if (!packName) {
+    void showAndLogErrorMessage(
+      logger,
+      `Could not automatically name extension pack for database ${name}`,
+    );
+
+    return undefined;
+  }
+
+  // Find any existing locations of this extension pack
+  const existingExtensionPackPaths =
+    extensionPacksInfo[formatPackName(packName)];
+
+  // If there is already an extension pack with this name, use it if it is valid
+  if (existingExtensionPackPaths?.length === 1) {
+    let extensionPack: ExtensionPack;
+    try {
+      extensionPack = await readExtensionPack(existingExtensionPackPaths[0]);
+    } catch (e: unknown) {
+      void showAndLogErrorMessage(
+        logger,
+        `Could not read extension pack ${formatPackName(packName)}`,
+        {
+          fullMessage: `Could not read extension pack ${formatPackName(
+            packName,
+          )} at ${existingExtensionPackPaths[0]}: ${getErrorMessage(e)}`,
+        },
+      );
+
+      return undefined;
+    }
+
+    return extensionPack;
+  }
+
+  // If there is already an existing extension pack with this name, but it resolves
+  // to multiple paths, then we can't use it
+  if (existingExtensionPackPaths?.length > 1) {
+    void showAndLogErrorMessage(
+      logger,
+      `Extension pack ${formatPackName(packName)} resolves to multiple paths`,
+      {
+        fullMessage: `Extension pack ${formatPackName(
+          packName,
+        )} resolves to multiple paths: ${existingExtensionPackPaths.join(
+          ", ",
+        )}`,
+      },
+    );
+
+    return undefined;
+  }
+
+  const packPath = join(extensionsDirectory.fsPath, packName.name);
+
+  if (await pathExists(packPath)) {
+    void showAndLogErrorMessage(
+      logger,
+      `Directory ${packPath} already exists for extension pack ${formatPackName(
+        packName,
+      )}`,
+    );
+
+    return undefined;
+  }
+
+  return writeExtensionPack(packPath, packName, language);
+}
+
+async function writeExtensionPack(
+  packPath: string,
+  packName: ExtensionPackName,
+  language: string,
+): Promise<ExtensionPack> {
   const packYamlPath = join(packPath, "codeql-pack.yml");
 
   const extensionPack: ExtensionPack = {
     path: packPath,
     yamlPath: packYamlPath,
-    name,
+    name: formatPackName(packName),
     version: "0.0.0",
     extensionTargets: {
-      [`codeql/${databaseItem.language}-all`]: "*",
+      [`codeql/${language}-all`]: "*",
     },
     dataExtensions: ["models/**/*.yml"],
   };
@@ -318,53 +337,6 @@ async function pickNewExtensionPack(
   return extensionPack;
 }
 
-async function pickNewModelFile(
-  databaseItem: Pick<DatabaseItem, "name">,
-  extensionPack: ExtensionPack,
-  token: CancellationToken,
-) {
-  const filename = await window.showInputBox(
-    {
-      title: "Enter the name of the new model file",
-      value: `models/${databaseItem.name.replaceAll("/", ".")}.model.yml`,
-      validateInput: async (value: string): Promise<string | undefined> => {
-        if (value === "") {
-          return "File name must not be empty";
-        }
-
-        const path = resolve(extensionPack.path, value);
-
-        if (await pathExists(path)) {
-          return "File already exists";
-        }
-
-        const notInExtensionPack = relative(
-          extensionPack.path,
-          path,
-        ).startsWith("..");
-        if (notInExtensionPack) {
-          return "File must be in the extension pack";
-        }
-
-        const matchesPattern = extensionPack.dataExtensions.some((pattern) =>
-          minimatch(value, pattern, { matchBase: true }),
-        );
-        if (!matchesPattern) {
-          return `File must match one of the patterns in 'dataExtensions' in ${extensionPack.yamlPath}`;
-        }
-
-        return undefined;
-      },
-    },
-    token,
-  );
-  if (!filename) {
-    return undefined;
-  }
-
-  return resolve(extensionPack.path, filename);
-}
-
 async function readExtensionPack(path: string): Promise<ExtensionPack> {
   const qlpackPath = await getQlPackPath(path);
   if (!qlpackPath) {

extensions/ql-vscode/src/data-extensions-editor/extensions-workspace-folder.ts

@@ -0,0 +1,224 @@
+import { FileType, Uri, window, workspace, WorkspaceFolder } from "vscode";
+import { getOnDiskWorkspaceFoldersObjects } from "../common/vscode/workspace-folders";
+import { extLogger } from "../common/logging/vscode";
+import { tmpdir } from "../common/files";
+
+/**
+ * Returns the ancestors of this path in order from furthest to closest (i.e. root of filesystem to parent directory)
+ */
+function getAncestors(uri: Uri): Uri[] {
+  const ancestors: Uri[] = [];
+  let current = uri;
+  while (current.fsPath !== Uri.joinPath(current, "..").fsPath) {
+    ancestors.push(current);
+    current = Uri.joinPath(current, "..");
+  }
+
+  // The ancestors are now in order from closest to furthest, so reverse them
+  ancestors.reverse();
+
+  return ancestors;
+}
+
+async function getRootWorkspaceDirectory(): Promise<Uri | undefined> {
+  // If there is a valid workspace file, just use its directory as the directory for the extensions
+  const workspaceFile = workspace.workspaceFile;
+  if (workspaceFile?.scheme === "file") {
+    return Uri.joinPath(workspaceFile, "..");
+  }
+
+  const allWorkspaceFolders = getOnDiskWorkspaceFoldersObjects();
+
+  // Get the system temp directory and convert it to a URI so it's normalized
+  const systemTmpdir = Uri.file(tmpdir());
+
+  const workspaceFolders = allWorkspaceFolders.filter((folder) => {
+    // Never use a workspace folder that is in the system temp directory
+    return !folder.uri.fsPath.startsWith(systemTmpdir.fsPath);
+  });
+
+  // Find the common root directory of all workspace folders by finding the longest common prefix
+  const commonRoot = workspaceFolders.reduce((commonRoot, folder) => {
+    const folderUri = folder.uri;
+    const ancestors = getAncestors(folderUri);
+
+    const minLength = Math.min(commonRoot.length, ancestors.length);
+    let commonLength = 0;
+    for (let i = 0; i < minLength; i++) {
+      if (commonRoot[i].fsPath === ancestors[i].fsPath) {
+        commonLength++;
+      } else {
+        break;
+      }
+    }
+
+    return commonRoot.slice(0, commonLength);
+  }, getAncestors(workspaceFolders[0].uri));
+
+  if (commonRoot.length === 0) {
+    return await findGitFolder(workspaceFolders);
+  }
+
+  // The path closest to the workspace folders is the last element of the common root
+  const commonRootUri = commonRoot[commonRoot.length - 1];
+
+  // If we are at the root of the filesystem, we can't go up any further and there's something
+  // wrong, so just return undefined
+  if (commonRootUri.fsPath === Uri.joinPath(commonRootUri, "..").fsPath) {
+    return await findGitFolder(workspaceFolders);
+  }
+
+  return commonRootUri;
+}
+
+async function findGitFolder(
+  workspaceFolders: WorkspaceFolder[],
+): Promise<Uri | undefined> {
+  // Go through all workspace folders one-by-one and try to find the closest .git folder for each one
+  const folders = await Promise.all(
+    workspaceFolders.map(async (folder) => {
+      const ancestors = getAncestors(folder.uri);
+
+      // Reverse the ancestors so we're going from closest to furthest
+      ancestors.reverse();
+
+      const gitFoldersExists = await Promise.all(
+        ancestors.map(async (uri) => {
+          const gitFolder = Uri.joinPath(uri, ".git");
+          try {
+            const stat = await workspace.fs.stat(gitFolder);
+            // Check whether it's a directory
+            return (stat.type & FileType.Directory) !== 0;
+          } catch (e) {
+            return false;
+          }
+        }),
+      );
+
+      // Find the first ancestor that has a .git folder
+      const ancestorIndex = gitFoldersExists.findIndex((exists) => exists);
+
+      if (ancestorIndex === -1) {
+        return undefined;
+      }
+
+      return [ancestorIndex, ancestors[ancestorIndex]];
+    }),
+  );
+
+  const validFolders = folders.filter(
+    (folder): folder is [number, Uri] => folder !== undefined,
+  );
+  if (validFolders.length === 0) {
+    return undefined;
+  }
+
+  // Find the .git folder which is closest to a workspace folder
+  const closestFolder = validFolders.reduce((closestFolder, folder) => {
+    if (folder[0] < closestFolder[0]) {
+      return folder;
+    }
+    return closestFolder;
+  }, validFolders[0]);
+
+  return closestFolder?.[1];
+}
+
+/**
+ * Finds a suitable directory for extension packs to be created in. This will
+ * always be a path ending in `.github/codeql/extensions`. The parent directory
+ * will be determined heuristically based on the on-disk workspace folders.
+ *
+ * The heuristic is as follows (`.github/codeql/extensions` is added automatically unless
+ * otherwise specified):
+ * 1. If there is only 1 workspace folder, use that folder
+ * 2. If there is a workspace folder for which the path ends in `.github/codeql/extensions`, use that folder
+ *   - If there are multiple such folders, use the first one
+ *   - Does not append `.github/codeql/extensions` to the path
+ * 3. If there is a workspace file (`<basename>.code-workspace`), use the directory containing that file
+ * 4. If there is a common root directory for all workspace folders, use that directory
+ *   - Workspace folders in the system temp directory are ignored
+ *   - If the common root directory is the root of the filesystem, then it's not used
+ * 5. If there is a .git directory in any workspace folder, use the directory containing that .git directory
+ *    for which the .git directory is closest to a workspace folder
+ * 6. If none of the above apply, return `undefined`
+ */
+export async function autoPickExtensionsDirectory(): Promise<Uri | undefined> {
+  const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
+
+  // If there's only 1 workspace folder, use the `.github/codeql/extensions` directory in that folder
+  if (workspaceFolders.length === 1) {
+    return Uri.joinPath(
+      workspaceFolders[0].uri,
+      ".github",
+      "codeql",
+      "extensions",
+    );
+  }
+
+  // Now try to find a workspace folder for which the path ends in `.github/codeql/extensions`
+  const workspaceFolderForExtensions = workspaceFolders.find((folder) =>
+    // Using path instead of fsPath because path always uses forward slashes
+    folder.uri.path.endsWith(".github/codeql/extensions"),
+  );
+  if (workspaceFolderForExtensions) {
+    return workspaceFolderForExtensions.uri;
+  }
+
+  // Get the root workspace directory, i.e. the common root directory of all workspace folders
+  const rootDirectory = await getRootWorkspaceDirectory();
+  if (!rootDirectory) {
+    void extLogger.log("Unable to determine root workspace directory");
+
+    return undefined;
+  }
+
+  // We'll create a new workspace folder for the extensions in the root workspace directory
+  // at `.github/codeql/extensions`
+  const extensionsUri = Uri.joinPath(
+    rootDirectory,
+    ".github",
+    "codeql",
+    "extensions",
+  );
+
+  if (
+    !workspace.updateWorkspaceFolders(
+      workspace.workspaceFolders?.length ?? 0,
+      0,
+      {
+        name: "CodeQL Extension Packs",
+        uri: extensionsUri,
+      },
+    )
+  ) {
+    void extLogger.log(
+      `Failed to add workspace folder for extensions at ${extensionsUri.fsPath}`,
+    );
+    return undefined;
+  }
+
+  return extensionsUri;
+}
+
+export async function askForWorkspaceFolder(): Promise<
+  WorkspaceFolder | undefined
+> {
+  const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
+  const workspaceFolderOptions = workspaceFolders.map((folder) => ({
+    label: folder.name,
+    detail: folder.uri.fsPath,
+    folder,
+  }));
+
+  // We're not using window.showWorkspaceFolderPick because that also includes the database source folders while
+  // we only want to include on-disk workspace folders.
+  const workspaceFolder = await window.showQuickPick(workspaceFolderOptions, {
+    title: "Select workspace folder to create extension pack in",
+  });
+  if (!workspaceFolder) {
+    return undefined;
+  }
+
+  return workspaceFolder.folder;
+}

extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts

@@ -2,20 +2,20 @@ import { CoreCompletedQuery, QueryRunner } from "../query-server";
 import { dir } from "tmp-promise";
 import { writeFile } from "fs-extra";
 import { dump as dumpYaml } from "js-yaml";
-import {
-  getOnDiskWorkspaceFolders,
-  showAndLogExceptionWithTelemetry,
-} from "../helpers";
-import { TeeLogger } from "../common";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { extLogger } from "../common/logging/vscode";
+import { showAndLogExceptionWithTelemetry, TeeLogger } from "../common/logging";
+import { isQueryLanguage } from "../common/query-language";
 import { CancellationToken } from "vscode";
-import { CodeQLCliServer } from "../cli";
-import { DatabaseItem } from "../local-databases";
-import { ProgressCallback } from "../progress";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { DatabaseItem } from "../databases/local-databases";
+import { ProgressCallback } from "../common/vscode/progress";
 import { fetchExternalApiQueries } from "./queries";
-import { QueryResultType } from "../pure/new-messages";
+import { QueryResultType } from "../query-server/new-messages";
 import { join } from "path";
-import { redactableError } from "../pure/errors";
-import { QueryLanguage } from "../common/query-language";
+import { redactableError } from "../common/errors";
+import { telemetryListener } from "../common/vscode/telemetry";
+import { Query } from "./queries/query";
 
 export type RunQueryOptions = {
   cliServer: Pick<CodeQLCliServer, "resolveQlpacks">;
@@ -27,23 +27,37 @@ export type RunQueryOptions = {
   token: CancellationToken;
 };
 
-export async function runQuery({
-  cliServer,
-  queryRunner,
-  databaseItem,
-  queryStorageDir,
-  progress,
-  token,
-}: RunQueryOptions): Promise<CoreCompletedQuery | undefined> {
+export async function runQuery(
+  queryName: keyof Omit<Query, "dependencies">,
+  {
+    cliServer,
+    queryRunner,
+    databaseItem,
+    queryStorageDir,
+    progress,
+    token,
+  }: RunQueryOptions,
+): Promise<CoreCompletedQuery | undefined> {
   // The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
   // move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
   // This is intentionally not pretty code, as it will be removed soon.
   // For a reference of what this should do in the future, see the previous implementation in
   // https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72
 
-  const query = fetchExternalApiQueries[databaseItem.language as QueryLanguage];
+  if (!isQueryLanguage(databaseItem.language)) {
+    void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
+      redactableError`Unsupported database language ${databaseItem.language}`,
+    );
+    return;
+  }
+
+  const query = fetchExternalApiQueries[databaseItem.language];
   if (!query) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`No external API usage query found for language ${databaseItem.language}`,
     );
     return;
@@ -51,7 +65,7 @@ export async function runQuery({
 
   const queryDir = (await dir({ unsafeCleanup: true })).path;
   const queryFile = join(queryDir, "FetchExternalApis.ql");
-  await writeFile(queryFile, query.mainQuery, "utf8");
+  await writeFile(queryFile, query[queryName], "utf8");
 
   if (query.dependencies) {
     for (const [filename, contents] of Object.entries(query.dependencies)) {
@@ -78,7 +92,11 @@ export async function runQuery({
 
   const queryRun = queryRunner.createQueryRun(
     databaseItem.databaseUri.fsPath,
-    { queryPath: queryFile, quickEvalPosition: undefined },
+    {
+      queryPath: queryFile,
+      quickEvalPosition: undefined,
+      quickEvalCountOnly: false,
+    },
     false,
     getOnDiskWorkspaceFolders(),
     extensionPacks,
@@ -95,6 +113,8 @@ export async function runQuery({
 
   if (completedQuery.resultType !== QueryResultType.SUCCESS) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`External API usage query failed: ${
         completedQuery.message ?? "No message"
       }`,
@@ -117,6 +137,8 @@ export async function readQueryResults({
   const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
   if (bqrsInfo["result-sets"].length !== 1) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`,
     );
     return undefined;

extensions/ql-vscode/src/data-extensions-editor/external-api-usage.ts

@@ -1,4 +1,4 @@
-import { ResolvableLocationValue } from "../pure/bqrs-cli-types";
+import { ResolvableLocationValue } from "../common/bqrs-cli-types";
 
 export type Call = {
   label: string;
@@ -6,6 +6,10 @@ export type Call = {
 };
 
 export type ExternalApiUsage = {
+  /**
+   * Contains the name of the library containing the method declaration, e.g. `sql2o-1.6.0.jar` or `System.Runtime.dll`
+   */
+  library: string;
   /**
    * Contains the full method signature, e.g. `org.sql2o.Connection#createQuery(String)`
    */

extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts

@@ -1,59 +1,103 @@
 import { CancellationToken } from "vscode";
-import { DatabaseItem } from "../local-databases";
-import { join } from "path";
+import { DatabaseItem } from "../databases/local-databases";
+import { basename } from "path";
 import { QueryRunner } from "../query-server";
-import { CodeQLCliServer } from "../cli";
-import { TeeLogger } from "../common";
+import { CodeQLCliServer } from "../codeql-cli/cli";
+import { showAndLogExceptionWithTelemetry, TeeLogger } from "../common/logging";
+import { extLogger } from "../common/logging/vscode";
 import { extensiblePredicateDefinitions } from "./predicates";
-import { ProgressCallback } from "../progress";
-import {
-  getOnDiskWorkspaceFolders,
-  showAndLogExceptionWithTelemetry,
-} from "../helpers";
+import { ProgressCallback } from "../common/vscode/progress";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import {
   ModeledMethodType,
   ModeledMethodWithSignature,
 } from "./modeled-method";
-import { redactableError } from "../pure/errors";
-import { QueryResultType } from "../pure/new-messages";
+import { redactableError } from "../common/errors";
+import { QueryResultType } from "../query-server/new-messages";
+import { file } from "tmp-promise";
+import { writeFile } from "fs-extra";
+import { dump } from "js-yaml";
+import { qlpackOfDatabase } from "../language-support";
+import { telemetryListener } from "../common/vscode/telemetry";
 
 type FlowModelOptions = {
   cliServer: CodeQLCliServer;
   queryRunner: QueryRunner;
   queryStorageDir: string;
-  qlDir: string;
   databaseItem: DatabaseItem;
   progress: ProgressCallback;
   token: CancellationToken;
   onResults: (results: ModeledMethodWithSignature[]) => void | Promise<void>;
 };
 
+async function resolveQueries(
+  cliServer: CodeQLCliServer,
+  databaseItem: DatabaseItem,
+): Promise<string[]> {
+  const qlpacks = await qlpackOfDatabase(cliServer, databaseItem);
+
+  const packsToSearch: string[] = [];
+
+  // The CLI can handle both library packs and query packs, so search both packs in order.
+  packsToSearch.push(qlpacks.dbschemePack);
+  if (qlpacks.queryPack !== undefined) {
+    packsToSearch.push(qlpacks.queryPack);
+  }
+
+  const suiteFile = (
+    await file({
+      postfix: ".qls",
+    })
+  ).path;
+  const suiteYaml = [];
+  for (const qlpack of packsToSearch) {
+    suiteYaml.push({
+      from: qlpack,
+      queries: ".",
+      include: {
+        "tags contain": "modelgenerator",
+      },
+    });
+  }
+  await writeFile(suiteFile, dump(suiteYaml), "utf8");
+
+  return await cliServer.resolveQueriesInSuite(
+    suiteFile,
+    getOnDiskWorkspaceFolders(),
+  );
+}
+
 async function getModeledMethodsFromFlow(
   type: Exclude<ModeledMethodType, "none">,
-  queryName: string,
+  queryPath: string | undefined,
   queryStep: number,
   {
     cliServer,
     queryRunner,
     queryStorageDir,
-    qlDir,
     databaseItem,
     progress,
     token,
   }: Omit<FlowModelOptions, "onResults">,
 ): Promise<ModeledMethodWithSignature[]> {
-  const definition = extensiblePredicateDefinitions[type];
+  if (queryPath === undefined) {
+    void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
+      redactableError`Failed to find ${type} query`,
+    );
+    return [];
+  }
 
-  const query = join(
-    qlDir,
-    databaseItem.language,
-    "ql/src/utils/modelgenerator",
-    queryName,
-  );
+  const definition = extensiblePredicateDefinitions[type];
 
   const queryRun = queryRunner.createQueryRun(
     databaseItem.databaseUri.fsPath,
-    { queryPath: query, quickEvalPosition: undefined },
+    {
+      queryPath,
+      quickEvalPosition: undefined,
+      quickEvalCountOnly: false,
+    },
     false,
     getOnDiskWorkspaceFolders(),
     undefined,
@@ -74,7 +118,9 @@ async function getModeledMethodsFromFlow(
   );
   if (queryResult.resultType !== QueryResultType.SUCCESS) {
     void showAndLogExceptionWithTelemetry(
-      redactableError`Failed to run ${queryName} query: ${
+      extLogger,
+      telemetryListener,
+      redactableError`Failed to run ${basename(queryPath)} query: ${
         queryResult.message ?? "No message"
       }`,
     );
@@ -86,7 +132,11 @@ async function getModeledMethodsFromFlow(
   const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
   if (bqrsInfo["result-sets"].length !== 1) {
     void showAndLogExceptionWithTelemetry(
-      redactableError`Expected exactly one result set, got ${bqrsInfo["result-sets"].length} for ${queryName}`,
+      extLogger,
+      telemetryListener,
+      redactableError`Expected exactly one result set, got ${
+        bqrsInfo["result-sets"].length
+      } for ${basename(queryPath)}`,
     );
   }
 
@@ -112,9 +162,16 @@ export async function generateFlowModel({
   onResults,
   ...options
 }: FlowModelOptions) {
+  const queries = await resolveQueries(options.cliServer, options.databaseItem);
+
+  const queriesByBasename: Record<string, string> = {};
+  for (const query of queries) {
+    queriesByBasename[basename(query)] = query;
+  }
+
   const summaryResults = await getModeledMethodsFromFlow(
     "summary",
-    "CaptureSummaryModels.ql",
+    queriesByBasename["CaptureSummaryModels.ql"],
     0,
     options,
   );
@@ -124,7 +181,7 @@ export async function generateFlowModel({
 
   const sinkResults = await getModeledMethodsFromFlow(
     "sink",
-    "CaptureSinkModels.ql",
+    queriesByBasename["CaptureSinkModels.ql"],
     1,
     options,
   );
@@ -134,7 +191,7 @@ export async function generateFlowModel({
 
   const sourceResults = await getModeledMethodsFromFlow(
     "source",
-    "CaptureSourceModels.ql",
+    queriesByBasename["CaptureSourceModels.ql"],
     2,
     options,
   );
@@ -144,7 +201,7 @@ export async function generateFlowModel({
 
   const neutralResults = await getModeledMethodsFromFlow(
     "neutral",
-    "CaptureNeutralModels.ql",
+    queriesByBasename["CaptureNeutralModels.ql"],
     3,
     options,
   );

extensions/ql-vscode/src/data-extensions-editor/modeled-method.ts

@@ -5,11 +5,24 @@ export type ModeledMethodType =
   | "summary"
   | "neutral";
 
+export type Provenance =
+  // Generated by the dataflow model
+  | "df-generated"
+  // Generated by the dataflow model and manually edited
+  | "df-manual"
+  // Generated by the auto-model
+  | "ai-generated"
+  // Generated by the auto-model and manually edited
+  | "ai-manual"
+  // Entered by the user in the editor manually
+  | "manual";
+
 export type ModeledMethod = {
   type: ModeledMethodType;
   input: string;
   output: string;
   kind: string;
+  provenance: Provenance;
 };
 
 export type ModeledMethodWithSignature = {

extensions/ql-vscode/src/data-extensions-editor/predicates.ts

@@ -3,6 +3,7 @@ import {
   ModeledMethod,
   ModeledMethodType,
   ModeledMethodWithSignature,
+  Provenance,
 } from "./modeled-method";
 
 export type ExternalApiUsageByType = {
@@ -43,7 +44,7 @@ export const extensiblePredicateDefinitions: Record<
       "",
       method.modeledMethod.output,
       method.modeledMethod.kind,
-      "manual",
+      method.modeledMethod.provenance,
     ],
     readModeledMethod: (row) => ({
       signature: readRowToMethod(row),
@@ -52,6 +53,7 @@ export const extensiblePredicateDefinitions: Record<
         input: "",
         output: row[6] as string,
         kind: row[7] as string,
+        provenance: row[8] as Provenance,
       },
     }),
     supportedKinds: ["remote"],
@@ -71,7 +73,7 @@ export const extensiblePredicateDefinitions: Record<
       "",
       method.modeledMethod.input,
       method.modeledMethod.kind,
-      "manual",
+      method.modeledMethod.provenance,
     ],
     readModeledMethod: (row) => ({
       signature: readRowToMethod(row),
@@ -80,6 +82,7 @@ export const extensiblePredicateDefinitions: Record<
         input: row[6] as string,
         output: "",
         kind: row[7] as string,
+        provenance: row[8] as Provenance,
       },
     }),
     supportedKinds: ["sql", "xss", "logging"],
@@ -100,7 +103,7 @@ export const extensiblePredicateDefinitions: Record<
       method.modeledMethod.input,
       method.modeledMethod.output,
       method.modeledMethod.kind,
-      "manual",
+      method.modeledMethod.provenance,
     ],
     readModeledMethod: (row) => ({
       signature: readRowToMethod(row),
@@ -109,6 +112,7 @@ export const extensiblePredicateDefinitions: Record<
         input: row[6] as string,
         output: row[7] as string,
         kind: row[8] as string,
+        provenance: row[9] as Provenance,
       },
     }),
     supportedKinds: ["taint", "value"],
@@ -116,14 +120,15 @@ export const extensiblePredicateDefinitions: Record<
   neutral: {
     extensiblePredicate: "neutralModel",
     // extensible predicate neutralModel(
-    //   string package, string type, string name, string signature, string provenance
+    //   string package, string type, string name, string signature, string kind, string provenance
     // );
     generateMethodDefinition: (method) => [
       method.externalApiUsage.packageName,
       method.externalApiUsage.typeName,
       method.externalApiUsage.methodName,
       method.externalApiUsage.methodParameters,
-      "manual",
+      method.modeledMethod.kind,
+      method.modeledMethod.provenance,
     ],
     readModeledMethod: (row) => ({
       signature: `${row[0]}.${row[1]}#${row[2]}${row[3]}`,
@@ -131,8 +136,10 @@ export const extensiblePredicateDefinitions: Record<
         type: "neutral",
         input: "",
         output: "",
-        kind: "",
+        kind: row[4] as string,
+        provenance: row[5] as Provenance,
       },
     }),
+    supportedKinds: ["summary", "source", "sink"],
   },
 };

extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts

@@ -1,37 +1,60 @@
 import { Query } from "./query";
 
 export const fetchExternalApisQuery: Query = {
-  mainQuery: `/**
+  applicationModeQuery: `/**
  * @name Usage of APIs coming from external libraries
  * @description A list of 3rd party APIs used in the codebase.
  * @tags telemetry
+ * @kind problem
  * @id cs/telemetry/fetch-external-apis
  */
 
- import csharp
- import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
- import ExternalApi
- 
- private Call aUsage(ExternalApi api) {
-   result.getTarget().getUnboundDeclaration() = api
- }
- 
- private boolean isSupported(ExternalApi api) {
-   api.isSupported() and result = true
-   or
-   not api.isSupported() and
-   result = false
- }
- 
- from ExternalApi api, string apiName, boolean supported, Call usage
- where
-   apiName = api.getApiName() and
-   supported = isSupported(api) and
-   usage = aUsage(api)
- select apiName, supported, usage
+private import csharp
+private import AutomodelVsCode
+
+class ExternalApi extends CallableMethod {
+  ExternalApi() {
+    this.isUnboundDeclaration() and
+    this.fromLibrary() and
+    this.(Modifiable).isEffectivelyPublic()
+  }
+}
+
+private Call aUsage(ExternalApi api) { result.getTarget().getUnboundDeclaration() = api }
+
+from ExternalApi api, string apiName, boolean supported, Call usage
+where
+  apiName = api.getApiName() and
+  supported = isSupported(api) and
+  usage = aUsage(api)
+select usage, apiName, supported.toString(), "supported", api.getFile().getBaseName(), "library"
+`,
+  frameworkModeQuery: `/**
+ * @name Public methods
+ * @description A list of APIs callable by consumers. Excludes test and generated code.
+ * @tags telemetry
+ * @kind problem
+ * @id cs/telemetry/fetch-public-methods
+ */
+
+private import csharp
+private import dotnet
+private import semmle.code.csharp.frameworks.Test
+private import AutomodelVsCode
+
+class PublicMethod extends CallableMethod {
+  PublicMethod() { this.fromSource() and not this.getFile() instanceof TestFile }
+}
+
+from PublicMethod publicMethod, string apiName, boolean supported
+where
+  apiName = publicMethod.getApiName() and
+  supported = isSupported(publicMethod)
+select publicMethod, apiName, supported.toString(), "supported",
+  publicMethod.getFile().getBaseName(), "library"
 `,
   dependencies: {
-    "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */
+    "AutomodelVsCode.qll": `/** Provides classes and predicates related to handling APIs for the VS Code extension. */
 
 private import csharp
 private import dotnet
@@ -61,22 +84,31 @@ class TestLibrary extends RefType {
 }
 
 /** Holds if the given callable is not worth supporting. */
-private predicate isUninteresting(DotNet::Callable c) {
+private predicate isUninteresting(DotNet::Declaration c) {
   c.getDeclaringType() instanceof TestLibrary or
-  c.(Constructor).isParameterless()
+  c.(Constructor).isParameterless() or
+  c.getDeclaringType() instanceof AnonymousClass
 }
 
 /**
- * An external API from either the C# Standard Library or a 3rd party library.
+ * An callable method from either the C# Standard Library, a 3rd party library, or from the source.
  */
-class ExternalApi extends DotNet::Callable {
-  ExternalApi() {
-    this.isUnboundDeclaration() and
-    this.fromLibrary() and
+class CallableMethod extends DotNet::Declaration {
+  CallableMethod() {
     this.(Modifiable).isEffectivelyPublic() and
     not isUninteresting(this)
   }
 
+  /**
+   * Gets the unbound type, name and parameter types of this API.
+   */
+  bindingset[this]
+  private string getSignature() {
+    result =
+      nestedName(this.getDeclaringType().getUnboundDeclaration()) + "." + this.getName() + "(" +
+        parameterQualifiedTypeNamesToString(this) + ")"
+  }
+
   /**
    * Gets the namespace of this API.
    */
@@ -87,8 +119,7 @@ class ExternalApi extends DotNet::Callable {
    * Gets the namespace and signature of this API.
    */
   bindingset[this]
-  string getApiName() { result = this.getNamespace() + "." + this.getDeclaringType().getUnboundDeclaration() + "#" + this.getName() + "(" +
-  parameterQualifiedTypeNamesToString(this) + ")" }
+  string getApiName() { result = this.getNamespace() + "#" + this.getSignature() }
 
   /** Gets a node that is an input to a call to this API. */
   private ArgumentNode getAnInput() {
@@ -142,47 +173,26 @@ class ExternalApi extends DotNet::Callable {
   }
 }
 
-/**
- * Gets the limit for the number of results produced by a telemetry query.
- */
-int resultLimit() { result = 1000 }
+boolean isSupported(CallableMethod callableMethod) {
+  callableMethod.isSupported() and result = true
+  or
+  not callableMethod.isSupported() and
+  result = false
+}
 
 /**
- * Holds if it is relevant to count usages of "api".
+ * Gets the nested name of the declaration.
+ *
+ * If the declaration is not a nested type, the result is the same as \`getName()\`.
+ * Otherwise the name of the nested type is prefixed with a \`+\` and appended to
+ * the name of the enclosing type, which might be a nested type as well.
  */
-signature predicate relevantApi(ExternalApi api);
-
-/**
- * Given a predicate to count relevant API usages, this module provides a predicate
- * for restricting the number or returned results based on a certain limit.
- */
-module Results<relevantApi/1 getRelevantUsages> {
-  private int getUsages(string apiName) {
-    result =
-      strictcount(Call c, ExternalApi api |
-        c.getTarget().getUnboundDeclaration() = api and
-        apiName = api.getApiName() and
-        getRelevantUsages(api)
-      )
-  }
-
-  private int getOrder(string apiName) {
-    apiName =
-      rank[result](string name, int usages |
-        usages = getUsages(name)
-      |
-        name order by usages desc, name
-      )
-  }
-
-  /**
-   * Holds if there exists an API with "apiName" that is being used "usages" times
-   * and if it is in the top results (guarded by resultLimit).
-   */
-  predicate restrict(string apiName, int usages) {
-    usages = getUsages(apiName) and
-    getOrder(apiName) <= resultLimit()
-  }
+private string nestedName(Declaration declaration) {
+  not exists(declaration.getDeclaringType().getUnboundDeclaration()) and
+  result = declaration.getName()
+  or
+  nestedName(declaration.getDeclaringType().getUnboundDeclaration()) + "+" + declaration.getName() =
+    result
 }
 `,
   },

extensions/ql-vscode/src/data-extensions-editor/queries/java.ts

@@ -1,37 +1,55 @@
 import { Query } from "./query";
 
 export const fetchExternalApisQuery: Query = {
-  mainQuery: `/**
+  applicationModeQuery: `/**
  * @name Usage of APIs coming from external libraries
  * @description A list of 3rd party APIs used in the codebase. Excludes test and generated code.
  * @tags telemetry
+ * @kind problem
  * @id java/telemetry/fetch-external-apis
  */
 
 import java
-import semmle.code.java.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
-import ExternalApi
+import AutomodelVsCode
+
+class ExternalApi extends CallableMethod {
+  ExternalApi() { not this.fromSource() }
+}
 
 private Call aUsage(ExternalApi api) {
   result.getCallee().getSourceDeclaration() = api and
   not result.getFile() instanceof GeneratedFile
 }
 
-private boolean isSupported(ExternalApi api) {
-  api.isSupported() and result = true
-  or
-  not api.isSupported() and result = false
-}
-
-from ExternalApi api, string apiName, boolean supported, Call usage
+from ExternalApi externalApi, string apiName, boolean supported, Call usage
 where
-  apiName = api.getApiName() and
-  supported = isSupported(api) and
-  usage = aUsage(api)
-select apiName, supported, usage
+  apiName = externalApi.getApiName() and
+  supported = isSupported(externalApi) and
+  usage = aUsage(externalApi)
+select usage, apiName, supported.toString(), "supported", externalApi.jarContainer(), "library"
+`,
+  frameworkModeQuery: `/**
+ * @name Public methods
+ * @description A list of APIs callable by consumers. Excludes test and generated code.
+ * @tags telemetry
+ * @kind problem
+ * @id java/telemetry/fetch-public-methods
+ */
+
+import java
+import AutomodelVsCode
+
+class PublicMethodFromSource extends CallableMethod, ModelApi { }
+
+from PublicMethodFromSource publicMethod, string apiName, boolean supported
+where
+  apiName = publicMethod.getApiName() and
+  supported = isSupported(publicMethod)
+select publicMethod, apiName, supported.toString(), "supported",
+  publicMethod.getCompilationUnit().getParentContainer().getBaseName(), "library"
 `,
   dependencies: {
-    "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */
+    "AutomodelVsCode.qll": `/** Provides classes and predicates related to handling APIs for the VS Code extension. */
 
 private import java
 private import semmle.code.java.dataflow.DataFlow
@@ -41,57 +59,42 @@ private import semmle.code.java.dataflow.FlowSummary
 private import semmle.code.java.dataflow.internal.DataFlowPrivate
 private import semmle.code.java.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
 private import semmle.code.java.dataflow.TaintTracking
+private import semmle.code.java.dataflow.internal.ModelExclusions
 
-pragma[nomagic]
-private predicate isTestPackage(Package p) {
-  p.getName()
-      .matches([
-          "org.junit%", "junit.%", "org.mockito%", "org.assertj%",
-          "com.github.tomakehurst.wiremock%", "org.hamcrest%", "org.springframework.test.%",
-          "org.springframework.mock.%", "org.springframework.boot.test.%", "reactor.test%",
-          "org.xmlunit%", "org.testcontainers.%", "org.opentest4j%", "org.mockserver%",
-          "org.powermock%", "org.skyscreamer.jsonassert%", "org.rnorth.visibleassertions",
-          "org.openqa.selenium%", "com.gargoylesoftware.htmlunit%", "org.jboss.arquillian.testng%",
-          "org.testng%"
-        ])
-}
-
-/**
- * A test library.
- */
-private class TestLibrary extends RefType {
-  TestLibrary() { isTestPackage(this.getPackage()) }
-}
-
-private string containerAsJar(Container container) {
-  if container instanceof JarFile then result = container.getBaseName() else result = "rt.jar"
-}
-
-/** Holds if the given callable is not worth supporting. */
+/** Holds if the given callable/method is not worth supporting. */
 private predicate isUninteresting(Callable c) {
   c.getDeclaringType() instanceof TestLibrary or
-  c.(Constructor).isParameterless()
+  c.(Constructor).isParameterless() or
+  c.getDeclaringType() instanceof AnonymousClass
 }
 
 /**
- * An external API from either the Standard Library or a 3rd party library.
+ * A callable method from either the Standard Library, a 3rd party library or from the source.
  */
-class ExternalApi extends Callable {
-  ExternalApi() { not this.fromSource() and not isUninteresting(this) }
+class CallableMethod extends Method {
+  CallableMethod() { not isUninteresting(this) }
 
   /**
    * Gets information about the external API in the form expected by the MaD modeling framework.
    */
   string getApiName() {
     result =
-      this.getDeclaringType().getPackage() + "." + this.getDeclaringType().getSourceDeclaration() +
-        "#" + this.getName() + paramsString(this)
+      this.getDeclaringType().getPackage() + "." + this.getDeclaringType().nestedName() + "#" +
+        this.getName() + paramsString(this)
+  }
+
+  private string getJarName() {
+    result = this.getCompilationUnit().getParentContainer*().(JarFile).getBaseName()
   }
 
   /**
    * Gets the jar file containing this API. Normalizes the Java Runtime to "rt.jar" despite the presence of modules.
    */
-  string jarContainer() { result = containerAsJar(this.getCompilationUnit().getParentContainer*()) }
+  string jarContainer() {
+    result = this.getJarName()
+    or
+    not exists(this.getJarName()) and result = "rt.jar"
+  }
 
   /** Gets a node that is an input to a call to this API. */
   private DataFlow::Node getAnInput() {
@@ -138,50 +141,85 @@ class ExternalApi extends Callable {
   }
 }
 
-/** DEPRECATED: Alias for ExternalApi */
-deprecated class ExternalAPI = ExternalApi;
+boolean isSupported(CallableMethod method) {
+  method.isSupported() and result = true
+  or
+  not method.isSupported() and result = false
+}
+
+// The below is a copy of https://github.com/github/codeql/blob/249f9f863db1e94e3c46ca85b49fb0ec32f8ca92/java/ql/lib/semmle/code/java/dataflow/internal/ModelExclusions.qll
+// to avoid the use of internal modules.
+/** Holds if the given package \`p\` is a test package. */
+pragma[nomagic]
+private predicate isTestPackage(Package p) {
+  p.getName()
+      .matches([
+          "org.junit%", "junit.%", "org.mockito%", "org.assertj%",
+          "com.github.tomakehurst.wiremock%", "org.hamcrest%", "org.springframework.test.%",
+          "org.springframework.mock.%", "org.springframework.boot.test.%", "reactor.test%",
+          "org.xmlunit%", "org.testcontainers.%", "org.opentest4j%", "org.mockserver%",
+          "org.powermock%", "org.skyscreamer.jsonassert%", "org.rnorth.visibleassertions",
+          "org.openqa.selenium%", "com.gargoylesoftware.htmlunit%", "org.jboss.arquillian.testng%",
+          "org.testng%"
+        ])
+}
 
 /**
- * Gets the limit for the number of results produced by a telemetry query.
+ * A test library.
  */
-int resultLimit() { result = 1000 }
+class TestLibrary extends RefType {
+  TestLibrary() { isTestPackage(this.getPackage()) }
+}
+
+/** Holds if the given file is a test file. */
+private predicate isInTestFile(File file) {
+  file.getAbsolutePath().matches(["%/test/%", "%/guava-tests/%", "%/guava-testlib/%"]) and
+  not file.getAbsolutePath().matches("%/ql/test/%") // allows our test cases to work
+}
+
+/** Holds if the given compilation unit's package is a JDK internal. */
+private predicate isJdkInternal(CompilationUnit cu) {
+  cu.getPackage().getName().matches("org.graalvm%") or
+  cu.getPackage().getName().matches("com.sun%") or
+  cu.getPackage().getName().matches("sun%") or
+  cu.getPackage().getName().matches("jdk%") or
+  cu.getPackage().getName().matches("java2d%") or
+  cu.getPackage().getName().matches("build.tools%") or
+  cu.getPackage().getName().matches("propertiesparser%") or
+  cu.getPackage().getName().matches("org.jcp%") or
+  cu.getPackage().getName().matches("org.w3c%") or
+  cu.getPackage().getName().matches("org.ietf.jgss%") or
+  cu.getPackage().getName().matches("org.xml.sax%") or
+  cu.getPackage().getName().matches("com.oracle%") or
+  cu.getPackage().getName().matches("org.omg%") or
+  cu.getPackage().getName().matches("org.relaxng%") or
+  cu.getPackage().getName() = "compileproperties" or
+  cu.getPackage().getName() = "transparentruler" or
+  cu.getPackage().getName() = "genstubs" or
+  cu.getPackage().getName() = "netscape.javascript" or
+  cu.getPackage().getName() = ""
+}
+
+/** Holds if the given callable is not worth modeling. */
+predicate isUninterestingForModels(Callable c) {
+  isInTestFile(c.getCompilationUnit().getFile()) or
+  isJdkInternal(c.getCompilationUnit()) or
+  c instanceof MainMethod or
+  c instanceof StaticInitializer or
+  exists(FunctionalExpr funcExpr | c = funcExpr.asMethod()) or
+  c.getDeclaringType() instanceof TestLibrary or
+  c.(Constructor).isParameterless()
+}
 
 /**
- * Holds if it is relevant to count usages of \`api\`.
+ * A class that represents all callables for which we might be
+ * interested in having a MaD model.
  */
-signature predicate relevantApi(ExternalApi api);
-
-/**
- * Given a predicate to count relevant API usages, this module provides a predicate
- * for restricting the number or returned results based on a certain limit.
- */
-module Results<relevantApi/1 getRelevantUsages> {
-  private int getUsages(string apiName) {
-    result =
-      strictcount(Call c, ExternalApi api |
-        c.getCallee().getSourceDeclaration() = api and
-        not c.getFile() instanceof GeneratedFile and
-        apiName = api.getApiName() and
-        getRelevantUsages(api)
-      )
-  }
-
-  private int getOrder(string apiInfo) {
-    apiInfo =
-      rank[result](string info, int usages |
-        usages = getUsages(info)
-      |
-        info order by usages desc, info
-      )
-  }
-
-  /**
-   * Holds if there exists an API with \`apiName\` that is being used \`usages\` times
-   * and if it is in the top results (guarded by resultLimit).
-   */
-  predicate restrict(string apiName, int usages) {
-    usages = getUsages(apiName) and
-    getOrder(apiName) <= resultLimit()
+class ModelApi extends SrcCallable {
+  ModelApi() {
+    this.fromSource() and
+    this.isEffectivelyPublic() and
+    not isUninterestingForModels(this)
   }
 }
 `,

extensions/ql-vscode/src/data-extensions-editor/queries/query.ts

@@ -1,5 +1,29 @@
 export type Query = {
-  mainQuery: string;
+  /**
+   * The application query.
+   *
+   * It should select all usages of external APIs, and return the following result pattern:
+   * - usage: the usage of the external API. This is an entity.
+   * - apiName: the name of the external API. This is a string.
+   * - supported: whether the external API is modeled. This should be a string representation of a boolean to satify the result pattern for a problem query.
+   * - "supported": a string literal. This is required to make the query a valid problem query.
+   * - libraryName: the name of the library that contains the external API. This is a string and usually the basename of a file.
+   * - "library": a string literal. This is required to make the query a valid problem query.
+   */
+  applicationModeQuery: string;
+  /**
+   * The framework query.
+   *
+   * It should select all methods that are callable by applications, which is usually all public methods (and constructors).
+   * The result pattern should be as follows:
+   * - method: the method that is callable by applications. This is an entity.
+   * - apiName: the name of the external API. This is a string.
+   * - supported: whether this method is modeled. This should be a string representation of a boolean to satify the result pattern for a problem query.
+   * - "supported": a string literal. This is required to make the query a valid problem query.
+   * - libraryName: an arbitrary string. This is required to make it match the structure of the application query.
+   * - "library": a string literal. This is required to make the query a valid problem query.
+   */
+  frameworkModeQuery: string;
   dependencies?: {
     [filename: string]: string;
   };

extensions/ql-vscode/src/data-extensions-editor/shared/extension-pack.ts

@@ -8,8 +8,3 @@ export interface ExtensionPack {
   extensionTargets: Record<string, string>;
   dataExtensions: string[];
 }
-
-export interface ExtensionPackModelFile {
-  filename: string;
-  extensionPack: ExtensionPack;
-}

extensions/ql-vscode/src/data-extensions-editor/shared/mode.ts

@@ -0,0 +1,4 @@
+export enum Mode {
+  Application = "application",
+  Framework = "framework",
+}

extensions/ql-vscode/src/data-extensions-editor/shared/modeled-percentage.ts

@@ -1,4 +1,4 @@
-import { ExternalApiUsage } from "../../data-extensions-editor/external-api-usage";
+import { ExternalApiUsage } from "../external-api-usage";
 
 export function calculateModeledPercentage(
   externalApiUsages: Array<Pick<ExternalApiUsage, "supported">>,

extensions/ql-vscode/src/data-extensions-editor/shared/sorting.ts

@@ -0,0 +1,88 @@
+import { ExternalApiUsage } from "../external-api-usage";
+import { Mode } from "./mode";
+import { calculateModeledPercentage } from "./modeled-percentage";
+
+export function groupMethods(
+  externalApiUsages: ExternalApiUsage[],
+  mode: Mode,
+): Record<string, ExternalApiUsage[]> {
+  const groupedByLibrary: Record<string, ExternalApiUsage[]> = {};
+
+  for (const externalApiUsage of externalApiUsages) {
+    // Group by package if using framework mode
+    const key =
+      mode === Mode.Framework
+        ? externalApiUsage.packageName
+        : externalApiUsage.library;
+
+    groupedByLibrary[key] ??= [];
+    groupedByLibrary[key].push(externalApiUsage);
+  }
+
+  return groupedByLibrary;
+}
+
+export function sortGroupNames(
+  methods: Record<string, ExternalApiUsage[]>,
+): string[] {
+  return Object.keys(methods).sort((a, b) =>
+    compareGroups(methods[a], a, methods[b], b),
+  );
+}
+
+export function sortMethods(
+  externalApiUsages: ExternalApiUsage[],
+): ExternalApiUsage[] {
+  const sortedExternalApiUsages = [...externalApiUsages];
+  sortedExternalApiUsages.sort((a, b) => compareMethod(a, b));
+  return sortedExternalApiUsages;
+}
+
+function compareGroups(
+  a: ExternalApiUsage[],
+  aName: string,
+  b: ExternalApiUsage[],
+  bName: string,
+): number {
+  const supportedPercentageA = calculateModeledPercentage(a);
+  const supportedPercentageB = calculateModeledPercentage(b);
+
+  // Sort first by supported percentage ascending
+  if (supportedPercentageA > supportedPercentageB) {
+    return 1;
+  }
+  if (supportedPercentageA < supportedPercentageB) {
+    return -1;
+  }
+
+  const numberOfUsagesA = a.reduce((acc, curr) => acc + curr.usages.length, 0);
+  const numberOfUsagesB = b.reduce((acc, curr) => acc + curr.usages.length, 0);
+
+  // If the number of usages is equal, sort by number of methods descending
+  if (numberOfUsagesA === numberOfUsagesB) {
+    const numberOfMethodsA = a.length;
+    const numberOfMethodsB = b.length;
+
+    // If the number of methods is equal, sort by library name ascending
+    if (numberOfMethodsA === numberOfMethodsB) {
+      return aName.localeCompare(bName);
+    }
+
+    return numberOfMethodsB - numberOfMethodsA;
+  }
+
+  // Then sort by number of usages descending
+  return numberOfUsagesB - numberOfUsagesA;
+}
+
+function compareMethod(a: ExternalApiUsage, b: ExternalApiUsage): number {
+  // Sort first by supported, putting unmodeled methods first.
+  if (a.supported && !b.supported) {
+    return 1;
+  }
+  if (!a.supported && b.supported) {
+    return -1;
+  }
+  // Then sort by number of usages descending
+  return b.usages.length - a.usages.length;
+}