Merge pull request #2612 from github/v1.8.8

Release prep for v1.8.8

.github/codeql/queries/assert-no-vscode-dependency.ql

@@ -2,9 +2,8 @@
  * @name Unwanted dependency on vscode API
  * @kind path-problem
  * @problem.severity error
- * @id vscode-codeql/assert-pure
- * @description The modules stored under `pure` and tested in the `pure-tests`
- * are intended to be "pure".
+ * @id vscode-codeql/assert-no-vscode-dependency
+ * @description The modules stored under `common` should not have dependencies on the VS Code API
  */
 
 import javascript
@@ -13,12 +12,9 @@ class VSCodeImport extends ImportDeclaration {
   VSCodeImport() { this.getImportedPath().getValue() = "vscode" }
 }
 
-class PureFile extends File {
-  PureFile() {
-    (
-      this.getRelativePath().regexpMatch(".*/src/pure/.*") or
-      this.getRelativePath().regexpMatch(".*/src/common/.*")
-    ) and
+class CommonFile extends File {
+  CommonFile() {
+    this.getRelativePath().regexpMatch(".*/src/common/.*") and
     not this.getRelativePath().regexpMatch(".*/vscode/.*")
   }
 }
@@ -34,7 +30,8 @@ query predicate edges(AstNode a, AstNode b) {
 
 from Module m, VSCodeImport v
 where
-  m.getFile() instanceof PureFile and
+  m.getFile() instanceof CommonFile and
   edges+(m, v)
 select m, m, v,
-  "This module is not pure: it has a transitive dependency on the vscode API imported $@", v, "here"
+  "This module is in the 'common' directory but has a transitive dependency on the vscode API imported $@",
+  v, "here"

.github/codeql/queries/qlpack.yml

@@ -1,3 +1,4 @@
 name: vscode-codeql-custom-queries-javascript
 version: 0.0.0
-libraryPathDependencies: codeql-javascript
+dependencies:
+  codeql/javascript-queries: "*"

.github/workflows/main.yml

@@ -209,6 +209,7 @@ jobs:
     name: CLI Test
     runs-on: ${{ matrix.os }}
     needs: [find-nightly, set-matrix]
+    timeout-minutes: 30
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]

CODEOWNERS

@@ -2,3 +2,4 @@
 **/variant-analysis/ @github/code-scanning-secexp-reviewers
 **/databases/ @github/code-scanning-secexp-reviewers
 **/data-extensions-editor/ @github/code-scanning-secexp-reviewers
+**/queries-panel/ @github/code-scanning-secexp-reviewers

docs/test-plan.md

@@ -378,6 +378,7 @@ This requires running a MRVA query and seeing the results view.
 1. Make changes via config file (ensure JSON schema is helping out)
 1. Close and re-open VS Code (ensure lists are there)
 1. Collapse/expand tree nodes
+1. Create a new list, right click and select "Add repositories with GitHub Code Search". Enter the language 'python' and the query "UserMixin". This should show a rate limiting notification after a while but eventually populate the list with roughly 770 items.
 
 Error cases that trigger an error notification:
 

extensions/ql-vscode/.eslintrc.js

@@ -65,10 +65,6 @@ const baseConfig = {
     "import/no-namespace": "off",
     "import/no-unresolved": "off",
     "import/no-webpack-loader-syntax": "off",
-    "jsx-a11y/anchor-is-valid": "off",
-    "jsx-a11y/no-noninteractive-element-interactions": "off",
-    "jsx-a11y/no-static-element-interactions": "off",
-    "jsx-a11y/click-events-have-key-events": "off",
     "no-invalid-this": "off",
     "no-fallthrough": "off",
     "no-console": "off",

extensions/ql-vscode/CHANGELOG.md

@@ -1,5 +1,20 @@
 # CodeQL for Visual Studio Code: Changelog
 
+## 1.8.8 - 17 July 2023
+
+- Remove support for CodeQL CLI versions older than 2.9.4. [#2610](https://github.com/github/vscode-codeql/pull/2610)
+- Implement syntax highlighting for the `additional` and `default` keywords. [#2609](https://github.com/github/vscode-codeql/pull/2609)
+
+## 1.8.7 - 29 June 2023
+
+- Show a run button on the file tab for query files, that will start a local query. This button will only show when a local database is selected in the extension. [#2544](https://github.com/github/vscode-codeql/pull/2544)
+- Add a `CodeQL: Quick Evaluation Count` command to generate the count summary statistics of the results set
+  without spending the time to compute locations and strings. [#2475](https://github.com/github/vscode-codeql/pull/2475)
+
+## 1.8.6 - 14 June 2023
+
+- Add repositories to a variant analysis list with GitHub Code Search. [#2439](https://github.com/github/vscode-codeql/pull/2439) and [#2476](https://github.com/github/vscode-codeql/pull/2476)
+
 ## 1.8.5 - 6 June 2023
 
 - Add settings `codeQL.variantAnalysis.defaultResultsFilter` and `codeQL.variantAnalysis.defaultResultsSort` for configuring how variant analysis results are filtered and sorted in the results view. The default is to show all repositories, and to sort by the number of results. [#2392](https://github.com/github/vscode-codeql/pull/2392)
@@ -7,7 +22,7 @@
 - Fix bug where the `CodeQL: Compare Query` command did not work for comparing quick-eval queries. [#2422](https://github.com/github/vscode-codeql/pull/2422)
 - Update text of copy and export buttons in variant analysis results view to clarify that they only copy/export the selected/filtered results. [#2427](https://github.com/github/vscode-codeql/pull/2427)
 - Add warning when using unsupported CodeQL CLI version. [#2428](https://github.com/github/vscode-codeql/pull/2428)
-- Retry MRVA results download if connection times out. [#2440](https://github.com/github/vscode-codeql/pull/2440)
+- Retry variant analysis results download if connection times out. [#2440](https://github.com/github/vscode-codeql/pull/2440)
 
 ## 1.8.4 - 3 May 2023
 

extensions/ql-vscode/gulpfile.ts/webpack.config.ts

@@ -62,18 +62,10 @@ export const config: webpack.Configuration = {
       },
       {
         test: /\.(woff(2)?|ttf|eot)$/,
-        use: [
-          {
-            loader: "file-loader",
-            options: {
-              name: "[name].[ext]",
-              outputPath: "fonts/",
-              // We need this to make Webpack use the correct path for the fonts.
-              // Without this, the CSS file will use `url([object Module])`
-              esModule: false,
-            },
-          },
-        ],
+        type: "asset/resource",
+        generator: {
+          filename: "fonts/[hash][ext][query]",
+        },
       },
     ],
   },

extensions/ql-vscode/media/dark/symbol-misc.svg

@@ -0,0 +1 @@
+<svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 2h8v4c.341.035.677.112 1 .23V1H3v8.48l1-1.75V2zm2.14 8L5 8 4 9.75 3.29 11 1 15h8l-2.29-4-.57-1zm-3.42 4l1.72-3L5 10l.56 1 1.72 3H2.72zm6.836-6.41a3.5 3.5 0 1 1 3.888 5.82 3.5 3.5 0 0 1-3.888-5.82zm.555 4.989a2.5 2.5 0 1 0 2.778-4.157 2.5 2.5 0 0 0-2.778 4.157z" fill="#C5C5C5"/></svg>

extensions/ql-vscode/media/light/symbol-misc.svg

@@ -0,0 +1 @@
+<svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M4 2h8v4c.341.035.677.112 1 .23V1H3v8.48l1-1.75V2zm2.14 8L5 8 4 9.75 3.29 11 1 15h8l-2.29-4-.57-1zm-3.42 4l1.72-3L5 10l.56 1 1.72 3H2.72zm6.836-6.41a3.5 3.5 0 1 1 3.888 5.82 3.5 3.5 0 0 1-3.888-5.82zm.555 4.989a2.5 2.5 0 1 0 2.778-4.157 2.5 2.5 0 0 0-2.778 4.157z" fill="#424242"/></svg>

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.8.5",
+  "version": "1.8.8",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -71,6 +71,7 @@
   "contributes": {
     "configurationDefaults": {
       "[ql]": {
+        "debug.saveBeforeStart": "nonUntitledEditorsInActiveGroup",
         "editor.wordBasedSuggestions": false
       },
       "[dbscheme]": {
@@ -189,7 +190,7 @@
           "scope": "machine-overridable",
           "type": "string",
           "default": "",
-          "markdownDescription": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable."
+          "markdownDescription": "Path to the CodeQL executable that should be used by the CodeQL extension. The executable is named `codeql` on Linux/Mac and `codeql.exe` on Windows. If empty, the extension will look for a CodeQL executable on your shell PATH, or if CodeQL is not on your PATH, download and manage its own CodeQL executable (note: if you later introduce CodeQL on your PATH, the extension will prefer a CodeQL executable it has downloaded itself)."
         },
         "codeQL.runningQueries.numberOfThreads": {
           "type": "integer",
@@ -246,8 +247,8 @@
         },
         "codeQL.runningQueries.autoSave": {
           "type": "boolean",
-          "default": false,
-          "description": "Enable automatically saving a modified query file when running a query."
+          "description": "Enable automatically saving a modified query file when running a query.",
+          "markdownDeprecationMessage": "This property is deprecated and no longer has any effect. To control automatic saving of documents before running queries, use the `debug.saveBeforeStart` setting."
         },
         "codeQL.runningQueries.maxQueries": {
           "type": "integer",
@@ -457,6 +458,10 @@
         "command": "codeQL.quickEval",
         "title": "CodeQL: Quick Evaluation"
       },
+      {
+        "command": "codeQL.quickEvalCount",
+        "title": "CodeQL: Quick Evaluation Count"
+      },
       {
         "command": "codeQL.quickEvalContextEditor",
         "title": "CodeQL: Quick Evaluation"
@@ -501,6 +506,33 @@
         "command": "codeQL.copyVersion",
         "title": "CodeQL: Copy Version Information"
       },
+      {
+        "command": "codeQLQueries.runLocalQueryFromQueriesPanel",
+        "title": "Run local query",
+        "icon": "$(run)"
+      },
+      {
+        "command": "codeQLQueries.runLocalQueriesFromPanel",
+        "title": "Run local queries",
+        "icon": "$(run-all)"
+      },
+      {
+        "command": "codeQL.runLocalQueryFromFileTab",
+        "title": "CodeQL: Run local query",
+        "icon": "$(run)"
+      },
+      {
+        "command": "codeQLQueries.runLocalQueryContextMenu",
+        "title": "Run against local database"
+      },
+      {
+        "command": "codeQLQueries.runLocalQueriesContextMenu",
+        "title": "Run against local database"
+      },
+      {
+        "command": "codeQLQueries.runVariantAnalysisContextMenu",
+        "title": "Run against variant analysis repositories"
+      },
       {
         "command": "codeQLVariantAnalysisRepositories.openConfigFile",
         "title": "Open database configuration file",
@@ -872,6 +904,13 @@
       }
     ],
     "menus": {
+      "editor/title": [
+        {
+          "command": "codeQL.runLocalQueryFromFileTab",
+          "group": "navigation",
+          "when": "resourceExtname == .ql && codeQL.currentDatabaseItem"
+        }
+      ],
       "view/title": [
         {
           "command": "codeQLDatabases.sortByName",
@@ -967,7 +1006,7 @@
         },
         {
           "command": "codeQLVariantAnalysisRepositories.importFromCodeSearch",
-          "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/ && config.codeQL.codeSearch",
+          "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/",
           "group": "2_qlContextMenu@1"
         },
         {
@@ -1095,6 +1134,31 @@
           "group": "1_queryHistory@1",
           "when": "viewItem == remoteResultsItem"
         },
+        {
+          "command": "codeQLQueries.runLocalQueryFromQueriesPanel",
+          "group": "inline",
+          "when": "view == codeQLQueries && viewItem == queryFile && codeQL.currentDatabaseItem"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueryContextMenu",
+          "group": "queriesPanel@1",
+          "when": "view == codeQLQueries && viewItem == queryFile && codeQL.currentDatabaseItem"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesContextMenu",
+          "group": "queriesPanel@1",
+          "when": "view == codeQLQueries && viewItem == queryFolder && codeQL.currentDatabaseItem"
+        },
+        {
+          "command": "codeQLQueries.runVariantAnalysisContextMenu",
+          "group": "queriesPanel@1",
+          "when": "view == codeQLQueries && viewItem == queryFile"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesFromPanel",
+          "group": "inline",
+          "when": "view == codeQLQueries && viewItem == queryFolder && codeQL.currentDatabaseItem"
+        },
         {
           "command": "codeQLTests.showOutputDifferences",
           "group": "qltest@1",
@@ -1154,6 +1218,18 @@
           "command": "codeQL.runQuery",
           "when": "resourceLangId == ql && resourceExtname == .ql"
         },
+        {
+          "command": "codeQLQueries.runLocalQueryFromQueriesPanel",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesFromPanel",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.runLocalQueryFromFileTab",
+          "when": "false"
+        },
         {
           "command": "codeQL.runQueryContextEditor",
           "when": "false"
@@ -1206,6 +1282,10 @@
           "command": "codeQL.quickEval",
           "when": "editorLangId == ql"
         },
+        {
+          "command": "codeQL.quickEvalCount",
+          "when": "editorLangId == ql && codeql.supportsQuickEvalCount"
+        },
         {
           "command": "codeQL.quickEvalContextEditor",
           "when": "false"
@@ -1274,6 +1354,18 @@
           "command": "codeQL.openDataExtensionsEditor",
           "when": "config.codeQL.canary && config.codeQL.dataExtensions.editor"
         },
+        {
+          "command": "codeQLQueries.runLocalQueryContextMenu",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueries.runLocalQueriesContextMenu",
+          "when": "false"
+        },
+        {
+          "command": "codeQLQueries.runVariantAnalysisContextMenu",
+          "when": "false"
+        },
         {
           "command": "codeQLVariantAnalysisRepositories.openConfigFile",
           "when": "false"
@@ -1608,7 +1700,7 @@
       },
       {
         "view": "codeQLQueries",
-        "contents": "This workspace doesn't contain any CodeQL queries at the moment."
+        "contents": "Looking for queries..."
       },
       {
         "view": "codeQLDatabases",
@@ -1635,7 +1727,7 @@
     "test:vscode-integration:activated-extension": "jest --projects test/vscode-tests/activated-extension",
     "test:vscode-integration:no-workspace": "jest --projects test/vscode-tests/no-workspace",
     "test:vscode-integration:minimal-workspace": "jest --projects test/vscode-tests/minimal-workspace",
-    "test:cli-integration": "jest --projects test/vscode-tests/cli-integration",
+    "test:cli-integration": "jest --projects test/vscode-tests/cli-integration --verbose",
     "update-vscode": "node ./node_modules/vscode/bin/install",
     "format": "prettier --write **/*.{ts,tsx} && eslint . --ext .ts,.tsx --fix",
     "lint": "eslint . --ext .js,.ts,.tsx --max-warnings=0",
@@ -1649,7 +1741,7 @@
     "prepare": "cd ../.. && husky install"
   },
   "dependencies": {
-    "@octokit/plugin-retry": "^3.0.9",
+    "@octokit/plugin-retry": "^4.1.6",
     "@octokit/rest": "^19.0.4",
     "@vscode/codicons": "^0.0.31",
     "@vscode/debugadapter": "^1.59.0",
@@ -1673,7 +1765,7 @@
     "path-browserify": "^1.0.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
-    "semver": "~7.3.2",
+    "semver": "~7.5.2",
     "source-map": "^0.7.4",
     "source-map-support": "^0.5.21",
     "stream": "^0.0.2",
@@ -1689,12 +1781,12 @@
     "vscode-languageclient": "^8.0.2",
     "vscode-test-adapter-api": "~1.7.0",
     "vscode-test-adapter-util": "~0.7.0",
-    "zip-a-folder": "~1.1.3"
+    "zip-a-folder": "~2.0.0"
   },
   "devDependencies": {
     "@babel/core": "^7.18.13",
     "@babel/plugin-transform-modules-commonjs": "^7.18.6",
-    "@faker-js/faker": "^7.5.0",
+    "@faker-js/faker": "^8.0.2",
     "@github/markdownlint-github": "^0.3.0",
     "@octokit/plugin-throttling": "^5.0.1",
     "@storybook/addon-actions": "^6.5.17-alpha.0",
@@ -1705,6 +1797,7 @@
     "@storybook/manager-webpack5": "^6.5.17-alpha.0",
     "@storybook/react": "^6.5.17-alpha.0",
     "@storybook/testing-library": "^0.0.13",
+    "@testing-library/dom": "^9.3.0",
     "@testing-library/jest-dom": "^5.16.5",
     "@testing-library/react": "^14.0.0",
     "@testing-library/user-event": "^14.4.3",
@@ -1738,23 +1831,22 @@
     "@types/vscode": "^1.67.0",
     "@types/webpack": "^5.28.0",
     "@types/webpack-env": "^1.18.0",
-    "@types/xml2js": "~0.4.4",
     "@typescript-eslint/eslint-plugin": "^5.38.0",
     "@typescript-eslint/parser": "^5.38.0",
     "@vscode/test-electron": "^2.2.0",
-    "@vscode/vsce": "^2.15.0",
+    "@vscode/vsce": "^2.19.0",
     "ansi-colors": "^4.1.1",
     "applicationinsights": "^2.3.5",
     "cross-env": "^7.0.3",
-    "css-loader": "~3.1.0",
+    "css-loader": "~6.8.1",
     "del": "^6.0.0",
     "esbuild": "^0.15.15",
     "eslint": "^8.23.1",
     "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-etc": "^2.0.2",
     "eslint-plugin-github": "^4.4.1",
-    "eslint-plugin-jest-dom": "^4.0.2",
-    "eslint-plugin-prettier": "^4.2.1",
+    "eslint-plugin-jest-dom": "^5.0.1",
+    "eslint-plugin-prettier": "^5.0.0",
     "eslint-plugin-react": "^7.31.8",
     "eslint-plugin-react-hooks": "^4.6.0",
     "eslint-plugin-storybook": "^0.6.4",
@@ -1775,7 +1867,7 @@
     "mini-css-extract-plugin": "^2.6.1",
     "npm-run-all": "^4.1.5",
     "patch-package": "^7.0.0",
-    "prettier": "^2.7.1",
+    "prettier": "^3.0.0",
     "tar-stream": "^3.0.0",
     "through2": "^4.0.2",
     "ts-jest": "^29.0.1",

extensions/ql-vscode/src/code-tour/code-tour.ts

@@ -0,0 +1,55 @@
+import { AppCommandManager } from "../common/commands";
+import { Uri, workspace } from "vscode";
+import { join } from "path";
+import { pathExists } from "fs-extra";
+import { isCodespacesTemplate } from "../config";
+import { showBinaryChoiceDialog } from "../common/vscode/dialog";
+import { extLogger } from "../common/logging/vscode";
+
+/**
+ * Check if the current workspace is the CodeTour and open the workspace folder.
+ * Without this, we can't run the code tour correctly.
+ **/
+export async function prepareCodeTour(
+  commandManager: AppCommandManager,
+): Promise<void> {
+  if (workspace.workspaceFolders?.length) {
+    const currentFolder = workspace.workspaceFolders[0].uri.fsPath;
+
+    const tutorialWorkspacePath = join(
+      currentFolder,
+      "tutorial.code-workspace",
+    );
+    const toursFolderPath = join(currentFolder, ".tours");
+
+    /** We're opening the tutorial workspace, if we detect it.
+     * This will only happen if the following three conditions are met:
+     * - the .tours folder exists
+     * - the tutorial.code-workspace file exists
+     * - the CODESPACES_TEMPLATE setting doesn't exist (it's only set if the user has already opened
+     * the tutorial workspace so it's a good indicator that the user is in the folder but has ignored
+     * the prompt to open the workspace)
+     */
+    if (
+      (await pathExists(tutorialWorkspacePath)) &&
+      (await pathExists(toursFolderPath)) &&
+      !isCodespacesTemplate()
+    ) {
+      const answer = await showBinaryChoiceDialog(
+        "We've detected you're in the CodeQL Tour repo. We will need to open the workspace file to continue. Reload?",
+      );
+
+      if (!answer) {
+        return;
+      }
+
+      const tutorialWorkspaceUri = Uri.file(tutorialWorkspacePath);
+
+      void extLogger.log(
+        `In prepareCodeTour() method, going to open the tutorial workspace file: ${tutorialWorkspacePath}`,
+      );
+
+      await commandManager.execute("vscode.openFolder", tutorialWorkspaceUri);
+    }
+  }
+}

extensions/ql-vscode/src/codeql-cli/cli-version.ts

@@ -1,7 +1,7 @@
 import * as semver from "semver";
 import { runCodeQlCliCommand } from "./cli";
-import { Logger } from "../common";
-import { getErrorMessage } from "../pure/helpers-pure";
+import { Logger } from "../common/logging";
+import { getErrorMessage } from "../common/helpers-pure";
 
 /**
  * Get the version of a CodeQL CLI.

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -11,7 +11,7 @@ import tk from "tree-kill";
 import { promisify } from "util";
 import { CancellationToken, Disposable, Uri } from "vscode";
 
-import { BQRSInfo, DecodedBqrsChunk } from "../pure/bqrs-cli-types";
+import { BQRSInfo, DecodedBqrsChunk } from "../common/bqrs-cli-types";
 import { allowCanaryQueryServer, CliConfig } from "../config";
 import {
   DistributionProvider,
@@ -21,12 +21,13 @@ import {
   assertNever,
   getErrorMessage,
   getErrorStack,
-} from "../pure/helpers-pure";
-import { QueryMetadata, SortDirection } from "../pure/interface-types";
-import { BaseLogger, Logger, ProgressReporter } from "../common";
-import { CompilationMessage } from "../pure/legacy-messages";
+} from "../common/helpers-pure";
+import { walkDirectory } from "../common/files";
+import { QueryMetadata, SortDirection } from "../common/interface-types";
+import { BaseLogger, Logger } from "../common/logging";
+import { ProgressReporter } from "../common/logging/vscode";
+import { CompilationMessage } from "../query-server/legacy-messages";
 import { sarifParser } from "../common/sarif-parser";
-import { walkDirectory } from "../helpers";
 import { App } from "../common/app";
 import { QueryLanguage } from "../common/query-language";
 
@@ -718,6 +719,7 @@ export class CodeQLCliServer implements Disposable {
   async resolveLibraryPath(
     workspaces: string[],
     queryPath: string,
+    silent = false,
   ): Promise<QuerySetup> {
     const subcommandArgs = [
       "--query",
@@ -728,6 +730,7 @@ export class CodeQLCliServer implements Disposable {
       ["resolve", "library-path"],
       subcommandArgs,
       "Resolving library paths",
+      { silent },
     );
   }
 
@@ -1480,6 +1483,13 @@ export class CodeQLCliServer implements Disposable {
             CliVersionConstraint.CLI_VERSION_WITH_PER_QUERY_EVAL_LOG,
           ) >= 0,
         );
+        await this.app.commands.execute(
+          "setContext",
+          "codeql.supportsQuickEvalCount",
+          newVersion.compare(
+            CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
+          ) >= 0,
+        );
       } catch (e) {
         this._versionChangedListeners.forEach((listener) =>
           listener(undefined),
@@ -1756,34 +1766,31 @@ async function logStream(stream: Readable, logger: BaseLogger): Promise<void> {
   }
 }
 
-export function shouldDebugIdeServer() {
+function isEnvTrue(name: string): boolean {
   return (
-    "IDE_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.IDE_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.IDE_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
+    name in process.env &&
+    process.env[name] !== "0" &&
+    // Use en-US since we expect the value to be either "false" or "FALSE", not a localized version.
+    process.env[name]?.toLocaleLowerCase("en-US") !== "false"
   );
 }
 
+export function shouldDebugIdeServer() {
+  return isEnvTrue("IDE_SERVER_JAVA_DEBUG");
+}
+
 export function shouldDebugQueryServer() {
-  return (
-    "QUERY_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.QUERY_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.QUERY_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
-  );
+  return isEnvTrue("QUERY_SERVER_JAVA_DEBUG");
 }
 
 export function shouldDebugCliServer() {
-  return (
-    "CLI_SERVER_JAVA_DEBUG" in process.env &&
-    process.env.CLI_SERVER_JAVA_DEBUG !== "0" &&
-    process.env.CLI_SERVER_JAVA_DEBUG?.toLocaleLowerCase() !== "false"
-  );
+  return isEnvTrue("CLI_SERVER_JAVA_DEBUG");
 }
 
 export class CliVersionConstraint {
   // The oldest version of the CLI that we support. This is used to determine
   // whether to show a warning about the CLI being too old on startup.
-  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.7.6");
+  public static OLDEST_SUPPORTED_CLI_VERSION = new SemVer("2.9.4");
 
   /**
    * CLI version where building QLX packs for remote queries is supported.
@@ -1845,6 +1852,18 @@ export class CliVersionConstraint {
 
   public static CLI_VERSION_GLOBAL_CACHE = new SemVer("2.12.4");
 
+  /**
+   * CLI version where the query server supports quick-eval count mode.
+   */
+  public static CLI_VERSION_WITH_QUICK_EVAL_COUNT = new SemVer("2.13.3");
+
+  /**
+   * CLI version where the langauge server supports visisbility change notifications.
+   */
+  public static CLI_VERSION_WITH_VISIBILITY_NOTIFICATIONS = new SemVer(
+    "2.14.0",
+  );
+
   constructor(private readonly cli: CodeQLCliServer) {
     /**/
   }
@@ -1918,4 +1937,16 @@ export class CliVersionConstraint {
   async usesGlobalCompilationCache() {
     return this.isVersionAtLeast(CliVersionConstraint.CLI_VERSION_GLOBAL_CACHE);
   }
+
+  async supportsVisibilityNotifications() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_VISIBILITY_NOTIFICATIONS,
+    );
+  }
+
+  async supportsQuickEvalCount() {
+    return this.isVersionAtLeast(
+      CliVersionConstraint.CLI_VERSION_WITH_QUICK_EVAL_COUNT,
+    );
+  }
 }

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -6,8 +6,7 @@ import * as semver from "semver";
 import { URL } from "url";
 import { ExtensionContext, Event } from "vscode";
 import { DistributionConfig } from "../config";
-import { showAndLogErrorMessage, showAndLogWarningMessage } from "../helpers";
-import { extLogger } from "../common";
+import { extLogger } from "../common/logging/vscode";
 import { getCodeQlCliVersion } from "./cli-version";
 import {
   ProgressCallback,
@@ -18,11 +17,15 @@ import {
   deprecatedCodeQlLauncherName,
   extractZipArchive,
   getRequiredAssetName,
-} from "../pure/distribution";
+} from "../common/distribution";
 import {
   InvocationRateLimiter,
   InvocationRateLimiterResultKind,
 } from "../common/invocation-rate-limiter";
+import {
+  showAndLogErrorMessage,
+  showAndLogWarningMessage,
+} from "../common/logging";
 
 /**
  * distribution.ts
@@ -157,6 +160,7 @@ export class DistributionManager implements DistributionProvider {
     if (this.config.customCodeQlPath) {
       if (!(await pathExists(this.config.customCodeQlPath))) {
         void showAndLogErrorMessage(
+          extLogger,
           `The CodeQL executable path is specified as "${this.config.customCodeQlPath}" ` +
             "by a configuration setting, but a CodeQL executable could not be found at that path. Please check " +
             "that a CodeQL executable exists at the specified path or remove the setting.",
@@ -849,6 +853,7 @@ export async function getExecutableFromDirectory(
 
 function warnDeprecatedLauncher() {
   void showAndLogWarningMessage(
+    extLogger,
     `The "${deprecatedCodeQlLauncherName()!}" launcher has been deprecated and will be removed in a future version. ` +
       `Please use "${codeQlLauncherName()}" instead. It is recommended to update to the latest CodeQL binaries.`,
   );
@@ -949,7 +954,10 @@ export interface GithubReleaseAsset {
 }
 
 export class GithubApiError extends Error {
-  constructor(public status: number, public body: string) {
+  constructor(
+    public status: number,
+    public body: string,
+  ) {
     super(`API call failed with status code ${status}, body: ${body}`);
   }
 }

extensions/ql-vscode/src/codeql-cli/query-language.ts

@@ -0,0 +1,80 @@
+import { CodeQLCliServer } from "./cli";
+import { Uri, window } from "vscode";
+import { isQueryLanguage, QueryLanguage } from "../common/query-language";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { extLogger } from "../common/logging/vscode";
+import { UserCancellationException } from "../common/vscode/progress";
+import { showAndLogErrorMessage } from "../common/logging";
+
+/**
+ * Finds the language that a query targets.
+ * If it can't be autodetected, prompt the user to specify the language manually.
+ */
+export async function findLanguage(
+  cliServer: CodeQLCliServer,
+  queryUri: Uri | undefined,
+): Promise<QueryLanguage | undefined> {
+  const uri = queryUri || window.activeTextEditor?.document.uri;
+  if (uri !== undefined) {
+    try {
+      const queryInfo = await cliServer.resolveQueryByLanguage(
+        getOnDiskWorkspaceFolders(),
+        uri,
+      );
+      const language = Object.keys(queryInfo.byLanguage)[0];
+      void extLogger.log(`Detected query language: ${language}`);
+
+      if (isQueryLanguage(language)) {
+        return language;
+      }
+
+      void extLogger.log(
+        "Query language is unsupported. Select language manually.",
+      );
+    } catch (e) {
+      void extLogger.log(
+        "Could not autodetect query language. Select language manually.",
+      );
+    }
+  }
+
+  // will be undefined if user cancels the quick pick.
+  return await askForLanguage(cliServer, false);
+}
+
+export async function askForLanguage(
+  cliServer: CodeQLCliServer,
+  throwOnEmpty = true,
+): Promise<QueryLanguage | undefined> {
+  const language = await window.showQuickPick(
+    await cliServer.getSupportedLanguages(),
+    {
+      placeHolder: "Select target language for your query",
+      ignoreFocusOut: true,
+    },
+  );
+  if (!language) {
+    // This only happens if the user cancels the quick pick.
+    if (throwOnEmpty) {
+      throw new UserCancellationException("Cancelled.");
+    } else {
+      void showAndLogErrorMessage(
+        extLogger,
+        "Language not found. Language must be specified manually.",
+      );
+    }
+    return undefined;
+  }
+
+  if (!isQueryLanguage(language)) {
+    void showAndLogErrorMessage(
+      extLogger,
+      `Language '${language}' is not supported. Only languages ${Object.values(
+        QueryLanguage,
+      ).join(", ")} are supported.`,
+    );
+    return undefined;
+  }
+
+  return language;
+}

extensions/ql-vscode/src/codeql-cli/query-metadata.ts

@@ -0,0 +1,22 @@
+import { CodeQLCliServer } from "./cli";
+import { QueryMetadata } from "../common/interface-types";
+import { extLogger } from "../common/logging/vscode";
+
+/**
+ * Gets metadata for a query, if it exists.
+ * @param cliServer The CLI server.
+ * @param queryPath The path to the query.
+ * @returns A promise that resolves to the query metadata, if available.
+ */
+export async function tryGetQueryMetadata(
+  cliServer: CodeQLCliServer,
+  queryPath: string,
+): Promise<QueryMetadata | undefined> {
+  try {
+    return await cliServer.resolveMetadata(queryPath);
+  } catch (e) {
+    // Ignore errors and provide no metadata.
+    void extLogger.log(`Couldn't resolve metadata for ${queryPath}: ${e}`);
+    return;
+  }
+}

extensions/ql-vscode/src/common/app.ts

@@ -1,26 +1,21 @@
 import { Credentials } from "./authentication";
-import { Disposable } from "../pure/disposable-object";
+import { Disposable } from "./disposable-object";
 import { AppEventEmitter } from "./events";
-import { Logger } from "./logging";
+import { NotificationLogger } from "./logging";
 import { Memento } from "./memento";
 import { AppCommandManager } from "./commands";
-import type {
-  WorkspaceFolder,
-  Event,
-  WorkspaceFoldersChangeEvent,
-} from "vscode";
+import { AppTelemetry } from "./telemetry";
 
 export interface App {
   createEventEmitter<T>(): AppEventEmitter<T>;
   readonly mode: AppMode;
-  readonly logger: Logger;
+  readonly logger: NotificationLogger;
+  readonly telemetry?: AppTelemetry;
   readonly subscriptions: Disposable[];
   readonly extensionPath: string;
   readonly globalStoragePath: string;
   readonly workspaceStoragePath?: string;
   readonly workspaceState: Memento;
-  readonly workspaceFolders: readonly WorkspaceFolder[] | undefined;
-  readonly onDidChangeWorkspaceFolders: Event<WorkspaceFoldersChangeEvent>;
   readonly credentials: Credentials;
   readonly commands: AppCommandManager;
   readonly environment: EnvironmentContext;

extensions/ql-vscode/src/common/bqrs-utils.ts

@@ -4,7 +4,7 @@ import {
   LineColumnLocation,
   WholeFileLocation,
 } from "./bqrs-cli-types";
-import { createRemoteFileRef } from "./location-link-utils";
+import { createRemoteFileRef } from "../common/location-link-utils";
 
 /**
  * The CodeQL filesystem libraries use this pattern in `getURL()` predicates

extensions/ql-vscode/src/common/commands.ts

@@ -4,7 +4,7 @@ import type { AstItem } from "../language-support";
 import type { DbTreeViewItem } from "../databases/ui/db-tree-view-item";
 import type { DatabaseItem } from "../databases/local-databases";
 import type { QueryHistoryInfo } from "../query-history/query-history-info";
-import type { RepositoriesFilterSortStateWithIds } from "../pure/variant-analysis-filter-sort";
+import type { RepositoriesFilterSortStateWithIds } from "../variant-analysis/shared/variant-analysis-filter-sort";
 import type { TestTreeNode } from "../query-testing/test-tree-node";
 import type {
   VariantAnalysis,
@@ -12,6 +12,7 @@ import type {
   VariantAnalysisScannedRepositoryResult,
 } from "../variant-analysis/shared/variant-analysis";
 import type { QLDebugConfiguration } from "../debugger/debug-configuration";
+import type { QueryTreeViewItem } from "../queries-panel/query-tree-view-item";
 
 // A command function matching the signature that VS Code calls when
 // a command is invoked from a context menu on a TreeView with
@@ -129,8 +130,14 @@ export type LocalQueryCommands = {
   "codeQL.runQueryOnMultipleDatabasesContextEditor": (
     uri?: Uri,
   ) => Promise<void>;
+  "codeQLQueries.runLocalQueryFromQueriesPanel": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueryContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueriesContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQLQueries.runLocalQueriesFromPanel": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
+  "codeQL.runLocalQueryFromFileTab": (uri: Uri) => Promise<void>;
   "codeQL.runQueries": ExplorerSelectionCommandFunction<Uri>;
   "codeQL.quickEval": (uri: Uri) => Promise<void>;
+  "codeQL.quickEvalCount": (uri: Uri) => Promise<void>;
   "codeQL.quickEvalContextEditor": (uri: Uri) => Promise<void>;
   "codeQL.codeLensQuickEval": (uri: Uri, range: Range) => Promise<void>;
   "codeQL.quickQuery": () => Promise<void>;
@@ -262,6 +269,7 @@ export type VariantAnalysisCommands = {
   ) => Promise<void>;
   "codeQL.runVariantAnalysis": (uri?: Uri) => Promise<void>;
   "codeQL.runVariantAnalysisContextEditor": (uri?: Uri) => Promise<void>;
+  "codeQLQueries.runVariantAnalysisContextMenu": TreeViewContextSingleSelectionCommandFunction<QueryTreeViewItem>;
 };
 
 export type DatabasePanelCommands = {

extensions/ql-vscode/src/common/discovery.ts

@@ -1,5 +1,5 @@
-import { DisposableObject } from "../pure/disposable-object";
-import { getErrorMessage } from "../pure/helpers-pure";
+import { DisposableObject } from "./disposable-object";
+import { getErrorMessage } from "./helpers-pure";
 import { Logger } from "./logging";
 
 /**
@@ -7,11 +7,14 @@ import { Logger } from "./logging";
  * files. This class automatically prevents more than one discovery operation from running at the
  * same time.
  */
-export abstract class Discovery<T> extends DisposableObject {
+export abstract class Discovery extends DisposableObject {
   private restartWhenFinished = false;
   private currentDiscoveryPromise: Promise<void> | undefined;
 
-  constructor(private readonly name: string, private readonly logger: Logger) {
+  constructor(
+    protected readonly name: string,
+    private readonly logger: Logger,
+  ) {
     super();
   }
 
@@ -64,14 +67,12 @@ export abstract class Discovery<T> extends DisposableObject {
    * discovery.
    */
   private async launchDiscovery(): Promise<void> {
-    let results: T | undefined;
     try {
-      results = await this.discover();
+      await this.discover();
     } catch (err) {
       void this.logger.log(
         `${this.name} failed. Reason: ${getErrorMessage(err)}`,
       );
-      results = undefined;
     }
 
     if (this.restartWhenFinished) {
@@ -82,24 +83,11 @@ export abstract class Discovery<T> extends DisposableObject {
       // succeeded or failed.
       this.restartWhenFinished = false;
       await this.launchDiscovery();
-    } else {
-      // If the discovery was successful, then update any listeners with the results.
-      if (results !== undefined) {
-        this.update(results);
-      }
     }
   }
 
   /**
    * Overridden by the derived class to spawn the actual discovery operation, returning the results.
    */
-  protected abstract discover(): Promise<T>;
-
-  /**
-   * Overridden by the derived class to atomically update the `Discovery` object with the results of
-   * the discovery operation, and to notify any listeners that the discovery results may have
-   * changed.
-   * @param results The discovery results returned by the `discover` function.
-   */
-  protected abstract update(results: T): void;
+  protected abstract discover(): Promise<void>;
 }

extensions/ql-vscode/src/common/events.ts

@@ -1,4 +1,4 @@
-import { Disposable } from "../pure/disposable-object";
+import { Disposable } from "./disposable-object";
 
 export interface AppEvent<T> {
   (listener: (event: T) => void): Disposable;

extensions/ql-vscode/src/common/files.ts

@@ -1,5 +1,6 @@
-import { pathExists, stat, readdir } from "fs-extra";
-import { join, resolve } from "path";
+import { pathExists, stat, readdir, opendir } from "fs-extra";
+import { isAbsolute, join, relative, resolve } from "path";
+import { tmpdir as osTmpdir } from "os";
 
 /**
  * Recursively finds all .ql files in this set of Uris.
@@ -51,36 +52,32 @@ export async function getDirectoryNamesInsidePath(
   return dirNames;
 }
 
-function normalizePath(path: string, platform: NodeJS.Platform): string {
+export function normalizePath(path: string): string {
   // On Windows, "C:/", "C:\", and "c:/" are all equivalent. We need
   // to normalize the paths to ensure they all get resolved to the
   // same format. On Windows, we also need to do the comparison
   // case-insensitively.
   path = resolve(path);
-  if (platform === "win32") {
+  if (process.platform === "win32") {
     path = path.toLowerCase();
   }
   return path;
 }
 
-export function pathsEqual(
-  path1: string,
-  path2: string,
-  platform: NodeJS.Platform,
-): boolean {
-  return normalizePath(path1, platform) === normalizePath(path2, platform);
+export function pathsEqual(path1: string, path2: string): boolean {
+  return normalizePath(path1) === normalizePath(path2);
 }
 
 /**
- * Returns true if path1 contains path2.
+ * Returns true if `parent` contains `child`, or if they are equal.
  */
-export function containsPath(
-  path1: string,
-  path2: string,
-  platform: NodeJS.Platform,
-): boolean {
-  return normalizePath(path2, platform).startsWith(
-    normalizePath(path1, platform),
+export function containsPath(parent: string, child: string): boolean {
+  const relativePath = relative(parent, child);
+  return (
+    !relativePath.startsWith("..") &&
+    // On windows, if the two paths are in different drives, then the
+    // relative path will be an absolute path to the other drive.
+    !isAbsolute(relativePath)
   );
 }
 
@@ -88,3 +85,45 @@ export async function readDirFullPaths(path: string): Promise<string[]> {
   const baseNames = await readdir(path);
   return baseNames.map((baseName) => join(path, baseName));
 }
+
+/**
+ * Recursively walk a directory and return the full path to all files found.
+ * Symbolic links are ignored.
+ *
+ * @param dir the directory to walk
+ *
+ * @return An iterator of the full path to all files recursively found in the directory.
+ */
+export async function* walkDirectory(
+  dir: string,
+): AsyncIterableIterator<string> {
+  const seenFiles = new Set<string>();
+  for await (const d of await opendir(dir)) {
+    const entry = join(dir, d.name);
+    seenFiles.add(entry);
+    if (d.isDirectory()) {
+      yield* walkDirectory(entry);
+    } else if (d.isFile()) {
+      yield entry;
+    }
+  }
+}
+
+/**
+ * Error thrown from methods from the `fs` module.
+ *
+ * In practice, any error matching this is likely an instance of `NodeJS.ErrnoException`.
+ * If desired in the future, we could model more fields or use `NodeJS.ErrnoException` directly.
+ */
+export interface IOError {
+  readonly code: string;
+}
+
+export function isIOError(e: any): e is IOError {
+  return e.code !== undefined && typeof e.code === "string";
+}
+
+// This function is a wrapper around `os.tmpdir()` to make it easier to mock in tests.
+export function tmpdir(): string {
+  return osTmpdir();
+}

extensions/ql-vscode/src/common/github-url-identifier-helper.ts

@@ -1,4 +1,4 @@
-import { OWNER_REGEX, REPO_REGEX } from "../pure/helpers-pure";
+import { OWNER_REGEX, REPO_REGEX } from "./helpers-pure";
 
 /**
  * Checks if a string is a valid GitHub NWO.

extensions/ql-vscode/src/common/index.ts

@@ -1 +0,0 @@
-export * from "./logging";

extensions/ql-vscode/src/common/interface-types.ts

@@ -5,7 +5,7 @@ import {
   ResultSetSchema,
   Column,
   ResolvableLocationValue,
-} from "./bqrs-cli-types";
+} from "../common/bqrs-cli-types";
 import {
   VariantAnalysis,
   VariantAnalysisScannedRepositoryResult,
@@ -14,12 +14,13 @@ import {
 import {
   RepositoriesFilterSortState,
   RepositoriesFilterSortStateWithIds,
-} from "./variant-analysis-filter-sort";
-import { ErrorLike } from "./errors";
+} from "../variant-analysis/shared/variant-analysis-filter-sort";
+import { ErrorLike } from "../common/errors";
 import { DataFlowPaths } from "../variant-analysis/shared/data-flow-paths";
 import { ExternalApiUsage } from "../data-extensions-editor/external-api-usage";
 import { ModeledMethod } from "../data-extensions-editor/modeled-method";
 import { DataExtensionEditorViewState } from "../data-extensions-editor/shared/view-state";
+import { Mode } from "../data-extensions-editor/shared/mode";
 
 /**
  * This module contains types and code that are shared between
@@ -508,17 +509,19 @@ export interface ShowProgressMessage {
   message: string;
 }
 
+export interface LoadModeledMethodsMessage {
+  t: "loadModeledMethods";
+  modeledMethods: Record<string, ModeledMethod>;
+}
+
 export interface AddModeledMethodsMessage {
   t: "addModeledMethods";
   modeledMethods: Record<string, ModeledMethod>;
+}
 
-  /**
-   * If true, then any existing modeled methods set to "none" will be
-   * overwritten by the new modeled methods. Otherwise, the "none" modeled
-   * methods will not be overwritten, even if the new modeled methods
-   * contain a better model.
-   */
-  overrideNone?: boolean;
+export interface SwitchModeMessage {
+  t: "switchMode";
+  mode: Mode;
 }
 
 export interface JumpToUsageMessage {
@@ -530,8 +533,8 @@ export interface OpenExtensionPackMessage {
   t: "openExtensionPack";
 }
 
-export interface OpenModelFileMessage {
-  t: "openModelFile";
+export interface RefreshExternalApiUsages {
+  t: "refreshExternalApiUsages";
 }
 
 export interface SaveModeledMethods {
@@ -554,11 +557,13 @@ export type ToDataExtensionsEditorMessage =
   | SetExtensionPackStateMessage
   | SetExternalApiUsagesMessage
   | ShowProgressMessage
+  | LoadModeledMethodsMessage
   | AddModeledMethodsMessage;
 
 export type FromDataExtensionsEditorMessage =
   | ViewLoadedMsg
-  | OpenModelFileMessage
+  | SwitchModeMessage
+  | RefreshExternalApiUsages
   | OpenExtensionPackMessage
   | JumpToUsageMessage
   | SaveModeledMethods

extensions/ql-vscode/src/common/logging/index.ts

@@ -1,4 +1,4 @@
 export * from "./logger";
+export * from "./notification-logger";
+export * from "./notifications";
 export * from "./tee-logger";
-export * from "./vscode/loggers";
-export * from "./vscode/output-channel-logger";

extensions/ql-vscode/src/common/logging/notification-logger.ts

@@ -0,0 +1,7 @@
+import { Logger } from "./logger";
+
+export interface NotificationLogger extends Logger {
+  showErrorMessage(message: string): Promise<void>;
+  showWarningMessage(message: string): Promise<void>;
+  showInformationMessage(message: string): Promise<void>;
+}

extensions/ql-vscode/src/common/logging/notifications.ts

@@ -0,0 +1,116 @@
+import { NotificationLogger } from "./notification-logger";
+import { AppTelemetry } from "../telemetry";
+import { RedactableError } from "../errors";
+
+export interface ShowAndLogOptions {
+  /**
+   * An alternate message that is added to the log, but not displayed in the popup.
+   * This is useful for adding extra detail to the logs that would be too noisy for the popup.
+   */
+  fullMessage?: string;
+}
+
+/**
+ * Show an error message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogErrorMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    dropLinesExceptInitial(message),
+    logger.showErrorMessage,
+    { fullMessage: message, ...options },
+  );
+}
+
+function dropLinesExceptInitial(message: string, n = 2) {
+  return message.toString().split(/\r?\n/).slice(0, n).join("\n");
+}
+
+/**
+ * Show a warning message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogWarningMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    message,
+    logger.showWarningMessage,
+    options,
+  );
+}
+
+/**
+ * Show an information message and log it to the console
+ *
+ * @param logger The logger that will receive the message.
+ * @param message The message to show.
+ * @param options? See individual fields on `ShowAndLogOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogInformationMessage(
+  logger: NotificationLogger,
+  message: string,
+  options?: ShowAndLogOptions,
+): Promise<void> {
+  return internalShowAndLog(
+    logger,
+    message,
+    logger.showInformationMessage,
+    options,
+  );
+}
+
+async function internalShowAndLog(
+  logger: NotificationLogger,
+  message: string,
+  fn: (message: string) => Promise<void>,
+  { fullMessage }: ShowAndLogOptions = {},
+): Promise<void> {
+  void logger.log(fullMessage || message);
+  await fn.bind(logger)(message);
+}
+
+interface ShowAndLogExceptionOptions extends ShowAndLogOptions {
+  /** Custom properties to include in the telemetry report. */
+  extraTelemetryProperties?: { [key: string]: string };
+}
+
+/**
+ * Show an error message, log it to the console, and emit redacted information as telemetry
+ *
+ * @param logger The logger that will receive the message.
+ * @param telemetry The telemetry instance to use for reporting.
+ * @param error The error to show. Only redacted information will be included in the telemetry.
+ * @param options See individual fields on `ShowAndLogExceptionOptions` type.
+ *
+ * @return A promise that resolves to the selected item or undefined when being dismissed.
+ */
+export async function showAndLogExceptionWithTelemetry(
+  logger: NotificationLogger,
+  telemetry: AppTelemetry | undefined,
+  error: RedactableError,
+  options: ShowAndLogExceptionOptions = {},
+): Promise<void> {
+  telemetry?.sendError(error, options.extraTelemetryProperties);
+  return showAndLogErrorMessage(logger, error.fullMessage, options);
+}

extensions/ql-vscode/src/common/logging/tee-logger.ts

@@ -1,6 +1,6 @@
 import { appendFile, ensureFile } from "fs-extra";
 import { isAbsolute } from "path";
-import { getErrorMessage } from "../../pure/helpers-pure";
+import { getErrorMessage } from "../helpers-pure";
 import { Logger, LogOptions } from "./logger";
 
 /**

extensions/ql-vscode/src/common/logging/vscode/index.ts

@@ -0,0 +1,2 @@
+export * from "./loggers";
+export * from "./output-channel-logger";

extensions/ql-vscode/src/common/logging/vscode/output-channel-logger.ts

@@ -1,11 +1,15 @@
 import { window as Window, OutputChannel, Progress } from "vscode";
 import { Logger, LogOptions } from "../logger";
-import { DisposableObject } from "../../../pure/disposable-object";
+import { DisposableObject } from "../../disposable-object";
+import { NotificationLogger } from "../notification-logger";
 
 /**
  * A logger that writes messages to an output channel in the VS Code Output tab.
  */
-export class OutputChannelLogger extends DisposableObject implements Logger {
+export class OutputChannelLogger
+  extends DisposableObject
+  implements Logger, NotificationLogger
+{
   public readonly outputChannel: OutputChannel;
   isCustomLogDirectory: boolean;
 
@@ -42,6 +46,30 @@ export class OutputChannelLogger extends DisposableObject implements Logger {
   show(preserveFocus?: boolean): void {
     this.outputChannel.show(preserveFocus);
   }
+
+  async showErrorMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showErrorMessage);
+  }
+
+  async showInformationMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showInformationMessage);
+  }
+
+  async showWarningMessage(message: string): Promise<void> {
+    await this.showMessage(message, Window.showWarningMessage);
+  }
+
+  private async showMessage(
+    message: string,
+    show: (message: string, ...items: string[]) => Thenable<string | undefined>,
+  ): Promise<void> {
+    const label = "Show Log";
+    const result = await show(message, label);
+
+    if (result === label) {
+      this.show();
+    }
+  }
 }
 
 export type ProgressReporter = Progress<{ message: string }>;

extensions/ql-vscode/src/common/path.ts

@@ -19,3 +19,11 @@ export const basename = (path: string): string => {
   const index = path.lastIndexOf("\\");
   return index === -1 ? path : path.slice(index + 1);
 };
+
+// Returns the extension of a path, including the leading dot.
+export const extname = (path: string): string => {
+  const name = basename(path);
+
+  const index = name.lastIndexOf(".");
+  return index === -1 ? "" : name.slice(index);
+};

extensions/ql-vscode/src/common/query-language.ts

@@ -25,13 +25,17 @@ export const PACKS_BY_QUERY_LANGUAGE = {
   [QueryLanguage.Ruby]: ["codeql/ruby-queries"],
 };
 
-export const dbSchemeToLanguage = {
-  "semmlecode.javascript.dbscheme": "javascript",
-  "semmlecode.cpp.dbscheme": "cpp",
-  "semmlecode.dbscheme": "java",
-  "semmlecode.python.dbscheme": "python",
-  "semmlecode.csharp.dbscheme": "csharp",
-  "go.dbscheme": "go",
-  "ruby.dbscheme": "ruby",
-  "swift.dbscheme": "swift",
+export const dbSchemeToLanguage: Record<string, QueryLanguage> = {
+  "semmlecode.javascript.dbscheme": QueryLanguage.Javascript,
+  "semmlecode.cpp.dbscheme": QueryLanguage.Cpp,
+  "semmlecode.dbscheme": QueryLanguage.Java,
+  "semmlecode.python.dbscheme": QueryLanguage.Python,
+  "semmlecode.csharp.dbscheme": QueryLanguage.CSharp,
+  "go.dbscheme": QueryLanguage.Go,
+  "ruby.dbscheme": QueryLanguage.Ruby,
+  "swift.dbscheme": QueryLanguage.Swift,
 };
+
+export function isQueryLanguage(language: string): language is QueryLanguage {
+  return Object.values(QueryLanguage).includes(language as QueryLanguage);
+}

extensions/ql-vscode/src/common/sarif-parser.ts

@@ -1,7 +1,7 @@
 import * as Sarif from "sarif";
 import { createReadStream } from "fs-extra";
 import { connectTo } from "stream-json/Assembler";
-import { getErrorMessage } from "../pure/helpers-pure";
+import { getErrorMessage } from "./helpers-pure";
 import { withParser } from "stream-json/filters/Pick";
 
 const DUMMY_TOOL: Sarif.Tool = { driver: { name: "" } };

extensions/ql-vscode/src/common/sarif-utils.ts

@@ -1,6 +1,6 @@
 import * as Sarif from "sarif";
 import type { HighlightedRegion } from "../variant-analysis/shared/analysis-result";
-import { ResolvableLocationValue } from "./bqrs-cli-types";
+import { ResolvableLocationValue } from "../common/bqrs-cli-types";
 
 export interface SarifLink {
   dest: number;

extensions/ql-vscode/src/common/telemetry.ts

@@ -0,0 +1,10 @@
+import { RedactableError } from "./errors";
+
+export interface AppTelemetry {
+  sendCommandUsage(name: string, executionTime: number, error?: Error): void;
+  sendUIInteraction(name: string): void;
+  sendError(
+    error: RedactableError,
+    extraProperties?: { [key: string]: string },
+  ): void;
+}

extensions/ql-vscode/src/common/vscode/abstract-webview.ts

@@ -9,13 +9,9 @@ import {
 } from "vscode";
 import { join } from "path";
 
-import { DisposableObject, DisposeHandler } from "../../pure/disposable-object";
-import { tmpDir } from "../../helpers";
-import {
-  getHtmlForWebview,
-  WebviewMessage,
-  WebviewView,
-} from "../../interface-utils";
+import { DisposableObject, DisposeHandler } from "../disposable-object";
+import { tmpDir } from "../../tmp-dir";
+import { getHtmlForWebview, WebviewMessage, WebviewView } from "./webview-html";
 
 export type WebviewPanelConfig = {
   viewId: string;
@@ -23,6 +19,7 @@ export type WebviewPanelConfig = {
   viewColumn: ViewColumn;
   view: WebviewView;
   preserveFocus?: boolean;
+  iconPath?: Uri | { dark: Uri; light: Uri };
   additionalOptions?: WebviewPanelOptions & WebviewOptions;
   allowWasmEval?: boolean;
 };
@@ -90,6 +87,8 @@ export abstract class AbstractWebview<
       );
       this.panel = panel;
 
+      this.panel.iconPath = config.iconPath;
+
       this.setupPanel(panel, config);
 
       this.panelResolves.forEach((resolve) => resolve(panel));

extensions/ql-vscode/src/common/vscode/archive-filesystem-provider.ts

@@ -1,7 +1,7 @@
 import { pathExists } from "fs-extra";
 import * as unzipper from "unzipper";
 import * as vscode from "vscode";
-import { extLogger } from "..";
+import { extLogger } from "../logging/vscode";
 
 // All path operations in this file must be on paths *within* the zip
 // archive.
@@ -14,7 +14,10 @@ export class File implements vscode.FileStat {
   mtime: number;
   size: number;
 
-  constructor(public name: string, public data: Uint8Array) {
+  constructor(
+    public name: string,
+    public data: Uint8Array,
+  ) {
     this.type = vscode.FileType.File;
     this.ctime = Date.now();
     this.mtime = Date.now();

extensions/ql-vscode/src/common/vscode/commands.ts

@@ -1,18 +1,20 @@
 import { commands, Disposable } from "vscode";
 import { CommandFunction, CommandManager } from "../../packages/commands";
-import { extLogger, OutputChannelLogger } from "../logging";
+import {
+  NotificationLogger,
+  showAndLogWarningMessage,
+  showAndLogExceptionWithTelemetry,
+} from "../logging";
+import { extLogger } from "../logging/vscode";
 import {
   asError,
   getErrorMessage,
   getErrorStack,
-} from "../../pure/helpers-pure";
-import { redactableError } from "../../pure/errors";
+} from "../../common/helpers-pure";
+import { redactableError } from "../../common/errors";
 import { UserCancellationException } from "./progress";
-import {
-  showAndLogExceptionWithTelemetry,
-  showAndLogWarningMessage,
-} from "../../helpers";
-import { telemetryListener } from "../../telemetry";
+import { telemetryListener } from "./telemetry";
+import { AppTelemetry } from "../telemetry";
 
 /**
  * Create a command manager for VSCode, wrapping registerCommandWithErrorHandling
@@ -20,9 +22,12 @@ import { telemetryListener } from "../../telemetry";
  */
 export function createVSCodeCommandManager<
   Commands extends Record<string, CommandFunction>,
->(outputLogger?: OutputChannelLogger): CommandManager<Commands> {
+>(
+  logger?: NotificationLogger,
+  telemetry?: AppTelemetry,
+): CommandManager<Commands> {
   return new CommandManager((commandId, task) => {
-    return registerCommandWithErrorHandling(commandId, task, outputLogger);
+    return registerCommandWithErrorHandling(commandId, task, logger, telemetry);
   }, wrapExecuteCommand);
 }
 
@@ -32,11 +37,14 @@ export function createVSCodeCommandManager<
  * @param commandId The ID of the command to register.
  * @param task The task to run. It is passed directly to `commands.registerCommand`. Any
  * arguments to the command handler are passed on to the task.
+ * @param logger The logger to use for error reporting.
+ * @param telemetry The telemetry listener to use for error reporting.
  */
 export function registerCommandWithErrorHandling(
   commandId: string,
   task: (...args: any[]) => Promise<any>,
-  outputLogger = extLogger,
+  logger: NotificationLogger = extLogger,
+  telemetry: AppTelemetry | undefined = telemetryListener,
 ): Disposable {
   return commands.registerCommand(commandId, async (...args: any[]) => {
     const startTime = Date.now();
@@ -52,11 +60,9 @@ export function registerCommandWithErrorHandling(
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
         if (e.silent) {
-          void outputLogger.log(errorMessage.fullMessage);
+          void logger.log(errorMessage.fullMessage);
         } else {
-          void showAndLogWarningMessage(errorMessage.fullMessage, {
-            outputLogger,
-          });
+          void showAndLogWarningMessage(logger, errorMessage.fullMessage);
         }
       } else {
         // Include the full stack in the error log only.
@@ -64,8 +70,7 @@ export function registerCommandWithErrorHandling(
         const fullMessage = errorStack
           ? `${errorMessage.fullMessage}\n${errorStack}`
           : errorMessage.fullMessage;
-        void showAndLogExceptionWithTelemetry(errorMessage, {
-          outputLogger,
+        void showAndLogExceptionWithTelemetry(logger, telemetry, errorMessage, {
           fullMessage,
           extraTelemetryProperties: {
             command: commandId,

extensions/ql-vscode/src/common/vscode/dialog.ts

@@ -0,0 +1,135 @@
+import { env, Uri, window } from "vscode";
+
+/**
+ * Opens a modal dialog for the user to make a yes/no choice.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ * @param yesTitle The text in the box indicating the affirmative choice.
+ * @param noTitle The text in the box indicating the negative choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showBinaryChoiceDialog(
+  message: string,
+  modal = true,
+  yesTitle = "Yes",
+  noTitle = "No",
+): Promise<boolean | undefined> {
+  const yesItem = { title: yesTitle, isCloseAffordance: false };
+  const noItem = { title: noTitle, isCloseAffordance: true };
+  const chosenItem = await window.showInformationMessage(
+    message,
+    { modal },
+    yesItem,
+    noItem,
+  );
+  if (!chosenItem) {
+    return undefined;
+  }
+  return chosenItem?.title === yesItem.title;
+}
+
+/**
+ * Opens a modal dialog for the user to make a yes/no choice.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ *
+ * @return
+ *  `true` if the user clicks 'Yes',
+ *  `false` if the user clicks 'No' or cancels the dialog,
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showBinaryChoiceWithUrlDialog(
+  message: string,
+  url: string,
+): Promise<boolean | undefined> {
+  const urlItem = { title: "More Information", isCloseAffordance: false };
+  const yesItem = { title: "Yes", isCloseAffordance: false };
+  const noItem = { title: "No", isCloseAffordance: true };
+  let chosenItem;
+
+  // Keep the dialog open as long as the user is clicking the 'more information' option.
+  // To prevent an infinite loop, if the user clicks 'more information' 5 times, close the dialog and return cancelled
+  let count = 0;
+  do {
+    chosenItem = await window.showInformationMessage(
+      message,
+      { modal: true },
+      urlItem,
+      yesItem,
+      noItem,
+    );
+    if (chosenItem === urlItem) {
+      await env.openExternal(Uri.parse(url, true));
+    }
+    count++;
+  } while (chosenItem === urlItem && count < 5);
+
+  if (!chosenItem || chosenItem.title === urlItem.title) {
+    return undefined;
+  }
+  return chosenItem.title === yesItem.title;
+}
+
+/**
+ * Show an information message with a customisable action.
+ * @param message The message to show.
+ * @param actionMessage The call to action message.
+ *
+ * @return `true` if the user clicks the action, `false` if the user cancels the dialog.
+ */
+export async function showInformationMessageWithAction(
+  message: string,
+  actionMessage: string,
+): Promise<boolean> {
+  const actionItem = { title: actionMessage, isCloseAffordance: false };
+  const chosenItem = await window.showInformationMessage(message, actionItem);
+  return chosenItem === actionItem;
+}
+
+/**
+ * Opens a modal dialog for the user to make a choice between yes/no/never be asked again.
+ *
+ * @param message The message to show.
+ * @param modal If true (the default), show a modal dialog box, otherwise dialog is non-modal and can
+ *        be closed even if the user does not make a choice.
+ * @param yesTitle The text in the box indicating the affirmative choice.
+ * @param noTitle The text in the box indicating the negative choice.
+ * @param neverTitle The text in the box indicating the opt out choice.
+ *
+ * @return
+ *  `Yes` if the user clicks 'Yes',
+ *  `No` if the user clicks 'No' or cancels the dialog,
+ *  `No, and never ask me again` if the user clicks 'No, and never ask me again',
+ *  `undefined` if the dialog is closed without the user making a choice.
+ */
+export async function showNeverAskAgainDialog(
+  message: string,
+  modal = true,
+  yesTitle = "Yes",
+  noTitle = "No",
+  neverAskAgainTitle = "No, and never ask me again",
+): Promise<string | undefined> {
+  const yesItem = { title: yesTitle, isCloseAffordance: true };
+  const noItem = { title: noTitle, isCloseAffordance: false };
+  const neverAskAgainItem = {
+    title: neverAskAgainTitle,
+    isCloseAffordance: false,
+  };
+  const chosenItem = await window.showInformationMessage(
+    message,
+    { modal },
+    yesItem,
+    noItem,
+    neverAskAgainItem,
+  );
+
+  return chosenItem?.title;
+}

extensions/ql-vscode/src/common/vscode/external-files.ts

@@ -1,15 +1,15 @@
 import { Uri, window } from "vscode";
 import { AppCommandManager } from "../commands";
-import {
-  showAndLogExceptionWithTelemetry,
-  showBinaryChoiceDialog,
-} from "../../helpers";
-import { redactableError } from "../../pure/errors";
+import { showBinaryChoiceDialog } from "./dialog";
+import { redactableError } from "../../common/errors";
 import {
   asError,
   getErrorMessage,
   getErrorStack,
-} from "../../pure/helpers-pure";
+} from "../../common/helpers-pure";
+import { showAndLogExceptionWithTelemetry } from "../logging";
+import { extLogger } from "../logging/vscode";
+import { telemetryListener } from "./telemetry";
 
 export async function tryOpenExternalFile(
   commandManager: AppCommandManager,
@@ -36,6 +36,8 @@ the file in the file explorer and dragging it into the workspace.`,
           await commandManager.execute("revealFileInOS", uri);
         } catch (e) {
           void showAndLogExceptionWithTelemetry(
+            extLogger,
+            telemetryListener,
             redactableError(
               asError(e),
             )`Failed to reveal file in OS: ${getErrorMessage(e)}`,
@@ -44,6 +46,8 @@ the file in the file explorer and dragging it into the workspace.`,
       }
     } else {
       void showAndLogExceptionWithTelemetry(
+        extLogger,
+        telemetryListener,
         redactableError(asError(e))`Could not open file ${fileLocation}`,
         {
           fullMessage: `${getErrorMessage(e)}\n${getErrorStack(e)}`,

extensions/ql-vscode/src/common/vscode/file-path-discovery.ts

@@ -0,0 +1,272 @@
+import { Discovery } from "../discovery";
+import {
+  Event,
+  EventEmitter,
+  RelativePattern,
+  Uri,
+  WorkspaceFoldersChangeEvent,
+  workspace,
+} from "vscode";
+import { MultiFileSystemWatcher } from "./multi-file-system-watcher";
+import { AppEventEmitter } from "../events";
+import { extLogger } from "../logging/vscode";
+import { lstat } from "fs-extra";
+import { containsPath, isIOError } from "../files";
+import {
+  getOnDiskWorkspaceFolders,
+  getOnDiskWorkspaceFoldersObjects,
+} from "./workspace-folders";
+import { getErrorMessage } from "../../common/helpers-pure";
+
+interface PathData {
+  path: string;
+}
+
+/**
+ * Discovers and watches for changes to all files matching a given filter
+ * contained in the workspace. Also allows computing extra data about each
+ * file path, and only recomputing the data when the file changes.
+ *
+ * Scans the whole workspace on startup, and then watches for changes to files
+ * to do the minimum work to keep up with changes.
+ *
+ * Can configure which changes it watches for, which files are considered
+ * relevant, and what extra data to compute for each file.
+ */
+export abstract class FilePathDiscovery<T extends PathData> extends Discovery {
+  /**
+   * Has `discover` been called. This allows distinguishing between
+   * "no paths found" and not having scanned yet.
+   */
+  private discoverHasCompletedOnce = false;
+
+  /** The set of known paths and associated data that we are tracking */
+  private pathData: T[] = [];
+
+  /** Event that fires whenever the contents of `pathData` changes */
+  private readonly onDidChangePathDataEmitter: AppEventEmitter<void>;
+
+  /**
+   * The set of file paths that may have changed on disk since the last time
+   * refresh was run. Whenever a watcher reports some change to a file we add
+   * it to this set, and then during the next refresh we will process all
+   * file paths from this set and update our internal state to match whatever
+   * we find on disk (i.e. the file exists, doesn't exist, computed data has
+   * changed).
+   */
+  private readonly changedFilePaths = new Set<string>();
+
+  /**
+   * Watches for changes to files and directories in all workspace folders.
+   */
+  private readonly watcher: MultiFileSystemWatcher = this.push(
+    new MultiFileSystemWatcher(),
+  );
+
+  /**
+   * @param name Name of the discovery operation, for logging purposes.
+   * @param fileWatchPattern Passed to `vscode.RelativePattern` to determine the files to watch for changes to.
+   */
+  constructor(
+    name: string,
+    private readonly fileWatchPattern: string,
+  ) {
+    super(name, extLogger);
+
+    this.onDidChangePathDataEmitter = this.push(new EventEmitter<void>());
+    this.push(
+      workspace.onDidChangeWorkspaceFolders(
+        this.workspaceFoldersChanged.bind(this),
+      ),
+    );
+    this.push(this.watcher.onDidChange(this.fileChanged.bind(this)));
+  }
+
+  protected getPathData(): ReadonlyArray<Readonly<T>> | undefined {
+    if (!this.discoverHasCompletedOnce) {
+      return undefined;
+    }
+    return this.pathData;
+  }
+
+  protected get onDidChangePathData(): Event<void> {
+    return this.onDidChangePathDataEmitter.event;
+  }
+
+  /**
+   * Compute any extra data to be stored regarding the given path.
+   */
+  protected abstract getDataForPath(path: string): Promise<T>;
+
+  /**
+   * Is the given path relevant to this discovery operation?
+   */
+  protected abstract pathIsRelevant(path: string): boolean;
+
+  /**
+   * Should the given new data overwrite the existing data we have stored?
+   */
+  protected abstract shouldOverwriteExistingData(
+    newData: T,
+    existingData: T,
+  ): boolean;
+
+  /**
+   * Update the data for every path by calling `getDataForPath`.
+   */
+  protected async recomputeAllData() {
+    this.pathData = await Promise.all(
+      this.pathData.map((p) => this.getDataForPath(p.path)),
+    );
+    this.onDidChangePathDataEmitter.fire();
+  }
+
+  /**
+   * Do the initial scan of the entire workspace and set up watchers for future changes.
+   */
+  public async initialRefresh() {
+    getOnDiskWorkspaceFolders().forEach((workspaceFolder) => {
+      this.changedFilePaths.add(workspaceFolder);
+    });
+
+    this.updateWatchers();
+    await this.refresh();
+    this.onDidChangePathDataEmitter.fire();
+  }
+
+  private workspaceFoldersChanged(event: WorkspaceFoldersChangeEvent) {
+    event.added.forEach((workspaceFolder) => {
+      this.changedFilePaths.add(workspaceFolder.uri.fsPath);
+    });
+    event.removed.forEach((workspaceFolder) => {
+      this.changedFilePaths.add(workspaceFolder.uri.fsPath);
+    });
+
+    this.updateWatchers();
+    void this.refresh();
+  }
+
+  private updateWatchers() {
+    this.watcher.clear();
+    for (const workspaceFolder of getOnDiskWorkspaceFoldersObjects()) {
+      // Watch for changes to individual files
+      this.watcher.addWatch(
+        new RelativePattern(workspaceFolder, this.fileWatchPattern),
+      );
+      // need to explicitly watch for changes to directories themselves.
+      this.watcher.addWatch(new RelativePattern(workspaceFolder, "**/"));
+    }
+  }
+
+  private fileChanged(uri: Uri) {
+    this.changedFilePaths.add(uri.fsPath);
+    void this.refresh();
+  }
+
+  protected async discover() {
+    let pathsUpdated = false;
+    for (const path of this.changedFilePaths) {
+      try {
+        this.changedFilePaths.delete(path);
+        if (await this.handleChangedPath(path)) {
+          pathsUpdated = true;
+        }
+      } catch (e) {
+        // If we get an error while processing a path, just log it and continue.
+        // There aren't any network operations happening here or anything else
+        // that's likely to succeed on a retry, so don't bother adding it back
+        // to the changedFilePaths set.
+        void extLogger.log(
+          `${
+            this.name
+          } failed while processing path "${path}": ${getErrorMessage(e)}`,
+        );
+      }
+    }
+
+    this.discoverHasCompletedOnce = true;
+    if (pathsUpdated) {
+      this.onDidChangePathDataEmitter.fire();
+    }
+  }
+
+  private async handleChangedPath(path: string): Promise<boolean> {
+    try {
+      // If the path is not in the workspace then we don't want to be
+      // tracking or displaying it, so treat it as if it doesn't exist.
+      if (!this.pathIsInWorkspace(path)) {
+        return this.handleRemovedPath(path);
+      }
+
+      if ((await lstat(path)).isDirectory()) {
+        return await this.handleChangedDirectory(path);
+      } else {
+        return this.handleChangedFile(path);
+      }
+    } catch (e) {
+      if (isIOError(e) && e.code === "ENOENT") {
+        return this.handleRemovedPath(path);
+      }
+      throw e;
+    }
+  }
+
+  private pathIsInWorkspace(path: string): boolean {
+    return getOnDiskWorkspaceFolders().some((workspaceFolder) =>
+      containsPath(workspaceFolder, path),
+    );
+  }
+
+  private handleRemovedPath(path: string): boolean {
+    const oldLength = this.pathData.length;
+    this.pathData = this.pathData.filter(
+      (existingPathData) => !containsPath(path, existingPathData.path),
+    );
+    return this.pathData.length !== oldLength;
+  }
+
+  private async handleChangedDirectory(path: string): Promise<boolean> {
+    const newPaths = await workspace.findFiles(
+      new RelativePattern(path, this.fileWatchPattern),
+    );
+
+    let pathsUpdated = false;
+    for (const path of newPaths) {
+      if (await this.addOrUpdatePath(path.fsPath)) {
+        pathsUpdated = true;
+      }
+    }
+    return pathsUpdated;
+  }
+
+  private async handleChangedFile(path: string): Promise<boolean> {
+    if (this.pathIsRelevant(path)) {
+      return await this.addOrUpdatePath(path);
+    } else {
+      return false;
+    }
+  }
+
+  private async addOrUpdatePath(path: string): Promise<boolean> {
+    const data = await this.getDataForPath(path);
+    const existingPathDataIndex = this.pathData.findIndex(
+      (existingPathData) => existingPathData.path === path,
+    );
+    if (existingPathDataIndex !== -1) {
+      if (
+        this.shouldOverwriteExistingData(
+          data,
+          this.pathData[existingPathDataIndex],
+        )
+      ) {
+        this.pathData.splice(existingPathDataIndex, 1, data);
+        return true;
+      } else {
+        return false;
+      }
+    } else {
+      this.pathData.push(data);
+      return true;
+    }
+  }
+}

extensions/ql-vscode/src/common/vscode/multi-file-system-watcher.ts

@@ -1,4 +1,4 @@
-import { DisposableObject } from "../../pure/disposable-object";
+import { DisposableObject } from "../disposable-object";
 import { EventEmitter, Event, Uri, GlobPattern, workspace } from "vscode";
 
 /**

extensions/ql-vscode/src/common/vscode/progress.ts

@@ -10,7 +10,10 @@ export class UserCancellationException extends Error {
    * @param message The error message
    * @param silent If silent is true, then this exception will avoid showing a warning message to the user.
    */
-  constructor(message?: string, public readonly silent = false) {
+  constructor(
+    message?: string,
+    public readonly silent = false,
+  ) {
     super(message);
   }
 }

extensions/ql-vscode/src/common/vscode/selection-commands.ts

@@ -1,9 +1,9 @@
-import { showAndLogErrorMessage } from "../../helpers";
 import {
   ExplorerSelectionCommandFunction,
   TreeViewContextMultiSelectionCommandFunction,
   TreeViewContextSingleSelectionCommandFunction,
 } from "../commands";
+import { showAndLogErrorMessage, NotificationLogger } from "../logging";
 
 // A hack to match types that are not an array, which is useful to help avoid
 // misusing createSingleSelectionCommand, e.g. where T accidentally gets instantiated
@@ -25,6 +25,7 @@ type SelectionCommand<T extends NotArray> = CreateSupertypeOf<
 >;
 
 export function createSingleSelectionCommand<T extends NotArray>(
+  logger: NotificationLogger,
   f: (argument: T) => Promise<void>,
   itemName: string,
 ): SelectionCommand<T> {
@@ -32,7 +33,10 @@ export function createSingleSelectionCommand<T extends NotArray>(
     if (multiSelect === undefined || multiSelect.length === 1) {
       return f(singleItem);
     } else {
-      void showAndLogErrorMessage(`Please select a single ${itemName}.`);
+      void showAndLogErrorMessage(
+        logger,
+        `Please select a single ${itemName}.`,
+      );
       return;
     }
   };

extensions/ql-vscode/src/common/vscode/telemetry.ts

@@ -13,13 +13,14 @@ import {
   LOG_TELEMETRY,
   isIntegrationTestMode,
   isCanary,
-} from "./config";
+} from "../../config";
 import * as appInsights from "applicationinsights";
-import { extLogger } from "./common";
-import { UserCancellationException } from "./common/vscode/progress";
-import { showBinaryChoiceWithUrlDialog } from "./helpers";
-import { RedactableError } from "./pure/errors";
+import { extLogger } from "../logging/vscode";
+import { UserCancellationException } from "./progress";
+import { showBinaryChoiceWithUrlDialog } from "./dialog";
+import { RedactableError } from "../errors";
 import { SemVer } from "semver";
+import { AppTelemetry } from "../telemetry";
 
 // Key is injected at build time through the APP_INSIGHTS_KEY environment variable.
 const key = "REPLACE-APP-INSIGHTS-KEY";
@@ -54,7 +55,10 @@ const baseDataPropertiesToRemove = [
 
 const NOT_SET_CLI_VERSION = "not-set";
 
-export class TelemetryListener extends ConfigListener {
+export class ExtensionTelemetryListener
+  extends ConfigListener
+  implements AppTelemetry
+{
   static relevantSettings = [ENABLE_TELEMETRY, CANARY_FEATURES];
 
   private reporter?: TelemetryReporter;
@@ -152,7 +156,7 @@ export class TelemetryListener extends ConfigListener {
     void this.reporter?.dispose();
   }
 
-  sendCommandUsage(name: string, executionTime: number, error?: Error) {
+  sendCommandUsage(name: string, executionTime: number, error?: Error): void {
     if (!this.reporter) {
       return;
     }
@@ -174,7 +178,7 @@ export class TelemetryListener extends ConfigListener {
     );
   }
 
-  sendUIInteraction(name: string) {
+  sendUIInteraction(name: string): void {
     if (!this.reporter) {
       return;
     }
@@ -193,7 +197,7 @@ export class TelemetryListener extends ConfigListener {
   sendError(
     error: RedactableError,
     extraProperties?: { [key: string]: string },
-  ) {
+  ): void {
     if (!this.reporter) {
       return;
     }
@@ -272,16 +276,16 @@ export class TelemetryListener extends ConfigListener {
 /**
  * The global Telemetry instance
  */
-export let telemetryListener: TelemetryListener | undefined;
+export let telemetryListener: ExtensionTelemetryListener | undefined;
 
 export async function initializeTelemetry(
   extension: Extension<any>,
   ctx: ExtensionContext,
-): Promise<TelemetryListener> {
+): Promise<ExtensionTelemetryListener> {
   if (telemetryListener !== undefined) {
     throw new Error("Telemetry is already initialized");
   }
-  telemetryListener = new TelemetryListener(
+  telemetryListener = new ExtensionTelemetryListener(
     extension.id,
     extension.packageJSON.version,
     key,

extensions/ql-vscode/src/common/vscode/vscode-app.ts

@@ -1,14 +1,17 @@
 import * as vscode from "vscode";
 import { VSCodeCredentials } from "./authentication";
-import { Disposable } from "../../pure/disposable-object";
+import { Disposable } from "../disposable-object";
 import { App, AppMode, EnvironmentContext } from "../app";
 import { AppEventEmitter } from "../events";
-import { extLogger, Logger, queryServerLogger } from "../logging";
+import { NotificationLogger } from "../logging";
+import { extLogger, queryServerLogger } from "../logging/vscode";
 import { Memento } from "../memento";
 import { VSCodeAppEventEmitter } from "./events";
 import { AppCommandManager, QueryServerCommandManager } from "../commands";
 import { createVSCodeCommandManager } from "./commands";
 import { AppEnvironmentContext } from "./environment-context";
+import { AppTelemetry } from "../telemetry";
+import { telemetryListener } from "./telemetry";
 
 export class ExtensionApp implements App {
   public readonly credentials: VSCodeCredentials;
@@ -40,14 +43,6 @@ export class ExtensionApp implements App {
     return this.extensionContext.workspaceState;
   }
 
-  public get workspaceFolders(): readonly vscode.WorkspaceFolder[] | undefined {
-    return vscode.workspace.workspaceFolders;
-  }
-
-  public get onDidChangeWorkspaceFolders(): vscode.Event<vscode.WorkspaceFoldersChangeEvent> {
-    return vscode.workspace.onDidChangeWorkspaceFolders;
-  }
-
   public get subscriptions(): Disposable[] {
     return this.extensionContext.subscriptions;
   }
@@ -63,10 +58,14 @@ export class ExtensionApp implements App {
     }
   }
 
-  public get logger(): Logger {
+  public get logger(): NotificationLogger {
     return extLogger;
   }
 
+  public get telemetry(): AppTelemetry | undefined {
+    return telemetryListener;
+  }
+
   public createEventEmitter<T>(): AppEventEmitter<T> {
     return new VSCodeAppEventEmitter<T>();
   }

extensions/ql-vscode/src/common/vscode/webview-html.ts

@@ -0,0 +1,101 @@
+import { ExtensionContext, Uri, Webview } from "vscode";
+import { randomBytes } from "crypto";
+import { EOL } from "os";
+
+export type WebviewView =
+  | "results"
+  | "compare"
+  | "variant-analysis"
+  | "data-flow-paths"
+  | "data-extensions-editor";
+
+export interface WebviewMessage {
+  t: string;
+}
+
+/**
+ * Returns HTML to populate the given webview.
+ * Uses a content security policy that only loads the given script.
+ */
+export function getHtmlForWebview(
+  ctx: ExtensionContext,
+  webview: Webview,
+  view: WebviewView,
+  {
+    allowInlineStyles,
+    allowWasmEval,
+  }: {
+    allowInlineStyles?: boolean;
+    allowWasmEval?: boolean;
+  } = {
+    allowInlineStyles: false,
+    allowWasmEval: false,
+  },
+): string {
+  const scriptUriOnDisk = Uri.file(ctx.asAbsolutePath("out/webview.js"));
+
+  const stylesheetUrisOnDisk = [
+    Uri.file(ctx.asAbsolutePath("out/webview.css")),
+  ];
+
+  // Convert the on-disk URIs into webview URIs.
+  const scriptWebviewUri = webview.asWebviewUri(scriptUriOnDisk);
+  const stylesheetWebviewUris = stylesheetUrisOnDisk.map(
+    (stylesheetUriOnDisk) => webview.asWebviewUri(stylesheetUriOnDisk),
+  );
+
+  // Use a nonce in the content security policy to uniquely identify the above resources.
+  const nonce = getNonce();
+
+  const stylesheetsHtmlLines = allowInlineStyles
+    ? stylesheetWebviewUris.map((uri) => createStylesLinkWithoutNonce(uri))
+    : stylesheetWebviewUris.map((uri) => createStylesLinkWithNonce(nonce, uri));
+
+  const styleSrc = allowInlineStyles
+    ? `${webview.cspSource} vscode-file: 'unsafe-inline'`
+    : `'nonce-${nonce}'`;
+
+  const fontSrc = webview.cspSource;
+
+  /*
+   * Content security policy:
+   * default-src: allow nothing by default.
+   * script-src:
+   *   - allow the given script, using the nonce.
+   *   - 'wasm-unsafe-eval: allow loading WebAssembly modules if necessary.
+   * style-src: allow only the given stylesheet, using the nonce.
+   * connect-src: only allow fetch calls to webview resource URIs
+   * (this is used to load BQRS result files).
+   */
+  return `
+<html>
+  <head>
+    <meta http-equiv="Content-Security-Policy"
+          content="default-src 'none'; script-src 'nonce-${nonce}'${
+            allowWasmEval ? " 'wasm-unsafe-eval'" : ""
+          }; font-src ${fontSrc}; style-src ${styleSrc}; connect-src ${
+            webview.cspSource
+          };">
+        ${stylesheetsHtmlLines.join(`    ${EOL}`)}
+  </head>
+  <body>
+    <div id=root data-view="${view}">
+    </div>
+      <script nonce="${nonce}" src="${scriptWebviewUri}">
+    </script>
+  </body>
+</html>`;
+}
+
+/** Gets a nonce string created with 128 bits of entropy. */
+function getNonce(): string {
+  return randomBytes(16).toString("base64");
+}
+
+function createStylesLinkWithNonce(nonce: string, uri: Uri): string {
+  return `<link nonce="${nonce}" rel="stylesheet" href="${uri}">`;
+}
+
+function createStylesLinkWithoutNonce(uri: Uri): string {
+  return `<link rel="stylesheet" href="${uri}">`;
+}

extensions/ql-vscode/src/common/vscode/workspace-folders.ts

@@ -0,0 +1,64 @@
+import { dirname, join } from "path";
+import { workspace, WorkspaceFolder } from "vscode";
+
+/** Returns true if the specified workspace folder is on the file system. */
+export function isWorkspaceFolderOnDisk(
+  workspaceFolder: WorkspaceFolder,
+): boolean {
+  return workspaceFolder.uri.scheme === "file";
+}
+
+/** Gets all active workspace folders that are on the filesystem. */
+export function getOnDiskWorkspaceFoldersObjects() {
+  const workspaceFolders = workspace.workspaceFolders ?? [];
+  return workspaceFolders.filter(isWorkspaceFolderOnDisk);
+}
+
+/** Gets all active workspace folders that are on the filesystem. */
+export function getOnDiskWorkspaceFolders() {
+  return getOnDiskWorkspaceFoldersObjects().map((folder) => folder.uri.fsPath);
+}
+
+/** Check if folder is already present in workspace */
+export function isFolderAlreadyInWorkspace(folderName: string) {
+  const workspaceFolders = workspace.workspaceFolders || [];
+
+  return !!workspaceFolders.find(
+    (workspaceFolder) => workspaceFolder.name === folderName,
+  );
+}
+
+/**
+ * Returns the path of the first folder in the workspace.
+ * This is used to decide where to create skeleton QL packs.
+ *
+ * If the first folder is a QL pack, then the parent folder is returned.
+ * This is because the vscode-codeql-starter repo contains a ql pack in
+ * the first folder.
+ *
+ * This is a temporary workaround until we can retire the
+ * vscode-codeql-starter repo.
+ */
+export function getFirstWorkspaceFolder() {
+  const workspaceFolders = getOnDiskWorkspaceFolders();
+
+  if (!workspaceFolders || workspaceFolders.length === 0) {
+    throw new Error("No workspace folders found");
+  }
+
+  const firstFolderFsPath = workspaceFolders[0];
+
+  // For the vscode-codeql-starter repo, the first folder will be a ql pack
+  // so we need to get the parent folder
+  if (
+    firstFolderFsPath.includes(
+      join("vscode-codeql-starter", "codeql-custom-queries"),
+    )
+  ) {
+    // return the parent folder
+    return dirname(firstFolderFsPath);
+  } else {
+    // if the first folder is not a ql pack, then we are in a normal workspace
+    return firstFolderFsPath;
+  }
+}

extensions/ql-vscode/src/common/word.ts

@@ -7,8 +7,9 @@ export function pluralize(
   numItems: number | undefined,
   singular: string,
   plural: string,
+  numberFormatter: (value: number) => string = (value) => value.toString(),
 ): string {
   return numItems !== undefined
-    ? `${numItems} ${numItems === 1 ? singular : plural}`
+    ? `${numberFormatter(numItems)} ${numItems === 1 ? singular : plural}`
     : "";
 }

extensions/ql-vscode/src/compare/compare-view.ts

@@ -4,27 +4,27 @@ import {
   FromCompareViewMessage,
   ToCompareViewMessage,
   QueryCompareResult,
-} from "../pure/interface-types";
-import { Logger } from "../common";
+} from "../common/interface-types";
+import { Logger, showAndLogExceptionWithTelemetry } from "../common/logging";
+import { extLogger } from "../common/logging/vscode";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { DatabaseManager } from "../databases/local-databases";
-import { jumpToLocation } from "../interface-utils";
+import { jumpToLocation } from "../databases/local-databases/locations";
 import {
   transformBqrsResultSet,
   RawResultSet,
   BQRSInfo,
-} from "../pure/bqrs-cli-types";
+} from "../common/bqrs-cli-types";
 import resultsDiff from "./resultsDiff";
 import { CompletedLocalQueryInfo } from "../query-results";
-import { assertNever, getErrorMessage } from "../pure/helpers-pure";
+import { assertNever, getErrorMessage } from "../common/helpers-pure";
 import { HistoryItemLabelProvider } from "../query-history/history-item-label-provider";
 import {
   AbstractWebview,
   WebviewPanelConfig,
 } from "../common/vscode/abstract-webview";
-import { telemetryListener } from "../telemetry";
-import { redactableError } from "../pure/errors";
-import { showAndLogExceptionWithTelemetry } from "../helpers";
+import { telemetryListener } from "../common/vscode/telemetry";
+import { redactableError } from "../common/errors";
 
 interface ComparePair {
   from: CompletedLocalQueryInfo;
@@ -130,7 +130,12 @@ export class CompareView extends AbstractWebview<
         break;
 
       case "viewSourceFile":
-        await jumpToLocation(msg, this.databaseManager, this.logger);
+        await jumpToLocation(
+          msg.databaseUri,
+          msg.loc,
+          this.databaseManager,
+          this.logger,
+        );
         break;
 
       case "openQuery":
@@ -146,6 +151,8 @@ export class CompareView extends AbstractWebview<
 
       case "unhandledError":
         void showAndLogExceptionWithTelemetry(
+          extLogger,
+          telemetryListener,
           redactableError(
             msg.error,
           )`Unhandled error in result comparison view: ${msg.error.message}`,

extensions/ql-vscode/src/compare/resultsDiff.ts

@@ -1,5 +1,5 @@
-import { RawResultSet } from "../pure/bqrs-cli-types";
-import { QueryCompareResult } from "../pure/interface-types";
+import { RawResultSet } from "../common/bqrs-cli-types";
+import { QueryCompareResult } from "../common/interface-types";
 
 /**
  * Compare the rows of two queries. Use deep equality to determine if

extensions/ql-vscode/src/config.ts

@@ -1,19 +1,20 @@
-import { DisposableObject } from "./pure/disposable-object";
+import { DisposableObject } from "./common/disposable-object";
 import {
   workspace,
   Event,
   EventEmitter,
   ConfigurationChangeEvent,
   ConfigurationTarget,
+  ConfigurationScope,
 } from "vscode";
 import { DistributionManager } from "./codeql-cli/distribution";
-import { extLogger } from "./common";
-import { ONE_DAY_IN_MS } from "./pure/time";
+import { extLogger } from "./common/logging/vscode";
+import { ONE_DAY_IN_MS } from "./common/time";
 import {
   FilterKey,
   SortKey,
   defaultFilterSortState,
-} from "./pure/variant-analysis-filter-sort";
+} from "./variant-analysis/shared/variant-analysis-filter-sort";
 
 export const ALL_SETTINGS: Setting[] = [];
 
@@ -44,12 +45,12 @@ export class Setting {
     }
   }
 
-  getValue<T>(): T {
+  getValue<T>(scope?: ConfigurationScope | null): T {
     if (this.parent === undefined) {
       throw new Error("Cannot get the value of a root setting.");
     }
     return workspace
-      .getConfiguration(this.parent.qualifiedName)
+      .getConfiguration(this.parent.qualifiedName, scope)
       .get<T>(this.name)!;
   }
 
@@ -69,6 +70,12 @@ export interface InspectionResult<T> {
   workspaceFolderValue?: T;
 }
 
+const VSCODE_DEBUG_SETTING = new Setting("debug", undefined);
+export const VSCODE_SAVE_BEFORE_START_SETTING = new Setting(
+  "saveBeforeStart",
+  VSCODE_DEBUG_SETTING,
+);
+
 const ROOT_SETTING = new Setting("codeQL");
 
 // Global configuration
@@ -160,10 +167,6 @@ export const NUMBER_OF_TEST_THREADS_SETTING = new Setting(
   RUNNING_TESTS_SETTING,
 );
 export const MAX_QUERIES = new Setting("maxQueries", RUNNING_QUERIES_SETTING);
-export const AUTOSAVE_SETTING = new Setting(
-  "autoSave",
-  RUNNING_QUERIES_SETTING,
-);
 export const PAGE_SIZE = new Setting("pageSize", RESULTS_DISPLAY_SETTING);
 const CUSTOM_LOG_DIRECTORY_SETTING = new Setting(
   "customLogDirectory",
@@ -692,7 +695,7 @@ const AUTOGENERATE_QL_PACKS = new Setting(
 );
 
 const AutogenerateQLPacksValues = ["ask", "never"] as const;
-type AutogenerateQLPacks = typeof AutogenerateQLPacksValues[number];
+type AutogenerateQLPacks = (typeof AutogenerateQLPacksValues)[number];
 
 export function getAutogenerateQlPacks(): AutogenerateQLPacks {
   const value = AUTOGENERATE_QL_PACKS.getValue<AutogenerateQLPacks>();
@@ -714,7 +717,30 @@ export function showQueriesPanel(): boolean {
 
 const DATA_EXTENSIONS = new Setting("dataExtensions", ROOT_SETTING);
 const LLM_GENERATION = new Setting("llmGeneration", DATA_EXTENSIONS);
+const FRAMEWORK_MODE = new Setting("frameworkMode", DATA_EXTENSIONS);
+const DISABLE_AUTO_NAME_EXTENSION_PACK = new Setting(
+  "disableAutoNameExtensionPack",
+  DATA_EXTENSIONS,
+);
+const EXTENSIONS_DIRECTORY = new Setting(
+  "extensionsDirectory",
+  DATA_EXTENSIONS,
+);
 
 export function showLlmGeneration(): boolean {
   return !!LLM_GENERATION.getValue<boolean>();
 }
+
+export function enableFrameworkMode(): boolean {
+  return !!FRAMEWORK_MODE.getValue<boolean>();
+}
+
+export function disableAutoNameExtensionPack(): boolean {
+  return !!DISABLE_AUTO_NAME_EXTENSION_PACK.getValue<boolean>();
+}
+
+export function getExtensionsDirectory(languageId: string): string | undefined {
+  return EXTENSIONS_DIRECTORY.getValue<string>({
+    languageId,
+  });
+}

extensions/ql-vscode/src/data-extensions-editor/auto-model-api.ts

@@ -34,7 +34,7 @@ export interface ModelRequest {
 
 export interface ModelResponse {
   language: string;
-  predicted: Method[];
+  predicted?: Method[];
 }
 
 export async function autoModel(

extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts

@@ -32,7 +32,7 @@ export async function getAutoModelUsages({
   // This will re-run the query that was already run when opening the data extensions editor. This
   // might be unnecessary, but this makes it really easy to get the path to the BQRS file which we
   // need to interpret the results.
-  const queryResult = await runQuery({
+  const queryResult = await runQuery("applicationModeQuery", {
     cliServer,
     queryRunner,
     queryStorageDir,

extensions/ql-vscode/src/data-extensions-editor/auto-model.ts

@@ -7,12 +7,21 @@ import {
   ModelRequest,
 } from "./auto-model-api";
 import type { UsageSnippetsBySignature } from "./auto-model-usages-query";
+import { groupMethods, sortGroupNames, sortMethods } from "./shared/sorting";
+import { Mode } from "./shared/mode";
+
+// Soft limit on the number of candidates to send to the model.
+// Note that the model may return fewer than this number of candidates.
+const candidateLimit = 20;
+// Soft limit on the number of samples to send to the model.
+const sampleLimit = 100;
 
 export function createAutoModelRequest(
   language: string,
   externalApiUsages: ExternalApiUsage[],
   modeledMethods: Record<string, ModeledMethod>,
   usages: UsageSnippetsBySignature,
+  mode: Mode,
 ): ModelRequest {
   const request: ModelRequest = {
     language,
@@ -20,11 +29,14 @@ export function createAutoModelRequest(
     candidates: [],
   };
 
-  // Sort by number of usages so we always send the most used methods first
-  externalApiUsages = [...externalApiUsages];
-  externalApiUsages.sort((a, b) => b.usages.length - a.usages.length);
+  // Sort the same way as the UI so we send the first ones listed in the UI first
+  const grouped = groupMethods(externalApiUsages, mode);
+  const sortedGroupNames = sortGroupNames(grouped);
+  const sortedExternalApiUsages = sortedGroupNames.flatMap((name) =>
+    sortMethods(grouped[name]),
+  );
 
-  for (const externalApiUsage of externalApiUsages) {
+  for (const externalApiUsage of sortedExternalApiUsages) {
     const modeledMethod: ModeledMethod = modeledMethods[
       externalApiUsage.signature
     ] ?? {
@@ -40,11 +52,15 @@ export function createAutoModelRequest(
         ? 0
         : externalApiUsage.methodParameters.split(",").length;
 
+    const candidates: Method[] = [];
+    const samples: Method[] = [];
     for (
-      let argumentIndex = 0;
+      let argumentIndex = -1; // Start at -1 which means `this` as in `this.method()`
       argumentIndex < numberOfArguments;
       argumentIndex++
     ) {
+      const argumentInput: string =
+        argumentIndex === -1 ? "Argument[this]" : `Argument[${argumentIndex}]`;
       const method: Method = {
         package: externalApiUsage.packageName,
         type: externalApiUsage.typeName,
@@ -54,21 +70,35 @@ export function createAutoModelRequest(
           modeledMethod.type === "none"
             ? undefined
             : toMethodClassification(modeledMethod),
-        usages: usagesForMethod.slice(0, 10),
-        input: `Argument[${argumentIndex}]`,
+        usages: usagesForMethod.slice(0, 6), // At most 6 usages per argument
+        input: argumentInput,
       };
 
+      // A method that is supported is modeled outside of the model file, so it is not a candidate.
+      // We also do not want it as a sample because we do not know the classification.
+      if (modeledMethod.type === "none" && externalApiUsage.supported) {
+        continue;
+      }
+
+      // Candidates are methods that are not currently modeled
       if (modeledMethod.type === "none") {
-        request.candidates.push(method);
+        candidates.push(method);
       } else {
-        request.samples.push(method);
+        samples.push(method);
       }
     }
+    // If there is room for at least one candidate, add all candidates.
+    // This ensures that we send all arguments for a method together.
+    // NOTE: this might go above the candidate limit, but that's okay.
+    if (request.candidates.length < candidateLimit) {
+      request.candidates.push(...candidates);
+    }
+    // Same for samples
+    if (request.samples.length < sampleLimit) {
+      request.samples.push(...samples);
+    }
   }
 
-  request.candidates = request.candidates.slice(0, 20);
-  request.samples = request.samples.slice(0, 100);
-
   return request;
 }
 
@@ -110,6 +140,12 @@ export function parsePredictedClassifications(
         input: "",
         output: "",
         provenance: "ai-generated",
+        signature,
+        // predictedBySignature[signature] always has at least element
+        packageName: predictedMethods[0].package,
+        typeName: predictedMethods[0].type,
+        methodName: predictedMethods[0].name,
+        methodParameters: predictedMethods[0].signature,
       };
       continue;
     }
@@ -127,6 +163,11 @@ export function parsePredictedClassifications(
       input: sink.input ?? "",
       output: sink.output ?? "",
       provenance: "ai-generated",
+      signature,
+      packageName: sink.package,
+      typeName: sink.type,
+      methodName: sink.name,
+      methodParameters: sink.signature,
     };
   }
 

extensions/ql-vscode/src/data-extensions-editor/bqrs.ts

@@ -1,5 +1,11 @@
-import { DecodedBqrsChunk } from "../pure/bqrs-cli-types";
-import { Call, ExternalApiUsage } from "./external-api-usage";
+import { DecodedBqrsChunk } from "../common/bqrs-cli-types";
+import {
+  Call,
+  CallClassification,
+  ExternalApiUsage,
+} from "./external-api-usage";
+import { ModeledMethodType } from "./modeled-method";
+import { parseLibraryFilename } from "./library";
 
 export function decodeBqrsToExternalApiUsages(
   chunk: DecodedBqrsChunk,
@@ -10,6 +16,10 @@ export function decodeBqrsToExternalApiUsages(
     const usage = tuple[0] as Call;
     const signature = tuple[1] as string;
     const supported = (tuple[2] as string) === "true";
+    let library = tuple[4] as string;
+    let libraryVersion: string | undefined = tuple[5] as string;
+    const type = tuple[6] as ModeledMethodType;
+    const classification = tuple[8] as CallClassification;
 
     const [packageWithType, methodDeclaration] = signature.split("#");
 
@@ -29,33 +39,42 @@ export function decodeBqrsToExternalApiUsages(
       methodDeclaration.indexOf("("),
     );
 
+    // For Java, we'll always get back a .jar file, and the library version may be bad because not all library authors
+    // properly specify the version. Therefore, we'll always try to parse the name and version from the library filename
+    // for Java.
+    if (library.endsWith(".jar") || libraryVersion === "") {
+      const { name, version } = parseLibraryFilename(library);
+      library = name;
+      if (version) {
+        libraryVersion = version;
+      }
+    }
+
+    if (libraryVersion === "") {
+      libraryVersion = undefined;
+    }
+
     if (!methodsByApiName.has(signature)) {
       methodsByApiName.set(signature, {
+        library,
+        libraryVersion,
         signature,
         packageName,
         typeName,
         methodName,
         methodParameters,
         supported,
+        supportedType: type,
         usages: [],
       });
     }
 
     const method = methodsByApiName.get(signature)!;
-    method.usages.push(usage);
+    method.usages.push({
+      ...usage,
+      classification,
+    });
   });
 
-  const externalApiUsages = Array.from(methodsByApiName.values());
-  externalApiUsages.sort((a, b) => {
-    // Sort first by supported, putting unmodeled methods first.
-    if (a.supported && !b.supported) {
-      return 1;
-    }
-    if (!a.supported && b.supported) {
-      return -1;
-    }
-    // Then sort by number of usages descending
-    return b.usages.length - a.usages.length;
-  });
-  return externalApiUsages;
+  return Array.from(methodsByApiName.values());
 }

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts

@@ -7,9 +7,9 @@ import { DatabaseManager } from "../databases/local-databases";
 import { ensureDir } from "fs-extra";
 import { join } from "path";
 import { App } from "../common/app";
-import { showAndLogErrorMessage } from "../helpers";
 import { withProgress } from "../common/vscode/progress";
-import { pickExtensionPackModelFile } from "./extension-pack-picker";
+import { pickExtensionPack } from "./extension-pack-picker";
+import { showAndLogErrorMessage } from "../common/logging";
 
 const SUPPORTED_LANGUAGES: string[] = ["java", "csharp"];
 
@@ -56,12 +56,13 @@ export class DataExtensionsEditorModule {
       "codeQL.openDataExtensionsEditor": async () => {
         const db = this.databaseManager.currentDatabaseItem;
         if (!db) {
-          void showAndLogErrorMessage("No database selected");
+          void showAndLogErrorMessage(this.app.logger, "No database selected");
           return;
         }
 
         if (!SUPPORTED_LANGUAGES.includes(db.language)) {
           void showAndLogErrorMessage(
+            this.app.logger,
             `The data extensions editor is not supported for ${db.language} databases.`,
           );
           return;
@@ -71,14 +72,16 @@ export class DataExtensionsEditorModule {
           async (progress, token) => {
             if (!(await this.cliServer.cliConstraints.supportsQlpacksKind())) {
               void showAndLogErrorMessage(
+                this.app.logger,
                 `This feature requires CodeQL CLI version ${CliVersionConstraint.CLI_VERSION_WITH_QLPACKS_KIND.format()} or later.`,
               );
               return;
             }
 
-            const modelFile = await pickExtensionPackModelFile(
+            const modelFile = await pickExtensionPack(
               this.cliServer,
               db,
+              this.app.logger,
               progress,
               token,
             );

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts

@@ -4,8 +4,8 @@ import {
   Uri,
   ViewColumn,
   window,
-  workspace,
 } from "vscode";
+import { join } from "path";
 import { RequestError } from "@octokit/request-error";
 import {
   AbstractWebview,
@@ -14,38 +14,43 @@ import {
 import {
   FromDataExtensionsEditorMessage,
   ToDataExtensionsEditorMessage,
-} from "../pure/interface-types";
+} from "../common/interface-types";
 import { ProgressUpdate } from "../common/vscode/progress";
 import { QueryRunner } from "../query-server";
 import {
-  showAndLogErrorMessage,
   showAndLogExceptionWithTelemetry,
-} from "../helpers";
-import { extLogger } from "../common";
-import { outputFile, pathExists, readFile } from "fs-extra";
+  showAndLogErrorMessage,
+} from "../common/logging";
+import { outputFile, readFile } from "fs-extra";
 import { load as loadYaml } from "js-yaml";
 import { DatabaseItem, DatabaseManager } from "../databases/local-databases";
 import { CodeQLCliServer } from "../codeql-cli/cli";
-import { asError, assertNever, getErrorMessage } from "../pure/helpers-pure";
+import { asError, assertNever, getErrorMessage } from "../common/helpers-pure";
 import { generateFlowModel } from "./generate-flow-model";
 import { promptImportGithubDatabase } from "../databases/database-fetcher";
 import { App } from "../common/app";
-import { ResolvableLocationValue } from "../pure/bqrs-cli-types";
-import { showResolvableLocation } from "../interface-utils";
+import { ResolvableLocationValue } from "../common/bqrs-cli-types";
+import { showResolvableLocation } from "../databases/local-databases/locations";
 import { decodeBqrsToExternalApiUsages } from "./bqrs";
-import { redactableError } from "../pure/errors";
+import { redactableError } from "../common/errors";
 import { readQueryResults, runQuery } from "./external-api-usage-query";
-import { createDataExtensionYaml, loadDataExtensionYaml } from "./yaml";
+import {
+  createDataExtensionYamlsForApplicationMode,
+  createDataExtensionYamlsForFrameworkMode,
+  loadDataExtensionYaml,
+} from "./yaml";
 import { ExternalApiUsage } from "./external-api-usage";
 import { ModeledMethod } from "./modeled-method";
-import { ExtensionPackModelFile } from "./shared/extension-pack";
+import { ExtensionPack } from "./shared/extension-pack";
 import { autoModel, ModelRequest, ModelResponse } from "./auto-model-api";
 import {
   createAutoModelRequest,
   parsePredictedClassifications,
 } from "./auto-model";
-import { showLlmGeneration } from "../config";
+import { enableFrameworkMode, showLlmGeneration } from "../config";
 import { getAutoModelUsages } from "./auto-model-usages-query";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { Mode } from "./shared/mode";
 
 export class DataExtensionsEditorView extends AbstractWebview<
   ToDataExtensionsEditorMessage,
@@ -59,7 +64,8 @@ export class DataExtensionsEditorView extends AbstractWebview<
     private readonly queryRunner: QueryRunner,
     private readonly queryStorageDir: string,
     private readonly databaseItem: DatabaseItem,
-    private readonly modelFile: ExtensionPackModelFile,
+    private readonly extensionPack: ExtensionPack,
+    private mode: Mode = Mode.Application,
   ) {
     super(ctx);
   }
@@ -78,6 +84,14 @@ export class DataExtensionsEditorView extends AbstractWebview<
       viewColumn: ViewColumn.Active,
       preserveFocus: true,
       view: "data-extensions-editor",
+      iconPath: {
+        dark: Uri.file(
+          join(this.ctx.extensionPath, "media/dark/symbol-misc.svg"),
+        ),
+        light: Uri.file(
+          join(this.ctx.extensionPath, "media/light/symbol-misc.svg"),
+        ),
+      },
     };
   }
 
@@ -96,14 +110,12 @@ export class DataExtensionsEditorView extends AbstractWebview<
       case "openExtensionPack":
         await this.app.commands.execute(
           "revealInExplorer",
-          Uri.file(this.modelFile.extensionPack.path),
+          Uri.file(this.extensionPack.path),
         );
 
         break;
-      case "openModelFile":
-        await window.showTextDocument(
-          await workspace.openTextDocument(this.modelFile.filename),
-        );
+      case "refreshExternalApiUsages":
+        await this.loadExternalApiUsages();
 
         break;
       case "jumpToUsage":
@@ -128,6 +140,12 @@ export class DataExtensionsEditorView extends AbstractWebview<
           msg.modeledMethods,
         );
 
+        break;
+      case "switchMode":
+        this.mode = msg.mode;
+
+        await Promise.all([this.setViewState(), this.loadExternalApiUsages()]);
+
         break;
       default:
         assertNever(msg);
@@ -148,9 +166,10 @@ export class DataExtensionsEditorView extends AbstractWebview<
     await this.postMessage({
       t: "setDataExtensionEditorViewState",
       viewState: {
-        extensionPackModelFile: this.modelFile,
-        modelFileExists: await pathExists(this.modelFile.filename),
+        extensionPack: this.extensionPack,
+        enableFrameworkMode: enableFrameworkMode(),
         showLlmButton: showLlmGeneration(),
+        mode: this.mode,
       },
     });
   }
@@ -167,10 +186,10 @@ export class DataExtensionsEditorView extends AbstractWebview<
             "Original file of this result is not in the database's source archive.",
           );
         } else {
-          void extLogger.log(`Unable to handleMsgFromView: ${e.message}`);
+          void this.app.logger.log(`Unable to handleMsgFromView: ${e.message}`);
         }
       } else {
-        void extLogger.log(`Unable to handleMsgFromView: ${e}`);
+        void this.app.logger.log(`Unable to handleMsgFromView: ${e}`);
       }
     }
   }
@@ -179,49 +198,80 @@ export class DataExtensionsEditorView extends AbstractWebview<
     externalApiUsages: ExternalApiUsage[],
     modeledMethods: Record<string, ModeledMethod>,
   ): Promise<void> {
-    const yaml = createDataExtensionYaml(
-      this.databaseItem.language,
-      externalApiUsages,
-      modeledMethods,
-    );
+    let yamls: Record<string, string>;
+    switch (this.mode) {
+      case Mode.Application:
+        yamls = createDataExtensionYamlsForApplicationMode(
+          this.databaseItem.language,
+          externalApiUsages,
+          modeledMethods,
+        );
+        break;
+      case Mode.Framework:
+        yamls = createDataExtensionYamlsForFrameworkMode(
+          this.databaseItem.name,
+          this.databaseItem.language,
+          externalApiUsages,
+          modeledMethods,
+        );
+        break;
+      default:
+        assertNever(this.mode);
+    }
 
-    await outputFile(this.modelFile.filename, yaml);
+    for (const [filename, yaml] of Object.entries(yamls)) {
+      await outputFile(join(this.extensionPack.path, filename), yaml);
+    }
 
-    void extLogger.log(
-      `Saved data extension YAML to ${this.modelFile.filename}`,
-    );
+    void this.app.logger.log(`Saved data extension YAML`);
   }
 
   protected async loadExistingModeledMethods(): Promise<void> {
     try {
-      if (!(await pathExists(this.modelFile.filename))) {
-        return;
+      const extensions = await this.cliServer.resolveExtensions(
+        this.extensionPack.path,
+        getOnDiskWorkspaceFolders(),
+      );
+
+      const modelFiles = new Set<string>();
+
+      if (this.extensionPack.path in extensions.data) {
+        for (const extension of extensions.data[this.extensionPack.path]) {
+          modelFiles.add(extension.file);
+        }
       }
 
-      const yaml = await readFile(this.modelFile.filename, "utf8");
+      const existingModeledMethods: Record<string, ModeledMethod> = {};
 
-      const data = loadYaml(yaml, {
-        filename: this.modelFile.filename,
-      });
+      for (const modelFile of modelFiles) {
+        const yaml = await readFile(modelFile, "utf8");
 
-      const existingModeledMethods = loadDataExtensionYaml(data);
+        const data = loadYaml(yaml, {
+          filename: modelFile,
+        });
 
-      if (!existingModeledMethods) {
-        void showAndLogErrorMessage(
-          `Failed to parse data extension YAML ${this.modelFile.filename}.`,
-        );
-        return;
+        const modeledMethods = loadDataExtensionYaml(data);
+        if (!modeledMethods) {
+          void showAndLogErrorMessage(
+            this.app.logger,
+            `Failed to parse data extension YAML ${modelFile}.`,
+          );
+          continue;
+        }
+
+        for (const [key, value] of Object.entries(modeledMethods)) {
+          existingModeledMethods[key] = value;
+        }
       }
 
       await this.postMessage({
-        t: "addModeledMethods",
+        t: "loadModeledMethods",
         modeledMethods: existingModeledMethods,
       });
     } catch (e: unknown) {
       void showAndLogErrorMessage(
-        `Unable to read data extension YAML ${
-          this.modelFile.filename
-        }: ${getErrorMessage(e)}`,
+        this.app.logger,
+        `Unable to read data extension YAML: ${getErrorMessage(e)}`,
       );
     }
   }
@@ -230,16 +280,21 @@ export class DataExtensionsEditorView extends AbstractWebview<
     const cancellationTokenSource = new CancellationTokenSource();
 
     try {
-      const queryResult = await runQuery({
-        cliServer: this.cliServer,
-        queryRunner: this.queryRunner,
-        databaseItem: this.databaseItem,
-        queryStorageDir: this.queryStorageDir,
-        progress: (progressUpdate: ProgressUpdate) => {
-          void this.showProgress(progressUpdate, 1500);
+      const queryResult = await runQuery(
+        this.mode === Mode.Framework
+          ? "frameworkModeQuery"
+          : "applicationModeQuery",
+        {
+          cliServer: this.cliServer,
+          queryRunner: this.queryRunner,
+          databaseItem: this.databaseItem,
+          queryStorageDir: this.queryStorageDir,
+          progress: (progressUpdate: ProgressUpdate) => {
+            void this.showProgress(progressUpdate, 1500);
+          },
+          token: cancellationTokenSource.token,
         },
-        token: cancellationTokenSource.token,
-      });
+      );
       if (!queryResult) {
         await this.clearProgress();
         return;
@@ -276,6 +331,8 @@ export class DataExtensionsEditorView extends AbstractWebview<
       await this.clearProgress();
     } catch (err) {
       void showAndLogExceptionWithTelemetry(
+        this.app.logger,
+        this.app.telemetry,
         redactableError(
           asError(err),
         )`Failed to load external API usages: ${getErrorMessage(err)}`,
@@ -286,31 +343,36 @@ export class DataExtensionsEditorView extends AbstractWebview<
   protected async generateModeledMethods(): Promise<void> {
     const tokenSource = new CancellationTokenSource();
 
-    const selectedDatabase = this.databaseManager.currentDatabaseItem;
+    let addedDatabase: DatabaseItem | undefined;
 
-    // The external API methods are in the library source code, so we need to ask
-    // the user to import the library database. We need to have the database
-    // imported to the query server, so we need to register it to our workspace.
-    const database = await promptImportGithubDatabase(
-      this.app.commands,
-      this.databaseManager,
-      this.app.workspaceStoragePath ?? this.app.globalStoragePath,
-      this.app.credentials,
-      (update) => this.showProgress(update),
-      tokenSource.token,
-      this.cliServer,
-    );
-    if (!database) {
-      await this.clearProgress();
-      void extLogger.log("No database chosen");
+    // In application mode, we need the database of a specific library to generate
+    // the modeled methods. In framework mode, we'll use the current database.
+    if (this.mode === Mode.Application) {
+      const selectedDatabase = this.databaseManager.currentDatabaseItem;
 
-      return;
+      // The external API methods are in the library source code, so we need to ask
+      // the user to import the library database. We need to have the database
+      // imported to the query server, so we need to register it to our workspace.
+      addedDatabase = await promptImportGithubDatabase(
+        this.app.commands,
+        this.databaseManager,
+        this.app.workspaceStoragePath ?? this.app.globalStoragePath,
+        this.app.credentials,
+        (update) => this.showProgress(update),
+        this.cliServer,
+      );
+      if (!addedDatabase) {
+        await this.clearProgress();
+        void this.app.logger.log("No database chosen");
+
+        return;
+      }
+
+      // The library database was set as the current database by importing it,
+      // but we need to set it back to the originally selected database.
+      await this.databaseManager.setCurrentDatabaseItem(selectedDatabase);
     }
 
-    // The library database was set as the current database by importing it,
-    // but we need to set it back to the originally selected database.
-    await this.databaseManager.setCurrentDatabaseItem(selectedDatabase);
-
     await this.showProgress({
       step: 0,
       maxStep: 4000,
@@ -322,18 +384,17 @@ export class DataExtensionsEditorView extends AbstractWebview<
         cliServer: this.cliServer,
         queryRunner: this.queryRunner,
         queryStorageDir: this.queryStorageDir,
-        databaseItem: database,
-        onResults: async (results) => {
+        databaseItem: addedDatabase ?? this.databaseItem,
+        onResults: async (modeledMethods) => {
           const modeledMethodsByName: Record<string, ModeledMethod> = {};
 
-          for (const result of results) {
-            modeledMethodsByName[result.signature] = result.modeledMethod;
+          for (const modeledMethod of modeledMethods) {
+            modeledMethodsByName[modeledMethod.signature] = modeledMethod;
           }
 
           await this.postMessage({
             t: "addModeledMethods",
             modeledMethods: modeledMethodsByName,
-            overrideNone: true,
           });
         },
         progress: (update) => this.showProgress(update),
@@ -341,24 +402,24 @@ export class DataExtensionsEditorView extends AbstractWebview<
       });
     } catch (e: unknown) {
       void showAndLogExceptionWithTelemetry(
+        this.app.logger,
+        this.app.telemetry,
         redactableError(
           asError(e),
         )`Failed to generate flow model: ${getErrorMessage(e)}`,
       );
     }
 
-    // After the flow model has been generated, we can remove the temporary database
-    // which we used for generating the flow model.
-    await this.databaseManager.removeDatabaseItem(
-      () =>
-        this.showProgress({
-          step: 3900,
-          maxStep: 4000,
-          message: "Removing temporary database",
-        }),
-      tokenSource.token,
-      database,
-    );
+    if (addedDatabase) {
+      // After the flow model has been generated, we can remove the temporary database
+      // which we used for generating the flow model.
+      await this.showProgress({
+        step: 3900,
+        maxStep: 4000,
+        message: "Removing temporary database",
+      });
+      await this.databaseManager.removeDatabaseItem(addedDatabase);
+    }
 
     await this.clearProgress();
   }
@@ -394,6 +455,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
       externalApiUsages,
       modeledMethods,
       usages,
+      this.mode,
     );
 
     await this.showProgress({
@@ -414,7 +476,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
     });
 
     const predictedModeledMethods = parsePredictedClassifications(
-      response.predicted,
+      response.predicted || [],
     );
 
     await this.showProgress({
@@ -426,7 +488,6 @@ export class DataExtensionsEditorView extends AbstractWebview<
     await this.postMessage({
       t: "addModeledMethods",
       modeledMethods: predictedModeledMethods,
-      overrideNone: true,
     });
 
     await this.clearProgress();
@@ -474,6 +535,8 @@ export class DataExtensionsEditorView extends AbstractWebview<
 
       if (e instanceof RequestError && e.status === 429) {
         void showAndLogExceptionWithTelemetry(
+          this.app.logger,
+          this.app.telemetry,
           redactableError(e)`Rate limit hit, please try again soon.`,
         );
         return null;

extensions/ql-vscode/src/data-extensions-editor/extension-pack-name.ts

@@ -0,0 +1,93 @@
+const packNamePartRegex = /[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/;
+const packNameRegex = new RegExp(
+  `^(?<scope>${packNamePartRegex.source})/(?<name>${packNamePartRegex.source})$`,
+);
+const packNameLength = 128;
+
+export interface ExtensionPackName {
+  scope: string;
+  name: string;
+}
+
+export function formatPackName(packName: ExtensionPackName): string {
+  return `${packName.scope}/${packName.name}`;
+}
+
+export function autoNameExtensionPack(
+  name: string,
+  language: string,
+): ExtensionPackName | undefined {
+  let packName = `${name}-${language}`;
+  if (!packName.includes("/")) {
+    packName = `pack/${packName}`;
+  }
+
+  const parts = packName.split("/");
+  const sanitizedParts = parts.map((part) => sanitizeExtensionPackName(part));
+
+  // If the scope is empty (e.g. if the given name is "-/b"), then we need to still set a scope
+  if (sanitizedParts[0].length === 0) {
+    sanitizedParts[0] = "pack";
+  }
+
+  return {
+    scope: sanitizedParts[0],
+    // This will ensure there's only 1 slash
+    name: sanitizedParts.slice(1).join("-"),
+  };
+}
+
+export function sanitizeExtensionPackName(name: string) {
+  // Lowercase everything
+  name = name.toLowerCase();
+
+  // Replace all spaces, dots, and underscores with hyphens
+  name = name.replaceAll(/[\s._]+/g, "-");
+
+  // Replace all characters which are not allowed by empty strings
+  name = name.replaceAll(/[^a-z0-9-]/g, "");
+
+  // Remove any leading or trailing hyphens
+  name = name.replaceAll(/^-|-$/g, "");
+
+  // Remove any duplicate hyphens
+  name = name.replaceAll(/-{2,}/g, "-");
+
+  return name;
+}
+
+export function parsePackName(packName: string): ExtensionPackName | undefined {
+  const matches = packNameRegex.exec(packName);
+  if (!matches?.groups) {
+    return;
+  }
+
+  const scope = matches.groups.scope;
+  const name = matches.groups.name;
+
+  return {
+    scope,
+    name,
+  };
+}
+
+export function validatePackName(name: string): string | undefined {
+  if (!name) {
+    return "Pack name must not be empty";
+  }
+
+  if (name.length > packNameLength) {
+    return `Pack name must be no longer than ${packNameLength} characters`;
+  }
+
+  const matches = packNameRegex.exec(name);
+  if (!matches?.groups) {
+    if (!name.includes("/")) {
+      return "Invalid package name: a pack name must contain a slash to separate the scope from the pack name";
+    }
+
+    return "Invalid package name: a pack name must contain only lowercase ASCII letters, ASCII digits, and hyphens";
+  }
+
+  return undefined;
+}

extensions/ql-vscode/src/data-extensions-editor/extension-pack-picker.ts

@@ -1,64 +1,37 @@
-import { join, relative, resolve, sep } from "path";
+import { join } from "path";
 import { outputFile, pathExists, readFile } from "fs-extra";
 import { dump as dumpYaml, load as loadYaml } from "js-yaml";
-import { minimatch } from "minimatch";
-import { CancellationToken, window } from "vscode";
-import { CodeQLCliServer } from "../codeql-cli/cli";
-import {
-  getOnDiskWorkspaceFolders,
-  getOnDiskWorkspaceFoldersObjects,
-  showAndLogErrorMessage,
-} from "../helpers";
+import { CancellationToken, Uri, window } from "vscode";
+import { CodeQLCliServer, QlpacksInfo } from "../codeql-cli/cli";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { ProgressCallback } from "../common/vscode/progress";
 import { DatabaseItem } from "../databases/local-databases";
-import { getQlPackPath, QLPACK_FILENAMES } from "../pure/ql";
-import { getErrorMessage } from "../pure/helpers-pure";
-import { ExtensionPack, ExtensionPackModelFile } from "./shared/extension-pack";
+import { getQlPackPath, QLPACK_FILENAMES } from "../common/ql";
+import { getErrorMessage } from "../common/helpers-pure";
+import { ExtensionPack } from "./shared/extension-pack";
+import { NotificationLogger, showAndLogErrorMessage } from "../common/logging";
+import {
+  disableAutoNameExtensionPack,
+  getExtensionsDirectory,
+} from "../config";
+import {
+  autoNameExtensionPack,
+  ExtensionPackName,
+  formatPackName,
+  parsePackName,
+  validatePackName,
+} from "./extension-pack-name";
+import {
+  askForWorkspaceFolder,
+  autoPickExtensionsDirectory,
+} from "./extensions-workspace-folder";
 
 const maxStep = 3;
 
-const packNamePartRegex = /[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/;
-const packNameRegex = new RegExp(
-  `^(?<scope>${packNamePartRegex.source})/(?<name>${packNamePartRegex.source})$`,
-);
-const packNameLength = 128;
-
-export async function pickExtensionPackModelFile(
-  cliServer: Pick<CodeQLCliServer, "resolveQlpacks" | "resolveExtensions">,
-  databaseItem: Pick<DatabaseItem, "name" | "language">,
-  progress: ProgressCallback,
-  token: CancellationToken,
-): Promise<ExtensionPackModelFile | undefined> {
-  const extensionPack = await pickExtensionPack(
-    cliServer,
-    databaseItem,
-    progress,
-    token,
-  );
-  if (!extensionPack) {
-    return undefined;
-  }
-
-  const modelFile = await pickModelFile(
-    cliServer,
-    databaseItem,
-    extensionPack,
-    progress,
-    token,
-  );
-  if (!modelFile) {
-    return;
-  }
-
-  return {
-    filename: modelFile,
-    extensionPack,
-  };
-}
-
-async function pickExtensionPack(
+export async function pickExtensionPack(
   cliServer: Pick<CodeQLCliServer, "resolveQlpacks">,
   databaseItem: Pick<DatabaseItem, "name" | "language">,
+  logger: NotificationLogger,
   progress: ProgressCallback,
   token: CancellationToken,
 ): Promise<ExtensionPack | undefined> {
@@ -75,6 +48,21 @@ async function pickExtensionPack(
     true,
   );
 
+  if (!disableAutoNameExtensionPack()) {
+    progress({
+      message: "Creating extension pack...",
+      step: 2,
+      maxStep,
+    });
+
+    return autoCreateExtensionPack(
+      databaseItem.name,
+      databaseItem.language,
+      extensionPacksInfo,
+      logger,
+    );
+  }
+
   if (Object.keys(extensionPacksInfo).length === 0) {
     return pickNewExtensionPack(databaseItem, token);
   }
@@ -84,6 +72,7 @@ async function pickExtensionPack(
       Object.entries(extensionPacksInfo).map(async ([name, paths]) => {
         if (paths.length !== 1) {
           void showAndLogErrorMessage(
+            logger,
             `Extension pack ${name} resolves to multiple paths`,
             {
               fullMessage: `Extension pack ${name} resolves to multiple paths: ${paths.join(
@@ -101,11 +90,15 @@ async function pickExtensionPack(
         try {
           extensionPack = await readExtensionPack(path);
         } catch (e: unknown) {
-          void showAndLogErrorMessage(`Could not read extension pack ${name}`, {
-            fullMessage: `Could not read extension pack ${name} at ${path}: ${getErrorMessage(
-              e,
-            )}`,
-          });
+          void showAndLogErrorMessage(
+            logger,
+            `Could not read extension pack ${name}`,
+            {
+              fullMessage: `Could not read extension pack ${name} at ${path}: ${getErrorMessage(
+                e,
+              )}`,
+            },
+          );
 
           return undefined;
         }
@@ -163,118 +156,39 @@ async function pickExtensionPack(
   return extensionPackOption.extensionPack;
 }
 
-async function pickModelFile(
-  cliServer: Pick<CodeQLCliServer, "resolveExtensions">,
-  databaseItem: Pick<DatabaseItem, "name">,
-  extensionPack: ExtensionPack,
-  progress: ProgressCallback,
-  token: CancellationToken,
-): Promise<string | undefined> {
-  // Find the existing model files in the extension pack
-  const additionalPacks = getOnDiskWorkspaceFolders();
-  const extensions = await cliServer.resolveExtensions(
-    extensionPack.path,
-    additionalPacks,
-  );
-
-  const modelFiles = new Set<string>();
-
-  if (extensionPack.path in extensions.data) {
-    for (const extension of extensions.data[extensionPack.path]) {
-      modelFiles.add(extension.file);
-    }
-  }
-
-  if (modelFiles.size === 0) {
-    return pickNewModelFile(databaseItem, extensionPack, token);
-  }
-
-  const fileOptions: Array<{ label: string; file: string | null }> = [];
-  for (const file of modelFiles) {
-    fileOptions.push({
-      label: relative(extensionPack.path, file).replaceAll(sep, "/"),
-      file,
-    });
-  }
-  fileOptions.push({
-    label: "Create new model file",
-    file: null,
-  });
-
-  progress({
-    message: "Choosing model file...",
-    step: 3,
-    maxStep,
-  });
-
-  const fileOption = await window.showQuickPick(
-    fileOptions,
-    {
-      title: "Select model file to use",
-    },
-    token,
-  );
-
-  if (!fileOption) {
-    return undefined;
-  }
-
-  if (fileOption.file) {
-    return fileOption.file;
-  }
-
-  return pickNewModelFile(databaseItem, extensionPack, token);
-}
-
 async function pickNewExtensionPack(
   databaseItem: Pick<DatabaseItem, "name" | "language">,
   token: CancellationToken,
 ): Promise<ExtensionPack | undefined> {
-  const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
-  const workspaceFolderOptions = workspaceFolders.map((folder) => ({
-    label: folder.name,
-    detail: folder.uri.fsPath,
-    path: folder.uri.fsPath,
-  }));
-
-  // We're not using window.showWorkspaceFolderPick because that also includes the database source folders while
-  // we only want to include on-disk workspace folders.
-  const workspaceFolder = await window.showQuickPick(workspaceFolderOptions, {
-    title: "Select workspace folder to create extension pack in",
-  });
+  const workspaceFolder = await askForWorkspaceFolder();
   if (!workspaceFolder) {
     return undefined;
   }
 
-  let examplePackName = `${databaseItem.name}-extensions`;
-  if (!examplePackName.includes("/")) {
-    examplePackName = `pack/${examplePackName}`;
-  }
+  const examplePackName = autoNameExtensionPack(
+    databaseItem.name,
+    databaseItem.language,
+  );
 
-  const packName = await window.showInputBox(
+  const name = await window.showInputBox(
     {
       title: "Create new extension pack",
       prompt: "Enter name of extension pack",
-      placeHolder: `e.g. ${examplePackName}`,
+      placeHolder: examplePackName
+        ? `e.g. ${formatPackName(examplePackName)}`
+        : "",
       validateInput: async (value: string): Promise<string | undefined> => {
-        if (!value) {
-          return "Pack name must not be empty";
+        const message = validatePackName(value);
+        if (message) {
+          return message;
         }
 
-        if (value.length > packNameLength) {
-          return `Pack name must be no longer than ${packNameLength} characters`;
+        const packName = parsePackName(value);
+        if (!packName) {
+          return "Invalid pack name";
         }
 
-        const matches = packNameRegex.exec(value);
-        if (!matches?.groups) {
-          if (!value.includes("/")) {
-            return "Invalid package name: a pack name must contain a slash to separate the scope from the pack name";
-          }
-
-          return "Invalid package name: a pack name must contain only lowercase ASCII letters, ASCII digits, and hyphens";
-        }
-
-        const packPath = join(workspaceFolder.path, matches.groups.name);
+        const packPath = join(workspaceFolder.uri.fsPath, packName.name);
         if (await pathExists(packPath)) {
           return `A pack already exists at ${packPath}`;
         }
@@ -284,31 +198,127 @@ async function pickNewExtensionPack(
     },
     token,
   );
+  if (!name) {
+    return undefined;
+  }
+
+  const packName = parsePackName(name);
   if (!packName) {
     return undefined;
   }
 
-  const matches = packNameRegex.exec(packName);
-  if (!matches?.groups) {
-    return;
-  }
-
-  const name = matches.groups.name;
-  const packPath = join(workspaceFolder.path, name);
+  const packPath = join(workspaceFolder.uri.fsPath, packName.name);
 
   if (await pathExists(packPath)) {
     return undefined;
   }
 
+  return writeExtensionPack(packPath, packName, databaseItem.language);
+}
+
+async function autoCreateExtensionPack(
+  name: string,
+  language: string,
+  extensionPacksInfo: QlpacksInfo,
+  logger: NotificationLogger,
+): Promise<ExtensionPack | undefined> {
+  // Get the `codeQL.dataExtensions.extensionsDirectory` setting for the language
+  const userExtensionsDirectory = getExtensionsDirectory(language);
+
+  // If the setting is not set, automatically pick a suitable directory
+  const extensionsDirectory = userExtensionsDirectory
+    ? Uri.file(userExtensionsDirectory)
+    : await autoPickExtensionsDirectory();
+
+  if (!extensionsDirectory) {
+    return undefined;
+  }
+
+  // Generate the name of the extension pack
+  const packName = autoNameExtensionPack(name, language);
+  if (!packName) {
+    void showAndLogErrorMessage(
+      logger,
+      `Could not automatically name extension pack for database ${name}`,
+    );
+
+    return undefined;
+  }
+
+  // Find any existing locations of this extension pack
+  const existingExtensionPackPaths =
+    extensionPacksInfo[formatPackName(packName)];
+
+  // If there is already an extension pack with this name, use it if it is valid
+  if (existingExtensionPackPaths?.length === 1) {
+    let extensionPack: ExtensionPack;
+    try {
+      extensionPack = await readExtensionPack(existingExtensionPackPaths[0]);
+    } catch (e: unknown) {
+      void showAndLogErrorMessage(
+        logger,
+        `Could not read extension pack ${formatPackName(packName)}`,
+        {
+          fullMessage: `Could not read extension pack ${formatPackName(
+            packName,
+          )} at ${existingExtensionPackPaths[0]}: ${getErrorMessage(e)}`,
+        },
+      );
+
+      return undefined;
+    }
+
+    return extensionPack;
+  }
+
+  // If there is already an existing extension pack with this name, but it resolves
+  // to multiple paths, then we can't use it
+  if (existingExtensionPackPaths?.length > 1) {
+    void showAndLogErrorMessage(
+      logger,
+      `Extension pack ${formatPackName(packName)} resolves to multiple paths`,
+      {
+        fullMessage: `Extension pack ${formatPackName(
+          packName,
+        )} resolves to multiple paths: ${existingExtensionPackPaths.join(
+          ", ",
+        )}`,
+      },
+    );
+
+    return undefined;
+  }
+
+  const packPath = join(extensionsDirectory.fsPath, packName.name);
+
+  if (await pathExists(packPath)) {
+    void showAndLogErrorMessage(
+      logger,
+      `Directory ${packPath} already exists for extension pack ${formatPackName(
+        packName,
+      )}`,
+    );
+
+    return undefined;
+  }
+
+  return writeExtensionPack(packPath, packName, language);
+}
+
+async function writeExtensionPack(
+  packPath: string,
+  packName: ExtensionPackName,
+  language: string,
+): Promise<ExtensionPack> {
   const packYamlPath = join(packPath, "codeql-pack.yml");
 
   const extensionPack: ExtensionPack = {
     path: packPath,
     yamlPath: packYamlPath,
-    name: packName,
+    name: formatPackName(packName),
     version: "0.0.0",
     extensionTargets: {
-      [`codeql/${databaseItem.language}-all`]: "*",
+      [`codeql/${language}-all`]: "*",
     },
     dataExtensions: ["models/**/*.yml"],
   };
@@ -327,53 +337,6 @@ async function pickNewExtensionPack(
   return extensionPack;
 }
 
-async function pickNewModelFile(
-  databaseItem: Pick<DatabaseItem, "name">,
-  extensionPack: ExtensionPack,
-  token: CancellationToken,
-) {
-  const filename = await window.showInputBox(
-    {
-      title: "Enter the name of the new model file",
-      value: `models/${databaseItem.name.replaceAll("/", ".")}.model.yml`,
-      validateInput: async (value: string): Promise<string | undefined> => {
-        if (value === "") {
-          return "File name must not be empty";
-        }
-
-        const path = resolve(extensionPack.path, value);
-
-        if (await pathExists(path)) {
-          return "File already exists";
-        }
-
-        const notInExtensionPack = relative(
-          extensionPack.path,
-          path,
-        ).startsWith("..");
-        if (notInExtensionPack) {
-          return "File must be in the extension pack";
-        }
-
-        const matchesPattern = extensionPack.dataExtensions.some((pattern) =>
-          minimatch(value, pattern, { matchBase: true }),
-        );
-        if (!matchesPattern) {
-          return `File must match one of the patterns in 'dataExtensions' in ${extensionPack.yamlPath}`;
-        }
-
-        return undefined;
-      },
-    },
-    token,
-  );
-  if (!filename) {
-    return undefined;
-  }
-
-  return resolve(extensionPack.path, filename);
-}
-
 async function readExtensionPack(path: string): Promise<ExtensionPack> {
   const qlpackPath = await getQlPackPath(path);
   if (!qlpackPath) {

extensions/ql-vscode/src/data-extensions-editor/extensions-workspace-folder.ts

@@ -0,0 +1,224 @@
+import { FileType, Uri, window, workspace, WorkspaceFolder } from "vscode";
+import { getOnDiskWorkspaceFoldersObjects } from "../common/vscode/workspace-folders";
+import { extLogger } from "../common/logging/vscode";
+import { tmpdir } from "../common/files";
+
+/**
+ * Returns the ancestors of this path in order from furthest to closest (i.e. root of filesystem to parent directory)
+ */
+function getAncestors(uri: Uri): Uri[] {
+  const ancestors: Uri[] = [];
+  let current = uri;
+  while (current.fsPath !== Uri.joinPath(current, "..").fsPath) {
+    ancestors.push(current);
+    current = Uri.joinPath(current, "..");
+  }
+
+  // The ancestors are now in order from closest to furthest, so reverse them
+  ancestors.reverse();
+
+  return ancestors;
+}
+
+async function getRootWorkspaceDirectory(): Promise<Uri | undefined> {
+  // If there is a valid workspace file, just use its directory as the directory for the extensions
+  const workspaceFile = workspace.workspaceFile;
+  if (workspaceFile?.scheme === "file") {
+    return Uri.joinPath(workspaceFile, "..");
+  }
+
+  const allWorkspaceFolders = getOnDiskWorkspaceFoldersObjects();
+
+  // Get the system temp directory and convert it to a URI so it's normalized
+  const systemTmpdir = Uri.file(tmpdir());
+
+  const workspaceFolders = allWorkspaceFolders.filter((folder) => {
+    // Never use a workspace folder that is in the system temp directory
+    return !folder.uri.fsPath.startsWith(systemTmpdir.fsPath);
+  });
+
+  // Find the common root directory of all workspace folders by finding the longest common prefix
+  const commonRoot = workspaceFolders.reduce((commonRoot, folder) => {
+    const folderUri = folder.uri;
+    const ancestors = getAncestors(folderUri);
+
+    const minLength = Math.min(commonRoot.length, ancestors.length);
+    let commonLength = 0;
+    for (let i = 0; i < minLength; i++) {
+      if (commonRoot[i].fsPath === ancestors[i].fsPath) {
+        commonLength++;
+      } else {
+        break;
+      }
+    }
+
+    return commonRoot.slice(0, commonLength);
+  }, getAncestors(workspaceFolders[0].uri));
+
+  if (commonRoot.length === 0) {
+    return await findGitFolder(workspaceFolders);
+  }
+
+  // The path closest to the workspace folders is the last element of the common root
+  const commonRootUri = commonRoot[commonRoot.length - 1];
+
+  // If we are at the root of the filesystem, we can't go up any further and there's something
+  // wrong, so just return undefined
+  if (commonRootUri.fsPath === Uri.joinPath(commonRootUri, "..").fsPath) {
+    return await findGitFolder(workspaceFolders);
+  }
+
+  return commonRootUri;
+}
+
+async function findGitFolder(
+  workspaceFolders: WorkspaceFolder[],
+): Promise<Uri | undefined> {
+  // Go through all workspace folders one-by-one and try to find the closest .git folder for each one
+  const folders = await Promise.all(
+    workspaceFolders.map(async (folder) => {
+      const ancestors = getAncestors(folder.uri);
+
+      // Reverse the ancestors so we're going from closest to furthest
+      ancestors.reverse();
+
+      const gitFoldersExists = await Promise.all(
+        ancestors.map(async (uri) => {
+          const gitFolder = Uri.joinPath(uri, ".git");
+          try {
+            const stat = await workspace.fs.stat(gitFolder);
+            // Check whether it's a directory
+            return (stat.type & FileType.Directory) !== 0;
+          } catch (e) {
+            return false;
+          }
+        }),
+      );
+
+      // Find the first ancestor that has a .git folder
+      const ancestorIndex = gitFoldersExists.findIndex((exists) => exists);
+
+      if (ancestorIndex === -1) {
+        return undefined;
+      }
+
+      return [ancestorIndex, ancestors[ancestorIndex]];
+    }),
+  );
+
+  const validFolders = folders.filter(
+    (folder): folder is [number, Uri] => folder !== undefined,
+  );
+  if (validFolders.length === 0) {
+    return undefined;
+  }
+
+  // Find the .git folder which is closest to a workspace folder
+  const closestFolder = validFolders.reduce((closestFolder, folder) => {
+    if (folder[0] < closestFolder[0]) {
+      return folder;
+    }
+    return closestFolder;
+  }, validFolders[0]);
+
+  return closestFolder?.[1];
+}
+
+/**
+ * Finds a suitable directory for extension packs to be created in. This will
+ * always be a path ending in `.github/codeql/extensions`. The parent directory
+ * will be determined heuristically based on the on-disk workspace folders.
+ *
+ * The heuristic is as follows (`.github/codeql/extensions` is added automatically unless
+ * otherwise specified):
+ * 1. If there is only 1 workspace folder, use that folder
+ * 2. If there is a workspace folder for which the path ends in `.github/codeql/extensions`, use that folder
+ *   - If there are multiple such folders, use the first one
+ *   - Does not append `.github/codeql/extensions` to the path
+ * 3. If there is a workspace file (`<basename>.code-workspace`), use the directory containing that file
+ * 4. If there is a common root directory for all workspace folders, use that directory
+ *   - Workspace folders in the system temp directory are ignored
+ *   - If the common root directory is the root of the filesystem, then it's not used
+ * 5. If there is a .git directory in any workspace folder, use the directory containing that .git directory
+ *    for which the .git directory is closest to a workspace folder
+ * 6. If none of the above apply, return `undefined`
+ */
+export async function autoPickExtensionsDirectory(): Promise<Uri | undefined> {
+  const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
+
+  // If there's only 1 workspace folder, use the `.github/codeql/extensions` directory in that folder
+  if (workspaceFolders.length === 1) {
+    return Uri.joinPath(
+      workspaceFolders[0].uri,
+      ".github",
+      "codeql",
+      "extensions",
+    );
+  }
+
+  // Now try to find a workspace folder for which the path ends in `.github/codeql/extensions`
+  const workspaceFolderForExtensions = workspaceFolders.find((folder) =>
+    // Using path instead of fsPath because path always uses forward slashes
+    folder.uri.path.endsWith(".github/codeql/extensions"),
+  );
+  if (workspaceFolderForExtensions) {
+    return workspaceFolderForExtensions.uri;
+  }
+
+  // Get the root workspace directory, i.e. the common root directory of all workspace folders
+  const rootDirectory = await getRootWorkspaceDirectory();
+  if (!rootDirectory) {
+    void extLogger.log("Unable to determine root workspace directory");
+
+    return undefined;
+  }
+
+  // We'll create a new workspace folder for the extensions in the root workspace directory
+  // at `.github/codeql/extensions`
+  const extensionsUri = Uri.joinPath(
+    rootDirectory,
+    ".github",
+    "codeql",
+    "extensions",
+  );
+
+  if (
+    !workspace.updateWorkspaceFolders(
+      workspace.workspaceFolders?.length ?? 0,
+      0,
+      {
+        name: "CodeQL Extension Packs",
+        uri: extensionsUri,
+      },
+    )
+  ) {
+    void extLogger.log(
+      `Failed to add workspace folder for extensions at ${extensionsUri.fsPath}`,
+    );
+    return undefined;
+  }
+
+  return extensionsUri;
+}
+
+export async function askForWorkspaceFolder(): Promise<
+  WorkspaceFolder | undefined
+> {
+  const workspaceFolders = getOnDiskWorkspaceFoldersObjects();
+  const workspaceFolderOptions = workspaceFolders.map((folder) => ({
+    label: folder.name,
+    detail: folder.uri.fsPath,
+    folder,
+  }));
+
+  // We're not using window.showWorkspaceFolderPick because that also includes the database source folders while
+  // we only want to include on-disk workspace folders.
+  const workspaceFolder = await window.showQuickPick(workspaceFolderOptions, {
+    title: "Select workspace folder to create extension pack in",
+  });
+  if (!workspaceFolder) {
+    return undefined;
+  }
+
+  return workspaceFolder.folder;
+}

extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts

@@ -2,20 +2,20 @@ import { CoreCompletedQuery, QueryRunner } from "../query-server";
 import { dir } from "tmp-promise";
 import { writeFile } from "fs-extra";
 import { dump as dumpYaml } from "js-yaml";
-import {
-  getOnDiskWorkspaceFolders,
-  isQueryLanguage,
-  showAndLogExceptionWithTelemetry,
-} from "../helpers";
-import { TeeLogger } from "../common";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { extLogger } from "../common/logging/vscode";
+import { showAndLogExceptionWithTelemetry, TeeLogger } from "../common/logging";
+import { isQueryLanguage } from "../common/query-language";
 import { CancellationToken } from "vscode";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { DatabaseItem } from "../databases/local-databases";
 import { ProgressCallback } from "../common/vscode/progress";
 import { fetchExternalApiQueries } from "./queries";
-import { QueryResultType } from "../pure/new-messages";
+import { QueryResultType } from "../query-server/new-messages";
 import { join } from "path";
-import { redactableError } from "../pure/errors";
+import { redactableError } from "../common/errors";
+import { telemetryListener } from "../common/vscode/telemetry";
+import { Query } from "./queries/query";
 
 export type RunQueryOptions = {
   cliServer: Pick<CodeQLCliServer, "resolveQlpacks">;
@@ -27,14 +27,17 @@ export type RunQueryOptions = {
   token: CancellationToken;
 };
 
-export async function runQuery({
-  cliServer,
-  queryRunner,
-  databaseItem,
-  queryStorageDir,
-  progress,
-  token,
-}: RunQueryOptions): Promise<CoreCompletedQuery | undefined> {
+export async function runQuery(
+  queryName: keyof Omit<Query, "dependencies">,
+  {
+    cliServer,
+    queryRunner,
+    databaseItem,
+    queryStorageDir,
+    progress,
+    token,
+  }: RunQueryOptions,
+): Promise<CoreCompletedQuery | undefined> {
   // The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
   // move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
   // This is intentionally not pretty code, as it will be removed soon.
@@ -43,6 +46,8 @@ export async function runQuery({
 
   if (!isQueryLanguage(databaseItem.language)) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`Unsupported database language ${databaseItem.language}`,
     );
     return;
@@ -51,6 +56,8 @@ export async function runQuery({
   const query = fetchExternalApiQueries[databaseItem.language];
   if (!query) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`No external API usage query found for language ${databaseItem.language}`,
     );
     return;
@@ -58,7 +65,7 @@ export async function runQuery({
 
   const queryDir = (await dir({ unsafeCleanup: true })).path;
   const queryFile = join(queryDir, "FetchExternalApis.ql");
-  await writeFile(queryFile, query.mainQuery, "utf8");
+  await writeFile(queryFile, query[queryName], "utf8");
 
   if (query.dependencies) {
     for (const [filename, contents] of Object.entries(query.dependencies)) {
@@ -106,6 +113,8 @@ export async function runQuery({
 
   if (completedQuery.resultType !== QueryResultType.SUCCESS) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`External API usage query failed: ${
         completedQuery.message ?? "No message"
       }`,
@@ -128,6 +137,8 @@ export async function readQueryResults({
   const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
   if (bqrsInfo["result-sets"].length !== 1) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`,
     );
     return undefined;

extensions/ql-vscode/src/data-extensions-editor/external-api-usage.ts

@@ -1,19 +1,54 @@
-import { ResolvableLocationValue } from "../pure/bqrs-cli-types";
+import { ResolvableLocationValue } from "../common/bqrs-cli-types";
+import { ModeledMethodType } from "./modeled-method";
 
 export type Call = {
   label: string;
   url: ResolvableLocationValue;
 };
 
-export type ExternalApiUsage = {
+export enum CallClassification {
+  Unknown = "unknown",
+  Source = "source",
+  Test = "test",
+  Generated = "generated",
+}
+
+export type Usage = Call & {
+  classification: CallClassification;
+};
+
+export interface MethodSignature {
   /**
-   * Contains the full method signature, e.g. `org.sql2o.Connection#createQuery(String)`
+   * Contains the version of the library if it can be determined by CodeQL, e.g. `4.2.2.2`
+   */
+  libraryVersion?: string;
+  /**
+   * A unique signature that can be used to identify this external API usage.
+   *
+   * The signature contains the package name, type name, method name, and method parameters
+   * in the form "packageName.typeName#methodName(methodParameters)".
+   * e.g. `org.sql2o.Connection#createQuery(String)`
    */
   signature: string;
   packageName: string;
   typeName: string;
   methodName: string;
+  /**
+   * The method parameters, including enclosing parentheses, e.g. `(String, String)`
+   */
   methodParameters: string;
+}
+
+export interface ExternalApiUsage extends MethodSignature {
+  /**
+   * Contains the name of the library containing the method declaration, e.g. `sql2o-1.6.0.jar` or `System.Runtime.dll`
+   */
+  library: string;
+  /**
+   * Is this method already supported by CodeQL standard libraries.
+   * If so, there is no need for the user to model it themselves.
+   */
   supported: boolean;
-  usages: Call[];
-};
+  supportedType: ModeledMethodType;
+  usages: Usage[];
+}

extensions/ql-vscode/src/data-extensions-editor/generate-flow-model.ts

@@ -3,23 +3,19 @@ import { DatabaseItem } from "../databases/local-databases";
 import { basename } from "path";
 import { QueryRunner } from "../query-server";
 import { CodeQLCliServer } from "../codeql-cli/cli";
-import { TeeLogger } from "../common";
+import { showAndLogExceptionWithTelemetry, TeeLogger } from "../common/logging";
+import { extLogger } from "../common/logging/vscode";
 import { extensiblePredicateDefinitions } from "./predicates";
 import { ProgressCallback } from "../common/vscode/progress";
-import {
-  getOnDiskWorkspaceFolders,
-  showAndLogExceptionWithTelemetry,
-} from "../helpers";
-import {
-  ModeledMethodType,
-  ModeledMethodWithSignature,
-} from "./modeled-method";
-import { redactableError } from "../pure/errors";
-import { QueryResultType } from "../pure/new-messages";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
+import { ModeledMethod, ModeledMethodType } from "./modeled-method";
+import { redactableError } from "../common/errors";
+import { QueryResultType } from "../query-server/new-messages";
 import { file } from "tmp-promise";
 import { writeFile } from "fs-extra";
 import { dump } from "js-yaml";
 import { qlpackOfDatabase } from "../language-support";
+import { telemetryListener } from "../common/vscode/telemetry";
 
 type FlowModelOptions = {
   cliServer: CodeQLCliServer;
@@ -28,7 +24,7 @@ type FlowModelOptions = {
   databaseItem: DatabaseItem;
   progress: ProgressCallback;
   token: CancellationToken;
-  onResults: (results: ModeledMethodWithSignature[]) => void | Promise<void>;
+  onResults: (results: ModeledMethod[]) => void | Promise<void>;
 };
 
 async function resolveQueries(
@@ -80,9 +76,11 @@ async function getModeledMethodsFromFlow(
     progress,
     token,
   }: Omit<FlowModelOptions, "onResults">,
-): Promise<ModeledMethodWithSignature[]> {
+): Promise<ModeledMethod[]> {
   if (queryPath === undefined) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`Failed to find ${type} query`,
     );
     return [];
@@ -117,6 +115,8 @@ async function getModeledMethodsFromFlow(
   );
   if (queryResult.resultType !== QueryResultType.SUCCESS) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`Failed to run ${basename(queryPath)} query: ${
         queryResult.message ?? "No message"
       }`,
@@ -129,6 +129,8 @@ async function getModeledMethodsFromFlow(
   const bqrsInfo = await cliServer.bqrsInfo(bqrsPath);
   if (bqrsInfo["result-sets"].length !== 1) {
     void showAndLogExceptionWithTelemetry(
+      extLogger,
+      telemetryListener,
       redactableError`Expected exactly one result set, got ${
         bqrsInfo["result-sets"].length
       } for ${basename(queryPath)}`,

extensions/ql-vscode/src/data-extensions-editor/library.ts

@@ -0,0 +1,58 @@
+import { basename, extname } from "../common/path";
+
+// From the semver package using
+// const { re, t } = require("semver/internal/re");
+// console.log(re[t.LOOSE]);
+// Modifications:
+// - Added version named group which does not capture the v prefix
+// - Removed the ^ and $ anchors
+// - Made the minor and patch versions optional
+// - Added a hyphen to the start of the version
+// - Added a dot as a valid separator between the version and the label
+// - Made the patch version optional even if a label is given
+// This will match any semver string at the end of a larger string
+const semverRegex =
+  /-[v=\s]*(?<version>([0-9]+)(\.([0-9]+)(?:(\.([0-9]+))?(?:[-.]?((?:[0-9]+|\d*[a-zA-Z-][a-zA-Z0-9-]*)(?:\.(?:[0-9]+|\d*[a-zA-Z-][a-zA-Z0-9-]*))*))?(?:\+([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?)?)?)/g;
+
+export interface Library {
+  name: string;
+  version?: string;
+}
+
+export function parseLibraryFilename(filename: string): Library {
+  let libraryName = basename(filename);
+  const extension = extname(libraryName);
+  libraryName = libraryName.slice(0, -extension.length);
+
+  let libraryVersion: string | undefined;
+
+  let match: RegExpMatchArray | null = null;
+
+  // Reset the regex
+  semverRegex.lastIndex = 0;
+
+  // Find the last occurence of the regex within the library name
+  // eslint-disable-next-line no-constant-condition
+  while (true) {
+    const currentMatch = semverRegex.exec(libraryName);
+    if (currentMatch === null) {
+      break;
+    }
+
+    match = currentMatch;
+  }
+
+  if (match?.groups) {
+    libraryVersion = match.groups?.version;
+    // Remove everything after the start of the match
+    libraryName = libraryName.slice(0, match.index);
+  }
+
+  // Remove any leading or trailing hyphens or dots
+  libraryName = libraryName.replaceAll(/^[.-]+|[.-]+$/g, "");
+
+  return {
+    name: libraryName,
+    version: libraryVersion,
+  };
+}

extensions/ql-vscode/src/data-extensions-editor/modeled-method.ts

@@ -1,3 +1,5 @@
+import { MethodSignature } from "./external-api-usage";
+
 export type ModeledMethodType =
   | "none"
   | "source"
@@ -17,15 +19,10 @@ export type Provenance =
   // Entered by the user in the editor manually
   | "manual";
 
-export type ModeledMethod = {
+export interface ModeledMethod extends MethodSignature {
   type: ModeledMethodType;
   input: string;
   output: string;
   kind: string;
   provenance: Provenance;
-};
-
-export type ModeledMethodWithSignature = {
-  signature: string;
-  modeledMethod: ModeledMethod;
-};
+}

extensions/ql-vscode/src/data-extensions-editor/predicates.ts

@@ -1,20 +1,9 @@
-import { ExternalApiUsage } from "./external-api-usage";
-import {
-  ModeledMethod,
-  ModeledMethodType,
-  ModeledMethodWithSignature,
-  Provenance,
-} from "./modeled-method";
-
-export type ExternalApiUsageByType = {
-  externalApiUsage: ExternalApiUsage;
-  modeledMethod: ModeledMethod;
-};
+import { ModeledMethod, ModeledMethodType, Provenance } from "./modeled-method";
 
 export type ExtensiblePredicateDefinition = {
   extensiblePredicate: string;
-  generateMethodDefinition: (method: ExternalApiUsageByType) => Tuple[];
-  readModeledMethod: (row: Tuple[]) => ModeledMethodWithSignature;
+  generateMethodDefinition: (method: ModeledMethod) => Tuple[];
+  readModeledMethod: (row: Tuple[]) => ModeledMethod;
 
   supportedKinds?: string[];
 };
@@ -36,25 +25,27 @@ export const extensiblePredicateDefinitions: Record<
     //   string output, string kind, string provenance
     // );
     generateMethodDefinition: (method) => [
-      method.externalApiUsage.packageName,
-      method.externalApiUsage.typeName,
+      method.packageName,
+      method.typeName,
       true,
-      method.externalApiUsage.methodName,
-      method.externalApiUsage.methodParameters,
+      method.methodName,
+      method.methodParameters,
       "",
-      method.modeledMethod.output,
-      method.modeledMethod.kind,
-      method.modeledMethod.provenance,
+      method.output,
+      method.kind,
+      method.provenance,
     ],
     readModeledMethod: (row) => ({
+      type: "source",
+      input: "",
+      output: row[6] as string,
+      kind: row[7] as string,
+      provenance: row[8] as Provenance,
       signature: readRowToMethod(row),
-      modeledMethod: {
-        type: "source",
-        input: "",
-        output: row[6] as string,
-        kind: row[7] as string,
-        provenance: row[8] as Provenance,
-      },
+      packageName: row[0] as string,
+      typeName: row[1] as string,
+      methodName: row[3] as string,
+      methodParameters: row[4] as string,
     }),
     supportedKinds: ["remote"],
   },
@@ -65,25 +56,27 @@ export const extensiblePredicateDefinitions: Record<
     //   string input, string kind, string provenance
     // );
     generateMethodDefinition: (method) => [
-      method.externalApiUsage.packageName,
-      method.externalApiUsage.typeName,
+      method.packageName,
+      method.typeName,
       true,
-      method.externalApiUsage.methodName,
-      method.externalApiUsage.methodParameters,
+      method.methodName,
+      method.methodParameters,
       "",
-      method.modeledMethod.input,
-      method.modeledMethod.kind,
-      method.modeledMethod.provenance,
+      method.input,
+      method.kind,
+      method.provenance,
     ],
     readModeledMethod: (row) => ({
+      type: "sink",
+      input: row[6] as string,
+      output: "",
+      kind: row[7] as string,
+      provenance: row[8] as Provenance,
       signature: readRowToMethod(row),
-      modeledMethod: {
-        type: "sink",
-        input: row[6] as string,
-        output: "",
-        kind: row[7] as string,
-        provenance: row[8] as Provenance,
-      },
+      packageName: row[0] as string,
+      typeName: row[1] as string,
+      methodName: row[3] as string,
+      methodParameters: row[4] as string,
     }),
     supportedKinds: ["sql", "xss", "logging"],
   },
@@ -94,26 +87,28 @@ export const extensiblePredicateDefinitions: Record<
     //   string input, string output, string kind, string provenance
     // );
     generateMethodDefinition: (method) => [
-      method.externalApiUsage.packageName,
-      method.externalApiUsage.typeName,
+      method.packageName,
+      method.typeName,
       true,
-      method.externalApiUsage.methodName,
-      method.externalApiUsage.methodParameters,
+      method.methodName,
+      method.methodParameters,
       "",
-      method.modeledMethod.input,
-      method.modeledMethod.output,
-      method.modeledMethod.kind,
-      method.modeledMethod.provenance,
+      method.input,
+      method.output,
+      method.kind,
+      method.provenance,
     ],
     readModeledMethod: (row) => ({
+      type: "summary",
+      input: row[6] as string,
+      output: row[7] as string,
+      kind: row[8] as string,
+      provenance: row[9] as Provenance,
       signature: readRowToMethod(row),
-      modeledMethod: {
-        type: "summary",
-        input: row[6] as string,
-        output: row[7] as string,
-        kind: row[8] as string,
-        provenance: row[9] as Provenance,
-      },
+      packageName: row[0] as string,
+      typeName: row[1] as string,
+      methodName: row[3] as string,
+      methodParameters: row[4] as string,
     }),
     supportedKinds: ["taint", "value"],
   },
@@ -123,22 +118,24 @@ export const extensiblePredicateDefinitions: Record<
     //   string package, string type, string name, string signature, string kind, string provenance
     // );
     generateMethodDefinition: (method) => [
-      method.externalApiUsage.packageName,
-      method.externalApiUsage.typeName,
-      method.externalApiUsage.methodName,
-      method.externalApiUsage.methodParameters,
-      method.modeledMethod.kind,
-      method.modeledMethod.provenance,
+      method.packageName,
+      method.typeName,
+      method.methodName,
+      method.methodParameters,
+      method.kind,
+      method.provenance,
     ],
     readModeledMethod: (row) => ({
+      type: "neutral",
+      input: "",
+      output: "",
+      kind: row[4] as string,
+      provenance: row[5] as Provenance,
       signature: `${row[0]}.${row[1]}#${row[2]}${row[3]}`,
-      modeledMethod: {
-        type: "neutral",
-        input: "",
-        output: "",
-        kind: row[4] as string,
-        provenance: row[5] as Provenance,
-      },
+      packageName: row[0] as string,
+      typeName: row[1] as string,
+      methodName: row[2] as string,
+      methodParameters: row[3] as string,
     }),
     supportedKinds: ["summary", "source", "sink"],
   },

extensions/ql-vscode/src/data-extensions-editor/queries/csharp.ts

@@ -1,7 +1,7 @@
 import { Query } from "./query";
 
 export const fetchExternalApisQuery: Query = {
-  mainQuery: `/**
+  applicationModeQuery: `/**
  * @name Usage of APIs coming from external libraries
  * @description A list of 3rd party APIs used in the codebase.
  * @tags telemetry
@@ -9,27 +9,57 @@ export const fetchExternalApisQuery: Query = {
  * @id cs/telemetry/fetch-external-apis
  */
 
-import csharp
-import ExternalApi
+private import csharp
+private import AutomodelVsCode
+
+class ExternalApi extends CallableMethod {
+  ExternalApi() {
+    this.isUnboundDeclaration() and
+    this.fromLibrary() and
+    this.(Modifiable).isEffectivelyPublic()
+  }
+}
 
 private Call aUsage(ExternalApi api) { result.getTarget().getUnboundDeclaration() = api }
 
-private boolean isSupported(ExternalApi api) {
-  api.isSupported() and result = true
-  or
-  not api.isSupported() and
-  result = false
-}
-
-from ExternalApi api, string apiName, boolean supported, Call usage
+from
+  ExternalApi api, string apiName, boolean supported, Call usage, string type, string classification
 where
   apiName = api.getApiName() and
   supported = isSupported(api) and
-  usage = aUsage(api)
-select usage, apiName, supported.toString(), "supported"
+  usage = aUsage(api) and
+  type = supportedType(api) and
+  classification = methodClassification(usage)
+select usage, apiName, supported.toString(), "supported", api.dllName(), api.dllVersion(), type,
+  "type", classification, "classification"
+`,
+  frameworkModeQuery: `/**
+ * @name Public methods
+ * @description A list of APIs callable by consumers. Excludes test and generated code.
+ * @tags telemetry
+ * @kind problem
+ * @id cs/telemetry/fetch-public-methods
+ */
+
+private import csharp
+private import dotnet
+private import semmle.code.csharp.frameworks.Test
+private import AutomodelVsCode
+
+class PublicMethod extends CallableMethod {
+  PublicMethod() { this.fromSource() and not this.getFile() instanceof TestFile }
+}
+
+from PublicMethod publicMethod, string apiName, boolean supported, string type
+where
+  apiName = publicMethod.getApiName() and
+  supported = isSupported(publicMethod) and
+  type = supportedType(publicMethod)
+select publicMethod, apiName, supported.toString(), "supported",
+  publicMethod.getFile().getBaseName(), "library", type, "type", "unknown", "classification"
 `,
   dependencies: {
-    "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */
+    "AutomodelVsCode.qll": `/** Provides classes and predicates related to handling APIs for the VS Code extension. */
 
 private import csharp
 private import dotnet
@@ -41,6 +71,7 @@ private import semmle.code.csharp.dataflow.internal.DataFlowPrivate
 private import semmle.code.csharp.dataflow.internal.DataFlowDispatch as DataFlowDispatch
 private import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
 private import semmle.code.csharp.dataflow.internal.TaintTrackingPrivate
+private import semmle.code.csharp.frameworks.Test
 private import semmle.code.csharp.security.dataflow.flowsources.Remote
 
 pragma[nomagic]
@@ -59,22 +90,31 @@ class TestLibrary extends RefType {
 }
 
 /** Holds if the given callable is not worth supporting. */
-private predicate isUninteresting(DotNet::Callable c) {
+private predicate isUninteresting(DotNet::Declaration c) {
   c.getDeclaringType() instanceof TestLibrary or
-  c.(Constructor).isParameterless()
+  c.(Constructor).isParameterless() or
+  c.getDeclaringType() instanceof AnonymousClass
 }
 
 /**
- * An external API from either the C# Standard Library or a 3rd party library.
+ * An callable method from either the C# Standard Library, a 3rd party library, or from the source.
  */
-class ExternalApi extends DotNet::Callable {
-  ExternalApi() {
-    this.isUnboundDeclaration() and
-    this.fromLibrary() and
+class CallableMethod extends DotNet::Declaration {
+  CallableMethod() {
     this.(Modifiable).isEffectivelyPublic() and
     not isUninteresting(this)
   }
 
+  /**
+   * Gets the unbound type, name and parameter types of this API.
+   */
+  bindingset[this]
+  private string getSignature() {
+    result =
+      nestedName(this.getDeclaringType().getUnboundDeclaration()) + "#" + this.getName() + "(" +
+        parameterQualifiedTypeNamesToString(this) + ")"
+  }
+
   /**
    * Gets the namespace of this API.
    */
@@ -85,8 +125,23 @@ class ExternalApi extends DotNet::Callable {
    * Gets the namespace and signature of this API.
    */
   bindingset[this]
-  string getApiName() { result = this.getNamespace() + "." + this.getDeclaringType().getUnboundDeclaration() + "#" + this.getName() + "(" +
-  parameterQualifiedTypeNamesToString(this) + ")" }
+  string getApiName() { result = this.getNamespace() + "." + this.getSignature() }
+
+  private string getDllName() { result = this.getLocation().(Assembly).getName() }
+
+  private string getDllVersion() { result = this.getLocation().(Assembly).getVersion().toString() }
+
+  string dllName() {
+    result = this.getDllName()
+    or
+    not exists(this.getDllName()) and result = this.getFile().getBaseName()
+  }
+
+  string dllVersion() {
+    result = this.getDllVersion()
+    or
+    not exists(this.getDllVersion()) and result = ""
+  }
 
   /** Gets a node that is an input to a call to this API. */
   private ArgumentNode getAnInput() {
@@ -140,47 +195,45 @@ class ExternalApi extends DotNet::Callable {
   }
 }
 
-/**
- * Gets the limit for the number of results produced by a telemetry query.
- */
-int resultLimit() { result = 1000 }
+boolean isSupported(CallableMethod callableMethod) {
+  callableMethod.isSupported() and result = true
+  or
+  not callableMethod.isSupported() and
+  result = false
+}
+
+string supportedType(CallableMethod method) {
+  method.isSink() and result = "sink"
+  or
+  method.isSource() and result = "source"
+  or
+  method.hasSummary() and result = "summary"
+  or
+  method.isNeutral() and result = "neutral"
+  or
+  not method.isSupported() and result = ""
+}
+
+string methodClassification(Call method) {
+  method.getFile() instanceof TestFile and result = "test"
+  or
+  not method.getFile() instanceof TestFile and
+  result = "source"
+}
 
 /**
- * Holds if it is relevant to count usages of "api".
+ * Gets the nested name of the declaration.
+ *
+ * If the declaration is not a nested type, the result is the same as \`getName()\`.
+ * Otherwise the name of the nested type is prefixed with a \`+\` and appended to
+ * the name of the enclosing type, which might be a nested type as well.
  */
-signature predicate relevantApi(ExternalApi api);
-
-/**
- * Given a predicate to count relevant API usages, this module provides a predicate
- * for restricting the number or returned results based on a certain limit.
- */
-module Results<relevantApi/1 getRelevantUsages> {
-  private int getUsages(string apiName) {
-    result =
-      strictcount(Call c, ExternalApi api |
-        c.getTarget().getUnboundDeclaration() = api and
-        apiName = api.getApiName() and
-        getRelevantUsages(api)
-      )
-  }
-
-  private int getOrder(string apiName) {
-    apiName =
-      rank[result](string name, int usages |
-        usages = getUsages(name)
-      |
-        name order by usages desc, name
-      )
-  }
-
-  /**
-   * Holds if there exists an API with "apiName" that is being used "usages" times
-   * and if it is in the top results (guarded by resultLimit).
-   */
-  predicate restrict(string apiName, int usages) {
-    usages = getUsages(apiName) and
-    getOrder(apiName) <= resultLimit()
-  }
+private string nestedName(Declaration declaration) {
+  not exists(declaration.getDeclaringType().getUnboundDeclaration()) and
+  result = declaration.getName()
+  or
+  nestedName(declaration.getDeclaringType().getUnboundDeclaration()) + "+" + declaration.getName() =
+    result
 }
 `,
   },

extensions/ql-vscode/src/data-extensions-editor/queries/java.ts

@@ -1,7 +1,7 @@
 import { Query } from "./query";
 
 export const fetchExternalApisQuery: Query = {
-  mainQuery: `/**
+  applicationModeQuery: `/**
  * @name Usage of APIs coming from external libraries
  * @description A list of 3rd party APIs used in the codebase. Excludes test and generated code.
  * @tags telemetry
@@ -10,28 +10,50 @@ export const fetchExternalApisQuery: Query = {
  */
 
 import java
-import ExternalApi
+import AutomodelVsCode
 
-private Call aUsage(ExternalApi api) {
-  result.getCallee().getSourceDeclaration() = api and
-  not result.getFile() instanceof GeneratedFile
+class ExternalApi extends CallableMethod {
+  ExternalApi() { not this.fromSource() }
 }
 
-private boolean isSupported(ExternalApi api) {
-  api.isSupported() and result = true
-  or
-  not api.isSupported() and result = false
-}
+private Call aUsage(ExternalApi api) { result.getCallee().getSourceDeclaration() = api }
 
-from ExternalApi api, string apiName, boolean supported, Call usage
+from
+  ExternalApi externalApi, string apiName, boolean supported, Call usage, string type,
+  string classification
 where
-  apiName = api.getApiName() and
-  supported = isSupported(api) and
-  usage = aUsage(api)
-select usage, apiName, supported.toString(), "supported"
+  apiName = externalApi.getApiName() and
+  supported = isSupported(externalApi) and
+  usage = aUsage(externalApi) and
+  type = supportedType(externalApi) and
+  classification = methodClassification(usage)
+select usage, apiName, supported.toString(), "supported", externalApi.jarContainer(),
+  externalApi.jarVersion(), type, "type", classification, "classification"
+`,
+  frameworkModeQuery: `/**
+ * @name Public methods
+ * @description A list of APIs callable by consumers. Excludes test and generated code.
+ * @tags telemetry
+ * @kind problem
+ * @id java/telemetry/fetch-public-methods
+ */
+
+import java
+import AutomodelVsCode
+
+class PublicMethodFromSource extends CallableMethod, ModelApi { }
+
+from PublicMethodFromSource publicMethod, string apiName, boolean supported, string type
+where
+  apiName = publicMethod.getApiName() and
+  supported = isSupported(publicMethod) and
+  type = supportedType(publicMethod)
+select publicMethod, apiName, supported.toString(), "supported",
+  publicMethod.getCompilationUnit().getParentContainer().getBaseName(), "library", type, "type",
+  "unknown", "classification"
 `,
   dependencies: {
-    "ExternalApi.qll": `/** Provides classes and predicates related to handling APIs from external libraries. */
+    "AutomodelVsCode.qll": `/** Provides classes and predicates related to handling APIs for the VS Code extension. */
 
 private import java
 private import semmle.code.java.dataflow.DataFlow
@@ -41,57 +63,55 @@ private import semmle.code.java.dataflow.FlowSummary
 private import semmle.code.java.dataflow.internal.DataFlowPrivate
 private import semmle.code.java.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
 private import semmle.code.java.dataflow.TaintTracking
+private import semmle.code.java.dataflow.internal.ModelExclusions
 
-pragma[nomagic]
-private predicate isTestPackage(Package p) {
-  p.getName()
-      .matches([
-          "org.junit%", "junit.%", "org.mockito%", "org.assertj%",
-          "com.github.tomakehurst.wiremock%", "org.hamcrest%", "org.springframework.test.%",
-          "org.springframework.mock.%", "org.springframework.boot.test.%", "reactor.test%",
-          "org.xmlunit%", "org.testcontainers.%", "org.opentest4j%", "org.mockserver%",
-          "org.powermock%", "org.skyscreamer.jsonassert%", "org.rnorth.visibleassertions",
-          "org.openqa.selenium%", "com.gargoylesoftware.htmlunit%", "org.jboss.arquillian.testng%",
-          "org.testng%"
-        ])
-}
-
-/**
- * A test library.
- */
-private class TestLibrary extends RefType {
-  TestLibrary() { isTestPackage(this.getPackage()) }
-}
-
-private string containerAsJar(Container container) {
-  if container instanceof JarFile then result = container.getBaseName() else result = "rt.jar"
-}
-
-/** Holds if the given callable is not worth supporting. */
+/** Holds if the given callable/method is not worth supporting. */
 private predicate isUninteresting(Callable c) {
   c.getDeclaringType() instanceof TestLibrary or
-  c.(Constructor).isParameterless()
+  c.(Constructor).isParameterless() or
+  c.getDeclaringType() instanceof AnonymousClass
 }
 
 /**
- * An external API from either the Standard Library or a 3rd party library.
+ * A callable method from either the Standard Library, a 3rd party library or from the source.
  */
-class ExternalApi extends Callable {
-  ExternalApi() { not this.fromSource() and not isUninteresting(this) }
+class CallableMethod extends Callable {
+  CallableMethod() { not isUninteresting(this) }
 
   /**
    * Gets information about the external API in the form expected by the MaD modeling framework.
    */
   string getApiName() {
     result =
-      this.getDeclaringType().getPackage() + "." + this.getDeclaringType().getSourceDeclaration() +
-        "#" + this.getName() + paramsString(this)
+      this.getDeclaringType().getPackage() + "." + this.getDeclaringType().nestedName() + "#" +
+        this.getName() + paramsString(this)
+  }
+
+  private string getJarName() {
+    result = this.getCompilationUnit().getParentContainer*().(JarFile).getBaseName()
+  }
+
+  private string getJarVersion() {
+    result = this.getCompilationUnit().getParentContainer*().(JarFile).getSpecificationVersion()
   }
 
   /**
    * Gets the jar file containing this API. Normalizes the Java Runtime to "rt.jar" despite the presence of modules.
    */
-  string jarContainer() { result = containerAsJar(this.getCompilationUnit().getParentContainer*()) }
+  string jarContainer() {
+    result = this.getJarName()
+    or
+    not exists(this.getJarName()) and result = "rt.jar"
+  }
+
+  /**
+   * Gets the version of the JAR file containing this API. Empty if no version is found in the JAR.
+   */
+  string jarVersion() {
+    result = this.getJarVersion()
+    or
+    not exists(this.getJarVersion()) and result = ""
+  }
 
   /** Gets a node that is an input to a call to this API. */
   private DataFlow::Node getAnInput() {
@@ -138,50 +158,107 @@ class ExternalApi extends Callable {
   }
 }
 
-/** DEPRECATED: Alias for ExternalApi */
-deprecated class ExternalAPI = ExternalApi;
+boolean isSupported(CallableMethod method) {
+  method.isSupported() and result = true
+  or
+  not method.isSupported() and result = false
+}
+
+string supportedType(CallableMethod method) {
+  method.isSink() and result = "sink"
+  or
+  method.isSource() and result = "source"
+  or
+  method.hasSummary() and result = "summary"
+  or
+  method.isNeutral() and result = "neutral"
+  or
+  not method.isSupported() and result = ""
+}
+
+string methodClassification(Call method) {
+  isInTestFile(method.getLocation().getFile()) and result = "test"
+  or
+  method.getFile() instanceof GeneratedFile and result = "generated"
+  or
+  not isInTestFile(method.getLocation().getFile()) and
+  not method.getFile() instanceof GeneratedFile and
+  result = "source"
+}
+
+// The below is a copy of https://github.com/github/codeql/blob/249f9f863db1e94e3c46ca85b49fb0ec32f8ca92/java/ql/lib/semmle/code/java/dataflow/internal/ModelExclusions.qll
+// to avoid the use of internal modules.
+/** Holds if the given package \`p\` is a test package. */
+pragma[nomagic]
+private predicate isTestPackage(Package p) {
+  p.getName()
+      .matches([
+          "org.junit%", "junit.%", "org.mockito%", "org.assertj%",
+          "com.github.tomakehurst.wiremock%", "org.hamcrest%", "org.springframework.test.%",
+          "org.springframework.mock.%", "org.springframework.boot.test.%", "reactor.test%",
+          "org.xmlunit%", "org.testcontainers.%", "org.opentest4j%", "org.mockserver%",
+          "org.powermock%", "org.skyscreamer.jsonassert%", "org.rnorth.visibleassertions",
+          "org.openqa.selenium%", "com.gargoylesoftware.htmlunit%", "org.jboss.arquillian.testng%",
+          "org.testng%"
+        ])
+}
 
 /**
- * Gets the limit for the number of results produced by a telemetry query.
+ * A test library.
  */
-int resultLimit() { result = 1000 }
+class TestLibrary extends RefType {
+  TestLibrary() { isTestPackage(this.getPackage()) }
+}
+
+/** Holds if the given file is a test file. */
+private predicate isInTestFile(File file) {
+  file.getAbsolutePath().matches(["%/test/%", "%/guava-tests/%", "%/guava-testlib/%"]) and
+  not file.getAbsolutePath().matches("%/ql/test/%") // allows our test cases to work
+}
+
+/** Holds if the given compilation unit's package is a JDK internal. */
+private predicate isJdkInternal(CompilationUnit cu) {
+  cu.getPackage().getName().matches("org.graalvm%") or
+  cu.getPackage().getName().matches("com.sun%") or
+  cu.getPackage().getName().matches("sun%") or
+  cu.getPackage().getName().matches("jdk%") or
+  cu.getPackage().getName().matches("java2d%") or
+  cu.getPackage().getName().matches("build.tools%") or
+  cu.getPackage().getName().matches("propertiesparser%") or
+  cu.getPackage().getName().matches("org.jcp%") or
+  cu.getPackage().getName().matches("org.w3c%") or
+  cu.getPackage().getName().matches("org.ietf.jgss%") or
+  cu.getPackage().getName().matches("org.xml.sax%") or
+  cu.getPackage().getName().matches("com.oracle%") or
+  cu.getPackage().getName().matches("org.omg%") or
+  cu.getPackage().getName().matches("org.relaxng%") or
+  cu.getPackage().getName() = "compileproperties" or
+  cu.getPackage().getName() = "transparentruler" or
+  cu.getPackage().getName() = "genstubs" or
+  cu.getPackage().getName() = "netscape.javascript" or
+  cu.getPackage().getName() = ""
+}
+
+/** Holds if the given callable is not worth modeling. */
+predicate isUninterestingForModels(Callable c) {
+  isInTestFile(c.getCompilationUnit().getFile()) or
+  isJdkInternal(c.getCompilationUnit()) or
+  c instanceof MainMethod or
+  c instanceof StaticInitializer or
+  exists(FunctionalExpr funcExpr | c = funcExpr.asMethod()) or
+  c.getDeclaringType() instanceof TestLibrary or
+  c.(Constructor).isParameterless()
+}
 
 /**
- * Holds if it is relevant to count usages of \`api\`.
+ * A class that represents all callables for which we might be
+ * interested in having a MaD model.
  */
-signature predicate relevantApi(ExternalApi api);
-
-/**
- * Given a predicate to count relevant API usages, this module provides a predicate
- * for restricting the number or returned results based on a certain limit.
- */
-module Results<relevantApi/1 getRelevantUsages> {
-  private int getUsages(string apiName) {
-    result =
-      strictcount(Call c, ExternalApi api |
-        c.getCallee().getSourceDeclaration() = api and
-        not c.getFile() instanceof GeneratedFile and
-        apiName = api.getApiName() and
-        getRelevantUsages(api)
-      )
-  }
-
-  private int getOrder(string apiInfo) {
-    apiInfo =
-      rank[result](string info, int usages |
-        usages = getUsages(info)
-      |
-        info order by usages desc, info
-      )
-  }
-
-  /**
-   * Holds if there exists an API with \`apiName\` that is being used \`usages\` times
-   * and if it is in the top results (guarded by resultLimit).
-   */
-  predicate restrict(string apiName, int usages) {
-    usages = getUsages(apiName) and
-    getOrder(apiName) <= resultLimit()
+class ModelApi extends SrcCallable {
+  ModelApi() {
+    this.fromSource() and
+    this.isEffectivelyPublic() and
+    not isUninterestingForModels(this)
   }
 }
 `,

extensions/ql-vscode/src/data-extensions-editor/queries/query.ts

@@ -1,14 +1,37 @@
 export type Query = {
   /**
-   * The main query.
+   * The application query.
    *
    * It should select all usages of external APIs, and return the following result pattern:
    * - usage: the usage of the external API. This is an entity.
    * - apiName: the name of the external API. This is a string.
-   * - supported: whether the external API is supported by the extension. This should be a string representation of a boolean to satify the result pattern for a problem query.
+   * - supported: whether the external API is modeled. This should be a string representation of a boolean to satify the result pattern for a problem query.
    * - "supported": a string literal. This is required to make the query a valid problem query.
+   * - libraryName: the name of the library that contains the external API. This is a string and usually the basename of a file.
+   * - libraryVersion: the version of the library that contains the external API. This is a string and can be empty if the version cannot be determined.
+   * - type: the modeled kind of the method, either "sink", "source", "summary", or "neutral"
+   * - "type": a string literal. This is required to make the query a valid problem query.
+   * - classification: the classification of the use of the method, either "source", "test", "generated", or "unknown"
+   * - "classification: a string literal. This is required to make the query a valid problem query.
    */
-  mainQuery: string;
+  applicationModeQuery: string;
+  /**
+   * The framework query.
+   *
+   * It should select all methods that are callable by applications, which is usually all public methods (and constructors).
+   * The result pattern should be as follows:
+   * - method: the method that is callable by applications. This is an entity.
+   * - apiName: the name of the external API. This is a string.
+   * - supported: whether this method is modeled. This should be a string representation of a boolean to satify the result pattern for a problem query.
+   * - "supported": a string literal. This is required to make the query a valid problem query.
+   * - libraryName: an arbitrary string. This is required to make it match the structure of the application query.
+   * - libraryVersion: an arbitrary string. This is required to make it match the structure of the application query.
+   * - type: the modeled kind of the method, either "sink", "source", "summary", or "neutral"
+   * - "type": a string literal. This is required to make the query a valid problem query.
+   * - "unknown": a string literal. This is required to make it match the structure of the application query.
+   * - "classification: a string literal. This is required to make the query a valid problem query.
+   */
+  frameworkModeQuery: string;
   dependencies?: {
     [filename: string]: string;
   };

extensions/ql-vscode/src/data-extensions-editor/shared/extension-pack.ts

@@ -8,8 +8,3 @@ export interface ExtensionPack {
   extensionTargets: Record<string, string>;
   dataExtensions: string[];
 }
-
-export interface ExtensionPackModelFile {
-  filename: string;
-  extensionPack: ExtensionPack;
-}

extensions/ql-vscode/src/data-extensions-editor/shared/mode.ts

@@ -0,0 +1,4 @@
+export enum Mode {
+  Application = "application",
+  Framework = "framework",
+}

extensions/ql-vscode/src/data-extensions-editor/shared/modeled-percentage.ts

@@ -1,4 +1,4 @@
-import { ExternalApiUsage } from "../../data-extensions-editor/external-api-usage";
+import { ExternalApiUsage } from "../external-api-usage";
 
 export function calculateModeledPercentage(
   externalApiUsages: Array<Pick<ExternalApiUsage, "supported">>,

extensions/ql-vscode/src/data-extensions-editor/shared/sorting.ts

@@ -0,0 +1,88 @@
+import { ExternalApiUsage } from "../external-api-usage";
+import { Mode } from "./mode";
+import { calculateModeledPercentage } from "./modeled-percentage";
+
+export function groupMethods(
+  externalApiUsages: ExternalApiUsage[],
+  mode: Mode,
+): Record<string, ExternalApiUsage[]> {
+  const groupedByLibrary: Record<string, ExternalApiUsage[]> = {};
+
+  for (const externalApiUsage of externalApiUsages) {
+    // Group by package if using framework mode
+    const key =
+      mode === Mode.Framework
+        ? externalApiUsage.packageName
+        : externalApiUsage.library;
+
+    groupedByLibrary[key] ??= [];
+    groupedByLibrary[key].push(externalApiUsage);
+  }
+
+  return groupedByLibrary;
+}
+
+export function sortGroupNames(
+  methods: Record<string, ExternalApiUsage[]>,
+): string[] {
+  return Object.keys(methods).sort((a, b) =>
+    compareGroups(methods[a], a, methods[b], b),
+  );
+}
+
+export function sortMethods(
+  externalApiUsages: ExternalApiUsage[],
+): ExternalApiUsage[] {
+  const sortedExternalApiUsages = [...externalApiUsages];
+  sortedExternalApiUsages.sort((a, b) => compareMethod(a, b));
+  return sortedExternalApiUsages;
+}
+
+function compareGroups(
+  a: ExternalApiUsage[],
+  aName: string,
+  b: ExternalApiUsage[],
+  bName: string,
+): number {
+  const supportedPercentageA = calculateModeledPercentage(a);
+  const supportedPercentageB = calculateModeledPercentage(b);
+
+  // Sort first by supported percentage ascending
+  if (supportedPercentageA > supportedPercentageB) {
+    return 1;
+  }
+  if (supportedPercentageA < supportedPercentageB) {
+    return -1;
+  }
+
+  const numberOfUsagesA = a.reduce((acc, curr) => acc + curr.usages.length, 0);
+  const numberOfUsagesB = b.reduce((acc, curr) => acc + curr.usages.length, 0);
+
+  // If the number of usages is equal, sort by number of methods descending
+  if (numberOfUsagesA === numberOfUsagesB) {
+    const numberOfMethodsA = a.length;
+    const numberOfMethodsB = b.length;
+
+    // If the number of methods is equal, sort by library name ascending
+    if (numberOfMethodsA === numberOfMethodsB) {
+      return aName.localeCompare(bName);
+    }
+
+    return numberOfMethodsB - numberOfMethodsA;
+  }
+
+  // Then sort by number of usages descending
+  return numberOfUsagesB - numberOfUsagesA;
+}
+
+function compareMethod(a: ExternalApiUsage, b: ExternalApiUsage): number {
+  // Sort first by supported, putting unmodeled methods first.
+  if (a.supported && !b.supported) {
+    return 1;
+  }
+  if (!a.supported && b.supported) {
+    return -1;
+  }
+  // Then sort by number of usages descending
+  return b.usages.length - a.usages.length;
+}

extensions/ql-vscode/src/data-extensions-editor/shared/view-state.ts

@@ -1,7 +1,9 @@
-import { ExtensionPackModelFile } from "./extension-pack";
+import { ExtensionPack } from "./extension-pack";
+import { Mode } from "./mode";
 
 export interface DataExtensionEditorViewState {
-  extensionPackModelFile: ExtensionPackModelFile;
-  modelFileExists: boolean;
+  extensionPack: ExtensionPack;
+  enableFrameworkMode: boolean;
   showLlmButton: boolean;
+  mode: Mode;
 }

extensions/ql-vscode/src/data-extensions-editor/yaml.ts

@@ -1,31 +1,20 @@
 import Ajv from "ajv";
 
 import { ExternalApiUsage } from "./external-api-usage";
+import { ModeledMethod, ModeledMethodType } from "./modeled-method";
 import {
-  ModeledMethod,
-  ModeledMethodType,
-  ModeledMethodWithSignature,
-} from "./modeled-method";
-import { extensiblePredicateDefinitions } from "./predicates";
+  ExtensiblePredicateDefinition,
+  extensiblePredicateDefinitions,
+} from "./predicates";
 
 import * as dataSchemaJson from "./data-schema.json";
+import { sanitizeExtensionPackName } from "./extension-pack-name";
 
 const ajv = new Ajv({ allErrors: true });
 const dataSchemaValidate = ajv.compile(dataSchemaJson);
 
-type ExternalApiUsageByType = {
-  externalApiUsage: ExternalApiUsage;
-  modeledMethod: ModeledMethod;
-};
-
-type ExtensiblePredicateDefinition = {
-  extensiblePredicate: string;
-  generateMethodDefinition: (method: ExternalApiUsageByType) => any[];
-  readModeledMethod: (row: any[]) => ModeledMethodWithSignature;
-};
-
 function createDataProperty(
-  methods: ExternalApiUsageByType[],
+  methods: ModeledMethod[],
   definition: ExtensiblePredicateDefinition,
 ) {
   if (methods.length === 0) {
@@ -44,12 +33,11 @@ function createDataProperty(
 
 export function createDataExtensionYaml(
   language: string,
-  externalApiUsages: ExternalApiUsage[],
-  modeledMethods: Record<string, ModeledMethod>,
+  modeledMethods: ModeledMethod[],
 ) {
   const methodsByType: Record<
     Exclude<ModeledMethodType, "none">,
-    ExternalApiUsageByType[]
+    ModeledMethod[]
   > = {
     source: [],
     sink: [],
@@ -57,14 +45,9 @@ export function createDataExtensionYaml(
     neutral: [],
   };
 
-  for (const externalApiUsage of externalApiUsages) {
-    const modeledMethod = modeledMethods[externalApiUsage.signature];
-
+  for (const modeledMethod of modeledMethods) {
     if (modeledMethod?.type && modeledMethod.type !== "none") {
-      methodsByType[modeledMethod.type].push({
-        externalApiUsage,
-        modeledMethod,
-      });
+      methodsByType[modeledMethod.type].push(modeledMethod);
     }
   }
 
@@ -83,6 +66,88 @@ export function createDataExtensionYaml(
 ${extensions.join("\n")}`;
 }
 
+export function createDataExtensionYamlsForApplicationMode(
+  language: string,
+  externalApiUsages: ExternalApiUsage[],
+  modeledMethods: Record<string, ModeledMethod>,
+): Record<string, string> {
+  const methodsByLibraryFilename: Record<string, ModeledMethod[]> = {};
+
+  for (const externalApiUsage of externalApiUsages) {
+    const modeledMethod = modeledMethods[externalApiUsage.signature];
+    if (!modeledMethod) {
+      continue;
+    }
+
+    const filename = createFilenameForLibrary(externalApiUsage.library);
+
+    methodsByLibraryFilename[filename] =
+      methodsByLibraryFilename[filename] || [];
+    methodsByLibraryFilename[filename].push(modeledMethod);
+  }
+
+  const result: Record<string, string> = {};
+
+  for (const [filename, methods] of Object.entries(methodsByLibraryFilename)) {
+    result[filename] = createDataExtensionYaml(language, methods);
+  }
+
+  return result;
+}
+
+export function createDataExtensionYamlsForFrameworkMode(
+  databaseName: string,
+  language: string,
+  externalApiUsages: ExternalApiUsage[],
+  modeledMethods: Record<string, ModeledMethod>,
+  prefix = "models/",
+  suffix = ".model",
+): Record<string, string> {
+  const parts = databaseName.split("/");
+  const libraryName = parts
+    .slice(1)
+    .map((part) => sanitizeExtensionPackName(part))
+    .join("-");
+
+  const methods = externalApiUsages
+    .map((externalApiUsage) => modeledMethods[externalApiUsage.signature])
+    .filter((modeledMethod) => modeledMethod !== undefined);
+
+  return {
+    [`${prefix}${libraryName}${suffix}.yml`]: createDataExtensionYaml(
+      language,
+      methods,
+    ),
+  };
+}
+
+export function createFilenameForLibrary(
+  library: string,
+  prefix = "models/",
+  suffix = ".model",
+) {
+  let libraryName = library;
+
+  // Lowercase everything
+  libraryName = libraryName.toLowerCase();
+
+  // Replace all spaces and underscores with hyphens
+  libraryName = libraryName.replaceAll(/[\s_]+/g, "-");
+
+  // Replace all characters which are not allowed by empty strings
+  libraryName = libraryName.replaceAll(/[^a-z0-9.-]/g, "");
+
+  // Remove any leading or trailing hyphens or dots
+  libraryName = libraryName.replaceAll(/^[.-]+|[.-]+$/g, "");
+
+  // Remove any duplicate hyphens
+  libraryName = libraryName.replaceAll(/-{2,}/g, "-");
+  // Remove any duplicate dots
+  libraryName = libraryName.replaceAll(/\.{2,}/g, ".");
+
+  return `${prefix}${libraryName}${suffix}.yml`;
+}
+
 export function loadDataExtensionYaml(
   data: any,
 ): Record<string, ModeledMethod> | undefined {
@@ -116,14 +181,11 @@ export function loadDataExtensionYaml(
     }
 
     for (const row of data) {
-      const result = definition.readModeledMethod(row);
-      if (!result) {
+      const modeledMethod = definition.readModeledMethod(row);
+      if (!modeledMethod) {
         continue;
       }
-
-      const { signature, modeledMethod } = result;
-
-      modeledMethods[signature] = modeledMethod;
+      modeledMethods[modeledMethod.signature] = modeledMethod;
     }
   }
 

extensions/ql-vscode/src/databases/code-search-api.ts

@@ -2,8 +2,11 @@ import { retry } from "@octokit/plugin-retry";
 import { throttling } from "@octokit/plugin-throttling";
 import { Octokit } from "@octokit/rest";
 import { Progress, CancellationToken } from "vscode";
-import { showAndLogWarningMessage } from "../helpers";
 import { Credentials } from "../common/authentication";
+import {
+  NotificationLogger,
+  showAndLogWarningMessage,
+} from "../common/logging";
 
 export async function getCodeSearchRepositories(
   query: string,
@@ -13,9 +16,10 @@ export async function getCodeSearchRepositories(
   }>,
   token: CancellationToken,
   credentials: Credentials,
+  logger: NotificationLogger,
 ): Promise<string[]> {
   let nwos: string[] = [];
-  const octokit = await provideOctokitWithThrottling(credentials);
+  const octokit = await provideOctokitWithThrottling(credentials, logger);
 
   for await (const response of octokit.paginate.iterator(
     octokit.rest.search.code,
@@ -43,6 +47,7 @@ export async function getCodeSearchRepositories(
 
 async function provideOctokitWithThrottling(
   credentials: Credentials,
+  logger: NotificationLogger,
 ): Promise<Octokit> {
   const MyOctokit = Octokit.plugin(throttling);
   const auth = await credentials.getAccessToken();
@@ -53,6 +58,7 @@ async function provideOctokitWithThrottling(
     throttle: {
       onRateLimit: (retryAfter: number, options: any): boolean => {
         void showAndLogWarningMessage(
+          logger,
           `Rate Limit detected for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`,
         );
 
@@ -60,6 +66,7 @@ async function provideOctokitWithThrottling(
       },
       onSecondaryRateLimit: (_retryAfter: number, options: any): void => {
         void showAndLogWarningMessage(
+          logger,
           `Secondary Rate Limit detected for request ${options.method} ${options.url}`,
         );
       },

extensions/ql-vscode/src/databases/config/db-config-store.ts

@@ -18,7 +18,10 @@ import {
   SelectedDbItemKind,
 } from "./db-config";
 import * as chokidar from "chokidar";
-import { DisposableObject, DisposeHandler } from "../../pure/disposable-object";
+import {
+  DisposableObject,
+  DisposeHandler,
+} from "../../common/disposable-object";
 import { DbConfigValidator } from "./db-config-validator";
 import { App } from "../../common/app";
 import { AppEvent, AppEventEmitter } from "../../common/events";
@@ -147,14 +150,10 @@ export class DbConfigStore extends DisposableObject {
     await this.writeConfig(config);
   }
 
-  /**
-   * Adds a list of remote repositories to an existing repository list and removes duplicates.
-   * @returns a list of repositories that were not added because the list reached 1000 entries.
-   */
   public async addRemoteReposToList(
     repoNwoList: string[],
     parentList: string,
-  ): Promise<string[]> {
+  ): Promise<void> {
     if (!this.config) {
       throw Error("Cannot add variant analysis repos if config is not loaded");
     }
@@ -172,21 +171,15 @@ export class DbConfigStore extends DisposableObject {
       ...new Set([...parent.repositories, ...repoNwoList]),
     ];
 
-    parent.repositories = newRepositoriesList.slice(0, 1000);
-    const truncatedRepositories = newRepositoriesList.slice(1000);
+    parent.repositories = newRepositoriesList;
 
     await this.writeConfig(config);
-    return truncatedRepositories;
   }
 
-  /**
-   * Adds one remote repository
-   * @returns either nothing, or, if a parentList is given AND the number of repos on that list reaches 1000 returns the repo that was not added.
-   */
   public async addRemoteRepo(
     repoNwo: string,
     parentList?: string,
-  ): Promise<string[]> {
+  ): Promise<void> {
     if (!this.config) {
       throw Error("Cannot add variant analysis repo if config is not loaded");
     }
@@ -201,7 +194,6 @@ export class DbConfigStore extends DisposableObject {
       );
     }
 
-    const truncatedRepositories = [];
     const config = cloneDbConfig(this.config);
     if (parentList) {
       const parent = config.databases.variantAnalysis.repositoryLists.find(
@@ -210,15 +202,12 @@ export class DbConfigStore extends DisposableObject {
       if (!parent) {
         throw Error(`Cannot find parent list '${parentList}'`);
       } else {
-        const newRepositories = [...parent.repositories, repoNwo];
-        parent.repositories = newRepositories.slice(0, 1000);
-        truncatedRepositories.push(...newRepositories.slice(1000));
+        parent.repositories = [...parent.repositories, repoNwo];
       }
     } else {
       config.databases.variantAnalysis.repositories.push(repoNwo);
     }
     await this.writeConfig(config);
-    return truncatedRepositories;
   }
 
   public async addRemoteOwner(owner: string): Promise<void> {

extensions/ql-vscode/src/databases/config/db-config-validator.ts

@@ -2,7 +2,7 @@ import { readJsonSync } from "fs-extra";
 import { resolve } from "path";
 import Ajv, { ValidateFunction } from "ajv";
 import { clearLocalDbConfig, DbConfig } from "./db-config";
-import { findDuplicateStrings } from "../../pure/text-utils";
+import { findDuplicateStrings } from "../../common/text-utils";
 import {
   DbConfigValidationError,
   DbConfigValidationErrorKind,

extensions/ql-vscode/src/databases/database-fetcher.ts

@@ -1,7 +1,7 @@
 import fetch, { Response } from "node-fetch";
 import { zip } from "zip-a-folder";
 import { Open } from "unzipper";
-import { Uri, CancellationToken, window, InputBoxOptions } from "vscode";
+import { Uri, window, InputBoxOptions } from "vscode";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import {
   ensureDir,
@@ -17,13 +17,13 @@ import * as Octokit from "@octokit/rest";
 import { retry } from "@octokit/plugin-retry";
 
 import { DatabaseManager, DatabaseItem } from "./local-databases";
-import { showAndLogInformationMessage, tmpDir } from "../helpers";
+import { tmpDir } from "../tmp-dir";
 import {
   reportStreamProgress,
   ProgressCallback,
 } from "../common/vscode/progress";
-import { extLogger } from "../common";
-import { getErrorMessage } from "../pure/helpers-pure";
+import { extLogger } from "../common/logging/vscode";
+import { getErrorMessage } from "../common/helpers-pure";
 import {
   getNwoFromGitHubUrl,
   isValidGitHubNwo,
@@ -31,6 +31,7 @@ import {
 import { Credentials } from "../common/authentication";
 import { AppCommandManager } from "../common/commands";
 import { ALLOW_HTTP_SETTING } from "../config";
+import { showAndLogInformationMessage } from "../common/logging";
 
 /**
  * Prompts a user to fetch a database from a remote location. Database is assumed to be an archive file.
@@ -43,7 +44,6 @@ export async function promptImportInternetDatabase(
   databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
-  token: CancellationToken,
   cli?: CodeQLCliServer,
 ): Promise<DatabaseItem | undefined> {
   const databaseUrl = await window.showInputBox({
@@ -62,13 +62,13 @@ export async function promptImportInternetDatabase(
     storagePath,
     undefined,
     progress,
-    token,
     cli,
   );
 
   if (item) {
     await commandManager.execute("codeQLDatabases.focus");
     void showAndLogInformationMessage(
+      extLogger,
       "Database downloaded and imported successfully.",
     );
   }
@@ -84,7 +84,6 @@ export async function promptImportInternetDatabase(
  * @param storagePath where to store the unzipped database.
  * @param credentials the credentials to use to authenticate with GitHub
  * @param progress the progress callback
- * @param token the cancellation token
  * @param cli the CodeQL CLI server
  */
 export async function promptImportGithubDatabase(
@@ -93,7 +92,6 @@ export async function promptImportGithubDatabase(
   storagePath: string,
   credentials: Credentials | undefined,
   progress: ProgressCallback,
-  token: CancellationToken,
   cli?: CodeQLCliServer,
 ): Promise<DatabaseItem | undefined> {
   const githubRepo = await askForGitHubRepo(progress);
@@ -107,13 +105,13 @@ export async function promptImportGithubDatabase(
     storagePath,
     credentials,
     progress,
-    token,
     cli,
   );
 
   if (databaseItem) {
     await commandManager.execute("codeQLDatabases.focus");
     void showAndLogInformationMessage(
+      extLogger,
       "Database downloaded and imported successfully.",
     );
     return databaseItem;
@@ -154,7 +152,6 @@ export async function askForGitHubRepo(
  * @param storagePath where to store the unzipped database.
  * @param credentials the credentials to use to authenticate with GitHub
  * @param progress the progress callback
- * @param token the cancellation token
  * @param cli the CodeQL CLI server
  * @param language the language to download. If undefined, the user will be prompted to choose a language.
  **/
@@ -164,7 +161,6 @@ export async function downloadGitHubDatabase(
   storagePath: string,
   credentials: Credentials | undefined,
   progress: ProgressCallback,
-  token: CancellationToken,
   cli?: CodeQLCliServer,
   language?: string,
 ): Promise<DatabaseItem | undefined> {
@@ -210,7 +206,6 @@ export async function downloadGitHubDatabase(
     storagePath,
     `${owner}/${name}`,
     progress,
-    token,
     cli,
   );
 }
@@ -228,7 +223,6 @@ export async function importArchiveDatabase(
   databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
-  token: CancellationToken,
   cli?: CodeQLCliServer,
 ): Promise<DatabaseItem | undefined> {
   try {
@@ -239,12 +233,12 @@ export async function importArchiveDatabase(
       storagePath,
       undefined,
       progress,
-      token,
       cli,
     );
     if (item) {
       await commandManager.execute("codeQLDatabases.focus");
       void showAndLogInformationMessage(
+        extLogger,
         "Database unzipped and imported successfully.",
       );
     }
@@ -271,7 +265,6 @@ export async function importArchiveDatabase(
  * @param storagePath where to store the unzipped database.
  * @param nameOverride a name for the database that overrides the default
  * @param progress callback to send progress messages to
- * @param token cancellation token
  */
 async function databaseArchiveFetcher(
   databaseUrl: string,
@@ -280,7 +273,6 @@ async function databaseArchiveFetcher(
   storagePath: string,
   nameOverride: string | undefined,
   progress: ProgressCallback,
-  token: CancellationToken,
   cli?: CodeQLCliServer,
 ): Promise<DatabaseItem> {
   progress({
@@ -323,8 +315,6 @@ async function databaseArchiveFetcher(
     const makeSelected = true;
 
     const item = await databaseManager.openDatabase(
-      progress,
-      token,
       Uri.file(dbPath),
       makeSelected,
       nameOverride,

extensions/ql-vscode/src/databases/db-manager.ts

@@ -1,7 +1,7 @@
 import { App } from "../common/app";
 import { AppEvent, AppEventEmitter } from "../common/events";
 import { ValueResult } from "../common/value-result";
-import { DisposableObject } from "../pure/disposable-object";
+import { DisposableObject } from "../common/disposable-object";
 import { DbConfigStore } from "./config/db-config-store";
 import {
   DbItem,
@@ -101,15 +101,15 @@ export class DbManager extends DisposableObject {
   public async addNewRemoteRepo(
     nwo: string,
     parentList?: string,
-  ): Promise<string[]> {
-    return await this.dbConfigStore.addRemoteRepo(nwo, parentList);
+  ): Promise<void> {
+    await this.dbConfigStore.addRemoteRepo(nwo, parentList);
   }
 
   public async addNewRemoteReposToList(
     nwoList: string[],
     parentList: string,
-  ): Promise<string[]> {
-    return await this.dbConfigStore.addRemoteReposToList(nwoList, parentList);
+  ): Promise<void> {
+    await this.dbConfigStore.addRemoteReposToList(nwoList, parentList);
   }
 
   public async addNewRemoteOwner(owner: string): Promise<void> {

extensions/ql-vscode/src/databases/db-module.ts

@@ -1,7 +1,7 @@
 import { window } from "vscode";
 import { App } from "../common/app";
-import { extLogger } from "../common";
-import { DisposableObject } from "../pure/disposable-object";
+import { extLogger } from "../common/logging/vscode";
+import { DisposableObject } from "../common/disposable-object";
 import { DbConfigStore } from "./config/db-config-store";
 import { DbManager } from "./db-manager";
 import { DbPanel } from "./ui/db-panel";

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -1,5 +1,5 @@
 import { join, basename, dirname as path_dirname } from "path";
-import { DisposableObject } from "../pure/disposable-object";
+import { DisposableObject } from "../common/disposable-object";
 import {
   Event,
   EventEmitter,
@@ -31,20 +31,21 @@ import {
 import {
   isLikelyDatabaseRoot,
   isLikelyDbLanguageFolder,
-  showAndLogErrorMessage,
+} from "./local-databases/db-contents-heuristics";
+import {
   showAndLogExceptionWithTelemetry,
-} from "../helpers";
-import { extLogger } from "../common";
+  showAndLogErrorMessage,
+} from "../common/logging";
 import {
   importArchiveDatabase,
   promptImportGithubDatabase,
   promptImportInternetDatabase,
 } from "./database-fetcher";
-import { asError, asyncFilter, getErrorMessage } from "../pure/helpers-pure";
+import { asError, asyncFilter, getErrorMessage } from "../common/helpers-pure";
 import { QueryRunner } from "../query-server";
 import { isCanary } from "../config";
 import { App } from "../common/app";
-import { redactableError } from "../pure/errors";
+import { redactableError } from "../common/errors";
 import { LocalDatabasesCommands } from "../common/commands";
 import {
   createMultiSelectionCommand,
@@ -251,6 +252,7 @@ export class DatabaseUI extends DisposableObject {
         this.handleUpgradeDatabase.bind(this),
       ),
       "codeQLDatabases.renameDatabase": createSingleSelectionCommand(
+        this.app.logger,
         this.handleRenameDatabase.bind(this),
         "database",
       ),
@@ -279,6 +281,8 @@ export class DatabaseUI extends DisposableObject {
       await this.chooseAndSetDatabase(true, { progress, token });
     } catch (e) {
       void showAndLogExceptionWithTelemetry(
+        this.app.logger,
+        this.app.telemetry,
         redactableError(
           asError(e),
         )`Failed to choose and set database: ${getErrorMessage(e)}`,
@@ -310,7 +314,7 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleSetDefaultTourDatabase(): Promise<void> {
     return withProgress(
-      async (progress, token) => {
+      async () => {
         try {
           if (!workspace.workspaceFolders?.length) {
             throw new Error("No workspace folder is open.");
@@ -328,8 +332,6 @@ export class DatabaseUI extends DisposableObject {
               const isTutorialDatabase = true;
 
               await this.databaseManager.openDatabase(
-                progress,
-                token,
                 uri,
                 makeSelected,
                 nameOverride,
@@ -371,14 +373,16 @@ export class DatabaseUI extends DisposableObject {
 
   // Public because it's used in tests
   public async handleRemoveOrphanedDatabases(): Promise<void> {
-    void extLogger.log("Removing orphaned databases from workspace storage.");
+    void this.app.logger.log(
+      "Removing orphaned databases from workspace storage.",
+    );
     let dbDirs = undefined;
 
     if (
       !(await pathExists(this.storagePath)) ||
       !(await stat(this.storagePath)).isDirectory()
     ) {
-      void extLogger.log(
+      void this.app.logger.log(
         "Missing or invalid storage directory. Not trying to remove orphaned databases.",
       );
       return;
@@ -403,7 +407,7 @@ export class DatabaseUI extends DisposableObject {
     dbDirs = await asyncFilter(dbDirs, isLikelyDatabaseRoot);
 
     if (!dbDirs.length) {
-      void extLogger.log("No orphaned databases found.");
+      void this.app.logger.log("No orphaned databases found.");
       return;
     }
 
@@ -412,10 +416,12 @@ export class DatabaseUI extends DisposableObject {
     await Promise.all(
       dbDirs.map(async (dbDir) => {
         try {
-          void extLogger.log(`Deleting orphaned database '${dbDir}'.`);
+          void this.app.logger.log(`Deleting orphaned database '${dbDir}'.`);
           await remove(dbDir);
         } catch (e) {
           void showAndLogExceptionWithTelemetry(
+            this.app.logger,
+            this.app.telemetry,
             redactableError(
               asError(e),
             )`Failed to delete orphaned database: ${getErrorMessage(e)}`,
@@ -428,6 +434,7 @@ export class DatabaseUI extends DisposableObject {
     if (failures.length) {
       const dirname = path_dirname(failures[0]);
       void showAndLogErrorMessage(
+        this.app.logger,
         `Failed to delete unused databases (${failures.join(
           ", ",
         )}).\nTo delete unused databases, please remove them manually from the storage folder ${dirname}.`,
@@ -443,6 +450,8 @@ export class DatabaseUI extends DisposableObject {
       await this.chooseAndSetDatabase(false, { progress, token });
     } catch (e: unknown) {
       void showAndLogExceptionWithTelemetry(
+        this.app.logger,
+        this.app.telemetry,
         redactableError(
           asError(e),
         )`Failed to choose and set database: ${getErrorMessage(e)}`,
@@ -474,13 +483,12 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleChooseDatabaseInternet(): Promise<void> {
     return withProgress(
-      async (progress, token) => {
+      async (progress) => {
         await promptImportInternetDatabase(
           this.app.commands,
           this.databaseManager,
           this.storagePath,
           progress,
-          token,
           this.queryServer?.cliServer,
         );
       },
@@ -492,7 +500,7 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleChooseDatabaseGithub(): Promise<void> {
     return withProgress(
-      async (progress, token) => {
+      async (progress) => {
         const credentials = isCanary() ? this.app.credentials : undefined;
 
         await promptImportGithubDatabase(
@@ -501,7 +509,6 @@ export class DatabaseUI extends DisposableObject {
           this.storagePath,
           credentials,
           progress,
-          token,
           this.queryServer?.cliServer,
         );
       },
@@ -597,14 +604,13 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleClearCache(): Promise<void> {
     return withProgress(
-      async (progress, token) => {
+      async (_progress, token) => {
         if (
           this.queryServer !== undefined &&
           this.databaseManager.currentDatabaseItem !== undefined
         ) {
           await this.queryServer.clearCacheInDatabase(
             this.databaseManager.currentDatabaseItem,
-            progress,
             token,
           );
         }
@@ -622,7 +628,7 @@ export class DatabaseUI extends DisposableObject {
 
   private async handleSetCurrentDatabase(uri: Uri): Promise<void> {
     return withProgress(
-      async (progress, token) => {
+      async (progress) => {
         try {
           // Assume user has selected an archive if the file has a .zip extension
           if (uri.path.endsWith(".zip")) {
@@ -632,11 +638,10 @@ export class DatabaseUI extends DisposableObject {
               this.databaseManager,
               this.storagePath,
               progress,
-              token,
               this.queryServer?.cliServer,
             );
           } else {
-            await this.databaseManager.openDatabase(progress, token, uri);
+            await this.databaseManager.openDatabase(uri);
           }
         } catch (e) {
           // rethrow and let this be handled by default error handling.
@@ -657,10 +662,10 @@ export class DatabaseUI extends DisposableObject {
     databaseItems: DatabaseItem[],
   ): Promise<void> {
     return withProgress(
-      async (progress, token) => {
+      async () => {
         await Promise.all(
           databaseItems.map((dbItem) =>
-            this.databaseManager.removeDatabaseItem(progress, token, dbItem),
+            this.databaseManager.removeDatabaseItem(dbItem),
           ),
         );
       },
@@ -747,15 +752,11 @@ export class DatabaseUI extends DisposableObject {
 
     return await withInheritedProgress(
       progress,
-      async (progress, token) => {
+      async (progress) => {
         if (byFolder) {
           const fixedUri = await this.fixDbUri(uri);
           // we are selecting a database folder
-          return await this.databaseManager.openDatabase(
-            progress,
-            token,
-            fixedUri,
-          );
+          return await this.databaseManager.openDatabase(fixedUri);
         } else {
           // we are selecting a database archive. Must unzip into a workspace-controlled area
           // before importing.
@@ -765,7 +766,6 @@ export class DatabaseUI extends DisposableObject {
             this.databaseManager,
             this.storagePath,
             progress,
-            token,
             this.queryServer?.cliServer,
           );
         }

extensions/ql-vscode/src/databases/local-databases/database-item-impl.ts

@@ -2,7 +2,7 @@
 import * as cli from "../../codeql-cli/cli";
 import vscode from "vscode";
 import { FullDatabaseOptions } from "./database-options";
-import { basename, dirname, extname, join, relative } from "path";
+import { basename, dirname, extname, join } from "path";
 import {
   decodeSourceArchiveUri,
   encodeArchiveBasePath,
@@ -10,9 +10,9 @@ import {
   zipArchiveScheme,
 } from "../../common/vscode/archive-filesystem-provider";
 import { DatabaseItem, PersistedDatabaseItem } from "./database-item";
-import { isLikelyDatabaseRoot } from "../../helpers";
+import { isLikelyDatabaseRoot } from "./db-contents-heuristics";
 import { stat } from "fs-extra";
-import { pathsEqual } from "../../pure/files";
+import { containsPath, pathsEqual } from "../../common/files";
 import { DatabaseContents } from "./database-contents";
 
 export class DatabaseItemImpl implements DatabaseItem {
@@ -199,7 +199,7 @@ export class DatabaseItemImpl implements DatabaseItem {
     try {
       const stats = await stat(testPath);
       if (stats.isDirectory()) {
-        return !relative(testPath, databasePath).startsWith("..");
+        return containsPath(testPath, databasePath);
       } else {
         // database for /one/two/three/test.ql is at /one/two/three/three.testproj
         const testdir = dirname(testPath);
@@ -207,7 +207,6 @@ export class DatabaseItemImpl implements DatabaseItem {
         return pathsEqual(
           databasePath,
           join(testdir, `${testdirbase}.testproj`),
-          process.platform,
         );
       }
     } catch {

extensions/ql-vscode/src/databases/local-databases/database-manager.ts

@@ -1,6 +1,7 @@
 import vscode, { ExtensionContext } from "vscode";
-import { extLogger, Logger } from "../../common";
-import { DisposableObject } from "../../pure/disposable-object";
+import { Logger, showAndLogExceptionWithTelemetry } from "../../common/logging";
+import { extLogger } from "../../common/logging/vscode";
+import { DisposableObject } from "../../common/disposable-object";
 import { App } from "../../common/app";
 import { QueryRunner } from "../../query-server";
 import * as cli from "../../codeql-cli/cli";
@@ -13,22 +14,22 @@ import {
 import { join } from "path";
 import { FullDatabaseOptions } from "./database-options";
 import { DatabaseItemImpl } from "./database-item-impl";
+import { showNeverAskAgainDialog } from "../../common/vscode/dialog";
 import {
   getFirstWorkspaceFolder,
   isFolderAlreadyInWorkspace,
-  isQueryLanguage,
-  showAndLogExceptionWithTelemetry,
-  showNeverAskAgainDialog,
-} from "../../helpers";
+} from "../../common/vscode/workspace-folders";
+import { isQueryLanguage } from "../../common/query-language";
 import { existsSync } from "fs";
-import { QlPackGenerator } from "../../qlpack-generator";
-import { asError, getErrorMessage } from "../../pure/helpers-pure";
+import { QlPackGenerator } from "../../local-queries/qlpack-generator";
+import { asError, getErrorMessage } from "../../common/helpers-pure";
 import { DatabaseItem, PersistedDatabaseItem } from "./database-item";
-import { redactableError } from "../../pure/errors";
+import { redactableError } from "../../common/errors";
 import { remove } from "fs-extra";
-import { containsPath } from "../../pure/files";
+import { containsPath } from "../../common/files";
 import { DatabaseChangedEvent, DatabaseEventKind } from "./database-events";
 import { DatabaseResolver } from "./database-resolver";
+import { telemetryListener } from "../../common/vscode/telemetry";
 
 /**
  * The name of the key in the workspaceState dictionary in which we
@@ -103,8 +104,6 @@ export class DatabaseManager extends DisposableObject {
    * databases.
    */
   public async openDatabase(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
     uri: vscode.Uri,
     makeSelected = true,
     displayName?: string,
@@ -114,9 +113,7 @@ export class DatabaseManager extends DisposableObject {
 
     return await this.addExistingDatabaseItem(
       databaseItem,
-      progress,
       makeSelected,
-      token,
       isTutorialDatabase,
     );
   }
@@ -129,9 +126,7 @@ export class DatabaseManager extends DisposableObject {
    */
   private async addExistingDatabaseItem(
     databaseItem: DatabaseItemImpl,
-    progress: ProgressCallback,
     makeSelected: boolean,
-    token: vscode.CancellationToken,
     isTutorialDatabase?: boolean,
   ): Promise<DatabaseItem> {
     const existingItem = this.findDatabaseItem(databaseItem.databaseUri);
@@ -142,7 +137,7 @@ export class DatabaseManager extends DisposableObject {
       return existingItem;
     }
 
-    await this.addDatabaseItem(progress, token, databaseItem);
+    await this.addDatabaseItem(databaseItem);
     if (makeSelected) {
       await this.setCurrentDatabaseItem(databaseItem);
     }
@@ -259,14 +254,11 @@ export class DatabaseManager extends DisposableObject {
     }
   }
 
-  private async reregisterDatabases(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
-  ) {
+  private async reregisterDatabases(progress: ProgressCallback) {
     let completed = 0;
     await Promise.all(
       this._databaseItems.map(async (databaseItem) => {
-        await this.registerDatabase(progress, token, databaseItem);
+        await this.registerDatabase(databaseItem);
         completed++;
         progress({
           maxStep: this._databaseItems.length,
@@ -323,8 +315,6 @@ export class DatabaseManager extends DisposableObject {
   }
 
   private async createDatabaseItemFromPersistedState(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
     state: PersistedDatabaseItem,
   ): Promise<DatabaseItemImpl> {
     let displayName: string | undefined = undefined;
@@ -355,12 +345,12 @@ export class DatabaseManager extends DisposableObject {
 
     // Avoid persisting the database state after adding since that should happen only after
     // all databases have been added.
-    await this.addDatabaseItem(progress, token, item, false);
+    await this.addDatabaseItem(item, false);
     return item;
   }
 
   public async loadPersistedState(): Promise<void> {
-    return withProgress(async (progress, token) => {
+    return withProgress(async (progress) => {
       const currentDatabaseUri =
         this.ctx.workspaceState.get<string>(CURRENT_DB);
       const databases = this.ctx.workspaceState.get<PersistedDatabaseItem[]>(
@@ -387,13 +377,11 @@ export class DatabaseManager extends DisposableObject {
           });
 
           const databaseItem = await this.createDatabaseItemFromPersistedState(
-            progress,
-            token,
             database,
           );
           try {
             await this.refreshDatabase(databaseItem);
-            await this.registerDatabase(progress, token, databaseItem);
+            await this.registerDatabase(databaseItem);
             if (currentDatabaseUri === database.uri) {
               await this.setCurrentDatabaseItem(databaseItem, true);
             }
@@ -412,6 +400,8 @@ export class DatabaseManager extends DisposableObject {
       } catch (e) {
         // database list had an unexpected type - nothing to be done?
         void showAndLogExceptionWithTelemetry(
+          extLogger,
+          telemetryListener,
           redactableError(
             asError(e),
           )`Database list loading failed: ${getErrorMessage(e)}`,
@@ -486,8 +476,6 @@ export class DatabaseManager extends DisposableObject {
   }
 
   private async addDatabaseItem(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
     item: DatabaseItemImpl,
     updatePersistedState = true,
   ) {
@@ -501,7 +489,7 @@ export class DatabaseManager extends DisposableObject {
     // Database items reconstituted from persisted state
     // will not have their contents yet.
     if (item.contents?.datasetUri) {
-      await this.registerDatabase(progress, token, item);
+      await this.registerDatabase(item);
     }
     // note that we use undefined as the item in order to reset the entire tree
     this._onDidChangeDatabaseItem.fire({
@@ -520,11 +508,7 @@ export class DatabaseManager extends DisposableObject {
     });
   }
 
-  public async removeDatabaseItem(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
-    item: DatabaseItem,
-  ) {
+  public async removeDatabaseItem(item: DatabaseItem) {
     if (this._currentDatabaseItem === item) {
       this._currentDatabaseItem = undefined;
     }
@@ -546,7 +530,7 @@ export class DatabaseManager extends DisposableObject {
     }
 
     // Remove this database item from the allow-list
-    await this.deregisterDatabase(progress, token, item);
+    await this.deregisterDatabase(item);
 
     // Delete folder from file system only if it is controlled by the extension
     if (this.isExtensionControlledLocation(item.databaseUri)) {
@@ -569,22 +553,15 @@ export class DatabaseManager extends DisposableObject {
     });
   }
 
-  public async removeAllDatabases(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
-  ) {
+  public async removeAllDatabases() {
     for (const item of this.databaseItems) {
-      await this.removeDatabaseItem(progress, token, item);
+      await this.removeDatabaseItem(item);
     }
   }
 
-  private async deregisterDatabase(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
-    dbItem: DatabaseItem,
-  ) {
+  private async deregisterDatabase(dbItem: DatabaseItem) {
     try {
-      await this.qs.deregisterDatabase(progress, token, dbItem);
+      await this.qs.deregisterDatabase(dbItem);
     } catch (e) {
       const message = getErrorMessage(e);
       if (message === "Connection is disposed.") {
@@ -597,12 +574,8 @@ export class DatabaseManager extends DisposableObject {
       throw e;
     }
   }
-  private async registerDatabase(
-    progress: ProgressCallback,
-    token: vscode.CancellationToken,
-    dbItem: DatabaseItem,
-  ) {
-    await this.qs.registerDatabase(progress, token, dbItem);
+  private async registerDatabase(dbItem: DatabaseItem) {
+    await this.qs.registerDatabase(dbItem);
   }
 
   /**
@@ -652,7 +625,7 @@ export class DatabaseManager extends DisposableObject {
   private isExtensionControlledLocation(uri: vscode.Uri) {
     const storageUri = this.ctx.storageUri || this.ctx.globalStorageUri;
     if (storageUri) {
-      return containsPath(storageUri.fsPath, uri.fsPath, process.platform);
+      return containsPath(storageUri.fsPath, uri.fsPath);
     }
     return false;
   }