Merge remote-tracking branch 'origin/main' into dbartol/new-test-api

extensions/ql-vscode/CHANGELOG.md

@@ -2,10 +2,15 @@
 
 ## [UNRELEASED]
 
+- If you run a query without having selected a database, we show a more intuitive prompt to help you select a database. [#3214](https://github.com/github/vscode-codeql/pull/3214)
+
+## 1.12.0 - 11 January 2024
+
 - Add a prompt for downloading a GitHub database when opening a GitHub repository. [#3138](https://github.com/github/vscode-codeql/pull/3138)
 - Avoid showing a popup when hovering over source elements in database source files. [#3125](https://github.com/github/vscode-codeql/pull/3125)
 - Add comparison of alerts when comparing query results. This allows viewing path explanations for differences in alerts. [#3113](https://github.com/github/vscode-codeql/pull/3113)
 - Fix a bug where the CodeQL CLI and variant analysis results were corrupted after extraction in VS Code Insiders. [#3151](https://github.com/github/vscode-codeql/pull/3151) & [#3152](https://github.com/github/vscode-codeql/pull/3152)
+- Show progress when extracting the CodeQL CLI distribution during installation. [#3157](https://github.com/github/vscode-codeql/pull/3157)
 - Add option to cancel opening the model editor. [#3189](https://github.com/github/vscode-codeql/pull/3189)
 
 ## 1.11.0 - 13 December 2023

extensions/ql-vscode/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vscode-codeql",
-  "version": "1.11.1",
+  "version": "1.12.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vscode-codeql",
-      "version": "1.11.1",
+      "version": "1.12.1",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {
@@ -108,7 +108,7 @@
         "eslint-plugin-github": "^4.4.1",
         "eslint-plugin-import": "^2.29.1",
         "eslint-plugin-jest-dom": "^5.0.1",
-        "eslint-plugin-prettier": "^5.0.0",
+        "eslint-plugin-prettier": "^5.1.3",
         "eslint-plugin-react": "^7.31.8",
         "eslint-plugin-react-hooks": "^4.6.0",
         "eslint-plugin-storybook": "^0.6.4",
@@ -15856,23 +15856,24 @@
       }
     },
     "node_modules/eslint-plugin-prettier": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.0.1.tgz",
-      "integrity": "sha512-m3u5RnR56asrwV/lDC4GHorlW75DsFfmUcjfCYylTUs85dBRnB7VM6xG8eCMJdeDRnppzmxZVf1GEPJvl1JmNg==",
+      "version": "5.1.3",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.1.3.tgz",
+      "integrity": "sha512-C9GCVAs4Eq7ZC/XFQHITLiHJxQngdtraXaM+LoUFoFp/lHNl2Zn8f3WQbe9HvTBBQ9YnKFB0/2Ajdqwo5D1EAw==",
       "dev": true,
       "dependencies": {
         "prettier-linter-helpers": "^1.0.0",
-        "synckit": "^0.8.5"
+        "synckit": "^0.8.6"
       },
       "engines": {
         "node": "^14.18.0 || >=16.0.0"
       },
       "funding": {
-        "url": "https://opencollective.com/prettier"
+        "url": "https://opencollective.com/eslint-plugin-prettier"
       },
       "peerDependencies": {
         "@types/eslint": ">=8.0.0",
         "eslint": ">=8.0.0",
+        "eslint-config-prettier": "*",
         "prettier": ">=3.0.0"
       },
       "peerDependenciesMeta": {

extensions/ql-vscode/package.json

@@ -4,7 +4,7 @@
   "description": "CodeQL for Visual Studio Code",
   "author": "GitHub",
   "private": true,
-  "version": "1.11.1",
+  "version": "1.12.1",
   "publisher": "GitHub",
   "license": "MIT",
   "icon": "media/VS-marketplace-CodeQL-icon.png",
@@ -2005,7 +2005,7 @@
     "eslint-plugin-github": "^4.4.1",
     "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-jest-dom": "^5.0.1",
-    "eslint-plugin-prettier": "^5.0.0",
+    "eslint-plugin-prettier": "^5.1.3",
     "eslint-plugin-react": "^7.31.8",
     "eslint-plugin-react-hooks": "^4.6.0",
     "eslint-plugin-storybook": "^0.6.4",

extensions/ql-vscode/src/databases/local-databases-ui.ts

@@ -5,6 +5,7 @@ import type {
   ProviderResult,
   TreeDataProvider,
   CancellationToken,
+  QuickPickItem,
 } from "vscode";
 import {
   EventEmitter,
@@ -28,7 +29,11 @@ import type {
   ProgressCallback,
   ProgressContext,
 } from "../common/vscode/progress";
-import { withInheritedProgress, withProgress } from "../common/vscode/progress";
+import {
+  UserCancellationException,
+  withInheritedProgress,
+  withProgress,
+} from "../common/vscode/progress";
 import {
   isLikelyDatabaseRoot,
   isLikelyDbLanguageFolder,
@@ -52,7 +57,10 @@ import {
   createMultiSelectionCommand,
   createSingleSelectionCommand,
 } from "../common/vscode/selection-commands";
-import { tryGetQueryLanguage } from "../common/query-language";
+import {
+  getLanguageDisplayName,
+  tryGetQueryLanguage,
+} from "../common/query-language";
 import type { LanguageContextStore } from "../language-context-store";
 
 enum SortOrder {
@@ -227,6 +235,18 @@ async function chooseDatabaseDir(byFolder: boolean): Promise<Uri | undefined> {
   return getFirst(chosen);
 }
 
+interface DatabaseSelectionQuickPickItem extends QuickPickItem {
+  databaseKind: "new" | "existing";
+}
+
+export interface DatabaseQuickPickItem extends QuickPickItem {
+  databaseItem: DatabaseItem;
+}
+
+interface DatabaseImportQuickPickItems extends QuickPickItem {
+  importType: "URL" | "github" | "archive" | "folder";
+}
+
 export class DatabaseUI extends DisposableObject {
   private treeDataProvider: DatabaseTreeDataProvider;
 
@@ -794,13 +814,120 @@ export class DatabaseUI extends DisposableObject {
    * notification if it tries to perform any long-running operations.
    */
   private async getDatabaseItemInternal(
-    progress: ProgressContext | undefined,
+    progressContext: ProgressContext | undefined,
   ): Promise<DatabaseItem | undefined> {
     if (this.databaseManager.currentDatabaseItem === undefined) {
-      await this.chooseAndSetDatabase(false, progress);
+      progressContext?.progress({
+        maxStep: 2,
+        step: 1,
+        message: "Choosing database",
+      });
+      await this.promptForDatabase();
+    }
+    return this.databaseManager.currentDatabaseItem;
+  }
+
+  private async promptForDatabase(): Promise<void> {
+    const quickPickItems: DatabaseSelectionQuickPickItem[] = [
+      {
+        label: "$(database) Existing database",
+        detail: "Select an existing database from your workspace",
+        alwaysShow: true,
+        databaseKind: "existing",
+      },
+      {
+        label: "$(arrow-down) New database",
+        detail: "Import a new database from the cloud or your local machine",
+        alwaysShow: true,
+        databaseKind: "new",
+      },
+    ];
+    const selectedOption =
+      await window.showQuickPick<DatabaseSelectionQuickPickItem>(
+        quickPickItems,
+        {
+          placeHolder: "Select an option",
+          ignoreFocusOut: true,
+        },
+      );
+
+    if (!selectedOption) {
+      throw new UserCancellationException("No database selected", true);
     }
 
-    return this.databaseManager.currentDatabaseItem;
+    if (selectedOption.databaseKind === "existing") {
+      await this.selectExistingDatabase();
+    } else if (selectedOption.databaseKind === "new") {
+      await this.importNewDatabase();
+    }
+  }
+
+  private async selectExistingDatabase() {
+    const dbItems: DatabaseQuickPickItem[] =
+      this.databaseManager.databaseItems.map((dbItem) => ({
+        label: dbItem.name,
+        description: getLanguageDisplayName(dbItem.language),
+        databaseItem: dbItem,
+      }));
+
+    const selectedDatabase = await window.showQuickPick(dbItems, {
+      placeHolder: "Select a database",
+      ignoreFocusOut: true,
+    });
+
+    if (!selectedDatabase) {
+      throw new UserCancellationException("No database selected", true);
+    }
+
+    await this.databaseManager.setCurrentDatabaseItem(
+      selectedDatabase.databaseItem,
+    );
+  }
+
+  private async importNewDatabase() {
+    const importOptions: DatabaseImportQuickPickItems[] = [
+      {
+        label: "$(github) GitHub",
+        detail: "Import a database from a GitHub repository",
+        alwaysShow: true,
+        importType: "github",
+      },
+      {
+        label: "$(link) URL",
+        detail: "Import a database archive or folder from a remote URL",
+        alwaysShow: true,
+        importType: "URL",
+      },
+      {
+        label: "$(file-zip) Archive",
+        detail: "Import a database from a local ZIP archive",
+        alwaysShow: true,
+        importType: "archive",
+      },
+      {
+        label: "$(folder) Folder",
+        detail: "Import a database from a local folder",
+        alwaysShow: true,
+        importType: "folder",
+      },
+    ];
+    const selectedImportOption =
+      await window.showQuickPick<DatabaseImportQuickPickItems>(importOptions, {
+        placeHolder: "Import a database from...",
+        ignoreFocusOut: true,
+      });
+    if (!selectedImportOption) {
+      throw new UserCancellationException("No database selected", true);
+    }
+    if (selectedImportOption.importType === "github") {
+      await this.handleChooseDatabaseGithub();
+    } else if (selectedImportOption.importType === "URL") {
+      await this.handleChooseDatabaseInternet();
+    } else if (selectedImportOption.importType === "archive") {
+      await this.handleChooseDatabaseArchive();
+    } else if (selectedImportOption.importType === "folder") {
+      await this.handleChooseDatabaseFolder();
+    }
   }
 
   /**

extensions/ql-vscode/src/local-queries/local-queries.ts

@@ -3,12 +3,7 @@ import type {
   ProgressUpdate,
 } from "../common/vscode/progress";
 import { withProgress } from "../common/vscode/progress";
-import type {
-  CancellationToken,
-  QuickPickItem,
-  Range,
-  TabInputText,
-} from "vscode";
+import type { CancellationToken, Range, TabInputText } from "vscode";
 import { CancellationTokenSource, Uri, window } from "vscode";
 import {
   TeeLogger,
@@ -23,7 +18,10 @@ import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { displayQuickQuery } from "./quick-query";
 import type { CoreCompletedQuery, QueryRunner } from "../query-server";
 import type { QueryHistoryManager } from "../query-history/query-history-manager";
-import type { DatabaseUI } from "../databases/local-databases-ui";
+import type {
+  DatabaseQuickPickItem,
+  DatabaseUI,
+} from "../databases/local-databases-ui";
 import type { ResultsView } from "./results-view";
 import type {
   DatabaseItem,
@@ -55,10 +53,6 @@ import { tryGetQueryLanguage } from "../common/query-language";
 import type { LanguageContextStore } from "../language-context-store";
 import type { ExtensionApp } from "../common/vscode/vscode-app";
 
-interface DatabaseQuickPickItem extends QuickPickItem {
-  databaseItem: DatabaseItem;
-}
-
 export enum QuickEvalType {
   None,
   QuickEval,

extensions/ql-vscode/src/model-editor/languages/ruby/access-paths.ts

@@ -0,0 +1,41 @@
+export function parseRubyMethodFromPath(path: string): string {
+  const match = path.match(/Method\[([^\]]+)].*/);
+  if (match) {
+    return match[1];
+  } else {
+    return "";
+  }
+}
+
+export function parseRubyAccessPath(path: string): {
+  methodName: string;
+  path: string;
+} {
+  const match = path.match(/Method\[([^\]]+)]\.(.*)/);
+  if (match) {
+    return { methodName: match[1], path: match[2] };
+  } else {
+    return { methodName: "", path: "" };
+  }
+}
+
+export function rubyMethodSignature(typeName: string, methodName: string) {
+  return `${typeName}#${methodName}`;
+}
+
+export function rubyMethodPath(methodName: string) {
+  if (methodName === "") {
+    return "";
+  }
+
+  return `Method[${methodName}]`;
+}
+
+export function rubyPath(methodName: string, path: string) {
+  const methodPath = rubyMethodPath(methodName);
+  if (methodPath === "") {
+    return path;
+  }
+
+  return `${methodPath}.${path}`;
+}

extensions/ql-vscode/src/model-editor/languages/ruby/index.ts

@@ -4,48 +4,13 @@ import { Mode } from "../../shared/mode";
 import { parseGenerateModelResults } from "./generate";
 import type { MethodArgument } from "../../method";
 import { getArgumentsList } from "../../method";
-
-function parseRubyMethodFromPath(path: string): string {
-  const match = path.match(/Method\[([^\]]+)].*/);
-  if (match) {
-    return match[1];
-  } else {
-    return "";
-  }
-}
-
-function parseRubyAccessPath(path: string): {
-  methodName: string;
-  path: string;
-} {
-  const match = path.match(/Method\[([^\]]+)]\.(.*)/);
-  if (match) {
-    return { methodName: match[1], path: match[2] };
-  } else {
-    return { methodName: "", path: "" };
-  }
-}
-
-function rubyMethodSignature(typeName: string, methodName: string) {
-  return `${typeName}#${methodName}`;
-}
-
-function rubyMethodPath(methodName: string) {
-  if (methodName === "") {
-    return "";
-  }
-
-  return `Method[${methodName}]`;
-}
-
-function rubyPath(methodName: string, path: string) {
-  const methodPath = rubyMethodPath(methodName);
-  if (methodPath === "") {
-    return path;
-  }
-
-  return `${methodPath}.${path}`;
-}
+import {
+  parseRubyAccessPath,
+  parseRubyMethodFromPath,
+  rubyMethodPath,
+  rubyMethodSignature,
+  rubyPath,
+} from "./access-paths";
 
 export const ruby: ModelsAsDataLanguage = {
   availableModes: [Mode.Framework],

extensions/ql-vscode/src/model-editor/shared/access-paths.ts

@@ -0,0 +1,128 @@
+/**
+ * This file contains functions for parsing and validating access paths.
+ *
+ * This intentionally does not simply split by '.' since tokens may contain dots,
+ * e.g. `Field[foo.Bar.x]`. Instead, it uses some simple parsing to match valid tokens.
+ *
+ * Valid syntax was determined based on this file:
+ * https://github.com/github/codeql/blob/a04830b8b2d3e5f7df8e1f80f06c020b987a89a3/ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll
+ *
+ * In contrast to that file, we do not use a regex for parsing to allow us to be more lenient.
+ * For example, we can parse partial access paths such as `Field[foo.Bar.x` without error.
+ */
+
+/**
+ * A range of characters in an access path. The start position is inclusive, the end position is exclusive.
+ */
+type AccessPathRange = {
+  /**
+   * Zero-based index of the first character of the token.
+   */
+  start: number;
+  /**
+   * Zero-based index of the character after the last character of the token.
+   */
+  end: number;
+};
+
+/**
+ * A token in an access path. For example, `Argument[foo]` is a token.
+ */
+type AccessPartToken = {
+  text: string;
+  range: AccessPathRange;
+};
+
+/**
+ * Parses an access path into tokens.
+ *
+ * @param path The access path to parse.
+ * @returns An array of tokens.
+ */
+export function parseAccessPathTokens(path: string): AccessPartToken[] {
+  const parts: AccessPartToken[] = [];
+
+  let currentPart = "";
+  let currentPathStart = 0;
+  // Keep track of the number of brackets we can parse the path correctly when it contains
+  // nested brackets such as `Argument[foo[bar].test].Element`.
+  let bracketCounter = 0;
+  for (let i = 0; i < path.length; i++) {
+    const c = path[i];
+
+    if (c === "[") {
+      bracketCounter++;
+    } else if (c === "]") {
+      bracketCounter--;
+    } else if (c === "." && bracketCounter === 0) {
+      // A part ends when we encounter a dot that is not inside brackets.
+      parts.push({
+        text: currentPart,
+        range: {
+          start: currentPathStart,
+          end: i,
+        },
+      });
+      currentPart = "";
+      currentPathStart = i + 1;
+      continue;
+    }
+
+    currentPart += c;
+  }
+
+  // The last part should not be followed by a dot, so we need to add it manually.
+  // If the path is empty, such as for `Argument[foo].`, then this is still correct
+  // since the `validateAccessPath` function will check that none of the tokens are
+  // empty.
+  parts.push({
+    text: currentPart,
+    range: {
+      start: currentPathStart,
+      end: path.length,
+    },
+  });
+
+  return parts;
+}
+
+// Regex for a single part of the access path
+const tokenRegex = /^(\w+)(?:\[([^\]]*)])?$/;
+
+type AccessPathDiagnostic = {
+  range: AccessPathRange;
+  message: string;
+};
+
+/**
+ * Validates an access path and returns any errors. This requires that the path is a valid path
+ * and does not allow partial access paths.
+ *
+ * @param path The access path to validate.
+ * @returns An array of diagnostics for any errors in the access path.
+ */
+export function validateAccessPath(path: string): AccessPathDiagnostic[] {
+  if (path === "") {
+    return [];
+  }
+
+  const tokens = parseAccessPathTokens(path);
+
+  return tokens
+    .map((token): AccessPathDiagnostic | null => {
+      if (tokenRegex.test(token.text)) {
+        return null;
+      }
+
+      let message = "Invalid access path";
+      if (token.range.start === token.range.end) {
+        message = "Unexpected empty token";
+      }
+
+      return {
+        range: token.range,
+        message,
+      };
+    })
+    .filter((token): token is AccessPathDiagnostic => token !== null);
+}

extensions/ql-vscode/test/unit-tests/model-editor/shared/access-paths.test.ts

@@ -0,0 +1,251 @@
+import {
+  parseAccessPathTokens,
+  validateAccessPath,
+} from "../../../../src/model-editor/shared/access-paths";
+
+describe("parseAccessPathTokens", () => {
+  it.each([
+    {
+      path: "Argument[foo].Element.Field[@test]",
+      parts: [
+        {
+          range: {
+            start: 0,
+            end: 13,
+          },
+          text: "Argument[foo]",
+        },
+        {
+          range: {
+            start: 14,
+            end: 21,
+          },
+          text: "Element",
+        },
+        {
+          range: {
+            start: 22,
+            end: 34,
+          },
+          text: "Field[@test]",
+        },
+      ],
+    },
+    {
+      path: "Argument[foo].Element.Field[foo.Bar.x]",
+      parts: [
+        {
+          range: {
+            start: 0,
+            end: 13,
+          },
+          text: "Argument[foo]",
+        },
+        {
+          range: {
+            start: 14,
+            end: 21,
+          },
+          text: "Element",
+        },
+        {
+          range: {
+            start: 22,
+            end: 38,
+          },
+          text: "Field[foo.Bar.x]",
+        },
+      ],
+    },
+    {
+      path: "Argument[",
+      parts: [
+        {
+          range: {
+            start: 0,
+            end: 9,
+          },
+          text: "Argument[",
+        },
+      ],
+    },
+    {
+      path: "Argument[se",
+      parts: [
+        {
+          range: {
+            start: 0,
+            end: 11,
+          },
+          text: "Argument[se",
+        },
+      ],
+    },
+    {
+      path: "Argument[foo].Field[",
+      parts: [
+        {
+          range: {
+            start: 0,
+            end: 13,
+          },
+          text: "Argument[foo]",
+        },
+        {
+          range: {
+            start: 14,
+            end: 20,
+          },
+          text: "Field[",
+        },
+      ],
+    },
+    {
+      path: "Argument[foo].",
+      parts: [
+        {
+          text: "Argument[foo]",
+          range: {
+            end: 13,
+            start: 0,
+          },
+        },
+        {
+          text: "",
+          range: {
+            end: 14,
+            start: 14,
+          },
+        },
+      ],
+    },
+    {
+      path: "Argument[foo]..",
+      parts: [
+        {
+          text: "Argument[foo]",
+          range: {
+            end: 13,
+            start: 0,
+          },
+        },
+        {
+          text: "",
+          range: {
+            end: 14,
+            start: 14,
+          },
+        },
+        {
+          text: "",
+          range: {
+            end: 15,
+            start: 15,
+          },
+        },
+      ],
+    },
+    {
+      path: "Argument[foo[bar].test].Element.",
+      parts: [
+        {
+          range: {
+            start: 0,
+            end: 23,
+          },
+          text: "Argument[foo[bar].test]",
+        },
+        {
+          range: {
+            start: 24,
+            end: 31,
+          },
+          text: "Element",
+        },
+        {
+          range: {
+            start: 32,
+            end: 32,
+          },
+          text: "",
+        },
+      ],
+    },
+  ])(`parses correctly for $path`, ({ path, parts }) => {
+    expect(parseAccessPathTokens(path)).toEqual(parts);
+  });
+});
+
+describe("validateAccessPath", () => {
+  it.each([
+    {
+      path: "Argument[foo].Element.Field[@test]",
+      diagnostics: [],
+    },
+    {
+      path: "Argument[foo].Element.Field[foo.Bar.x]",
+      diagnostics: [],
+    },
+    {
+      path: "Argument[",
+      diagnostics: [
+        {
+          message: "Invalid access path",
+          range: {
+            start: 0,
+            end: 9,
+          },
+        },
+      ],
+    },
+    {
+      path: "Argument[se",
+      diagnostics: [
+        {
+          message: "Invalid access path",
+          range: {
+            start: 0,
+            end: 11,
+          },
+        },
+      ],
+    },
+    {
+      path: "Argument[foo].Field[",
+      diagnostics: [
+        {
+          message: "Invalid access path",
+          range: {
+            start: 14,
+            end: 20,
+          },
+        },
+      ],
+    },
+    {
+      path: "Argument[foo].",
+      diagnostics: [
+        { message: "Unexpected empty token", range: { start: 14, end: 14 } },
+      ],
+    },
+    {
+      path: "Argument[foo]..",
+      diagnostics: [
+        { message: "Unexpected empty token", range: { start: 14, end: 14 } },
+        { message: "Unexpected empty token", range: { start: 15, end: 15 } },
+      ],
+    },
+    {
+      path: "Argument[foo[bar].test].Element.",
+      diagnostics: [
+        { message: "Invalid access path", range: { start: 0, end: 23 } },
+        { message: "Unexpected empty token", range: { start: 32, end: 32 } },
+      ],
+    },
+  ])(
+    `validates $path correctly with $diagnostics.length errors`,
+    ({ path, diagnostics }) => {
+      expect(validateAccessPath(path)).toEqual(diagnostics);
+    },
+  );
+});