Merge branch 'main' into simpler-location-url

extensions/ql-vscode/package.json

@@ -36,6 +36,7 @@
   "activationEvents": [
     "onLanguage:ql",
     "onLanguage:ql-summary",
+    "onView:codeQLQueries",
     "onView:codeQLDatabases",
     "onView:codeQLVariantAnalysisRepositories",
     "onView:codeQLQueryHistory",
@@ -476,6 +477,14 @@
         "command": "codeQL.previewQueryHelp",
         "title": "CodeQL: Preview Query Help"
       },
+      {
+        "command": "codeQL.previewQueryHelpContextExplorer",
+        "title": "CodeQL: Preview Query Help"
+      },
+      {
+        "command": "codeQL.previewQueryHelpContextEditor",
+        "title": "CodeQL: Preview Query Help"
+      },
       {
         "command": "codeQL.quickQuery",
         "title": "CodeQL: Quick Query"
@@ -828,6 +837,11 @@
         "title": "CodeQL: Go to QL Code",
         "enablement": "codeql.hasQLSource"
       },
+      {
+        "command": "codeQL.gotoQLContextEditor",
+        "title": "CodeQL: Go to QL Code",
+        "enablement": "codeql.hasQLSource"
+      },
       {
         "command": "codeQL.openDataExtensionsEditor",
         "title": "CodeQL: Open Data Extensions Editor"
@@ -1117,7 +1131,7 @@
           "when": "resourceExtname == .qlref"
         },
         {
-          "command": "codeQL.previewQueryHelp",
+          "command": "codeQL.previewQueryHelpContextExplorer",
           "group": "9_qlCommands",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         }
@@ -1203,6 +1217,14 @@
           "command": "codeQL.previewQueryHelp",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         },
+        {
+          "command": "codeQL.previewQueryHelpContextEditor",
+          "when": "false"
+        },
+        {
+          "command": "codeQL.previewQueryHelpContextExplorer",
+          "when": "false"
+        },
         {
           "command": "codeQL.setCurrentDatabase",
           "when": "false"
@@ -1466,6 +1488,10 @@
         {
           "command": "codeQLTests.acceptOutputContextTestItem",
           "when": "false"
+        },
+        {
+          "command": "codeQL.gotoQLContextEditor",
+          "when": "false"
         }
       ],
       "editor/context": [
@@ -1510,11 +1536,11 @@
           "when": "resourceExtname == .qlref"
         },
         {
-          "command": "codeQL.previewQueryHelp",
+          "command": "codeQL.previewQueryHelpContextEditor",
           "when": "resourceExtname == .qhelp && isWorkspaceTrusted"
         },
         {
-          "command": "codeQL.gotoQL",
+          "command": "codeQL.gotoQLContextEditor",
           "when": "editorLangId == ql-summary && config.codeQL.canary"
         }
       ]
@@ -1530,6 +1556,11 @@
     },
     "views": {
       "ql-container": [
+        {
+          "id": "codeQLQueries",
+          "name": "Queries",
+          "when": "config.codeQL.canary && config.codeQL.queriesPanel"
+        },
         {
           "id": "codeQLDatabases",
           "name": "Databases"

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -1525,10 +1525,23 @@ export function spawnServer(
     );
   }
 
+  let lastStdout: any = undefined;
+  child.stdout!.on("data", (data) => {
+    lastStdout = data;
+  });
   // Set up event listeners.
-  child.on("close", (code) =>
-    logger.log(`Child process exited with code ${code}`),
-  );
+  child.on("close", async (code, signal) => {
+    if (code !== null)
+      void logger.log(`Child process exited with code ${code}`);
+    if (signal)
+      void logger.log(
+        `Child process exited due to receipt of signal ${signal}`,
+      );
+    // If the process exited abnormally, log the last stdout message,
+    // It may be from the jvm.
+    if (code !== 0 && lastStdout !== undefined)
+      void logger.log(`Last stdout was "${lastStdout.toString()}"`);
+  });
   child.stderr!.on("data", stderrListener);
   if (stdoutListener !== undefined) {
     child.stdout!.on("data", stdoutListener);

extensions/ql-vscode/src/codeql-cli/distribution.ts

@@ -3,7 +3,7 @@ import { pathExists, mkdtemp, createWriteStream, remove } from "fs-extra";
 import { tmpdir } from "os";
 import { delimiter, dirname, join } from "path";
 import * as semver from "semver";
-import { parse } from "url";
+import { URL } from "url";
 import { ExtensionContext, Event } from "vscode";
 import { DistributionConfig } from "../config";
 import {
@@ -505,7 +505,7 @@ class ExtensionSpecificDistributionManager {
         0,
       ) || "";
     return join(
-      this.extensionContext.globalStoragePath,
+      this.extensionContext.globalStorageUri.fsPath,
       ExtensionSpecificDistributionManager._currentDistributionFolderBaseName +
         distributionFolderIndex,
     );
@@ -670,7 +670,7 @@ export class ReleasesApiConsumer {
       redirectUrl &&
       redirectCount < ReleasesApiConsumer._maxRedirects
     ) {
-      const parsedRedirectUrl = parse(redirectUrl);
+      const parsedRedirectUrl = new URL(redirectUrl);
       if (parsedRedirectUrl.protocol !== "https:") {
         throw new Error("Encountered a non-https redirect, rejecting");
       }

extensions/ql-vscode/src/common/commands.ts

@@ -115,6 +115,10 @@ export type QueryEditorCommands = {
     selectedQuery: Uri,
   ) => Promise<void>;
   "codeQL.previewQueryHelp": (selectedQuery: Uri) => Promise<void>;
+  "codeQL.previewQueryHelpContextEditor": (selectedQuery: Uri) => Promise<void>;
+  "codeQL.previewQueryHelpContextExplorer": (
+    selectedQuery: Uri,
+  ) => Promise<void>;
 };
 
 // Commands used for running local queries
@@ -299,6 +303,7 @@ export type EvalLogViewerCommands = {
 
 export type SummaryLanguageSupportCommands = {
   "codeQL.gotoQL": () => Promise<void>;
+  "codeQL.gotoQLContextEditor": () => Promise<void>;
 };
 
 export type TestUICommands = {

extensions/ql-vscode/src/common/discovery.ts

@@ -1,5 +1,5 @@
 import { DisposableObject } from "../pure/disposable-object";
-import { extLogger } from "../common";
+import { extLogger } from "./logging/vscode/loggers";
 import { getErrorMessage } from "../pure/helpers-pure";
 
 /**

extensions/ql-vscode/src/config.ts

@@ -702,3 +702,12 @@ export function getAutogenerateQlPacks(): AutogenerateQLPacks {
 export async function setAutogenerateQlPacks(choice: AutogenerateQLPacks) {
   await AUTOGENERATE_QL_PACKS.updateValue(choice, ConfigurationTarget.Global);
 }
+
+/**
+ * A flag indicating whether to show the queries panel in the QL view container.
+ */
+const QUERIES_PANEL = new Setting("queriesPanel", ROOT_SETTING);
+
+export function showQueriesPanel(): boolean {
+  return !!QUERIES_PANEL.getValue<boolean>();
+}

extensions/ql-vscode/src/databases/local-databases.ts

@@ -24,7 +24,7 @@ import { DisposableObject } from "../pure/disposable-object";
 import { Logger, extLogger } from "../common";
 import { asError, getErrorMessage } from "../pure/helpers-pure";
 import { QueryRunner } from "../query-server";
-import { pathsEqual } from "../pure/files";
+import { containsPath, pathsEqual } from "../pure/files";
 import { redactableError } from "../pure/errors";
 import {
   getAutogenerateQlPacks,
@@ -1152,12 +1152,9 @@ export class DatabaseManager extends DisposableObject {
   }
 
   private isExtensionControlledLocation(uri: vscode.Uri) {
-    const storagePath = this.ctx.storagePath || this.ctx.globalStoragePath;
-    // the uri.fsPath function on windows returns a lowercase drive letter,
-    // but storagePath will have an uppercase drive letter. Be sure to compare
-    // URIs to URIs only
-    if (storagePath) {
-      return uri.fsPath.startsWith(vscode.Uri.file(storagePath).fsPath);
+    const storageUri = this.ctx.storageUri || this.ctx.globalStorageUri;
+    if (storageUri) {
+      return containsPath(storageUri.fsPath, uri.fsPath, process.platform);
     }
     return false;
   }

extensions/ql-vscode/src/extension.ts

@@ -125,6 +125,7 @@ import { TestManager } from "./query-testing/test-manager";
 import { TestRunner } from "./query-testing/test-runner";
 import { TestManagerBase } from "./query-testing/test-manager-base";
 import { NewQueryRunner, QueryRunner, QueryServerClient } from "./query-server";
+import { QueriesModule } from "./queries-panel/queries-module";
 
 /**
  * extension.ts
@@ -732,6 +733,8 @@ async function activateWithInstalledDistribution(
   );
   ctx.subscriptions.push(databaseUI);
 
+  QueriesModule.initialize(app);
+
   void extLogger.log("Initializing evaluator log viewer.");
   const evalLogViewer = new EvalLogViewer();
   ctx.subscriptions.push(evalLogViewer);

extensions/ql-vscode/src/language-support/query-editor.ts

@@ -38,6 +38,20 @@ export function getQueryEditorCommands({
         qhelpTmpDir,
         selectedQuery,
       ),
+    "codeQL.previewQueryHelpContextEditor": async (selectedQuery: Uri) =>
+      await previewQueryHelp(
+        commandManager,
+        cliServer,
+        qhelpTmpDir,
+        selectedQuery,
+      ),
+    "codeQL.previewQueryHelpContextExplorer": async (selectedQuery: Uri) =>
+      await previewQueryHelp(
+        commandManager,
+        cliServer,
+        qhelpTmpDir,
+        selectedQuery,
+      ),
   };
 }
 

extensions/ql-vscode/src/log-insights/summary-language-support.ts

@@ -78,6 +78,7 @@ export class SummaryLanguageSupport extends DisposableObject {
   public getCommands(): SummaryLanguageSupportCommands {
     return {
       "codeQL.gotoQL": this.handleGotoQL.bind(this),
+      "codeQL.gotoQLContextEditor": this.handleGotoQL.bind(this),
     };
   }
 

extensions/ql-vscode/src/pure/files.ts

@@ -51,21 +51,37 @@ export async function getDirectoryNamesInsidePath(
   return dirNames;
 }
 
+function normalizePath(path: string, platform: NodeJS.Platform): string {
+  // On Windows, "C:/", "C:\", and "c:/" are all equivalent. We need
+  // to normalize the paths to ensure they all get resolved to the
+  // same format. On Windows, we also need to do the comparison
+  // case-insensitively.
+  path = resolve(path);
+  if (platform === "win32") {
+    path = path.toLowerCase();
+  }
+  return path;
+}
+
 export function pathsEqual(
   path1: string,
   path2: string,
   platform: NodeJS.Platform,
 ): boolean {
-  // On Windows, "C:/", "C:\", and "c:/" are all equivalent. We need
-  // to normalize the paths to ensure they all get resolved to the
-  // same format. On Windows, we also need to do the comparison
-  // case-insensitively.
-  path1 = resolve(path1);
-  path2 = resolve(path2);
-  if (platform === "win32") {
-    return path1.toLowerCase() === path2.toLowerCase();
-  }
-  return path1 === path2;
+  return normalizePath(path1, platform) === normalizePath(path2, platform);
+}
+
+/**
+ * Returns true if path1 contains path2.
+ */
+export function containsPath(
+  path1: string,
+  path2: string,
+  platform: NodeJS.Platform,
+): boolean {
+  return normalizePath(path2, platform).startsWith(
+    normalizePath(path1, platform),
+  );
 }
 
 export async function readDirFullPaths(path: string): Promise<string[]> {

extensions/ql-vscode/src/queries-panel/queries-module.ts

@@ -0,0 +1,33 @@
+import { extLogger } from "../common";
+import { App, AppMode } from "../common/app";
+import { isCanary, showQueriesPanel } from "../config";
+import { DisposableObject } from "../pure/disposable-object";
+import { QueriesPanel } from "./queries-panel";
+
+export class QueriesModule extends DisposableObject {
+  private queriesPanel: QueriesPanel | undefined;
+
+  private constructor(readonly app: App) {
+    super();
+  }
+
+  private initialize(app: App): void {
+    if (app.mode === AppMode.Production || !isCanary() || !showQueriesPanel()) {
+      // Currently, we only want to expose the new panel when we are in development and canary mode
+      // and the developer has enabled the "Show queries panel" flag.
+      return;
+    }
+    void extLogger.log("Initializing queries panel.");
+
+    this.queriesPanel = new QueriesPanel();
+    this.push(this.queriesPanel);
+  }
+
+  public static initialize(app: App): QueriesModule {
+    const queriesModule = new QueriesModule(app);
+    app.subscriptions.push(queriesModule);
+
+    queriesModule.initialize(app);
+    return queriesModule;
+  }
+}

extensions/ql-vscode/src/queries-panel/queries-panel.ts

@@ -0,0 +1,21 @@
+import * as vscode from "vscode";
+import { DisposableObject } from "../pure/disposable-object";
+import { QueryTreeDataProvider } from "./query-tree-data-provider";
+import { QueryTreeViewItem } from "./query-tree-view-item";
+
+export class QueriesPanel extends DisposableObject {
+  private readonly dataProvider: QueryTreeDataProvider;
+  private readonly treeView: vscode.TreeView<QueryTreeViewItem>;
+
+  public constructor() {
+    super();
+
+    this.dataProvider = new QueryTreeDataProvider();
+
+    this.treeView = vscode.window.createTreeView("codeQLQueries", {
+      treeDataProvider: this.dataProvider,
+    });
+
+    this.push(this.treeView);
+  }
+}

extensions/ql-vscode/src/queries-panel/query-tree-data-provider.ts

@@ -0,0 +1,54 @@
+import * as vscode from "vscode";
+import { QueryTreeViewItem } from "./query-tree-view-item";
+import { DisposableObject } from "../pure/disposable-object";
+
+export class QueryTreeDataProvider
+  extends DisposableObject
+  implements vscode.TreeDataProvider<QueryTreeViewItem>
+{
+  private queryTreeItems: QueryTreeViewItem[];
+
+  public constructor() {
+    super();
+
+    this.queryTreeItems = this.createTree();
+  }
+
+  private createTree(): QueryTreeViewItem[] {
+    // Temporary mock data, just to populate the tree view.
+    return [
+      {
+        label: "name1",
+        tooltip: "path1",
+        children: [],
+      },
+    ];
+  }
+
+  /**
+   * Returns the UI presentation of the element that gets displayed in the view.
+   * @param item The item to represent.
+   * @returns The UI presentation of the item.
+   */
+  public getTreeItem(
+    item: QueryTreeViewItem,
+  ): vscode.TreeItem | Thenable<vscode.TreeItem> {
+    return item;
+  }
+
+  /**
+   * Called when expanding an item (including the root item).
+   * @param item The item to expand.
+   * @returns The children of the item.
+   */
+  public getChildren(
+    item?: QueryTreeViewItem,
+  ): vscode.ProviderResult<QueryTreeViewItem[]> {
+    if (!item) {
+      // We're at the root.
+      return Promise.resolve(this.queryTreeItems);
+    } else {
+      return Promise.resolve(item.children);
+    }
+  }
+}

extensions/ql-vscode/src/queries-panel/query-tree-view-item.ts

@@ -0,0 +1,11 @@
+import * as vscode from "vscode";
+
+export class QueryTreeViewItem extends vscode.TreeItem {
+  constructor(
+    public readonly label: string,
+    public readonly tooltip: string | undefined,
+    public readonly children: QueryTreeViewItem[],
+  ) {
+    super(label);
+  }
+}

extensions/ql-vscode/src/query-server/query-server-client.ts

@@ -11,9 +11,14 @@ import {
   ProgressMessage,
   WithProgressId,
 } from "../pure/new-messages";
-import { ProgressCallback, ProgressTask } from "../common/vscode/progress";
+import {
+  ProgressCallback,
+  ProgressTask,
+  withProgress,
+} from "../common/vscode/progress";
 import { ServerProcess } from "./server-process";
 import { App } from "../common/app";
+import { showAndLogErrorMessage } from "../helpers";
 
 type ServerOpts = {
   logger: Logger;
@@ -27,6 +32,8 @@ type WithProgressReporting = (
   ) => Thenable<void>,
 ) => Thenable<void>;
 
+const MAX_UNEXPECTED_TERMINATIONS = 5;
+
 /**
  * Client that manages a query server process.
  * The server process is started upon initialization and tracked during its lifetime.
@@ -40,6 +47,9 @@ export class QueryServerClient extends DisposableObject {
   };
   nextCallback: number;
   nextProgress: number;
+
+  unexpectedTerminationCount = 0;
+
   withProgressReporting: WithProgressReporting;
 
   private readonly queryServerStartListeners = [] as Array<ProgressTask<void>>;
@@ -91,10 +101,50 @@ export class QueryServerClient extends DisposableObject {
     }
   }
 
-  /** Restarts the query server by disposing of the current server process and then starting a new one. */
+  /**
+   * Restarts the query server by disposing of the current server process and then starting a new one.
+   * This resets the unexpected termination count. As hopefully it is an indication that the user has fixed the
+   * issue.
+   */
   async restartQueryServer(
     progress: ProgressCallback,
     token: CancellationToken,
+  ): Promise<void> {
+    // Reset the unexpected termination count when we restart the query server manually
+    // or due to config change
+    this.unexpectedTerminationCount = 0;
+    await this.restartQueryServerInternal(progress, token);
+  }
+
+  /**
+   * Try and restart the query server if it has unexpectedly terminated.
+   */
+  private restartQueryServerOnFailure() {
+    if (this.unexpectedTerminationCount < MAX_UNEXPECTED_TERMINATIONS) {
+      void withProgress(
+        async (progress, token) =>
+          this.restartQueryServerInternal(progress, token),
+        {
+          title: "Restarting CodeQL query server due to unexpected termination",
+        },
+      );
+    } else {
+      void showAndLogErrorMessage(
+        "The CodeQL query server has unexpectedly terminated too many times. Please check the logs for errors. You can manually restart the query server using the command 'CodeQL: Restart query server'.",
+      );
+      // Make sure we dispose anyway to reject all pending requests.
+      this.serverProcess?.dispose();
+    }
+    this.unexpectedTerminationCount++;
+  }
+
+  /**
+   * Restarts the query server by disposing of the current server process and then starting a new one.
+   * This does not reset the unexpected termination count.
+   */
+  private async restartQueryServerInternal(
+    progress: ProgressCallback,
+    token: CancellationToken,
   ): Promise<void> {
     this.stopQueryServer();
     await this.startQueryServer();
@@ -196,6 +246,9 @@ export class QueryServerClient extends DisposableObject {
     this.nextCallback = 0;
     this.nextProgress = 0;
     this.progressCallbacks = {};
+    child.on("close", () => {
+      this.restartQueryServerOnFailure();
+    });
   }
 
   get serverProcessPid(): number {

extensions/ql-vscode/src/query-testing/qltest-discovery.ts

@@ -1,5 +1,5 @@
 import { dirname, basename, join, normalize, relative, extname } from "path";
-import { Discovery } from "./discovery";
+import { Discovery } from "../common/discovery";
 import {
   EventEmitter,
   Event,

extensions/ql-vscode/supported_cli_versions.json

@@ -1,5 +1,5 @@
 [
-  "v2.13.0",
+  "v2.13.1",
   "v2.12.7",
   "v2.11.6",
   "v2.7.6",

extensions/ql-vscode/test/unit-tests/pure/files.test.ts

@@ -1,6 +1,7 @@
 import { join } from "path";
 
 import {
+  containsPath,
   gatherQlFiles,
   getDirectoryNamesInsidePath,
   pathsEqual,
@@ -203,3 +204,98 @@ describe("pathsEqual", () => {
     },
   );
 });
+
+describe("containsPath", () => {
+  const testCases: Array<{
+    path1: string;
+    path2: string;
+    platform: NodeJS.Platform;
+    expected: boolean;
+  }> = [
+    {
+      path1:
+        "/home/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript",
+      path2:
+        "/home/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript/example.ql",
+      platform: "linux",
+      expected: true,
+    },
+    {
+      path1:
+        "/HOME/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript",
+      path2:
+        "/home/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript/example.ql",
+      platform: "linux",
+      expected: false,
+    },
+    {
+      path1:
+        "/home/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript/example.ql",
+      path2:
+        "/home/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript",
+      platform: "linux",
+      expected: false,
+    },
+    {
+      path1:
+        "C:/Users/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript",
+      path2:
+        "C:/Users/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript/example.ql",
+      platform: "win32",
+      expected: true,
+    },
+    {
+      path1:
+        "C:/Users/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript",
+      path2:
+        "c:/Users/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript/example.ql",
+      platform: "win32",
+      expected: true,
+    },
+    {
+      path1:
+        "C:/Users/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript",
+      path2:
+        "D:/Users/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript/example.ql",
+      platform: "win32",
+      expected: false,
+    },
+    {
+      path1:
+        "C:/Users/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript/example.ql",
+      path2:
+        "C:/Users/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript",
+      platform: "win32",
+      expected: false,
+    },
+    {
+      path1:
+        "C:/Users/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript",
+      path2:
+        "C:\\Users\\github\\projects\\vscode-codeql-starter\\codeql-custom-queries-javascript\\example.ql",
+      platform: "win32",
+      expected: true,
+    },
+    {
+      path1:
+        "C:/Users/github/projects/vscode-codeql-starter/codeql-custom-queries-javascript",
+      path2:
+        "D:\\Users\\github\\projects\\vscode-codeql-starter\\codeql-custom-queries-javascript\\example.ql",
+      platform: "win32",
+      expected: false,
+    },
+  ];
+
+  test.each(testCases)(
+    "$path1 contains $path2 on $platform = $expected",
+    ({ path1, path2, platform, expected }) => {
+      if (platform !== process.platform) {
+        // We're using the platform-specific path.resolve, so we can't really run
+        // these tests on all platforms.
+        return;
+      }
+
+      expect(containsPath(path1, path2, platform)).toEqual(expected);
+    },
+  );
+});

extensions/ql-vscode/test/vscode-tests/minimal-workspace/local-databases.test.ts

@@ -80,8 +80,7 @@ describe("local databases", () => {
         dynamicProperties: {
           // pretend like databases added in the temp dir are controlled by the extension
           // so that they are deleted upon removal
-          storagePath: () => extensionContextStoragePath,
-          storageUri: () => Uri.parse(extensionContextStoragePath),
+          storageUri: () => Uri.file(extensionContextStoragePath),
         },
       },
     );
@@ -277,7 +276,7 @@ describe("local databases", () => {
       updateSpy.mockClear();
 
       // pretend that the database location is not controlled by the extension
-      (databaseManager as any).ctx.storagePath = "hucairz";
+      (databaseManager as any).ctx.storageUri = Uri.file("hucairz");
       extensionContextStoragePath = "hucairz";
 
       await databaseManager.removeDatabaseItem(