Better error handling for unexpected query server termination. (#2404)

* Log stdout when servers are terminated with errors.

This logs the last stdout chunk (probabaly the last line) if things
went wrong. This can sometimes be useful for debugging.

It also prints the signal when killed by a signal
(rather than printing null)

* Restart/Abort the queryserver if the process dies.

This cancels any running tasks and gives a limited number of restarts.

* Update extensions/ql-vscode/src/codeql-cli/cli.ts

Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>

* Update extensions/ql-vscode/src/query-server/query-server-client.ts

Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>

---------

Co-authored-by: Andrew Eisenberg <aeisenberg@github.com>

extensions/ql-vscode/src/codeql-cli/cli.ts

@@ -1525,10 +1525,23 @@ export function spawnServer(
     );
   }
 
+  let lastStdout: any = undefined;
+  child.stdout!.on("data", (data) => {
+    lastStdout = data;
+  });
   // Set up event listeners.
-  child.on("close", (code) =>
-    logger.log(`Child process exited with code ${code}`),
-  );
+  child.on("close", async (code, signal) => {
+    if (code !== null)
+      void logger.log(`Child process exited with code ${code}`);
+    if (signal)
+      void logger.log(
+        `Child process exited due to receipt of signal ${signal}`,
+      );
+    // If the process exited abnormally, log the last stdout message,
+    // It may be from the jvm.
+    if (code !== 0 && lastStdout !== undefined)
+      void logger.log(`Last stdout was "${lastStdout.toString()}"`);
+  });
   child.stderr!.on("data", stderrListener);
   if (stdoutListener !== undefined) {
     child.stdout!.on("data", stdoutListener);

extensions/ql-vscode/src/query-server/query-server-client.ts

@@ -11,9 +11,14 @@ import {
   ProgressMessage,
   WithProgressId,
 } from "../pure/new-messages";
-import { ProgressCallback, ProgressTask } from "../common/vscode/progress";
+import {
+  ProgressCallback,
+  ProgressTask,
+  withProgress,
+} from "../common/vscode/progress";
 import { ServerProcess } from "./server-process";
 import { App } from "../common/app";
+import { showAndLogErrorMessage } from "../helpers";
 
 type ServerOpts = {
   logger: Logger;
@@ -27,6 +32,8 @@ type WithProgressReporting = (
   ) => Thenable<void>,
 ) => Thenable<void>;
 
+const MAX_UNEXPECTED_TERMINATIONS = 5;
+
 /**
  * Client that manages a query server process.
  * The server process is started upon initialization and tracked during its lifetime.
@@ -40,6 +47,9 @@ export class QueryServerClient extends DisposableObject {
   };
   nextCallback: number;
   nextProgress: number;
+
+  unexpectedTerminationCount = 0;
+
   withProgressReporting: WithProgressReporting;
 
   private readonly queryServerStartListeners = [] as Array<ProgressTask<void>>;
@@ -91,10 +101,50 @@ export class QueryServerClient extends DisposableObject {
     }
   }
 
-  /** Restarts the query server by disposing of the current server process and then starting a new one. */
+  /**
+   * Restarts the query server by disposing of the current server process and then starting a new one.
+   * This resets the unexpected termination count. As hopefully it is an indication that the user has fixed the
+   * issue.
+   */
   async restartQueryServer(
     progress: ProgressCallback,
     token: CancellationToken,
+  ): Promise<void> {
+    // Reset the unexpected termination count when we restart the query server manually
+    // or due to config change
+    this.unexpectedTerminationCount = 0;
+    await this.restartQueryServerInternal(progress, token);
+  }
+
+  /**
+   * Try and restart the query server if it has unexpectedly terminated.
+   */
+  private restartQueryServerOnFailure() {
+    if (this.unexpectedTerminationCount < MAX_UNEXPECTED_TERMINATIONS) {
+      void withProgress(
+        async (progress, token) =>
+          this.restartQueryServerInternal(progress, token),
+        {
+          title: "Restarting CodeQL query server due to unexpected termination",
+        },
+      );
+    } else {
+      void showAndLogErrorMessage(
+        "The CodeQL query server has unexpectedly terminated too many times. Please check the logs for errors. You can manually restart the query server using the command 'CodeQL: Restart query server'.",
+      );
+      // Make sure we dispose anyway to reject all pending requests.
+      this.serverProcess?.dispose();
+    }
+    this.unexpectedTerminationCount++;
+  }
+
+  /**
+   * Restarts the query server by disposing of the current server process and then starting a new one.
+   * This does not reset the unexpected termination count.
+   */
+  private async restartQueryServerInternal(
+    progress: ProgressCallback,
+    token: CancellationToken,
   ): Promise<void> {
     this.stopQueryServer();
     await this.startQueryServer();
@@ -196,6 +246,9 @@ export class QueryServerClient extends DisposableObject {
     this.nextCallback = 0;
     this.nextProgress = 0;
     this.progressCallbacks = {};
+    child.on("close", () => {
+      this.restartQueryServerOnFailure();
+    });
   }
 
   get serverProcessPid(): number {