Keep shell processes alive between network offline periods (#4258)

src/main/index.ts

@@ -65,6 +65,7 @@ import { initMenu } from "./menu";
 import { initTray } from "./tray";
 import { kubeApiRequest, shellApiRequest, ShellRequestAuthenticator } from "./proxy-functions";
 import { AppPaths } from "../common/app-paths";
+import { ShellSession } from "./shell-session/shell-session";
 
 injectSystemCAs();
 
@@ -228,6 +229,7 @@ app.on("ready", async () => {
   onQuitCleanup.push(
     initMenu(windowManager),
     initTray(windowManager),
+    () => ShellSession.cleanup(),
   );
 
   installDeveloperTools();

src/main/proxy-functions/shell-api-request.ts

@@ -87,8 +87,8 @@ export function shellApiRequest({ req, socket, head }: ProxyApiRequestArgs): voi
 
   ws.handleUpgrade(req, socket, head, (webSocket) => {
     const shell = node
-      ? new NodeShellSession(webSocket, cluster, node)
-      : new LocalShellSession(webSocket, cluster);
+      ? new NodeShellSession(webSocket, cluster, node, tabId)
+      : new LocalShellSession(webSocket, cluster, tabId);
 
     shell.open()
       .catch(error => logger.error(`[SHELL-SESSION]: failed to open a ${node ? "node" : "local"} shell`, error));

src/main/shell-session/local-shell-session.ts

@@ -40,7 +40,7 @@ export class LocalShellSession extends ShellSession {
     const shell = env.PTYSHELL;
     const args = await this.getShellArgs(shell);
 
-    super.open(env.PTYSHELL, args, env);
+    await this.openShellProcess(env.PTYSHELL, args, env);
   }
 
   protected async getShellArgs(shell: string): Promise<string[]> {

src/main/shell-session/node-shell-session.ts

@@ -40,8 +40,8 @@ export class NodeShellSession extends ShellSession {
     return undefined;
   }
 
-  constructor(socket: WebSocket, cluster: Cluster, protected nodeName: string) {
-    super(socket, cluster);
+  constructor(socket: WebSocket, cluster: Cluster, protected nodeName: string, terminalId: string) {
+    super(socket, cluster, terminalId);
   }
 
   public async open() {
@@ -78,7 +78,7 @@ export class NodeShellSession extends ShellSession {
         break;
     }
 
-    return super.open(shell, args, env);
+    await this.openShellProcess(shell, args, env);
   }
 
   protected createNodeShellPod() {

src/main/shell-session/shell-session.ts

@@ -31,6 +31,7 @@ import { isWindows } from "../../common/vars";
 import { UserStore } from "../../common/user-store";
 import * as pty from "node-pty";
 import { appEventBus } from "../../common/event-bus";
+import logger from "../logger";
 
 export class ShellOpenError extends Error {
   constructor(message: string, public cause: Error) {
@@ -40,41 +41,140 @@ export class ShellOpenError extends Error {
   }
 }
 
+export enum WebSocketCloseEvent {
+  /**
+   * The connection successfully completed the purpose for which it was created.
+   */
+  NormalClosure = 1000,
+  /**
+   * The endpoint is going away, either because of a server failure or because
+   * the browser is navigating away from the page that opened the connection.
+   */
+  GoingAway = 1001,
+  /**
+   * The endpoint is terminating the connection due to a protocol error.
+   */
+  ProtocolError = 1002,
+  /**
+   * The connection is being terminated because the endpoint received data of a
+   * type it cannot accept. (For example, a text-only endpoint received binary
+   * data.)
+   */
+  UnsupportedData = 1003,
+  /**
+   * Indicates that no status code was provided even though one was expected.
+   */
+  NoStatusReceived = 1005,
+  /**
+   * Indicates that a connection was closed abnormally (that is, with no close
+   * frame being sent) when a status code is expected.
+   */
+  AbnormalClosure = 1006,
+  /**
+   *  The endpoint is terminating the connection because a message was received
+   * that contained inconsistent data (e.g., non-UTF-8 data within a text message).
+   */
+  InvalidFramePayloadData = 1007,
+  /**
+   * The endpoint is terminating the connection because it received a message
+   * that violates its policy. This is a generic status code, used when codes
+   * 1003 and 1009 are not suitable.
+   */
+  PolicyViolation = 1008,
+  /**
+   * The endpoint is terminating the connection because a data frame was
+   * received that is too large.
+   */
+  MessageTooBig = 1009,
+  /**
+   * The client is terminating the connection because it expected the server to
+   * negotiate one or more extension, but the server didn't.
+   */
+  MissingExtension = 1010,
+  /**
+   * The server is terminating the connection because it encountered an
+   * unexpected condition that prevented it from fulfilling the request.
+   */
+  InternalError = 1011,
+  /**
+   * The server is terminating the connection because it is restarting.
+   */
+  ServiceRestart = 1012,
+  /**
+   * The server is terminating the connection due to a temporary condition,
+   * e.g. it is overloaded and is casting off some of its clients.
+   */
+  TryAgainLater = 1013,
+  /**
+   * The server was acting as a gateway or proxy and received an invalid
+   * response from the upstream server. This is similar to 502 HTTP Status Code.
+   */
+  BadGateway = 1014,
+  /**
+   * Indicates that the connection was closed due to a failure to perform a TLS
+   * handshake (e.g., the server certificate can't be verified).
+   */
+  TlsHandshake = 1015,
+}
+
 export abstract class ShellSession {
   abstract ShellType: string;
 
-  static shellEnvs: Map<string, Record<string, any>> = new Map();
+  private static shellEnvs = new Map<string, Record<string, string>>();
+  private static processes = new Map<string, pty.IPty>();
+
+  /**
+   * Kill all remaining shell backing processes. Should be called when about to
+   * quit
+   */
+  public static cleanup(): void {
+    for (const shellProcess of this.processes.values()) {
+      try {
+        process.kill(shellProcess.pid);
+      } catch {}
+    }
+
+    this.processes.clear();
+  }
 
   protected kubectl: Kubectl;
   protected running = false;
-  protected shellProcess: pty.IPty;
   protected kubectlBinDirP: Promise<string>;
   protected kubeconfigPathP: Promise<string>;
+  protected readonly terminalId: string;
 
   protected abstract get cwd(): string | undefined;
 
-  constructor(protected websocket: WebSocket, protected cluster: Cluster) {
+  protected ensureShellProcess(shell: string, args: string[], env: Record<string, string>, cwd: string): pty.IPty {
+    if (!ShellSession.processes.has(this.terminalId)) {
+      ShellSession.processes.set(this.terminalId, pty.spawn(shell, args, {
+        cols: 80,
+        cwd,
+        env,
+        name: "xterm-256color",
+        rows: 30,
+      }));
+    }
+
+    return ShellSession.processes.get(this.terminalId);
+  }
+
+  constructor(protected websocket: WebSocket, protected cluster: Cluster, terminalId: string) {
     this.kubectl = new Kubectl(cluster.version);
     this.kubeconfigPathP = this.cluster.getProxyKubeconfigPath();
     this.kubectlBinDirP = this.kubectl.binDir();
+    this.terminalId = `${cluster.id}:${terminalId}`;
   }
 
-  protected async open(shell: string, args: string[], env: Record<string, any>) {
+  protected async openShellProcess(shell: string, args: string[], env: Record<string, any>) {
     const cwd = (this.cwd && await fse.pathExists(this.cwd))
     	? this.cwd
     	: env.HOME;
+    const shellProcess = this.ensureShellProcess(shell, args, env, cwd);
 
-    this.shellProcess = pty.spawn(shell, args, {
-      cols: 80,
-      cwd,
-      env,
-      name: "xterm-256color",
-      rows: 30,
-    });
     this.running = true;
-
-    this.shellProcess.onData(data => this.sendResponse(data));
-    this.shellProcess.onExit(({ exitCode }) => {
+    shellProcess.onData(data => this.sendResponse(data));
+    shellProcess.onExit(({ exitCode }) => {
       this.running = false;
 
       if (exitCode > 0) {
@@ -95,24 +195,28 @@ export abstract class ShellSession {
 
         switch (data[0]) {
           case "0":
-            this.shellProcess.write(message);
+            shellProcess.write(message);
             break;
           case "4":
             const { Width, Height } = JSON.parse(message);
 
-            this.shellProcess.resize(Width, Height);
+            shellProcess.resize(Width, Height);
             break;
         }
       })
-      .on("close", () => {
-        if (this.running) {
+      .on("close", (code) => {
+        logger.debug(`[SHELL-SESSION]: websocket for ${this.terminalId} closed with code=${code}`);
+
+        if (this.running && code !== WebSocketCloseEvent.AbnormalClosure) {
+          // This code is the one that gets sent when the network is turned off
           try {
-            process.kill(this.shellProcess.pid);
+            logger.info(`[SHELL-SESSION]: Killing shell process for ${this.terminalId}`);
+            process.kill(shellProcess.pid);
+            ShellSession.processes.delete(this.terminalId);
           } catch (e) {
           }
+          this.running = false;
         }
-
-        this.running = false;
       });
 
     appEventBus.emit({ name: this.ShellType, action: "open" });
@@ -191,7 +295,7 @@ export abstract class ShellSession {
     return env;
   }
 
-  protected exit(code = 1000) {
+  protected exit(code = WebSocketCloseEvent.NormalClosure) {
     if (this.websocket.readyState == this.websocket.OPEN) {
       this.websocket.close(code);
     }

src/renderer/api/terminal-api.ts

@@ -73,7 +73,13 @@ export class TerminalApi extends WebSocketApi {
   }
 
   async connect() {
-    this.emitStatus("Connecting ...");
+    if (!this.socket) {
+      /**
+       * Only emit this message if we are not "reconnecting", so as to keep the
+       * output display clean when the computer wakes from sleep
+       */
+      this.emitStatus("Connecting ...");
+    }
 
     const shellToken = await ipcRenderer.invoke("cluster:shell-api", getHostedClusterId(), this.query.id);
     const { hostname, protocol, port } = location;