Add more resiliancy to listing kube API resource kinds (#6995)

Signed-off-by: Sebastian Malton <sebastian@malton.name>

Signed-off-by: Sebastian Malton <sebastian@malton.name>

packages/core/src/main/__test__/kube-auth-proxy.test.ts

@@ -200,34 +200,34 @@ describe("kube auth proxy tests", () => {
       await proxy.run();
       listeners.emit("error", { message: "foobarbat" });
 
-      expect(broadcastMessageMock).toBeCalledWith("cluster:foobar:connection-update", { message: "foobarbat", isError: true });
+      expect(broadcastMessageMock).toBeCalledWith("cluster:foobar:connection-update", { message: "foobarbat", level: "error" });
     });
 
     it("should call spawn and broadcast exit", async () => {
       await proxy.run();
       listeners.emit("exit", 0);
 
-      expect(broadcastMessageMock).toBeCalledWith("cluster:foobar:connection-update", { message: "proxy exited with code: 0", isError: false });
+      expect(broadcastMessageMock).toBeCalledWith("cluster:foobar:connection-update", { message: "proxy exited with code: 0", level: "info" });
     });
 
     it("should call spawn and broadcast errors from stderr", async () => {
       await proxy.run();
       listeners.emit("stderr/data", "an error");
 
-      expect(broadcastMessageMock).toBeCalledWith("cluster:foobar:connection-update", { message: "an error", isError: true });
+      expect(broadcastMessageMock).toBeCalledWith("cluster:foobar:connection-update", { message: "an error", level: "error" });
     });
 
     it("should call spawn and broadcast stdout serving info", async () => {
       await proxy.run();
 
-      expect(broadcastMessageMock).toBeCalledWith("cluster:foobar:connection-update", { message: "Authentication proxy started", isError: false });
+      expect(broadcastMessageMock).toBeCalledWith("cluster:foobar:connection-update", { message: "Authentication proxy started", level: "info" });
     });
 
     it("should call spawn and broadcast stdout other info", async () => {
       await proxy.run();
       listeners.emit("stdout/data", "some info");
 
-      expect(broadcastMessageMock).toBeCalledWith("cluster:foobar:connection-update", { message: "some info", isError: false });
+      expect(broadcastMessageMock).toBeCalledWith("cluster:foobar:connection-update", { message: "some info", level: "info" });
     });
   });
 });

packages/core/src/main/cluster/request-api-resources.injectable.ts

@@ -10,8 +10,10 @@ import type { Cluster } from "../../common/cluster/cluster";
 import { requestApiVersionsInjectionToken } from "./request-api-versions";
 import { withConcurrencyLimit } from "../../common/utils/with-concurrency-limit";
 import requestKubeApiResourcesForInjectable from "./request-kube-api-resources-for.injectable";
+import type { AsyncResult } from "../../common/utils/async-result";
+import { backoffCaller } from "../../common/utils/backoff-caller";
 
-export type RequestApiResources = (cluster: Cluster) => Promise<KubeApiResource[]>;
+export type RequestApiResources = (cluster: Cluster) => Promise<AsyncResult<KubeApiResource[], Error>>;
 
 export interface KubeResourceListGroup {
   group: string;
@@ -25,23 +27,46 @@ const requestApiResourcesInjectable = getInjectable({
     const apiVersionRequesters = di.injectMany(requestApiVersionsInjectionToken);
     const requestKubeApiResourcesFor = di.inject(requestKubeApiResourcesForInjectable);
 
-    return async (cluster) => {
+    return async (...args) => {
+      const [cluster] = args;
       const requestKubeApiResources = withConcurrencyLimit(5)(requestKubeApiResourcesFor(cluster));
 
-      try {
-        const requests = await Promise.all(apiVersionRequesters.map(fn => fn(cluster)));
-        const resources = await Promise.all((
-          requests
-            .flat()
-            .map(requestKubeApiResources)
-        ));
+      const groupLists: KubeResourceListGroup[] = [];
 
-        return resources.flat();
-      } catch (error) {
-        logger.error(`[LIST-API-RESOURCES]: failed to list api resources: ${error}`);
+      for (const apiVersionRequester of apiVersionRequesters) {
+        const result = await backoffCaller(() => apiVersionRequester(cluster), {
+          onIntermediateError: (error, attempt) => {
+            cluster.broadcastConnectUpdate(`Failed to list kube API resource kinds, attempt ${attempt}: ${error}`, "warning");
+            logger.warn(`[LIST-API-RESOURCES]: failed to list kube api resources: ${error}`, { attempt, clusterId: cluster.id });
+          },
+        });
 
-        return [];
+        if (!result.callWasSuccessful) {
+          return result;
+        }
+
+        groupLists.push(...result.response);
       }
+
+      const apiResourceRequests = groupLists.map(async listGroup => (
+        Object.assign(await requestKubeApiResources(listGroup), { listGroup })
+      ));
+      const results = await Promise.all(apiResourceRequests);
+      const resources: KubeApiResource[] = [];
+
+      for (const result of results) {
+        if (!result.callWasSuccessful) {
+          cluster.broadcastConnectUpdate(`Kube APIs under "${result.listGroup.path}" may not be displayed`, "warning");
+          continue;
+        }
+
+        resources.push(...result.response);
+      }
+
+      return {
+        callWasSuccessful: true,
+        response: resources,
+      };
     };
   },
 });

packages/core/src/main/cluster/request-api-versions.ts

@@ -5,13 +5,14 @@
 
 import { getInjectionToken } from "@ogre-tools/injectable";
 import type { Cluster } from "../../common/cluster/cluster";
+import type { AsyncResult } from "../../common/utils/async-result";
 
 export interface KubeResourceListGroup {
   group: string;
   path: string;
 }
 
-export type RequestApiVersions = (cluster: Cluster) => Promise<KubeResourceListGroup[]>;
+export type RequestApiVersions = (cluster: Cluster) => Promise<AsyncResult<KubeResourceListGroup[], Error>>;
 
 export const requestApiVersionsInjectionToken = getInjectionToken<RequestApiVersions>({
   id: "request-api-versions-token",

packages/core/src/main/cluster/request-core-api-versions.injectable.ts

@@ -13,12 +13,22 @@ const requestCoreApiVersionsInjectable = getInjectable({
     const k8sRequest = di.inject(k8sRequestInjectable);
 
     return async (cluster) => {
-      const { versions } = await k8sRequest(cluster, "/api") as V1APIVersions;
+      try {
+        const { versions } = await k8sRequest(cluster, "/api") as V1APIVersions;
 
-      return versions.map(version => ({
-        group: "",
-        path: `/api/${version}`,
-      }));
+        return {
+          callWasSuccessful: true,
+          response: versions.map(version => ({
+            group: "",
+            path: `/api/${version}`,
+          })),
+        };
+      } catch (error) {
+        return {
+          callWasSuccessful: false,
+          error: error as Error,
+        };
+      }
     };
   },
   injectionToken: requestApiVersionsInjectionToken,

packages/core/src/main/cluster/request-kube-api-resources-for.injectable.ts

@@ -6,10 +6,11 @@ import type { V1APIResourceList } from "@kubernetes/client-node";
 import { getInjectable } from "@ogre-tools/injectable";
 import type { Cluster } from "../../common/cluster/cluster";
 import type { KubeApiResource } from "../../common/rbac";
+import type { AsyncResult } from "../../common/utils/async-result";
 import k8sRequestInjectable from "../k8s-request.injectable";
 import type { KubeResourceListGroup } from "./request-api-versions";
 
-export type RequestKubeApiResources = (grouping: KubeResourceListGroup) => Promise<KubeApiResource[]>;
+export type RequestKubeApiResources = (grouping: KubeResourceListGroup) => Promise<AsyncResult<KubeApiResource[], Error>>;
 
 export type RequestKubeApiResourcesFor = (cluster: Cluster) => RequestKubeApiResources;
 
@@ -19,14 +20,24 @@ const requestKubeApiResourcesForInjectable = getInjectable({
     const k8sRequest = di.inject(k8sRequestInjectable);
 
     return (cluster) => async ({ group, path }) => {
-      const { resources } = await k8sRequest(cluster, path) as V1APIResourceList;
+      try {
+        const { resources } = await k8sRequest(cluster, path) as V1APIResourceList;
 
-      return resources.map(resource => ({
-        apiName: resource.name,
-        kind: resource.kind,
-        group,
-        namespaced: resource.namespaced,
-      }));
+        return {
+          callWasSuccessful: true,
+          response: resources.map(resource => ({
+            apiName: resource.name,
+            kind: resource.kind,
+            group,
+            namespaced: resource.namespaced,
+          })),
+        };
+      } catch (error) {
+        return {
+          callWasSuccessful: false,
+          error: error as Error,
+        };
+      }
     };
   },
 });

packages/core/src/main/cluster/request-non-core-api-versions.injectable.ts

@@ -14,14 +14,24 @@ const requestNonCoreApiVersionsInjectable = getInjectable({
     const k8sRequest = di.inject(k8sRequestInjectable);
 
     return async (cluster) => {
-      const { groups } = await k8sRequest(cluster, "/apis") as V1APIGroupList;
+      try {
+        const { groups } = await k8sRequest(cluster, "/apis") as V1APIGroupList;
 
-      return chain(groups.values())
-        .filterMap(group => group.preferredVersion?.groupVersion && ({
-          group: group.name,
-          path: `/apis/${group.preferredVersion.groupVersion}`,
-        }))
-        .collect(v => [...v]);
+        return {
+          callWasSuccessful: true,
+          response: chain(groups.values())
+            .filterMap(group => group.preferredVersion?.groupVersion && ({
+              group: group.name,
+              path: `/apis/${group.preferredVersion.groupVersion}`,
+            }))
+            .collect(v => [...v]),
+        };
+      } catch (error) {
+        return {
+          callWasSuccessful: false,
+          error: error as Error,
+        };
+      }
     };
   },
   injectionToken: requestApiVersionsInjectionToken,

packages/core/src/main/kube-auth-proxy/kube-auth-proxy.ts

@@ -71,17 +71,17 @@ export class KubeAuthProxy {
       },
     });
     this.proxyProcess.on("error", (error) => {
-      this.cluster.broadcastConnectUpdate(error.message, true);
+      this.cluster.broadcastConnectUpdate(error.message, "error");
       this.exit();
     });
 
     this.proxyProcess.on("exit", (code) => {
-      this.cluster.broadcastConnectUpdate(`proxy exited with code: ${code}`, code ? code > 0: false);
+      this.cluster.broadcastConnectUpdate(`proxy exited with code: ${code}`, code ? "error" : "info");
       this.exit();
     });
 
     this.proxyProcess.on("disconnect", () => {
-      this.cluster.broadcastConnectUpdate("Proxy disconnected communications", true );
+      this.cluster.broadcastConnectUpdate("Proxy disconnected communications", "error");
       this.exit();
     });
 
@@ -93,7 +93,7 @@ export class KubeAuthProxy {
         return;
       }
 
-      this.cluster.broadcastConnectUpdate(data.toString(), true);
+      this.cluster.broadcastConnectUpdate(data.toString(), "error");
     });
 
     this.proxyProcess.stdout.on("data", (data: Buffer) => {
@@ -114,7 +114,7 @@ export class KubeAuthProxy {
       this.ready = true;
     } catch (error) {
       this.dependencies.logger.warn("[KUBE-AUTH-PROXY]: waitUntilUsed failed", error);
-      this.cluster.broadcastConnectUpdate("Proxy port failed to be used within timelimit, restarting...", true);
+      this.cluster.broadcastConnectUpdate("Proxy port failed to be used within timelimit, restarting...", "error");
       this.exit();
 
       return this.run();