Cleanup Prometheus detection (#4316)

src/main/__test__/context-handler.test.ts

@@ -110,7 +110,7 @@ describe("ContextHandler", () => {
       [0, 1],
       [0, 2],
       [0, 3],
-    ])("should return undefined from %d success(es) after %d failure(s)", async (successes, failures) => {
+    ])("should throw from %d success(es) after %d failure(s)", async (successes, failures) => {
       const reg = PrometheusProviderRegistry.getInstance();
       let count = 0;
 
@@ -124,9 +124,7 @@ describe("ContextHandler", () => {
         reg.registerProvider(new TestProvider(`id_${count++}`, ServiceResult.Success));
       }
 
-      const service = await getHandler().getPrometheusService();
-
-      expect(service).toBeUndefined();
+      expect(() => (getHandler() as any).getPrometheusService()).rejects.toBeDefined();
     });
 
     it.each([
@@ -152,7 +150,7 @@ describe("ContextHandler", () => {
         reg.registerProvider(new TestProvider(`id_${count++}`, ServiceResult.Success));
       }
 
-      const service = await getHandler().getPrometheusService();
+      const service = await (getHandler() as any).getPrometheusService();
 
       expect(service.id === `id_${failures}`);
     });
@@ -180,7 +178,7 @@ describe("ContextHandler", () => {
         reg.registerProvider(new TestProvider(`id_${count++}`, serviceResult));
       }
 
-      const service = await getHandler().getPrometheusService();
+      const service = await (getHandler() as any).getPrometheusService();
 
       expect(service.id === "id_0");
     });
@@ -214,7 +212,7 @@ describe("ContextHandler", () => {
         reg.registerProvider(new TestProvider(`id_${count++}`, ServiceResult.Success));
       }
 
-      const service = await getHandler().getPrometheusService();
+      const service = await (getHandler() as any).getPrometheusService();
 
       expect(service.id === "id_0");
     });
@@ -227,7 +225,7 @@ describe("ContextHandler", () => {
       reg.registerProvider(new TestProvider(`id_${count++}`, ServiceResult.Success));
       reg.registerProvider(new TestProvider(`id_${count++}`, ServiceResult.Success));
 
-      const service = await getHandler().getPrometheusService();
+      const service = await (getHandler() as any).getPrometheusService();
 
       expect(service.id).not.toBe("id_2");
     });

src/main/context-handler.ts

@@ -29,6 +29,11 @@ import { CoreV1Api } from "@kubernetes/client-node";
 import logger from "./logger";
 import { KubeAuthProxy } from "./kube-auth-proxy";
 
+export interface PrometheusDetails {
+  prometheusPath: string;
+  provider: PrometheusProvider;
+}
+
 export class ContextHandler {
   public clusterUrl: UrlWithStringQuery;
   protected kubeAuthProxy?: KubeAuthProxy;
@@ -52,23 +57,21 @@ export class ContextHandler {
     }
   }
 
-  protected async resolvePrometheusPath(): Promise<string> {
-    const prometheusService = await this.getPrometheusService();
-
-    if (!prometheusService) return null;
-    const { service, namespace, port } = prometheusService;
-
-    return `${namespace}/services/${service}:${port}`;
-  }
-
-  async getPrometheusProvider() {
-    if (!this.prometheusProvider) {
+  public async getPrometheusDetails(): Promise<PrometheusDetails> {
     const service = await this.getPrometheusService();
+    const prometheusPath = this.ensurePrometheusPath(service);
+    const provider = this.ensurePrometheusProvider(service);
 
-      if (!service) {
-        return null;
+    return { prometheusPath, provider };
   }
-      logger.info(`using ${service.id} as prometheus provider`);
+
+  protected ensurePrometheusPath({ service, namespace, port }: PrometheusService): string {
+    return this.prometheusPath ||= `${namespace}/services/${service}:${port}`;
+  }
+
+  protected ensurePrometheusProvider(service: PrometheusService): PrometheusProvider {
+    if (!this.prometheusProvider) {
+      logger.info(`[CONTEXT-HANDLER]: using ${service.id} as prometheus provider for clusterId=${this.cluster.id}`);
       this.prometheusProvider = service.id;
     }
 
@@ -77,37 +80,40 @@ export class ContextHandler {
 
   protected listPotentialProviders(): PrometheusProvider[] {
     const registry = PrometheusProviderRegistry.getInstance();
+    const provider = this.prometheusProvider && registry.getByKind(this.prometheusProvider);
 
-    if (typeof this.prometheusProvider === "string") {
-      return [registry.getByKind(this.prometheusProvider)];
+    if (provider) {
+      return [provider];
     }
 
     return Array.from(registry.providers.values());
   }
 
-  async getPrometheusService(): Promise<PrometheusService | undefined> {
+  protected async getPrometheusService(): Promise<PrometheusService> {
     const providers = this.listPotentialProviders();
     const proxyConfig = await this.cluster.getProxyKubeconfig();
     const apiClient = proxyConfig.makeApiClient(CoreV1Api);
     const potentialServices = await Promise.allSettled(
       providers.map(provider => provider.getPrometheusService(apiClient)),
     );
+    const errors: any[] = [];
 
-    for (const result of potentialServices) {
-      if (result.status === "fulfilled" && result.value) {
-        return result.value;
+    for (const res of potentialServices) {
+      switch (res.status) {
+        case "rejected":
+          if (res.reason) {
+            errors.push(String(res.reason));
+          }
+          break;
+
+        case "fulfilled":
+          if (res.value) {
+            return res.value;
+          }
       }
     }
 
-    return undefined;
-  }
-
-  async getPrometheusPath(): Promise<string> {
-    if (!this.prometheusPath) {
-      this.prometheusPath = await this.resolvePrometheusPath();
-    }
-
-    return this.prometheusPath;
+    throw Object.assign(new Error("No Prometheus service found"), { cause: errors });
   }
 
   async resolveAuthProxyUrl() {

src/main/prometheus/helm.ts

@@ -29,7 +29,7 @@ export class PrometheusHelm extends PrometheusLens {
   readonly rateAccuracy: string = "5m";
   readonly isConfigurable: boolean = true;
 
-  public async getPrometheusService(client: CoreV1Api): Promise<PrometheusService | undefined> {
-    return this.getFirstNamespacedServer(client, "app=prometheus,component=server,heritage=Helm");
+  public async getPrometheusService(client: CoreV1Api): Promise<PrometheusService> {
+    return this.getFirstNamespacedService(client, "app=prometheus,component=server,heritage=Helm");
   }
 }

src/main/prometheus/lens.ts

@@ -21,7 +21,6 @@
 
 import { PrometheusProvider, PrometheusService } from "./provider-registry";
 import type { CoreV1Api } from "@kubernetes/client-node";
-import logger from "../logger";
 import { inspect } from "util";
 
 export class PrometheusLens extends PrometheusProvider {
@@ -30,22 +29,8 @@ export class PrometheusLens extends PrometheusProvider {
   readonly rateAccuracy: string = "1m";
   readonly isConfigurable: boolean = false;
 
-  public async getPrometheusService(client: CoreV1Api): Promise<PrometheusService | undefined> {
-    try {
-      const resp = await client.readNamespacedService("prometheus", "lens-metrics");
-      const service = resp.body;
-
-      return {
-        id: this.id,
-        namespace: service.metadata.namespace,
-        service: service.metadata.name,
-        port: service.spec.ports[0].port,
-      };
-    } catch(error) {
-      logger.warn(`PrometheusLens: failed to list services: ${error.response.body.message}`);
-
-      return undefined;
-    }
+  public getPrometheusService(client: CoreV1Api): Promise<PrometheusService> {
+    return this.getNamespacedService(client, "prometheus", "lens-metrics");
   }
 
   public getQuery(opts: Record<string, string>, queryName: string): string {

src/main/prometheus/operator.ts

@@ -29,8 +29,8 @@ export class PrometheusOperator extends PrometheusProvider {
   readonly name: string = "Prometheus Operator";
   readonly isConfigurable: boolean = true;
 
-  public async getPrometheusService(client: CoreV1Api): Promise<PrometheusService | undefined> {
-    return this.getFirstNamespacedServer(client, "operated-prometheus=true", "self-monitor=true");
+  public async getPrometheusService(client: CoreV1Api): Promise<PrometheusService> {
+    return this.getFirstNamespacedService(client, "operated-prometheus=true", "self-monitor=true");
   }
 
   public getQuery(opts: Record<string, string>, queryName: string): string {

src/main/prometheus/provider-registry.ts

@@ -20,8 +20,8 @@
  */
 
 import type { CoreV1Api } from "@kubernetes/client-node";
+import { inspect } from "util";
 import { Singleton } from "../../common/utils";
-import logger from "../logger";
 
 export type PrometheusService = {
   id: string;
@@ -43,7 +43,7 @@ export abstract class PrometheusProvider {
     return `sum(rate(nginx_ingress_controller_bytes_sent_sum{ingress="${ingress}",namespace="${namespace}",status=~"${statuses}"}[${this.rateAccuracy}])) by (ingress, namespace)`;
   }
 
-  protected async getFirstNamespacedServer(client: CoreV1Api, ...selectors: string[]): Promise<PrometheusService | undefined> {
+  protected async getFirstNamespacedService(client: CoreV1Api, ...selectors: string[]): Promise<PrometheusService> {
     try {
       for (const selector of selectors) {
         const { body: { items: [service] }} = await client.listServiceForAllNamespaces(null, null, null, selector);
@@ -58,10 +58,26 @@ export abstract class PrometheusProvider {
         }
       }
     } catch (error) {
-      logger.warn(`${this.name}: failed to list services: ${error.toString()}`);
+      throw new Error(`Failed to list services for Prometheus${this.name} in all namespaces: ${error.response.body.message}`);
     }
 
-    return undefined;
+    throw new Error(`No service found for Prometheus${this.name} from any namespace`);
+  }
+
+  protected async getNamespacedService(client: CoreV1Api, name: string, namespace: string): Promise<PrometheusService> {
+    try {
+      const resp = await client.readNamespacedService(name, namespace);
+      const service = resp.body;
+
+      return {
+        id: this.id,
+        namespace: service.metadata.namespace,
+        service: service.metadata.name,
+        port: service.spec.ports[0].port,
+      };
+    } catch(error) {
+      throw new Error(`Failed to list services for Prometheus${this.name} in namespace=${inspect(namespace, false, undefined, false)}: ${error.response.body.message}`);
+    }
   }
 }
 

src/main/prometheus/stacklight.ts

@@ -21,7 +21,6 @@
 
 import { PrometheusProvider, PrometheusService } from "./provider-registry";
 import type { CoreV1Api } from "@kubernetes/client-node";
-import logger from "../logger";
 import { inspect } from "util";
 
 export class PrometheusStacklight extends PrometheusProvider {
@@ -30,22 +29,8 @@ export class PrometheusStacklight extends PrometheusProvider {
   readonly rateAccuracy: string = "1m";
   readonly isConfigurable: boolean = true;
 
-  public async getPrometheusService(client: CoreV1Api): Promise<PrometheusService | undefined> {
-    try {
-      const resp = await client.readNamespacedService("prometheus-server", "stacklight");
-      const service = resp.body;
-
-      return {
-        id: this.id,
-        namespace: service.metadata.namespace,
-        service: service.metadata.name,
-        port: service.spec.ports[0].port,
-      };
-    } catch(error) {
-      logger.warn(`PrometheusLens: failed to list services: ${error.response.body.message}`);
-
-      return undefined;
-    }
+  public getPrometheusService(client: CoreV1Api): Promise<PrometheusService> {
+    return this.getNamespacedService(client, "prometheus-server", "stacklight");
   }
 
   public getQuery(opts: Record<string, string>, queryName: string): string {

src/main/routes/metrics-route.ts

@@ -73,12 +73,9 @@ export class MetricsRoute {
     const prometheusMetadata: ClusterPrometheusMetadata = {};
 
     try {
-      const [prometheusPath, prometheusProvider] = await Promise.all([
-        cluster.contextHandler.getPrometheusPath(),
-        cluster.contextHandler.getPrometheusProvider(),
-      ]);
+      const { prometheusPath, provider } = await cluster.contextHandler.getPrometheusDetails();
 
-      prometheusMetadata.provider = prometheusProvider?.id;
+      prometheusMetadata.provider = provider?.id;
       prometheusMetadata.autoDetected = !cluster.preferences.prometheusProvider?.type;
 
       if (!prometheusPath) {
@@ -99,7 +96,7 @@ export class MetricsRoute {
       } else {
         const queries = Object.entries<Record<string, string>>(payload)
           .map(([queryName, queryOpts]) => (
-            prometheusProvider.getQuery(queryOpts, queryName)
+            provider.getQuery(queryOpts, queryName)
           ));
         const result = await loadMetrics(queries, cluster, prometheusPath, queryParams);
         const data = Object.fromEntries(Object.keys(payload).map((metricName, i) => [metricName, result[i]]));
@@ -107,9 +104,10 @@ export class MetricsRoute {
         respondJson(response, data);
       }
       prometheusMetadata.success = true;
-    } catch {
+    } catch (error) {
       prometheusMetadata.success = false;
       respondJson(response, {});
+      logger.warn(`[METRICS-ROUTE]: failed to get metrics for clusterId=${cluster.id}:`, error);
     } finally {
       cluster.metadata[ClusterMetadataKey.PROMETHEUS] = prometheusMetadata;
     }