Add setting for filtering empty containers on Prometheus queries

Signed-off-by: Juho Heikka <juho.heikka@gmail.com>

packages/core/src/common/cluster-types.ts

@@ -112,6 +112,10 @@ export interface ClusterPreferences extends ClusterPrometheusPreferences {
   defaultNamespace?: string;
 }
 
+interface QueryFilterOptions {
+  hideEmptyContainers: boolean;
+}
+
 /**
  * A cluster's prometheus settings (a subset of cluster settings)
  */
@@ -125,6 +129,7 @@ export interface ClusterPrometheusPreferences {
   prometheusProvider?: {
     type: string;
   };
+  prometheusQueryOptions?: QueryFilterOptions;
 }
 
 /**

packages/core/src/main/prometheus/operator-provider.injectable.ts.ts

@@ -9,6 +9,9 @@ import { getInjectable } from "@ogre-tools/injectable";
 
 export const getOperatorLikeQueryFor = ({ rateAccuracy }: { rateAccuracy: string }): PrometheusProvider["getQuery"] => (
   (opts, queryName) => {
+    const emptyContainerAndImageFilter = opts.hideEmptyContainers === "true" ? `container!="", image!="",` : "";
+    const emptyContainerFilter = opts.hideEmptyContainers === "true" ? `container!="",` : "";
+
     switch(opts.category) {
       case "cluster":
         switch (queryName) {
@@ -71,19 +74,19 @@ export const getOperatorLikeQueryFor = ({ rateAccuracy }: { rateAccuracy: string
       case "pods":
         switch (queryName) {
           case "cpuUsage":
-            return `sum(rate(container_cpu_usage_seconds_total{pod=~"${opts.pods}", namespace="${opts.namespace}"}[${rateAccuracy}])) by (${opts.selector})`;
+            return `sum(rate(container_cpu_usage_seconds_total{${emptyContainerAndImageFilter} pod=~"${opts.pods}", namespace="${opts.namespace}"}[${rateAccuracy}])) by (${opts.selector})`;
           case "cpuRequests":
             return `sum(kube_pod_container_resource_requests{pod=~"${opts.pods}", resource="cpu", namespace="${opts.namespace}"}) by (${opts.selector})`;
           case "cpuLimits":
-            return `sum(kube_pod_container_resource_limits{pod=~"${opts.pods}", resource="cpu", namespace="${opts.namespace}"}) by (${opts.selector})`;
+            return `sum(kube_pod_container_resource_limits{${emptyContainerAndImageFilter} pod=~"${opts.pods}", resource="cpu", namespace="${opts.namespace}"}) by (${opts.selector})`;
           case "memoryUsage":
             return `sum(container_memory_working_set_bytes{pod=~"${opts.pods}", namespace="${opts.namespace}"}) by (${opts.selector})`;
           case "memoryRequests":
-            return `sum(kube_pod_container_resource_requests{pod=~"${opts.pods}", resource="memory", namespace="${opts.namespace}"}) by (${opts.selector})`;
+            return `sum(kube_pod_container_resource_requests{${emptyContainerFilter} pod=~"${opts.pods}", resource="memory", namespace="${opts.namespace}"}) by (${opts.selector})`;
           case "memoryLimits":
-            return `sum(kube_pod_container_resource_limits{pod=~"${opts.pods}", resource="memory", namespace="${opts.namespace}"}) by (${opts.selector})`;
+            return `sum(kube_pod_container_resource_limits{${emptyContainerFilter} pod=~"${opts.pods}", resource="memory", namespace="${opts.namespace}"}) by (${opts.selector})`;
           case "fsUsage":
-            return `sum(container_fs_usage_bytes{pod=~"${opts.pods}", namespace="${opts.namespace}"}) by (${opts.selector})`;
+            return `sum(container_fs_usage_bytes{${emptyContainerFilter} pod=~"${opts.pods}", namespace="${opts.namespace}"}) by (${opts.selector})`;
           case "fsWrites":
             return `sum(rate(container_fs_writes_bytes_total{pod=~"${opts.pods}", namespace="${opts.namespace}"}[${rateAccuracy}])) by (${opts.selector})`;
           case "fsReads":

packages/core/src/main/routes/metrics/add-metrics-route.injectable.ts

@@ -97,9 +97,13 @@ const addMetricsRouteInjectable = getRouteInjectable({
 
         if (isObject(payload)) {
           const data = payload as Record<string, Record<string, string>>;
+          const queryFilterPreferences: Record<string, string> = cluster.preferences.prometheusQueryOptions ?
+            Object.fromEntries(Object.entries(cluster.preferences.prometheusQueryOptions).map(([k, v]) => [k, String(v)]))
+            : {};
+
           const queries = object.entries(data)
             .map(([queryName, queryOpts]) => (
-              provider.getQuery(queryOpts, queryName)
+              provider.getQuery({ ...queryOpts, ...queryFilterPreferences }, queryName)
             ));
 
           const result = await loadMetrics(queries, cluster, prometheusPath, queryParams);

packages/core/src/renderer/components/cluster-settings/prometheus-setting.tsx

@@ -16,6 +16,9 @@ import type { MetricProviderInfo, RequestMetricsProviders } from "../../../commo
 import { withInjectables } from "@ogre-tools/injectable-react";
 import requestMetricsProvidersInjectable from "../../../common/k8s-api/endpoints/metrics.api/request-providers.injectable";
 import productNameInjectable from "../../../common/vars/product-name.injectable";
+import { Checkbox } from "../checkbox";
+import Gutter from "../gutter/gutter";
+
 
 export interface ClusterPrometheusSettingProps {
   cluster: Cluster;
@@ -123,6 +126,9 @@ class NonInjectedClusterPrometheusSetting extends React.Component<ClusterPrometh
   };
 
   render() {
+    const providerType = this.props.cluster.preferences.prometheusProvider?.type;
+    const showQueryFilters = providerType === "operator" || !providerType;
+
     return (
       <>
         <section>
@@ -147,6 +153,28 @@ class NonInjectedClusterPrometheusSetting extends React.Component<ClusterPrometh
               )
           }
         </section>
+        {
+          showQueryFilters && (
+            <>
+              <Gutter />
+              <section>
+                <SubTitle title="Prometheus queries" />
+                <Checkbox
+                  label="Filter empty containers on pod metrics"
+                  value={this.props.cluster.preferences.prometheusQueryOptions?.hideEmptyContainers ?? false}
+                  onChange={v => {
+                    this.props.cluster.preferences.prometheusQueryOptions = {
+                      hideEmptyContainers: v,
+                    };
+                  }}
+                />
+                <small className="hint">
+                  In certain metric setups, pod metrics may be observed as double values. This filter can be helpful in ensuring accurate metric are shown.
+                </small>
+              </section>
+            </>
+          )
+        }
         {this.canEditPrometheusPath && (
           <>
             <hr />