using ?limit=1 and metadata.remainingItemCount for counting all items / refactoring

Signed-off-by: Roman <ixrock@gmail.com>

src/renderer/api/kube-api.ts

@@ -1,4 +1,5 @@
 // Base class for building all kubernetes apis
+// Docs: https://kubernetes.io/docs/reference/using-api/api-concepts
 
 import merge from "lodash/merge";
 import { stringify } from "querystring";
@@ -7,7 +8,7 @@ import logger from "../../main/logger";
 import { apiManager } from "./api-manager";
 import { apiKube } from "./index";
 import { createKubeApiURL, parseKubeApi } from "./kube-api-parse";
-import { KubeJsonApi, KubeJsonApiData, KubeJsonApiDataList } from "./kube-json-api";
+import { KubeJsonApi, KubeJsonApiData, KubeJsonApiDataList, KubeJsonApiListMetadata, KubeJsonApiListMetadataParsed, KubeJsonApiResponse } from "./kube-json-api";
 import { IKubeObjectConstructor, KubeObject } from "./kube-object";
 import { kubeWatchApi } from "./kube-watch-api";
 
@@ -233,8 +234,43 @@ export class KubeApi<T extends KubeObject = any> {
     return this.resourceVersions.get(namespace);
   }
 
-  async refreshResourceVersion(params?: { namespace: string }) {
+  async listMetadata({ namespace = "" } = {}): Promise<KubeJsonApiListMetadataParsed> {
-    return this.list(params, { limit: 1 });
+    const response = await this.rawList({ namespace }, {
+      limit: 1, // specifying a limit to get metadata.remainingItemCount in response
+      resourceVersion: this.getResourceVersion(namespace) ?? "1",
+      resourceVersionMatch: "NotOlderThan",
+    });
+
+    const { remainingItemCount, ...metadata } = this.parseMetadata(namespace, response.metadata);
+
+    return {
+      ...metadata,
+      itemsCount: remainingItemCount + 1, // +1 from limit=1
+    };
+  }
+
+  async getItemsCount(): Promise<number> {
+    try {
+      const { itemsCount } = await this.listMetadata(); // list request for all namespaces
+
+      return itemsCount;
+    } catch (error) {
+      logger.error(`[KUBE-API]: getItemsTotal() has failed: ${error}`);
+    }
+
+    return 0;
+  }
+
+  async refreshResourceVersion(params?: { namespace: string }): Promise<string> {
+    try {
+      const { resourceVersion } = await this.listMetadata(params);
+
+      return resourceVersion;
+    } catch (error) {
+      logger.error(`[KUBE-API]: refreshing resourceVersion has failed: ${error}`, { params });
+    }
+
+    return "";
   }
 
   getUrl({ name = "", namespace = "" } = {}, query?: Partial<IKubeApiQueryParams>) {
@@ -261,7 +297,7 @@ export class KubeApi<T extends KubeObject = any> {
     return query;
   }
 
-  protected parseResponse(data: KubeJsonApiData | KubeJsonApiData[] | KubeJsonApiDataList, namespace?: string): any {
+  protected parseResponse(data: KubeJsonApiResponse, namespace?: string): any {
     const KubeObjectConstructor = this.objectConstructor;
 
     if (KubeObject.isJsonApiData(data)) {
@@ -276,9 +312,8 @@ export class KubeApi<T extends KubeObject = any> {
     if (KubeObject.isJsonApiDataList(data)) {
       const { apiVersion, items, metadata } = data;
 
-      // save "resourceVersion" metadata from list requests
+      // parse & process metadata
-      this.setResourceVersion(namespace, metadata.resourceVersion);
+      this.parseMetadata(namespace, metadata);
-      this.setResourceVersion("", metadata.resourceVersion);
 
       return items.map((item) => {
         const object = new KubeObjectConstructor({
@@ -301,11 +336,32 @@ export class KubeApi<T extends KubeObject = any> {
     return data;
   }
 
-  async list({ namespace = "" } = {}, query?: IKubeApiQueryParams): Promise<T[]> {
+  protected parseMetadata(namespace: string, metadata: KubeJsonApiListMetadata): KubeJsonApiListMetadata & { namespace: string } {
+    const {
+      resourceVersion = "1", // optimization for "?limit=1&resourceVersionMatch=[non-empty]&resourceVersionMatch=NotOlderThan"
+      remainingItemCount = 0, // this is undefined for requests without ?limit
+      ...unprocessedMeta
+    } = metadata;
+
+    // save "resourceVersion" for requests optimization
+    this.setResourceVersion(namespace, resourceVersion);
+
+    return {
+      namespace,
+      resourceVersion,
+      remainingItemCount,
+      ...unprocessedMeta,
+    };
+  }
+
+  async rawList({ namespace = "" } = {}, query?: IKubeApiQueryParams): Promise<KubeJsonApiDataList> {
     await this.checkPreferredVersion();
 
-    return this.request
+    return this.request.get(this.getUrl({ namespace }), { query });
-      .get(this.getUrl({ namespace }), { query })
+  }
+
+  async list({ namespace = "" } = {}, query?: IKubeApiQueryParams): Promise<T[]> {
+    return this.rawList({ namespace }, query)
       .then(data => this.parseResponse(data, namespace));
   }
 

src/renderer/api/kube-json-api.ts

@@ -1,15 +1,25 @@
 import { JsonApi, JsonApiData, JsonApiError } from "./json-api";
 
+export type KubeJsonApiResponse = KubeJsonApiData | KubeJsonApiData[] | KubeJsonApiDataList;
+
 export interface KubeJsonApiDataList<T = KubeJsonApiData> {
   kind: string;
   apiVersion: string;
   items: T[];
-  metadata: {
+  metadata: KubeJsonApiListMetadata;
-    resourceVersion: string;
-    selfLink: string;
-  };
 }
 
+export interface KubeJsonApiListMetadata {
+  resourceVersion: string;
+  selfLink: string;
+  continue?: string; // hash-token, ?limit=N is required to use for request
+  remainingItemCount?: number; // remained items count from list request with ?limit=
+}
+
+export type KubeJsonApiListMetadataParsed = Omit<KubeJsonApiListMetadata, "remainingItemCount"> & {
+  itemsCount?: number;
+};
+
 export interface KubeJsonApiData extends JsonApiData {
   kind: string;
   apiVersion: string;

src/renderer/components/+user-management-roles-bindings/role-bindings.store.ts

@@ -7,7 +7,7 @@ import { apiManager } from "../../api/api-manager";
 
 @autobind()
 export class RoleBindingsStore extends KubeObjectStore<RoleBinding> {
-  api = roleBindingApi;
+  api = clusterRoleBindingApi;
 
   getSubscribeApis() {
     return [clusterRoleBindingApi, roleBindingApi];
@@ -29,7 +29,7 @@ export class RoleBindingsStore extends KubeObjectStore<RoleBinding> {
   protected async loadItems(params: KubeStoreLoadItemsOptions): Promise<RoleBinding[]> {
     const items = await Promise.all([
       super.loadItems({ ...params, api: clusterRoleBindingApi }),
-      super.loadItems({ ...params, api: roleBindingApi }),
+      super.loadItems({ ...params, api: roleBindingApi, refreshMeta: true }),
     ]);
 
     return items.flat();

src/renderer/components/+user-management-roles/roles.store.ts

@@ -5,7 +5,7 @@ import { apiManager } from "../../api/api-manager";
 
 @autobind()
 export class RolesStore extends KubeObjectStore<Role> {
-  api = roleApi;
+  api = clusterRoleApi;
 
   getSubscribeApis() {
     return [roleApi, clusterRoleApi];
@@ -27,7 +27,7 @@ export class RolesStore extends KubeObjectStore<Role> {
   protected async loadItems(params: KubeStoreLoadItemsOptions): Promise<Role[]> {
     const items = await Promise.all([
       super.loadItems({ ...params, api: clusterRoleApi }),
-      super.loadItems({ ...params, api: roleApi }),
+      super.loadItems({ ...params, api: roleApi, refreshMeta: true }),
     ]);
 
     return items.flat();

src/renderer/components/item-object-list/item-list-layout.tsx

@@ -341,20 +341,19 @@ export class ItemListLayout extends React.Component<ItemListLayoutProps> {
   }
 
   renderInfo() {
-    const { allItems, items, isReady, userSettings, filters } = this;
+    const { items, isReady, userSettings, filters } = this;
-    const allItemsCount = allItems.length;
+    const totalCount = this.props.store.getItemsCount();
-    const itemsCount = items.length;
     const isFiltered = isReady && filters.length > 0;
 
     if (isFiltered) {
       const toggleFilters = () => userSettings.showAppliedFilters = !userSettings.showAppliedFilters;
 
       return (
-        <><a onClick={toggleFilters}>Filtered</a>: {itemsCount} / {allItemsCount}</>
+        <><a onClick={toggleFilters}>Filtered</a>: {items.length} / {totalCount}</>
       );
     }
 
-    return allItemsCount <= 1 ? `${allItemsCount} item` : `${allItemsCount} items`;
+    return totalCount <= 1 ? `${totalCount} item` : `${totalCount} items`;
   }
 
   renderHeader() {

src/renderer/item.store.ts

@@ -26,6 +26,10 @@ export abstract class ItemStore<T extends ItemObject = ItemObject> {
     return this.items.toJS();
   }
 
+  public getItemsCount(): number {
+    return this.items.length;
+  }
+
   getByName(name: string, ...args: any[]): T;
   getByName(name: string): T {
     return this.items.find(item => item.getName() === name);

src/renderer/kube-object.store.ts

@@ -18,6 +18,7 @@ export interface KubeStoreLoadItemsOptions {
   namespaces: string[]; // list of namespaces for loading into store with following merge-update
   api?: KubeApi; // api for loading resources, used for overriding, see: roles-store.ts
   merge?: boolean; // merge items into store, default: false
+  refreshMeta?: boolean;
 }
 
 export interface KubeStoreMergeItemsOptions {
@@ -34,6 +35,7 @@ export abstract class KubeObjectStore<T extends KubeObject = any> extends ItemSt
   abstract api: KubeApi<T>;
   public readonly limit?: number;
   public readonly bufferSize: number = 50000;
+  protected itemsCount = observable.map<KubeApi, number>();
 
   contextReady = when(() => Boolean(this.context));
 
@@ -113,7 +115,28 @@ export abstract class KubeObjectStore<T extends KubeObject = any> extends ItemSt
     }
   }
 
-  protected async loadItems({ namespaces, api = this.api, merge = false }: KubeStoreLoadItemsOptions): Promise<T[]> {
+  getItemsCount(): number {
+    const itemsCountFromMetadataLists = Array.from(this.itemsCount.values()).reduce((total, itemsCount) => {
+      return total + itemsCount;
+    }, 0);
+
+    return Math.max(itemsCountFromMetadataLists, this.items.length);
+  }
+
+  @action
+  protected async refreshItemsCount({ api = this.api } = {}): Promise<void> {
+    await this.contextReady;
+
+    try {
+      const itemsCount = await api.getItemsCount();
+
+      this.itemsCount.set(api, itemsCount);
+    } catch (error) {
+      console.error(`Refreshing metadata has failed: ${error}`, { api });
+    }
+  }
+
+  protected async loadItems({ namespaces, api = this.api, merge = false, refreshMeta = false }: KubeStoreLoadItemsOptions): Promise<T[]> {
     await this.contextReady;
     const { allNamespaces, cluster } = this.context;
     let items: T[] = [];
@@ -143,6 +166,10 @@ export abstract class KubeObjectStore<T extends KubeObject = any> extends ItemSt
       this.mergeItems(items, { replaceAll: false, updateStore: true });
     }
 
+    if (refreshMeta) {
+      this.refreshItemsCount({ api });
+    }
+
     return items;
   }
 
@@ -156,25 +183,28 @@ export abstract class KubeObjectStore<T extends KubeObject = any> extends ItemSt
     this.isLoading = true;
 
     try {
-      const newItems = await this.loadItems({
+      namespaces ??= this.context.allNamespaces; // load from all namespaces by default
-        namespaces: namespaces ?? this.context.allNamespaces, // load all by default
+      const items = await this.loadItems({ namespaces, api: this.api, });
-        api: this.api
-      });
 
       if (updateStore) {
-        this.mergeItems(newItems, {
+        this.mergeItems(items, {
           replaceAll: false, // partial update
           updateStore: true,
         });
       } else {
-        return newItems;
+        return items;
       }
 
       // clean up possibly stale items and reload removed namespaces
       if (autoCleanUp) {
-        await this.cleanUpAfterLoad(newItems).refreshRemovedItems();
+        const { refreshRemovedItems } = this.cleanUpAfterLoad(items);
+
+        await refreshRemovedItems();
       }
 
+      // refresh total items count with help of "/api/list?limit=1" requests
+      this.refreshItemsCount();
+
       this.isLoaded = true;
     } catch (error) {
       console.error("Loading store items failed", { error, store: this });