From 273c61a05943d1505470818c96155757d680229f Mon Sep 17 00:00:00 2001 From: Roman Date: Sun, 7 Feb 2021 23:55:57 +0200 Subject: [PATCH] using ?limit=1 and metadata.remainingItemCount for counting all items / refactoring Signed-off-by: Roman --- src/renderer/api/kube-api.ts | 76 ++++++++++++++++--- src/renderer/api/kube-json-api.ts | 18 ++++- .../role-bindings.store.ts | 4 +- .../+user-management-roles/roles.store.ts | 4 +- .../item-object-list/item-list-layout.tsx | 9 +-- src/renderer/item.store.ts | 4 + src/renderer/kube-object.store.ts | 46 +++++++++-- 7 files changed, 130 insertions(+), 31 deletions(-) diff --git a/src/renderer/api/kube-api.ts b/src/renderer/api/kube-api.ts index 199a8c7d30..e07f8a37e2 100644 --- a/src/renderer/api/kube-api.ts +++ b/src/renderer/api/kube-api.ts @@ -1,4 +1,5 @@ // Base class for building all kubernetes apis +// Docs: https://kubernetes.io/docs/reference/using-api/api-concepts import merge from "lodash/merge"; import { stringify } from "querystring"; @@ -7,7 +8,7 @@ import logger from "../../main/logger"; import { apiManager } from "./api-manager"; import { apiKube } from "./index"; import { createKubeApiURL, parseKubeApi } from "./kube-api-parse"; -import { KubeJsonApi, KubeJsonApiData, KubeJsonApiDataList } from "./kube-json-api"; +import { KubeJsonApi, KubeJsonApiData, KubeJsonApiDataList, KubeJsonApiListMetadata, KubeJsonApiListMetadataParsed, KubeJsonApiResponse } from "./kube-json-api"; import { IKubeObjectConstructor, KubeObject } from "./kube-object"; import { kubeWatchApi } from "./kube-watch-api"; @@ -233,8 +234,43 @@ export class KubeApi { return this.resourceVersions.get(namespace); } - async refreshResourceVersion(params?: { namespace: string }) { - return this.list(params, { limit: 1 }); + async listMetadata({ namespace = "" } = {}): Promise { + const response = await this.rawList({ namespace }, { + limit: 1, // specifying a limit to get metadata.remainingItemCount in response + resourceVersion: this.getResourceVersion(namespace) ?? "1", + resourceVersionMatch: "NotOlderThan", + }); + + const { remainingItemCount, ...metadata } = this.parseMetadata(namespace, response.metadata); + + return { + ...metadata, + itemsCount: remainingItemCount + 1, // +1 from limit=1 + }; + } + + async getItemsCount(): Promise { + try { + const { itemsCount } = await this.listMetadata(); // list request for all namespaces + + return itemsCount; + } catch (error) { + logger.error(`[KUBE-API]: getItemsTotal() has failed: ${error}`); + } + + return 0; + } + + async refreshResourceVersion(params?: { namespace: string }): Promise { + try { + const { resourceVersion } = await this.listMetadata(params); + + return resourceVersion; + } catch (error) { + logger.error(`[KUBE-API]: refreshing resourceVersion has failed: ${error}`, { params }); + } + + return ""; } getUrl({ name = "", namespace = "" } = {}, query?: Partial) { @@ -261,7 +297,7 @@ export class KubeApi { return query; } - protected parseResponse(data: KubeJsonApiData | KubeJsonApiData[] | KubeJsonApiDataList, namespace?: string): any { + protected parseResponse(data: KubeJsonApiResponse, namespace?: string): any { const KubeObjectConstructor = this.objectConstructor; if (KubeObject.isJsonApiData(data)) { @@ -276,9 +312,8 @@ export class KubeApi { if (KubeObject.isJsonApiDataList(data)) { const { apiVersion, items, metadata } = data; - // save "resourceVersion" metadata from list requests - this.setResourceVersion(namespace, metadata.resourceVersion); - this.setResourceVersion("", metadata.resourceVersion); + // parse & process metadata + this.parseMetadata(namespace, metadata); return items.map((item) => { const object = new KubeObjectConstructor({ @@ -301,11 +336,32 @@ export class KubeApi { return data; } - async list({ namespace = "" } = {}, query?: IKubeApiQueryParams): Promise { + protected parseMetadata(namespace: string, metadata: KubeJsonApiListMetadata): KubeJsonApiListMetadata & { namespace: string } { + const { + resourceVersion = "1", // optimization for "?limit=1&resourceVersionMatch=[non-empty]&resourceVersionMatch=NotOlderThan" + remainingItemCount = 0, // this is undefined for requests without ?limit + ...unprocessedMeta + } = metadata; + + // save "resourceVersion" for requests optimization + this.setResourceVersion(namespace, resourceVersion); + + return { + namespace, + resourceVersion, + remainingItemCount, + ...unprocessedMeta, + }; + } + + async rawList({ namespace = "" } = {}, query?: IKubeApiQueryParams): Promise { await this.checkPreferredVersion(); - return this.request - .get(this.getUrl({ namespace }), { query }) + return this.request.get(this.getUrl({ namespace }), { query }); + } + + async list({ namespace = "" } = {}, query?: IKubeApiQueryParams): Promise { + return this.rawList({ namespace }, query) .then(data => this.parseResponse(data, namespace)); } diff --git a/src/renderer/api/kube-json-api.ts b/src/renderer/api/kube-json-api.ts index 362ee5438e..db5ace3803 100644 --- a/src/renderer/api/kube-json-api.ts +++ b/src/renderer/api/kube-json-api.ts @@ -1,15 +1,25 @@ import { JsonApi, JsonApiData, JsonApiError } from "./json-api"; +export type KubeJsonApiResponse = KubeJsonApiData | KubeJsonApiData[] | KubeJsonApiDataList; + export interface KubeJsonApiDataList { kind: string; apiVersion: string; items: T[]; - metadata: { - resourceVersion: string; - selfLink: string; - }; + metadata: KubeJsonApiListMetadata; } +export interface KubeJsonApiListMetadata { + resourceVersion: string; + selfLink: string; + continue?: string; // hash-token, ?limit=N is required to use for request + remainingItemCount?: number; // remained items count from list request with ?limit= +} + +export type KubeJsonApiListMetadataParsed = Omit & { + itemsCount?: number; +}; + export interface KubeJsonApiData extends JsonApiData { kind: string; apiVersion: string; diff --git a/src/renderer/components/+user-management-roles-bindings/role-bindings.store.ts b/src/renderer/components/+user-management-roles-bindings/role-bindings.store.ts index a126ab9d18..3e384f5628 100644 --- a/src/renderer/components/+user-management-roles-bindings/role-bindings.store.ts +++ b/src/renderer/components/+user-management-roles-bindings/role-bindings.store.ts @@ -7,7 +7,7 @@ import { apiManager } from "../../api/api-manager"; @autobind() export class RoleBindingsStore extends KubeObjectStore { - api = roleBindingApi; + api = clusterRoleBindingApi; getSubscribeApis() { return [clusterRoleBindingApi, roleBindingApi]; @@ -29,7 +29,7 @@ export class RoleBindingsStore extends KubeObjectStore { protected async loadItems(params: KubeStoreLoadItemsOptions): Promise { const items = await Promise.all([ super.loadItems({ ...params, api: clusterRoleBindingApi }), - super.loadItems({ ...params, api: roleBindingApi }), + super.loadItems({ ...params, api: roleBindingApi, refreshMeta: true }), ]); return items.flat(); diff --git a/src/renderer/components/+user-management-roles/roles.store.ts b/src/renderer/components/+user-management-roles/roles.store.ts index e4bb9f26f8..dd3df2798c 100644 --- a/src/renderer/components/+user-management-roles/roles.store.ts +++ b/src/renderer/components/+user-management-roles/roles.store.ts @@ -5,7 +5,7 @@ import { apiManager } from "../../api/api-manager"; @autobind() export class RolesStore extends KubeObjectStore { - api = roleApi; + api = clusterRoleApi; getSubscribeApis() { return [roleApi, clusterRoleApi]; @@ -27,7 +27,7 @@ export class RolesStore extends KubeObjectStore { protected async loadItems(params: KubeStoreLoadItemsOptions): Promise { const items = await Promise.all([ super.loadItems({ ...params, api: clusterRoleApi }), - super.loadItems({ ...params, api: roleApi }), + super.loadItems({ ...params, api: roleApi, refreshMeta: true }), ]); return items.flat(); diff --git a/src/renderer/components/item-object-list/item-list-layout.tsx b/src/renderer/components/item-object-list/item-list-layout.tsx index 3dbb16956a..e18108009a 100644 --- a/src/renderer/components/item-object-list/item-list-layout.tsx +++ b/src/renderer/components/item-object-list/item-list-layout.tsx @@ -341,20 +341,19 @@ export class ItemListLayout extends React.Component { } renderInfo() { - const { allItems, items, isReady, userSettings, filters } = this; - const allItemsCount = allItems.length; - const itemsCount = items.length; + const { items, isReady, userSettings, filters } = this; + const totalCount = this.props.store.getItemsCount(); const isFiltered = isReady && filters.length > 0; if (isFiltered) { const toggleFilters = () => userSettings.showAppliedFilters = !userSettings.showAppliedFilters; return ( - <>Filtered: {itemsCount} / {allItemsCount} + <>Filtered: {items.length} / {totalCount} ); } - return allItemsCount <= 1 ? `${allItemsCount} item` : `${allItemsCount} items`; + return totalCount <= 1 ? `${totalCount} item` : `${totalCount} items`; } renderHeader() { diff --git a/src/renderer/item.store.ts b/src/renderer/item.store.ts index ce2c5eac25..a5235afe10 100644 --- a/src/renderer/item.store.ts +++ b/src/renderer/item.store.ts @@ -26,6 +26,10 @@ export abstract class ItemStore { return this.items.toJS(); } + public getItemsCount(): number { + return this.items.length; + } + getByName(name: string, ...args: any[]): T; getByName(name: string): T { return this.items.find(item => item.getName() === name); diff --git a/src/renderer/kube-object.store.ts b/src/renderer/kube-object.store.ts index d9e6fedef5..7019f671dc 100644 --- a/src/renderer/kube-object.store.ts +++ b/src/renderer/kube-object.store.ts @@ -18,6 +18,7 @@ export interface KubeStoreLoadItemsOptions { namespaces: string[]; // list of namespaces for loading into store with following merge-update api?: KubeApi; // api for loading resources, used for overriding, see: roles-store.ts merge?: boolean; // merge items into store, default: false + refreshMeta?: boolean; } export interface KubeStoreMergeItemsOptions { @@ -34,6 +35,7 @@ export abstract class KubeObjectStore extends ItemSt abstract api: KubeApi; public readonly limit?: number; public readonly bufferSize: number = 50000; + protected itemsCount = observable.map(); contextReady = when(() => Boolean(this.context)); @@ -113,7 +115,28 @@ export abstract class KubeObjectStore extends ItemSt } } - protected async loadItems({ namespaces, api = this.api, merge = false }: KubeStoreLoadItemsOptions): Promise { + getItemsCount(): number { + const itemsCountFromMetadataLists = Array.from(this.itemsCount.values()).reduce((total, itemsCount) => { + return total + itemsCount; + }, 0); + + return Math.max(itemsCountFromMetadataLists, this.items.length); + } + + @action + protected async refreshItemsCount({ api = this.api } = {}): Promise { + await this.contextReady; + + try { + const itemsCount = await api.getItemsCount(); + + this.itemsCount.set(api, itemsCount); + } catch (error) { + console.error(`Refreshing metadata has failed: ${error}`, { api }); + } + } + + protected async loadItems({ namespaces, api = this.api, merge = false, refreshMeta = false }: KubeStoreLoadItemsOptions): Promise { await this.contextReady; const { allNamespaces, cluster } = this.context; let items: T[] = []; @@ -143,6 +166,10 @@ export abstract class KubeObjectStore extends ItemSt this.mergeItems(items, { replaceAll: false, updateStore: true }); } + if (refreshMeta) { + this.refreshItemsCount({ api }); + } + return items; } @@ -156,25 +183,28 @@ export abstract class KubeObjectStore extends ItemSt this.isLoading = true; try { - const newItems = await this.loadItems({ - namespaces: namespaces ?? this.context.allNamespaces, // load all by default - api: this.api - }); + namespaces ??= this.context.allNamespaces; // load from all namespaces by default + const items = await this.loadItems({ namespaces, api: this.api, }); if (updateStore) { - this.mergeItems(newItems, { + this.mergeItems(items, { replaceAll: false, // partial update updateStore: true, }); } else { - return newItems; + return items; } // clean up possibly stale items and reload removed namespaces if (autoCleanUp) { - await this.cleanUpAfterLoad(newItems).refreshRemovedItems(); + const { refreshRemovedItems } = this.cleanUpAfterLoad(items); + + await refreshRemovedItems(); } + // refresh total items count with help of "/api/list?limit=1" requests + this.refreshItemsCount(); + this.isLoaded = true; } catch (error) { console.error("Loading store items failed", { error, store: this });