Watch-api streaming reworks (#1990)

* loading k8s resources into stores per selected namespaces -- part 1

Signed-off-by: Roman <ixrock@gmail.com>

* loading k8s resources into stores per selected namespaces -- part 2
- fix: generating helm chart id

Signed-off-by: Roman <ixrock@gmail.com>

* loading k8s resources into stores per selected namespaces -- part 3

Signed-off-by: Roman <ixrock@gmail.com>

* fixes

Signed-off-by: Roman <ixrock@gmail.com>

* fixes / responding to comments

Signed-off-by: Roman <ixrock@gmail.com>

* chore / small fixes

Signed-off-by: Roman <ixrock@gmail.com>

* Watch api does not work for non-admins with lots of namespaces #1898 -- part 1

Signed-off-by: Roman <ixrock@gmail.com>

* fixes & refactoring

Signed-off-by: Roman <ixrock@gmail.com>

* make lint happy

Signed-off-by: Roman <ixrock@gmail.com>

* reset store on loading error

Signed-off-by: Roman <ixrock@gmail.com>

* added new cluster method: cluster.isAllowedResource

Signed-off-by: Roman <ixrock@gmail.com>

* fix: loading namespaces optimizations

Signed-off-by: Roman <ixrock@gmail.com>

* fixes & refactoring

Signed-off-by: Roman <ixrock@gmail.com>

* fix: parse multiple kube-events from stream's chunk

Signed-off-by: Roman <ixrock@gmail.com>

* fix: mobx issue with accessing empty observable array by index (removes warning), use common logger

Signed-off-by: Roman <ixrock@gmail.com>

* fine-tuning

Signed-off-by: Roman <ixrock@gmail.com>

* fix: parse json stream chunks at client-side (might be partial, depends on network speed)

Signed-off-by: Roman <ixrock@gmail.com>

* store subscribing refactoring -- part 1

Signed-off-by: Roman <ixrock@gmail.com>

* store subscribing refactoring -- part 2

Signed-off-by: Roman <ixrock@gmail.com>

* store subscribing refactoring -- part 3

Signed-off-by: Roman <ixrock@gmail.com>

* store subscribing refactoring -- part 4

Signed-off-by: Roman <ixrock@gmail.com>

* auto-reconnect on online/offline status change, interval connection check

Signed-off-by: Roman <ixrock@gmail.com>

* check connection every 5m

Signed-off-by: Roman <ixrock@gmail.com>

* split concurrent watch-api requests by 10 at a time + 150ms delay before next call

Signed-off-by: Roman <ixrock@gmail.com>

* refactoring / clean up

Signed-off-by: Roman <ixrock@gmail.com>

* use `plimit` + delay for k8s watch requests

Signed-off-by: Roman <ixrock@gmail.com>

* lint fix

Signed-off-by: Roman <ixrock@gmail.com>

* added explicit `preload: true` when subscribing stores

Signed-off-by: Roman <ixrock@gmail.com>

* kubeWatchApi refactoring / fine-tuning

Signed-off-by: Roman <ixrock@gmail.com>

* clean up

Signed-off-by: Roman <ixrock@gmail.com>
This commit is contained in:
Roman authored and GitHub committed 2021-02-01 15:49:32 +02:00
1 parent 7490b15aad
commit 078f952b36
20 files changed
+519 -395

No files matched your search

@@ -3,13 +3,9 @@ import "./cluster-overview.scss";
import React from "react";
import { reaction } from "mobx";
import { disposeOnUnmount, observer } from "mobx-react";
import { eventStore } from "../+events/event.store";
import { nodesStore } from "../+nodes/nodes.store";
import { podsStore } from "../+workloads-pods/pods.store";
import { getHostedCluster } from "../../../common/cluster-store";
import { isAllowedResource } from "../../../common/rbac";
import { KubeObjectStore } from "../../kube-object.store";
import { interval } from "../../utils";
import { TabLayout } from "../layout/tab-layout";
import { Spinner } from "../spinner";
@@ -17,45 +13,33 @@ import { ClusterIssues } from "./cluster-issues";
import { ClusterMetrics } from "./cluster-metrics";
import { clusterOverviewStore } from "./cluster-overview.store";
import { ClusterPieCharts } from "./cluster-pie-charts";
import { eventStore } from "../+events/event.store";
import { kubeWatchApi } from "../../api/kube-watch-api";
@observer
export class ClusterOverview extends React.Component {
private stores: KubeObjectStore<any>[] = [];
private subscribers: Array<() => void> = [];
private metricPoller = interval(60, this.loadMetrics);
@disposeOnUnmount
fetchMetrics = reaction(
() => clusterOverviewStore.metricNodeRole, // Toggle Master/Worker node switcher
() => this.metricPoller.restart(true)
);
private metricPoller = interval(60, () => this.loadMetrics());
loadMetrics() {
getHostedCluster().available && clusterOverviewStore.loadMetrics();
}
async componentDidMount() {
if (isAllowedResource("nodes")) {
this.stores.push(nodesStore);
}
componentDidMount() {
this.metricPoller.start(true);
if (isAllowedResource("pods")) {
this.stores.push(podsStore);
}
disposeOnUnmount(this, [
kubeWatchApi.subscribeStores([nodesStore, podsStore, eventStore], {
preload: true,
}),
if (isAllowedResource("events")) {
this.stores.push(eventStore);
}
await Promise.all(this.stores.map(store => store.loadAll()));
this.loadMetrics();
this.subscribers = this.stores.map(store => store.subscribe());
this.metricPoller.start();
reaction(
() => clusterOverviewStore.metricNodeRole, // Toggle Master/Worker node switcher
() => this.metricPoller.restart(true)
),
]);
}
componentWillUnmount() {
this.subscribers.forEach(dispose => dispose()); // unsubscribe all
this.metricPoller.stop();
}
@@ -52,6 +52,10 @@ export class EventStore extends KubeObjectStore<KubeEvent> {
return compact(eventsWithError);
}
getWarningsCount() {
return this.getWarnings().length;
}
}
export const eventStore = new EventStore();
@@ -2,13 +2,14 @@ import "./namespace-select.scss";
import React from "react";
import { computed } from "mobx";
import { observer } from "mobx-react";
import { disposeOnUnmount, observer } from "mobx-react";
import { Select, SelectOption, SelectProps } from "../select";
import { cssNames, noop } from "../../utils";
import { cssNames } from "../../utils";
import { Icon } from "../icon";
import { namespaceStore } from "./namespace.store";
import { FilterIcon } from "../item-object-list/filter-icon";
import { FilterType } from "../item-object-list/page-filters.store";
import { kubeWatchApi } from "../../api/kube-watch-api";
interface Props extends SelectProps {
showIcons?: boolean;
@@ -28,17 +29,13 @@ const defaultProps: Partial<Props> = {
@observer
export class NamespaceSelect extends React.Component<Props> {
static defaultProps = defaultProps as object;
private unsubscribe = noop;
async componentDidMount() {
if (!namespaceStore.isLoaded) {
await namespaceStore.loadAll();
}
this.unsubscribe = namespaceStore.subscribe();
}
componentWillUnmount() {
this.unsubscribe();
componentDidMount() {
disposeOnUnmount(this, [
kubeWatchApi.subscribeStores([namespaceStore], {
preload: true,
})
]);
}
@computed get options(): SelectOption[] {
@@ -60,7 +57,7 @@ export class NamespaceSelect extends React.Component<Props> {
return label || (
<>
{showIcons && <Icon small material="layers" />}
{showIcons && <Icon small material="layers"/>}
{value}
</>
);
@@ -103,9 +100,9 @@ export class NamespaceSelectFilter extends React.Component {
return (
<div className="flex gaps align-center">
<FilterIcon type={FilterType.NAMESPACE} />
<FilterIcon type={FilterType.NAMESPACE}/>
<span>{namespace}</span>
{isSelected && <Icon small material="check" className="box right" />}
{isSelected && <Icon small material="check" className="box right"/>}
</div>
);
}}
@@ -117,15 +117,15 @@ export class NamespaceStore extends KubeObjectStore<Namespace> {
return namespaces;
}
subscribe(apis = [this.api]) {
getSubscribeApis() {
const { accessibleNamespaces } = getHostedCluster();
// if user has given static list of namespaces let's not start watches because watch adds stuff that's not wanted
if (accessibleNamespaces.length > 0) {
return Function; // no-op
return [];
}
return super.subscribe(apis);
return super.getSubscribeApis();
}
protected async loadItems(params: KubeObjectStoreLoadingParams) {
@@ -1,17 +1,18 @@
import "./service-details.scss";
import React from "react";
import { observer } from "mobx-react";
import { disposeOnUnmount, observer } from "mobx-react";
import { DrawerItem, DrawerTitle } from "../drawer";
import { Badge } from "../badge";
import { KubeEventDetails } from "../+events/kube-event-details";
import { KubeObjectDetailsProps } from "../kube-object";
import { Service, endpointApi } from "../../api/endpoints";
import { Service } from "../../api/endpoints";
import { KubeObjectMeta } from "../kube-object/kube-object-meta";
import { ServicePortComponent } from "./service-port-component";
import { endpointStore } from "../+network-endpoints/endpoints.store";
import { ServiceDetailsEndpoint } from "./service-details-endpoint";
import { kubeObjectDetailRegistry } from "../../api/kube-object-detail-registry";
import { kubeWatchApi } from "../../api/kube-watch-api";
interface Props extends KubeObjectDetailsProps<Service> {
}
@@ -19,10 +20,11 @@ interface Props extends KubeObjectDetailsProps<Service> {
@observer
export class ServiceDetails extends React.Component<Props> {
componentDidMount() {
if (!endpointStore.isLoaded) {
endpointStore.loadAll();
}
endpointApi.watch();
disposeOnUnmount(this, [
kubeWatchApi.subscribeStores([endpointStore], {
preload: true,
}),
]);
}
render() {
@@ -77,7 +79,7 @@ export class ServiceDetails extends React.Component<Props> {
)}
<DrawerTitle title={`Endpoint`}/>
<ServiceDetailsEndpoint endpoint={endpoint} />
<ServiceDetailsEndpoint endpoint={endpoint}/>
</div>
);
}
@@ -1,3 +1,4 @@
import { sum } from "lodash";
import { action, computed, observable } from "mobx";
import { clusterApi, IClusterMetrics, INodeMetrics, Node, nodesApi } from "../../api/endpoints";
import { autobind } from "../../utils";
@@ -62,6 +63,10 @@ export class NodesStore extends KubeObjectStore<Node> {
});
}
getWarningsCount(): number {
return sum(this.items.map((node: Node) => node.getWarningConditions().length));
}
reset() {
super.reset();
this.metrics = {};
@@ -9,8 +9,8 @@ import { apiManager } from "../../api/api-manager";
export class RoleBindingsStore extends KubeObjectStore<RoleBinding> {
api = clusterRoleBindingApi;
subscribe() {
return super.subscribe([clusterRoleBindingApi, roleBindingApi]);
getSubscribeApis() {
return [clusterRoleBindingApi, roleBindingApi];
}
protected sortItems(items: RoleBinding[]) {
@@ -7,8 +7,8 @@ import { apiManager } from "../../api/api-manager";
export class RolesStore extends KubeObjectStore<Role> {
api = clusterRoleApi;
subscribe() {
return super.subscribe([roleApi, clusterRoleApi]);
getSubscribeApis() {
return [roleApi, clusterRoleApi];
}
protected sortItems(items: Role[]) {
@@ -1,8 +1,7 @@
import "./overview.scss";
import React from "react";
import { observable, when } from "mobx";
import { observer } from "mobx-react";
import { disposeOnUnmount, observer } from "mobx-react";
import { OverviewStatuses } from "./overview-statuses";
import { RouteComponentProps } from "react-router";
import { IWorkloadsOverviewRouteParams } from "../+workloads";
@@ -15,60 +14,23 @@ import { replicaSetStore } from "../+workloads-replicasets/replicasets.store";
import { jobStore } from "../+workloads-jobs/job.store";
import { cronJobStore } from "../+workloads-cronjobs/cronjob.store";
import { Events } from "../+events";
import { KubeObjectStore } from "../../kube-object.store";
import { isAllowedResource } from "../../../common/rbac";
import { namespaceStore } from "../+namespaces/namespace.store";
import { kubeWatchApi } from "../../api/kube-watch-api";
interface Props extends RouteComponentProps<IWorkloadsOverviewRouteParams> {
}
@observer
export class WorkloadsOverview extends React.Component<Props> {
@observable isLoading = false;
@observable isUnmounting = false;
async componentDidMount() {
const stores: KubeObjectStore[] = [
isAllowedResource("pods") && podsStore,
isAllowedResource("deployments") && deploymentStore,
isAllowedResource("daemonsets") && daemonSetStore,
isAllowedResource("statefulsets") && statefulSetStore,
isAllowedResource("replicasets") && replicaSetStore,
isAllowedResource("jobs") && jobStore,
isAllowedResource("cronjobs") && cronJobStore,
isAllowedResource("events") && eventStore,
].filter(Boolean);
const unsubscribeMap = new Map<KubeObjectStore, () => void>();
const loadStores = async () => {
this.isLoading = true;
for (const store of stores) {
if (this.isUnmounting) break;
try {
await store.loadAll();
unsubscribeMap.get(store)?.(); // unsubscribe previous watcher
unsubscribeMap.set(store, store.subscribe());
} catch (error) {
console.error("loading store error", error);
}
}
this.isLoading = false;
};
namespaceStore.onContextChange(loadStores, {
fireImmediately: true,
});
await when(() => this.isUnmounting && !this.isLoading);
unsubscribeMap.forEach(dispose => dispose());
unsubscribeMap.clear();
}
componentWillUnmount() {
this.isUnmounting = true;
componentDidMount() {
disposeOnUnmount(this, [
kubeWatchApi.subscribeStores([
podsStore, deploymentStore, daemonSetStore, statefulSetStore, replicaSetStore,
jobStore, cronJobStore, eventStore,
], {
preload: true,
}),
]);
}
render() {
+39 -47
View File
@@ -1,5 +1,5 @@
import React from "react";
import { observer } from "mobx-react";
import { disposeOnUnmount, observer } from "mobx-react";
import { Redirect, Route, Router, Switch } from "react-router";
import { history } from "../navigation";
import { Notifications } from "./notifications";
@@ -42,10 +42,10 @@ import { ClusterPageMenuRegistration, clusterPageMenuRegistry } from "../../exte
import { TabLayout, TabLayoutRoute } from "./layout/tab-layout";
import { StatefulSetScaleDialog } from "./+workloads-statefulsets/statefulset-scale-dialog";
import { eventStore } from "./+events/event.store";
import { reaction, computed, observable } from "mobx";
import { computed, reaction, observable } from "mobx";
import { nodesStore } from "./+nodes/nodes.store";
import { podsStore } from "./+workloads-pods/pods.store";
import { sum } from "lodash";
import { kubeWatchApi } from "../api/kube-watch-api";
import { ReplicaSetScaleDialog } from "./+workloads-replicasets/replicaset-scale-dialog";
@observer
@@ -75,50 +75,26 @@ export class App extends React.Component {
whatInput.ask(); // Start to monitor user input device
}
@observable extensionRoutes: Map<ClusterPageMenuRegistration, React.ReactNode> = new Map();
componentDidMount() {
disposeOnUnmount(this, [
kubeWatchApi.subscribeStores([podsStore, nodesStore, eventStore], {
preload: true,
}),
async componentDidMount() {
const cluster = getHostedCluster();
const promises: Promise<void>[] = [];
reaction(() => this.warningsTotal, (count: number) => {
broadcastMessage(`cluster-warning-event-count:${getHostedCluster().id}`, count);
}),
if (isAllowedResource("events") && isAllowedResource("pods")) {
promises.push(eventStore.loadAll());
promises.push(podsStore.loadAll());
}
if (isAllowedResource("nodes")) {
promises.push(nodesStore.loadAll());
}
await Promise.all(promises);
if (eventStore.isLoaded && podsStore.isLoaded) {
eventStore.subscribe();
podsStore.subscribe();
}
if (nodesStore.isLoaded) {
nodesStore.subscribe();
}
reaction(() => this.warningsCount, (count) => {
broadcastMessage(`cluster-warning-event-count:${cluster.id}`, count);
});
reaction(() => clusterPageMenuRegistry.getRootItems(), (rootItems) => {
this.generateExtensionTabLayoutRoutes(rootItems);
}, {
fireImmediately: true
});
reaction(() => clusterPageMenuRegistry.getRootItems(), (rootItems) => {
this.generateExtensionTabLayoutRoutes(rootItems);
}, {
fireImmediately: true
})
]);
}
@computed
get warningsCount() {
let warnings = sum(nodesStore.items
.map(node => node.getWarningConditions().length));
warnings = warnings + eventStore.getWarnings().length;
return warnings;
@computed get warningsTotal(): number {
return nodesStore.getWarningsCount() + eventStore.getWarningsCount();
}
get startURL() {
@@ -151,6 +127,26 @@ export class App extends React.Component {
return routes;
}
renderExtensionTabLayoutRoutes() {
return clusterPageMenuRegistry.getRootItems().map((menu, index) => {
const tabRoutes = this.getTabLayoutRoutes(menu);
if (tabRoutes.length > 0) {
const pageComponent = () => <TabLayout tabs={tabRoutes}/>;
return <Route key={`extension-tab-layout-route-${index}`} component={pageComponent} path={tabRoutes.map((tab) => tab.routePath)}/>;
} else {
const page = clusterPageRegistry.getByPageTarget(menu.target);
if (page) {
return <Route key={`extension-tab-layout-route-${index}`} path={page.url} component={page.components.Page}/>;
}
}
});
}
@observable extensionRoutes: Map<ClusterPageMenuRegistration, React.ReactNode> = new Map();
generateExtensionTabLayoutRoutes(rootItems: ClusterPageMenuRegistration[]) {
rootItems.forEach((menu, index) => {
let route = this.extensionRoutes.get(menu);
@@ -181,10 +177,6 @@ export class App extends React.Component {
}
}
renderExtensionTabLayoutRoutes() {
return Array.from(this.extensionRoutes.values());
}
renderExtensionRoutes() {
return clusterPageRegistry.getItems().map((page, index) => {
const menu = clusterPageMenuRegistry.getByPage(page);
@@ -2,7 +2,7 @@ import "./item-list-layout.scss";
import groupBy from "lodash/groupBy";
import React, { ReactNode } from "react";
import { computed, IReactionDisposer, observable, reaction, toJS } from "mobx";
import { computed, observable, reaction, toJS } from "mobx";
import { disposeOnUnmount, observer } from "mobx-react";
import { ConfirmDialog, ConfirmDialogParams } from "../confirm-dialog";
import { Table, TableCell, TableCellProps, TableHead, TableProps, TableRow, TableRowProps, TableSortCallback } from "../table";
@@ -12,7 +12,6 @@ import { NoItems } from "../no-items";
import { Spinner } from "../spinner";
import { ItemObject, ItemStore } from "../../item.store";
import { SearchInputUrl } from "../input";
import { namespaceStore } from "../+namespaces/namespace.store";
import { Filter, FilterType, pageFilters } from "./page-filters.store";
import { PageFiltersList } from "./page-filters-list";
import { PageFiltersSelect } from "./page-filters-select";
@@ -22,6 +21,7 @@ import { MenuActions } from "../menu/menu-actions";
import { MenuItem } from "../menu";
import { Checkbox } from "../checkbox";
import { userStore } from "../../../common/user-store";
import { namespaceStore } from "../+namespaces/namespace.store";
// todo: refactor, split to small re-usable components
@@ -40,6 +40,7 @@ export interface ItemListLayoutProps<T extends ItemObject = ItemObject> {
className: IClassName;
store: ItemStore<T>;
dependentStores?: ItemStore[];
preloadStores?: boolean;
isClusterScoped?: boolean;
hideFilters?: boolean;
searchFilters?: SearchFilter<T>[];
@@ -82,6 +83,7 @@ const defaultProps: Partial<ItemListLayoutProps> = {
isSelectable: true,
isConfigurable: false,
copyClassNameFromHeadCells: true,
preloadStores: true,
dependentStores: [],
filterItems: [],
hasDetailsView: true,
@@ -97,10 +99,6 @@ interface ItemListLayoutUserSettings {
export class ItemListLayout extends React.Component<ItemListLayoutProps> {
static defaultProps = defaultProps as object;
private watchDisposers: IReactionDisposer[] = [];
@observable isUnmounting = false;
@observable userSettings: ItemListLayoutUserSettings = {
showAppliedFilters: false,
};
@@ -119,54 +117,28 @@ export class ItemListLayout extends React.Component<ItemListLayoutProps> {
}
async componentDidMount() {
const { isClusterScoped, isConfigurable, tableId } = this.props;
const { isClusterScoped, isConfigurable, tableId, preloadStores } = this.props;
if (isConfigurable && !tableId) {
throw new Error("[ItemListLayout]: configurable list require props.tableId to be specified");
}
this.loadStores();
if (preloadStores) {
this.loadStores();
if (!isClusterScoped) {
disposeOnUnmount(this, [
namespaceStore.onContextChange(() => this.loadStores())
]);
if (!isClusterScoped) {
disposeOnUnmount(this, [
namespaceStore.onContextChange(() => this.loadStores())
]);
}
}
}
async componentWillUnmount() {
this.isUnmounting = true;
this.unsubscribeStores();
}
@computed get stores() {
private loadStores() {
const { store, dependentStores } = this.props;
const stores = Array.from(new Set([store, ...dependentStores]));
return new Set([store, ...dependentStores]);
}
async loadStores() {
this.unsubscribeStores(); // reset first
// load
for (const store of this.stores) {
if (this.isUnmounting) {
this.unsubscribeStores();
break;
}
try {
await store.loadAll();
this.watchDisposers.push(store.subscribe());
} catch (error) {
console.error("loading store error", error);
}
}
}
unsubscribeStores() {
this.watchDisposers.forEach(dispose => dispose());
this.watchDisposers.length = 0;
stores.forEach(store => store.loadAll());
}
private filterCallbacks: { [type: string]: ItemsFilter } = {
@@ -1,15 +1,17 @@
import React from "react";
import { computed } from "mobx";
import { observer } from "mobx-react";
import { disposeOnUnmount, observer } from "mobx-react";
import { cssNames } from "../../utils";
import { KubeObject } from "../../api/kube-object";
import { ItemListLayout, ItemListLayoutProps } from "../item-object-list/item-list-layout";
import { KubeObjectStore } from "../../kube-object.store";
import { KubeObjectMenu } from "./kube-object-menu";
import { kubeSelectedUrlParam, showDetails } from "./kube-object-details";
import { kubeWatchApi } from "../../api/kube-watch-api";
export interface KubeObjectListLayoutProps extends ItemListLayoutProps {
store: KubeObjectStore;
dependentStores?: KubeObjectStore[];
}
@observer
@@ -18,6 +20,17 @@ export class KubeObjectListLayout extends React.Component<KubeObjectListLayoutPr
return this.props.store.getByPath(kubeSelectedUrlParam.get());
}
componentDidMount() {
const { store, dependentStores = [] } = this.props;
const stores = Array.from(new Set([store, ...dependentStores]));
disposeOnUnmount(this, [
kubeWatchApi.subscribeStores(stores, {
preload: true
})
]);
}
onDetails = (item: KubeObject) => {
if (this.props.onDetails) {
this.props.onDetails(item);
@@ -33,6 +46,7 @@ export class KubeObjectListLayout extends React.Component<KubeObjectListLayoutPr
<ItemListLayout
{...layoutProps}
className={cssNames("KubeObjectListLayout", className)}
preloadStores={false} // loading handled in kubeWatchApi.subscribeStores()
detailsItem={this.selectedItem}
onDetails={this.onDetails}
renderItemMenu={(item) => {