Watch-api streaming reworks (#1990)

* loading k8s resources into stores per selected namespaces -- part 1

Signed-off-by: Roman <ixrock@gmail.com>

* loading k8s resources into stores per selected namespaces -- part 2
- fix: generating helm chart id

Signed-off-by: Roman <ixrock@gmail.com>

* loading k8s resources into stores per selected namespaces -- part 3

Signed-off-by: Roman <ixrock@gmail.com>

* fixes

Signed-off-by: Roman <ixrock@gmail.com>

* fixes / responding to comments

Signed-off-by: Roman <ixrock@gmail.com>

* chore / small fixes

Signed-off-by: Roman <ixrock@gmail.com>

* Watch api does not work for non-admins with lots of namespaces #1898 -- part 1

Signed-off-by: Roman <ixrock@gmail.com>

* fixes & refactoring

Signed-off-by: Roman <ixrock@gmail.com>

* make lint happy

Signed-off-by: Roman <ixrock@gmail.com>

* reset store on loading error

Signed-off-by: Roman <ixrock@gmail.com>

* added new cluster method: cluster.isAllowedResource

Signed-off-by: Roman <ixrock@gmail.com>

* fix: loading namespaces optimizations

Signed-off-by: Roman <ixrock@gmail.com>

* fixes & refactoring

Signed-off-by: Roman <ixrock@gmail.com>

* fix: parse multiple kube-events from stream's chunk

Signed-off-by: Roman <ixrock@gmail.com>

* fix: mobx issue with accessing empty observable array by index (removes warning), use common logger

Signed-off-by: Roman <ixrock@gmail.com>

* fine-tuning

Signed-off-by: Roman <ixrock@gmail.com>

* fix: parse json stream chunks at client-side (might be partial, depends on network speed)

Signed-off-by: Roman <ixrock@gmail.com>

* store subscribing refactoring -- part 1

Signed-off-by: Roman <ixrock@gmail.com>

* store subscribing refactoring -- part 2

Signed-off-by: Roman <ixrock@gmail.com>

* store subscribing refactoring -- part 3

Signed-off-by: Roman <ixrock@gmail.com>

* store subscribing refactoring -- part 4

Signed-off-by: Roman <ixrock@gmail.com>

* auto-reconnect on online/offline status change, interval connection check

Signed-off-by: Roman <ixrock@gmail.com>

* check connection every 5m

Signed-off-by: Roman <ixrock@gmail.com>

* split concurrent watch-api requests by 10 at a time + 150ms delay before next call

Signed-off-by: Roman <ixrock@gmail.com>

* refactoring / clean up

Signed-off-by: Roman <ixrock@gmail.com>

* use `plimit` + delay for k8s watch requests

Signed-off-by: Roman <ixrock@gmail.com>

* lint fix

Signed-off-by: Roman <ixrock@gmail.com>

* added explicit `preload: true` when subscribing stores

Signed-off-by: Roman <ixrock@gmail.com>

* kubeWatchApi refactoring / fine-tuning

Signed-off-by: Roman <ixrock@gmail.com>

* clean up

Signed-off-by: Roman <ixrock@gmail.com>
This commit is contained in:
Roman authored and GitHub committed 2021-02-01 15:49:32 +02:00
1 parent 7490b15aad
commit 078f952b36
20 files changed
+519 -395

No files matched your search

+2 -2
View File
@@ -2,7 +2,7 @@ import type { KubeObjectStore } from "../kube-object.store";
import { action, observable } from "mobx";
import { autobind } from "../utils";
import { KubeApi } from "./kube-api";
import { KubeApi, parseKubeApi } from "./kube-api";
@autobind()
export class ApiManager {
@@ -11,7 +11,7 @@ export class ApiManager {
getApi(pathOrCallback: string | ((api: KubeApi) => boolean)) {
if (typeof pathOrCallback === "string") {
return this.apis.get(pathOrCallback) || this.apis.get(KubeApi.parseApi(pathOrCallback).apiBase);
return this.apis.get(pathOrCallback) || this.apis.get(parseKubeApi(pathOrCallback).apiBase);
}
return Array.from(this.apis.values()).find(pathOrCallback ?? (() => true));
+3 -11
View File
@@ -92,14 +92,6 @@ export function ensureObjectSelfLink(api: KubeApi, object: KubeJsonApiData) {
}
export class KubeApi<T extends KubeObject = any> {
static parseApi = parseKubeApi;
static watchAll(...apis: KubeApi[]) {
const disposers = apis.map(api => api.watch());
return () => disposers.forEach(unwatch => unwatch());
}
readonly kind: string;
readonly apiBase: string;
readonly apiPrefix: string;
@@ -124,7 +116,7 @@ export class KubeApi<T extends KubeObject = any> {
if (!options.apiBase) {
options.apiBase = objectConstructor.apiBase;
}
const { apiBase, apiPrefix, apiGroup, apiVersion, resource } = KubeApi.parseApi(options.apiBase);
const { apiBase, apiPrefix, apiGroup, apiVersion, resource } = parseKubeApi(options.apiBase);
this.kind = kind;
this.isNamespaced = isNamespaced;
@@ -157,7 +149,7 @@ export class KubeApi<T extends KubeObject = any> {
for (const apiUrl of apiBases) {
// Split e.g. "/apis/extensions/v1beta1/ingresses" to parts
const { apiPrefix, apiGroup, apiVersionWithGroup, resource } = KubeApi.parseApi(apiUrl);
const { apiPrefix, apiGroup, apiVersionWithGroup, resource } = parseKubeApi(apiUrl);
// Request available resources
try {
@@ -366,7 +358,7 @@ export class KubeApi<T extends KubeObject = any> {
}
watch(): () => void {
return kubeWatchApi.subscribe(this);
return kubeWatchApi.subscribeApi(this);
}
}
+289 -142
View File
@@ -1,202 +1,349 @@
// Kubernetes watch-api consumer
// Kubernetes watch-api client
// API: https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams
import { computed, observable, reaction } from "mobx";
import { stringify } from "querystring";
import { autobind, EventEmitter } from "../utils";
import { KubeJsonApiData } from "./kube-json-api";
import type { Cluster } from "../../main/cluster";
import type { IKubeWatchEvent, IKubeWatchEventStreamEnd, IWatchRoutePayload } from "../../main/routes/watch-route";
import type { KubeObject } from "./kube-object";
import type { KubeObjectStore } from "../kube-object.store";
import { ensureObjectSelfLink, KubeApi } from "./kube-api";
import type { NamespaceStore } from "../components/+namespaces/namespace.store";
import plimit from "p-limit";
import debounce from "lodash/debounce";
import { comparer, computed, observable, reaction } from "mobx";
import { autobind, EventEmitter } from "../utils";
import { ensureObjectSelfLink, KubeApi, parseKubeApi } from "./kube-api";
import { KubeJsonApiData, KubeJsonApiError } from "./kube-json-api";
import { apiPrefix, isDebugging, isProduction } from "../../common/vars";
import { apiManager } from "./api-manager";
import { apiPrefix, isDevelopment } from "../../common/vars";
import { getHostedCluster } from "../../common/cluster-store";
export interface IKubeWatchEvent<T = any> {
type: "ADDED" | "MODIFIED" | "DELETED" | "ERROR";
object?: T;
export { IKubeWatchEvent, IKubeWatchEventStreamEnd };
export interface IKubeWatchMessage<T extends KubeObject = any> {
data?: IKubeWatchEvent<KubeJsonApiData>
error?: IKubeWatchEvent<KubeJsonApiError>;
api?: KubeApi<T>;
store?: KubeObjectStore<T>;
}
export interface IKubeWatchRouteEvent {
type: "STREAM_END";
url: string;
status: number;
export interface IKubeWatchSubscribeStoreOptions {
preload?: boolean; // preload store items, default: true
waitUntilLoaded?: boolean; // subscribe only after loading all stores, default: true
cacheLoading?: boolean; // when enabled loading store will be skipped, default: false
}
export interface IKubeWatchRouteQuery {
api: string | string[];
export interface IKubeWatchReconnectOptions {
reconnectAttempts: number;
timeout: number;
}
export interface IKubeWatchLog {
message: string | Error;
meta?: object;
}
@autobind()
export class KubeWatchApi {
protected evtSource: EventSource;
protected onData = new EventEmitter<[IKubeWatchEvent]>();
protected subscribers = observable.map<KubeApi, number>();
protected reconnectTimeoutMs = 5000;
protected maxReconnectsOnError = 10;
protected reconnectAttempts = this.maxReconnectsOnError;
private cluster: Cluster;
private namespaceStore: NamespaceStore;
constructor() {
reaction(() => this.activeApis, () => this.connect(), {
fireImmediately: true,
delay: 500,
});
private requestId = 0;
private isConnected = false;
private reader: ReadableStreamReader<string>;
private subscribers = observable.map<KubeApi, number>();
// events
public onMessage = new EventEmitter<[IKubeWatchMessage]>();
@computed get isActive(): boolean {
return this.apis.length > 0;
}
@computed get activeApis() {
return Array.from(this.subscribers.keys());
@computed get apis(): string[] {
const { cluster, namespaceStore } = this;
const activeApis = Array.from(this.subscribers.keys());
return activeApis.map(api => {
if (!cluster.isAllowedResource(api.kind)) {
return [];
}
if (api.isNamespaced) {
return namespaceStore.getContextNamespaces().map(namespace => api.getWatchUrl(namespace));
} else {
return api.getWatchUrl();
}
}).flat();
}
constructor() {
this.init();
}
private async init() {
const { getHostedCluster } = await import("../../common/cluster-store");
const { namespaceStore } = await import("../components/+namespaces/namespace.store");
await namespaceStore.whenReady;
this.cluster = getHostedCluster();
this.namespaceStore = namespaceStore;
this.bindAutoConnect();
}
private bindAutoConnect() {
const connect = debounce(() => this.connect(), 1000);
reaction(() => this.apis, connect, {
fireImmediately: true,
equals: comparer.structural,
});
window.addEventListener("online", () => this.connect());
window.addEventListener("offline", () => this.disconnect());
setInterval(() => this.connectionCheck(), 60000 * 5); // every 5m
}
getSubscribersCount(api: KubeApi) {
return this.subscribers.get(api) || 0;
}
subscribe(...apis: KubeApi[]) {
isAllowedApi(api: KubeApi): boolean {
return !!this?.cluster.isAllowedResource(api.kind);
}
subscribeApi(api: KubeApi | KubeApi[]): () => void {
const apis: KubeApi[] = [api].flat();
apis.forEach(api => {
if (!this.isAllowedApi(api)) return; // skip
this.subscribers.set(api, this.getSubscribersCount(api) + 1);
});
return () => apis.forEach(api => {
const count = this.getSubscribersCount(api) - 1;
return () => {
apis.forEach(api => {
const count = this.getSubscribersCount(api) - 1;
if (count <= 0) this.subscribers.delete(api);
else this.subscribers.set(api, count);
});
}
// FIXME: use POST to send apis for subscribing (list could be huge)
// TODO: try to use normal fetch res.body stream to consume watch-api updates
// https://github.com/lensapp/lens/issues/1898
protected async getQuery() {
const { namespaceStore } = await import("../components/+namespaces/namespace.store");
await namespaceStore.whenReady;
const { isAdmin } = getHostedCluster();
return {
api: this.activeApis.map(api => {
if (isAdmin && !api.isNamespaced) {
return api.getWatchUrl();
}
if (api.isNamespaced) {
return namespaceStore.getContextNamespaces().map(namespace => api.getWatchUrl(namespace));
}
return [];
}).flat()
if (count <= 0) this.subscribers.delete(api);
else this.subscribers.set(api, count);
});
};
}
// todo: maybe switch to websocket to avoid often reconnects
@autobind()
protected async connect() {
if (this.evtSource) this.disconnect(); // close previous connection
subscribeStores(stores: KubeObjectStore[], options: IKubeWatchSubscribeStoreOptions = {}): () => void {
const { preload = true, waitUntilLoaded = true, cacheLoading = false } = options;
const limitRequests = plimit(1); // load stores one by one to allow quick skipping when fast clicking btw pages
const preloading: Promise<any>[] = [];
const apis = new Set(stores.map(store => store.getSubscribeApis()).flat());
const unsubscribeList: (() => void)[] = [];
let isUnsubscribed = false;
const query = await this.getQuery();
const subscribe = () => {
if (isUnsubscribed) return;
apis.forEach(api => unsubscribeList.push(this.subscribeApi(api)));
};
if (preload) {
for (const store of stores) {
preloading.push(limitRequests(async () => {
if (cacheLoading && store.isLoaded) return; // skip
return store.loadAll();
}));
}
}
if (waitUntilLoaded) {
Promise.all(preloading).then(subscribe, error => {
this.log({
message: new Error("Loading stores has failed"),
meta: { stores, error, options },
});
});
} else {
subscribe();
}
// unsubscribe
return () => {
if (isUnsubscribed) return;
isUnsubscribed = true;
limitRequests.clearQueue();
unsubscribeList.forEach(unsubscribe => unsubscribe());
};
}
protected async connectionCheck() {
if (!this.isConnected) {
this.log({ message: "Offline: reconnecting.." });
await this.connect();
}
this.log({
message: `Connection check: ${this.isConnected ? "online" : "offline"}`,
meta: { connected: this.isConnected },
});
}
protected async connect(apis = this.apis) {
this.disconnect(); // close active connections first
if (!navigator.onLine || !apis.length) {
this.isConnected = false;
if (!this.activeApis.length || !query.api.length) {
return;
}
const apiUrl = `${apiPrefix}/watch?${stringify(query)}`;
this.log({
message: "Connecting",
meta: { apis }
});
this.evtSource = new EventSource(apiUrl);
this.evtSource.onmessage = this.onMessage;
this.evtSource.onerror = this.onError;
this.writeLog("CONNECTING", query.api);
}
try {
const requestId = ++this.requestId;
const abortController = new AbortController();
reconnect() {
if (!this.evtSource || this.evtSource.readyState !== EventSource.OPEN) {
this.reconnectAttempts = this.maxReconnectsOnError;
this.connect();
const request = await fetch(`${apiPrefix}/watch`, {
method: "POST",
body: JSON.stringify({ apis } as IWatchRoutePayload),
signal: abortController.signal,
headers: {
"content-type": "application/json"
}
});
// request above is stale since new request-id has been issued
if (this.requestId !== requestId) {
abortController.abort();
return;
}
let jsonBuffer = "";
const stream = request.body.pipeThrough(new TextDecoderStream());
const reader = stream.getReader();
this.isConnected = true;
this.reader = reader;
while (true) {
const { done, value } = await reader.read();
if (done) break; // exit
const events = (jsonBuffer + value).split("\n");
jsonBuffer = this.processBuffer(events);
}
} catch (error) {
this.log({ message: error });
} finally {
this.isConnected = false;
}
}
protected disconnect() {
if (!this.evtSource) return;
this.evtSource.close();
this.evtSource.onmessage = null;
this.evtSource = null;
this.reader?.cancel();
this.reader = null;
this.isConnected = false;
}
protected onMessage(evt: MessageEvent) {
if (!evt.data) return;
const data = JSON.parse(evt.data);
// process received stream events, returns unprocessed buffer chunk if any
protected processBuffer(events: string[]): string {
for (const json of events) {
try {
const kubeEvent: IKubeWatchEvent = JSON.parse(json);
const message = this.getMessage(kubeEvent);
if ((data as IKubeWatchEvent).object) {
this.onData.emit(data);
} else {
this.onRouteEvent(data);
this.onMessage.emit(message);
} catch (error) {
return json;
}
}
return "";
}
protected async onRouteEvent(event: IKubeWatchRouteEvent) {
if (event.type === "STREAM_END") {
this.disconnect();
const { apiBase, namespace } = KubeApi.parseApi(event.url);
const api = apiManager.getApi(apiBase);
protected getMessage(event: IKubeWatchEvent): IKubeWatchMessage {
const message: IKubeWatchMessage = {};
if (api) {
try {
await api.refreshResourceVersion({ namespace });
this.reconnect();
} catch (error) {
console.error("failed to refresh resource version", error);
switch (event.type) {
case "ADDED":
case "DELETED":
if (this.subscribers.size > 0) {
setTimeout(() => {
this.onRouteEvent(event);
}, 1000);
}
case "MODIFIED": {
const data = event as IKubeWatchEvent<KubeJsonApiData>;
const api = apiManager.getApiByKind(data.object.kind, data.object.apiVersion);
message.data = data;
if (api) {
ensureObjectSelfLink(api, data.object);
const { namespace, resourceVersion } = data.object.metadata;
api.setResourceVersion(namespace, resourceVersion);
api.setResourceVersion("", resourceVersion);
message.api = api;
message.store = apiManager.getStore(api);
}
break;
}
case "ERROR":
message.error = event as IKubeWatchEvent<KubeJsonApiError>;
break;
case "STREAM_END": {
this.onServerStreamEnd(event as IKubeWatchEventStreamEnd, {
reconnectAttempts: 5,
timeout: 1000,
});
break;
}
}
return message;
}
protected async onServerStreamEnd(event: IKubeWatchEventStreamEnd, opts?: IKubeWatchReconnectOptions) {
const { apiBase, namespace } = parseKubeApi(event.url);
const api = apiManager.getApi(apiBase);
if (!api) return;
try {
await api.refreshResourceVersion({ namespace });
this.connect();
} catch (error) {
this.log({
message: new Error(`Failed to connect on single stream end: ${error}`),
meta: { event, error },
});
if (this.isActive && opts?.reconnectAttempts > 0) {
opts.reconnectAttempts--;
setTimeout(() => this.onServerStreamEnd(event, opts), opts.timeout); // repeat event
}
}
}
protected onError(evt: MessageEvent) {
const { reconnectAttempts: attemptsRemain, reconnectTimeoutMs } = this;
if (evt.eventPhase === EventSource.CLOSED) {
if (attemptsRemain > 0) {
this.reconnectAttempts--;
setTimeout(() => this.connect(), reconnectTimeoutMs);
}
protected log({ message, meta = {} }: IKubeWatchLog) {
if (isProduction && !isDebugging) {
return;
}
}
protected writeLog(...data: any[]) {
if (isDevelopment) {
console.log("%cKUBE-WATCH-API:", `font-weight: bold`, ...data);
const logMessage = `%c[KUBE-WATCH-API]: ${String(message).toUpperCase()}`;
const isError = message instanceof Error;
const textStyle = `font-weight: bold;`;
const time = new Date().toLocaleString();
if (isError) {
console.error(logMessage, textStyle, { time, ...meta });
} else {
console.info(logMessage, textStyle, { time, ...meta });
}
}
addListener(store: KubeObjectStore, callback: (evt: IKubeWatchEvent) => void) {
const listener = (evt: IKubeWatchEvent<KubeJsonApiData>) => {
if (evt.type === "ERROR") {
return; // e.g. evt.object.message == "too old resource version"
}
const { namespace, resourceVersion } = evt.object.metadata;
const api = apiManager.getApiByKind(evt.object.kind, evt.object.apiVersion);
api.setResourceVersion(namespace, resourceVersion);
api.setResourceVersion("", resourceVersion);
ensureObjectSelfLink(api, evt.object);
if (store == apiManager.getStore(api)) {
callback(evt);
}
};
this.onData.addListener(listener);
return () => this.onData.removeListener(listener);
}
reset() {
this.subscribers.clear();
}
}
export const kubeWatchApi = new KubeWatchApi();