From ad23e51704f11e06ca30fd933ad42de5c6cec9e5 Mon Sep 17 00:00:00 2001 From: Sebastian Malton Date: Fri, 9 Oct 2020 11:47:10 -0400 Subject: [PATCH] Implement base metadata gathering system Signed-off-by: Sebastian Malton --- src/common/base-store.ts | 11 +- src/common/cluster-meta-manager.ts | 196 +++++++++++++++++++++ src/common/cluster-meta-store.ts | 131 ++++++++++++++ src/common/cluster-store.ts | 2 +- src/common/meta-collectors/distribution.ts | 16 ++ src/common/system-ca.ts | 8 +- src/main/cluster.ts | 2 + src/main/index.ts | 3 + src/main/register-collectors.ts | 6 + src/migrations/cluster-meta-store/index.ts | 1 + 10 files changed, 364 insertions(+), 12 deletions(-) create mode 100644 src/common/cluster-meta-manager.ts create mode 100644 src/common/cluster-meta-store.ts create mode 100644 src/common/meta-collectors/distribution.ts create mode 100644 src/main/register-collectors.ts create mode 100644 src/migrations/cluster-meta-store/index.ts diff --git a/src/common/base-store.ts b/src/common/base-store.ts index f476965736..05c361a332 100644 --- a/src/common/base-store.ts +++ b/src/common/base-store.ts @@ -9,12 +9,12 @@ import logger from "../main/logger"; import { broadcastIpc, IpcBroadcastParams } from "./ipc"; import isEqual from "lodash/isEqual"; -export interface BaseStoreParams extends ConfOptions { +export interface BaseStoreParams extends ConfOptions { autoLoad?: boolean; syncEnabled?: boolean; } -export class BaseStore extends Singleton { +export abstract class BaseStore extends Singleton { protected storeConfig: Config; protected syncDisposers: Function[] = []; @@ -22,7 +22,7 @@ export class BaseStore extends Singleton { @observable isLoaded = false; @observable protected data: T; - protected constructor(protected params: BaseStoreParams) { + protected constructor(protected params: BaseStoreParams) { super(); this.params = { autoLoad: false, @@ -157,10 +157,7 @@ export class BaseStore extends Singleton { return subFrames; } - @action - protected fromStore(data: T) { - this.data = data; - } + protected abstract fromStore(data: T): void; // todo: use "serializr" ? toJSON(): T { diff --git a/src/common/cluster-meta-manager.ts b/src/common/cluster-meta-manager.ts new file mode 100644 index 0000000000..3796edd62f --- /dev/null +++ b/src/common/cluster-meta-manager.ts @@ -0,0 +1,196 @@ +import { autobind } from "../renderer/utils"; +import { clusterMetaStore } from "./cluster-meta-store"; +import { ClusterId } from "./cluster-store"; +import Singleton from "./utils/singleton"; + +export abstract class ClusterMetaCollector { + /** + * start tells the collector to start collecting its metadata once. + * + * - If finished collecting last time (either producing a value or error) + * then start to collect again + * - If still collected since last time then should continue and **not** + * restart + */ + abstract start(): void; + + /** + * stop tells the collector to stop collecting its metadata. If `start()` is + * called after `stop()` then the collector should begin completely fresh. + * + * Should not throw if called multiple times. + */ + abstract stop(): void; +} + +/** + * This is the constructor for a type that extends the abstract class + * `ClusterMetaCollector` + * + * @param clusterId the ID of the cluster that the collector should be targeting + * @param onSuccess the function that should be called when the collector has + * collected its metadata successfully + * @param onError the function that should be called when the collector + * encounters an error during the collection process + */ +export type MetadataConstructor = new (clusterId: ClusterId, onSuccess: (result: any) => void, onError: (err: string) => void) => T; + +export class ClusterMetaManager extends Singleton { + /** + * registeredCollectors is a mapping between the name of the metadata and the + * means of creating new collectors when new clusters are activated. + */ + protected registeredCollectors = new Map>(); + + /** + * createdCollectors is the mapping between clusters and those collectors + * targeting that cluster. Each are stored so that it can be easily iterated + * over by name of metadata to be collected and by cluster ID + */ + protected createdCollectors = new Map>(); + + protected interval: NodeJS.Timeout + + // the milliseconds since 1970 when the last interval fired + protected lastInterval = Date.now() + + protected constructor(protected collectionPeriod = 10 * 1000) { + super() + + this.interval = setInterval(this.onInterval, this.collectionPeriod) + } + + @autobind() + protected onInterval() { + this.lastInterval = Date.now() + + for (const byCluster of this.createdCollectors.values()) { + for (const collector of byCluster.values()) { + collector.start() + } + } + } + + /** + * registerCollector adds the collector to the list of current collectors and + * starts collection of metadata on all currently active clusters + * + * @param name is the name of the metadata to be collected, will be the + * name of the field in the metadata store + * @param Collector the type of the collector. This is so that the manager + * can create new instances on demand + * + * @throws if `name` has already been registered + */ + public registerCollector(name: string, CollectorType: MetadataConstructor) { + if (this.registeredCollectors.has(name)) { + throw new Error(`A collector for ${name} has already been registered`) + } + + this.registeredCollectors.set(name, CollectorType) + + /** + * add the collector to all the clusters currently been collected on + */ + for (const [clusterId, byCluster] of this.createdCollectors) { + const collector = new CollectorType( + clusterId, + this.onCollectionSuccess.bind(clusterId, name), + this.onCollectionError.bind(clusterId, name) + ) + collector.start() + byCluster.set(name, collector) + } + } + + public startCollectingFor(clusterId: ClusterId): () => void { + if (this.createdCollectors.has(clusterId)) { + console.log(`CLUSTER-META-MANAGER: already collecting for cluster ID ${clusterId}`) + } else { + const collectorsForCluster = new Map() + + for (const [name, CollectorType] of this.registeredCollectors) { + const collector = new CollectorType( + clusterId, + this.onCollectionSuccess.bind(clusterId, name), + this.onCollectionError.bind(clusterId, name) + ) + collector.start() + collectorsForCluster.set(name, collector) + } + + this.createdCollectors.set(clusterId, collectorsForCluster) + } + + return () => { + if (!this.createdCollectors.has(clusterId)) { + console.log(`CLUSTER-META-MANAGER: already stopped collecting for cluster ID ${clusterId}`) + return + } + + for (const collector of this.createdCollectors.get(clusterId).values()) { + collector.stop() + } + + this.createdCollectors.delete(clusterId) + } + } + + @autobind() + private onCollectionSuccess(clusterId: ClusterId, metadataName: string, value: any): void { + clusterMetaStore.updateMetadataValue(clusterId, metadataName, value) + } + + @autobind() + private onCollectionError(clusterId: ClusterId, metadataName: string, err: string): void { + clusterMetaStore.updateMetadataError(clusterId, metadataName, err) + } + + /** + * deregisterCollector removes the currently registered collector and stops + * all instances from collecting + * + * @param name the name of the metadata collector to stop collecting + * + * @throws if `name` isn't the name of a currently registered metadata collector + */ + public deregisterCollector(name: string) { + if (!this.registeredCollectors.has(name)) { + throw new Error(`No collector for ${name} has been registered`) + } + + /** + * stop all collectors by for metadata `name` + */ + for (const byCluster of this.createdCollectors.values()) { + byCluster.get(name).stop() + byCluster.delete(name) + } + + this.registeredCollectors.delete(name) + } + + /** + * setCollectionPeriod updates the time between collection starts and + * optionally starts one immediately + * + * @param newPeriod the new number of milliseconds between interval + * fires + * @param fireImmediately if true and `newPeriod` implies the interval would + * have already fired, then fire the interval + */ + public setCollectionPeriod(newPeriod: number, fireImmediately = true) { + this.collectionPeriod = newPeriod + + clearInterval(this.interval) + setInterval(this.onInterval, this.collectionPeriod) + + if (fireImmediately && (this.lastInterval + newPeriod) >= Date.now()) { + // the last time the interval fired is more than `newPeriod` in the past + // then fire immediately + this.onInterval() + } + } +} + +export const clusterMetaManager = ClusterMetaManager.getInstance() diff --git a/src/common/cluster-meta-store.ts b/src/common/cluster-meta-store.ts new file mode 100644 index 0000000000..c62e18b6b0 --- /dev/null +++ b/src/common/cluster-meta-store.ts @@ -0,0 +1,131 @@ +import { BaseStore } from "./base-store"; +import { ClusterId } from "./cluster-store"; +import migrations from "../migrations/cluster-meta-store" +import { action, computed, observable, toJS } from "mobx"; + +interface ClusterMetadataModel { + reportingTime: string; + value?: any; + error?: string; +} + +export interface ClusterMetaStoreModel { + metadata?: Record +} + +export interface ClusterMetaData { + [name: string]: MetadataContainer; +} + +export interface MetadataContainer { + reportingTime: Date; + value?: any; + error?: string; +} + +export class ClusterMetaStore extends BaseStore { + @observable metadata = observable.map() + + private constructor() { + super({ + configName: "lens-cluster-meta-store", + accessPropertiesByDotNotation: false, // To make dots safe in cluster context names + migrations: migrations, + }); + } + + private updateMetadata(clusterId: ClusterId, name: string, report: MetadataContainer) { + if (!this.metadata.has(clusterId)) { + this.metadata.set(clusterId, {}) + } + + this.metadata.get(clusterId)[name] = report + } + + public updateMetadataValue(clusterId: ClusterId, name: string, value: any) { + this.updateMetadata(clusterId, name, { + reportingTime: new Date(), + value, + }) + } + + public updateMetadataError(clusterId: ClusterId, name: string, error: string) { + this.updateMetadata(clusterId, name, { + reportingTime: new Date(), + error, + }) + } + + @action + protected fromStore({ metadata = {} }: ClusterMetaStoreModel = {}): void { + const updatedMetadata = this.metadata.toJS() + + for (const [curCluster, curCollected] of this.metadata) { + if (!(curCluster in metadata)) { + continue + } + + const prevCollected = metadata[curCluster] + delete metadata[curCluster] + + for (const [name, curValue] of Object.entries(curCollected)) { + if (!(name in prevCollected)) { + continue + } + + const prevValue = prevCollected[name] + delete prevCollected[name] + + const prevReportingTime = new Date(prevValue.reportingTime) + if (prevReportingTime.getTime() >= curValue.reportingTime.getTime()) { + updatedMetadata.get(curCluster)[name] = { + ...prevValue, + reportingTime: prevReportingTime, + } + } + } + + for (const [newName, newValue] of Object.entries(prevCollected)) { + const newReportingTime = new Date(newValue.reportingTime) + updatedMetadata.get(curCluster)[newName] = { + ...newValue, + reportingTime: newReportingTime, + } + } + } + + for (const [newCluster, newCollected] of Object.entries(metadata)) { + updatedMetadata.set(newCluster, {}) + + for (const [newName, newValue] of Object.entries(newCollected)) { + const newReportingTime = new Date(newValue.reportingTime) + updatedMetadata.get(newCluster)[newName] = { + ...newValue, + reportingTime: newReportingTime, + } + } + } + + this.metadata = observable.map(updatedMetadata) + } + + toJSON() { + const metadata: ClusterMetaStoreModel["metadata"] = {} + for (const [clusterId, collected] of this.metadata) { + const converted: Record = {} + for (const [name, value] of Object.entries(collected)) { + converted[name] = { + ...value, + reportingTime: value.reportingTime.toISOString(), + } + } + metadata[clusterId] = converted + } + + return toJS({ metadata, }, { recurseEverything: true }) + } +} + +export const clusterMetaStore = ClusterMetaStore.getInstance() diff --git a/src/common/cluster-store.ts b/src/common/cluster-store.ts index a382006cdc..42721c2973 100644 --- a/src/common/cluster-store.ts +++ b/src/common/cluster-store.ts @@ -1,5 +1,5 @@ import path from "path"; -import { app, ipcRenderer, remote, webFrame, webContents } from "electron"; +import { app, ipcRenderer, remote, webFrame } from "electron"; import { unlink } from "fs-extra"; import { action, computed, observable, toJS } from "mobx"; import { BaseStore } from "./base-store"; diff --git a/src/common/meta-collectors/distribution.ts b/src/common/meta-collectors/distribution.ts new file mode 100644 index 0000000000..fbfb488fb7 --- /dev/null +++ b/src/common/meta-collectors/distribution.ts @@ -0,0 +1,16 @@ +import { ClusterMetaCollector } from "../cluster-meta-manager"; +import { ClusterId } from "../cluster-store"; + +export class Distribution extends ClusterMetaCollector { + constructor(protected clusterId: ClusterId, protected onSuccess: (result: any) => void, protected onError: (err: string) => void) { + super() + } + + start(): void { + // TODO + } + + stop(): void { + // TODO + } +} diff --git a/src/common/system-ca.ts b/src/common/system-ca.ts index 957f374785..4b78f4eb26 100644 --- a/src/common/system-ca.ts +++ b/src/common/system-ca.ts @@ -1,14 +1,14 @@ import { isMac, isWindows } from "./vars"; -import winca from "win-ca" -import macca from "mac-ca" +import winCa from "win-ca" +import macCa from "mac-ca" import logger from "../main/logger" if (isMac) { - for (const crt of macca.all()) { + for (const crt of macCa.all()) { const attributes = crt.issuer?.attributes?.map((a: any) => `${a.name}=${a.value}`) logger.debug("Using host CA: " + attributes.join(",")) } } if (isWindows) { - winca.inject("+") // see: https://github.com/ukoloff/win-ca#caveats + winCa.inject("+") // see: https://github.com/ukoloff/win-ca#caveats } diff --git a/src/main/cluster.ts b/src/main/cluster.ts index 9c4afd71d0..71c55395b1 100644 --- a/src/main/cluster.ts +++ b/src/main/cluster.ts @@ -14,6 +14,7 @@ import { getFeatures, installFeature, uninstallFeature, upgradeFeature } from ". import request, { RequestPromiseOptions } from "request-promise-native" import { apiResources } from "../common/rbac"; import logger from "./logger" +import { clusterMetaManager } from "../common/cluster-meta-manager"; export enum ClusterStatus { AccessGranted = 2, @@ -115,6 +116,7 @@ export class Cluster implements ClusterModel { this.eventDisposers.push( reaction(this.getState, this.pushState), + clusterMetaManager.startCollectingFor(this.id), () => clearInterval(refreshTimer), ); } diff --git a/src/main/index.ts b/src/main/index.ts index e4fd246467..1c29c46b53 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -18,6 +18,7 @@ import { userStore } from "../common/user-store"; import { workspaceStore } from "../common/workspace-store"; import { tracker } from "../common/tracker"; import logger from "./logger" +import { registerCollectors } from "./register-collectors"; const workingDir = path.join(app.getPath("appData"), appName); app.setName(appName); @@ -44,6 +45,8 @@ async function main() { registerFileProtocol("static", __static); + registerCollectors() + // find free port let proxyPort: number try { diff --git a/src/main/register-collectors.ts b/src/main/register-collectors.ts new file mode 100644 index 0000000000..84bcdcc5fc --- /dev/null +++ b/src/main/register-collectors.ts @@ -0,0 +1,6 @@ +import { clusterMetaManager } from "../common/cluster-meta-manager"; +import { Distribution } from "../common/meta-collectors/distribution"; + +export function registerCollectors() { + clusterMetaManager.registerCollector("distribution", Distribution) +} diff --git a/src/migrations/cluster-meta-store/index.ts b/src/migrations/cluster-meta-store/index.ts new file mode 100644 index 0000000000..b1c6ea436a --- /dev/null +++ b/src/migrations/cluster-meta-store/index.ts @@ -0,0 +1 @@ +export default {}