Implement base metadata gathering system

Signed-off-by: Sebastian Malton <sebastian@malton.name>

src/common/base-store.ts

@@ -9,12 +9,12 @@ import logger from "../main/logger";
 import { broadcastIpc, IpcBroadcastParams } from "./ipc";
 import isEqual from "lodash/isEqual";
 
-export interface BaseStoreParams<T = any> extends ConfOptions<T> {
+export interface BaseStoreParams<T> extends ConfOptions<T> {
   autoLoad?: boolean;
   syncEnabled?: boolean;
 }
 
-export class BaseStore<T = any> extends Singleton {
+export abstract class BaseStore<T> extends Singleton {
   protected storeConfig: Config<T>;
   protected syncDisposers: Function[] = [];
 
@@ -22,7 +22,7 @@ export class BaseStore<T = any> extends Singleton {
   @observable isLoaded = false;
   @observable protected data: T;
 
-  protected constructor(protected params: BaseStoreParams) {
+  protected constructor(protected params: BaseStoreParams<T>) {
     super();
     this.params = {
       autoLoad: false,
@@ -157,10 +157,7 @@ export class BaseStore<T = any> extends Singleton {
     return subFrames;
   }
 
-  @action
+  protected abstract fromStore(data: T): void;
-  protected fromStore(data: T) {
-    this.data = data;
-  }
 
   // todo: use "serializr" ?
   toJSON(): T {

src/common/cluster-meta-manager.ts

@@ -0,0 +1,196 @@
+import { autobind } from "../renderer/utils";
+import { clusterMetaStore } from "./cluster-meta-store";
+import { ClusterId } from "./cluster-store";
+import Singleton from "./utils/singleton";
+
+export abstract class ClusterMetaCollector {
+  /**
+   * start tells the collector to start collecting its metadata once.
+   *
+   *  - If finished collecting last time (either producing a value or error)
+   *    then start to collect again
+   *  - If still collected since last time then should continue and **not**
+   *    restart
+   */
+  abstract start(): void;
+
+  /**
+   * stop tells the collector to stop collecting its metadata. If `start()` is
+   * called after `stop()` then the collector should begin completely fresh.
+   *
+   * Should not throw if called multiple times.
+   */
+  abstract stop(): void;
+}
+
+/**
+ * This is the constructor for a type that extends the abstract class
+ * `ClusterMetaCollector`
+ *
+ * @param clusterId the ID of the cluster that the collector should be targeting
+ * @param onSuccess the function that should be called when the collector has
+ *                  collected its metadata successfully
+ * @param onError   the function that should be called when the collector
+ *                  encounters an error during the collection process
+ */
+export type MetadataConstructor<T extends ClusterMetaCollector> = new (clusterId: ClusterId, onSuccess: (result: any) => void, onError: (err: string) => void) => T;
+
+export class ClusterMetaManager extends Singleton {
+  /**
+   * registeredCollectors is a mapping between the name of the metadata and the
+   * means of creating new collectors when new clusters are activated.
+   */
+  protected registeredCollectors = new Map<ClusterId, MetadataConstructor<ClusterMetaCollector>>();
+
+  /**
+   * createdCollectors is the mapping between clusters and those collectors
+   * targeting that cluster. Each are stored so that it can be easily iterated
+   * over by name of metadata to be collected and by cluster ID
+   */
+  protected createdCollectors = new Map<ClusterId, Map<string, ClusterMetaCollector>>();
+
+  protected interval: NodeJS.Timeout
+
+  // the milliseconds since 1970 when the last interval fired
+  protected lastInterval = Date.now()
+
+  protected constructor(protected collectionPeriod = 10 * 1000) {
+    super()
+
+    this.interval = setInterval(this.onInterval, this.collectionPeriod)
+  }
+
+  @autobind()
+  protected onInterval() {
+    this.lastInterval = Date.now()
+
+    for (const byCluster of this.createdCollectors.values()) {
+      for (const collector of byCluster.values()) {
+        collector.start()
+      }
+    }
+  }
+
+  /**
+   * registerCollector adds the collector to the list of current collectors and
+   * starts collection of metadata on all currently active clusters
+   *
+   * @param name      is the name of the metadata to be collected, will be the
+   *                  name of the field in the metadata store
+   * @param Collector the type of the collector. This is so that the manager
+   *                  can create new instances on demand
+   *
+   * @throws if `name` has already been registered
+   */
+  public registerCollector<T extends ClusterMetaCollector>(name: string, CollectorType: MetadataConstructor<T>) {
+    if (this.registeredCollectors.has(name)) {
+      throw new Error(`A collector for ${name} has already been registered`)
+    }
+
+    this.registeredCollectors.set(name, CollectorType)
+
+    /**
+     * add the collector to all the clusters currently been collected on
+     */
+    for (const [clusterId, byCluster] of this.createdCollectors) {
+      const collector = new CollectorType(
+        clusterId,
+        this.onCollectionSuccess.bind(clusterId, name),
+        this.onCollectionError.bind(clusterId, name)
+      )
+      collector.start()
+      byCluster.set(name, collector)
+    }
+  }
+
+  public startCollectingFor(clusterId: ClusterId): () => void {
+    if (this.createdCollectors.has(clusterId)) {
+      console.log(`CLUSTER-META-MANAGER: already collecting for cluster ID ${clusterId}`)
+    } else {
+      const collectorsForCluster = new Map()
+
+      for (const [name, CollectorType] of this.registeredCollectors) {
+        const collector = new CollectorType(
+          clusterId,
+          this.onCollectionSuccess.bind(clusterId, name),
+          this.onCollectionError.bind(clusterId, name)
+        )
+        collector.start()
+        collectorsForCluster.set(name, collector)
+      }
+
+      this.createdCollectors.set(clusterId, collectorsForCluster)
+    }
+
+    return () => {
+      if (!this.createdCollectors.has(clusterId)) {
+        console.log(`CLUSTER-META-MANAGER: already stopped collecting for cluster ID ${clusterId}`)
+        return
+      }
+
+      for (const collector of this.createdCollectors.get(clusterId).values()) {
+        collector.stop()
+      }
+
+      this.createdCollectors.delete(clusterId)
+    }
+  }
+
+  @autobind()
+  private onCollectionSuccess(clusterId: ClusterId, metadataName: string, value: any): void {
+    clusterMetaStore.updateMetadataValue(clusterId, metadataName, value)
+  }
+
+  @autobind()
+  private onCollectionError(clusterId: ClusterId, metadataName: string, err: string): void {
+    clusterMetaStore.updateMetadataError(clusterId, metadataName, err)
+  }
+
+  /**
+   * deregisterCollector removes the currently registered collector and stops
+   * all instances from collecting
+   *
+   * @param name the name of the metadata collector to stop collecting
+   *
+   * @throws if `name` isn't the name of a currently registered metadata collector
+   */
+  public deregisterCollector(name: string) {
+    if (!this.registeredCollectors.has(name)) {
+      throw new Error(`No collector for ${name} has been registered`)
+    }
+
+    /**
+     * stop all collectors by for metadata `name`
+     */
+    for (const byCluster of this.createdCollectors.values()) {
+      byCluster.get(name).stop()
+      byCluster.delete(name)
+    }
+
+    this.registeredCollectors.delete(name)
+  }
+
+  /**
+   * setCollectionPeriod updates the time between collection starts and
+   * optionally starts one immediately
+   *
+   * @param newPeriod       the new number of milliseconds between interval
+   *                        fires
+   * @param fireImmediately if true and `newPeriod` implies the interval would
+   *                        have already fired, then fire the interval
+   */
+  public setCollectionPeriod(newPeriod: number, fireImmediately = true) {
+    this.collectionPeriod = newPeriod
+
+    clearInterval(this.interval)
+    setInterval(this.onInterval, this.collectionPeriod)
+
+    if (fireImmediately && (this.lastInterval + newPeriod) >= Date.now()) {
+      // the last time the interval fired is more than `newPeriod` in the past
+      // then fire immediately
+      this.onInterval()
+    }
+  }
+}
+
+export const clusterMetaManager = ClusterMetaManager.getInstance<ClusterMetaManager>()

src/common/cluster-meta-store.ts

@@ -0,0 +1,131 @@
+import { BaseStore } from "./base-store";
+import { ClusterId } from "./cluster-store";
+import migrations from "../migrations/cluster-meta-store"
+import { action, computed, observable, toJS } from "mobx";
+
+interface ClusterMetadataModel {
+  reportingTime: string;
+  value?: any;
+  error?: string;
+}
+
+export interface ClusterMetaStoreModel {
+  metadata?: Record<ClusterId, {
+    [name: string]: ClusterMetadataModel
+  }>
+}
+
+export interface ClusterMetaData {
+  [name: string]: MetadataContainer;
+}
+
+export interface MetadataContainer {
+  reportingTime: Date;
+  value?: any;
+  error?: string;
+}
+
+export class ClusterMetaStore extends BaseStore<ClusterMetaStoreModel> {
+  @observable metadata = observable.map<ClusterId, ClusterMetaData>()
+
+  private constructor() {
+    super({
+      configName: "lens-cluster-meta-store",
+      accessPropertiesByDotNotation: false, // To make dots safe in cluster context names
+      migrations: migrations,
+    });
+  }
+
+  private updateMetadata(clusterId: ClusterId, name: string, report: MetadataContainer) {
+    if (!this.metadata.has(clusterId)) {
+      this.metadata.set(clusterId, {})
+    }
+
+    this.metadata.get(clusterId)[name] = report
+  }
+
+  public updateMetadataValue(clusterId: ClusterId, name: string, value: any) {
+    this.updateMetadata(clusterId, name, {
+      reportingTime: new Date(),
+      value,
+    })
+  }
+
+  public updateMetadataError(clusterId: ClusterId, name: string, error: string) {
+    this.updateMetadata(clusterId, name, {
+      reportingTime: new Date(),
+      error,
+    })
+  }
+
+  @action
+  protected fromStore({ metadata = {} }: ClusterMetaStoreModel = {}): void {
+    const updatedMetadata = this.metadata.toJS()
+
+    for (const [curCluster, curCollected] of this.metadata) {
+      if (!(curCluster in metadata)) {
+        continue
+      }
+
+      const prevCollected = metadata[curCluster]
+      delete metadata[curCluster]
+
+      for (const [name, curValue] of Object.entries(curCollected)) {
+        if (!(name in prevCollected)) {
+          continue
+        }
+
+        const prevValue = prevCollected[name]
+        delete prevCollected[name]
+
+        const prevReportingTime = new Date(prevValue.reportingTime)
+        if (prevReportingTime.getTime() >= curValue.reportingTime.getTime()) {
+          updatedMetadata.get(curCluster)[name] = {
+            ...prevValue,
+            reportingTime: prevReportingTime,
+          }
+        }
+      }
+
+      for (const [newName, newValue] of Object.entries(prevCollected)) {
+        const newReportingTime = new Date(newValue.reportingTime)
+        updatedMetadata.get(curCluster)[newName] = {
+          ...newValue,
+          reportingTime: newReportingTime,
+        }
+      }
+    }
+
+    for (const [newCluster, newCollected] of Object.entries(metadata)) {
+      updatedMetadata.set(newCluster, {})
+
+      for (const [newName, newValue] of Object.entries(newCollected)) {
+        const newReportingTime = new Date(newValue.reportingTime)
+        updatedMetadata.get(newCluster)[newName] = {
+          ...newValue,
+          reportingTime: newReportingTime,
+        }
+      }
+    }
+
+    this.metadata = observable.map(updatedMetadata)
+  }
+
+  toJSON() {
+    const metadata: ClusterMetaStoreModel["metadata"] = {}
+    for (const [clusterId, collected] of this.metadata) {
+      const converted: Record<string, ClusterMetadataModel> = {}
+      for (const [name, value] of Object.entries(collected)) {
+        converted[name] = {
+          ...value,
+          reportingTime: value.reportingTime.toISOString(),
+        }
+      }
+      metadata[clusterId] = converted
+    }
+
+    return toJS({ metadata, }, { recurseEverything: true })
+  }
+}
+
+export const clusterMetaStore = ClusterMetaStore.getInstance<ClusterMetaStore>()

src/common/cluster-store.ts

@@ -1,5 +1,5 @@
 import path from "path";
-import { app, ipcRenderer, remote, webFrame, webContents } from "electron";
+import { app, ipcRenderer, remote, webFrame } from "electron";
 import { unlink } from "fs-extra";
 import { action, computed, observable, toJS } from "mobx";
 import { BaseStore } from "./base-store";

src/common/meta-collectors/distribution.ts

@@ -0,0 +1,16 @@
+import { ClusterMetaCollector } from "../cluster-meta-manager";
+import { ClusterId } from "../cluster-store";
+
+export class Distribution extends ClusterMetaCollector {
+  constructor(protected clusterId: ClusterId, protected onSuccess: (result: any) => void, protected onError: (err: string) => void) {
+    super()
+  }
+
+  start(): void {
+    // TODO
+  }
+
+  stop(): void {
+    // TODO
+  }
+}

src/common/system-ca.ts

@@ -1,14 +1,14 @@
 import { isMac, isWindows } from "./vars";
-import winca from "win-ca"
+import winCa from "win-ca"
-import macca from "mac-ca"
+import macCa from "mac-ca"
 import logger from "../main/logger"
 
 if (isMac) {
-  for (const crt of macca.all()) {
+  for (const crt of macCa.all()) {
     const attributes = crt.issuer?.attributes?.map((a: any) => `${a.name}=${a.value}`)
     logger.debug("Using host CA: " + attributes.join(","))
   }
 }
 if (isWindows) {
-  winca.inject("+") // see: https://github.com/ukoloff/win-ca#caveats
+  winCa.inject("+") // see: https://github.com/ukoloff/win-ca#caveats
 }

src/main/cluster.ts

@@ -14,6 +14,7 @@ import { getFeatures, installFeature, uninstallFeature, upgradeFeature } from ".
 import request, { RequestPromiseOptions } from "request-promise-native"
 import { apiResources } from "../common/rbac";
 import logger from "./logger"
+import { clusterMetaManager } from "../common/cluster-meta-manager";
 
 export enum ClusterStatus {
   AccessGranted = 2,
@@ -115,6 +116,7 @@ export class Cluster implements ClusterModel {
 
     this.eventDisposers.push(
       reaction(this.getState, this.pushState),
+      clusterMetaManager.startCollectingFor(this.id),
       () => clearInterval(refreshTimer),
     );
   }

src/main/index.ts

@@ -18,6 +18,7 @@ import { userStore } from "../common/user-store";
 import { workspaceStore } from "../common/workspace-store";
 import { tracker } from "../common/tracker";
 import logger from "./logger"
+import { registerCollectors } from "./register-collectors";
 
 const workingDir = path.join(app.getPath("appData"), appName);
 app.setName(appName);
@@ -44,6 +45,8 @@ async function main() {
 
   registerFileProtocol("static", __static);
 
+  registerCollectors()
+
   // find free port
   let proxyPort: number
   try {

src/main/register-collectors.ts

@@ -0,0 +1,6 @@
+import { clusterMetaManager } from "../common/cluster-meta-manager";
+import { Distribution } from "../common/meta-collectors/distribution";
+
+export function registerCollectors() {
+  clusterMetaManager.registerCollector("distribution", Distribution)
+}

src/migrations/cluster-meta-store/index.ts

@@ -0,0 +1 @@
+export default {}