diff --git a/src/common/cluster-store/get-by-id.injectable.ts b/src/common/cluster-store/get-by-id.injectable.ts new file mode 100644 index 0000000000..534bdb5e76 --- /dev/null +++ b/src/common/cluster-store/get-by-id.injectable.ts @@ -0,0 +1,21 @@ +/** + * Copyright (c) OpenLens Authors. All rights reserved. + * Licensed under MIT License. See LICENSE in root directory for more information. + */ +import { getInjectable } from "@ogre-tools/injectable"; +import type { ClusterId } from "../cluster-types"; +import type { Cluster } from "../cluster/cluster"; +import clusterStoreInjectable from "./cluster-store.injectable"; + +export type GetClusterById = (id: ClusterId) => Cluster | undefined; + +const getClusterByIdInjectable = getInjectable({ + id: "get-cluster-by-id", + instantiate: (di): GetClusterById => { + const store = di.inject(clusterStoreInjectable); + + return (id) => store.getById(id); + }, +}); + +export default getClusterByIdInjectable; diff --git a/src/common/fs/watch.injectable.ts b/src/common/fs/watch.injectable.ts new file mode 100644 index 0000000000..eddee2da2e --- /dev/null +++ b/src/common/fs/watch.injectable.ts @@ -0,0 +1,14 @@ +/** + * Copyright (c) OpenLens Authors. All rights reserved. + * Licensed under MIT License. See LICENSE in root directory for more information. + */ +import { getInjectable } from "@ogre-tools/injectable"; +import { watch } from "chokidar"; + +const watchInjectable = getInjectable({ + id: "watch", + instantiate: () => watch, + causesSideEffects: true, +}); + +export default watchInjectable; diff --git a/src/common/logger/child-logger.injectable.ts b/src/common/logger/child-logger.injectable.ts new file mode 100644 index 0000000000..f0e2fa07c8 --- /dev/null +++ b/src/common/logger/child-logger.injectable.ts @@ -0,0 +1,37 @@ +/** + * Copyright (c) OpenLens Authors. All rights reserved. + * Licensed under MIT License. See LICENSE in root directory for more information. + */ +import { getInjectable, lifecycleEnum } from "@ogre-tools/injectable"; +import type { Logger } from "../logger"; +import loggerInjectable from "../logger.injectable"; + +const childLoggerInjectable = getInjectable({ + id: "child-logger", + instantiate: (di, prefix): Logger => { + const logger = di.inject(loggerInjectable); + + return { + debug: (message, ...args) => { + logger.debug(`[${prefix}]: ${message}`, ...args); + }, + error: (message, ...args) => { + logger.error(`[${prefix}]: ${message}`, ...args); + }, + info: (message, ...args) => { + logger.info(`[${prefix}]: ${message}`, ...args); + }, + silly: (message, ...args) => { + logger.silly(`[${prefix}]: ${message}`, ...args); + }, + warn: (message, ...args) => { + logger.warn(`[${prefix}]: ${message}`, ...args); + }, + }; + }, + lifecycle: lifecycleEnum.keyedSingleton({ + getInstanceKey: (di, prefix: string) => prefix, + }), +}); + +export default childLoggerInjectable; diff --git a/src/common/user-store/kubeconfig-syncs.injectable.ts b/src/common/user-store/kubeconfig-syncs.injectable.ts new file mode 100644 index 0000000000..bbe02fffad --- /dev/null +++ b/src/common/user-store/kubeconfig-syncs.injectable.ts @@ -0,0 +1,17 @@ +/** + * Copyright (c) OpenLens Authors. All rights reserved. + * Licensed under MIT License. See LICENSE in root directory for more information. + */ +import { getInjectable } from "@ogre-tools/injectable"; +import userStoreInjectable from "./user-store.injectable"; + +const kubeconfigSyncsInjectable = getInjectable({ + id: "kubeconfig-syncs", + instantiate: (di) => { + const store = di.inject(userStoreInjectable); + + return store.syncKubeconfigEntries; + }, +}); + +export default kubeconfigSyncsInjectable; diff --git a/src/common/utils/iter.ts b/src/common/utils/iter.ts index 05d097ed78..9752e56dc5 100644 --- a/src/common/utils/iter.ts +++ b/src/common/utils/iter.ts @@ -11,6 +11,7 @@ interface Iterator { find(fn: (val: T) => unknown): T | undefined; collect(fn: (values: Iterable) => U): U; map(fn: (val: T) => U): Iterator; + flatMap(fn: (val: T) => U[]): Iterator; join(sep?: string): string; } @@ -19,6 +20,7 @@ export function pipeline(src: IterableIterator): Iterator { filter: (fn) => pipeline(filter(src, fn)), filterMap: (fn) => pipeline(filterMap(src, fn)), map: (fn) => pipeline(map(src, fn)), + flatMap: (fn) => pipeline(flatMap(src, fn)), find: (fn) => find(src, fn), join: (sep) => join(src, sep), collect: (fn) => fn(src), diff --git a/src/main/catalog-sources/__test__/kubeconfig-sync.test.ts b/src/main/catalog-sources/__test__/kubeconfig-sync.test.ts index 08ff464bf2..767eb8d088 100644 --- a/src/main/catalog-sources/__test__/kubeconfig-sync.test.ts +++ b/src/main/catalog-sources/__test__/kubeconfig-sync.test.ts @@ -3,19 +3,15 @@ * Licensed under MIT License. See LICENSE in root directory for more information. */ -import { ObservableMap } from "mobx"; +import { observable, ObservableMap, when } from "mobx"; import type { CatalogEntity } from "../../../common/catalog"; import { loadFromOptions } from "../../../common/kube-helpers"; import type { Cluster } from "../../../common/cluster/cluster"; -import { computeDiff as computeDiffFor, configToModels } from "../kubeconfig-sync/manager"; import mockFs from "mock-fs"; import fs from "fs"; import clusterStoreInjectable from "../../../common/cluster-store/cluster-store.injectable"; import { getDiForUnitTesting } from "../../getDiForUnitTesting"; -import { createClusterInjectionToken } from "../../../common/cluster/create-cluster-injection-token"; -import directoryForKubeConfigsInjectable from "../../../common/app-paths/directory-for-kube-configs/directory-for-kube-configs.injectable"; import getConfigurationFileModelInjectable from "../../../common/get-configuration-file-model/get-configuration-file-model.injectable"; -import clusterManagerInjectable from "../../cluster-manager.injectable"; import directoryForUserDataInjectable from "../../../common/app-paths/directory-for-user-data/directory-for-user-data.injectable"; import directoryForTempInjectable from "../../../common/app-paths/directory-for-temp/directory-for-temp.injectable"; import kubectlBinaryNameInjectable from "../../kubectl/binary-name.injectable"; @@ -23,6 +19,17 @@ import kubectlDownloadingNormalizedArchInjectable from "../../kubectl/normalized import normalizedPlatformInjectable from "../../../common/vars/normalized-platform.injectable"; import { iter } from "../../../common/utils"; import fsInjectable from "../../../common/fs/fs.injectable"; +import type { ComputeKubeconfigDiff } from "../kubeconfig-sync/compute-diff.injectable"; +import computeKubeconfigDiffInjectable from "../kubeconfig-sync/compute-diff.injectable"; +import watchInjectable from "../../../common/fs/watch.injectable"; +import type { ConfigToModels } from "../kubeconfig-sync/config-to-models.injectable"; +import configToModelsInjectable from "../kubeconfig-sync/config-to-models.injectable"; +import kubeconfigSyncManagerInjectable from "../kubeconfig-sync/manager.injectable"; +import type { KubeconfigSyncManager } from "../kubeconfig-sync/manager"; +import type { KubeconfigSyncValue } from "../../../common/user-store"; +import kubeconfigSyncsInjectable from "../../../common/user-store/kubeconfig-syncs.injectable"; + +console.log("This is a reminder that mockFS breaks things and needs to be removed"); jest.mock("electron", () => ({ app: { @@ -41,7 +48,10 @@ jest.mock("electron", () => ({ })); describe("kubeconfig-sync.source tests", () => { - let computeDiff: ReturnType; + let computeKubeconfigDiff: ComputeKubeconfigDiff; + let configToModels: ConfigToModels; + let manager: KubeconfigSyncManager; + let kubeconfigSyncs: ObservableMap; beforeEach(async () => { const di = getDiForUnitTesting({ doGeneralOverrides: true }); @@ -55,15 +65,18 @@ describe("kubeconfig-sync.source tests", () => { di.override(normalizedPlatformInjectable, () => "darwin"); di.permitSideEffects(fsInjectable); + di.permitSideEffects(watchInjectable); di.unoverride(clusterStoreInjectable); di.permitSideEffects(clusterStoreInjectable); di.permitSideEffects(getConfigurationFileModelInjectable); - computeDiff = computeDiffFor({ - directoryForKubeConfigs: di.inject(directoryForKubeConfigsInjectable), - createCluster: di.inject(createClusterInjectionToken), - clusterManager: di.inject(clusterManagerInjectable), - }); + kubeconfigSyncs = observable.map(); + + di.override(kubeconfigSyncsInjectable, () => kubeconfigSyncs); + + computeKubeconfigDiff = di.inject(computeKubeconfigDiffInjectable); + configToModels = di.inject(configToModelsInjectable); + manager = di.inject(kubeconfigSyncManagerInjectable); }); afterEach(() => { @@ -108,13 +121,13 @@ describe("kubeconfig-sync.source tests", () => { }); }); - describe("computeDiff", () => { + describe("computeKubeconfigDiff", () => { it("should leave an empty source empty if there are no entries", () => { const contents = ""; const rootSource = new ObservableMap(); const filePath = "/bar"; - computeDiff(contents, rootSource, filePath); + computeKubeconfigDiff(contents, rootSource, filePath); expect(rootSource.size).toBe(0); }); @@ -151,7 +164,7 @@ describe("kubeconfig-sync.source tests", () => { fs.writeFileSync(filePath, contents); - computeDiff(contents, rootSource, filePath); + computeKubeconfigDiff(contents, rootSource, filePath); expect(rootSource.size).toBe(1); @@ -195,7 +208,7 @@ describe("kubeconfig-sync.source tests", () => { fs.writeFileSync(filePath, contents); - computeDiff(contents, rootSource, filePath); + computeKubeconfigDiff(contents, rootSource, filePath); expect(rootSource.size).toBe(1); @@ -204,7 +217,7 @@ describe("kubeconfig-sync.source tests", () => { expect(c.kubeConfigPath).toBe("/bar"); expect(c.contextName).toBe("context-name"); - computeDiff("{}", rootSource, filePath); + computeKubeconfigDiff("{}", rootSource, filePath); expect(rootSource.size).toBe(0); }); @@ -249,7 +262,7 @@ describe("kubeconfig-sync.source tests", () => { fs.writeFileSync(filePath, contents); - computeDiff(contents, rootSource, filePath); + computeKubeconfigDiff(contents, rootSource, filePath); expect(rootSource.size).toBe(2); @@ -289,7 +302,7 @@ describe("kubeconfig-sync.source tests", () => { currentContext: "foobar", }); - computeDiff(newContents, rootSource, filePath); + computeKubeconfigDiff(newContents, rootSource, filePath); expect(rootSource.size).toBe(1); @@ -301,4 +314,80 @@ describe("kubeconfig-sync.source tests", () => { } }); }); + + describe("given a config file at /foobar/config", () => { + beforeEach(() => { + fs.mkdirSync("/foobar"); + fs.writeFileSync("/foobar/config", JSON.stringify({ + clusters: [{ + name: "cluster-name", + cluster: { + server: "1.2.3.4", + }, + skipTLSVerify: false, + }], + users: [{ + name: "user-name", + }], + contexts: [{ + name: "context-name", + context: { + cluster: "cluster-name", + user: "user-name", + }, + }, { + name: "context-the-second", + context: { + cluster: "missing-cluster", + user: "user-name", + }, + }], + currentContext: "foobar", + })); + }); + + it("should not find any entities", () => { + expect(manager.source.get()).toEqual([]); + }); + + describe("when sync has started", () => { + beforeEach(() => { + manager.startSync(); + }); + + it("should not find any entities", () => { + expect(manager.source.get()).toEqual([]); + }); + + describe("when a file sync target for /foobar/config is added", () => { + beforeEach(() => { + kubeconfigSyncs.set("/foobar/config", {}); + }); + + it("should find a single entity", (done) => { + when(() => manager.source.get().length === 1, () => done()); + }); + + describe("when a folder sync target for /foobar is added", () => { + beforeEach(() => { + kubeconfigSyncs.set("/foobar", {}); + }); + + it("should still only find a single entity", (done) => { + when(() => manager.source.get().length === 1, () => done()); + }); + }); + }); + + describe("when a folder sync target for /foobar is added", () => { + beforeEach(() => { + kubeconfigSyncs.set("/foobar", {}); + }); + + it("should find a single entity", (done) => { + when(() => manager.source.get().length === 1, () => done()); + }); + }); + }); + }); }); diff --git a/src/main/catalog-sources/kubeconfig-sync/compute-diff.injectable.ts b/src/main/catalog-sources/kubeconfig-sync/compute-diff.injectable.ts new file mode 100644 index 0000000000..31745d116f --- /dev/null +++ b/src/main/catalog-sources/kubeconfig-sync/compute-diff.injectable.ts @@ -0,0 +1,100 @@ +/** + * Copyright (c) OpenLens Authors. All rights reserved. + * Licensed under MIT License. See LICENSE in root directory for more information. + */ +import { getInjectable } from "@ogre-tools/injectable"; +import { createHash } from "crypto"; +import type { ObservableMap } from "mobx"; +import { action } from "mobx"; +import { homedir } from "os"; +import directoryForKubeConfigsInjectable from "../../../common/app-paths/directory-for-kube-configs/directory-for-kube-configs.injectable"; +import type { CatalogEntity } from "../../../common/catalog"; +import getClusterByIdInjectable from "../../../common/cluster-store/get-by-id.injectable"; +import type { Cluster } from "../../../common/cluster/cluster"; +import { loadConfigFromString } from "../../../common/kube-helpers"; +import { catalogEntityFromCluster } from "../../cluster-manager"; +import clusterManagerInjectable from "../../cluster-manager.injectable"; +import createClusterInjectable from "../../create-cluster/create-cluster.injectable"; +import configToModelsInjectable from "./config-to-models.injectable"; +import kubeconfigSyncLoggerInjectable from "./logger.injectable"; + +export type ComputeKubeconfigDiff = (contents: string, source: ObservableMap, filePath: string) => void; + +const computeKubeconfigDiffInjectable = getInjectable({ + id: "compute-kubeconfig-diff", + instantiate: (di): ComputeKubeconfigDiff => { + const directoryForKubeConfigs = di.inject(directoryForKubeConfigsInjectable); + const createCluster = di.inject(createClusterInjectable); + const clusterManager = di.inject(clusterManagerInjectable); + const configToModels = di.inject(configToModelsInjectable); + const logger = di.inject(kubeconfigSyncLoggerInjectable); + const getClusterById = di.inject(getClusterByIdInjectable); + + return action((contents, source, filePath) => { + try { + const { config, error } = loadConfigFromString(contents); + + if (error) { + logger.warn(`encountered errors while loading config: ${error.message}`, { filePath, details: error.details }); + } + + const rawModels = configToModels(config, filePath); + const models = new Map(rawModels.map(([model, configData]) => [model.contextName, [model, configData] as const])); + + logger.debug(`File now has ${models.size} entries`, { filePath }); + + for (const [contextName, value] of source) { + const data = models.get(contextName); + + // remove and disconnect clusters that were removed from the config + if (!data) { + // remove from the deleting set, so that if a new context of the same name is added, it isn't marked as deleting + clusterManager.deleting.delete(value[0].id); + + value[0].disconnect(); + source.delete(contextName); + logger.debug(`Removed old cluster from sync`, { filePath, contextName }); + continue; + } + + // TODO: For the update check we need to make sure that the config itself hasn't changed. + // Probably should make it so that cluster keeps a copy of the config in its memory and + // diff against that + + // or update the model and mark it as not needed to be added + value[0].updateModel(data[0]); + models.delete(contextName); + logger.debug(`Updated old cluster from sync`, { filePath, contextName }); + } + + for (const [contextName, [model, configData]] of models) { + // add new clusters to the source + try { + const clusterId = createHash("md5").update(`${filePath}:${contextName}`).digest("hex"); + const cluster = getClusterById(clusterId) ?? createCluster({ ...model, id: clusterId }, configData); + + if (!cluster.apiUrl) { + throw new Error("Cluster constructor failed, see above error"); + } + + const entity = catalogEntityFromCluster(cluster); + + if (!filePath.startsWith(directoryForKubeConfigs)) { + entity.metadata.labels.file = filePath.replace(homedir(), "~"); + } + source.set(contextName, [cluster, entity]); + + logger.debug(`Added new cluster from sync`, { filePath, contextName }); + } catch (error) { + logger.warn(`Failed to create cluster from model: ${error}`, { filePath, contextName }); + } + } + } catch (error) { + logger.warn(`Failed to compute diff: ${error}`, { filePath }); + source.clear(); // clear source if we have failed so as to not show outdated information + } + }); + }, +}); + +export default computeKubeconfigDiffInjectable; diff --git a/src/main/catalog-sources/kubeconfig-sync/config-to-models.injectable.ts b/src/main/catalog-sources/kubeconfig-sync/config-to-models.injectable.ts new file mode 100644 index 0000000000..8240d4e914 --- /dev/null +++ b/src/main/catalog-sources/kubeconfig-sync/config-to-models.injectable.ts @@ -0,0 +1,42 @@ +/** + * Copyright (c) OpenLens Authors. All rights reserved. + * Licensed under MIT License. See LICENSE in root directory for more information. + */ +import type { KubeConfig } from "@kubernetes/client-node"; +import { getInjectable } from "@ogre-tools/injectable"; +import type { ClusterConfigData, UpdateClusterModel } from "../../../common/cluster-types"; +import { splitConfig } from "../../../common/kube-helpers"; +import kubeconfigSyncLoggerInjectable from "./logger.injectable"; + +export type ConfigToModels = (rootConfig: KubeConfig, filePath: string) => [UpdateClusterModel, ClusterConfigData][]; + +const configToModelsInjectable = getInjectable({ + id: "config-to-models", + instantiate: (di): ConfigToModels => { + const logger = di.inject(kubeconfigSyncLoggerInjectable); + + return (rootConfig, filePath) => { + const validConfigs: ReturnType = []; + + for (const { config, validationResult } of splitConfig(rootConfig)) { + if (validationResult.error) { + logger.debug(`context failed validation: ${validationResult.error}`, { context: config.currentContext, filePath }); + } else { + validConfigs.push([ + { + kubeConfigPath: filePath, + contextName: config.currentContext, + }, + { + clusterServerUrl: validationResult.cluster.server, + }, + ]); + } + } + + return validConfigs; + }; + }, +}); + +export default configToModelsInjectable; diff --git a/src/main/catalog-sources/kubeconfig-sync/diff-changed-kubeconfig.injectable.ts b/src/main/catalog-sources/kubeconfig-sync/diff-changed-kubeconfig.injectable.ts new file mode 100644 index 0000000000..a35f41c028 --- /dev/null +++ b/src/main/catalog-sources/kubeconfig-sync/diff-changed-kubeconfig.injectable.ts @@ -0,0 +1,90 @@ +/** + * Copyright (c) OpenLens Authors. All rights reserved. + * Licensed under MIT License. See LICENSE in root directory for more information. + */ +import { getInjectable } from "@ogre-tools/injectable"; +import type { Stats } from "fs"; +import { constants } from "fs"; +import type { ObservableMap } from "mobx"; +import type { Readable } from "stream"; +import type { CatalogEntity } from "../../../common/catalog"; +import type { Cluster } from "../../../common/cluster/cluster"; +import fsInjectable from "../../../common/fs/fs.injectable"; +import type { Disposer } from "../../../common/utils"; +import { bytesToUnits, noop } from "../../../common/utils"; +import computeKubeconfigDiffInjectable from "./compute-diff.injectable"; +import kubeconfigSyncLoggerInjectable from "./logger.injectable"; + +export interface DiffChangedKubeconfigArgs { + filePath: string; + source: ObservableMap; + stats: Stats; + maxAllowedFileReadSize: number; +} +export type DiffChangedKubeconfig = (args: DiffChangedKubeconfigArgs) => Disposer; + +const diffChangedKubeconfigInjectable = getInjectable({ + id: "diff-changed-kubeconfig", + instantiate: (di): DiffChangedKubeconfig => { + const computeKubeconfigDiff = di.inject(computeKubeconfigDiffInjectable); + const logger = di.inject(kubeconfigSyncLoggerInjectable); + const { createReadStream } = di.inject(fsInjectable); + + return ({ filePath, maxAllowedFileReadSize, source, stats }) => { + logger.debug(`file changed`, { filePath }); + + if (stats.size >= maxAllowedFileReadSize) { + logger.warn(`skipping ${filePath}: size=${bytesToUnits(stats.size)} is larger than maxSize=${bytesToUnits(maxAllowedFileReadSize)}`); + source.clear(); + + return noop; + } + + const fileReader = createReadStream(filePath, { + mode: constants.O_RDONLY, + }); + const readStream = fileReader as Readable; + const decoder = new TextDecoder("utf-8", { fatal: true }); + let fileString = ""; + let closed = false; + + const cleanup = () => { + closed = true; + fileReader.close(); // This may not close the stream. + // Artificially marking end-of-stream, as if the underlying resource had + // indicated end-of-file by itself, allows the stream to close. + // This does not cancel pending read operations, and if there is such an + // operation, the process may still not be able to exit successfully + // until it finishes. + fileReader.push(null); + fileReader.read(0); + readStream.removeAllListeners(); + }; + + readStream + .on("data", (chunk: Buffer) => { + try { + fileString += decoder.decode(chunk, { stream: true }); + } catch (error) { + logger.warn(`skipping ${filePath}: ${error}`); + source.clear(); + cleanup(); + } + }) + .on("close", () => cleanup()) + .on("error", error => { + cleanup(); + logger.warn(`failed to read file: ${error}`, { filePath }); + }) + .on("end", () => { + if (!closed) { + computeKubeconfigDiff(fileString, source, filePath); + } + }); + + return cleanup; + }; + }, +}); + +export default diffChangedKubeconfigInjectable; diff --git a/src/main/catalog-sources/kubeconfig-sync/logger.injectable.ts b/src/main/catalog-sources/kubeconfig-sync/logger.injectable.ts new file mode 100644 index 0000000000..66a70ce42d --- /dev/null +++ b/src/main/catalog-sources/kubeconfig-sync/logger.injectable.ts @@ -0,0 +1,13 @@ +/** + * Copyright (c) OpenLens Authors. All rights reserved. + * Licensed under MIT License. See LICENSE in root directory for more information. + */ +import { getInjectable } from "@ogre-tools/injectable"; +import childLoggerInjectable from "../../../common/logger/child-logger.injectable"; + +const kubeconfigSyncLoggerInjectable = getInjectable({ + id: "kubeconfig-sync-logger", + instantiate: (di) => di.inject(childLoggerInjectable, "KUBECONFIG-SYNC"), +}); + +export default kubeconfigSyncLoggerInjectable; diff --git a/src/main/catalog-sources/kubeconfig-sync/manager.injectable.ts b/src/main/catalog-sources/kubeconfig-sync/manager.injectable.ts index f95fa0fb17..06265764c8 100644 --- a/src/main/catalog-sources/kubeconfig-sync/manager.injectable.ts +++ b/src/main/catalog-sources/kubeconfig-sync/manager.injectable.ts @@ -5,18 +5,18 @@ import { getInjectable } from "@ogre-tools/injectable"; import directoryForKubeConfigsInjectable from "../../../common/app-paths/directory-for-kube-configs/directory-for-kube-configs.injectable"; import { KubeconfigSyncManager } from "./manager"; -import { createClusterInjectionToken } from "../../../common/cluster/create-cluster-injection-token"; -import clusterManagerInjectable from "../../cluster-manager.injectable"; -import catalogEntityRegistryInjectable from "../../catalog/entity-registry.injectable"; +import kubeconfigSyncLoggerInjectable from "./logger.injectable"; +import watchKubeconfigFileChangesInjectable from "./watch-file-changes.injectable"; +import kubeconfigSyncsInjectable from "../../../common/user-store/kubeconfig-syncs.injectable"; const kubeconfigSyncManagerInjectable = getInjectable({ id: "kubeconfig-sync-manager", instantiate: (di) => new KubeconfigSyncManager({ directoryForKubeConfigs: di.inject(directoryForKubeConfigsInjectable), - createCluster: di.inject(createClusterInjectionToken), - clusterManager: di.inject(clusterManagerInjectable), - entityRegistry: di.inject(catalogEntityRegistryInjectable), + logger: di.inject(kubeconfigSyncLoggerInjectable), + watchKubeconfigFileChanges: di.inject(watchKubeconfigFileChangesInjectable), + kubeconfigSyncs: di.inject(kubeconfigSyncsInjectable), }), }); diff --git a/src/main/catalog-sources/kubeconfig-sync/manager.ts b/src/main/catalog-sources/kubeconfig-sync/manager.ts index 1d281bc936..6e284bd2ca 100644 --- a/src/main/catalog-sources/kubeconfig-sync/manager.ts +++ b/src/main/catalog-sources/kubeconfig-sync/manager.ts @@ -4,97 +4,59 @@ */ import type { IComputedValue, ObservableMap } from "mobx"; -import { action, observable, computed, runInAction, makeObservable, observe } from "mobx"; +import { action, observable, computed, makeObservable, observe } from "mobx"; import type { CatalogEntity } from "../../../common/catalog"; -import type { FSWatcher } from "chokidar"; -import { watch } from "chokidar"; -import type { Stats } from "fs"; -import fs from "fs"; -import path from "path"; import type { Disposer } from "../../../common/utils"; -import { disposer, bytesToUnits, getOrInsertWith, iter, noop } from "../../../common/utils"; -import logger from "../../logger"; -import type { KubeConfig } from "@kubernetes/client-node"; -import { loadConfigFromString, splitConfig } from "../../../common/kube-helpers"; -import type { ClusterManager } from "../../cluster-manager"; -import { catalogEntityFromCluster } from "../../cluster-manager"; -import { UserStore } from "../../../common/user-store"; -import { ClusterStore } from "../../../common/cluster-store/cluster-store"; -import { createHash } from "crypto"; -import { homedir } from "os"; -import globToRegExp from "glob-to-regexp"; -import { inspect } from "util"; -import type { ClusterConfigData, UpdateClusterModel } from "../../../common/cluster-types"; -import type { Cluster } from "../../../common/cluster/cluster"; -import type { CatalogEntityRegistry } from "../../catalog/entity-registry"; -import type { CreateCluster } from "../../../common/cluster/create-cluster-injection-token"; - -const logPrefix = "[KUBECONFIG-SYNC]:"; - -/** - * This is the list of globs of which files are ignored when under a folder sync - */ -const ignoreGlobs = [ - "*.lock", // kubectl lock files - "*.swp", // vim swap files - ".DS_Store", // macOS specific -].map(rawGlob => ({ - rawGlob, - matcher: globToRegExp(rawGlob), -})); - -/** - * This should be much larger than any kubeconfig text file - * - * Even if you have a cert-file, key-file, and client-cert files that is only - * 12kb of extra data (at 4096 bytes each) which allows for around 150 entries. - */ -const folderSyncMaxAllowedFileReadSize = 2 * 1024 * 1024; // 2 MiB -const fileSyncMaxAllowedFileReadSize = 16 * folderSyncMaxAllowedFileReadSize; // 32 MiB +import { iter } from "../../../common/utils"; +import type { KubeconfigSyncValue } from "../../../common/user-store"; +import type { Logger } from "../../../common/logger"; +import type { WatchKubeconfigFileChanges } from "./watch-file-changes.injectable"; interface KubeconfigSyncManagerDependencies { readonly directoryForKubeConfigs: string; - readonly entityRegistry: CatalogEntityRegistry; - readonly clusterManager: ClusterManager; - createCluster: CreateCluster; + readonly logger: Logger; + readonly kubeconfigSyncs: ObservableMap; + watchKubeconfigFileChanges: WatchKubeconfigFileChanges; } -const kubeConfigSyncName = "lens:kube-sync"; - export class KubeconfigSyncManager { protected readonly sources = observable.map, Disposer]>(); - protected syncing = false; protected syncListDisposer?: Disposer; constructor(protected readonly dependencies: KubeconfigSyncManagerDependencies) { makeObservable(this); } + public readonly source = computed(() => { + /** + * This prevents multiple overlapping syncs from leading to multiple entities with the same IDs + */ + const seenIds = new Set(); + + return ( + iter.pipeline(this.sources.values()) + .flatMap(([entities]) => entities.get()) + .filter(entity => ( + seenIds.has(entity.getId()) + ? false + : seenIds.add(entity.getId()) + )) + .collect(items => [...items]) + ); + }); + @action startSync(): void { - if (this.syncing) { - return; - } - - this.syncing = true; - - logger.info(`${logPrefix} starting requested syncs`); - - this.dependencies.entityRegistry.addComputedSource(kubeConfigSyncName, computed(() => ( - Array.from(iter.flatMap( - this.sources.values(), - ([entities]) => entities.get(), - )) - ))); + this.dependencies.logger.info(`starting requested syncs`); // This must be done so that c&p-ed clusters are visible this.startNewSync(this.dependencies.directoryForKubeConfigs); - for (const filePath of UserStore.getInstance().syncKubeconfigEntries.keys()) { + for (const filePath of this.dependencies.kubeconfigSyncs.keys()) { this.startNewSync(filePath); } - this.syncListDisposer = observe(UserStore.getInstance().syncKubeconfigEntries, change => { + this.syncListDisposer = observe(this.dependencies.kubeconfigSyncs, change => { switch (change.type) { case "add": this.startNewSync(change.name); @@ -108,275 +70,38 @@ export class KubeconfigSyncManager { @action stopSync() { + this.dependencies.logger.info(`stopping requested syncs`); this.syncListDisposer?.(); for (const filePath of this.sources.keys()) { this.stopOldSync(filePath); } - - this.dependencies.entityRegistry.removeSource(kubeConfigSyncName); - this.syncing = false; } @action protected startNewSync(filePath: string): void { if (this.sources.has(filePath)) { // don't start a new sync if we already have one - return void logger.debug(`${logPrefix} already syncing file/folder`, { filePath }); + return this.dependencies.logger.debug(`already syncing file/folder`, { filePath }); } this.sources.set( filePath, - watchFileChanges(filePath, this.dependencies), + this.dependencies.watchKubeconfigFileChanges(filePath), ); - logger.info(`${logPrefix} starting sync of file/folder`, { filePath }); - logger.debug(`${logPrefix} ${this.sources.size} files/folders watched`, { files: Array.from(this.sources.keys()) }); + this.dependencies.logger.info(`starting sync of file/folder`, { filePath }); + this.dependencies.logger.debug(`${this.sources.size} files/folders watched`, { files: Array.from(this.sources.keys()) }); } @action protected stopOldSync(filePath: string): void { if (!this.sources.delete(filePath)) { // already stopped - return void logger.debug(`${logPrefix} no syncing file/folder to stop`, { filePath }); + return this.dependencies.logger.debug(`no syncing file/folder to stop`, { filePath }); } - logger.info(`${logPrefix} stopping sync of file/folder`, { filePath }); - logger.debug(`${logPrefix} ${this.sources.size} files/folders watched`, { files: Array.from(this.sources.keys()) }); + this.dependencies.logger.info(`stopping sync of file/folder`, { filePath }); + this.dependencies.logger.debug(`${this.sources.size} files/folders watched`, { files: Array.from(this.sources.keys()) }); } } - -// exported for testing -export function configToModels(rootConfig: KubeConfig, filePath: string): [UpdateClusterModel, ClusterConfigData][] { - const validConfigs: ReturnType = []; - - for (const { config, validationResult } of splitConfig(rootConfig)) { - if (validationResult.error) { - logger.debug(`${logPrefix} context failed validation: ${validationResult.error}`, { context: config.currentContext, filePath }); - } else { - validConfigs.push([ - { - kubeConfigPath: filePath, - contextName: config.currentContext, - }, - { - clusterServerUrl: validationResult.cluster.server, - }, - ]); - } - } - - return validConfigs; -} - -type RootSourceValue = [Cluster, CatalogEntity]; -type RootSource = ObservableMap; - -interface ComputeDiffDependencies { - directoryForKubeConfigs: string; - createCluster: CreateCluster; - clusterManager: ClusterManager; -} - -// exported for testing -export const computeDiff = ({ directoryForKubeConfigs, createCluster, clusterManager }: ComputeDiffDependencies) => (contents: string, source: RootSource, filePath: string): void => { - runInAction(() => { - try { - const { config, error } = loadConfigFromString(contents); - - if (error) { - logger.warn(`${logPrefix} encountered errors while loading config: ${error.message}`, { filePath, details: error.details }); - } - - const rawModels = configToModels(config, filePath); - const models = new Map(rawModels.map(([model, configData]) => [model.contextName, [model, configData] as const])); - - logger.debug(`${logPrefix} File now has ${models.size} entries`, { filePath }); - - for (const [contextName, value] of source) { - const data = models.get(contextName); - - // remove and disconnect clusters that were removed from the config - if (!data) { - // remove from the deleting set, so that if a new context of the same name is added, it isn't marked as deleting - clusterManager.deleting.delete(value[0].id); - - value[0].disconnect(); - source.delete(contextName); - logger.debug(`${logPrefix} Removed old cluster from sync`, { filePath, contextName }); - continue; - } - - // TODO: For the update check we need to make sure that the config itself hasn't changed. - // Probably should make it so that cluster keeps a copy of the config in its memory and - // diff against that - - // or update the model and mark it as not needed to be added - value[0].updateModel(data[0]); - models.delete(contextName); - logger.debug(`${logPrefix} Updated old cluster from sync`, { filePath, contextName }); - } - - for (const [contextName, [model, configData]] of models) { - // add new clusters to the source - try { - const clusterId = createHash("md5").update(`${filePath}:${contextName}`).digest("hex"); - - const cluster = ClusterStore.getInstance().getById(clusterId) || createCluster({ ...model, id: clusterId }, configData); - - if (!cluster.apiUrl) { - throw new Error("Cluster constructor failed, see above error"); - } - - const entity = catalogEntityFromCluster(cluster); - - if (!filePath.startsWith(directoryForKubeConfigs)) { - entity.metadata.labels.file = filePath.replace(homedir(), "~"); - } - source.set(contextName, [cluster, entity]); - - logger.debug(`${logPrefix} Added new cluster from sync`, { filePath, contextName }); - } catch (error) { - logger.warn(`${logPrefix} Failed to create cluster from model: ${error}`, { filePath, contextName }); - } - } - } catch (error) { - logger.warn(`${logPrefix} Failed to compute diff: ${error}`, { filePath }); - source.clear(); // clear source if we have failed so as to not show outdated information - } - }); -}; - -interface DiffChangedConfigArgs { - filePath: string; - source: RootSource; - stats: fs.Stats; - maxAllowedFileReadSize: number; -} - -const diffChangedConfigFor = (dependencies: ComputeDiffDependencies) => ({ filePath, source, stats, maxAllowedFileReadSize }: DiffChangedConfigArgs): Disposer => { - logger.debug(`${logPrefix} file changed`, { filePath }); - - if (stats.size >= maxAllowedFileReadSize) { - logger.warn(`${logPrefix} skipping ${filePath}: size=${bytesToUnits(stats.size)} is larger than maxSize=${bytesToUnits(maxAllowedFileReadSize)}`); - source.clear(); - - return noop; - } - - const controller = new AbortController(); - const fileContentsP = fs.promises.readFile(filePath, { - signal: controller.signal, - }); - const cleanup = disposer( - () => controller.abort(), - ); - - fileContentsP - .then((fileData) => { - const decoder = new TextDecoder("utf-8", { fatal: true }); - - try { - const fileString = decoder.decode(fileData); - - computeDiff(dependencies)(fileString, source, filePath); - } catch (error) { - logger.warn(`${logPrefix} skipping ${filePath}: ${error}`); - source.clear(); - cleanup(); - } - }) - .catch(error => { - if (controller.signal.aborted) { - return; - } - - logger.warn(`${logPrefix} failed to read file: ${error}`, { filePath }); - cleanup(); - }); - - return cleanup; -}; - -const watchFileChanges = (filePath: string, dependencies: ComputeDiffDependencies): [IComputedValue, Disposer] => { - const rootSource = observable.map>(); - const derivedSource = computed(() => Array.from(iter.flatMap(rootSource.values(), from => iter.map(from.values(), child => child[1])))); - - let watcher: FSWatcher; - - (async () => { - try { - const stat = await fs.promises.stat(filePath); - const isFolderSync = stat.isDirectory(); - const cleanupFns = new Map(); - const maxAllowedFileReadSize = isFolderSync - ? folderSyncMaxAllowedFileReadSize - : fileSyncMaxAllowedFileReadSize; - - watcher = watch(filePath, { - followSymlinks: true, - depth: isFolderSync ? 0 : 1, // DIRs works with 0 but files need 1 (bug: https://github.com/paulmillr/chokidar/issues/1095) - disableGlobbing: true, - ignorePermissionErrors: true, - usePolling: false, - awaitWriteFinish: { - pollInterval: 100, - stabilityThreshold: 1000, - }, - atomic: 150, // for "atomic writes" - alwaysStat: true, - }); - - const diffChangedConfig = diffChangedConfigFor(dependencies); - - watcher - .on("change", (childFilePath, stats: Stats): void => { - const cleanup = cleanupFns.get(childFilePath); - - if (!cleanup) { - // file was previously ignored, do nothing - return void logger.debug(`${logPrefix} ${inspect(childFilePath)} that should have been previously ignored has changed. Doing nothing`); - } - - cleanup(); - cleanupFns.set(childFilePath, diffChangedConfig({ - filePath: childFilePath, - source: getOrInsertWith(rootSource, childFilePath, observable.map), - stats, - maxAllowedFileReadSize, - })); - }) - .on("add", (childFilePath, stats: Stats): void => { - if (isFolderSync) { - const fileName = path.basename(childFilePath); - - for (const ignoreGlob of ignoreGlobs) { - if (ignoreGlob.matcher.test(fileName)) { - return void logger.info(`${logPrefix} ignoring ${inspect(childFilePath)} due to ignore glob: ${ignoreGlob.rawGlob}`); - } - } - } - - cleanupFns.set(childFilePath, diffChangedConfig({ - filePath: childFilePath, - source: getOrInsertWith(rootSource, childFilePath, observable.map), - stats, - maxAllowedFileReadSize, - })); - }) - .on("unlink", (childFilePath) => { - cleanupFns.get(childFilePath)?.(); - cleanupFns.delete(childFilePath); - rootSource.delete(childFilePath); - }) - .on("error", error => logger.error(`${logPrefix} watching file/folder failed: ${error}`, { filePath })); - } catch (error) { - console.log((error as { stack: unknown }).stack); - logger.warn(`${logPrefix} failed to start watching changes: ${error}`); - } - })(); - - return [derivedSource, () => { - watcher?.close(); - }]; -}; diff --git a/src/main/catalog-sources/kubeconfig-sync/watch-file-changes.injectable.ts b/src/main/catalog-sources/kubeconfig-sync/watch-file-changes.injectable.ts new file mode 100644 index 0000000000..91709afee0 --- /dev/null +++ b/src/main/catalog-sources/kubeconfig-sync/watch-file-changes.injectable.ts @@ -0,0 +1,135 @@ +/** + * Copyright (c) OpenLens Authors. All rights reserved. + * Licensed under MIT License. See LICENSE in root directory for more information. + */ +import { getInjectable } from "@ogre-tools/injectable"; +import type { FSWatcher } from "chokidar"; +import type { Stats } from "fs"; +import GlobToRegExp from "glob-to-regexp"; +import type { IComputedValue, ObservableMap } from "mobx"; +import { computed, observable } from "mobx"; +import path from "path"; +import { inspect } from "util"; +import type { CatalogEntity } from "../../../common/catalog"; +import type { Cluster } from "../../../common/cluster/cluster"; +import fsInjectable from "../../../common/fs/fs.injectable"; +import watchInjectable from "../../../common/fs/watch.injectable"; +import type { Disposer } from "../../../common/utils"; +import { getOrInsertWith, iter } from "../../../common/utils"; +import diffChangedKubeconfigInjectable from "./diff-changed-kubeconfig.injectable"; +import kubeconfigSyncLoggerInjectable from "./logger.injectable"; + +export type WatchKubeconfigFileChanges = (filepath: string) => [IComputedValue, Disposer]; + +/** + * This is the list of globs of which files are ignored when under a folder sync + */ +const ignoreGlobs = [ + "*.lock", // kubectl lock files + "*.swp", // vim swap files + ".DS_Store", // macOS specific +].map(rawGlob => ({ + rawGlob, + matcher: GlobToRegExp(rawGlob), +})); + +/** + * This should be much larger than any kubeconfig text file + * + * Even if you have a cert-file, key-file, and client-cert files that is only + * 12kb of extra data (at 4096 bytes each) which allows for around 150 entries. + */ +const folderSyncMaxAllowedFileReadSize = 2 * 1024 * 1024; // 2 MiB +const fileSyncMaxAllowedFileReadSize = 16 * folderSyncMaxAllowedFileReadSize; // 32 MiB + +const watchKubeconfigFileChangesInjectable = getInjectable({ + id: "watch-kubeconfig-file-changes", + instantiate: (di): WatchKubeconfigFileChanges => { + const diffChangedKubeconfig = di.inject(diffChangedKubeconfigInjectable); + const logger = di.inject(kubeconfigSyncLoggerInjectable); + const { stat } = di.inject(fsInjectable); + const watch = di.inject(watchInjectable); + + return (filePath) => { + const rootSource = observable.map>(); + const derivedSource = computed(() => Array.from(iter.flatMap(rootSource.values(), from => iter.map(from.values(), child => child[1])))); + + let watcher: FSWatcher; + + (async () => { + try { + const stats = await stat(filePath); + const isFolderSync = stats.isDirectory(); + const cleanupFns = new Map(); + const maxAllowedFileReadSize = isFolderSync + ? folderSyncMaxAllowedFileReadSize + : fileSyncMaxAllowedFileReadSize; + + watcher = watch(filePath, { + followSymlinks: true, + depth: isFolderSync ? 0 : 1, // DIRs works with 0 but files need 1 (bug: https://github.com/paulmillr/chokidar/issues/1095) + disableGlobbing: true, + ignorePermissionErrors: true, + usePolling: false, + awaitWriteFinish: { + pollInterval: 100, + stabilityThreshold: 1000, + }, + atomic: 150, // for "atomic writes" + alwaysStat: true, + }); + + watcher + .on("change", (childFilePath, stats: Stats): void => { + const cleanup = cleanupFns.get(childFilePath); + + if (!cleanup) { + // file was previously ignored, do nothing + return void logger.debug(`${inspect(childFilePath)} that should have been previously ignored has changed. Doing nothing`); + } + + cleanup(); + cleanupFns.set(childFilePath, diffChangedKubeconfig({ + filePath: childFilePath, + source: getOrInsertWith(rootSource, childFilePath, observable.map), + stats, + maxAllowedFileReadSize, + })); + }) + .on("add", (childFilePath, stats: Stats): void => { + if (isFolderSync) { + const fileName = path.basename(childFilePath); + + for (const ignoreGlob of ignoreGlobs) { + if (ignoreGlob.matcher.test(fileName)) { + return void logger.info(`ignoring ${inspect(childFilePath)} due to ignore glob: ${ignoreGlob.rawGlob}`); + } + } + } + + cleanupFns.set(childFilePath, diffChangedKubeconfig({ + filePath: childFilePath, + source: getOrInsertWith(rootSource, childFilePath, observable.map), + stats, + maxAllowedFileReadSize, + })); + }) + .on("unlink", (childFilePath) => { + cleanupFns.get(childFilePath)?.(); + cleanupFns.delete(childFilePath); + rootSource.delete(childFilePath); + }) + .on("error", error => logger.error(`watching file/folder failed: ${error}`, { filePath })); + } catch (error) { + logger.warn(`failed to start watching changes: ${error}`); + } + })(); + + return [derivedSource, () => { + watcher?.close(); + }]; + }; + }, +}); + +export default watchKubeconfigFileChangesInjectable; diff --git a/src/main/start-main-application/runnables/kube-config-sync/add-source.injectable.ts b/src/main/start-main-application/runnables/kube-config-sync/add-source.injectable.ts new file mode 100644 index 0000000000..fec87c16ce --- /dev/null +++ b/src/main/start-main-application/runnables/kube-config-sync/add-source.injectable.ts @@ -0,0 +1,25 @@ +/** + * Copyright (c) OpenLens Authors. All rights reserved. + * Licensed under MIT License. See LICENSE in root directory for more information. + */ +import { getInjectable } from "@ogre-tools/injectable"; +import kubeconfigSyncManagerInjectable from "../../../catalog-sources/kubeconfig-sync/manager.injectable"; +import catalogEntityRegistryInjectable from "../../../catalog/entity-registry.injectable"; +import { afterRootFrameIsReadyInjectionToken } from "../../runnable-tokens/after-root-frame-is-ready-injection-token"; + +const addKubeconfigSyncAsEntitySourceInjectable = getInjectable({ + id: "add-kubeconfig-sync-as-entity-source", + instantiate: (di) => { + const kubeConfigSyncManager = di.inject(kubeconfigSyncManagerInjectable); + const entityRegistry = di.inject(catalogEntityRegistryInjectable); + + return { + run: () => { + entityRegistry.addComputedSource("kubeconfig-sync", kubeConfigSyncManager.source); + }, + }; + }, + injectionToken: afterRootFrameIsReadyInjectionToken, +}); + +export default addKubeconfigSyncAsEntitySourceInjectable; diff --git a/src/main/start-main-application/runnables/kube-config-sync/start-kube-config-sync.injectable.ts b/src/main/start-main-application/runnables/kube-config-sync/start-kube-config-sync.injectable.ts index 0b585f3d65..7be93de224 100644 --- a/src/main/start-main-application/runnables/kube-config-sync/start-kube-config-sync.injectable.ts +++ b/src/main/start-main-application/runnables/kube-config-sync/start-kube-config-sync.injectable.ts @@ -7,6 +7,7 @@ import { afterApplicationIsLoadedInjectionToken } from "../../runnable-tokens/af import directoryForKubeConfigsInjectable from "../../../../common/app-paths/directory-for-kube-configs/directory-for-kube-configs.injectable"; import ensureDirInjectable from "../../../../common/fs/ensure-dir.injectable"; import kubeconfigSyncManagerInjectable from "../../../catalog-sources/kubeconfig-sync/manager.injectable"; +import addKubeconfigSyncAsEntitySourceInjectable from "./add-source.injectable"; const startKubeConfigSyncInjectable = getInjectable({ id: "start-kubeconfig-sync", @@ -22,6 +23,7 @@ const startKubeConfigSyncInjectable = getInjectable({ kubeConfigSyncManager.startSync(); }, + runAfter: di.inject(addKubeconfigSyncAsEntitySourceInjectable), }; }, diff --git a/src/renderer/components/hotbar/hotbar-entity-icon.tsx b/src/renderer/components/hotbar/hotbar-entity-icon.tsx index e07f320fad..b3f635acb5 100644 --- a/src/renderer/components/hotbar/hotbar-entity-icon.tsx +++ b/src/renderer/components/hotbar/hotbar-entity-icon.tsx @@ -17,11 +17,12 @@ import { Icon } from "../icon"; import { HotbarIcon } from "./hotbar-icon"; import { LensKubernetesClusterStatus } from "../../../common/catalog-entities/kubernetes-cluster"; import type { VisitEntityContextMenu } from "../../../common/catalog/visit-entity-context-menu.injectable"; -import { navigate } from "../../navigation"; import { withInjectables } from "@ogre-tools/injectable-react"; import catalogCategoryRegistryInjectable from "../../../common/catalog/category-registry.injectable"; import visitEntityContextMenuInjectable from "../../../common/catalog/visit-entity-context-menu.injectable"; import activeEntityInjectable from "../../api/catalog/entity/active.injectable"; +import type { Navigate } from "../../navigation/navigate.injectable"; +import navigateInjectable from "../../navigation/navigate.injectable"; export interface HotbarEntityIconProps { entity: CatalogEntity; @@ -38,13 +39,14 @@ interface Dependencies { visitEntityContextMenu: VisitEntityContextMenu; catalogCategoryRegistry: CatalogCategoryRegistry; activeEntity: IComputedValue; + navigate: Navigate; } @observer class NonInjectedHotbarEntityIcon extends React.Component { private readonly menuItems = observable.array(); - get kindIcon() { + private renderKindIcon() { const className = styles.badge; const category = this.props.catalogCategoryRegistry.getCategoryForEntity(this.props.entity); @@ -59,7 +61,7 @@ class NonInjectedHotbarEntityIcon extends React.Component; } - get ledIcon() { + private renderLedIcon() { if (this.props.entity.kind !== "KubernetesCluster") { return null; } @@ -86,7 +88,7 @@ class NonInjectedHotbarEntityIcon extends React.Component - { this.ledIcon } - { this.kindIcon } + {this.renderLedIcon()} + {this.renderKindIcon()} ); } @@ -126,5 +128,6 @@ export const HotbarEntityIcon = withInjectables