Fix kubeconfig-sync sometimes producing multiple identical entities

- This happens when there is an overlapping sync of a folder and a file
  within that folder both producing the same entities

- This also fixes the LED not showing up on some KubernetesCluster
  instances because of this duplication the connected status was being
  set on the first instance, but renderer would use the second (because
  JS's Map constructor deduplicates its constructor initializer by
  taking the last)

- Added some tests to cover this case

Signed-off-by: Sebastian Malton <sebastian@malton.name>

src/main/catalog-sources/__test__/kubeconfig-sync.test.ts

@@ -3,19 +3,15 @@
  * Licensed under MIT License. See LICENSE in root directory for more information.
  */
 
-import { ObservableMap } from "mobx";
+import { observable, ObservableMap, when } from "mobx";
 import type { CatalogEntity } from "../../../common/catalog";
 import { loadFromOptions } from "../../../common/kube-helpers";
 import type { Cluster } from "../../../common/cluster/cluster";
-import { computeDiff as computeDiffFor, configToModels } from "../kubeconfig-sync/manager";
 import mockFs from "mock-fs";
 import fs from "fs";
 import clusterStoreInjectable from "../../../common/cluster-store/cluster-store.injectable";
 import { getDiForUnitTesting } from "../../getDiForUnitTesting";
-import { createClusterInjectionToken } from "../../../common/cluster/create-cluster-injection-token";
-import directoryForKubeConfigsInjectable from "../../../common/app-paths/directory-for-kube-configs/directory-for-kube-configs.injectable";
 import getConfigurationFileModelInjectable from "../../../common/get-configuration-file-model/get-configuration-file-model.injectable";
-import clusterManagerInjectable from "../../cluster-manager.injectable";
 import directoryForUserDataInjectable from "../../../common/app-paths/directory-for-user-data/directory-for-user-data.injectable";
 import directoryForTempInjectable from "../../../common/app-paths/directory-for-temp/directory-for-temp.injectable";
 import kubectlBinaryNameInjectable from "../../kubectl/binary-name.injectable";
@@ -23,6 +19,17 @@ import kubectlDownloadingNormalizedArchInjectable from "../../kubectl/normalized
 import normalizedPlatformInjectable from "../../../common/vars/normalized-platform.injectable";
 import { iter } from "../../../common/utils";
 import fsInjectable from "../../../common/fs/fs.injectable";
+import type { ComputeKubeconfigDiff } from "../kubeconfig-sync/compute-diff.injectable";
+import computeKubeconfigDiffInjectable from "../kubeconfig-sync/compute-diff.injectable";
+import watchInjectable from "../../../common/fs/watch.injectable";
+import type { ConfigToModels } from "../kubeconfig-sync/config-to-models.injectable";
+import configToModelsInjectable from "../kubeconfig-sync/config-to-models.injectable";
+import kubeconfigSyncManagerInjectable from "../kubeconfig-sync/manager.injectable";
+import type { KubeconfigSyncManager } from "../kubeconfig-sync/manager";
+import type { KubeconfigSyncValue } from "../../../common/user-store";
+import kubeconfigSyncsInjectable from "../../../common/user-store/kubeconfig-syncs.injectable";
+
+console.log("This is a reminder that mockFS breaks things and needs to be removed");
 
 jest.mock("electron", () => ({
   app: {
@@ -41,7 +48,10 @@ jest.mock("electron", () => ({
 }));
 
 describe("kubeconfig-sync.source tests", () => {
-  let computeDiff: ReturnType<typeof computeDiffFor>;
+  let computeKubeconfigDiff: ComputeKubeconfigDiff;
+  let configToModels: ConfigToModels;
+  let manager: KubeconfigSyncManager;
+  let kubeconfigSyncs: ObservableMap<string, KubeconfigSyncValue>;
 
   beforeEach(async () => {
     const di = getDiForUnitTesting({ doGeneralOverrides: true });
@@ -55,15 +65,18 @@ describe("kubeconfig-sync.source tests", () => {
     di.override(normalizedPlatformInjectable, () => "darwin");
 
     di.permitSideEffects(fsInjectable);
+    di.permitSideEffects(watchInjectable);
     di.unoverride(clusterStoreInjectable);
     di.permitSideEffects(clusterStoreInjectable);
     di.permitSideEffects(getConfigurationFileModelInjectable);
 
-    computeDiff = computeDiffFor({
-      directoryForKubeConfigs: di.inject(directoryForKubeConfigsInjectable),
-      createCluster: di.inject(createClusterInjectionToken),
-      clusterManager: di.inject(clusterManagerInjectable),
-    });
+    kubeconfigSyncs = observable.map();
+
+    di.override(kubeconfigSyncsInjectable, () => kubeconfigSyncs);
+
+    computeKubeconfigDiff = di.inject(computeKubeconfigDiffInjectable);
+    configToModels = di.inject(configToModelsInjectable);
+    manager = di.inject(kubeconfigSyncManagerInjectable);
   });
 
   afterEach(() => {
@@ -108,13 +121,13 @@ describe("kubeconfig-sync.source tests", () => {
     });
   });
 
-  describe("computeDiff", () => {
+  describe("computeKubeconfigDiff", () => {
     it("should leave an empty source empty if there are no entries", () => {
       const contents = "";
       const rootSource = new ObservableMap<string, [Cluster, CatalogEntity]>();
       const filePath = "/bar";
 
-      computeDiff(contents, rootSource, filePath);
+      computeKubeconfigDiff(contents, rootSource, filePath);
 
       expect(rootSource.size).toBe(0);
     });
@@ -151,7 +164,7 @@ describe("kubeconfig-sync.source tests", () => {
 
       fs.writeFileSync(filePath, contents);
 
-      computeDiff(contents, rootSource, filePath);
+      computeKubeconfigDiff(contents, rootSource, filePath);
 
       expect(rootSource.size).toBe(1);
 
@@ -195,7 +208,7 @@ describe("kubeconfig-sync.source tests", () => {
 
       fs.writeFileSync(filePath, contents);
 
-      computeDiff(contents, rootSource, filePath);
+      computeKubeconfigDiff(contents, rootSource, filePath);
 
       expect(rootSource.size).toBe(1);
 
@@ -204,7 +217,7 @@ describe("kubeconfig-sync.source tests", () => {
       expect(c.kubeConfigPath).toBe("/bar");
       expect(c.contextName).toBe("context-name");
 
-      computeDiff("{}", rootSource, filePath);
+      computeKubeconfigDiff("{}", rootSource, filePath);
 
       expect(rootSource.size).toBe(0);
     });
@@ -249,7 +262,7 @@ describe("kubeconfig-sync.source tests", () => {
 
       fs.writeFileSync(filePath, contents);
 
-      computeDiff(contents, rootSource, filePath);
+      computeKubeconfigDiff(contents, rootSource, filePath);
 
       expect(rootSource.size).toBe(2);
 
@@ -289,7 +302,7 @@ describe("kubeconfig-sync.source tests", () => {
         currentContext: "foobar",
       });
 
-      computeDiff(newContents, rootSource, filePath);
+      computeKubeconfigDiff(newContents, rootSource, filePath);
 
       expect(rootSource.size).toBe(1);
 
@@ -301,4 +314,80 @@ describe("kubeconfig-sync.source tests", () => {
       }
     });
   });
+
+  describe("given a config file at /foobar/config", () => {
+    beforeEach(() => {
+      fs.mkdirSync("/foobar");
+      fs.writeFileSync("/foobar/config", JSON.stringify({
+        clusters: [{
+          name: "cluster-name",
+          cluster: {
+            server: "1.2.3.4",
+          },
+          skipTLSVerify: false,
+        }],
+        users: [{
+          name: "user-name",
+        }],
+        contexts: [{
+          name: "context-name",
+          context: {
+            cluster: "cluster-name",
+            user: "user-name",
+          },
+        }, {
+          name: "context-the-second",
+          context: {
+            cluster: "missing-cluster",
+            user: "user-name",
+          },
+        }],
+        currentContext: "foobar",
+      }));
+    });
+
+    it("should not find any entities", () => {
+      expect(manager.source.get()).toEqual([]);
+    });
+
+    describe("when sync has started", () => {
+      beforeEach(() => {
+        manager.startSync();
+      });
+
+      it("should not find any entities", () => {
+        expect(manager.source.get()).toEqual([]);
+      });
+
+      describe("when a file sync target for /foobar/config is added", () => {
+        beforeEach(() => {
+          kubeconfigSyncs.set("/foobar/config", {});
+        });
+
+        it("should find a single entity", (done) => {
+          when(() => manager.source.get().length === 1, () => done());
+        });
+
+        describe("when a folder sync target for /foobar is added", () => {
+          beforeEach(() => {
+            kubeconfigSyncs.set("/foobar", {});
+          });
+
+          it("should still only find a single entity", (done) => {
+            when(() => manager.source.get().length === 1, () => done());
+          });
+        });
+      });
+
+      describe("when a folder sync target for /foobar is added", () => {
+        beforeEach(() => {
+          kubeconfigSyncs.set("/foobar", {});
+        });
+
+        it("should find a single entity", (done) => {
+          when(() => manager.source.get().length === 1, () => done());
+        });
+      });
+    });
+  });
 });

src/main/catalog-sources/kubeconfig-sync/compute-diff.injectable.ts

@@ -0,0 +1,100 @@
+/**
+ * Copyright (c) OpenLens Authors. All rights reserved.
+ * Licensed under MIT License. See LICENSE in root directory for more information.
+ */
+import { getInjectable } from "@ogre-tools/injectable";
+import { createHash } from "crypto";
+import type { ObservableMap } from "mobx";
+import { action } from "mobx";
+import { homedir } from "os";
+import directoryForKubeConfigsInjectable from "../../../common/app-paths/directory-for-kube-configs/directory-for-kube-configs.injectable";
+import type { CatalogEntity } from "../../../common/catalog";
+import getClusterByIdInjectable from "../../../common/cluster-store/get-by-id.injectable";
+import type { Cluster } from "../../../common/cluster/cluster";
+import { loadConfigFromString } from "../../../common/kube-helpers";
+import { catalogEntityFromCluster } from "../../cluster-manager";
+import clusterManagerInjectable from "../../cluster-manager.injectable";
+import createClusterInjectable from "../../create-cluster/create-cluster.injectable";
+import configToModelsInjectable from "./config-to-models.injectable";
+import kubeconfigSyncLoggerInjectable from "./logger.injectable";
+
+export type ComputeKubeconfigDiff = (contents: string, source: ObservableMap<string, [Cluster, CatalogEntity]>, filePath: string) => void;
+
+const computeKubeconfigDiffInjectable = getInjectable({
+  id: "compute-kubeconfig-diff",
+  instantiate: (di): ComputeKubeconfigDiff => {
+    const directoryForKubeConfigs = di.inject(directoryForKubeConfigsInjectable);
+    const createCluster = di.inject(createClusterInjectable);
+    const clusterManager = di.inject(clusterManagerInjectable);
+    const configToModels = di.inject(configToModelsInjectable);
+    const logger = di.inject(kubeconfigSyncLoggerInjectable);
+    const getClusterById = di.inject(getClusterByIdInjectable);
+
+    return action((contents, source, filePath) => {
+      try {
+        const { config, error } = loadConfigFromString(contents);
+
+        if (error) {
+          logger.warn(`encountered errors while loading config: ${error.message}`, { filePath, details: error.details });
+        }
+
+        const rawModels = configToModels(config, filePath);
+        const models = new Map(rawModels.map(([model, configData]) => [model.contextName, [model, configData] as const]));
+
+        logger.debug(`File now has ${models.size} entries`, { filePath });
+
+        for (const [contextName, value] of source) {
+          const data = models.get(contextName);
+
+          // remove and disconnect clusters that were removed from the config
+          if (!data) {
+          // remove from the deleting set, so that if a new context of the same name is added, it isn't marked as deleting
+            clusterManager.deleting.delete(value[0].id);
+
+            value[0].disconnect();
+            source.delete(contextName);
+            logger.debug(`Removed old cluster from sync`, { filePath, contextName });
+            continue;
+          }
+
+          // TODO: For the update check we need to make sure that the config itself hasn't changed.
+          // Probably should make it so that cluster keeps a copy of the config in its memory and
+          // diff against that
+
+          // or update the model and mark it as not needed to be added
+          value[0].updateModel(data[0]);
+          models.delete(contextName);
+          logger.debug(`Updated old cluster from sync`, { filePath, contextName });
+        }
+
+        for (const [contextName, [model, configData]] of models) {
+        // add new clusters to the source
+          try {
+            const clusterId = createHash("md5").update(`${filePath}:${contextName}`).digest("hex");
+            const cluster = getClusterById(clusterId) ?? createCluster({ ...model, id: clusterId }, configData);
+
+            if (!cluster.apiUrl) {
+              throw new Error("Cluster constructor failed, see above error");
+            }
+
+            const entity = catalogEntityFromCluster(cluster);
+
+            if (!filePath.startsWith(directoryForKubeConfigs)) {
+              entity.metadata.labels.file = filePath.replace(homedir(), "~");
+            }
+            source.set(contextName, [cluster, entity]);
+
+            logger.debug(`Added new cluster from sync`, { filePath, contextName });
+          } catch (error) {
+            logger.warn(`Failed to create cluster from model: ${error}`, { filePath, contextName });
+          }
+        }
+      } catch (error) {
+        logger.warn(`Failed to compute diff: ${error}`, { filePath });
+        source.clear(); // clear source if we have failed so as to not show outdated information
+      }
+    });
+  },
+});
+
+export default computeKubeconfigDiffInjectable;

src/main/catalog-sources/kubeconfig-sync/config-to-models.injectable.ts

@@ -0,0 +1,42 @@
+/**
+ * Copyright (c) OpenLens Authors. All rights reserved.
+ * Licensed under MIT License. See LICENSE in root directory for more information.
+ */
+import type { KubeConfig } from "@kubernetes/client-node";
+import { getInjectable } from "@ogre-tools/injectable";
+import type { ClusterConfigData, UpdateClusterModel } from "../../../common/cluster-types";
+import { splitConfig } from "../../../common/kube-helpers";
+import kubeconfigSyncLoggerInjectable from "./logger.injectable";
+
+export type ConfigToModels = (rootConfig: KubeConfig, filePath: string) => [UpdateClusterModel, ClusterConfigData][];
+
+const configToModelsInjectable = getInjectable({
+  id: "config-to-models",
+  instantiate: (di): ConfigToModels => {
+    const logger = di.inject(kubeconfigSyncLoggerInjectable);
+
+    return (rootConfig, filePath) => {
+      const validConfigs: ReturnType<ConfigToModels> = [];
+
+      for (const { config, validationResult } of splitConfig(rootConfig)) {
+        if (validationResult.error) {
+          logger.debug(`context failed validation: ${validationResult.error}`, { context: config.currentContext, filePath });
+        } else {
+          validConfigs.push([
+            {
+              kubeConfigPath: filePath,
+              contextName: config.currentContext,
+            },
+            {
+              clusterServerUrl: validationResult.cluster.server,
+            },
+          ]);
+        }
+      }
+
+      return validConfigs;
+    };
+  },
+});
+
+export default configToModelsInjectable;

src/main/catalog-sources/kubeconfig-sync/diff-changed-kubeconfig.injectable.ts

@@ -0,0 +1,90 @@
+/**
+ * Copyright (c) OpenLens Authors. All rights reserved.
+ * Licensed under MIT License. See LICENSE in root directory for more information.
+ */
+import { getInjectable } from "@ogre-tools/injectable";
+import type { Stats } from "fs";
+import { constants } from "fs";
+import type { ObservableMap } from "mobx";
+import type { Readable } from "stream";
+import type { CatalogEntity } from "../../../common/catalog";
+import type { Cluster } from "../../../common/cluster/cluster";
+import fsInjectable from "../../../common/fs/fs.injectable";
+import type { Disposer } from "../../../common/utils";
+import { bytesToUnits, noop } from "../../../common/utils";
+import computeKubeconfigDiffInjectable from "./compute-diff.injectable";
+import kubeconfigSyncLoggerInjectable from "./logger.injectable";
+
+export interface DiffChangedKubeconfigArgs {
+  filePath: string;
+  source: ObservableMap<string, [Cluster, CatalogEntity]>;
+  stats: Stats;
+  maxAllowedFileReadSize: number;
+}
+export type DiffChangedKubeconfig = (args: DiffChangedKubeconfigArgs) => Disposer;
+
+const diffChangedKubeconfigInjectable = getInjectable({
+  id: "diff-changed-kubeconfig",
+  instantiate: (di): DiffChangedKubeconfig => {
+    const computeKubeconfigDiff = di.inject(computeKubeconfigDiffInjectable);
+    const logger = di.inject(kubeconfigSyncLoggerInjectable);
+    const { createReadStream } = di.inject(fsInjectable);
+
+    return ({ filePath, maxAllowedFileReadSize, source, stats }) => {
+      logger.debug(`file changed`, { filePath });
+
+      if (stats.size >= maxAllowedFileReadSize) {
+        logger.warn(`skipping ${filePath}: size=${bytesToUnits(stats.size)} is larger than maxSize=${bytesToUnits(maxAllowedFileReadSize)}`);
+        source.clear();
+
+        return noop;
+      }
+
+      const fileReader = createReadStream(filePath, {
+        mode: constants.O_RDONLY,
+      });
+      const readStream = fileReader as Readable;
+      const decoder = new TextDecoder("utf-8", { fatal: true });
+      let fileString = "";
+      let closed = false;
+
+      const cleanup = () => {
+        closed = true;
+        fileReader.close(); // This may not close the stream.
+        // Artificially marking end-of-stream, as if the underlying resource had
+        // indicated end-of-file by itself, allows the stream to close.
+        // This does not cancel pending read operations, and if there is such an
+        // operation, the process may still not be able to exit successfully
+        // until it finishes.
+        fileReader.push(null);
+        fileReader.read(0);
+        readStream.removeAllListeners();
+      };
+
+      readStream
+        .on("data", (chunk: Buffer) => {
+          try {
+            fileString += decoder.decode(chunk, { stream: true });
+          } catch (error) {
+            logger.warn(`skipping ${filePath}: ${error}`);
+            source.clear();
+            cleanup();
+          }
+        })
+        .on("close", () => cleanup())
+        .on("error", error => {
+          cleanup();
+          logger.warn(`failed to read file: ${error}`, { filePath });
+        })
+        .on("end", () => {
+          if (!closed) {
+            computeKubeconfigDiff(fileString, source, filePath);
+          }
+        });
+
+      return cleanup;
+    };
+  },
+});
+
+export default diffChangedKubeconfigInjectable;

src/main/catalog-sources/kubeconfig-sync/logger.injectable.ts

@@ -0,0 +1,13 @@
+/**
+ * Copyright (c) OpenLens Authors. All rights reserved.
+ * Licensed under MIT License. See LICENSE in root directory for more information.
+ */
+import { getInjectable } from "@ogre-tools/injectable";
+import childLoggerInjectable from "../../../common/logger/child-logger.injectable";
+
+const kubeconfigSyncLoggerInjectable = getInjectable({
+  id: "kubeconfig-sync-logger",
+  instantiate: (di) => di.inject(childLoggerInjectable, "KUBECONFIG-SYNC"),
+});
+
+export default kubeconfigSyncLoggerInjectable;

src/main/catalog-sources/kubeconfig-sync/manager.injectable.ts

@@ -5,18 +5,18 @@
 import { getInjectable } from "@ogre-tools/injectable";
 import directoryForKubeConfigsInjectable from "../../../common/app-paths/directory-for-kube-configs/directory-for-kube-configs.injectable";
 import { KubeconfigSyncManager } from "./manager";
-import { createClusterInjectionToken } from "../../../common/cluster/create-cluster-injection-token";
-import clusterManagerInjectable from "../../cluster-manager.injectable";
-import catalogEntityRegistryInjectable from "../../catalog/entity-registry.injectable";
+import kubeconfigSyncLoggerInjectable from "./logger.injectable";
+import watchKubeconfigFileChangesInjectable from "./watch-file-changes.injectable";
+import kubeconfigSyncsInjectable from "../../../common/user-store/kubeconfig-syncs.injectable";
 
 const kubeconfigSyncManagerInjectable = getInjectable({
   id: "kubeconfig-sync-manager",
 
   instantiate: (di) => new KubeconfigSyncManager({
     directoryForKubeConfigs: di.inject(directoryForKubeConfigsInjectable),
-    createCluster: di.inject(createClusterInjectionToken),
-    clusterManager: di.inject(clusterManagerInjectable),
-    entityRegistry: di.inject(catalogEntityRegistryInjectable),
+    logger: di.inject(kubeconfigSyncLoggerInjectable),
+    watchKubeconfigFileChanges: di.inject(watchKubeconfigFileChangesInjectable),
+    kubeconfigSyncs: di.inject(kubeconfigSyncsInjectable),
   }),
 });
 

src/main/catalog-sources/kubeconfig-sync/manager.ts

@@ -4,97 +4,59 @@
  */
 
 import type { IComputedValue, ObservableMap } from "mobx";
-import { action, observable, computed, runInAction, makeObservable, observe } from "mobx";
+import { action, observable, computed, makeObservable, observe } from "mobx";
 import type { CatalogEntity } from "../../../common/catalog";
-import type { FSWatcher } from "chokidar";
-import { watch } from "chokidar";
-import type { Stats } from "fs";
-import fs from "fs";
-import path from "path";
 import type { Disposer } from "../../../common/utils";
-import { disposer, bytesToUnits, getOrInsertWith, iter, noop } from "../../../common/utils";
-import logger from "../../logger";
-import type { KubeConfig } from "@kubernetes/client-node";
-import { loadConfigFromString, splitConfig } from "../../../common/kube-helpers";
-import type { ClusterManager } from "../../cluster-manager";
-import { catalogEntityFromCluster } from "../../cluster-manager";
-import { UserStore } from "../../../common/user-store";
-import { ClusterStore } from "../../../common/cluster-store/cluster-store";
-import { createHash } from "crypto";
-import { homedir } from "os";
-import globToRegExp from "glob-to-regexp";
-import { inspect } from "util";
-import type { ClusterConfigData, UpdateClusterModel } from "../../../common/cluster-types";
-import type { Cluster } from "../../../common/cluster/cluster";
-import type { CatalogEntityRegistry } from "../../catalog/entity-registry";
-import type { CreateCluster } from "../../../common/cluster/create-cluster-injection-token";
-
-const logPrefix = "[KUBECONFIG-SYNC]:";
-
-/**
- * This is the list of globs of which files are ignored when under a folder sync
- */
-const ignoreGlobs = [
-  "*.lock", // kubectl lock files
-  "*.swp", // vim swap files
-  ".DS_Store", // macOS specific
-].map(rawGlob => ({
-  rawGlob,
-  matcher: globToRegExp(rawGlob),
-}));
-
-/**
- * This should be much larger than any kubeconfig text file
- *
- * Even if you have a cert-file, key-file, and client-cert files that is only
- * 12kb of extra data (at 4096 bytes each) which allows for around 150 entries.
- */
-const folderSyncMaxAllowedFileReadSize = 2 * 1024 * 1024; // 2 MiB
-const fileSyncMaxAllowedFileReadSize = 16 * folderSyncMaxAllowedFileReadSize; // 32 MiB
+import { iter } from "../../../common/utils";
+import type { KubeconfigSyncValue } from "../../../common/user-store";
+import type { Logger } from "../../../common/logger";
+import type { WatchKubeconfigFileChanges } from "./watch-file-changes.injectable";
 
 interface KubeconfigSyncManagerDependencies {
   readonly directoryForKubeConfigs: string;
-  readonly entityRegistry: CatalogEntityRegistry;
-  readonly clusterManager: ClusterManager;
-  createCluster: CreateCluster;
+  readonly logger: Logger;
+  readonly kubeconfigSyncs: ObservableMap<string, KubeconfigSyncValue>;
+  watchKubeconfigFileChanges: WatchKubeconfigFileChanges;
 }
 
-const kubeConfigSyncName = "lens:kube-sync";
-
 export class KubeconfigSyncManager {
   protected readonly sources = observable.map<string, [IComputedValue<CatalogEntity[]>, Disposer]>();
-  protected syncing = false;
   protected syncListDisposer?: Disposer;
 
   constructor(protected readonly dependencies: KubeconfigSyncManagerDependencies) {
     makeObservable(this);
   }
 
+  public readonly source = computed(() => {
+    /**
+     * This prevents multiple overlapping syncs from leading to multiple entities with the same IDs
+     */
+    const seenIds = new Set<string>();
+
+    return (
+      iter.pipeline(this.sources.values())
+        .flatMap(([entities]) => entities.get())
+        .filter(entity => (
+          seenIds.has(entity.getId())
+            ? false
+            : seenIds.add(entity.getId())
+        ))
+        .collect(items => [...items])
+    );
+  });
+
   @action
   startSync(): void {
-    if (this.syncing) {
-      return;
-    }
-
-    this.syncing = true;
-
-    logger.info(`${logPrefix} starting requested syncs`);
-
-    this.dependencies.entityRegistry.addComputedSource(kubeConfigSyncName, computed(() => (
-      Array.from(iter.flatMap(
-        this.sources.values(),
-        ([entities]) => entities.get(),
-      ))
-    )));
+    this.dependencies.logger.info(`starting requested syncs`);
 
     // This must be done so that c&p-ed clusters are visible
     this.startNewSync(this.dependencies.directoryForKubeConfigs);
 
-    for (const filePath of UserStore.getInstance().syncKubeconfigEntries.keys()) {
+    for (const filePath of this.dependencies.kubeconfigSyncs.keys()) {
       this.startNewSync(filePath);
     }
 
-    this.syncListDisposer = observe(UserStore.getInstance().syncKubeconfigEntries, change => {
+    this.syncListDisposer = observe(this.dependencies.kubeconfigSyncs, change => {
       switch (change.type) {
         case "add":
           this.startNewSync(change.name);
@@ -108,275 +70,38 @@ export class KubeconfigSyncManager {
 
   @action
   stopSync() {
+    this.dependencies.logger.info(`stopping requested syncs`);
     this.syncListDisposer?.();
 
     for (const filePath of this.sources.keys()) {
       this.stopOldSync(filePath);
     }
-
-    this.dependencies.entityRegistry.removeSource(kubeConfigSyncName);
-    this.syncing = false;
   }
 
   @action
   protected startNewSync(filePath: string): void {
     if (this.sources.has(filePath)) {
       // don't start a new sync if we already have one
-      return void logger.debug(`${logPrefix} already syncing file/folder`, { filePath });
+      return this.dependencies.logger.debug(`already syncing file/folder`, { filePath });
     }
 
     this.sources.set(
       filePath,
-      watchFileChanges(filePath, this.dependencies),
+      this.dependencies.watchKubeconfigFileChanges(filePath),
     );
 
-    logger.info(`${logPrefix} starting sync of file/folder`, { filePath });
-    logger.debug(`${logPrefix} ${this.sources.size} files/folders watched`, { files: Array.from(this.sources.keys()) });
+    this.dependencies.logger.info(`starting sync of file/folder`, { filePath });
+    this.dependencies.logger.debug(`${this.sources.size} files/folders watched`, { files: Array.from(this.sources.keys()) });
   }
 
   @action
   protected stopOldSync(filePath: string): void {
     if (!this.sources.delete(filePath)) {
       // already stopped
-      return void logger.debug(`${logPrefix} no syncing file/folder to stop`, { filePath });
+      return this.dependencies.logger.debug(`no syncing file/folder to stop`, { filePath });
     }
 
-    logger.info(`${logPrefix} stopping sync of file/folder`, { filePath });
-    logger.debug(`${logPrefix} ${this.sources.size} files/folders watched`, { files: Array.from(this.sources.keys()) });
+    this.dependencies.logger.info(`stopping sync of file/folder`, { filePath });
+    this.dependencies.logger.debug(`${this.sources.size} files/folders watched`, { files: Array.from(this.sources.keys()) });
   }
 }
-
-// exported for testing
-export function configToModels(rootConfig: KubeConfig, filePath: string): [UpdateClusterModel, ClusterConfigData][] {
-  const validConfigs: ReturnType<typeof configToModels> = [];
-
-  for (const { config, validationResult } of splitConfig(rootConfig)) {
-    if (validationResult.error) {
-      logger.debug(`${logPrefix} context failed validation: ${validationResult.error}`, { context: config.currentContext, filePath });
-    } else {
-      validConfigs.push([
-        {
-          kubeConfigPath: filePath,
-          contextName: config.currentContext,
-        },
-        {
-          clusterServerUrl: validationResult.cluster.server,
-        },
-      ]);
-    }
-  }
-
-  return validConfigs;
-}
-
-type RootSourceValue = [Cluster, CatalogEntity];
-type RootSource = ObservableMap<string, RootSourceValue>;
-
-interface ComputeDiffDependencies {
-  directoryForKubeConfigs: string;
-  createCluster: CreateCluster;
-  clusterManager: ClusterManager;
-}
-
-// exported for testing
-export const computeDiff = ({ directoryForKubeConfigs, createCluster, clusterManager }: ComputeDiffDependencies) => (contents: string, source: RootSource, filePath: string): void => {
-  runInAction(() => {
-    try {
-      const { config, error } = loadConfigFromString(contents);
-
-      if (error) {
-        logger.warn(`${logPrefix} encountered errors while loading config: ${error.message}`, { filePath, details: error.details });
-      }
-
-      const rawModels = configToModels(config, filePath);
-      const models = new Map(rawModels.map(([model, configData]) => [model.contextName, [model, configData] as const]));
-
-      logger.debug(`${logPrefix} File now has ${models.size} entries`, { filePath });
-
-      for (const [contextName, value] of source) {
-        const data = models.get(contextName);
-
-        // remove and disconnect clusters that were removed from the config
-        if (!data) {
-          // remove from the deleting set, so that if a new context of the same name is added, it isn't marked as deleting
-          clusterManager.deleting.delete(value[0].id);
-
-          value[0].disconnect();
-          source.delete(contextName);
-          logger.debug(`${logPrefix} Removed old cluster from sync`, { filePath, contextName });
-          continue;
-        }
-
-        // TODO: For the update check we need to make sure that the config itself hasn't changed.
-        // Probably should make it so that cluster keeps a copy of the config in its memory and
-        // diff against that
-
-        // or update the model and mark it as not needed to be added
-        value[0].updateModel(data[0]);
-        models.delete(contextName);
-        logger.debug(`${logPrefix} Updated old cluster from sync`, { filePath, contextName });
-      }
-
-      for (const [contextName, [model, configData]] of models) {
-        // add new clusters to the source
-        try {
-          const clusterId = createHash("md5").update(`${filePath}:${contextName}`).digest("hex");
-
-          const cluster = ClusterStore.getInstance().getById(clusterId) || createCluster({ ...model, id: clusterId }, configData);
-
-          if (!cluster.apiUrl) {
-            throw new Error("Cluster constructor failed, see above error");
-          }
-
-          const entity = catalogEntityFromCluster(cluster);
-
-          if (!filePath.startsWith(directoryForKubeConfigs)) {
-            entity.metadata.labels.file = filePath.replace(homedir(), "~");
-          }
-          source.set(contextName, [cluster, entity]);
-
-          logger.debug(`${logPrefix} Added new cluster from sync`, { filePath, contextName });
-        } catch (error) {
-          logger.warn(`${logPrefix} Failed to create cluster from model: ${error}`, { filePath, contextName });
-        }
-      }
-    } catch (error) {
-      logger.warn(`${logPrefix} Failed to compute diff: ${error}`, { filePath });
-      source.clear(); // clear source if we have failed so as to not show outdated information
-    }
-  });
-};
-
-interface DiffChangedConfigArgs {
-  filePath: string;
-  source: RootSource;
-  stats: fs.Stats;
-  maxAllowedFileReadSize: number;
-}
-
-const diffChangedConfigFor = (dependencies: ComputeDiffDependencies) => ({ filePath, source, stats, maxAllowedFileReadSize }: DiffChangedConfigArgs): Disposer => {
-  logger.debug(`${logPrefix} file changed`, { filePath });
-
-  if (stats.size >= maxAllowedFileReadSize) {
-    logger.warn(`${logPrefix} skipping ${filePath}: size=${bytesToUnits(stats.size)} is larger than maxSize=${bytesToUnits(maxAllowedFileReadSize)}`);
-    source.clear();
-
-    return noop;
-  }
-
-  const controller = new AbortController();
-  const fileContentsP = fs.promises.readFile(filePath, {
-    signal: controller.signal,
-  });
-  const cleanup = disposer(
-    () => controller.abort(),
-  );
-
-  fileContentsP
-    .then((fileData) => {
-      const decoder = new TextDecoder("utf-8", { fatal: true });
-
-      try {
-        const fileString = decoder.decode(fileData);
-
-        computeDiff(dependencies)(fileString, source, filePath);
-      } catch (error) {
-        logger.warn(`${logPrefix} skipping ${filePath}: ${error}`);
-        source.clear();
-        cleanup();
-      }
-    })
-    .catch(error => {
-      if (controller.signal.aborted) {
-        return;
-      }
-
-      logger.warn(`${logPrefix} failed to read file: ${error}`, { filePath });
-      cleanup();
-    });
-
-  return cleanup;
-};
-
-const watchFileChanges = (filePath: string, dependencies: ComputeDiffDependencies): [IComputedValue<CatalogEntity[]>, Disposer] => {
-  const rootSource = observable.map<string, ObservableMap<string, RootSourceValue>>();
-  const derivedSource = computed(() => Array.from(iter.flatMap(rootSource.values(), from => iter.map(from.values(), child => child[1]))));
-
-  let watcher: FSWatcher;
-
-  (async () => {
-    try {
-      const stat = await fs.promises.stat(filePath);
-      const isFolderSync = stat.isDirectory();
-      const cleanupFns = new Map<string, Disposer>();
-      const maxAllowedFileReadSize = isFolderSync
-        ? folderSyncMaxAllowedFileReadSize
-        : fileSyncMaxAllowedFileReadSize;
-
-      watcher = watch(filePath, {
-        followSymlinks: true,
-        depth: isFolderSync ? 0 : 1, // DIRs works with 0 but files need 1 (bug: https://github.com/paulmillr/chokidar/issues/1095)
-        disableGlobbing: true,
-        ignorePermissionErrors: true,
-        usePolling: false,
-        awaitWriteFinish: {
-          pollInterval: 100,
-          stabilityThreshold: 1000,
-        },
-        atomic: 150, // for "atomic writes"
-        alwaysStat: true,
-      });
-
-      const diffChangedConfig = diffChangedConfigFor(dependencies);
-
-      watcher
-        .on("change", (childFilePath, stats: Stats): void => {
-          const cleanup = cleanupFns.get(childFilePath);
-
-          if (!cleanup) {
-            // file was previously ignored, do nothing
-            return void logger.debug(`${logPrefix} ${inspect(childFilePath)} that should have been previously ignored has changed. Doing nothing`);
-          }
-
-          cleanup();
-          cleanupFns.set(childFilePath, diffChangedConfig({
-            filePath: childFilePath,
-            source: getOrInsertWith(rootSource, childFilePath, observable.map),
-            stats,
-            maxAllowedFileReadSize,
-          }));
-        })
-        .on("add", (childFilePath, stats: Stats): void => {
-          if (isFolderSync) {
-            const fileName = path.basename(childFilePath);
-
-            for (const ignoreGlob of ignoreGlobs) {
-              if (ignoreGlob.matcher.test(fileName)) {
-                return void logger.info(`${logPrefix} ignoring ${inspect(childFilePath)} due to ignore glob: ${ignoreGlob.rawGlob}`);
-              }
-            }
-          }
-
-          cleanupFns.set(childFilePath, diffChangedConfig({
-            filePath: childFilePath,
-            source: getOrInsertWith(rootSource, childFilePath, observable.map),
-            stats,
-            maxAllowedFileReadSize,
-          }));
-        })
-        .on("unlink", (childFilePath) => {
-          cleanupFns.get(childFilePath)?.();
-          cleanupFns.delete(childFilePath);
-          rootSource.delete(childFilePath);
-        })
-        .on("error", error => logger.error(`${logPrefix} watching file/folder failed: ${error}`, { filePath }));
-    } catch (error) {
-      console.log((error as { stack: unknown }).stack);
-      logger.warn(`${logPrefix} failed to start watching changes: ${error}`);
-    }
-  })();
-
-  return [derivedSource, () => {
-    watcher?.close();
-  }];
-};

src/main/catalog-sources/kubeconfig-sync/watch-file-changes.injectable.ts

@@ -0,0 +1,135 @@
+/**
+ * Copyright (c) OpenLens Authors. All rights reserved.
+ * Licensed under MIT License. See LICENSE in root directory for more information.
+ */
+import { getInjectable } from "@ogre-tools/injectable";
+import type { FSWatcher } from "chokidar";
+import type { Stats } from "fs";
+import GlobToRegExp from "glob-to-regexp";
+import type { IComputedValue, ObservableMap } from "mobx";
+import { computed, observable } from "mobx";
+import path from "path";
+import { inspect } from "util";
+import type { CatalogEntity } from "../../../common/catalog";
+import type { Cluster } from "../../../common/cluster/cluster";
+import fsInjectable from "../../../common/fs/fs.injectable";
+import watchInjectable from "../../../common/fs/watch.injectable";
+import type { Disposer } from "../../../common/utils";
+import { getOrInsertWith, iter } from "../../../common/utils";
+import diffChangedKubeconfigInjectable from "./diff-changed-kubeconfig.injectable";
+import kubeconfigSyncLoggerInjectable from "./logger.injectable";
+
+export type WatchKubeconfigFileChanges = (filepath: string) => [IComputedValue<CatalogEntity[]>, Disposer];
+
+/**
+ * This is the list of globs of which files are ignored when under a folder sync
+ */
+const ignoreGlobs = [
+  "*.lock", // kubectl lock files
+  "*.swp", // vim swap files
+  ".DS_Store", // macOS specific
+].map(rawGlob => ({
+  rawGlob,
+  matcher: GlobToRegExp(rawGlob),
+}));
+
+/**
+ * This should be much larger than any kubeconfig text file
+ *
+ * Even if you have a cert-file, key-file, and client-cert files that is only
+ * 12kb of extra data (at 4096 bytes each) which allows for around 150 entries.
+ */
+const folderSyncMaxAllowedFileReadSize = 2 * 1024 * 1024; // 2 MiB
+const fileSyncMaxAllowedFileReadSize = 16 * folderSyncMaxAllowedFileReadSize; // 32 MiB
+
+const watchKubeconfigFileChangesInjectable = getInjectable({
+  id: "watch-kubeconfig-file-changes",
+  instantiate: (di): WatchKubeconfigFileChanges => {
+    const diffChangedKubeconfig = di.inject(diffChangedKubeconfigInjectable);
+    const logger = di.inject(kubeconfigSyncLoggerInjectable);
+    const { stat } = di.inject(fsInjectable);
+    const watch = di.inject(watchInjectable);
+
+    return (filePath) => {
+      const rootSource = observable.map<string, ObservableMap<string, [Cluster, CatalogEntity]>>();
+      const derivedSource = computed(() => Array.from(iter.flatMap(rootSource.values(), from => iter.map(from.values(), child => child[1]))));
+
+      let watcher: FSWatcher;
+
+      (async () => {
+        try {
+          const stats = await stat(filePath);
+          const isFolderSync = stats.isDirectory();
+          const cleanupFns = new Map<string, Disposer>();
+          const maxAllowedFileReadSize = isFolderSync
+            ? folderSyncMaxAllowedFileReadSize
+            : fileSyncMaxAllowedFileReadSize;
+
+          watcher = watch(filePath, {
+            followSymlinks: true,
+            depth: isFolderSync ? 0 : 1, // DIRs works with 0 but files need 1 (bug: https://github.com/paulmillr/chokidar/issues/1095)
+            disableGlobbing: true,
+            ignorePermissionErrors: true,
+            usePolling: false,
+            awaitWriteFinish: {
+              pollInterval: 100,
+              stabilityThreshold: 1000,
+            },
+            atomic: 150, // for "atomic writes"
+            alwaysStat: true,
+          });
+
+          watcher
+            .on("change", (childFilePath, stats: Stats): void => {
+              const cleanup = cleanupFns.get(childFilePath);
+
+              if (!cleanup) {
+                // file was previously ignored, do nothing
+                return void logger.debug(`${inspect(childFilePath)} that should have been previously ignored has changed. Doing nothing`);
+              }
+
+              cleanup();
+              cleanupFns.set(childFilePath, diffChangedKubeconfig({
+                filePath: childFilePath,
+                source: getOrInsertWith(rootSource, childFilePath, observable.map),
+                stats,
+                maxAllowedFileReadSize,
+              }));
+            })
+            .on("add", (childFilePath, stats: Stats): void => {
+              if (isFolderSync) {
+                const fileName = path.basename(childFilePath);
+
+                for (const ignoreGlob of ignoreGlobs) {
+                  if (ignoreGlob.matcher.test(fileName)) {
+                    return void logger.info(`ignoring ${inspect(childFilePath)} due to ignore glob: ${ignoreGlob.rawGlob}`);
+                  }
+                }
+              }
+
+              cleanupFns.set(childFilePath, diffChangedKubeconfig({
+                filePath: childFilePath,
+                source: getOrInsertWith(rootSource, childFilePath, observable.map),
+                stats,
+                maxAllowedFileReadSize,
+              }));
+            })
+            .on("unlink", (childFilePath) => {
+              cleanupFns.get(childFilePath)?.();
+              cleanupFns.delete(childFilePath);
+              rootSource.delete(childFilePath);
+            })
+            .on("error", error => logger.error(`watching file/folder failed: ${error}`, { filePath }));
+        } catch (error) {
+          logger.warn(`failed to start watching changes: ${error}`);
+        }
+      })();
+
+      return [derivedSource, () => {
+        watcher?.close();
+      }];
+    };
+  },
+});
+
+export default watchKubeconfigFileChangesInjectable;

src/main/start-main-application/runnables/kube-config-sync/add-source.injectable.ts

@@ -0,0 +1,25 @@
+/**
+ * Copyright (c) OpenLens Authors. All rights reserved.
+ * Licensed under MIT License. See LICENSE in root directory for more information.
+ */
+import { getInjectable } from "@ogre-tools/injectable";
+import kubeconfigSyncManagerInjectable from "../../../catalog-sources/kubeconfig-sync/manager.injectable";
+import catalogEntityRegistryInjectable from "../../../catalog/entity-registry.injectable";
+import { afterRootFrameIsReadyInjectionToken } from "../../runnable-tokens/after-root-frame-is-ready-injection-token";
+
+const addKubeconfigSyncAsEntitySourceInjectable = getInjectable({
+  id: "add-kubeconfig-sync-as-entity-source",
+  instantiate: (di) => {
+    const kubeConfigSyncManager = di.inject(kubeconfigSyncManagerInjectable);
+    const entityRegistry = di.inject(catalogEntityRegistryInjectable);
+
+    return {
+      run: () => {
+        entityRegistry.addComputedSource("kubeconfig-sync", kubeConfigSyncManager.source);
+      },
+    };
+  },
+  injectionToken: afterRootFrameIsReadyInjectionToken,
+});
+
+export default addKubeconfigSyncAsEntitySourceInjectable;

src/main/start-main-application/runnables/kube-config-sync/start-kube-config-sync.injectable.ts

@@ -7,6 +7,7 @@ import { afterApplicationIsLoadedInjectionToken } from "../../runnable-tokens/af
 import directoryForKubeConfigsInjectable from "../../../../common/app-paths/directory-for-kube-configs/directory-for-kube-configs.injectable";
 import ensureDirInjectable from "../../../../common/fs/ensure-dir.injectable";
 import kubeconfigSyncManagerInjectable from "../../../catalog-sources/kubeconfig-sync/manager.injectable";
+import addKubeconfigSyncAsEntitySourceInjectable from "./add-source.injectable";
 
 const startKubeConfigSyncInjectable = getInjectable({
   id: "start-kubeconfig-sync",
@@ -22,6 +23,7 @@ const startKubeConfigSyncInjectable = getInjectable({
 
         kubeConfigSyncManager.startSync();
       },
+      runAfter: di.inject(addKubeconfigSyncAsEntitySourceInjectable),
     };
   },