hendrik/lens
1
0
Fork 0
mirror of https://github.com/lensapp/lens.git synced 2025-05-20 05:10:56 +00:00
Code Issues Projects Releases Wiki Activity

Fix kubeconfig-sync sometimes producing multiple identical entities

Browse Source 
- This happens when there is an overlapping sync of a folder and a file
  within that folder both producing the same entities

- This also fixes the LED not showing up on some KubernetesCluster
  instances because of this duplication the connected status was being
  set on the first instance, but renderer would use the second (because
  JS's Map constructor deduplicates its constructor initializer by
  taking the last)

- Added some tests to cover this case

Signed-off-by: Sebastian Malton <sebastian@malton.name>
This commit is contained in:
Sebastian Malton 2022-07-19 17:05:04 -04:00
parent 52b4c45e46
commit 8f3e851237
16 changed files with 657 additions and 342 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/common/cluster-store/get-by-id.injectable.ts Normal file
View File

@ -0,0 +1,21 @@
/**
* Copyright (c) OpenLens Authors. All rights reserved.
* Licensed under MIT License. See LICENSE in root directory for more information.
*/
import { getInjectable } from "@ogre-tools/injectable";
import type { ClusterId } from "../cluster-types";
import type { Cluster } from "../cluster/cluster";
import clusterStoreInjectable from "./cluster-store.injectable";
export type GetClusterById = (id: ClusterId) => Cluster | undefined;
const getClusterByIdInjectable = getInjectable({
id: "get-cluster-by-id",
instantiate: (di): GetClusterById => {
const store = di.inject(clusterStoreInjectable);
return (id) => store.getById(id);
},
});
export default getClusterByIdInjectable;

14
src/common/fs/watch.injectable.ts Normal file
View File

@ -0,0 +1,14 @@
/**
* Copyright (c) OpenLens Authors. All rights reserved.
* Licensed under MIT License. See LICENSE in root directory for more information.
*/
import { getInjectable } from "@ogre-tools/injectable";
import { watch } from "chokidar";
const watchInjectable = getInjectable({
id: "watch",
instantiate: () => watch,
causesSideEffects: true,
});
export default watchInjectable;

37
src/common/logger/child-logger.injectable.ts Normal file
View File

@ -0,0 +1,37 @@
/**
* Copyright (c) OpenLens Authors. All rights reserved.
* Licensed under MIT License. See LICENSE in root directory for more information.
*/
import { getInjectable, lifecycleEnum } from "@ogre-tools/injectable";
import type { Logger } from "../logger";
import loggerInjectable from "../logger.injectable";
const childLoggerInjectable = getInjectable({
id: "child-logger",
instantiate: (di, prefix): Logger => {
const logger = di.inject(loggerInjectable);
return {
debug: (message, ...args) => {
logger.debug(`[${prefix}]: ${message}`, ...args);
},
error: (message, ...args) => {
logger.error(`[${prefix}]: ${message}`, ...args);
},
info: (message, ...args) => {
logger.info(`[${prefix}]: ${message}`, ...args);
},
silly: (message, ...args) => {
logger.silly(`[${prefix}]: ${message}`, ...args);
},
warn: (message, ...args) => {
logger.warn(`[${prefix}]: ${message}`, ...args);
},
};
},
lifecycle: lifecycleEnum.keyedSingleton({
getInstanceKey: (di, prefix: string) => prefix,
}),
});
export default childLoggerInjectable;

17
src/common/user-store/kubeconfig-syncs.injectable.ts Normal file
View File

@ -0,0 +1,17 @@
/**
* Copyright (c) OpenLens Authors. All rights reserved.
* Licensed under MIT License. See LICENSE in root directory for more information.
*/
import { getInjectable } from "@ogre-tools/injectable";
import userStoreInjectable from "./user-store.injectable";
const kubeconfigSyncsInjectable = getInjectable({
id: "kubeconfig-syncs",
instantiate: (di) => {
const store = di.inject(userStoreInjectable);
return store.syncKubeconfigEntries;
},
});
export default kubeconfigSyncsInjectable;

2
src/common/utils/iter.ts
View File

@ -11,6 +11,7 @@ interface Iterator<T> {
find(fn: (val: T) => unknown): T | undefined;
collect<U>(fn: (values: Iterable<T>) => U): U;
map<U>(fn: (val: T) => U): Iterator<U>;
flatMap<U>(fn: (val: T) => U[]): Iterator<U>;
join(sep?: string): string;
}
@ -19,6 +20,7 @@ export function pipeline<T>(src: IterableIterator<T>): Iterator<T> {
filter: (fn) => pipeline(filter(src, fn)),
filterMap: (fn) => pipeline(filterMap(src, fn)),
map: (fn) => pipeline(map(src, fn)),
flatMap: (fn) => pipeline(flatMap(src, fn)),
find: (fn) => find(src, fn),
join: (sep) => join(src, sep),
collect: (fn) => fn(src),

125
src/main/catalog-sources/__test__/kubeconfig-sync.test.ts
View File

@ -3,19 +3,15 @@
* Licensed under MIT License. See LICENSE in root directory for more information.
*/
import { ObservableMap } from "mobx";
import { observable, ObservableMap, when } from "mobx";
import type { CatalogEntity } from "../../../common/catalog";
import { loadFromOptions } from "../../../common/kube-helpers";
import type { Cluster } from "../../../common/cluster/cluster";
import { computeDiff as computeDiffFor, configToModels } from "../kubeconfig-sync/manager";
import mockFs from "mock-fs";
import fs from "fs";
import clusterStoreInjectable from "../../../common/cluster-store/cluster-store.injectable";
import { getDiForUnitTesting } from "../../getDiForUnitTesting";
import { createClusterInjectionToken } from "../../../common/cluster/create-cluster-injection-token";
import directoryForKubeConfigsInjectable from "../../../common/app-paths/directory-for-kube-configs/directory-for-kube-configs.injectable";
import getConfigurationFileModelInjectable from "../../../common/get-configuration-file-model/get-configuration-file-model.injectable";
import clusterManagerInjectable from "../../cluster-manager.injectable";
import directoryForUserDataInjectable from "../../../common/app-paths/directory-for-user-data/directory-for-user-data.injectable";
import directoryForTempInjectable from "../../../common/app-paths/directory-for-temp/directory-for-temp.injectable";
import kubectlBinaryNameInjectable from "../../kubectl/binary-name.injectable";
@ -23,6 +19,17 @@ import kubectlDownloadingNormalizedArchInjectable from "../../kubectl/normalized
import normalizedPlatformInjectable from "../../../common/vars/normalized-platform.injectable";
import { iter } from "../../../common/utils";
import fsInjectable from "../../../common/fs/fs.injectable";
import type { ComputeKubeconfigDiff } from "../kubeconfig-sync/compute-diff.injectable";
import computeKubeconfigDiffInjectable from "../kubeconfig-sync/compute-diff.injectable";
import watchInjectable from "../../../common/fs/watch.injectable";
import type { ConfigToModels } from "../kubeconfig-sync/config-to-models.injectable";
import configToModelsInjectable from "../kubeconfig-sync/config-to-models.injectable";
import kubeconfigSyncManagerInjectable from "../kubeconfig-sync/manager.injectable";
import type { KubeconfigSyncManager } from "../kubeconfig-sync/manager";
import type { KubeconfigSyncValue } from "../../../common/user-store";
import kubeconfigSyncsInjectable from "../../../common/user-store/kubeconfig-syncs.injectable";
console.log("This is a reminder that mockFS breaks things and needs to be removed");
jest.mock("electron", () => ({
app: {
@ -41,7 +48,10 @@ jest.mock("electron", () => ({
}));
describe("kubeconfig-sync.source tests", () => {
let computeDiff: ReturnType<typeof computeDiffFor>;
let computeKubeconfigDiff: ComputeKubeconfigDiff;
let configToModels: ConfigToModels;
let manager: KubeconfigSyncManager;
let kubeconfigSyncs: ObservableMap<string, KubeconfigSyncValue>;
beforeEach(async () => {
const di = getDiForUnitTesting({ doGeneralOverrides: true });
@ -55,15 +65,18 @@ describe("kubeconfig-sync.source tests", () => {
di.override(normalizedPlatformInjectable, () => "darwin");
di.permitSideEffects(fsInjectable);
di.permitSideEffects(watchInjectable);
di.unoverride(clusterStoreInjectable);
di.permitSideEffects(clusterStoreInjectable);
di.permitSideEffects(getConfigurationFileModelInjectable);
computeDiff = computeDiffFor({
directoryForKubeConfigs: di.inject(directoryForKubeConfigsInjectable),
createCluster: di.inject(createClusterInjectionToken),
clusterManager: di.inject(clusterManagerInjectable),
});
kubeconfigSyncs = observable.map();
di.override(kubeconfigSyncsInjectable, () => kubeconfigSyncs);
computeKubeconfigDiff = di.inject(computeKubeconfigDiffInjectable);
configToModels = di.inject(configToModelsInjectable);
manager = di.inject(kubeconfigSyncManagerInjectable);
});
afterEach(() => {
@ -108,13 +121,13 @@ describe("kubeconfig-sync.source tests", () => {
});
});
describe("computeDiff", () => {
describe("computeKubeconfigDiff", () => {
it("should leave an empty source empty if there are no entries", () => {
const contents = "";
const rootSource = new ObservableMap<string, [Cluster, CatalogEntity]>();
const filePath = "/bar";
computeDiff(contents, rootSource, filePath);
computeKubeconfigDiff(contents, rootSource, filePath);
expect(rootSource.size).toBe(0);
});
@ -151,7 +164,7 @@ describe("kubeconfig-sync.source tests", () => {
fs.writeFileSync(filePath, contents);
computeDiff(contents, rootSource, filePath);
computeKubeconfigDiff(contents, rootSource, filePath);
expect(rootSource.size).toBe(1);
@ -195,7 +208,7 @@ describe("kubeconfig-sync.source tests", () => {
fs.writeFileSync(filePath, contents);
computeDiff(contents, rootSource, filePath);
computeKubeconfigDiff(contents, rootSource, filePath);
expect(rootSource.size).toBe(1);
@ -204,7 +217,7 @@ describe("kubeconfig-sync.source tests", () => {
expect(c.kubeConfigPath).toBe("/bar");
expect(c.contextName).toBe("context-name");
computeDiff("{}", rootSource, filePath);
computeKubeconfigDiff("{}", rootSource, filePath);
expect(rootSource.size).toBe(0);
});
@ -249,7 +262,7 @@ describe("kubeconfig-sync.source tests", () => {
fs.writeFileSync(filePath, contents);
computeDiff(contents, rootSource, filePath);
computeKubeconfigDiff(contents, rootSource, filePath);
expect(rootSource.size).toBe(2);
@ -289,7 +302,7 @@ describe("kubeconfig-sync.source tests", () => {
currentContext: "foobar",
});
computeDiff(newContents, rootSource, filePath);
computeKubeconfigDiff(newContents, rootSource, filePath);
expect(rootSource.size).toBe(1);
@ -301,4 +314,80 @@ describe("kubeconfig-sync.source tests", () => {
}
});
});
describe("given a config file at /foobar/config", () => {
beforeEach(() => {
fs.mkdirSync("/foobar");
fs.writeFileSync("/foobar/config", JSON.stringify({
clusters: [{
name: "cluster-name",
cluster: {
server: "1.2.3.4",
},
skipTLSVerify: false,
}],
users: [{
name: "user-name",
}],
contexts: [{
name: "context-name",
context: {
cluster: "cluster-name",
user: "user-name",
},
}, {
name: "context-the-second",
context: {
cluster: "missing-cluster",
user: "user-name",
},
}],
currentContext: "foobar",
}));
});
it("should not find any entities", () => {
expect(manager.source.get()).toEqual([]);
});
describe("when sync has started", () => {
beforeEach(() => {
manager.startSync();
});
it("should not find any entities", () => {
expect(manager.source.get()).toEqual([]);
});
describe("when a file sync target for /foobar/config is added", () => {
beforeEach(() => {
kubeconfigSyncs.set("/foobar/config", {});
});
it("should find a single entity", (done) => {
when(() => manager.source.get().length === 1, () => done());
});
describe("when a folder sync target for /foobar is added", () => {
beforeEach(() => {
kubeconfigSyncs.set("/foobar", {});
});
it("should still only find a single entity", (done) => {
when(() => manager.source.get().length === 1, () => done());
});
});
});
describe("when a folder sync target for /foobar is added", () => {
beforeEach(() => {
kubeconfigSyncs.set("/foobar", {});
});
it("should find a single entity", (done) => {
when(() => manager.source.get().length === 1, () => done());
});
});
});
});
});

100
src/main/catalog-sources/kubeconfig-sync/compute-diff.injectable.ts Normal file
View File

@ -0,0 +1,100 @@
/**
* Copyright (c) OpenLens Authors. All rights reserved.
* Licensed under MIT License. See LICENSE in root directory for more information.
*/
import { getInjectable } from "@ogre-tools/injectable";
import { createHash } from "crypto";
import type { ObservableMap } from "mobx";
import { action } from "mobx";
import { homedir } from "os";
import directoryForKubeConfigsInjectable from "../../../common/app-paths/directory-for-kube-configs/directory-for-kube-configs.injectable";
import type { CatalogEntity } from "../../../common/catalog";
import getClusterByIdInjectable from "../../../common/cluster-store/get-by-id.injectable";
import type { Cluster } from "../../../common/cluster/cluster";
import { loadConfigFromString } from "../../../common/kube-helpers";
import { catalogEntityFromCluster } from "../../cluster-manager";
import clusterManagerInjectable from "../../cluster-manager.injectable";
import createClusterInjectable from "../../create-cluster/create-cluster.injectable";
import configToModelsInjectable from "./config-to-models.injectable";
import kubeconfigSyncLoggerInjectable from "./logger.injectable";
export type ComputeKubeconfigDiff = (contents: string, source: ObservableMap<string, [Cluster, CatalogEntity]>, filePath: string) => void;
const computeKubeconfigDiffInjectable = getInjectable({
id: "compute-kubeconfig-diff",
instantiate: (di): ComputeKubeconfigDiff => {
const directoryForKubeConfigs = di.inject(directoryForKubeConfigsInjectable);
const createCluster = di.inject(createClusterInjectable);
const clusterManager = di.inject(clusterManagerInjectable);
const configToModels = di.inject(configToModelsInjectable);
const logger = di.inject(kubeconfigSyncLoggerInjectable);
const getClusterById = di.inject(getClusterByIdInjectable);
return action((contents, source, filePath) => {
try {
const { config, error } = loadConfigFromString(contents);
if (error) {
logger.warn(`encountered errors while loading config: ${error.message}`, { filePath, details: error.details });
}
const rawModels = configToModels(config, filePath);
const models = new Map(rawModels.map(([model, configData]) => [model.contextName, [model, configData] as const]));
logger.debug(`File now has ${models.size} entries`, { filePath });
for (const [contextName, value] of source) {
const data = models.get(contextName);
// remove and disconnect clusters that were removed from the config
if (!data) {
// remove from the deleting set, so that if a new context of the same name is added, it isn't marked as deleting
clusterManager.deleting.delete(value[0].id);
value[0].disconnect();
source.delete(contextName);
logger.debug(`Removed old cluster from sync`, { filePath, contextName });
continue;
}
// TODO: For the update check we need to make sure that the config itself hasn't changed.
// Probably should make it so that cluster keeps a copy of the config in its memory and
// diff against that
// or update the model and mark it as not needed to be added
value[0].updateModel(data[0]);
models.delete(contextName);
logger.debug(`Updated old cluster from sync`, { filePath, contextName });
}
for (const [contextName, [model, configData]] of models) {
// add new clusters to the source
try {
const clusterId = createHash("md5").update(`${filePath}:${contextName}`).digest("hex");
const cluster = getClusterById(clusterId) ?? createCluster({ ...model, id: clusterId }, configData);
if (!cluster.apiUrl) {
throw new Error("Cluster constructor failed, see above error");
}
const entity = catalogEntityFromCluster(cluster);
if (!filePath.startsWith(directoryForKubeConfigs)) {
entity.metadata.labels.file = filePath.replace(homedir(), "~");
}
source.set(contextName, [cluster, entity]);
logger.debug(`Added new cluster from sync`, { filePath, contextName });
} catch (error) {
logger.warn(`Failed to create cluster from model: ${error}`, { filePath, contextName });
}
}
} catch (error) {
logger.warn(`Failed to compute diff: ${error}`, { filePath });
source.clear(); // clear source if we have failed so as to not show outdated information
}
});
},
});
export default computeKubeconfigDiffInjectable;

42
src/main/catalog-sources/kubeconfig-sync/config-to-models.injectable.ts Normal file
View File

@ -0,0 +1,42 @@
/**
* Copyright (c) OpenLens Authors. All rights reserved.
* Licensed under MIT License. See LICENSE in root directory for more information.
*/
import type { KubeConfig } from "@kubernetes/client-node";
import { getInjectable } from "@ogre-tools/injectable";
import type { ClusterConfigData, UpdateClusterModel } from "../../../common/cluster-types";
import { splitConfig } from "../../../common/kube-helpers";
import kubeconfigSyncLoggerInjectable from "./logger.injectable";
export type ConfigToModels = (rootConfig: KubeConfig, filePath: string) => [UpdateClusterModel, ClusterConfigData][];
const configToModelsInjectable = getInjectable({
id: "config-to-models",
instantiate: (di): ConfigToModels => {
const logger = di.inject(kubeconfigSyncLoggerInjectable);
return (rootConfig, filePath) => {
const validConfigs: ReturnType<ConfigToModels> = [];
for (const { config, validationResult } of splitConfig(rootConfig)) {
if (validationResult.error) {
logger.debug(`context failed validation: ${validationResult.error}`, { context: config.currentContext, filePath });
} else {
validConfigs.push([
{
kubeConfigPath: filePath,
contextName: config.currentContext,
},
{
clusterServerUrl: validationResult.cluster.server,
},
]);
}
}
return validConfigs;
};
},
});
export default configToModelsInjectable;

90
src/main/catalog-sources/kubeconfig-sync/diff-changed-kubeconfig.injectable.ts Normal file
View File

@ -0,0 +1,90 @@
/**
* Copyright (c) OpenLens Authors. All rights reserved.
* Licensed under MIT License. See LICENSE in root directory for more information.
*/
import { getInjectable } from "@ogre-tools/injectable";
import type { Stats } from "fs";
import { constants } from "fs";
import type { ObservableMap } from "mobx";
import type { Readable } from "stream";
import type { CatalogEntity } from "../../../common/catalog";
import type { Cluster } from "../../../common/cluster/cluster";
import fsInjectable from "../../../common/fs/fs.injectable";
import type { Disposer } from "../../../common/utils";
import { bytesToUnits, noop } from "../../../common/utils";
import computeKubeconfigDiffInjectable from "./compute-diff.injectable";
import kubeconfigSyncLoggerInjectable from "./logger.injectable";
export interface DiffChangedKubeconfigArgs {
filePath: string;
source: ObservableMap<string, [Cluster, CatalogEntity]>;
stats: Stats;
maxAllowedFileReadSize: number;
}
export type DiffChangedKubeconfig = (args: DiffChangedKubeconfigArgs) => Disposer;
const diffChangedKubeconfigInjectable = getInjectable({
id: "diff-changed-kubeconfig",
instantiate: (di): DiffChangedKubeconfig => {
const computeKubeconfigDiff = di.inject(computeKubeconfigDiffInjectable);
const logger = di.inject(kubeconfigSyncLoggerInjectable);
const { createReadStream } = di.inject(fsInjectable);
return ({ filePath, maxAllowedFileReadSize, source, stats }) => {
logger.debug(`file changed`, { filePath });
if (stats.size >= maxAllowedFileReadSize) {
logger.warn(`skipping ${filePath}: size=${bytesToUnits(stats.size)} is larger than maxSize=${bytesToUnits(maxAllowedFileReadSize)}`);
source.clear();
return noop;
}
const fileReader = createReadStream(filePath, {
mode: constants.O_RDONLY,
});
const readStream = fileReader as Readable;
const decoder = new TextDecoder("utf-8", { fatal: true });
let fileString = "";
let closed = false;
const cleanup = () => {
closed = true;
fileReader.close(); // This may not close the stream.
// Artificially marking end-of-stream, as if the underlying resource had
// indicated end-of-file by itself, allows the stream to close.
// This does not cancel pending read operations, and if there is such an
// operation, the process may still not be able to exit successfully
// until it finishes.
fileReader.push(null);
fileReader.read(0);
readStream.removeAllListeners();
};
readStream
.on("data", (chunk: Buffer) => {
try {
fileString += decoder.decode(chunk, { stream: true });
} catch (error) {
logger.warn(`skipping ${filePath}: ${error}`);
source.clear();
cleanup();
}
})
.on("close", () => cleanup())
.on("error", error => {
cleanup();
logger.warn(`failed to read file: ${error}`, { filePath });
})
.on("end", () => {
if (!closed) {
computeKubeconfigDiff(fileString, source, filePath);
}
});
return cleanup;
};
},
});
export default diffChangedKubeconfigInjectable;

13
src/main/catalog-sources/kubeconfig-sync/logger.injectable.ts Normal file
View File

@ -0,0 +1,13 @@
/**
* Copyright (c) OpenLens Authors. All rights reserved.
* Licensed under MIT License. See LICENSE in root directory for more information.
*/
import { getInjectable } from "@ogre-tools/injectable";
import childLoggerInjectable from "../../../common/logger/child-logger.injectable";
const kubeconfigSyncLoggerInjectable = getInjectable({
id: "kubeconfig-sync-logger",
instantiate: (di) => di.inject(childLoggerInjectable, "KUBECONFIG-SYNC"),
});
export default kubeconfigSyncLoggerInjectable;

12
src/main/catalog-sources/kubeconfig-sync/manager.injectable.ts
View File

@ -5,18 +5,18 @@
import { getInjectable } from "@ogre-tools/injectable";
import directoryForKubeConfigsInjectable from "../../../common/app-paths/directory-for-kube-configs/directory-for-kube-configs.injectable";
import { KubeconfigSyncManager } from "./manager";
import { createClusterInjectionToken } from "../../../common/cluster/create-cluster-injection-token";
import clusterManagerInjectable from "../../cluster-manager.injectable";
import catalogEntityRegistryInjectable from "../../catalog/entity-registry.injectable";
import kubeconfigSyncLoggerInjectable from "./logger.injectable";
import watchKubeconfigFileChangesInjectable from "./watch-file-changes.injectable";
import kubeconfigSyncsInjectable from "../../../common/user-store/kubeconfig-syncs.injectable";
const kubeconfigSyncManagerInjectable = getInjectable({
id: "kubeconfig-sync-manager",
instantiate: (di) => new KubeconfigSyncManager({
directoryForKubeConfigs: di.inject(directoryForKubeConfigsInjectable),
createCluster: di.inject(createClusterInjectionToken),
clusterManager: di.inject(clusterManagerInjectable),
entityRegistry: di.inject(catalogEntityRegistryInjectable),
logger: di.inject(kubeconfigSyncLoggerInjectable),
watchKubeconfigFileChanges: di.inject(watchKubeconfigFileChangesInjectable),
kubeconfigSyncs: di.inject(kubeconfigSyncsInjectable),
}),
});

349
src/main/catalog-sources/kubeconfig-sync/manager.ts
View File

@ -4,97 +4,59 @@
*/
import type { IComputedValue, ObservableMap } from "mobx";
import { action, observable, computed, runInAction, makeObservable, observe } from "mobx";
import { action, observable, computed, makeObservable, observe } from "mobx";
import type { CatalogEntity } from "../../../common/catalog";
import type { FSWatcher } from "chokidar";
import { watch } from "chokidar";
import type { Stats } from "fs";
import fs from "fs";
import path from "path";
import type { Disposer } from "../../../common/utils";
import { disposer, bytesToUnits, getOrInsertWith, iter, noop } from "../../../common/utils";
import logger from "../../logger";
import type { KubeConfig } from "@kubernetes/client-node";
import { loadConfigFromString, splitConfig } from "../../../common/kube-helpers";
import type { ClusterManager } from "../../cluster-manager";
import { catalogEntityFromCluster } from "../../cluster-manager";
import { UserStore } from "../../../common/user-store";
import { ClusterStore } from "../../../common/cluster-store/cluster-store";
import { createHash } from "crypto";
import { homedir } from "os";
import globToRegExp from "glob-to-regexp";
import { inspect } from "util";
import type { ClusterConfigData, UpdateClusterModel } from "../../../common/cluster-types";
import type { Cluster } from "../../../common/cluster/cluster";
import type { CatalogEntityRegistry } from "../../catalog/entity-registry";
import type { CreateCluster } from "../../../common/cluster/create-cluster-injection-token";
const logPrefix = "[KUBECONFIG-SYNC]:";
/**
* This is the list of globs of which files are ignored when under a folder sync
*/
const ignoreGlobs = [
"*.lock", // kubectl lock files
"*.swp", // vim swap files
".DS_Store", // macOS specific
].map(rawGlob => ({
rawGlob,
matcher: globToRegExp(rawGlob),
}));
/**
* This should be much larger than any kubeconfig text file
*
* Even if you have a cert-file, key-file, and client-cert files that is only
* 12kb of extra data (at 4096 bytes each) which allows for around 150 entries.
*/
const folderSyncMaxAllowedFileReadSize = 2 * 1024 * 1024; // 2 MiB
const fileSyncMaxAllowedFileReadSize = 16 * folderSyncMaxAllowedFileReadSize; // 32 MiB
import { iter } from "../../../common/utils";
import type { KubeconfigSyncValue } from "../../../common/user-store";
import type { Logger } from "../../../common/logger";
import type { WatchKubeconfigFileChanges } from "./watch-file-changes.injectable";
interface KubeconfigSyncManagerDependencies {
readonly directoryForKubeConfigs: string;
readonly entityRegistry: CatalogEntityRegistry;
readonly clusterManager: ClusterManager;
createCluster: CreateCluster;
readonly logger: Logger;
readonly kubeconfigSyncs: ObservableMap<string, KubeconfigSyncValue>;
watchKubeconfigFileChanges: WatchKubeconfigFileChanges;
}
const kubeConfigSyncName = "lens:kube-sync";
export class KubeconfigSyncManager {
protected readonly sources = observable.map<string, [IComputedValue<CatalogEntity[]>, Disposer]>();
protected syncing = false;
protected syncListDisposer?: Disposer;
constructor(protected readonly dependencies: KubeconfigSyncManagerDependencies) {
makeObservable(this);
}
public readonly source = computed(() => {
/**
* This prevents multiple overlapping syncs from leading to multiple entities with the same IDs
*/
const seenIds = new Set<string>();
return (
iter.pipeline(this.sources.values())
.flatMap(([entities]) => entities.get())
.filter(entity => (
seenIds.has(entity.getId())
? false
: seenIds.add(entity.getId())
))
.collect(items => [...items])
);
});
@action
startSync(): void {
if (this.syncing) {
return;
}
this.syncing = true;
logger.info(`${logPrefix} starting requested syncs`);
this.dependencies.entityRegistry.addComputedSource(kubeConfigSyncName, computed(() => (
Array.from(iter.flatMap(
this.sources.values(),
([entities]) => entities.get(),
))
)));
this.dependencies.logger.info(`starting requested syncs`);
// This must be done so that c&p-ed clusters are visible
this.startNewSync(this.dependencies.directoryForKubeConfigs);
for (const filePath of UserStore.getInstance().syncKubeconfigEntries.keys()) {
for (const filePath of this.dependencies.kubeconfigSyncs.keys()) {
this.startNewSync(filePath);
}
this.syncListDisposer = observe(UserStore.getInstance().syncKubeconfigEntries, change => {
this.syncListDisposer = observe(this.dependencies.kubeconfigSyncs, change => {
switch (change.type) {
case "add":
this.startNewSync(change.name);
@ -108,275 +70,38 @@ export class KubeconfigSyncManager {
@action
stopSync() {
this.dependencies.logger.info(`stopping requested syncs`);
this.syncListDisposer?.();
for (const filePath of this.sources.keys()) {
this.stopOldSync(filePath);
}
this.dependencies.entityRegistry.removeSource(kubeConfigSyncName);
this.syncing = false;
}
@action
protected startNewSync(filePath: string): void {
if (this.sources.has(filePath)) {
// don't start a new sync if we already have one
return void logger.debug(`${logPrefix} already syncing file/folder`, { filePath });
return this.dependencies.logger.debug(`already syncing file/folder`, { filePath });
}
this.sources.set(
filePath,
watchFileChanges(filePath, this.dependencies),
this.dependencies.watchKubeconfigFileChanges(filePath),
);
logger.info(`${logPrefix} starting sync of file/folder`, { filePath });
logger.debug(`${logPrefix} ${this.sources.size} files/folders watched`, { files: Array.from(this.sources.keys()) });
this.dependencies.logger.info(`starting sync of file/folder`, { filePath });
this.dependencies.logger.debug(`${this.sources.size} files/folders watched`, { files: Array.from(this.sources.keys()) });
}
@action
protected stopOldSync(filePath: string): void {
if (!this.sources.delete(filePath)) {
// already stopped
return void logger.debug(`${logPrefix} no syncing file/folder to stop`, { filePath });
return this.dependencies.logger.debug(`no syncing file/folder to stop`, { filePath });
}
logger.info(`${logPrefix} stopping sync of file/folder`, { filePath });
logger.debug(`${logPrefix} ${this.sources.size} files/folders watched`, { files: Array.from(this.sources.keys()) });
this.dependencies.logger.info(`stopping sync of file/folder`, { filePath });
this.dependencies.logger.debug(`${this.sources.size} files/folders watched`, { files: Array.from(this.sources.keys()) });
}
}
// exported for testing
export function configToModels(rootConfig: KubeConfig, filePath: string): [UpdateClusterModel, ClusterConfigData][] {
const validConfigs: ReturnType<typeof configToModels> = [];
for (const { config, validationResult } of splitConfig(rootConfig)) {
if (validationResult.error) {
logger.debug(`${logPrefix} context failed validation: ${validationResult.error}`, { context: config.currentContext, filePath });
} else {
validConfigs.push([
{
kubeConfigPath: filePath,
contextName: config.currentContext,
},
{
clusterServerUrl: validationResult.cluster.server,
},
]);
}
}
return validConfigs;
}
type RootSourceValue = [Cluster, CatalogEntity];
type RootSource = ObservableMap<string, RootSourceValue>;
interface ComputeDiffDependencies {
directoryForKubeConfigs: string;
createCluster: CreateCluster;
clusterManager: ClusterManager;
}
// exported for testing
export const computeDiff = ({ directoryForKubeConfigs, createCluster, clusterManager }: ComputeDiffDependencies) => (contents: string, source: RootSource, filePath: string): void => {
runInAction(() => {
try {
const { config, error } = loadConfigFromString(contents);
if (error) {
logger.warn(`${logPrefix} encountered errors while loading config: ${error.message}`, { filePath, details: error.details });
}
const rawModels = configToModels(config, filePath);
const models = new Map(rawModels.map(([model, configData]) => [model.contextName, [model, configData] as const]));
logger.debug(`${logPrefix} File now has ${models.size} entries`, { filePath });
for (const [contextName, value] of source) {
const data = models.get(contextName);
// remove and disconnect clusters that were removed from the config
if (!data) {
// remove from the deleting set, so that if a new context of the same name is added, it isn't marked as deleting
clusterManager.deleting.delete(value[0].id);
value[0].disconnect();
source.delete(contextName);
logger.debug(`${logPrefix} Removed old cluster from sync`, { filePath, contextName });
continue;
}
// TODO: For the update check we need to make sure that the config itself hasn't changed.
// Probably should make it so that cluster keeps a copy of the config in its memory and
// diff against that
// or update the model and mark it as not needed to be added
value[0].updateModel(data[0]);
models.delete(contextName);
logger.debug(`${logPrefix} Updated old cluster from sync`, { filePath, contextName });
}
for (const [contextName, [model, configData]] of models) {
// add new clusters to the source
try {
const clusterId = createHash("md5").update(`${filePath}:${contextName}`).digest("hex");
const cluster = ClusterStore.getInstance().getById(clusterId) || createCluster({ ...model, id: clusterId }, configData);
if (!cluster.apiUrl) {
throw new Error("Cluster constructor failed, see above error");
}
const entity = catalogEntityFromCluster(cluster);
if (!filePath.startsWith(directoryForKubeConfigs)) {
entity.metadata.labels.file = filePath.replace(homedir(), "~");
}
source.set(contextName, [cluster, entity]);
logger.debug(`${logPrefix} Added new cluster from sync`, { filePath, contextName });
} catch (error) {
logger.warn(`${logPrefix} Failed to create cluster from model: ${error}`, { filePath, contextName });
}
}
} catch (error) {
logger.warn(`${logPrefix} Failed to compute diff: ${error}`, { filePath });
source.clear(); // clear source if we have failed so as to not show outdated information
}
});
};
interface DiffChangedConfigArgs {
filePath: string;
source: RootSource;
stats: fs.Stats;
maxAllowedFileReadSize: number;
}
const diffChangedConfigFor = (dependencies: ComputeDiffDependencies) => ({ filePath, source, stats, maxAllowedFileReadSize }: DiffChangedConfigArgs): Disposer => {
logger.debug(`${logPrefix} file changed`, { filePath });
if (stats.size >= maxAllowedFileReadSize) {
logger.warn(`${logPrefix} skipping ${filePath}: size=${bytesToUnits(stats.size)} is larger than maxSize=${bytesToUnits(maxAllowedFileReadSize)}`);
source.clear();
return noop;
}
const controller = new AbortController();
const fileContentsP = fs.promises.readFile(filePath, {
signal: controller.signal,
});
const cleanup = disposer(
() => controller.abort(),
);
fileContentsP
.then((fileData) => {
const decoder = new TextDecoder("utf-8", { fatal: true });
try {
const fileString = decoder.decode(fileData);
computeDiff(dependencies)(fileString, source, filePath);
} catch (error) {
logger.warn(`${logPrefix} skipping ${filePath}: ${error}`);
source.clear();
cleanup();
}
})
.catch(error => {
if (controller.signal.aborted) {
return;
}
logger.warn(`${logPrefix} failed to read file: ${error}`, { filePath });
cleanup();
});
return cleanup;
};
const watchFileChanges = (filePath: string, dependencies: ComputeDiffDependencies): [IComputedValue<CatalogEntity[]>, Disposer] => {
const rootSource = observable.map<string, ObservableMap<string, RootSourceValue>>();
const derivedSource = computed(() => Array.from(iter.flatMap(rootSource.values(), from => iter.map(from.values(), child => child[1]))));
let watcher: FSWatcher;
(async () => {
try {
const stat = await fs.promises.stat(filePath);
const isFolderSync = stat.isDirectory();
const cleanupFns = new Map<string, Disposer>();
const maxAllowedFileReadSize = isFolderSync
? folderSyncMaxAllowedFileReadSize
: fileSyncMaxAllowedFileReadSize;
watcher = watch(filePath, {
followSymlinks: true,
depth: isFolderSync ? 0 : 1, // DIRs works with 0 but files need 1 (bug: https://github.com/paulmillr/chokidar/issues/1095)
disableGlobbing: true,
ignorePermissionErrors: true,
usePolling: false,
awaitWriteFinish: {
pollInterval: 100,
stabilityThreshold: 1000,
},
atomic: 150, // for "atomic writes"
alwaysStat: true,
});
const diffChangedConfig = diffChangedConfigFor(dependencies);
watcher
.on("change", (childFilePath, stats: Stats): void => {
const cleanup = cleanupFns.get(childFilePath);
if (!cleanup) {
// file was previously ignored, do nothing
return void logger.debug(`${logPrefix} ${inspect(childFilePath)} that should have been previously ignored has changed. Doing nothing`);
}
cleanup();
cleanupFns.set(childFilePath, diffChangedConfig({
filePath: childFilePath,
source: getOrInsertWith(rootSource, childFilePath, observable.map),
stats,
maxAllowedFileReadSize,
}));
})
.on("add", (childFilePath, stats: Stats): void => {
if (isFolderSync) {
const fileName = path.basename(childFilePath);
for (const ignoreGlob of ignoreGlobs) {
if (ignoreGlob.matcher.test(fileName)) {
return void logger.info(`${logPrefix} ignoring ${inspect(childFilePath)} due to ignore glob: ${ignoreGlob.rawGlob}`);
}
}
}
cleanupFns.set(childFilePath, diffChangedConfig({
filePath: childFilePath,
source: getOrInsertWith(rootSource, childFilePath, observable.map),
stats,
maxAllowedFileReadSize,
}));
})
.on("unlink", (childFilePath) => {
cleanupFns.get(childFilePath)?.();
cleanupFns.delete(childFilePath);
rootSource.delete(childFilePath);
})
.on("error", error => logger.error(`${logPrefix} watching file/folder failed: ${error}`, { filePath }));
} catch (error) {
console.log((error as { stack: unknown }).stack);
logger.warn(`${logPrefix} failed to start watching changes: ${error}`);
}
})();
return [derivedSource, () => {
watcher?.close();
}];
};

135
src/main/catalog-sources/kubeconfig-sync/watch-file-changes.injectable.ts Normal file
View File

@ -0,0 +1,135 @@
/**
* Copyright (c) OpenLens Authors. All rights reserved.
* Licensed under MIT License. See LICENSE in root directory for more information.
*/
import { getInjectable } from "@ogre-tools/injectable";
import type { FSWatcher } from "chokidar";
import type { Stats } from "fs";
import GlobToRegExp from "glob-to-regexp";
import type { IComputedValue, ObservableMap } from "mobx";
import { computed, observable } from "mobx";
import path from "path";
import { inspect } from "util";
import type { CatalogEntity } from "../../../common/catalog";
import type { Cluster } from "../../../common/cluster/cluster";
import fsInjectable from "../../../common/fs/fs.injectable";
import watchInjectable from "../../../common/fs/watch.injectable";
import type { Disposer } from "../../../common/utils";
import { getOrInsertWith, iter } from "../../../common/utils";
import diffChangedKubeconfigInjectable from "./diff-changed-kubeconfig.injectable";
import kubeconfigSyncLoggerInjectable from "./logger.injectable";
export type WatchKubeconfigFileChanges = (filepath: string) => [IComputedValue<CatalogEntity[]>, Disposer];
/**
* This is the list of globs of which files are ignored when under a folder sync
*/
const ignoreGlobs = [
"*.lock", // kubectl lock files
"*.swp", // vim swap files
".DS_Store", // macOS specific
].map(rawGlob => ({
rawGlob,
matcher: GlobToRegExp(rawGlob),
}));
/**
* This should be much larger than any kubeconfig text file
*
* Even if you have a cert-file, key-file, and client-cert files that is only
* 12kb of extra data (at 4096 bytes each) which allows for around 150 entries.
*/
const folderSyncMaxAllowedFileReadSize = 2 * 1024 * 1024; // 2 MiB
const fileSyncMaxAllowedFileReadSize = 16 * folderSyncMaxAllowedFileReadSize; // 32 MiB
const watchKubeconfigFileChangesInjectable = getInjectable({
id: "watch-kubeconfig-file-changes",
instantiate: (di): WatchKubeconfigFileChanges => {
const diffChangedKubeconfig = di.inject(diffChangedKubeconfigInjectable);
const logger = di.inject(kubeconfigSyncLoggerInjectable);
const { stat } = di.inject(fsInjectable);
const watch = di.inject(watchInjectable);
return (filePath) => {
const rootSource = observable.map<string, ObservableMap<string, [Cluster, CatalogEntity]>>();
const derivedSource = computed(() => Array.from(iter.flatMap(rootSource.values(), from => iter.map(from.values(), child => child[1]))));
let watcher: FSWatcher;
(async () => {
try {
const stats = await stat(filePath);
const isFolderSync = stats.isDirectory();
const cleanupFns = new Map<string, Disposer>();
const maxAllowedFileReadSize = isFolderSync
? folderSyncMaxAllowedFileReadSize
: fileSyncMaxAllowedFileReadSize;
watcher = watch(filePath, {
followSymlinks: true,
depth: isFolderSync ? 0 : 1, // DIRs works with 0 but files need 1 (bug: https://github.com/paulmillr/chokidar/issues/1095)
disableGlobbing: true,
ignorePermissionErrors: true,
usePolling: false,
awaitWriteFinish: {
pollInterval: 100,
stabilityThreshold: 1000,
},
atomic: 150, // for "atomic writes"
alwaysStat: true,
});
watcher
.on("change", (childFilePath, stats: Stats): void => {
const cleanup = cleanupFns.get(childFilePath);
if (!cleanup) {
// file was previously ignored, do nothing
return void logger.debug(`${inspect(childFilePath)} that should have been previously ignored has changed. Doing nothing`);
}
cleanup();
cleanupFns.set(childFilePath, diffChangedKubeconfig({
filePath: childFilePath,
source: getOrInsertWith(rootSource, childFilePath, observable.map),
stats,
maxAllowedFileReadSize,
}));
})
.on("add", (childFilePath, stats: Stats): void => {
if (isFolderSync) {
const fileName = path.basename(childFilePath);
for (const ignoreGlob of ignoreGlobs) {
if (ignoreGlob.matcher.test(fileName)) {
return void logger.info(`ignoring ${inspect(childFilePath)} due to ignore glob: ${ignoreGlob.rawGlob}`);
}
}
}
cleanupFns.set(childFilePath, diffChangedKubeconfig({
filePath: childFilePath,
source: getOrInsertWith(rootSource, childFilePath, observable.map),
stats,
maxAllowedFileReadSize,
}));
})
.on("unlink", (childFilePath) => {
cleanupFns.get(childFilePath)?.();
cleanupFns.delete(childFilePath);
rootSource.delete(childFilePath);
})
.on("error", error => logger.error(`watching file/folder failed: ${error}`, { filePath }));
} catch (error) {
logger.warn(`failed to start watching changes: ${error}`);
}
})();
return [derivedSource, () => {
watcher?.close();
}];
};
},
});
export default watchKubeconfigFileChangesInjectable;

25
src/main/start-main-application/runnables/kube-config-sync/add-source.injectable.ts Normal file
View File

@ -0,0 +1,25 @@
/**
* Copyright (c) OpenLens Authors. All rights reserved.
* Licensed under MIT License. See LICENSE in root directory for more information.
*/
import { getInjectable } from "@ogre-tools/injectable";
import kubeconfigSyncManagerInjectable from "../../../catalog-sources/kubeconfig-sync/manager.injectable";
import catalogEntityRegistryInjectable from "../../../catalog/entity-registry.injectable";
import { afterRootFrameIsReadyInjectionToken } from "../../runnable-tokens/after-root-frame-is-ready-injection-token";
const addKubeconfigSyncAsEntitySourceInjectable = getInjectable({
id: "add-kubeconfig-sync-as-entity-source",
instantiate: (di) => {
const kubeConfigSyncManager = di.inject(kubeconfigSyncManagerInjectable);
const entityRegistry = di.inject(catalogEntityRegistryInjectable);
return {
run: () => {
entityRegistry.addComputedSource("kubeconfig-sync", kubeConfigSyncManager.source);
},
};
},
injectionToken: afterRootFrameIsReadyInjectionToken,
});
export default addKubeconfigSyncAsEntitySourceInjectable;

2
src/main/start-main-application/runnables/kube-config-sync/start-kube-config-sync.injectable.ts
View File

@ -7,6 +7,7 @@ import { afterApplicationIsLoadedInjectionToken } from "../../runnable-tokens/af
import directoryForKubeConfigsInjectable from "../../../../common/app-paths/directory-for-kube-configs/directory-for-kube-configs.injectable";
import ensureDirInjectable from "../../../../common/fs/ensure-dir.injectable";
import kubeconfigSyncManagerInjectable from "../../../catalog-sources/kubeconfig-sync/manager.injectable";
import addKubeconfigSyncAsEntitySourceInjectable from "./add-source.injectable";
const startKubeConfigSyncInjectable = getInjectable({
id: "start-kubeconfig-sync",
@ -22,6 +23,7 @@ const startKubeConfigSyncInjectable = getInjectable({
kubeConfigSyncManager.startSync();
},
runAfter: di.inject(addKubeconfigSyncAsEntitySourceInjectable),
};
},

15
src/renderer/components/hotbar/hotbar-entity-icon.tsx
View File

@ -17,11 +17,12 @@ import { Icon } from "../icon";
import { HotbarIcon } from "./hotbar-icon";
import { LensKubernetesClusterStatus } from "../../../common/catalog-entities/kubernetes-cluster";
import type { VisitEntityContextMenu } from "../../../common/catalog/visit-entity-context-menu.injectable";
import { navigate } from "../../navigation";
import { withInjectables } from "@ogre-tools/injectable-react";
import catalogCategoryRegistryInjectable from "../../../common/catalog/category-registry.injectable";
import visitEntityContextMenuInjectable from "../../../common/catalog/visit-entity-context-menu.injectable";
import activeEntityInjectable from "../../api/catalog/entity/active.injectable";
import type { Navigate } from "../../navigation/navigate.injectable";
import navigateInjectable from "../../navigation/navigate.injectable";
export interface HotbarEntityIconProps {
entity: CatalogEntity;
@ -38,13 +39,14 @@ interface Dependencies {
visitEntityContextMenu: VisitEntityContextMenu;
catalogCategoryRegistry: CatalogCategoryRegistry;
activeEntity: IComputedValue<CatalogEntity | undefined>;
navigate: Navigate;
}
@observer
class NonInjectedHotbarEntityIcon extends React.Component<HotbarEntityIconProps & Dependencies> {
private readonly menuItems = observable.array<CatalogEntityContextMenu>();
get kindIcon() {
private renderKindIcon() {
const className = styles.badge;
const category = this.props.catalogCategoryRegistry.getCategoryForEntity(this.props.entity);
@ -59,7 +61,7 @@ class NonInjectedHotbarEntityIcon extends React.Component<HotbarEntityIconProps
return <Icon material={category.metadata.icon} className={className} />;
}
get ledIcon() {
private renderLedIcon() {
if (this.props.entity.kind !== "KubernetesCluster") {
return null;
}
@ -86,7 +88,7 @@ class NonInjectedHotbarEntityIcon extends React.Component<HotbarEntityIconProps
this.props.visitEntityContextMenu(this.props.entity, {
menuItems: this.menuItems,
navigate,
navigate: this.props.navigate,
});
}
@ -113,8 +115,8 @@ class NonInjectedHotbarEntityIcon extends React.Component<HotbarEntityIconProps
)}
onClick={onClick}
>
{ this.ledIcon }
{ this.kindIcon }
{this.renderLedIcon()}
{this.renderKindIcon()}
</HotbarIcon>
);
}
@ -126,5 +128,6 @@ export const HotbarEntityIcon = withInjectables<Dependencies, HotbarEntityIconPr
catalogCategoryRegistry: di.inject(catalogCategoryRegistryInjectable),
visitEntityContextMenu: di.inject(visitEntityContextMenuInjectable),
activeEntity: di.inject(activeEntityInjectable),
navigate: di.inject(navigateInjectable),
}),
});