Watch for added/removed local extensions (#1482)

Signed-off-by: Panu Horsmalahti <phorsmalahti@mirantis.com>
This commit is contained in:
Panu Horsmalahti authored and GitHub committed 2020-11-24 10:48:40 +02:00
1 parent 3922609204
commit c79cee0311
9 files changed
+457 -191

No files matched your search

+329
View File
@@ -0,0 +1,329 @@
import chokidar from "chokidar";
import { EventEmitter } from "events";
import fs from "fs-extra";
import os from "os";
import path from "path";
import { getBundledExtensions } from "../common/utils/app-version";
import logger from "../main/logger";
import { extensionInstaller, PackageJson } from "./extension-installer";
import type { LensExtensionId, LensExtensionManifest } from "./lens-extension";
export interface InstalledExtension {
readonly manifest: LensExtensionManifest;
readonly manifestPath: string;
readonly isBundled: boolean; // defined in project root's package.json
isEnabled: boolean;
}
const logModule = "[EXTENSION-DISCOVERY]";
const manifestFilename = "package.json";
/**
* Returns true if the lstat is for a directory-like file (e.g. isDirectory or symbolic link)
* @param lstat the stats to compare
*/
const isDirectoryLike = (lstat: fs.Stats) => lstat.isDirectory() || lstat.isSymbolicLink();
/**
* Discovers installed bundled and local extensions from the filesystem.
* Also watches for added and removed local extensions by watching the directory.
* Uses ExtensionInstaller to install dependencies for all of the extensions.
* This is also done when a new extension is copied to the local extensions directory.
* .init() must be called to start the directory watching.
* The class emits events for added and removed extensions:
* - "add": When extension is added. The event is of type InstalledExtension
* - "remove": When extension is removed. The event is of type LensExtensionId
*/
export class ExtensionDiscovery {
protected bundledFolderPath: string;
private loadStarted = false;
// This promise is resolved when .load() is finished.
// This allows operations to be added after .load() success.
private loaded: Promise<void>;
// These are called to either resolve or reject this.loaded promise
private resolveLoaded: () => void;
private rejectLoaded: (error: any) => void;
public events: EventEmitter;
constructor() {
this.loaded = new Promise((resolve, reject) => {
this.resolveLoaded = resolve;
this.rejectLoaded = reject;
});
this.events = new EventEmitter();
}
// Each extension is added as a single dependency to this object, which is written as package.json.
// Each dependency key is the name of the dependency, and
// each dependency value is the non-symlinked path to the dependency (folder).
protected packagesJson: PackageJson = {
dependencies: {}
};
get localFolderPath(): string {
return path.join(os.homedir(), ".k8slens", "extensions");
}
get packageJsonPath() {
return path.join(extensionInstaller.extensionPackagesRoot, manifestFilename);
}
get inTreeTargetPath() {
return path.join(extensionInstaller.extensionPackagesRoot, "extensions");
}
get inTreeFolderPath(): string {
return path.resolve(__static, "../extensions");
}
get nodeModulesPath(): string {
return path.join(extensionInstaller.extensionPackagesRoot, "node_modules");
}
/**
* Initializes the class and setups the file watcher for added/removed local extensions.
*/
init() {
this.watchExtensions();
}
/**
* Watches for added/removed local extensions.
* Dependencies are installed automatically after an extension folder is copied.
*/
async watchExtensions() {
logger.info(`${logModule} watching extension add/remove in ${this.localFolderPath}`);
// Wait until .load() has been called and has been resolved
await this.loaded;
// chokidar works better than fs.watch
chokidar.watch(this.localFolderPath, {
// Dont watch recursively into subdirectories
depth: 0,
// Try to wait until the file has been completely copied.
// The OS might emit an event for added file even it's not completely written to the filesysten.
awaitWriteFinish: {
// Wait 300ms until the file size doesn't change to consider the file written.
// For a small file like package.json this should be plenty of time.
stabilityThreshold: 300
}
})
// Extension add is detected by watching "<extensionDir>package.json" add
.on("add", this.handleWatchFileAdd)
// Extension remove is detected by watching <extensionDir>" unlink
.on("unlinkDir", this.handleWatchUnlinkDir);
}
handleWatchFileAdd = async (filePath: string) => {
if (path.basename(filePath) === manifestFilename) {
try {
const absPath = path.dirname(filePath);
// this.loadExtensionFromPath updates this.packagesJson
const extension = await this.loadExtensionFromPath(absPath);
if (extension) {
// Install dependencies for the new extension
await this.installPackages();
logger.info(`${logModule} Added extension ${extension.manifest.name}`);
this.events.emit("add", extension);
}
} catch (error) {
console.error(error);
}
}
};
handleWatchUnlinkDir = async (filePath: string) => {
// filePath is the non-symlinked path to the extension folder
// this.packagesJson.dependencies value is the non-symlinked path to the extension folder
// LensExtensionId in extension-loader is the symlinked path to the extension folder manifest file
// Check that the removed path is directly under this.localFolderPath
// Note that the watcher can create unlink events for subdirectories of the extension
const extensionFolderName = path.basename(filePath);
if (path.relative(this.localFolderPath, filePath) === extensionFolderName) {
const extensionName: string | undefined = Object
.entries(this.packagesJson.dependencies)
.find(([_name, extensionFolder]) => filePath === extensionFolder)?.[0];
if (extensionName !== undefined) {
delete this.packagesJson.dependencies[extensionName];
// Reinstall dependencies to remove the extension from package.json
await this.installPackages();
// The path to the manifest file is the lens extension id
// Note that we need to use the symlinked path
const lensExtensionId = path.join(this.nodeModulesPath, extensionName, "package.json");
logger.info(`${logModule} removed extension ${extensionName}`);
this.events.emit("remove", lensExtensionId as LensExtensionId);
} else {
logger.warn(`${logModule} extension ${extensionFolderName} not found, can't remove`);
}
}
};
async load(): Promise<Map<LensExtensionId, InstalledExtension>> {
if (this.loadStarted) {
// The class is simplified by only supporting .load() to be called once
throw new Error("ExtensionDiscovery.load() can be only be called once");
}
this.loadStarted = true;
try {
logger.info(`${logModule} loading extensions from ${extensionInstaller.extensionPackagesRoot}`);
if (fs.existsSync(path.join(extensionInstaller.extensionPackagesRoot, "package-lock.json"))) {
await fs.remove(path.join(extensionInstaller.extensionPackagesRoot, "package-lock.json"));
}
try {
await fs.access(this.inTreeFolderPath, fs.constants.W_OK);
this.bundledFolderPath = this.inTreeFolderPath;
} catch {
// we need to copy in-tree extensions so that we can symlink them properly on "npm install"
await fs.remove(this.inTreeTargetPath);
await fs.ensureDir(this.inTreeTargetPath);
await fs.copy(this.inTreeFolderPath, this.inTreeTargetPath);
this.bundledFolderPath = this.inTreeTargetPath;
}
await fs.ensureDir(this.nodeModulesPath);
await fs.ensureDir(this.localFolderPath);
const extensions = await this.loadExtensions();
// resolve the loaded promise
this.resolveLoaded();
return extensions;
} catch (error) {
this.rejectLoaded(error);
}
}
protected async getByManifest(manifestPath: string, { isBundled = false, isEnabled = isBundled }: {
isBundled?: boolean;
isEnabled?: boolean;
} = {}): Promise<InstalledExtension | null> {
let manifestJson: LensExtensionManifest;
try {
// check manifest file for existence
fs.accessSync(manifestPath, fs.constants.F_OK);
manifestJson = __non_webpack_require__(manifestPath);
this.packagesJson.dependencies[manifestJson.name] = path.dirname(manifestPath);
return {
manifestPath: path.join(this.nodeModulesPath, manifestJson.name, "package.json"),
manifest: manifestJson,
isBundled,
isEnabled,
};
} catch (error) {
logger.error(`${logModule}: can't install extension at ${manifestPath}: ${error}`, { manifestJson });
return null;
}
}
async loadExtensions(): Promise<Map<LensExtensionId, InstalledExtension>> {
const bundledExtensions = await this.loadBundledExtensions();
const localExtensions = await this.loadFromFolder(this.localFolderPath);
await this.installPackages();
const extensions = bundledExtensions.concat(localExtensions);
return new Map(extensions.map(ext => [ext.manifestPath, ext]));
}
/**
* Write package.json to file system and install dependencies.
*/
installPackages() {
return extensionInstaller.installPackages(this.packageJsonPath, this.packagesJson);
}
async loadBundledExtensions() {
const extensions: InstalledExtension[] = [];
const folderPath = this.bundledFolderPath;
const bundledExtensions = getBundledExtensions();
const paths = await fs.readdir(folderPath);
for (const fileName of paths) {
if (!bundledExtensions.includes(fileName)) {
continue;
}
const absPath = path.resolve(folderPath, fileName);
const extension = await this.loadExtensionFromPath(absPath, { isBundled: true });
if (extension) {
extensions.push(extension);
}
}
logger.debug(`${logModule}: ${extensions.length} extensions loaded`, { folderPath, extensions });
return extensions;
}
async loadFromFolder(folderPath: string): Promise<InstalledExtension[]> {
const bundledExtensions = getBundledExtensions();
const extensions: InstalledExtension[] = [];
const paths = await fs.readdir(folderPath);
for (const fileName of paths) {
// do not allow to override bundled extensions
if (bundledExtensions.includes(fileName)) {
continue;
}
const absPath = path.resolve(folderPath, fileName);
if (!fs.existsSync(absPath)) {
continue;
}
const lstat = await fs.lstat(absPath);
// skip non-directories
if (!isDirectoryLike(lstat)) {
continue;
}
const extension = await this.loadExtensionFromPath(absPath);
if (extension) {
extensions.push(extension);
}
}
logger.debug(`${logModule}: ${extensions.length} extensions loaded`, { folderPath, extensions });
return extensions;
}
/**
* Loads extension from absolute path, updates this.packagesJson to include it and returns the extension.
*/
async loadExtensionFromPath(absPath: string, { isBundled = false, isEnabled = isBundled }: {
isBundled?: boolean;
isEnabled?: boolean;
} = {}): Promise<InstalledExtension | null> {
const manifestPath = path.resolve(absPath, manifestFilename);
return this.getByManifest(manifestPath, { isBundled, isEnabled });
}
}
export const extensionDiscovery = new ExtensionDiscovery();
+69
View File
@@ -0,0 +1,69 @@
import AwaitLock from 'await-lock';
import child_process from "child_process";
import fs from "fs-extra";
import path from "path";
import logger from "../main/logger";
import { extensionPackagesRoot } from "./extension-loader";
const logModule = "[EXTENSION-INSTALLER]";
type Dependencies = {
[name: string]: string;
};
// Type for the package.json file that is written by ExtensionInstaller
export type PackageJson = {
dependencies: Dependencies;
};
/**
* Installs dependencies for extensions
*/
export class ExtensionInstaller {
private installLock = new AwaitLock();
get extensionPackagesRoot() {
return extensionPackagesRoot();
}
get npmPath() {
return __non_webpack_require__.resolve('npm/bin/npm-cli');
}
installDependencies(): Promise<void> {
return new Promise((resolve, reject) => {
logger.info(`${logModule} installing dependencies at ${extensionPackagesRoot()}`);
const child = child_process.fork(this.npmPath, ["install", "--silent", "--no-audit", "--only=prod", "--prefer-offline", "--no-package-lock"], {
cwd: extensionPackagesRoot(),
silent: true
});
child.on("close", () => {
resolve();
});
child.on("error", (err) => {
reject(err);
});
});
}
/**
* Write package.json to the file system and execute npm install for it.
*/
async installPackages(packageJsonPath: string, packagesJson: PackageJson): Promise<void> {
// Mutual exclusion to install packages in sequence
await this.installLock.acquireAsync();
try {
// Write the package.json which will be installed in .installDependencies()
await fs.writeFile(path.join(packageJsonPath), JSON.stringify(packagesJson, null, 2), {
mode: 0o600
});
await this.installDependencies();
} finally {
this.installLock.release();
}
}
}
export const extensionInstaller = new ExtensionInstaller();
+30 -13
View File
@@ -1,20 +1,25 @@
import { app, ipcRenderer, remote } from "electron";
import { action, computed, observable, reaction, toJS, when } from "mobx";
import path from "path";
import { broadcastMessage, handleRequest, requestMain, subscribeToBroadcast } from "../common/ipc";
import logger from "../main/logger";
import type { InstalledExtension } from "./extension-discovery";
import { extensionsStore } from "./extensions-store";
import type { LensExtension, LensExtensionConstructor, LensExtensionId } from "./lens-extension";
import type { LensMainExtension } from "./lens-main-extension";
import type { LensRendererExtension } from "./lens-renderer-extension";
import type { InstalledExtension } from "./extension-manager";
import path from "path";
import { broadcastMessage, handleRequest, requestMain, subscribeToBroadcast } from "../common/ipc";
import { action, computed, observable, reaction, toJS, when } from "mobx";
import logger from "../main/logger";
import { app, ipcRenderer, remote } from "electron";
import * as registries from "./registries";
import { extensionsStore } from "./extensions-store";
// lazy load so that we get correct userData
export function extensionPackagesRoot() {
return path.join((app || remote.app).getPath("userData"));
}
const logModule = "[EXTENSIONS-LOADER]";
/**
* Loads installed extensions to the Lens application
*/
export class ExtensionLoader {
protected extensions = observable.map<LensExtensionId, InstalledExtension>();
protected instances = observable.map<LensExtensionId, LensExtension>();
@@ -47,6 +52,17 @@ export class ExtensionLoader {
this.extensions.replace(extensions);
}
addExtension(extension: InstalledExtension) {
this.extensions.set(extension.manifestPath as LensExtensionId, extension);
}
removeExtension(lensExtensionId: LensExtensionId) {
// TODO: Remove the extension properly (from menus etc.)
if (!this.extensions.delete(lensExtensionId)) {
throw new Error(`Can't remove extension ${lensExtensionId}, doesn't exist.`);
}
}
protected async initMain() {
this.isLoaded = true;
this.loadOnMain();
@@ -77,14 +93,14 @@ export class ExtensionLoader {
}
loadOnMain() {
logger.info('[EXTENSIONS-LOADER]: load on main');
logger.info(`${logModule}: load on main`);
this.autoInitExtensions((ext: LensMainExtension) => [
registries.menuRegistry.add(ext.appMenus)
]);
}
loadOnClusterManagerRenderer() {
logger.info('[EXTENSIONS-LOADER]: load on main renderer (cluster manager)');
logger.info(`${logModule}: load on main renderer (cluster manager)`);
this.autoInitExtensions((ext: LensRendererExtension) => [
registries.globalPageRegistry.add(ext.globalPages, ext),
registries.globalPageMenuRegistry.add(ext.globalPageMenus, ext),
@@ -95,7 +111,7 @@ export class ExtensionLoader {
}
loadOnClusterRenderer() {
logger.info('[EXTENSIONS-LOADER]: load on cluster renderer (dashboard)');
logger.info(`${logModule}: load on cluster renderer (dashboard)`);
this.autoInitExtensions((ext: LensRendererExtension) => [
registries.clusterPageRegistry.add(ext.clusterPages, ext),
registries.clusterPageMenuRegistry.add(ext.clusterPageMenus, ext),
@@ -118,14 +134,15 @@ export class ExtensionLoader {
instance.enable();
this.instances.set(extId, instance);
} catch (err) {
logger.error(`[EXTENSION-LOADER]: activation extension error`, { ext, err });
logger.error(`${logModule}: activation extension error`, { ext, err });
}
} else if (!ext.isEnabled && instance) {
logger.info(`${logModule} deleting extension ${extId}`);
try {
instance.disable();
this.instances.delete(extId);
} catch (err) {
logger.error(`[EXTENSION-LOADER]: deactivation extension error`, { ext, err });
logger.error(`${logModule}: deactivation extension error`, { ext, err });
}
}
}
@@ -146,7 +163,7 @@ export class ExtensionLoader {
return __non_webpack_require__(extEntrypoint).default;
}
} catch (err) {
console.error(`[EXTENSION-LOADER]: can't load extension main at ${extEntrypoint}: ${err}`, { extension });
console.error(`${logModule}: can't load extension main at ${extEntrypoint}: ${err}`, { extension });
console.trace(err);
}
}
-172
View File
@@ -1,172 +0,0 @@
import type { LensExtensionId, LensExtensionManifest } from "./lens-extension";
import path from "path";
import os from "os";
import fs from "fs-extra";
import child_process from "child_process";
import logger from "../main/logger";
import { extensionPackagesRoot } from "./extension-loader";
import { getBundledExtensions } from "../common/utils/app-version";
export interface InstalledExtension {
readonly manifest: LensExtensionManifest;
readonly manifestPath: string;
readonly isBundled: boolean; // defined in project root's package.json
isEnabled: boolean;
}
type Dependencies = {
[name: string]: string;
};
type PackageJson = {
dependencies: Dependencies;
};
export class ExtensionManager {
protected bundledFolderPath: string;
protected packagesJson: PackageJson = {
dependencies: {}
};
get extensionPackagesRoot() {
return extensionPackagesRoot();
}
get inTreeTargetPath() {
return path.join(this.extensionPackagesRoot, "extensions");
}
get inTreeFolderPath(): string {
return path.resolve(__static, "../extensions");
}
get nodeModulesPath(): string {
return path.join(this.extensionPackagesRoot, "node_modules");
}
get localFolderPath(): string {
return path.join(os.homedir(), ".k8slens", "extensions");
}
get npmPath() {
return __non_webpack_require__.resolve('npm/bin/npm-cli');
}
get packageJsonPath() {
return path.join(this.extensionPackagesRoot, "package.json");
}
async load(): Promise<Map<LensExtensionId, InstalledExtension>> {
logger.info("[EXTENSION-MANAGER] loading extensions from " + this.extensionPackagesRoot);
if (fs.existsSync(path.join(this.extensionPackagesRoot, "package-lock.json"))) {
await fs.remove(path.join(this.extensionPackagesRoot, "package-lock.json"));
}
try {
await fs.access(this.inTreeFolderPath, fs.constants.W_OK);
this.bundledFolderPath = this.inTreeFolderPath;
} catch {
// we need to copy in-tree extensions so that we can symlink them properly on "npm install"
await fs.remove(this.inTreeTargetPath);
await fs.ensureDir(this.inTreeTargetPath);
await fs.copy(this.inTreeFolderPath, this.inTreeTargetPath);
this.bundledFolderPath = this.inTreeTargetPath;
}
await fs.ensureDir(this.nodeModulesPath);
await fs.ensureDir(this.localFolderPath);
return await this.loadExtensions();
}
protected async getByManifest(manifestPath: string, { isBundled = false } = {}): Promise<InstalledExtension> {
let manifestJson: LensExtensionManifest;
try {
fs.accessSync(manifestPath, fs.constants.F_OK); // check manifest file for existence
manifestJson = __non_webpack_require__(manifestPath);
this.packagesJson.dependencies[manifestJson.name] = path.dirname(manifestPath);
logger.info("[EXTENSION-MANAGER] installed extension " + manifestJson.name);
return {
manifestPath: path.join(this.nodeModulesPath, manifestJson.name, "package.json"),
manifest: manifestJson,
isBundled: isBundled,
isEnabled: isBundled,
};
} catch (err) {
logger.error(`[EXTENSION-MANAGER]: can't install extension at ${manifestPath}: ${err}`, { manifestJson });
}
}
protected installPackages(): Promise<void> {
return new Promise((resolve, reject) => {
const child = child_process.fork(this.npmPath, ["install", "--silent", "--no-audit", "--only=prod", "--prefer-offline", "--no-package-lock"], {
cwd: extensionPackagesRoot(),
silent: true
});
child.on("close", () => {
resolve();
});
child.on("error", (err) => {
reject(err);
});
});
}
async loadExtensions() {
const bundledExtensions = await this.loadBundledExtensions();
const localExtensions = await this.loadFromFolder(this.localFolderPath);
await fs.writeFile(path.join(this.packageJsonPath), JSON.stringify(this.packagesJson, null, 2), { mode: 0o600 });
await this.installPackages();
const extensions = bundledExtensions.concat(localExtensions);
return new Map(extensions.map(ext => [ext.manifestPath, ext]));
}
async loadBundledExtensions() {
const extensions: InstalledExtension[] = [];
const folderPath = this.bundledFolderPath;
const bundledExtensions = getBundledExtensions();
const paths = await fs.readdir(folderPath);
for (const fileName of paths) {
if (!bundledExtensions.includes(fileName)) {
continue;
}
const absPath = path.resolve(folderPath, fileName);
const manifestPath = path.resolve(absPath, "package.json");
const ext = await this.getByManifest(manifestPath, { isBundled: true }).catch(() => null);
if (ext) {
extensions.push(ext);
}
}
logger.debug(`[EXTENSION-MANAGER]: ${extensions.length} extensions loaded`, { folderPath, extensions });
return extensions;
}
async loadFromFolder(folderPath: string): Promise<InstalledExtension[]> {
const bundledExtensions = getBundledExtensions();
const extensions: InstalledExtension[] = [];
const paths = await fs.readdir(folderPath);
for (const fileName of paths) {
if (bundledExtensions.includes(fileName)) { // do no allow to override bundled extensions
continue;
}
const absPath = path.resolve(folderPath, fileName);
if (!fs.existsSync(absPath)) {
continue;
}
const lstat = await fs.lstat(absPath);
if (!lstat.isDirectory() && !lstat.isSymbolicLink()) { // skip non-directories
continue;
}
const manifestPath = path.resolve(absPath, "package.json");
const ext = await this.getByManifest(manifestPath).catch(() => null);
if (ext) {
extensions.push(ext);
}
}
logger.debug(`[EXTENSION-MANAGER]: ${extensions.length} extensions loaded`, { folderPath, extensions });
return extensions;
}
}
export const extensionManager = new ExtensionManager();
+2 -1
View File
@@ -1,4 +1,4 @@
import type { InstalledExtension } from "./extension-manager";
import type { InstalledExtension } from "./extension-discovery";
import { action, observable, reaction } from "mobx";
import logger from "../main/logger";
@@ -27,6 +27,7 @@ export class LensExtension {
}
get id(): LensExtensionId {
// This is the symlinked path under node_modules
return this.manifestPath;
}