Allow extensions to have external dependencies (#1018)

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

src/common/cluster-ipc.ts

@@ -1,5 +1,6 @@
 import { createIpcChannel } from "./ipc";
 import { ClusterId, clusterStore } from "./cluster-store";
+import { extensionLoader } from "../extensions/extension-loader"
 import { tracker } from "./tracker";
 
 export const clusterIpc = {
@@ -8,7 +9,10 @@ export const clusterIpc = {
     handle: (clusterId: ClusterId, frameId?: number) => {
       const cluster = clusterStore.getById(clusterId);
       if (cluster) {
-        if (frameId) cluster.frameId = frameId; // save cluster's webFrame.routingId to be able to send push-updates
+        if (frameId) {
+          cluster.frameId = frameId; // save cluster's webFrame.routingId to be able to send push-updates
+        }
+        extensionLoader.broadcastExtensions(frameId)
         return cluster.activate();
       }
     },

src/extensions/extension-loader.ts

@@ -0,0 +1,110 @@
+import type { ExtensionId, LensExtension, ExtensionManifest, ExtensionModel } from "./lens-extension"
+import { broadcastIpc } from "../common/ipc"
+import type { LensRuntimeRendererEnv } from "./lens-runtime"
+import path from "path"
+import { observable, reaction, toJS, } from "mobx"
+import logger from "../main/logger"
+import { app, remote, ipcRenderer } from "electron"
+
+export interface InstalledExtension extends ExtensionModel {
+  manifestPath: string;
+  manifest: ExtensionManifest;
+}
+
+// lazy load so that we get correct userData
+export function extensionPackagesRoot() {
+  return path.join((app || remote.app).getPath("userData"))
+}
+
+export function withExtensionPackagesRoot(callback: Function) {
+  const cwd = process.cwd()
+  try {
+    process.chdir(extensionPackagesRoot())
+    return callback()
+  } finally {
+    process.chdir(cwd)
+  }
+}
+
+export class ExtensionLoader {
+  @observable extensions = observable.map<ExtensionId, InstalledExtension>([], { deep: false });
+  @observable instances = observable.map<ExtensionId, LensExtension>([], { deep: false })
+
+  constructor() {
+    if (ipcRenderer) {
+      ipcRenderer.on("extensions:loaded", (event, extensions: InstalledExtension[]) => {
+        extensions.forEach((ext) => {
+          if (!this.getById(ext.manifestPath)) {
+            this.extensions.set(ext.manifestPath, ext)
+          }
+        })
+      })
+    }
+  }
+
+  autoEnableOnLoad(getLensRuntimeEnv: () => LensRuntimeRendererEnv, { delay = 0 } = {}) {
+    logger.info('[EXTENSIONS-LOADER]: auto-activation loaded extensions: ON');
+    return reaction(() => this.extensions.toJS(), installedExtensions => {
+      installedExtensions.forEach((ext) => {
+        let instance = this.instances.get(ext.manifestPath)
+        if (!instance) {
+          const extensionModule = this.requireExtension(ext)
+          if (!extensionModule) {
+            logger.error("[EXTENSION-LOADER] failed to load extension " + ext.manifestPath)
+            return
+          }
+          const LensExtensionClass = extensionModule.default;
+          instance = new LensExtensionClass({ ...ext.manifest, manifestPath: ext.manifestPath, id: ext.manifestPath }, ext.manifest);
+          instance.enable(getLensRuntimeEnv());
+          this.instances.set(ext.id, instance)
+        }
+      })
+    }, {
+      fireImmediately: true,
+      delay: delay,
+    })
+  }
+
+  protected requireExtension(extension: InstalledExtension) {
+    return withExtensionPackagesRoot(() => {
+      try {
+        const extMain = path.join(path.dirname(extension.manifestPath), extension.manifest.main)
+        return __non_webpack_require__(extMain)
+      } catch (err) {
+        console.error(`[EXTENSION-LOADER]: can't load extension main at ${extension.manifestPath}: ${err}`, { extension });
+      }
+    })
+  }
+
+  getById(id: ExtensionId): InstalledExtension {
+    return this.extensions.get(id);
+  }
+
+  async removeById(id: ExtensionId) {
+    const extension = this.getById(id);
+    if (extension) {
+      const instance = this.instances.get(extension.id)
+      if (instance) { await instance.uninstall() }
+      this.extensions.delete(id);
+    }
+  }
+
+  broadcastExtensions(frameId?: number) {
+    broadcastIpc({
+      channel: "extensions:loaded",
+      frameId: frameId,
+      frameOnly: !!frameId,
+      args: [this.toJSON().extensions],
+    })
+  }
+
+  toJSON() {
+    return toJS({
+      extensions: Array.from(this.extensions).map(([id, instance]) => instance),
+    }, {
+      recurseEverything: true,
+    })
+  }
+}
+
+export const extensionLoader = new ExtensionLoader()

src/extensions/extension-manager.ts

@@ -0,0 +1,97 @@
+import type { ExtensionManifest } from "./lens-extension"
+import path from "path"
+import fs from "fs-extra"
+import logger from "../main/logger"
+import { withExtensionPackagesRoot, extensionPackagesRoot, InstalledExtension } from "./extension-loader"
+import npm from "npm"
+
+export class ExtensionManager {
+  get extensionPackagesRoot() {
+    return extensionPackagesRoot()
+  }
+
+  get folderPath(): string {
+    return path.resolve(__static, "../extensions");
+  }
+
+  async load() {
+    logger.info("[EXTENSION-MANAGER] loading extensions from " + this.extensionPackagesRoot)
+
+    await fs.ensureDir(path.join(this.extensionPackagesRoot, "node_modules"))
+    await fs.writeFile(path.join(this.extensionPackagesRoot, "package.json"), `{"dependencies": []}`, {mode: 0o600})
+
+    return await this.loadExtensions();
+  }
+
+  async getExtensionByManifest(manifestPath: string): Promise<InstalledExtension> {
+    let manifestJson: ExtensionManifest;
+    try {
+      manifestJson = __non_webpack_require__(manifestPath)
+      withExtensionPackagesRoot(() => {
+        this.installPackageFromPath(path.dirname(manifestPath))
+      })
+
+      logger.info("[EXTENSION-MANAGER] installed extension " + manifestJson.name)
+      return {
+        id: manifestJson.name,
+        version: manifestJson.version,
+        name: manifestJson.name,
+        manifestPath: path.join(this.extensionPackagesRoot, "node_modules", manifestJson.name, "package.json"),
+        manifest: manifestJson
+      }
+    } catch (err) {
+      logger.error(`[EXTENSION-MANAGER]: can't install extension at ${manifestPath}: ${err}`, { manifestJson });
+    }
+  }
+
+  protected installPackageFromPath(path: string): Promise<void> {
+    const origLogger = console.log
+    return new Promise((resolve, reject) => {
+      npm.load({
+        production: true,
+        global: false,
+        prefix: this.extensionPackagesRoot,
+        dev: false,
+        spin: false,
+        "ignore-scripts": true,
+        loglevel: "silent"
+      }, (err) => {
+        console.log = function() {
+          // just to ignore ts empty function error
+        }
+        npm.commands.install([
+          path
+        ], (err) => {
+          console.log = origLogger
+          if (err) {
+            reject(err)
+          } else {
+            resolve()
+          }
+        })
+      })
+    })
+  }
+
+  async loadExtensions() {
+    const extensions = await this.loadFromFolder(this.folderPath);
+    return new Map(extensions.map(ext => [ext.id, ext]));
+  }
+
+  async loadFromFolder(folderPath: string): Promise<InstalledExtension[]> {
+    const paths = await fs.readdir(folderPath);
+    const manifestsLoading = paths.map(fileName => {
+      const absPath = path.resolve(folderPath, fileName);
+      const manifestPath = path.resolve(absPath, "package.json");
+      return fs.access(manifestPath, fs.constants.F_OK)
+        .then(async () => await this.getExtensionByManifest(manifestPath))
+        .catch(() => null)
+    });
+    let extensions = await Promise.all(manifestsLoading);
+    extensions = extensions.filter(v => !!v); // filter out files and invalid folders (without manifest.json)
+    logger.debug(`[EXTENSION-MANAGER]: ${extensions.length} extensions loaded`, { folderPath, extensions });
+    return extensions;
+  }
+}
+
+export const extensionManager = new ExtensionManager()

src/extensions/extension-store.ts

@@ -1,168 +0,0 @@
-import type { ExtensionId, ExtensionManifest, ExtensionVersion, LensExtension } from "./lens-extension";
-import type { LensRuntimeRendererEnv } from "./lens-runtime";
-import path from "path";
-import fs from "fs-extra";
-import { action, observable, reaction, toJS, } from "mobx";
-import { BaseStore } from "../common/base-store";
-import logger from "../main/logger";
-
-export interface ExtensionStoreModel {
-  version: ExtensionVersion;
-  extensions: [ExtensionId, ExtensionModel][]
-}
-
-export interface ExtensionModel {
-  id: ExtensionId;
-  version: ExtensionVersion;
-  name: string;
-  manifestPath: string;
-  description?: string;
-  enabled?: boolean;
-  updateUrl?: string;
-}
-
-export interface InstalledExtension<T extends ExtensionModel = any> {
-  manifestPath: string;
-  manifest: ExtensionManifest;
-  extensionModule: {
-    [name: string]: any;
-    default: new (model: ExtensionModel, manifest?: ExtensionManifest) => LensExtension
-  }
-}
-
-export class ExtensionStore extends BaseStore<ExtensionStoreModel> {
-  private constructor() {
-    super({
-      configName: "lens-extension-store",
-    });
-  }
-
-  @observable version: ExtensionVersion = "0.0.0";
-  @observable extensions = observable.map<ExtensionId, LensExtension>();
-  @observable removed = observable.map<ExtensionId, LensExtension>();
-  @observable installed = observable.map<ExtensionId, InstalledExtension>([], { deep: false });
-
-  get folderPath(): string {
-    return path.resolve(__static, "../extensions");
-  }
-
-  async load() {
-    await this.loadExtensions();
-    return super.load();
-  }
-
-  autoEnableOnLoad(getLensRuntimeEnv: () => LensRuntimeRendererEnv, { delay = 0 } = {}) {
-    logger.info('[EXTENSIONS-STORE]: auto-activation loaded extensions: ON');
-    return reaction(() => this.installed.toJS(), installedExtensions => {
-      installedExtensions.forEach(({ extensionModule, manifest, manifestPath }) => {
-        let instance = this.getById(manifestPath);
-        if (!instance) {
-          const LensExtensionClass = extensionModule.default;
-          instance = new LensExtensionClass({ ...manifest, manifestPath, id: manifestPath }, manifest);
-          instance.enable(getLensRuntimeEnv());
-          this.extensions.set(manifestPath, instance); // save
-        }
-      })
-    }, {
-      fireImmediately: true,
-      delay: delay,
-    })
-  }
-
-  getExtensionByManifest(manifestPath: string): InstalledExtension {
-    let manifestJson: ExtensionManifest;
-    let mainJs: string;
-    try {
-      manifestJson = __non_webpack_require__(manifestPath); // "__non_webpack_require__" converts to native node's require()-call
-      mainJs = path.resolve(path.dirname(manifestPath), manifestJson.main);
-      const extensionModule = __non_webpack_require__(mainJs);
-      return {
-        manifestPath: manifestPath,
-        manifest: manifestJson,
-        extensionModule: extensionModule,
-      }
-    } catch (err) {
-      console.error(`[EXTENSION-STORE]: can't load extension at ${manifestPath}: ${err}`, { manifestJson, mainJs });
-    }
-  }
-
-  @action
-  async loadExtensions() {
-    const extensions = await this.loadFromFolder(this.folderPath);
-    const extManifestMap = new Map(extensions.map(ext => [ext.manifestPath, ext]));
-    this.installed.replace(extManifestMap);
-  }
-
-  async loadFromFolder(folderPath: string): Promise<InstalledExtension[]> {
-    const paths = await fs.readdir(folderPath);
-    const manifestsLoading = paths.map(fileName => {
-      const absPath = path.resolve(folderPath, fileName);
-      const manifestPath = path.resolve(absPath, "package.json");
-      return fs.access(manifestPath, fs.constants.F_OK)
-        .then(() => this.getExtensionByManifest(manifestPath))
-        .catch(() => null)
-    });
-    let extensions = await Promise.all(manifestsLoading);
-    extensions = extensions.filter(v => !!v); // filter out files and invalid folders (without manifest.json)
-    console.info(`[EXTENSION-STORE]: ${extensions.length} extensions loaded`, { folderPath, extensions });
-    return extensions;
-  }
-
-  getById(id: ExtensionId): LensExtension {
-    return this.extensions.get(id);
-  }
-
-  async removeById(id: ExtensionId) {
-    const extension = this.getById(id);
-    if (extension) {
-      await extension.uninstall();
-      this.extensions.delete(id);
-    }
-  }
-
-  @action
-  protected fromStore({ extensions, version }: ExtensionStoreModel) {
-    if (version) {
-      this.version = version;
-    }
-    if (extensions) {
-      const currentExtensions = new Map(extensions);
-      this.extensions.forEach(extension => {
-        if (!currentExtensions.has(extension.id)) {
-          this.removed.set(extension.id, extension);
-        }
-      })
-      currentExtensions.forEach(model => {
-        const extensionId = model.id || model.manifestPath;
-        const manifest = this.installed.get(extensionId);
-        if (!manifest) {
-          console.error(`[EXTENSION-STORE]: can't load extension manifest at ${model.manifestPath}`, { model })
-          return;
-        }
-        const extensionInstance = this.getById(extensionId)
-        if (!extensionInstance) {
-          try {
-            const { manifest: manifestJson, extensionModule } = manifest;
-            const LensExtensionClass = extensionModule.default;
-            this.extensions.set(model.id, new LensExtensionClass(model, manifestJson));
-          } catch (err) {
-            console.error(`[EXTENSION-STORE]: init extension failed: ${err}`, { model, manifest })
-          }
-        } else {
-          extensionInstance.importModel(model);
-        }
-      })
-    }
-  }
-
-  toJSON(): ExtensionStoreModel {
-    return toJS({
-      version: this.version,
-      extensions: Array.from(this.extensions).map(([id, instance]) => [id, instance.toJSON()]),
-    }, {
-      recurseEverything: true,
-    })
-  }
-}
-
-export const extensionStore = ExtensionStore.getInstance<ExtensionStore>()

src/extensions/lens-extension.ts

@@ -1,4 +1,3 @@
-import type { ExtensionModel } from "./extension-store";
 import type { LensRuntimeRendererEnv } from "./lens-runtime";
 import type { PageRegistration } from "./register-page";
 import { readJsonSync } from "fs-extra";
@@ -9,6 +8,16 @@ export type ExtensionId = string | ExtensionPackageJsonPath;
 export type ExtensionPackageJsonPath = string;
 export type ExtensionVersion = string | number;
 
+export interface ExtensionModel {
+  id: ExtensionId;
+  version: ExtensionVersion;
+  name: string;
+  manifestPath: string;
+  description?: string;
+  enabled?: boolean;
+  updateUrl?: string;
+}
+
 export interface ExtensionManifest extends ExtensionModel {
   main: string;
   description?: string; // todo: add more fields similar to package.json + some extra

src/main/index.ts

@@ -17,6 +17,8 @@ import { clusterStore } from "../common/cluster-store"
 import { userStore } from "../common/user-store";
 import { workspaceStore } from "../common/workspace-store";
 import { tracker } from "../common/tracker";
+import { extensionManager } from "../extensions/extension-manager";
+import { extensionLoader } from "../extensions/extension-loader";
 import logger from "./logger"
 
 const workingDir = path.join(app.getPath("appData"), appName);
@@ -75,6 +77,8 @@ async function main() {
 
   // create window manager and open app
   windowManager = new WindowManager(proxyPort);
+
+  extensionLoader.extensions.replace(await extensionManager.load())
 }
 
 app.on("ready", main);

src/main/menu.ts

@@ -6,7 +6,6 @@ import { addClusterURL } from "../renderer/components/+add-cluster/add-cluster.r
 import { preferencesURL } from "../renderer/components/+preferences/preferences.route";
 import { whatsNewURL } from "../renderer/components/+whats-new/whats-new.route";
 import { clusterSettingsURL } from "../renderer/components/+cluster-settings/cluster-settings.route";
-import { extensionsURL } from "../renderer/components/+extensions/extensions.route";
 import logger from "./logger";
 
 export function initMenu(windowManager: WindowManager) {
@@ -73,13 +72,6 @@ export function buildMenu(windowManager: WindowManager) {
           navigate(preferencesURL())
         }
       },
-      {
-        label: 'Extensions',
-        accelerator: 'CmdOrCtrl+E',
-        click() {
-          navigate(extensionsURL())
-        }
-      },
       { type: 'separator' },
       { role: 'services' },
       { type: 'separator' },

src/renderer/bootstrap.tsx

@@ -4,7 +4,7 @@ import { render, unmountComponentAtNode } from "react-dom";
 import { isMac } from "../common/vars";
 import { userStore } from "../common/user-store";
 import { workspaceStore } from "../common/workspace-store";
-import { extensionStore } from "../extensions/extension-store";
+import { extensionLoader } from "../extensions/extension-loader";
 import { clusterStore } from "../common/cluster-store";
 import { i18nStore } from "./i18n";
 import { themeStore } from "./theme.store";
@@ -25,18 +25,17 @@ export async function bootstrap(App: AppComponent) {
     userStore.load(),
     workspaceStore.load(),
     clusterStore.load(),
-    extensionStore.load(),
     i18nStore.init(),
     themeStore.init(),
   ]);
 
   // Register additional store listeners
   clusterStore.registerIpcListener();
+  extensionLoader.autoEnableOnLoad(getLensRuntime);
 
   // init app's dependencies if any
   if (App.init) {
     await App.init();
-    extensionStore.autoEnableOnLoad(getLensRuntime);
   }
   window.addEventListener("message", (ev: MessageEvent) => {
     if (ev.data === "teardown") {

src/renderer/components/+extensions/extensions.route.ts

@@ -1,11 +0,0 @@
-import { RouteProps } from "react-router";
-import { buildURL } from "../../navigation";
-
-export const extensionsRoute: RouteProps = {
-  path: "/extensions"
-}
-
-export interface IExtensionsRouteParams {
-}
-
-export const extensionsURL = buildURL<IExtensionsRouteParams>(extensionsRoute.path);

src/renderer/components/+extensions/extensions.scss

@@ -1,4 +0,0 @@
-.Extensions {
-  $spacing: $padding * 2;
-  padding: $spacing;
-}

src/renderer/components/+extensions/extensions.tsx

@@ -1,31 +0,0 @@
-import "./extensions.scss"
-import React from "react";
-import { observer } from "mobx-react";
-import { extensionStore } from "../../../extensions/extension-store";
-import { WizardLayout } from "../layout/wizard-layout";
-import { Icon } from "../icon";
-
-@observer
-export class Extensions extends React.Component {
-  // todo: add input-select to customize extensions loading folder(s)
-  renderInfoPanel() {
-    return (
-      <div className="info-panel flex gaps align-center">
-        <Icon material="info"/>
-        <p>Extensions available to install</p>
-      </div>
-    );
-  }
-
-  render() {
-    const { installed: installedExtensions } = extensionStore;
-    return (
-      <WizardLayout className="Extensions" infoPanel={this.renderInfoPanel()}>
-        <h2>Extensions</h2>
-        <pre>
-          {JSON.stringify(installedExtensions.toJSON(), null, 2)}
-        </pre>
-      </WizardLayout>
-    );
-  }
-}

src/renderer/components/+extensions/index.ts

@@ -1,2 +0,0 @@
-export * from "./extensions.route"
-export * from "./extensions"

src/renderer/components/cluster-manager/cluster-manager.tsx

@@ -11,7 +11,6 @@ import { Workspaces, workspacesRoute } from "../+workspaces";
 import { AddCluster, addClusterRoute } from "../+add-cluster";
 import { ClusterView } from "./cluster-view";
 import { ClusterSettings, clusterSettingsRoute } from "../+cluster-settings";
-import { Extensions, extensionsRoute } from "../+extensions";
 import { clusterViewRoute, clusterViewURL, getMatchedCluster, getMatchedClusterId } from "./cluster-view.route";
 import { clusterStore } from "../../../common/cluster-store";
 import { hasLoadedView, initView, lensViews, refreshViews } from "./lens-views";
@@ -64,7 +63,6 @@ export class ClusterManager extends React.Component {
             <Route component={AddCluster} {...addClusterRoute} />
             <Route component={ClusterView} {...clusterViewRoute} />
             <Route component={ClusterSettings} {...clusterSettingsRoute} />
-            <Route component={Extensions} {...extensionsRoute}/>
             {dynamicPages.globalPages.map(({ path, components: { Page } }) => {
               return <Route key={path} path={path} component={Page}/>
             })}