installation flow, extracting .tgz

Signed-off-by: Roman <ixrock@gmail.com>

package.json

@@ -47,7 +47,7 @@
     "bundledHelmVersion": "3.3.4"
   },
   "engines": {
-    "node": ">=12.0 <13.0"
+    "node": ">=12 <=14"
   },
   "lingui": {
     "locales": [
@@ -215,7 +215,7 @@
     "@types/node": "^12.12.45",
     "@types/proper-lockfile": "^4.1.1",
     "@types/react-beautiful-dnd": "^13.0.0",
-    "@types/tar": "^4.0.3",
+    "@types/tar": "^4.0.4",
     "array-move": "^3.0.0",
     "chalk": "^4.1.0",
     "command-exists": "1.2.9",
@@ -249,7 +249,7 @@
     "serializr": "^2.0.3",
     "shell-env": "^3.0.0",
     "spdy": "^4.0.2",
-    "tar": "^6.0.2",
+    "tar": "^6.0.5",
     "tcp-port-used": "^1.0.1",
     "tempy": "^0.5.0",
     "uuid": "^8.1.0",
@@ -311,7 +311,6 @@
     "@types/sharp": "^0.26.0",
     "@types/shelljs": "^0.8.8",
     "@types/spdy": "^3.4.4",
-    "@types/tar": "^4.0.3",
     "@types/tcp-port-used": "^1.0.0",
     "@types/tempy": "^0.3.0",
     "@types/terser-webpack-plugin": "^3.0.0",

src/common/utils/downloadFile.ts

@@ -1,35 +1,33 @@
-import path from "path";
 import request from "request";
 
 export interface DownloadFileOptions {
   url: string;
-  fileName?: string; // default: based on filename from URL
+  gzip?: boolean;
-  gzip?: boolean; // default: true
 }
 
 export interface DownloadFileTicket {
-  fileName: string;
+  url: string;
-  promise: Promise<File>;
+  promise: Promise<Buffer>;
   cancel(): void;
 }
 
 export function downloadFile(opts: DownloadFileOptions): DownloadFileTicket {
-  const { url, gzip = true, fileName = path.basename(url) } = opts;
+  const { url, gzip = true } = opts;
   const fileChunks: Buffer[] = [];
   const req = request(url, { gzip });
-  const promise: Promise<File> = new Promise((resolve, reject) => {
+  const promise: Promise<Buffer> = new Promise((resolve, reject) => {
     req.on("data", (chunk: Buffer) => {
       fileChunks.push(chunk);
     });
     req.on("complete", () => {
-      resolve(new File(fileChunks, fileName));
+      resolve(Buffer.concat(fileChunks));
     });
     req.on("error", err => {
       reject({ url, err });
     });
   });
   return {
-    fileName: fileName,
+    url: url,
     promise: promise,
     cancel() {
       req.abort();

src/common/utils/tar.ts

@@ -0,0 +1,49 @@
+// Helper for working with tarball files (.tar, .tgz)
+// Docs: https://github.com/npm/node-tar
+import tar, { ExtractOptions, FileStat } from "tar";
+import path from "path";
+
+export interface ReadFileFromTarOpts {
+  fileName?: string;
+  fileMatcher?(path: string, entry: FileStat): boolean;
+  notFoundMessage?: string;
+}
+
+export function readFileFromTar(tarFilePath: string, opts: ReadFileFromTarOpts): Promise<Buffer> {
+  return new Promise(async (resolve, reject) => {
+    const fileChunks: Buffer[] = [];
+    const {
+      fileName,
+      fileMatcher = (path: string) => path === fileName,
+      notFoundMessage = "File not found",
+    } = opts;
+
+    await tar.list({
+      file: tarFilePath,
+      filter: fileMatcher,
+      onentry(entry: FileStat) {
+        entry.on("data", chunk => {
+          fileChunks.push(chunk);
+        });
+        entry.on("error", err => {
+          reject(`Reading ${entry.path} error: ${err}`);
+        });
+        entry.on("end", () => {
+          resolve(Buffer.concat(fileChunks));
+        });
+      },
+    });
+
+    if (!fileChunks.length) {
+      reject(notFoundMessage);
+    }
+  })
+}
+
+export function extractTar(filePath: string, opts: ExtractOptions & { sync?: boolean } = {}) {
+  return tar.extract({
+    file: filePath,
+    cwd: path.dirname(filePath),
+    ...opts,
+  })
+}

src/extensions/extension-manager.ts

@@ -98,8 +98,12 @@ export class ExtensionManager {
   }
 
   getNpmPackageTarballUrl(packageName: string) {
-    const command = [this.npmPath, "view", packageName, "dist.tarball", "--silent"];
+    try {
-    return child_process.execSync(command.join(" "), { encoding: "utf8" }).trim();
+      const command = [this.npmPath, "view", packageName, "dist.tarball", "--silent"];
+      return child_process.execSync(command.join(" "), { encoding: "utf8" }).trim();
+    } catch (err) {
+      return null;
+    }
   }
 
   protected installPackages(): Promise<void> {

src/extensions/lens-extension.ts

@@ -11,6 +11,7 @@ export interface LensExtensionManifest {
   description?: string;
   main?: string; // path to %ext/dist/main.js
   renderer?: string; // path to %ext/dist/renderer.js
+  lens?: object; // fixme: add more required fields for validation
 }
 
 export class LensExtension {
@@ -95,3 +96,7 @@ export class LensExtension {
     // mock
   }
 }
+
+export function sanitizeExtensionName(name: string) {
+  return name.replace("@", "").replace("/", "-");
+}

src/extensions/registries/page-registry.ts

@@ -5,7 +5,7 @@ import path from "path";
 import { action } from "mobx";
 import { compile } from "path-to-regexp";
 import { BaseRegistry } from "./base-registry";
-import { LensExtension } from "../lens-extension";
+import { LensExtension, sanitizeExtensionName } from "../lens-extension";
 import logger from "../../main/logger";
 
 export interface PageRegistration {
@@ -44,10 +44,6 @@ export interface PageComponents {
   Page: React.ComponentType<any>;
 }
 
-export function sanitizeExtensionName(name: string) {
-  return name.replace("@", "").replace("/", "-");
-}
-
 export function getExtensionPageUrl<P extends object>({ extensionId, pageId = "", params }: PageMenuTarget<P>): string {
   const extensionBaseUrl = compile(`/extension/:name`)({
     name: sanitizeExtensionName(extensionId), // compile only with extension-id first and define base path

src/renderer/components/+extensions/extensions.scss

@@ -43,6 +43,10 @@
     }
   }
 
+  .SearchInput {
+    --spacing: #{$padding};
+  }
+
   .WizardLayout {
     padding: 0;
 
@@ -54,6 +58,23 @@
 }
 
 .InstallingExtensionNotification {
+  .folder-remove-warning {
+    font-size: $font-size-small;
+    color: inherit;
+    cursor: pointer;
+    font-style: italic;
+    opacity: .8;
+
+    &:hover {
+      opacity: 1;
+    }
+
+    code {
+      display: inline;
+      color: inherit;
+    }
+  }
+
   .Button {
     background-color: unset;
     border: 1px solid currentColor;

src/renderer/components/+extensions/extensions.tsx

@@ -1,7 +1,7 @@
 import "./extensions.scss";
-import { app, remote, shell } from "electron";
+import { remote, shell } from "electron";
+import os from "os";
 import path from "path";
-import tar from "tar";
 import fse from "fs-extra";
 import React from "react";
 import { computed, observable } from "mobx";
@@ -14,14 +14,28 @@ import { DropFileInput, Input, InputValidators, SearchInput } from "../input";
 import { Icon } from "../icon";
 import { PageLayout } from "../layout/page-layout";
 import { Clipboard } from "../clipboard";
+import logger from "../../../main/logger";
 import { extensionLoader } from "../../../extensions/extension-loader";
 import { extensionManager } from "../../../extensions/extension-manager";
+import { LensExtensionManifest, sanitizeExtensionName } from "../../../extensions/lens-extension";
 import { Notifications } from "../notifications";
-import logger from "../../../main/logger";
 import { downloadFile } from "../../../common/utils";
+import { extractTar, readFileFromTar } from "../../../common/utils/tar";
+
+interface InstallRequest {
+  fileName: string;
+  filePath?: string;
+  data?: Buffer;
+}
+
+interface InstallRequestValidated extends InstallRequest {
+  manifest: LensExtensionManifest;
+  tmpFile: string; // temp file for unpacking
+}
 
 @observer
 export class Extensions extends React.Component {
+  private supportedFormats = [".tar", ".tgz"];
   @observable search = "";
   @observable downloadUrl = "";
 
@@ -40,96 +54,192 @@ export class Extensions extends React.Component {
     return extensionManager.localFolderPath;
   }
 
-  selectLocalExtensionsDialog = async () => {
+  getExtensionDestFolder(name: string) {
-    const supportedFormats = [".tgz", ".tar.gz"]
+    return path.join(this.extensionsPath, sanitizeExtensionName(name));
+  }
+
+  installFromSelectFileDialog = async () => {
     const { dialog, BrowserWindow, app } = remote;
     const { canceled, filePaths } = await dialog.showOpenDialog(BrowserWindow.getFocusedWindow(), {
       defaultPath: app.getPath("downloads"),
       properties: ["openFile", "multiSelections"],
-      message: _i18n._(t`Select extensions to install (supported formats: ${supportedFormats.join(", ")}), `),
+      message: _i18n._(t`Select extensions to install (formats: ${this.supportedFormats.join(", ")}), `),
       buttonLabel: _i18n._(t`Use configuration`),
       filters: [
-        { name: "tarball", extensions: supportedFormats }
+        { name: "tarball", extensions: this.supportedFormats }
       ]
     });
     if (!canceled && filePaths.length) {
-      this.installFromSelectFileDialog(filePaths);
+      this.requestInstall(
+        filePaths.map(filePath => ({
+          fileName: path.basename(filePath),
+          filePath: filePath,
+        }))
+      );
     }
   }
 
-  installFromUrl = async () => {
+  installExtensions = () => {
-    const { downloadUrl } = this;
+    if (this.downloadUrl) {
-    if (!downloadUrl) {
+      this.installFromNpmOrUrl(this.downloadUrl);
-      return;
+      this.downloadUrl = "";
-    }
-    let tarballUrl: string;
-    if (InputValidators.isUrl.validate(downloadUrl)) {
-      tarballUrl = downloadUrl;
     } else {
-      try {
+      this.installFromSelectFileDialog();
-        tarballUrl = extensionManager.getNpmPackageTarballUrl(downloadUrl);
+    }
-      } catch (err) {
+  }
-        Notifications.error(`Error: npm package "${downloadUrl}" not found`);
+
+  installFromNpmOrUrl = async (url = this.downloadUrl) => {
+    if (!InputValidators.isUrl.validate(url)) {
+      url = extensionManager.getNpmPackageTarballUrl(url);
+      if (!url) {
+        Notifications.error(`Error: npm package "${url}" not found!`);
         return;
       }
     }
-    logger.info('Install from packed extension URL', { tarballUrl });
+    try {
-    if (tarballUrl) {
+      const { promise: filePromise } = downloadFile({ url });
-      try {
+      this.requestInstall([{
-        const { promise: filePromise } = downloadFile({ url: tarballUrl });
+        fileName: path.basename(url),
-        this.requestInstall([await filePromise]);
+        data: await filePromise,
-      } catch (err) {
+      }]);
-        Notifications.error(`Installing extension from ${tarballUrl} has failed: ${String(err)}`);
+    } catch (err) {
-      }
+      Notifications.error(
+        <div className="flex column gaps">
+          <p>Installation from URL has failed: <b>{String(err)}</b></p>
+          <p>URL: <em>{url}</em></p>
+        </div>
+      );
     }
   }
 
-  installFromSelectFileDialog = async (filePaths: string[]) => {
-    logger.info('Install from file-select dialog', { files: filePaths });
-    const files: File[] = await Promise.all(
-      filePaths.map(filePath => {
-        const fileName = path.basename(filePath);
-        return fse.readFile(filePath).then(buffer => new File([buffer], fileName));
-      })
-    );
-    return this.requestInstall(files);
-  }
-
   installOnDrop = (files: File[]) => {
-    logger.info('Install from D&D', { files: files.map(file => file.path) });
+    logger.info('Install from D&D');
-    return this.requestInstall(files);
+    return this.requestInstall(
+      files.map(file => ({
+        fileName: path.basename(file.path),
+        filePath: file.path,
+      }))
+    );
   }
 
-  // todo
+  async requestInstall(installRequests: InstallRequest[]) {
-  async installExtension(tarball: File, cleanUp?: () => void) {
+    const pendingFiles: Promise<any>[] = [];
-    logger.info(`Installing extension ${tarball.name} to ${this.extensionsPath}`);
-    const tempDir = path.join(app.getPath("temp"), "extensions");
-    await fse.ensureDir(tempDir);
-    const unpack = () => {
-      tar.extract({
-        cwd: tempDir,
-      })
-    }
-    if (cleanUp) {
-      cleanUp();
-    }
-  }
 
-  // todo: show name and description from unpacked archive
+    // read extensions with provided system path if any
-  async requestInstall(files: File[]) {
+    installRequests.forEach(ext => {
-    files.forEach((ext: File) => {
+      if (ext.data) return;
+      const promise = fse.readFile(ext.filePath)
+        .then(data => ext.data = data)
+        .catch(err => {
+          Notifications.error(`Error while reading "${ext.filePath}": ${String(err)}`);
+        });
+      pendingFiles.push(promise)
+    });
+    await Promise.all(pendingFiles);
+    installRequests = installRequests.filter(item => item.data); // remove items with reading errors
+
+    // prepare temp folder
+    const tempFolder = path.join(os.tmpdir(), "lens-extensions");
+    await fse.ensureDir(tempFolder);
+
+    // copy files to temp, get extension info from package.json and do basic validation
+    let validatedInstalls: Promise<InstallRequestValidated>[] = installRequests.map(async installReq => {
+      const { fileName, data } = installReq;
+      const tempFile = path.join(tempFolder, fileName);
+      await fse.writeFileSync(tempFile, data); // copy to temp
+      try {
+        const packageJson: Buffer = await readFileFromTar(tempFile, {
+          // tarball from npm contains single root folder "package/*"
+          fileMatcher: (path: string) => !!path.match(/(\w+\/)?package\.json$/),
+          notFoundMessage: "Extension's manifest file (package.json) not found",
+        });
+        const manifest: LensExtensionManifest = JSON.parse(packageJson.toString("utf8"));
+        if (!manifest.lens && !manifest.renderer) {
+          throw `package.json must specify "main" and/or "renderer" fields`;
+        }
+        return {
+          ...installReq,
+          manifest: manifest,
+          tmpFile: tempFile,
+        }
+      } catch (err) {
+        fse.unlink(tempFile).catch(() => null); // remove invalid temp file
+        Notifications.error(
+          <div className="flex column gaps">
+            <p>Installing <em>{fileName}</em> has failed, skipping.</p>
+            <p>Reason: <em>{String(err)}</em></p>
+          </div>
+        );
+      }
+    });
+
+    // final step, provide UI with extension info for reviewing and confirming installation
+    const extensions = await Promise.all(validatedInstalls);
+    extensions.forEach(install => {
+      if (!install) {
+        return; // skip validating errors if any
+      }
+      const { fileName, manifest } = install;
+      const { name, version, description } = manifest;
+      const extensionFolder = this.getExtensionDestFolder(name);
+      const folderExists = fse.existsSync(extensionFolder);
       const removeNotification = Notifications.info(
-        <div className="InstallingExtensionNotification flex gaps">
+        <div className="InstallingExtensionNotification flex gaps align-center">
-          <p>Install extension <em>{ext.name}</em>?</p>
+          <div className="flex column gaps">
-          <Button
+            <p>Install extension <b title={fileName}>{name}@{version}</b>?</p>
-            label="Confirm"
+            <p>Description: <em>{description}</em></p>
-            onClick={() => this.installExtension(ext, removeNotification)}
+            {folderExists && (
-          />
+              <div className="folder-remove-warning flex gaps inline align-center" onClick={() => shell.openPath(extensionFolder)}>
+                <Icon small material="warning"/>
+                <p>
+                  <b>Warning:</b> <code>{extensionFolder}</code> will be removed before installation.
+                </p>
+              </div>
+            )}
+          </div>
+          <Button autoFocus label="Install" onClick={() => {
+            removeNotification();
+            this.unpackExtension(install);
+          }}/>
         </div>
       );
     })
   }
 
+  async unpackExtension({ fileName, tmpFile, manifest: { name, version } }: InstallRequestValidated) {
+    logger.info(`Unpacking extension ${name} from ${fileName}`);
+    const unpackingTempFolder = path.join(path.dirname(tmpFile), path.basename(tmpFile) + "-unpacked");
+    const extensionFolder = this.getExtensionDestFolder(name);
+    try {
+      // extract to temp folder first
+      await fse.remove(unpackingTempFolder).catch(Function);
+      await fse.ensureDir(unpackingTempFolder);
+      await extractTar(tmpFile, { cwd: unpackingTempFolder });
+
+      // move contents to extensions folder
+      const unpackedFiles = await fse.readdir(unpackingTempFolder);
+      let unpackedRootFolder = unpackingTempFolder;
+      if (unpackedFiles.length === 1) {
+        // handle case when extension.tgz packed with top root folder,
+        // e.g. "npm pack %ext_name" downloads file with "package" root folder within tarball
+        unpackedRootFolder = path.join(unpackingTempFolder, unpackedFiles[0]);
+      }
+      await fse.ensureDir(extensionFolder);
+      await fse.move(unpackedRootFolder, extensionFolder, { overwrite: true });
+      Notifications.ok(
+        <p>Extension <b>{name}/{version}</b> successfully installed!</p>
+      );
+    } catch (err) {
+      Notifications.error(
+        <p>Installing extension <b>{name}</b> has failed: <em>{err}</em></p>
+      );
+    } finally {
+      // clean up
+      fse.remove(unpackingTempFolder).catch(Function);
+      fse.unlink(tmpFile).catch(Function);
+    }
+  }
+
   renderInfo() {
     return (
       <div className="extensions-info flex column gaps">
@@ -146,31 +256,25 @@ export class Extensions extends React.Component {
           </div>
         </div>
         <div className="install-extension flex column gaps">
-          <p><em>Install extensions from archive (tarball.tgz):</em></p>
+          <em>
-          <div className="install-extension-by-url flex gaps align-center">
+            Install extensions from tarball ({this.supportedFormats.join(", ")}):
-            <Input
+          </em>
-              showErrorsAsTooltip={true}
+          <Input
-              className="box grow"
+            showErrorsAsTooltip={true}
-              theme="round-black"
+            className="box grow"
-              placeholder="URL or NPM package name"
+            theme="round-black"
-              value={this.downloadUrl}
+            placeholder="URL or npm-package-name"
-              onChange={v => this.downloadUrl = v}
+            value={this.downloadUrl}
-              onSubmit={this.installFromUrl}
+            onChange={v => this.downloadUrl = v}
-            />
+            onSubmit={this.installExtensions}
-            <Icon
+          />
-              material="get_app"
-              tooltip={{ children: "Install", preferredPositions: "bottom" }}
-              interactive={this.downloadUrl.length > 0}
-              onClick={this.installFromUrl}
-            />
-          </div>
           <Button
             primary
-            label="Select extensions to install"
+            label="Add extensions"
-            onClick={this.selectLocalExtensionsDialog}
+            onClick={this.installExtensions}
           />
           <p className="hint">
-            <Trans><b>Pro-Tip 1</b>: you can download tarball from NPM via</Trans>
+            <Trans><b>Pro-Tip 1</b>: you can download packed extension from NPM via</Trans>
             <Clipboard showNotification>
               <code>npm pack %package-name</code>
             </Clipboard>

src/renderer/components/input/search-input.scss

@@ -1,5 +1,6 @@
 .Input.SearchInput {
   --compact-focus-width: 190px;
+  --spacing: 6px 6px 6px 10px;
 
   max-width: 900px;
   min-width: 220px;
@@ -10,7 +11,7 @@
     border: none;
     border-radius: $radius;
     box-shadow: 0 0 0 1px $halfGray;
-    padding: 6px 6px 6px 10px;
+    padding: var(--spacing);
 
     .Icon {
       height: $margin * 2;

yarn.lock

@@ -2405,10 +2405,10 @@
   resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.5.tgz#9adbc12950582aa65ead76bffdf39fe0c27a3c02"
   integrity sha512-/gG2M/Imw7cQFp8PGvz/SwocNrmKFjFsm5Pb8HdbHkZ1K8pmuPzOX4VeVoiEecFCVf4CsN1r3/BRvx+6sNqwtQ==
 
-"@types/tar@^4.0.3":
+"@types/tar@^4.0.4":
-  version "4.0.3"
+  version "4.0.4"
-  resolved "https://registry.yarnpkg.com/@types/tar/-/tar-4.0.3.tgz#e2cce0b8ff4f285293243f5971bd7199176ac489"
+  resolved "https://registry.yarnpkg.com/@types/tar/-/tar-4.0.4.tgz#d680de60855e7778a51c672b755869a3b8d2889f"
-  integrity sha512-Z7AVMMlkI8NTWF0qGhC4QIX0zkV/+y0J8x7b/RsHrN0310+YNjoJd8UrApCiGBCWtKjxS9QhNqLi2UJNToh5hA==
+  integrity sha512-0Xv+xcmkTsOZdIF4yCnd7RkOOyfyqPaqJ7RZFKnwdxfDbkN3eAAE9sHl8zJFqBz4VhxolW9EErbjR1oyH7jK2A==
   dependencies:
     "@types/minipass" "*"
     "@types/node" "*"
@@ -9778,6 +9778,14 @@ minizlib@^2.1.0:
     minipass "^3.0.0"
     yallist "^4.0.0"
 
+minizlib@^2.1.1:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931"
+  integrity sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==
+  dependencies:
+    minipass "^3.0.0"
+    yallist "^4.0.0"
+
 mississippi@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/mississippi/-/mississippi-3.0.0.tgz#ea0a3291f97e0b5e8776b363d5f0a12d94c67022"
@@ -13685,6 +13693,18 @@ tar@^6.0.2:
     mkdirp "^1.0.3"
     yallist "^4.0.0"
 
+tar@^6.0.5:
+  version "6.0.5"
+  resolved "https://registry.yarnpkg.com/tar/-/tar-6.0.5.tgz#bde815086e10b39f1dcd298e89d596e1535e200f"
+  integrity sha512-0b4HOimQHj9nXNEAA7zWwMM91Zhhba3pspja6sQbgTpynOJf+bkjBnfybNYzbpLbnwXnbyB4LOREvlyXLkCHSg==
+  dependencies:
+    chownr "^2.0.0"
+    fs-minipass "^2.0.0"
+    minipass "^3.0.0"
+    minizlib "^2.1.1"
+    mkdirp "^1.0.3"
+    yallist "^4.0.0"
+
 tcp-port-used@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/tcp-port-used/-/tcp-port-used-1.0.1.tgz#46061078e2d38c73979a2c2c12b5a674e6689d70"