From 88950b0541909b3aaaf8b0760f4cd28dbe66ce35 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 23 Nov 2020 19:31:30 +0200 Subject: [PATCH] installation flow, extracting .tgz Signed-off-by: Roman --- package.json | 7 +- src/common/utils/downloadFile.ts | 16 +- src/common/utils/tar.ts | 49 ++++ src/extensions/extension-manager.ts | 8 +- src/extensions/lens-extension.ts | 5 + src/extensions/registries/page-registry.ts | 6 +- .../components/+extensions/extensions.scss | 21 ++ .../components/+extensions/extensions.tsx | 274 ++++++++++++------ .../components/input/search-input.scss | 3 +- yarn.lock | 28 +- 10 files changed, 307 insertions(+), 110 deletions(-) create mode 100644 src/common/utils/tar.ts diff --git a/package.json b/package.json index 49c7f58bf5..c3bc562ff3 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "bundledHelmVersion": "3.3.4" }, "engines": { - "node": ">=12.0 <13.0" + "node": ">=12 <=14" }, "lingui": { "locales": [ @@ -215,7 +215,7 @@ "@types/node": "^12.12.45", "@types/proper-lockfile": "^4.1.1", "@types/react-beautiful-dnd": "^13.0.0", - "@types/tar": "^4.0.3", + "@types/tar": "^4.0.4", "array-move": "^3.0.0", "chalk": "^4.1.0", "command-exists": "1.2.9", @@ -249,7 +249,7 @@ "serializr": "^2.0.3", "shell-env": "^3.0.0", "spdy": "^4.0.2", - "tar": "^6.0.2", + "tar": "^6.0.5", "tcp-port-used": "^1.0.1", "tempy": "^0.5.0", "uuid": "^8.1.0", @@ -311,7 +311,6 @@ "@types/sharp": "^0.26.0", "@types/shelljs": "^0.8.8", "@types/spdy": "^3.4.4", - "@types/tar": "^4.0.3", "@types/tcp-port-used": "^1.0.0", "@types/tempy": "^0.3.0", "@types/terser-webpack-plugin": "^3.0.0", diff --git a/src/common/utils/downloadFile.ts b/src/common/utils/downloadFile.ts index eea9f24d55..666f713d09 100644 --- a/src/common/utils/downloadFile.ts +++ b/src/common/utils/downloadFile.ts @@ -1,35 +1,33 @@ -import path from "path"; import request from "request"; export interface DownloadFileOptions { url: string; - fileName?: string; // default: based on filename from URL - gzip?: boolean; // default: true + gzip?: boolean; } export interface DownloadFileTicket { - fileName: string; - promise: Promise; + url: string; + promise: Promise; cancel(): void; } export function downloadFile(opts: DownloadFileOptions): DownloadFileTicket { - const { url, gzip = true, fileName = path.basename(url) } = opts; + const { url, gzip = true } = opts; const fileChunks: Buffer[] = []; const req = request(url, { gzip }); - const promise: Promise = new Promise((resolve, reject) => { + const promise: Promise = new Promise((resolve, reject) => { req.on("data", (chunk: Buffer) => { fileChunks.push(chunk); }); req.on("complete", () => { - resolve(new File(fileChunks, fileName)); + resolve(Buffer.concat(fileChunks)); }); req.on("error", err => { reject({ url, err }); }); }); return { - fileName: fileName, + url: url, promise: promise, cancel() { req.abort(); diff --git a/src/common/utils/tar.ts b/src/common/utils/tar.ts new file mode 100644 index 0000000000..39b0a97d9f --- /dev/null +++ b/src/common/utils/tar.ts @@ -0,0 +1,49 @@ +// Helper for working with tarball files (.tar, .tgz) +// Docs: https://github.com/npm/node-tar +import tar, { ExtractOptions, FileStat } from "tar"; +import path from "path"; + +export interface ReadFileFromTarOpts { + fileName?: string; + fileMatcher?(path: string, entry: FileStat): boolean; + notFoundMessage?: string; +} + +export function readFileFromTar(tarFilePath: string, opts: ReadFileFromTarOpts): Promise { + return new Promise(async (resolve, reject) => { + const fileChunks: Buffer[] = []; + const { + fileName, + fileMatcher = (path: string) => path === fileName, + notFoundMessage = "File not found", + } = opts; + + await tar.list({ + file: tarFilePath, + filter: fileMatcher, + onentry(entry: FileStat) { + entry.on("data", chunk => { + fileChunks.push(chunk); + }); + entry.on("error", err => { + reject(`Reading ${entry.path} error: ${err}`); + }); + entry.on("end", () => { + resolve(Buffer.concat(fileChunks)); + }); + }, + }); + + if (!fileChunks.length) { + reject(notFoundMessage); + } + }) +} + +export function extractTar(filePath: string, opts: ExtractOptions & { sync?: boolean } = {}) { + return tar.extract({ + file: filePath, + cwd: path.dirname(filePath), + ...opts, + }) +} diff --git a/src/extensions/extension-manager.ts b/src/extensions/extension-manager.ts index 910af8378d..45455a14e8 100644 --- a/src/extensions/extension-manager.ts +++ b/src/extensions/extension-manager.ts @@ -98,8 +98,12 @@ export class ExtensionManager { } getNpmPackageTarballUrl(packageName: string) { - const command = [this.npmPath, "view", packageName, "dist.tarball", "--silent"]; - return child_process.execSync(command.join(" "), { encoding: "utf8" }).trim(); + try { + const command = [this.npmPath, "view", packageName, "dist.tarball", "--silent"]; + return child_process.execSync(command.join(" "), { encoding: "utf8" }).trim(); + } catch (err) { + return null; + } } protected installPackages(): Promise { diff --git a/src/extensions/lens-extension.ts b/src/extensions/lens-extension.ts index 444cf449d6..20c3583669 100644 --- a/src/extensions/lens-extension.ts +++ b/src/extensions/lens-extension.ts @@ -11,6 +11,7 @@ export interface LensExtensionManifest { description?: string; main?: string; // path to %ext/dist/main.js renderer?: string; // path to %ext/dist/renderer.js + lens?: object; // fixme: add more required fields for validation } export class LensExtension { @@ -95,3 +96,7 @@ export class LensExtension { // mock } } + +export function sanitizeExtensionName(name: string) { + return name.replace("@", "").replace("/", "-"); +} diff --git a/src/extensions/registries/page-registry.ts b/src/extensions/registries/page-registry.ts index 4b9c872715..7141d0b17d 100644 --- a/src/extensions/registries/page-registry.ts +++ b/src/extensions/registries/page-registry.ts @@ -5,7 +5,7 @@ import path from "path"; import { action } from "mobx"; import { compile } from "path-to-regexp"; import { BaseRegistry } from "./base-registry"; -import { LensExtension } from "../lens-extension"; +import { LensExtension, sanitizeExtensionName } from "../lens-extension"; import logger from "../../main/logger"; export interface PageRegistration { @@ -44,10 +44,6 @@ export interface PageComponents { Page: React.ComponentType; } -export function sanitizeExtensionName(name: string) { - return name.replace("@", "").replace("/", "-"); -} - export function getExtensionPageUrl

({ extensionId, pageId = "", params }: PageMenuTarget

): string { const extensionBaseUrl = compile(`/extension/:name`)({ name: sanitizeExtensionName(extensionId), // compile only with extension-id first and define base path diff --git a/src/renderer/components/+extensions/extensions.scss b/src/renderer/components/+extensions/extensions.scss index 8d0582a102..9d47a21800 100644 --- a/src/renderer/components/+extensions/extensions.scss +++ b/src/renderer/components/+extensions/extensions.scss @@ -43,6 +43,10 @@ } } + .SearchInput { + --spacing: #{$padding}; + } + .WizardLayout { padding: 0; @@ -54,6 +58,23 @@ } .InstallingExtensionNotification { + .folder-remove-warning { + font-size: $font-size-small; + color: inherit; + cursor: pointer; + font-style: italic; + opacity: .8; + + &:hover { + opacity: 1; + } + + code { + display: inline; + color: inherit; + } + } + .Button { background-color: unset; border: 1px solid currentColor; diff --git a/src/renderer/components/+extensions/extensions.tsx b/src/renderer/components/+extensions/extensions.tsx index 1a4d7dc664..997b8ffec6 100644 --- a/src/renderer/components/+extensions/extensions.tsx +++ b/src/renderer/components/+extensions/extensions.tsx @@ -1,7 +1,7 @@ import "./extensions.scss"; -import { app, remote, shell } from "electron"; +import { remote, shell } from "electron"; +import os from "os"; import path from "path"; -import tar from "tar"; import fse from "fs-extra"; import React from "react"; import { computed, observable } from "mobx"; @@ -14,14 +14,28 @@ import { DropFileInput, Input, InputValidators, SearchInput } from "../input"; import { Icon } from "../icon"; import { PageLayout } from "../layout/page-layout"; import { Clipboard } from "../clipboard"; +import logger from "../../../main/logger"; import { extensionLoader } from "../../../extensions/extension-loader"; import { extensionManager } from "../../../extensions/extension-manager"; +import { LensExtensionManifest, sanitizeExtensionName } from "../../../extensions/lens-extension"; import { Notifications } from "../notifications"; -import logger from "../../../main/logger"; import { downloadFile } from "../../../common/utils"; +import { extractTar, readFileFromTar } from "../../../common/utils/tar"; + +interface InstallRequest { + fileName: string; + filePath?: string; + data?: Buffer; +} + +interface InstallRequestValidated extends InstallRequest { + manifest: LensExtensionManifest; + tmpFile: string; // temp file for unpacking +} @observer export class Extensions extends React.Component { + private supportedFormats = [".tar", ".tgz"]; @observable search = ""; @observable downloadUrl = ""; @@ -40,96 +54,192 @@ export class Extensions extends React.Component { return extensionManager.localFolderPath; } - selectLocalExtensionsDialog = async () => { - const supportedFormats = [".tgz", ".tar.gz"] + getExtensionDestFolder(name: string) { + return path.join(this.extensionsPath, sanitizeExtensionName(name)); + } + + installFromSelectFileDialog = async () => { const { dialog, BrowserWindow, app } = remote; const { canceled, filePaths } = await dialog.showOpenDialog(BrowserWindow.getFocusedWindow(), { defaultPath: app.getPath("downloads"), properties: ["openFile", "multiSelections"], - message: _i18n._(t`Select extensions to install (supported formats: ${supportedFormats.join(", ")}), `), + message: _i18n._(t`Select extensions to install (formats: ${this.supportedFormats.join(", ")}), `), buttonLabel: _i18n._(t`Use configuration`), filters: [ - { name: "tarball", extensions: supportedFormats } + { name: "tarball", extensions: this.supportedFormats } ] }); if (!canceled && filePaths.length) { - this.installFromSelectFileDialog(filePaths); + this.requestInstall( + filePaths.map(filePath => ({ + fileName: path.basename(filePath), + filePath: filePath, + })) + ); } } - installFromUrl = async () => { - const { downloadUrl } = this; - if (!downloadUrl) { - return; - } - let tarballUrl: string; - if (InputValidators.isUrl.validate(downloadUrl)) { - tarballUrl = downloadUrl; + installExtensions = () => { + if (this.downloadUrl) { + this.installFromNpmOrUrl(this.downloadUrl); + this.downloadUrl = ""; } else { - try { - tarballUrl = extensionManager.getNpmPackageTarballUrl(downloadUrl); - } catch (err) { - Notifications.error(`Error: npm package "${downloadUrl}" not found`); + this.installFromSelectFileDialog(); + } + } + + installFromNpmOrUrl = async (url = this.downloadUrl) => { + if (!InputValidators.isUrl.validate(url)) { + url = extensionManager.getNpmPackageTarballUrl(url); + if (!url) { + Notifications.error(`Error: npm package "${url}" not found!`); return; } } - logger.info('Install from packed extension URL', { tarballUrl }); - if (tarballUrl) { - try { - const { promise: filePromise } = downloadFile({ url: tarballUrl }); - this.requestInstall([await filePromise]); - } catch (err) { - Notifications.error(`Installing extension from ${tarballUrl} has failed: ${String(err)}`); - } + try { + const { promise: filePromise } = downloadFile({ url }); + this.requestInstall([{ + fileName: path.basename(url), + data: await filePromise, + }]); + } catch (err) { + Notifications.error( +

+

Installation from URL has failed: {String(err)}

+

URL: {url}

+
+ ); } } - installFromSelectFileDialog = async (filePaths: string[]) => { - logger.info('Install from file-select dialog', { files: filePaths }); - const files: File[] = await Promise.all( - filePaths.map(filePath => { - const fileName = path.basename(filePath); - return fse.readFile(filePath).then(buffer => new File([buffer], fileName)); - }) - ); - return this.requestInstall(files); - } - installOnDrop = (files: File[]) => { - logger.info('Install from D&D', { files: files.map(file => file.path) }); - return this.requestInstall(files); + logger.info('Install from D&D'); + return this.requestInstall( + files.map(file => ({ + fileName: path.basename(file.path), + filePath: file.path, + })) + ); } - // todo - async installExtension(tarball: File, cleanUp?: () => void) { - logger.info(`Installing extension ${tarball.name} to ${this.extensionsPath}`); - const tempDir = path.join(app.getPath("temp"), "extensions"); - await fse.ensureDir(tempDir); - const unpack = () => { - tar.extract({ - cwd: tempDir, - }) - } - if (cleanUp) { - cleanUp(); - } - } + async requestInstall(installRequests: InstallRequest[]) { + const pendingFiles: Promise[] = []; - // todo: show name and description from unpacked archive - async requestInstall(files: File[]) { - files.forEach((ext: File) => { + // read extensions with provided system path if any + installRequests.forEach(ext => { + if (ext.data) return; + const promise = fse.readFile(ext.filePath) + .then(data => ext.data = data) + .catch(err => { + Notifications.error(`Error while reading "${ext.filePath}": ${String(err)}`); + }); + pendingFiles.push(promise) + }); + await Promise.all(pendingFiles); + installRequests = installRequests.filter(item => item.data); // remove items with reading errors + + // prepare temp folder + const tempFolder = path.join(os.tmpdir(), "lens-extensions"); + await fse.ensureDir(tempFolder); + + // copy files to temp, get extension info from package.json and do basic validation + let validatedInstalls: Promise[] = installRequests.map(async installReq => { + const { fileName, data } = installReq; + const tempFile = path.join(tempFolder, fileName); + await fse.writeFileSync(tempFile, data); // copy to temp + try { + const packageJson: Buffer = await readFileFromTar(tempFile, { + // tarball from npm contains single root folder "package/*" + fileMatcher: (path: string) => !!path.match(/(\w+\/)?package\.json$/), + notFoundMessage: "Extension's manifest file (package.json) not found", + }); + const manifest: LensExtensionManifest = JSON.parse(packageJson.toString("utf8")); + if (!manifest.lens && !manifest.renderer) { + throw `package.json must specify "main" and/or "renderer" fields`; + } + return { + ...installReq, + manifest: manifest, + tmpFile: tempFile, + } + } catch (err) { + fse.unlink(tempFile).catch(() => null); // remove invalid temp file + Notifications.error( +
+

Installing {fileName} has failed, skipping.

+

Reason: {String(err)}

+
+ ); + } + }); + + // final step, provide UI with extension info for reviewing and confirming installation + const extensions = await Promise.all(validatedInstalls); + extensions.forEach(install => { + if (!install) { + return; // skip validating errors if any + } + const { fileName, manifest } = install; + const { name, version, description } = manifest; + const extensionFolder = this.getExtensionDestFolder(name); + const folderExists = fse.existsSync(extensionFolder); const removeNotification = Notifications.info( -
-

Install extension {ext.name}?

-
); }) } + async unpackExtension({ fileName, tmpFile, manifest: { name, version } }: InstallRequestValidated) { + logger.info(`Unpacking extension ${name} from ${fileName}`); + const unpackingTempFolder = path.join(path.dirname(tmpFile), path.basename(tmpFile) + "-unpacked"); + const extensionFolder = this.getExtensionDestFolder(name); + try { + // extract to temp folder first + await fse.remove(unpackingTempFolder).catch(Function); + await fse.ensureDir(unpackingTempFolder); + await extractTar(tmpFile, { cwd: unpackingTempFolder }); + + // move contents to extensions folder + const unpackedFiles = await fse.readdir(unpackingTempFolder); + let unpackedRootFolder = unpackingTempFolder; + if (unpackedFiles.length === 1) { + // handle case when extension.tgz packed with top root folder, + // e.g. "npm pack %ext_name" downloads file with "package" root folder within tarball + unpackedRootFolder = path.join(unpackingTempFolder, unpackedFiles[0]); + } + await fse.ensureDir(extensionFolder); + await fse.move(unpackedRootFolder, extensionFolder, { overwrite: true }); + Notifications.ok( +

Extension {name}/{version} successfully installed!

+ ); + } catch (err) { + Notifications.error( +

Installing extension {name} has failed: {err}

+ ); + } finally { + // clean up + fse.remove(unpackingTempFolder).catch(Function); + fse.unlink(tmpFile).catch(Function); + } + } + renderInfo() { return (
@@ -146,31 +256,25 @@ export class Extensions extends React.Component {
-

Install extensions from archive (tarball.tgz):

-
- this.downloadUrl = v} - onSubmit={this.installFromUrl} - /> - 0} - onClick={this.installFromUrl} - /> -
+ + Install extensions from tarball ({this.supportedFormats.join(", ")}): + + this.downloadUrl = v} + onSubmit={this.installExtensions} + />