Allow to install packed extensions from URL or local file (#1456)

* Option to install an extension from filesystem/url #1227 -- part 1 (UI)

Signed-off-by: Roman <ixrock@gmail.com>

* DropFileInput: common component to handle droped files (replaced also in add-cluster-page)

Signed-off-by: Roman <ixrock@gmail.com>

* fix: install via url-string on input.submit

Signed-off-by: Roman <ixrock@gmail.com>

* ui tweaks & minor fixes

Signed-off-by: Roman <ixrock@gmail.com>

* more ui/ux tweaks & fixes

Signed-off-by: Roman <ixrock@gmail.com>

* layout fixes

Signed-off-by: Roman <ixrock@gmail.com>

* component renaming: `copy-to-click` => `copy-to-clipboard` => `clipboard`

Signed-off-by: Roman <ixrock@gmail.com>

* reworks -- part 1

Signed-off-by: Roman <ixrock@gmail.com>

* fix downloading file, added common/utils/downloadFile

Signed-off-by: Roman <ixrock@gmail.com>

* confirm before install, unpack tar first steps

Signed-off-by: Roman <ixrock@gmail.com>

* installation flow, extracting .tgz

Signed-off-by: Roman <ixrock@gmail.com>

* clean up, fix lint issues

Signed-off-by: Roman <ixrock@gmail.com>

* update .azure-pipelines.yml

Signed-off-by: Roman <ixrock@gmail.com>

* fixes & refactoring

Signed-off-by: Roman <ixrock@gmail.com>

* fix lint harder :/

Signed-off-by: Roman <ixrock@gmail.com>

* fix validation

Signed-off-by: Roman <ixrock@gmail.com>

* fix validation harder

Signed-off-by: Roman <ixrock@gmail.com>

* responding to comments, fixed package validation

Signed-off-by: Roman <ixrock@gmail.com>

* common/utils/tar.ts: reject with Error-type

Signed-off-by: Roman <ixrock@gmail.com>

* fix: unit-tests

Signed-off-by: Roman <ixrock@gmail.com>

src/common/utils/downloadFile.ts

@@ -0,0 +1,35 @@
+import request from "request";
+
+export interface DownloadFileOptions {
+  url: string;
+  gzip?: boolean;
+}
+
+export interface DownloadFileTicket {
+  url: string;
+  promise: Promise<Buffer>;
+  cancel(): void;
+}
+
+export function downloadFile({ url, gzip = true }: DownloadFileOptions): DownloadFileTicket {
+  const fileChunks: Buffer[] = [];
+  const req = request(url, { gzip });
+  const promise: Promise<Buffer> = new Promise((resolve, reject) => {
+    req.on("data", (chunk: Buffer) => {
+      fileChunks.push(chunk);
+    });
+    req.once("error", err => {
+      reject({ url, err });
+    });
+    req.once("complete", () => {
+      resolve(Buffer.concat(fileChunks));
+    });
+  });
+  return {
+    url,
+    promise,
+    cancel() {
+      req.abort();
+    }
+  };
+}

src/common/utils/escapeRegExp.ts

@@ -0,0 +1,5 @@
+// Helper to sanitize / escape special chars for passing to RegExp-constructor
+
+export function escapeRegExp(str: string) {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
+}

src/common/utils/index.ts

@@ -15,3 +15,6 @@ export * from "./saveToAppFiles";
 export * from "./singleton";
 export * from "./openExternal";
 export * from "./rectify-array";
+export * from "./downloadFile";
+export * from "./escapeRegExp";
+export * from "./tar";

src/common/utils/tar.ts

@@ -0,0 +1,55 @@
+// Helper for working with tarball files (.tar, .tgz)
+// Docs: https://github.com/npm/node-tar
+import tar, { ExtractOptions, FileStat } from "tar";
+import path from "path";
+
+export interface ReadFileFromTarOpts {
+  tarPath: string;
+  filePath: string;
+  parseJson?: boolean;
+}
+
+export function readFileFromTar<R = Buffer>({ tarPath, filePath, parseJson }: ReadFileFromTarOpts): Promise<R> {
+  return new Promise(async (resolve, reject) => {
+    const fileChunks: Buffer[] = [];
+
+    await tar.list({
+      file: tarPath,
+      filter: path => path === filePath,
+      onentry(entry: FileStat) {
+        entry.on("data", chunk => {
+          fileChunks.push(chunk);
+        });
+        entry.once("error", err => {
+          reject(new Error(`reading file has failed ${entry.path}: ${err}`));
+        });
+        entry.once("end", () => {
+          const data = Buffer.concat(fileChunks);
+          const result = parseJson ? JSON.parse(data.toString("utf8")) : data;
+          resolve(result);
+        });
+      },
+    });
+
+    if (!fileChunks.length) {
+      reject(new Error("Not found"));
+    }
+  });
+}
+
+export async function listTarEntries(filePath: string): Promise<string[]> {
+  const entries: string[] = [];
+  await tar.list({
+    file: filePath,
+    onentry: (entry: FileStat) => entries.push(entry.path as any as string),
+  });
+  return entries;
+}
+
+export function extractTar(filePath: string, opts: ExtractOptions & { sync?: boolean } = {}) {
+  return tar.extract({
+    file: filePath,
+    cwd: path.dirname(filePath),
+    ...opts,
+  });
+}