Consolidate downloading binaries into one script (#4942)

* Consolidate downloading binaries into one script - Upgrade lens-k8s-proxy to 0.1.5 which allows us to remove the trailing slash in KubeAuthProxy - Consolidate nromalizing arch and os strings - Remove LensBinary as HelmCli is not just a raw object since it is always bundled - Introduce OnceCell and use it for the binary paths Signed-off-by: Sebastian Malton <sebastian@malton.name> * Fully remove helmCli, move associated variable to vars Signed-off-by: Sebastian Malton <sebastian@malton.name> * Fix helm downloading on windows Signed-off-by: Sebastian Malton <sebastian@malton.name> * Don't download binaries for unsupported windows Signed-off-by: Sebastian Malton <sebastian@malton.name> * Arch specific paths Signed-off-by: Sebastian Malton <sebastian@malton.name> * Fix downloading helm on windows Signed-off-by: Sebastian Malton <sebastian@malton.name> * rename once-cell file to lazy-initialized Signed-off-by: Sebastian Malton <sebastian@malton.name>
2025-05-20 05:10:56 +00:00 · 2022-03-09 17:51:51 -05:00
1 parent dd5dfb393d
commit a19f094a0d
@@ -0,0 +1,235 @@
+/**
+ * Copyright (c) OpenLens Authors. All rights reserved.
+ * Licensed under MIT License. See LICENSE in root directory for more information.
+ */
+import packageInfo from "../package.json";
+import { type WriteStream } from "fs";
+import { FileHandle, open } from "fs/promises";
+import { constants, ensureDir, unlink } from "fs-extra";
+import path from "path";
+import fetch from "node-fetch";
+import { promisify } from "util";
+import { pipeline as _pipeline, Transform, Writable } from "stream";
+import { MultiBar, SingleBar } from "cli-progress";
+import AbortController from "abort-controller";
+import { extract } from "tar-stream";
+import gunzip from "gunzip-maybe";
+import { getBinaryName, normalizedPlatform } from "../src/common/vars";
+
+const pipeline = promisify(_pipeline);
+
+interface BinaryDownloaderArgs {
+  readonly version: string;
+  readonly platform: SupportedPlatform;
+  readonly downloadArch: string;
+  readonly fileArch: string;
+  readonly binaryName: string;
+  readonly baseDir: string;
+}
+
+abstract class BinaryDownloader {
+  protected abstract readonly url: string;
+  protected readonly bar: SingleBar;
+  protected readonly target: string;
+
+  protected getTransformStreams(file: Writable): (NodeJS.ReadWriteStream | NodeJS.WritableStream)[] {
+    return [file];
+  }
+
+  constructor(public readonly args: BinaryDownloaderArgs, multiBar: MultiBar) {
+    this.bar = multiBar.create(1, 0, args);
+    this.target = path.join(args.baseDir, args.platform, args.fileArch, args.binaryName);
+  }
+
+  async ensureBinary(): Promise<void> {
+    const controller = new AbortController();
+    const stream = await fetch(this.url, {
+      timeout: 15 * 60 * 1000, // 15min
+      signal: controller.signal,
+    });
+    const total = Number(stream.headers.get("content-length"));
+    const bar = this.bar;
+    let fileHandle: FileHandle;
+
+    if (isNaN(total)) {
+      throw new Error("no content-length header was present");
+    }
+
+    bar.setTotal(total);
+
+    await ensureDir(path.dirname(this.target), 0o755);
+
+    try {
+      /**
+       * This is necessary because for some reason `createWriteStream({ flags: "wx" })`
+       * was throwing someplace else and not here
+       */
+      fileHandle = await open(this.target, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL);
+
+      await pipeline(
+        stream.body,
+        new Transform({
+          transform(chunk, encoding, callback) {
+            bar.increment(chunk.length);
+            this.push(chunk);
+            callback();
+          },
+        }),
+        ...this.getTransformStreams(new Writable({
+          write(chunk, encoding, cb) {
+            fileHandle.write(chunk)
+              .then(() => cb())
+              .catch(cb);
+          },
+        })),
+      );
+      await fileHandle.chmod(0o755);
+      await fileHandle.close();
+    } catch (error) {
+      await fileHandle?.close();
+
+      if (error.code === "EEXIST") {
+        bar.increment(total); // mark as finished
+        controller.abort(); // stop trying to download
+      } else {
+        await unlink(this.target);
+        throw error;
+      }
+    }
+  }
+}
+
+class LensK8sProxyDownloader extends BinaryDownloader {
+  protected readonly url: string;
+
+  constructor(args: Omit<BinaryDownloaderArgs, "binaryName">, bar: MultiBar) {
+    const binaryName = getBinaryName("lens-k8s-proxy", { forPlatform: args.platform });
+
+    super({ ...args, binaryName }, bar);
+    this.url = `https://github.com/lensapp/lens-k8s-proxy/releases/download/v${args.version}/lens-k8s-proxy-${args.platform}-${args.downloadArch}`;
+  }
+}
+
+class KubectlDownloader extends BinaryDownloader {
+  protected readonly url: string;
+
+  constructor(args: Omit<BinaryDownloaderArgs, "binaryName">, bar: MultiBar) {
+    const binaryName = getBinaryName("kubectl", { forPlatform: args.platform });
+
+    super({ ...args, binaryName }, bar);
+    this.url = `https://storage.googleapis.com/kubernetes-release/release/v${args.version}/bin/${args.platform}/${args.downloadArch}/${binaryName}`;
+  }
+}
+
+class HelmDownloader extends BinaryDownloader {
+  protected readonly url: string;
+
+  constructor(args: Omit<BinaryDownloaderArgs, "binaryName">, bar: MultiBar) {
+    const binaryName = getBinaryName("helm", { forPlatform: args.platform });
+
+    super({ ...args, binaryName }, bar);
+    this.url = `https://get.helm.sh/helm-v${args.version}-${args.platform}-${args.downloadArch}.tar.gz`;
+  }
+
+  protected getTransformStreams(file: WriteStream) {
+    const extracting = extract({
+      allowUnknownFormat: false,
+    });
+
+    extracting.on("entry", (headers, stream, next) => {
+      if (headers.name.endsWith(this.args.binaryName)) {
+        stream
+          .pipe(file)
+          .once("finish", () => next())
+          .once("error", next);
+      } else {
+        stream.resume();
+        next();
+      }
+    });
+
+    return [gunzip(3), extracting];
+  }
+}
+
+type SupportedPlatform = "darwin" | "linux" | "windows";
+
+async function main() {
+  const multiBar = new MultiBar({
+    align: "left",
+    clearOnComplete: false,
+    hideCursor: true,
+    autopadding: true,
+    noTTYOutput: true,
+    format: "[{bar}] {percentage}% | {downloadArch} {binaryName}",
+  });
+  const baseDir = path.join(__dirname, "..", "binaries", "client");
+  const downloaders: BinaryDownloader[] = [
+    new LensK8sProxyDownloader({
+      version: packageInfo.config.k8sProxyVersion,
+      platform: normalizedPlatform,
+      downloadArch: "amd64",
+      fileArch: "x64",
+      baseDir,
+    }, multiBar),
+    new KubectlDownloader({
+      version: packageInfo.config.bundledKubectlVersion,
+      platform: normalizedPlatform,
+      downloadArch: "amd64",
+      fileArch: "x64",
+      baseDir,
+    }, multiBar),
+    new HelmDownloader({
+      version: packageInfo.config.bundledHelmVersion,
+      platform: normalizedPlatform,
+      downloadArch: "amd64",
+      fileArch: "x64",
+      baseDir,
+    }, multiBar),
+  ];
+
+  if (normalizedPlatform === "darwin") {
+    downloaders.push(
+      new LensK8sProxyDownloader({
+        version: packageInfo.config.k8sProxyVersion,
+        platform: normalizedPlatform,
+        downloadArch: "arm64",
+        fileArch: "arm64",
+        baseDir,
+      }, multiBar),
+      new KubectlDownloader({
+        version: packageInfo.config.bundledKubectlVersion,
+        platform: normalizedPlatform,
+        downloadArch: "arm64",
+        fileArch: "arm64",
+        baseDir,
+      }, multiBar),
+      new HelmDownloader({
+        version: packageInfo.config.bundledHelmVersion,
+        platform: normalizedPlatform,
+        downloadArch: "arm64",
+        fileArch: "arm64",
+        baseDir,
+      }, multiBar),
+    );
+  }
+
+  const settledResults = await Promise.allSettled(downloaders.map(downloader => (
+    downloader.ensureBinary()
+      .catch(error => {
+        throw new Error(`Failed to download ${downloader.args.binaryName} for ${downloader.args.platform}/${downloader.args.downloadArch}: ${error}`);
+      })
+  )));
+
+  multiBar.stop();
+  const errorResult = settledResults.find(res => res.status === "rejected") as PromiseRejectedResult | undefined;
+
+  if (errorResult) {
+    console.error("234", String(errorResult.reason));
+    process.exit(1);
+  }
+
+  process.exit(0);
+}
+
+main().catch(error => console.error("from main", error));
@@ -1,19 +0,0 @@
-/**
- * Copyright (c) OpenLens Authors. All rights reserved.
- * Licensed under MIT License. See LICENSE in root directory for more information.
- */
-import packageInfo from "../package.json";
-import { isWindows } from "../src/common/vars";
-import { HelmCli } from "../src/main/helm/helm-cli";
-import * as path from "path";
-
-const helmVersion = packageInfo.config.bundledHelmVersion;
-
-if (!isWindows) {
-  Promise.all([
-    new HelmCli(path.join(process.cwd(), "binaries", "client", "x64"), helmVersion).ensureBinary(),
-    new HelmCli(path.join(process.cwd(), "binaries", "client", "arm64"), helmVersion).ensureBinary(),
-  ]);
-} else {
-  new HelmCli(path.join(process.cwd(), "binaries", "client", "x64"), helmVersion).ensureBinary();
-}
@@ -1,98 +0,0 @@
-/**
- * Copyright (c) OpenLens Authors. All rights reserved.
- * Licensed under MIT License. See LICENSE in root directory for more information.
- */
-import packageInfo from "../package.json";
-import fs from "fs";
-import request from "request";
-import { ensureDir, pathExists } from "fs-extra";
-import path from "path";
-import { noop } from "lodash";
-import { isLinux, isMac } from "../src/common/vars";
-
-class K8sProxyDownloader {
-  public version: string;
-  protected url: string;
-  protected path: string;
-  protected dirname: string;
-
-  constructor(version: string, platform: string, arch: string, target: string) {
-    this.version = version;
-    this.url = `https://github.com/lensapp/lens-k8s-proxy/releases/download/v${this.version}/lens-k8s-proxy-${platform}-${arch}`;
-    this.dirname = path.dirname(target);
-    this.path = target;
-  }
-
-  public async checkBinary() {
-    const exists = await pathExists(this.path);
-
-    if (exists) {
-      return true;
-    }
-
-    return false;
-  }
-
-  public async download() {
-    if (await this.checkBinary()) {
-      return console.log("Already exists");
-    }
-
-    await ensureDir(path.dirname(this.path), 0o755);
-
-    const file = fs.createWriteStream(this.path);
-
-    console.log(`Downloading lens-k8s-proxy ${this.version} from ${this.url} to ${this.path}`);
-    const requestOpts: request.UriOptions & request.CoreOptions = {
-      uri: this.url,
-      gzip: true,
-      followAllRedirects: true,
-    };
-    const stream = request(requestOpts);
-
-    stream.on("complete", () => {
-      console.log("lens-k8s-proxy binary download finished");
-      file.end(noop);
-    });
-
-    stream.on("error", (error) => {
-      console.log(error);
-      fs.unlink(this.path, noop);
-      throw error;
-    });
-
-    return new Promise<void>((resolve, reject) => {
-      file.on("close", () => {
-        console.log("lens-k8s-proxy binary download closed");
-        fs.chmod(this.path, 0o755, (err) => {
-          if (err) reject(err);
-        });
-        resolve();
-      });
-      stream.pipe(file);
-    });
-  }
-}
-const downloadVersion = packageInfo.config.k8sProxyVersion;
-const baseDir = path.join(__dirname, "..", "binaries", "client");
-
-const downloads = [];
-
-if (isMac) {
-  downloads.push({ platform: "darwin", arch: "amd64", target: path.join(baseDir, "darwin", "x64", "lens-k8s-proxy") });
-  downloads.push({ platform: "darwin", arch: "arm64", target: path.join(baseDir, "darwin", "arm64", "lens-k8s-proxy") });
-} else if (isLinux) {
-  downloads.push({ platform: "linux", arch: "amd64", target: path.join(baseDir, "linux", "x64", "lens-k8s-proxy") });
-  downloads.push({ platform: "linux", arch: "arm64", target: path.join(baseDir, "linux", "arm64", "lens-k8s-proxy") });
-} else {
-  downloads.push({ platform: "windows", arch: "amd64", target: path.join(baseDir, "windows", "x64", "lens-k8s-proxy.exe") });
-  downloads.push({ platform: "windows", arch: "386", target: path.join(baseDir, "windows", "ia32", "lens-k8s-proxy.exe") });
-}
-
-downloads.forEach((dlOpts) => {
-  console.log(dlOpts);
-  const downloader = new K8sProxyDownloader(downloadVersion, dlOpts.platform, dlOpts.arch, dlOpts.target);
-
-  console.log(`Downloading: ${JSON.stringify(dlOpts)}`);
-  downloader.download().then(() => downloader.checkBinary().then(() => console.log("Download complete")));
-});
@@ -1,125 +0,0 @@
-/**
- * Copyright (c) OpenLens Authors. All rights reserved.
- * Licensed under MIT License. See LICENSE in root directory for more information.
- */
-import packageInfo from "../package.json";
-import fs from "fs";
-import request from "request";
-import md5File from "md5-file";
-import requestPromise from "request-promise-native";
-import { ensureDir, pathExists } from "fs-extra";
-import path from "path";
-import { noop } from "lodash";
-import { isLinux, isMac } from "../src/common/vars";
-
-class KubectlDownloader {
-  public kubectlVersion: string;
-  protected url: string;
-  protected path: string;
-  protected dirname: string;
-
-  constructor(clusterVersion: string, platform: string, arch: string, target: string) {
-    this.kubectlVersion = clusterVersion;
-    const binaryName = platform === "windows" ? "kubectl.exe" : "kubectl";
-
-    this.url = `https://storage.googleapis.com/kubernetes-release/release/v${this.kubectlVersion}/bin/${platform}/${arch}/${binaryName}`;
-    this.dirname = path.dirname(target);
-    this.path = target;
-  }
-
-  protected async urlEtag() {
-    const response = await requestPromise({
-      method: "HEAD",
-      uri: this.url,
-      resolveWithFullResponse: true,
-    }).catch(console.error);
-
-    if (response.headers["etag"]) {
-      return response.headers["etag"].replace(/"/g, "");
-    }
-
-    return "";
-  }
-
-  public async checkBinary() {
-    const exists = await pathExists(this.path);
-
-    if (exists) {
-      const hash = md5File.sync(this.path);
-      const etag = await this.urlEtag();
-
-      if (hash == etag) {
-        console.log("Kubectl md5sum matches the remote etag");
-
-        return true;
-      }
-
-      console.log(`Kubectl md5sum ${hash} does not match the remote etag ${etag}, unlinking and downloading again`);
-      await fs.promises.unlink(this.path);
-    }
-
-    return false;
-  }
-
-  public async downloadKubectl() {
-    if (await this.checkBinary()) {
-      return console.log("Already exists and is valid");
-    }
-
-    await ensureDir(path.dirname(this.path), 0o755);
-
-    const file = fs.createWriteStream(this.path);
-
-    console.log(`Downloading kubectl ${this.kubectlVersion} from ${this.url} to ${this.path}`);
-    const requestOpts: request.UriOptions & request.CoreOptions = {
-      uri: this.url,
-      gzip: true,
-    };
-    const stream = request(requestOpts);
-
-    stream.on("complete", () => {
-      console.log("kubectl binary download finished");
-      file.end(noop);
-    });
-
-    stream.on("error", (error) => {
-      console.log(error);
-      fs.unlink(this.path, noop);
-      throw error;
-    });
-
-    return new Promise<void>((resolve, reject) => {
-      file.on("close", () => {
-        console.log("kubectl binary download closed");
-        fs.chmod(this.path, 0o755, (err) => {
-          if (err) reject(err);
-        });
-        resolve();
-      });
-      stream.pipe(file);
-    });
-  }
-}
-const downloadVersion = packageInfo.config.bundledKubectlVersion;
-const baseDir = path.join(__dirname, "..", "binaries", "client");
-
-const downloads = [];
-
-if (isMac) {
-  downloads.push({ platform: "darwin", arch: "amd64", target: path.join(baseDir, "darwin", "x64", "kubectl") });
-  downloads.push({ platform: "darwin", arch: "arm64", target: path.join(baseDir, "darwin", "arm64", "kubectl") });
-} else if (isLinux) {
-  downloads.push({ platform: "linux", arch: "amd64", target: path.join(baseDir, "linux", "x64", "kubectl") });
-  downloads.push({ platform: "linux", arch: "arm64", target: path.join(baseDir, "linux", "arm64", "kubectl") });
-} else {
-  downloads.push({ platform: "windows", arch: "amd64", target: path.join(baseDir, "windows", "x64", "kubectl.exe") });
-  downloads.push({ platform: "windows", arch: "386", target: path.join(baseDir, "windows", "ia32", "kubectl.exe") });
-}
-
-downloads.forEach((dlOpts) => {
-  console.log(dlOpts);
-  const downloader = new KubectlDownloader(downloadVersion, dlOpts.platform, dlOpts.arch, dlOpts.target);
-
-  console.log(`Downloading: ${JSON.stringify(dlOpts)}`);
-  downloader.downloadKubectl().then(() => downloader.checkBinary().then(() => console.log("Download complete")));
-});