Restructure to monorepo (#6907)

* wip: restructure to monorepo

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* refactor create-release-pr to a package

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* build fixes

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* github workflow fixes

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* fix typo

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* add webpack-env types to core

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* fix github workflows

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* refactor/fix integration tests

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* lint fix

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* yarn run dev

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* eslint settings for vscode

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* move templates to right package

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* open-lens build fixes

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* integration test fix

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* fix nx task dependencies

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* use bash shell for unit tests in test workflow

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* fix test:unit for windows

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* fix win-ca webpack error in open-lens

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* fix win-ca webpack error in open-lens

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* fix build:app on windows

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* remove ELECTRON_BUILDER_EXTRA_ARGS

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* sync src/ from master

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

* remove Makefile from core

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

Signed-off-by: Jari Kolehmainen <jari.kolehmainen@gmail.com>

packages/core/build/download_binaries.ts

@@ -0,0 +1,280 @@
+/**
+ * Copyright (c) OpenLens Authors. All rights reserved.
+ * Licensed under MIT License. See LICENSE in root directory for more information.
+ */
+import packageInfo from "../package.json";
+import type { FileHandle } from "fs/promises";
+import { open } from "fs/promises";
+import type { WriteStream } from "fs-extra";
+import { constants, ensureDir, unlink } from "fs-extra";
+import path from "path";
+import type * as FetchModule from "node-fetch";
+import { promisify } from "util";
+import { pipeline as _pipeline, Transform, Writable } from "stream";
+import type { SingleBar } from "cli-progress";
+import { MultiBar } from "cli-progress";
+import { extract } from "tar-stream";
+import gunzip from "gunzip-maybe";
+import { isErrnoException, setTimeoutFor } from "../src/common/utils";
+import AbortController from "abort-controller";
+
+type Response = FetchModule.Response;
+type RequestInfo = FetchModule.RequestInfo;
+type RequestInit = FetchModule.RequestInit;
+
+const pipeline = promisify(_pipeline);
+
+const getBinaryName = (binaryName: string, { forPlatform }: { forPlatform : string }) => {
+  if (forPlatform === "windows") {
+    return `${binaryName}.exe`;
+  }
+
+  return binaryName;
+};
+
+interface BinaryDownloaderArgs {
+  readonly version: string;
+  readonly platform: SupportedPlatform;
+  readonly downloadArch: string;
+  readonly fileArch: string;
+  readonly binaryName: string;
+  readonly baseDir: string;
+}
+
+interface BinaryDownloaderDependencies {
+  fetch: (url: RequestInfo, init?: RequestInit) => Promise<Response>;
+}
+
+abstract class BinaryDownloader {
+  protected abstract readonly url: string;
+  protected readonly bar: SingleBar;
+  protected readonly target: string;
+
+  protected getTransformStreams(file: Writable): (NodeJS.ReadWriteStream | NodeJS.WritableStream)[] {
+    return [file];
+  }
+
+  constructor(protected readonly dependencies: BinaryDownloaderDependencies, public readonly args: BinaryDownloaderArgs, multiBar: MultiBar) {
+    this.bar = multiBar.create(1, 0, args);
+    this.target = path.join(args.baseDir, args.platform, args.fileArch, args.binaryName);
+  }
+
+  async ensureBinary(): Promise<void> {
+    if (process.env.LENS_SKIP_DOWNLOAD_BINARIES === "true") {
+      return;
+    }
+
+    const controller = new AbortController();
+
+    setTimeoutFor(controller, 15 * 60 * 1000);
+
+    const stream = await this.dependencies.fetch(this.url, {
+      signal: controller.signal,
+    });
+    const total = Number(stream.headers.get("content-length"));
+    const bar = this.bar;
+    let fileHandle: FileHandle | undefined = undefined;
+
+    if (isNaN(total)) {
+      throw new Error("no content-length header was present");
+    }
+
+    bar.setTotal(total);
+
+    await ensureDir(path.dirname(this.target), 0o755);
+
+    try {
+      /**
+       * This is necessary because for some reason `createWriteStream({ flags: "wx" })`
+       * was throwing someplace else and not here
+       */
+      const handle = fileHandle = await open(this.target, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL);
+
+      if (!stream.body) {
+        throw new Error("no body on stream");
+      }
+
+      await pipeline(
+        stream.body,
+        new Transform({
+          transform(chunk, encoding, callback) {
+            bar.increment(chunk.length);
+            this.push(chunk);
+            callback();
+          },
+        }),
+        ...this.getTransformStreams(new Writable({
+          write(chunk, encoding, cb) {
+            handle.write(chunk)
+              .then(() => cb())
+              .catch(cb);
+          },
+        })),
+      );
+      await fileHandle.chmod(0o755);
+      await fileHandle.close();
+    } catch (error) {
+      await fileHandle?.close();
+
+      if (isErrnoException(error) && error.code === "EEXIST") {
+        bar.increment(total); // mark as finished
+        controller.abort(); // stop trying to download
+      } else {
+        await unlink(this.target);
+        throw error;
+      }
+    }
+  }
+}
+
+class LensK8sProxyDownloader extends BinaryDownloader {
+  protected readonly url: string;
+
+  constructor(deps: BinaryDownloaderDependencies, args: Omit<BinaryDownloaderArgs, "binaryName">, bar: MultiBar) {
+    const binaryName = getBinaryName("lens-k8s-proxy", { forPlatform: args.platform });
+
+    super(deps, { ...args, binaryName }, bar);
+    this.url = `https://github.com/lensapp/lens-k8s-proxy/releases/download/v${args.version}/lens-k8s-proxy-${args.platform}-${args.downloadArch}`;
+  }
+}
+
+class KubectlDownloader extends BinaryDownloader {
+  protected readonly url: string;
+
+  constructor(deps: BinaryDownloaderDependencies, args: Omit<BinaryDownloaderArgs, "binaryName">, bar: MultiBar) {
+    const binaryName = getBinaryName("kubectl", { forPlatform: args.platform });
+
+    super(deps, { ...args, binaryName }, bar);
+    this.url = `https://storage.googleapis.com/kubernetes-release/release/v${args.version}/bin/${args.platform}/${args.downloadArch}/${binaryName}`;
+  }
+}
+
+class HelmDownloader extends BinaryDownloader {
+  protected readonly url: string;
+
+  constructor(deps: BinaryDownloaderDependencies, args: Omit<BinaryDownloaderArgs, "binaryName">, bar: MultiBar) {
+    const binaryName = getBinaryName("helm", { forPlatform: args.platform });
+
+    super(deps, { ...args, binaryName }, bar);
+    this.url = `https://get.helm.sh/helm-v${args.version}-${args.platform}-${args.downloadArch}.tar.gz`;
+  }
+
+  protected getTransformStreams(file: WriteStream) {
+    const extracting = extract({
+      allowUnknownFormat: false,
+    });
+
+    extracting.on("entry", (headers, stream, next) => {
+      if (headers.name.endsWith(this.args.binaryName)) {
+        stream
+          .pipe(file)
+          .once("finish", () => next())
+          .once("error", next);
+      } else {
+        stream.resume();
+        next();
+      }
+    });
+
+    return [gunzip(3), extracting];
+  }
+}
+
+type SupportedPlatform = "darwin" | "linux" | "windows";
+
+const importFetchModule = new Function('return import("node-fetch")') as () => Promise<typeof FetchModule>;
+
+async function main() {
+  const deps: BinaryDownloaderDependencies = {
+    fetch: (await importFetchModule()).default,
+  };
+  const normalizedPlatform = (() => {
+    switch (process.platform) {
+      case "darwin":
+        return "darwin";
+      case "linux":
+        return "linux";
+      case "win32":
+        return "windows";
+      default:
+        throw new Error(`platform=${process.platform} is unsupported`);
+    }
+  })();
+  const multiBar = new MultiBar({
+    align: "left",
+    clearOnComplete: false,
+    hideCursor: true,
+    autopadding: true,
+    noTTYOutput: true,
+    format: "[{bar}] {percentage}% | {downloadArch} {binaryName}",
+  });
+  const baseDir = path.join(process.cwd(), "binaries", "client");
+  const downloaders: BinaryDownloader[] = [
+    new LensK8sProxyDownloader(deps, {
+      version: packageInfo.config.k8sProxyVersion,
+      platform: normalizedPlatform,
+      downloadArch: "amd64",
+      fileArch: "x64",
+      baseDir,
+    }, multiBar),
+    new KubectlDownloader(deps, {
+      version: packageInfo.config.bundledKubectlVersion,
+      platform: normalizedPlatform,
+      downloadArch: "amd64",
+      fileArch: "x64",
+      baseDir,
+    }, multiBar),
+    new HelmDownloader(deps, {
+      version: packageInfo.config.bundledHelmVersion,
+      platform: normalizedPlatform,
+      downloadArch: "amd64",
+      fileArch: "x64",
+      baseDir,
+    }, multiBar),
+  ];
+
+  if (normalizedPlatform !== "windows") {
+    downloaders.push(
+      new LensK8sProxyDownloader(deps, {
+        version: packageInfo.config.k8sProxyVersion,
+        platform: normalizedPlatform,
+        downloadArch: "arm64",
+        fileArch: "arm64",
+        baseDir,
+      }, multiBar),
+      new KubectlDownloader(deps, {
+        version: packageInfo.config.bundledKubectlVersion,
+        platform: normalizedPlatform,
+        downloadArch: "arm64",
+        fileArch: "arm64",
+        baseDir,
+      }, multiBar),
+      new HelmDownloader(deps, {
+        version: packageInfo.config.bundledHelmVersion,
+        platform: normalizedPlatform,
+        downloadArch: "arm64",
+        fileArch: "arm64",
+        baseDir,
+      }, multiBar),
+    );
+  }
+
+  const settledResults = await Promise.allSettled(downloaders.map(downloader => (
+    downloader.ensureBinary()
+      .catch(error => {
+        throw new Error(`Failed to download ${downloader.args.binaryName} for ${downloader.args.platform}/${downloader.args.downloadArch}: ${error}`);
+      })
+  )));
+
+  multiBar.stop();
+  const errorResult = settledResults.find(res => res.status === "rejected") as PromiseRejectedResult | undefined;
+
+  if (errorResult) {
+    console.error("234", String(errorResult.reason));
+    process.exit(1);
+  }
+
+  process.exit(0);
+}
+
+main().catch(error => console.error("from main", error));

packages/core/build/entitlements.mac.plist

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+  <dict>
+    <key>com.apple.security.cs.allow-jit</key>
+    <true/>
+    <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+    <true/>
+    <key>com.apple.security.cs.disable-library-validation</key>
+    <true/>
+  </dict>
+</plist>

packages/core/build/generate-tray-icons.ts

@@ -0,0 +1,139 @@
+/**
+ * Copyright (c) OpenLens Authors. All rights reserved.
+ * Licensed under MIT License. See LICENSE in root directory for more information.
+ */
+
+import { ensureDir, readFile } from "fs-extra";
+import { JSDOM } from "jsdom";
+import path from "path";
+import sharp from "sharp";
+
+const size = Number(process.env.OUTPUT_SIZE || "16");
+const outputFolder = process.env.OUTPUT_DIR || "./static/build/tray";
+const inputFile = process.env.INPUT_SVG_PATH || path.resolve(__dirname, "../src/renderer/components/icon/logo-lens.svg");
+const noticeFile = process.env.NOTICE_SVG_PATH || path.resolve(__dirname, "../src/renderer/components/icon/notice.svg");
+const spinnerFile = process.env.SPINNER_SVG_PATH || path.resolve(__dirname, "../src/renderer/components/icon/arrow-spinner.svg");
+
+async function ensureOutputFoler() {
+  await ensureDir(outputFolder);
+}
+
+function getSvgStyling(colouring: "dark" | "light"): string {
+  return `
+    <style>
+      ellipse {
+        stroke: ${colouring === "dark" ? "white" : "black"} !important;
+      }
+      path, rect {
+        fill: ${colouring === "dark" ? "white" : "black"} !important;
+      }
+    </style>
+  `;
+}
+
+type TargetSystems = "macos" | "windows-or-linux";
+
+async function getBaseIconImage(system: TargetSystems) {
+  const svgData = await readFile(inputFile, { encoding: "utf-8" });
+  const dom = new JSDOM(`<body>${svgData}</body>`);
+  const root = dom.window.document.body.getElementsByTagName("svg")[0];
+
+  root.innerHTML += getSvgStyling(system === "macos" ? "light" : "dark");
+
+  return Buffer.from(root.outerHTML);
+}
+
+async function generateImage(image: Buffer, size: number, namePrefix: string) {
+  sharp(image)
+    .resize({ width: size, height: size })
+    .png()
+    .toFile(path.join(outputFolder, `${namePrefix}.png`));
+}
+
+async function generateImages(image: Buffer, size: number, name: string) {
+  await Promise.all([
+    generateImage(image, size, name),
+    generateImage(image, size*2, `${name}@2x`),
+    generateImage(image, size*3, `${name}@3x`),
+    generateImage(image, size*4, `${name}@4x`),
+  ]);
+}
+
+async function generateImageWithSvg(baseImage: Buffer, system: TargetSystems, filePath: string) {
+  const svgFile = await getIconImage(system, filePath);
+
+  const circleBuffer = await sharp(Buffer.from(`
+    <svg viewBox="0 0 64 64">
+      <circle cx="32" cy="32" r="32" fill="black" />
+    </svg>
+  `))
+    .toBuffer();
+
+  return sharp(baseImage)
+    .resize({ width: 128, height: 128 })
+    .composite([
+      {
+        input: circleBuffer,
+        top: 64,
+        left: 64,
+        blend: "dest-out",
+      },
+      {
+        input: (
+          await sharp(svgFile)
+            .resize({
+              width: 60,
+              height: 60,
+            })
+            .toBuffer()
+        ),
+        top: 66,
+        left: 66,
+      },
+    ])
+    .toBuffer();
+}
+
+async function getIconImage(system: TargetSystems, filePath: string) {
+  const svgData = await readFile(filePath, { encoding: "utf-8" });
+  const root = new JSDOM(svgData).window.document.getElementsByTagName("svg")[0];
+
+  root.innerHTML += getSvgStyling(system === "macos" ? "light" : "dark");
+
+  return Buffer.from(root.outerHTML);
+}
+
+async function generateTrayIcons() {
+  try {
+    console.log("Generating tray icon pngs");
+    await ensureOutputFoler();
+
+    const baseIconTemplateImage = await getBaseIconImage("macos");
+    const baseIconImage = await getBaseIconImage("windows-or-linux");
+
+    const updateAvailableTemplateImage = await generateImageWithSvg(baseIconTemplateImage, "macos", noticeFile);
+    const updateAvailableImage = await generateImageWithSvg(baseIconImage, "windows-or-linux", noticeFile);
+
+    const checkingForUpdatesTemplateImage = await generateImageWithSvg(baseIconTemplateImage, "macos", spinnerFile);
+    const checkingForUpdatesImage = await generateImageWithSvg(baseIconImage, "windows-or-linux", spinnerFile);
+
+    await Promise.all([
+      // Templates are for macOS only
+      generateImages(baseIconTemplateImage, size, "trayIconTemplate"),
+      generateImages(updateAvailableTemplateImage, size, "trayIconUpdateAvailableTemplate"),
+      generateImages(updateAvailableTemplateImage, size, "trayIconUpdateAvailableTemplate"),
+      generateImages(checkingForUpdatesTemplateImage, size, "trayIconCheckingForUpdatesTemplate"),
+
+      // Non-templates are for windows and linux
+      generateImages(baseIconImage, size, "trayIcon"),
+      generateImages(updateAvailableImage, size, "trayIconUpdateAvailable"),
+      generateImages(checkingForUpdatesImage, size, "trayIconCheckingForUpdates"),
+    ]);
+
+    console.log("Generated all images");
+  } catch (error) {
+    console.error(error);
+  }
+}
+
+generateTrayIcons();

packages/core/build/installer.nsh

@@ -0,0 +1,15 @@
+!macro customInit
+  ; Make sure all old extensions are removed
+  RMDir /r "$INSTDIR\resources\extensions"
+
+  ; Workaround for installer handing when the app directory is removed manually
+  ${ifNot} ${FileExists} "$INSTDIR"
+    DeleteRegKey HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\{${UNINSTALL_APP_KEY}}"
+  ${EndIf}
+
+  ; Workaround for the old-format uninstall registry key (some people report it causes hangups, too)
+  ReadRegStr $0 HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\${UNINSTALL_APP_KEY}" "QuietUninstallString"
+  StrCmp $0 "" proceed 0
+  DeleteRegKey HKCU "Software\Microsoft\Windows\CurrentVersion\Uninstall\${UNINSTALL_APP_KEY}"
+  proceed:
+!macroend

packages/core/build/notarize.js

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) OpenLens Authors. All rights reserved.
+ * Licensed under MIT License. See LICENSE in root directory for more information.
+ */
+const { notarize } = require("electron-notarize");
+
+exports.default = async function notarizing(context) {
+  const { electronPlatformName, appOutDir } = context;
+
+  if (electronPlatformName !== "darwin") {
+    return;
+  }
+
+  if (!process.env.APPLEID || !process.env.APPLEIDPASS) {
+    return;
+  }
+
+  const appName = context.packager.appInfo.productFilename;
+
+  return await notarize({
+    appBundleId: process.env.APPBUNDLEID || "io.kontena.lens-app",
+    appPath: `${appOutDir}/${appName}.app`,
+    appleId: process.env.APPLEID,
+    appleIdPassword: process.env.APPLEIDPASS,
+    ascProvider:process.env.ASCPROVIDER,
+  });
+};

packages/core/build/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "../tsconfig.json",
+  "include": [
+    "./**/*",
+  ]
+}