Split integration tests into seperate jobs from unit tests for faster CI (#6310)

* Split integration tests into seperate jobs from unit tests for faster CI

Signed-off-by: Sebastian Malton <sebastian@malton.name>

* Add logging

Signed-off-by: Sebastian Malton <sebastian@malton.name>

* Simplify the matrix

Signed-off-by: Sebastian Malton <sebastian@malton.name>

* Remove steps that are part of Makefile

Signed-off-by: Sebastian Malton <sebastian@malton.name>

* Fix yml decl

Signed-off-by: Sebastian Malton <sebastian@malton.name>

* Switch to using single quotes

Signed-off-by: Sebastian Malton <sebastian@malton.name>

* Further clarify the test job names

Signed-off-by: Sebastian Malton <sebastian@malton.name>

* Fix invocation

Signed-off-by: Sebastian Malton <sebastian@malton.name>

* Attempt to fix traking stdout

Signed-off-by: Sebastian Malton <sebastian@malton.name>

* Fix lint

Signed-off-by: Sebastian Malton <sebastian@malton.name>

* And handling for tests failing to start

Signed-off-by: Sebastian Malton <sebastian@malton.name>

* Add check for app early exiting

Signed-off-by: Sebastian Malton <sebastian@malton.name>

* Add more logging to help with debugging

Signed-off-by: Sebastian Malton <sebastian@malton.name>

* Cleanup attemptStart code

Signed-off-by: Sebastian Malton <sebastian@malton.name>

Signed-off-by: Sebastian Malton <sebastian@malton.name>

.github/workflows/test.yml

@@ -7,13 +7,14 @@ on:
     branches:
       - master
 jobs:
-  build:
-    name: Test
+  test:
+    name: ${{ matrix.type }} tests on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-20.04, macos-11, windows-2019]
+        type: [unit, smoke]
         node-version: [16.x]
     steps:
       - name: Checkout Release from lens
@@ -51,25 +52,16 @@ jobs:
           retry_on: error
           command: make node_modules
 
-      - run: make build-npm
-        name: Generate npm package
-
-      - uses: nick-fields/retry@v2
-        name: Build bundled extensions
-        with:
-          timeout_minutes: 15
-          max_attempts: 3
-          retry_on: error
-          command: make -j2 build-extensions
-
       - run: make test
         name: Run tests
+        if: ${{ matrix.type == 'unit' }}
 
       - run: make test-extensions
         name: Run In-tree Extension tests
+        if: ${{ matrix.type == 'unit' }}
 
       - run: make ci-validate-dev
-        if: contains(github.event.pull_request.labels.*.name, 'dependencies')
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'dependencies') && matrix.type == 'unit' }}
         name: Validate dev mode will work
 
       - name: Install integration test dependencies
@@ -77,7 +69,7 @@ jobs:
         uses: medyagh/setup-minikube@master
         with:
           minikube-version: latest
-        if: runner.os == 'Linux'
+        if: ${{ runner.os == 'Linux' && matrix.type == 'smoke' }}
 
       - run: xvfb-run --auto-servernum --server-args='-screen 0, 1600x900x24' make integration
         name: Run Linux integration tests
@@ -88,11 +80,11 @@ jobs:
         shell: bash
         env:
           ELECTRON_BUILDER_EXTRA_ARGS: "--x64 --arm64"
-        if: runner.os == 'macOS'
+        if: ${{ runner.os == 'macOS' && matrix.type == 'smoke' }}
 
       - run: make integration
         name: Run Windows integration tests
         shell: bash
         env:
           ELECTRON_BUILDER_EXTRA_ARGS: "--x64 --ia32"
-        if: runner.os == 'Windows'
+        if: ${{ runner.os == 'Windows' && matrix.type == 'smoke' }}

integration/__tests__/app-preferences.tests.ts

@@ -13,7 +13,8 @@ import type { ElectronApplication, Page } from "playwright";
 import * as utils from "../helpers/utils";
 
 describe("preferences page tests", () => {
-  let window: Page, cleanup: () => Promise<void>;
+  let window: Page;
+  let cleanup: undefined | (() => Promise<void>);
 
   beforeEach(async () => {
     let app: ElectronApplication;
@@ -31,7 +32,7 @@ describe("preferences page tests", () => {
   }, 10*60*1000);
 
   afterEach(async () => {
-    await cleanup();
+    await cleanup?.();
   }, 10*60*1000);
 
   it('shows "preferences" and can navigate through the tabs', async () => {

integration/__tests__/cluster-pages.tests.ts

@@ -19,7 +19,9 @@ import { describeIf } from "../../src/test-utils/skippers";
 const TEST_NAMESPACE = "integration-tests";
 
 describeIf(minikubeReady(TEST_NAMESPACE))("Minikube based tests", () => {
-  let window: Page, cleanup: () => Promise<void>, frame: Frame;
+  let window: Page;
+  let cleanup: undefined | (() => Promise<void>);
+  let frame: Frame;
 
   beforeEach(async () => {
     ({ window, cleanup } = await utils.start());
@@ -29,7 +31,7 @@ describeIf(minikubeReady(TEST_NAMESPACE))("Minikube based tests", () => {
   }, 10 * 60 * 1000);
 
   afterEach(async () => {
-    await cleanup();
+    await cleanup?.();
   }, 10 * 60 * 1000);
 
   it("shows cluster context menu in sidebar", async () => {

integration/__tests__/command-palette.tests.ts

@@ -7,15 +7,17 @@ import type { ElectronApplication, Page } from "playwright";
 import * as utils from "../helpers/utils";
 
 describe("Lens command palette", () => {
-  let window: Page, cleanup: () => Promise<void>, app: ElectronApplication;
-
+  let window: Page;
+  let cleanup: undefined | (() => Promise<void>);
+  let app: ElectronApplication;
+  
   beforeEach(async () => {
     ({ window, cleanup, app } = await utils.start());
     await utils.clickWelcomeButton(window);
   }, 10*60*1000);
 
   afterEach(async () => {
-    await cleanup();
+    await cleanup?.();
   }, 10*60*1000);
 
   describe("menu", () => {

integration/helpers/utils.ts

@@ -10,6 +10,7 @@ import * as uuid from "uuid";
 import type { ElectronApplication, Frame, Page } from "playwright";
 import { _electron as electron } from "playwright";
 import { noop } from "lodash";
+import { disposer } from "../../src/common/utils";
 
 export const appPaths: Partial<Record<NodeJS.Platform, string>> = {
   "win32": "./dist/win-unpacked/OpenLens.exe",
@@ -18,19 +19,46 @@ export const appPaths: Partial<Record<NodeJS.Platform, string>> = {
 };
 
 async function getMainWindow(app: ElectronApplication, timeout = 50_000): Promise<Page> {
-  const deadline = Date.now() + timeout;
+  return new Promise((resolve, reject) => {
+    const cleanup = disposer();
+    let stdoutBuf = "";
+
+    const onWindow = (page: Page) => {
+      console.log(`Page opened: ${page.url()}`);
 
-  for (; Date.now() < deadline;) {
-    for (const page of app.windows()) {
       if (page.url().startsWith("http://localhost")) {
-        return page;
+        cleanup();
+        console.log(stdoutBuf);
+        resolve(page);
       }
-    }
+    };
 
-    await new Promise(resolve => setTimeout(resolve, 2_000));
-  }
+    app.on("window", onWindow);
+    cleanup.push(() => app.off("window", onWindow));
 
-  throw new Error(`Lens did not open the main window within ${timeout}ms`);
+    const onClose = () => {
+      cleanup();
+      reject(new Error("App has unnexpectedly closed"));
+    };
+
+    app.on("close", onClose);
+    cleanup.push(() => app.off("close", onClose));
+
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const stdout = app.process().stdout!;
+    const onData = (chunk: any) => stdoutBuf += chunk.toString();
+
+    stdout.on("data", onData);
+    cleanup.push(() => stdout.off("data", onData));
+
+    const timeoutId = setTimeout(() => {
+      cleanup();
+      console.log(stdoutBuf);
+      reject(new Error(`Lens did not open the main window within ${timeout}ms`));
+    }, timeout);
+
+    cleanup.push(() => clearTimeout(timeoutId));
+  });
 }
 
 async function attemptStart() {
@@ -49,7 +77,7 @@ async function attemptStart() {
       ...process.env,
     },
     timeout: 100_000,
-  } as Parameters<typeof electron["launch"]>[0]);
+  });
 
   try {
     const window = await getMainWindow(app);
@@ -70,6 +98,8 @@ async function attemptStart() {
 }
 
 export async function start() {
+  console.log(process.env);
+
   // this is an attempted workaround for an issue with playwright not always getting the main window when using Electron 14.2.4 (observed on windows)
   for (let i = 0; ; i++) {
     try {