ci: parallelize test suites and shard Web E2E

.github/workflows/ci.yml

@@ -12,9 +12,13 @@ concurrency:
 permissions:
   contents: read
 
+# Keep download caches outside the checkout and still install from the lockfile.
+env:
+  CI_APT_CACHE_DIR: /tmp/openpi-apt-download-cache
+
 jobs:
-  test:
-    name: Node ${{ matrix.node-version }}
+  test-node:
+    name: Runtime Node ${{ matrix.node-version }}
     runs-on: ubuntu-latest
     timeout-minutes: 20
     strategy:
@@ -30,6 +34,8 @@ jobs:
         with:
           bun-version: 1.3.14
       - name: Install dependencies
+        env:
+          BUN_INSTALL_CACHE_DIR: ${{ runner.temp }}/bun-install-cache
         run: bun install --frozen-lockfile
       - run: bun run check
       - name: Verify committed Web build
@@ -40,7 +46,7 @@ jobs:
             echo "web/dist is missing or stale; run bun run build:web and commit the result"
             exit 1
           fi
-      - run: bun run test
+      - run: bun run test node
       - name: Smoke-test Pi package discovery
         run: |
           agent_dir="$(mktemp -d)"
@@ -175,9 +181,13 @@ jobs:
           wait "$server_pid"
           server_pid=""
 
-  web-e2e:
-    name: Web E2E (Node 22.19.0)
-    runs-on: ubuntu-latest
+  web-e2e-shards:
+    name: Web E2E shard ${{ matrix.shard }}/2
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2]
+    runs-on: ubuntu-24.04
     timeout-minutes: 20
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -188,32 +198,69 @@ jobs:
         with:
           bun-version: 1.3.14
       - name: Install dependencies
+        env:
+          BUN_INSTALL_CACHE_DIR: ${{ runner.temp }}/bun-install-cache
         run: bun install --frozen-lockfile
+      - name: Cache Playwright browsers
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('bun.lock') }}
+      - name: Prepare system dependency download cache
+        run: |
+          mkdir -p "$CI_APT_CACHE_DIR"
+          sudo tee /etc/apt/apt.conf.d/99-openpi-ci-cache >/dev/null <<EOF
+          Dir::Cache::archives "$CI_APT_CACHE_DIR";
+          APT::Keep-Downloaded-Packages "true";
+          Binary::apt-get::APT::Keep-Downloaded-Packages "true";
+          Acquire::Retries "2";
+          Acquire::http::Timeout "30";
+          Acquire::https::Timeout "30";
+          EOF
+      - name: Cache Chromium system package downloads
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ${{ env.CI_APT_CACHE_DIR }}/*.deb
+          key: apt-${{ runner.os }}-${{ runner.arch }}-ubuntu24-${{ hashFiles('bun.lock') }}
+          restore-keys: apt-${{ runner.os }}-${{ runner.arch }}-ubuntu24-
       - name: Install Chromium and D-Bus
+        timeout-minutes: 10
         run: |
           bunx playwright install --with-deps chromium
           sudo apt-get install --no-install-recommends -y dbus-daemon
-      - run: dbus-run-session -- bun run test:web:e2e
+      # Independent runners isolate each shard's server, settings and filesystem.
+      - run: dbus-run-session -- bun run test:web:e2e --shard=${{ matrix.shard }}/2
         env:
           # The runner's installed Chrome has Ubuntu's native sandbox profile.
           OPENPI_CHROME_PATH: /opt/google/chrome/chrome
       - name: Test Plan Ready provider handoff
+        if: matrix.shard == 1
         run: bunx playwright test --config tests/web/playwright.plan-provider.config.ts
       - name: Upload Playwright failure artifacts
         if: failure()
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
-          name: openpi-web-playwright-results
+          name: openpi-web-playwright-results-${{ matrix.shard }}
           path: |
             /tmp/openpi-web-playwright-results
             /tmp/openpi-web-plan-provider-playwright-results
           if-no-files-found: ignore
           retention-days: 7
 
-  windows-background-terminals:
-    name: Background terminals (Windows)
+  windows-node:
+    name: Runtime Node (Windows ${{ matrix.group }})
     runs-on: windows-latest
     timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - group: parallel 1/2
+            arguments: node-parallel --shard=1/2
+          - group: parallel 2/2
+            arguments: node-parallel --shard=2/2
+          - group: isolated processes
+            arguments: node-serial
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
       - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
@@ -222,18 +269,97 @@ jobs:
       - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
         with:
           bun-version: 1.3.14
+      - name: Cache Bun downloads
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ${{ runner.temp }}/bun-install-cache
+          key: bun-${{ runner.os }}-${{ runner.arch }}-1.3.14-${{ hashFiles('bun.lock') }}
+          restore-keys: bun-${{ runner.os }}-${{ runner.arch }}-1.3.14-
       - name: Install dependencies
+        env:
+          BUN_INSTALL_CACHE_DIR: ${{ runner.temp }}/bun-install-cache
         run: bun install --frozen-lockfile
-      - name: Test Windows process lifecycle
-        run: |
-          $testPath = if (Test-Path "tests/extensions/background-terminals/manager.test.ts") {
-            "tests/extensions/background-terminals/manager.test.ts"
-          } else {
-            "extensions/background-terminals/manager.test.ts"
-          }
-          node --test --experimental-strip-types $testPath
-      - name: Test Windows full-suite isolation
-        # Allow the parallel Node suite, serial process tests, and Vitest to finish.
-        # Windows process identity checks make config-write coverage slower here.
+      - name: Test Windows Node suite with process isolation
+        # Process tests stay serial inside their own VM; ordinary files are
+        # split into disjoint shards on separate VMs, with every shard required.
         timeout-minutes: 10
-        run: bun run test
+        run: bun run test ${{ matrix.arguments }}
+
+  test-ui:
+    name: UI ${{ matrix.os }} Node ${{ matrix.node-version }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        node-version: [22.19.0, 24, 26]
+        include:
+          - os: windows-latest
+            node-version: 22.19.0
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: ${{ matrix.node-version }}
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.3.14
+      - name: Cache Bun downloads
+        if: runner.os == 'Windows'
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ${{ runner.temp }}/bun-install-cache
+          key: bun-${{ runner.os }}-${{ runner.arch }}-1.3.14-${{ hashFiles('bun.lock') }}
+          restore-keys: bun-${{ runner.os }}-${{ runner.arch }}-1.3.14-
+      - name: Install dependencies
+        env:
+          BUN_INSTALL_CACHE_DIR: ${{ runner.temp }}/bun-install-cache
+        run: bun install --frozen-lockfile
+      - run: bun run test ui
+
+  # Keep existing check names for branch protection. always() makes skipped,
+  # cancelled and failed dependencies fail the gates rather than bypass them.
+  test:
+    name: Node ${{ matrix.node-version }}
+    if: always()
+    needs: [test-node, test-ui, windows-node, web-e2e-shards]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [22.19.0, 24, 26]
+    steps:
+      - name: Require all test jobs
+        env:
+          NODE_RESULT: ${{ needs.test-node.result }}
+          UI_RESULT: ${{ needs.test-ui.result }}
+          WINDOWS_RESULT: ${{ needs.windows-node.result }}
+          E2E_RESULT: ${{ needs.web-e2e-shards.result }}
+        run: |
+          test "$NODE_RESULT" = success
+          test "$UI_RESULT" = success
+          test "$WINDOWS_RESULT" = success
+          test "$E2E_RESULT" = success
+
+  windows-background-terminals:
+    name: Background terminals (Windows)
+    if: always()
+    needs: [windows-node, test-ui]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Require Windows runtime and UI tests
+        env:
+          NODE_RESULT: ${{ needs.windows-node.result }}
+          UI_RESULT: ${{ needs.test-ui.result }}
+        run: test "$NODE_RESULT" = success && test "$UI_RESULT" = success
+
+  web-e2e:
+    name: Web E2E (Node 22.19.0)
+    if: always()
+    needs: [web-e2e-shards]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Require every E2E shard and provider handoff
+        env:
+          E2E_RESULT: ${{ needs.web-e2e-shards.result }}
+        run: test "$E2E_RESULT" = success

package.json

@@ -140,6 +140,8 @@
     "test:web:e2e:provider": "bun run build:web && playwright test --config tests/web/playwright.provider.config.ts",
     "benchmark:workflow-child-startup": "node --experimental-strip-types scripts/benchmark-workflow-child-startup.mjs",
     "test": "node scripts/run-tests.mjs",
+    "test:node": "node scripts/run-tests.mjs node",
+    "test:ui": "node scripts/run-tests.mjs ui",
     "provenance": "node scripts/provenance.mjs",
     "check:docs-contract": "node scripts/check-docs-contract.mjs"
   },

scripts/discover-tests.d.mts

@@ -0,0 +1 @@
+export function discoverTestFiles(root?: string): string[];

scripts/node-test-groups.d.mts

@@ -2,3 +2,5 @@ export function partitionNodeTestsByPlatform(
   files: string[],
   platform?: NodeJS.Platform,
 ): { parallel: string[]; serial: string[] };
+
+export function selectNodeTestShard(files: string[], shard: string): string[];

scripts/node-test-groups.mjs

@@ -31,3 +31,20 @@ export function partitionNodeTestsByPlatform(
   }
   return { parallel, serial };
 }
+
+export function selectNodeTestShard(files, shard) {
+  const match = /^([1-9]\d*)\/([1-9]\d*)$/.exec(shard);
+  const index = Number(match?.[1]);
+  const total = Number(match?.[2]);
+  if (
+    !Number.isSafeInteger(index) ||
+    !Number.isSafeInteger(total) ||
+    index > total ||
+    total > files.length
+  ) {
+    throw new Error(
+      "Invalid Node test shard: expected index/total with no empty shards.",
+    );
+  }
+  return files.filter((_, fileIndex) => fileIndex % total === index - 1);
+}

scripts/run-tests.mjs

@@ -1,7 +1,23 @@
 import { spawnSync } from "node:child_process";
 import { resolve } from "node:path";
 import { discoverTestFiles } from "./discover-tests.mjs";
-import { partitionNodeTestsByPlatform } from "./node-test-groups.mjs";
+import {
+  partitionNodeTestsByPlatform,
+  selectNodeTestShard,
+} from "./node-test-groups.mjs";
+
+const suite = process.argv[2] ?? "all";
+const shardArgument = process.argv[3];
+if (
+  process.argv.length > 4 ||
+  !["all", "node", "ui", "node-parallel", "node-serial"].includes(suite) ||
+  (shardArgument !== undefined &&
+    (suite !== "node-parallel" || !shardArgument.startsWith("--shard=")))
+) {
+  throw new Error(
+    "Usage: node scripts/run-tests.mjs [all|node|ui|node-parallel [--shard=index/total]|node-serial]",
+  );
+}
 
 const files = discoverTestFiles(resolve("tests"));
 const nodeTests = files.filter((file) => file.endsWith(".test.ts"));
@@ -23,21 +39,47 @@ function runNodeTests(files, options = []) {
   return result.status ?? 1;
 }
 
-const nodeTestGroups = partitionNodeTestsByPlatform(nodeTests);
-const parallelNodeResult = runNodeTests(nodeTestGroups.parallel);
-if (parallelNodeResult !== 0) {
-  process.exit(parallelNodeResult);
-}
+if (suite !== "ui") {
+  // Explicit groups use the Windows isolation boundary on every platform so
+  // their coverage can also be verified locally. Each CI group owns a VM.
+  const grouped = suite === "node-parallel" || suite === "node-serial";
+  const nodeTestGroups = partitionNodeTestsByPlatform(
+    nodeTests,
+    grouped ? "win32" : process.platform,
+  );
+  if (
+    grouped &&
+    (!nodeTestGroups.parallel.length || !nodeTestGroups.serial.length)
+  ) {
+    throw new Error("Both isolated Node test groups must be non-empty.");
+  }
+  if (suite === "node-serial") nodeTestGroups.parallel = [];
+  if (suite === "node-parallel") {
+    nodeTestGroups.serial = [];
+    if (shardArgument !== undefined) {
+      nodeTestGroups.parallel = selectNodeTestShard(
+        nodeTestGroups.parallel,
+        shardArgument.slice("--shard=".length),
+      );
+    }
+  }
+  const parallelNodeResult = runNodeTests(nodeTestGroups.parallel);
+  if (parallelNodeResult !== 0) {
+    process.exit(parallelNodeResult);
+  }
 
-// Windows process-tree tests must not overlap unrelated Node test files.
-// Keep the rest of the suite on Node's default file-level concurrency.
-const serialNodeResult = runNodeTests(nodeTestGroups.serial, [
-  "--test-concurrency=1",
-]);
-if (serialNodeResult !== 0) {
-  process.exit(serialNodeResult);
+  // Windows process-tree tests must not overlap unrelated Node test files.
+  // Keep the rest of the suite on Node's default file-level concurrency.
+  const serialNodeResult = runNodeTests(nodeTestGroups.serial, [
+    "--test-concurrency=1",
+  ]);
+  if (serialNodeResult !== 0) {
+    process.exit(serialNodeResult);
+  }
 }
 
+if (suite.startsWith("node")) process.exit(0);
+
 // Invoke the CLI module through Node instead of the package-manager shim.
 // Windows installs expose the shim as `vitest.cmd`, which cannot be launched
 // reliably by spawnSync without a shell.

tests/scripts/node-test-groups.test.ts

@@ -1,7 +1,11 @@
 import assert from "node:assert/strict";
 import { resolve } from "node:path";
 import test from "node:test";
-import { partitionNodeTestsByPlatform } from "../../scripts/node-test-groups.mjs";
+import {
+  partitionNodeTestsByPlatform,
+  selectNodeTestShard,
+} from "../../scripts/node-test-groups.mjs";
+import { discoverTestFiles } from "../../scripts/discover-tests.mjs";
 
 const backgroundTest = resolve(
   "tests",
@@ -42,3 +46,17 @@ test("non-Windows keeps all Node files in the parallel group", () => {
     },
   );
 });
+
+test("CI's Windows shards are disjoint and cover all discovered Node files", () => {
+  const files = discoverTestFiles().filter((file) => file.endsWith(".test.ts"));
+  const { parallel, serial } = partitionNodeTestsByPlatform(files, "win32");
+  const groups = [
+    selectNodeTestShard(parallel, "1/2"),
+    selectNodeTestShard(parallel, "2/2"),
+    serial,
+  ];
+  assert.ok(groups.every((group) => group.length > 0));
+  const selected = groups.flat();
+  assert.equal(new Set(selected).size, selected.length);
+  assert.deepEqual([...selected].sort(), [...files].sort());
+});