From 6ccf3e9d32058367fdbc92cc22abcf26a4174e55 Mon Sep 17 00:00:00 2001 From: tt-a1i Date: Fri, 2 Oct 2026 00:09:03 +0800 Subject: [PATCH 1/4] ci: parallelize suites and shard browser acceptance --- .github/workflows/ci.yml | 131 +++++++++++++++++++++++++++----- package.json | 2 + scripts/run-tests.mjs | 33 +++++--- tests/scripts/run-tests.test.ts | 119 +++++++++++++++++++++++++++++ tests/web/openpi-web.e2e.ts | 2 + tests/web/playwright.config.ts | 4 +- 6 files changed, 257 insertions(+), 34 deletions(-) create mode 100644 tests/scripts/run-tests.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fd63528c..1d0b14637 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,13 @@ concurrency: permissions: contents: read +# Keep download caches outside the checkout and still install from the lockfile. +env: + BUN_INSTALL_CACHE_DIR: ${{ github.workspace }}/../bun-install-cache + jobs: - test: - name: Node ${{ matrix.node-version }} + test-node: + name: Runtime Node ${{ matrix.node-version }} runs-on: ubuntu-latest timeout-minutes: 20 strategy: @@ -40,7 +44,7 @@ jobs: echo "web/dist is missing or stale; run bun run build:web and commit the result" exit 1 fi - - run: bun run test + - run: bun run test node - name: Smoke-test Pi package discovery run: | agent_dir="$(mktemp -d)" @@ -175,8 +179,12 @@ jobs: wait "$server_pid" server_pid="" - web-e2e: - name: Web E2E (Node 22.19.0) + web-e2e-shards: + name: Web E2E shard ${{ matrix.shard }}/2 + strategy: + fail-fast: false + matrix: + shard: [1, 2] runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -189,29 +197,36 @@ jobs: bun-version: 1.3.14 - name: Install dependencies run: bun install --frozen-lockfile + - name: Cache Playwright browsers + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ~/.cache/ms-playwright + key: playwright-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('bun.lock') }} - name: Install Chromium and D-Bus run: | bunx playwright install --with-deps chromium sudo apt-get install --no-install-recommends -y dbus-daemon - - run: dbus-run-session -- bun run test:web:e2e + # Independent runners isolate each shard's server, settings and filesystem. + - run: dbus-run-session -- bun run test:web:e2e --shard=${{ matrix.shard }}/2 env: # The runner's installed Chrome has Ubuntu's native sandbox profile. OPENPI_CHROME_PATH: /opt/google/chrome/chrome - name: Test Plan Ready provider handoff + if: matrix.shard == 1 run: bunx playwright test --config tests/web/playwright.plan-provider.config.ts - name: Upload Playwright failure artifacts if: failure() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: openpi-web-playwright-results + name: openpi-web-playwright-results-${{ matrix.shard }} path: | /tmp/openpi-web-playwright-results /tmp/openpi-web-plan-provider-playwright-results if-no-files-found: ignore retention-days: 7 - windows-background-terminals: - name: Background terminals (Windows) + windows-node: + name: Runtime Node (Windows) runs-on: windows-latest timeout-minutes: 15 steps: @@ -222,18 +237,92 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: bun-version: 1.3.14 + - name: Cache Bun downloads + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ${{ env.BUN_INSTALL_CACHE_DIR }} + key: bun-${{ runner.os }}-${{ runner.arch }}-1.3.14-${{ hashFiles('bun.lock') }} + restore-keys: bun-${{ runner.os }}-${{ runner.arch }}-1.3.14- - name: Install dependencies run: bun install --frozen-lockfile - - name: Test Windows process lifecycle - run: | - $testPath = if (Test-Path "tests/extensions/background-terminals/manager.test.ts") { - "tests/extensions/background-terminals/manager.test.ts" - } else { - "extensions/background-terminals/manager.test.ts" - } - node --test --experimental-strip-types $testPath - - name: Test Windows full-suite isolation - # Allow the parallel Node suite, serial process tests, and Vitest to finish. - # Windows process identity checks make config-write coverage slower here. + - name: Test Windows Node suite with process isolation + # run-tests.mjs runs background-terminal files after other Node files. timeout-minutes: 10 - run: bun run test + run: bun run test node + + test-ui: + name: UI ${{ matrix.os }} Node ${{ matrix.node-version }} + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + node-version: [22.19.0, 24, 26] + include: + - os: windows-latest + node-version: 22.19.0 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: ${{ matrix.node-version }} + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - name: Cache Bun downloads + if: runner.os == 'Windows' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ${{ env.BUN_INSTALL_CACHE_DIR }} + key: bun-${{ runner.os }}-${{ runner.arch }}-1.3.14-${{ hashFiles('bun.lock') }} + restore-keys: bun-${{ runner.os }}-${{ runner.arch }}-1.3.14- + - name: Install dependencies + run: bun install --frozen-lockfile + - run: bun run test ui + + # Keep existing check names for branch protection. always() makes skipped, + # cancelled and failed dependencies fail the gates rather than bypass them. + test: + name: Node ${{ matrix.node-version }} + if: always() + needs: [test-node, test-ui, windows-node, web-e2e-shards] + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [22.19.0, 24, 26] + steps: + - name: Require all test jobs + env: + NODE_RESULT: ${{ needs.test-node.result }} + UI_RESULT: ${{ needs.test-ui.result }} + WINDOWS_RESULT: ${{ needs.windows-node.result }} + E2E_RESULT: ${{ needs.web-e2e-shards.result }} + run: | + test "$NODE_RESULT" = success + test "$UI_RESULT" = success + test "$WINDOWS_RESULT" = success + test "$E2E_RESULT" = success + + windows-background-terminals: + name: Background terminals (Windows) + if: always() + needs: [windows-node, test-ui] + runs-on: ubuntu-latest + steps: + - name: Require Windows runtime and UI tests + env: + NODE_RESULT: ${{ needs.windows-node.result }} + UI_RESULT: ${{ needs.test-ui.result }} + run: test "$NODE_RESULT" = success && test "$UI_RESULT" = success + + web-e2e: + name: Web E2E (Node 22.19.0) + if: always() + needs: [web-e2e-shards] + runs-on: ubuntu-latest + steps: + - name: Require every E2E shard and provider handoff + env: + E2E_RESULT: ${{ needs.web-e2e-shards.result }} + run: test "$E2E_RESULT" = success diff --git a/package.json b/package.json index 82ecf090f..eaacfe668 100644 --- a/package.json +++ b/package.json @@ -140,6 +140,8 @@ "test:web:e2e:provider": "bun run build:web && playwright test --config tests/web/playwright.provider.config.ts", "benchmark:workflow-child-startup": "node --experimental-strip-types scripts/benchmark-workflow-child-startup.mjs", "test": "node scripts/run-tests.mjs", + "test:node": "node scripts/run-tests.mjs node", + "test:ui": "node scripts/run-tests.mjs ui", "provenance": "node scripts/provenance.mjs", "check:docs-contract": "node scripts/check-docs-contract.mjs" }, diff --git a/scripts/run-tests.mjs b/scripts/run-tests.mjs index 9e0f86314..ca6c5221a 100644 --- a/scripts/run-tests.mjs +++ b/scripts/run-tests.mjs @@ -3,6 +3,11 @@ import { resolve } from "node:path"; import { discoverTestFiles } from "./discover-tests.mjs"; import { partitionNodeTestsByPlatform } from "./node-test-groups.mjs"; +const suite = process.argv[2] ?? "all"; +if (process.argv.length > 3 || !["all", "node", "ui"].includes(suite)) { + throw new Error("Usage: node scripts/run-tests.mjs [all|node|ui]"); +} + const files = discoverTestFiles(resolve("tests")); const nodeTests = files.filter((file) => file.endsWith(".test.ts")); const vitestTests = files.filter((file) => file.endsWith(".spec.ts")); @@ -23,21 +28,25 @@ function runNodeTests(files, options = []) { return result.status ?? 1; } -const nodeTestGroups = partitionNodeTestsByPlatform(nodeTests); -const parallelNodeResult = runNodeTests(nodeTestGroups.parallel); -if (parallelNodeResult !== 0) { - process.exit(parallelNodeResult); -} +if (suite !== "ui") { + const nodeTestGroups = partitionNodeTestsByPlatform(nodeTests); + const parallelNodeResult = runNodeTests(nodeTestGroups.parallel); + if (parallelNodeResult !== 0) { + process.exit(parallelNodeResult); + } -// Windows process-tree tests must not overlap unrelated Node test files. -// Keep the rest of the suite on Node's default file-level concurrency. -const serialNodeResult = runNodeTests(nodeTestGroups.serial, [ - "--test-concurrency=1", -]); -if (serialNodeResult !== 0) { - process.exit(serialNodeResult); + // Windows process-tree tests must not overlap unrelated Node test files. + // Keep the rest of the suite on Node's default file-level concurrency. + const serialNodeResult = runNodeTests(nodeTestGroups.serial, [ + "--test-concurrency=1", + ]); + if (serialNodeResult !== 0) { + process.exit(serialNodeResult); + } } +if (suite === "node") process.exit(0); + // Invoke the CLI module through Node instead of the package-manager shim. // Windows installs expose the shim as `vitest.cmd`, which cannot be launched // reliably by spawnSync without a shell. diff --git a/tests/scripts/run-tests.test.ts b/tests/scripts/run-tests.test.ts new file mode 100644 index 000000000..41f2da2b9 --- /dev/null +++ b/tests/scripts/run-tests.test.ts @@ -0,0 +1,119 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { + copyFileSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; + +function fixture() { + const root = mkdtempSync(join(tmpdir(), "openpi-test-runner-")); + for (const directory of ["scripts", "tests", "node_modules/vitest"]) { + mkdirSync(join(root, directory), { recursive: true }); + } + for (const script of [ + "run-tests.mjs", + "discover-tests.mjs", + "node-test-groups.mjs", + ]) { + copyFileSync(resolve("scripts", script), join(root, "scripts", script)); + } + writeFileSync( + join(root, "tests", "sample.test.ts"), + ` + import assert from "node:assert/strict"; + import test from "node:test"; + test("node suite marker", () => assert.notEqual(process.env.FAIL_SUITE, "node")); + `, + ); + writeFileSync( + join(root, "tests", "sample.spec.ts"), + "// UI discovery fixture\n", + ); + // A tiny CLI stand-in isolates runner selection/exit behavior from Vitest itself. + writeFileSync( + join(root, "node_modules/vitest/vitest.mjs"), + ` + import assert from "node:assert/strict"; + assert.equal(process.argv[2], "run"); + assert.ok(process.argv[3].endsWith("sample.spec.ts")); + console.log("ui suite marker"); + process.exit(process.env.FAIL_SUITE === "ui" ? 7 : 0); + `, + ); + return { + root, + run(args: string[] = [], failSuite = "") { + const env: NodeJS.ProcessEnv = { ...process.env, FAIL_SUITE: failSuite }; + // Start a fresh test runner rather than inheriting this test worker context. + delete env.NODE_TEST_CONTEXT; + return spawnSync(process.execPath, ["scripts/run-tests.mjs", ...args], { + cwd: root, + env, + encoding: "utf8", + }); + }, + dispose() { + rmSync(root, { recursive: true, force: true }); + }, + }; +} + +test("split suite commands together cover the unchanged default command", () => { + const runner = fixture(); + try { + for (const [args, nodeExpected, uiExpected] of [ + [[], true, true], + [["all"], true, true], + [["node"], true, false], + [["ui"], false, true], + ] as const) { + const result = runner.run([...args]); + assert.equal(result.status, 0, result.stderr || result.stdout); + assert.equal(result.stdout.includes("node suite marker"), nodeExpected); + assert.equal(result.stdout.includes("ui suite marker"), uiExpected); + } + } finally { + runner.dispose(); + } +}); + +test("test failures propagate and default execution stops after Node failure", () => { + const runner = fixture(); + try { + const node = runner.run(["node"], "node"); + assert.notEqual(node.status, 0); + assert.equal(node.stdout.includes("ui suite marker"), false); + assert.equal(runner.run(["ui"], "ui").status, 7); + const combined = runner.run([], "node"); + assert.notEqual(combined.status, 0); + assert.equal(combined.stdout.includes("ui suite marker"), false); + assert.equal(runner.run([], "ui").status, 7); + } finally { + runner.dispose(); + } +}); + +test("invalid selection and empty discovery fail before running a partial suite", () => { + const runner = fixture(); + try { + for (const args of [["unknown"], ["node", "extra"]]) { + const result = runner.run(args); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Usage:/); + assert.equal(result.stdout.includes("suite marker"), false); + } + rmSync(join(runner.root, "tests", "sample.spec.ts")); + const empty = runner.run(["node"]); + assert.notEqual(empty.status, 0); + assert.match(empty.stderr, /both suites must be non-empty/); + assert.equal(empty.stdout.includes("suite marker"), false); + } finally { + runner.dispose(); + } +}); diff --git a/tests/web/openpi-web.e2e.ts b/tests/web/openpi-web.e2e.ts index 670fe0a9f..163a22956 100644 --- a/tests/web/openpi-web.e2e.ts +++ b/tests/web/openpi-web.e2e.ts @@ -570,6 +570,8 @@ test("provider failures remain visible after refresh without discarding partial await expect(page.getByText("Partial answer")).toBeVisible(); await expect(page.getByText("模型请求已停止")).toBeVisible(); await expect(page.getByRole("alert")).toHaveCount(0); + // Drain snapshot handlers before the per-test request context is disposed. + await page.unrouteAll({ behavior: "wait" }); }); async function openWorkbench(page: Page) { diff --git a/tests/web/playwright.config.ts b/tests/web/playwright.config.ts index 3da0c9ef8..b90916cf5 100644 --- a/tests/web/playwright.config.ts +++ b/tests/web/playwright.config.ts @@ -41,7 +41,9 @@ export default defineConfig({ "tests/web/subagent-inspection.e2e.ts", ], outputDir: outputDirectory, - fullyParallel: false, + // Shard individual cases, including the large workbench file. Each runner + // still uses one worker so tests cannot mutate its shared server concurrently. + fullyParallel: true, workers: 1, timeout: 30_000, reporter: "list", From 05023c65963a3341c028cbd3a26216df3f195ca6 Mon Sep 17 00:00:00 2001 From: tt-a1i Date: Fri, 2 Oct 2026 00:24:27 +0800 Subject: [PATCH 2/4] ci: cache system packages and isolate model fixtures --- .github/workflows/ci.yml | 20 +++++++++++++++++++- tests/web/openpi-web.e2e.ts | 14 ++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d0b14637..652d2e115 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,7 @@ permissions: # Keep download caches outside the checkout and still install from the lockfile. env: BUN_INSTALL_CACHE_DIR: ${{ github.workspace }}/../bun-install-cache + CI_APT_CACHE_DIR: ${{ github.workspace }}/../apt-download-cache jobs: test-node: @@ -185,7 +186,7 @@ jobs: fail-fast: false matrix: shard: [1, 2] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 20 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 @@ -202,7 +203,24 @@ jobs: with: path: ~/.cache/ms-playwright key: playwright-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('bun.lock') }} + - name: Prepare system dependency download cache + run: | + mkdir -p "$CI_APT_CACHE_DIR" + sudo tee /etc/apt/apt.conf.d/99-openpi-ci-cache >/dev/null < Date: Fri, 2 Oct 2026 00:36:45 +0800 Subject: [PATCH 3/4] ci: use restorable cache paths outside the checkout --- .github/workflows/ci.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 652d2e115..9dbcf44aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,8 +14,7 @@ permissions: # Keep download caches outside the checkout and still install from the lockfile. env: - BUN_INSTALL_CACHE_DIR: ${{ github.workspace }}/../bun-install-cache - CI_APT_CACHE_DIR: ${{ github.workspace }}/../apt-download-cache + CI_APT_CACHE_DIR: /tmp/openpi-apt-download-cache jobs: test-node: @@ -35,6 +34,8 @@ jobs: with: bun-version: 1.3.14 - name: Install dependencies + env: + BUN_INSTALL_CACHE_DIR: ${{ runner.temp }}/bun-install-cache run: bun install --frozen-lockfile - run: bun run check - name: Verify committed Web build @@ -197,6 +198,8 @@ jobs: with: bun-version: 1.3.14 - name: Install dependencies + env: + BUN_INSTALL_CACHE_DIR: ${{ runner.temp }}/bun-install-cache run: bun install --frozen-lockfile - name: Cache Playwright browsers uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 @@ -219,8 +222,9 @@ jobs: with: path: ${{ env.CI_APT_CACHE_DIR }}/*.deb key: apt-${{ runner.os }}-${{ runner.arch }}-ubuntu24-${{ hashFiles('bun.lock') }} + restore-keys: apt-${{ runner.os }}-${{ runner.arch }}-ubuntu24- - name: Install Chromium and D-Bus - timeout-minutes: 6 + timeout-minutes: 10 run: | bunx playwright install --with-deps chromium sudo apt-get install --no-install-recommends -y dbus-daemon @@ -258,10 +262,12 @@ jobs: - name: Cache Bun downloads uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: - path: ${{ env.BUN_INSTALL_CACHE_DIR }} + path: ${{ runner.temp }}/bun-install-cache key: bun-${{ runner.os }}-${{ runner.arch }}-1.3.14-${{ hashFiles('bun.lock') }} restore-keys: bun-${{ runner.os }}-${{ runner.arch }}-1.3.14- - name: Install dependencies + env: + BUN_INSTALL_CACHE_DIR: ${{ runner.temp }}/bun-install-cache run: bun install --frozen-lockfile - name: Test Windows Node suite with process isolation # run-tests.mjs runs background-terminal files after other Node files. @@ -292,10 +298,12 @@ jobs: if: runner.os == 'Windows' uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: - path: ${{ env.BUN_INSTALL_CACHE_DIR }} + path: ${{ runner.temp }}/bun-install-cache key: bun-${{ runner.os }}-${{ runner.arch }}-1.3.14-${{ hashFiles('bun.lock') }} restore-keys: bun-${{ runner.os }}-${{ runner.arch }}-1.3.14- - name: Install dependencies + env: + BUN_INSTALL_CACHE_DIR: ${{ runner.temp }}/bun-install-cache run: bun install --frozen-lockfile - run: bun run test ui From 4a31a17b2e4611fa04b8b1547e9a3e3f637e7609 Mon Sep 17 00:00:00 2001 From: tt-a1i Date: Fri, 2 Oct 2026 11:49:21 +0800 Subject: [PATCH 4/4] ci: shard Windows Node tests across isolated runners --- .github/workflows/ci.yml | 17 ++++- scripts/discover-tests.d.mts | 1 + scripts/node-test-groups.d.mts | 2 + scripts/node-test-groups.mjs | 17 +++++ scripts/run-tests.mjs | 43 ++++++++++-- tests/scripts/node-test-groups.test.ts | 20 +++++- tests/scripts/run-tests.test.ts | 90 +++++++++++++++++++++++++- 7 files changed, 180 insertions(+), 10 deletions(-) create mode 100644 scripts/discover-tests.d.mts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9dbcf44aa..fe7a54e9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -248,9 +248,19 @@ jobs: retention-days: 7 windows-node: - name: Runtime Node (Windows) + name: Runtime Node (Windows ${{ matrix.group }}) runs-on: windows-latest timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - group: parallel 1/2 + arguments: node-parallel --shard=1/2 + - group: parallel 2/2 + arguments: node-parallel --shard=2/2 + - group: isolated processes + arguments: node-serial steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -270,9 +280,10 @@ jobs: BUN_INSTALL_CACHE_DIR: ${{ runner.temp }}/bun-install-cache run: bun install --frozen-lockfile - name: Test Windows Node suite with process isolation - # run-tests.mjs runs background-terminal files after other Node files. + # Process tests stay serial inside their own VM; ordinary files are + # split into disjoint shards on separate VMs, with every shard required. timeout-minutes: 10 - run: bun run test node + run: bun run test ${{ matrix.arguments }} test-ui: name: UI ${{ matrix.os }} Node ${{ matrix.node-version }} diff --git a/scripts/discover-tests.d.mts b/scripts/discover-tests.d.mts new file mode 100644 index 000000000..043193453 --- /dev/null +++ b/scripts/discover-tests.d.mts @@ -0,0 +1 @@ +export function discoverTestFiles(root?: string): string[]; diff --git a/scripts/node-test-groups.d.mts b/scripts/node-test-groups.d.mts index ed8bfd983..25c69b9c3 100644 --- a/scripts/node-test-groups.d.mts +++ b/scripts/node-test-groups.d.mts @@ -2,3 +2,5 @@ export function partitionNodeTestsByPlatform( files: string[], platform?: NodeJS.Platform, ): { parallel: string[]; serial: string[] }; + +export function selectNodeTestShard(files: string[], shard: string): string[]; diff --git a/scripts/node-test-groups.mjs b/scripts/node-test-groups.mjs index a45b071d6..f0c7d2241 100644 --- a/scripts/node-test-groups.mjs +++ b/scripts/node-test-groups.mjs @@ -31,3 +31,20 @@ export function partitionNodeTestsByPlatform( } return { parallel, serial }; } + +export function selectNodeTestShard(files, shard) { + const match = /^([1-9]\d*)\/([1-9]\d*)$/.exec(shard); + const index = Number(match?.[1]); + const total = Number(match?.[2]); + if ( + !Number.isSafeInteger(index) || + !Number.isSafeInteger(total) || + index > total || + total > files.length + ) { + throw new Error( + "Invalid Node test shard: expected index/total with no empty shards.", + ); + } + return files.filter((_, fileIndex) => fileIndex % total === index - 1); +} diff --git a/scripts/run-tests.mjs b/scripts/run-tests.mjs index ca6c5221a..dfec12fe2 100644 --- a/scripts/run-tests.mjs +++ b/scripts/run-tests.mjs @@ -1,11 +1,22 @@ import { spawnSync } from "node:child_process"; import { resolve } from "node:path"; import { discoverTestFiles } from "./discover-tests.mjs"; -import { partitionNodeTestsByPlatform } from "./node-test-groups.mjs"; +import { + partitionNodeTestsByPlatform, + selectNodeTestShard, +} from "./node-test-groups.mjs"; const suite = process.argv[2] ?? "all"; -if (process.argv.length > 3 || !["all", "node", "ui"].includes(suite)) { - throw new Error("Usage: node scripts/run-tests.mjs [all|node|ui]"); +const shardArgument = process.argv[3]; +if ( + process.argv.length > 4 || + !["all", "node", "ui", "node-parallel", "node-serial"].includes(suite) || + (shardArgument !== undefined && + (suite !== "node-parallel" || !shardArgument.startsWith("--shard="))) +) { + throw new Error( + "Usage: node scripts/run-tests.mjs [all|node|ui|node-parallel [--shard=index/total]|node-serial]", + ); } const files = discoverTestFiles(resolve("tests")); @@ -29,7 +40,29 @@ function runNodeTests(files, options = []) { } if (suite !== "ui") { - const nodeTestGroups = partitionNodeTestsByPlatform(nodeTests); + // Explicit groups use the Windows isolation boundary on every platform so + // their coverage can also be verified locally. Each CI group owns a VM. + const grouped = suite === "node-parallel" || suite === "node-serial"; + const nodeTestGroups = partitionNodeTestsByPlatform( + nodeTests, + grouped ? "win32" : process.platform, + ); + if ( + grouped && + (!nodeTestGroups.parallel.length || !nodeTestGroups.serial.length) + ) { + throw new Error("Both isolated Node test groups must be non-empty."); + } + if (suite === "node-serial") nodeTestGroups.parallel = []; + if (suite === "node-parallel") { + nodeTestGroups.serial = []; + if (shardArgument !== undefined) { + nodeTestGroups.parallel = selectNodeTestShard( + nodeTestGroups.parallel, + shardArgument.slice("--shard=".length), + ); + } + } const parallelNodeResult = runNodeTests(nodeTestGroups.parallel); if (parallelNodeResult !== 0) { process.exit(parallelNodeResult); @@ -45,7 +78,7 @@ if (suite !== "ui") { } } -if (suite === "node") process.exit(0); +if (suite.startsWith("node")) process.exit(0); // Invoke the CLI module through Node instead of the package-manager shim. // Windows installs expose the shim as `vitest.cmd`, which cannot be launched diff --git a/tests/scripts/node-test-groups.test.ts b/tests/scripts/node-test-groups.test.ts index 86c0912b5..ab2b1ca2d 100644 --- a/tests/scripts/node-test-groups.test.ts +++ b/tests/scripts/node-test-groups.test.ts @@ -1,7 +1,11 @@ import assert from "node:assert/strict"; import { resolve } from "node:path"; import test from "node:test"; -import { partitionNodeTestsByPlatform } from "../../scripts/node-test-groups.mjs"; +import { + partitionNodeTestsByPlatform, + selectNodeTestShard, +} from "../../scripts/node-test-groups.mjs"; +import { discoverTestFiles } from "../../scripts/discover-tests.mjs"; const backgroundTest = resolve( "tests", @@ -42,3 +46,17 @@ test("non-Windows keeps all Node files in the parallel group", () => { }, ); }); + +test("CI's Windows shards are disjoint and cover all discovered Node files", () => { + const files = discoverTestFiles().filter((file) => file.endsWith(".test.ts")); + const { parallel, serial } = partitionNodeTestsByPlatform(files, "win32"); + const groups = [ + selectNodeTestShard(parallel, "1/2"), + selectNodeTestShard(parallel, "2/2"), + serial, + ]; + assert.ok(groups.every((group) => group.length > 0)); + const selected = groups.flat(); + assert.equal(new Set(selected).size, selected.length); + assert.deepEqual([...selected].sort(), [...files].sort()); +}); diff --git a/tests/scripts/run-tests.test.ts b/tests/scripts/run-tests.test.ts index 41f2da2b9..881da9491 100644 --- a/tests/scripts/run-tests.test.ts +++ b/tests/scripts/run-tests.test.ts @@ -13,7 +13,11 @@ import test from "node:test"; function fixture() { const root = mkdtempSync(join(tmpdir(), "openpi-test-runner-")); - for (const directory of ["scripts", "tests", "node_modules/vitest"]) { + for (const directory of [ + "scripts", + "tests/extensions/background-terminals", + "node_modules/vitest", + ]) { mkdirSync(join(root, directory), { recursive: true }); } for (const script of [ @@ -31,6 +35,18 @@ function fixture() { test("node suite marker", () => assert.notEqual(process.env.FAIL_SUITE, "node")); `, ); + writeFileSync( + join(root, "tests", "second.test.ts"), + `import test from "node:test"; test("second node marker", () => {});`, + ); + writeFileSync( + join(root, "tests/extensions/background-terminals", "process.test.ts"), + ` + import assert from "node:assert/strict"; + import test from "node:test"; + test("isolated process marker", () => assert.notEqual(process.env.FAIL_SUITE, "serial")); + `, + ); writeFileSync( join(root, "tests", "sample.spec.ts"), "// UI discovery fixture\n", @@ -99,6 +115,78 @@ test("test failures propagate and default execution stops after Node failure", ( } }); +test("isolated shards cover every Node file exactly once and propagate failures", () => { + const runner = fixture(); + try { + const outputs = [ + runner.run(["node-parallel", "--shard=1/2"]), + runner.run(["node-parallel", "--shard=2/2"]), + runner.run(["node-serial"]), + ]; + for (const result of outputs) { + assert.equal(result.status, 0, result.stderr || result.stdout); + assert.equal(result.stdout.includes("ui suite marker"), false); + } + for (const marker of [ + "node suite marker", + "second node marker", + "isolated process marker", + ]) { + assert.equal( + outputs.filter((result) => result.stdout.includes(marker)).length, + 1, + ); + } + assert.equal(outputs[2].stdout.includes("isolated process marker"), true); + assert.notEqual( + runner.run(["node-parallel", "--shard=1/2"], "node").status, + 0, + ); + assert.notEqual(runner.run(["node-serial"], "serial").status, 0); + assert.notEqual(runner.run([], "serial").status, 0); + } finally { + runner.dispose(); + } +}); + +test("invalid and empty shards fail before executing any tests", () => { + const runner = fixture(); + try { + for (const shard of [ + "0/2", + "3/2", + "1/0", + "1/3", + "1/2junk", + "1/9007199254740992", + ]) { + const result = runner.run(["node-parallel", `--shard=${shard}`]); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Invalid Node test shard/); + assert.equal(result.stdout.includes("marker"), false); + } + for (const args of [ + ["node-serial", "--shard=1/2"], + ["node-parallel", "extra"], + ["node-parallel", "--shard=1/2", "extra"], + ]) { + assert.match(runner.run(args).stderr, /Usage:/); + } + rmSync(join(runner.root, "tests/extensions/background-terminals"), { + recursive: true, + }); + const result = runner.run(["node-parallel"]); + assert.notEqual(result.status, 0); + assert.match( + result.stderr, + /Both isolated Node test groups must be non-empty/, + ); + assert.equal(result.stdout.includes("marker"), false); + } finally { + runner.dispose(); + } +}); + test("invalid selection and empty discovery fail before running a partial suite", () => { const runner = fixture(); try {