diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fd63528c..fe7a54e9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,13 @@ concurrency: permissions: contents: read +# Keep download caches outside the checkout and still install from the lockfile. +env: + CI_APT_CACHE_DIR: /tmp/openpi-apt-download-cache + jobs: - test: - name: Node ${{ matrix.node-version }} + test-node: + name: Runtime Node ${{ matrix.node-version }} runs-on: ubuntu-latest timeout-minutes: 20 strategy: @@ -30,6 +34,8 @@ jobs: with: bun-version: 1.3.14 - name: Install dependencies + env: + BUN_INSTALL_CACHE_DIR: ${{ runner.temp }}/bun-install-cache run: bun install --frozen-lockfile - run: bun run check - name: Verify committed Web build @@ -40,7 +46,7 @@ jobs: echo "web/dist is missing or stale; run bun run build:web and commit the result" exit 1 fi - - run: bun run test + - run: bun run test node - name: Smoke-test Pi package discovery run: | agent_dir="$(mktemp -d)" @@ -175,9 +181,13 @@ jobs: wait "$server_pid" server_pid="" - web-e2e: - name: Web E2E (Node 22.19.0) - runs-on: ubuntu-latest + web-e2e-shards: + name: Web E2E shard ${{ matrix.shard }}/2 + strategy: + fail-fast: false + matrix: + shard: [1, 2] + runs-on: ubuntu-24.04 timeout-minutes: 20 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 @@ -188,32 +198,69 @@ jobs: with: bun-version: 1.3.14 - name: Install dependencies + env: + BUN_INSTALL_CACHE_DIR: ${{ runner.temp }}/bun-install-cache run: bun install --frozen-lockfile + - name: Cache Playwright browsers + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ~/.cache/ms-playwright + key: playwright-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('bun.lock') }} + - name: Prepare system dependency download cache + run: | + mkdir -p "$CI_APT_CACHE_DIR" + sudo tee /etc/apt/apt.conf.d/99-openpi-ci-cache >/dev/null < total || + total > files.length + ) { + throw new Error( + "Invalid Node test shard: expected index/total with no empty shards.", + ); + } + return files.filter((_, fileIndex) => fileIndex % total === index - 1); +} diff --git a/scripts/run-tests.mjs b/scripts/run-tests.mjs index 9e0f86314..dfec12fe2 100644 --- a/scripts/run-tests.mjs +++ b/scripts/run-tests.mjs @@ -1,7 +1,23 @@ import { spawnSync } from "node:child_process"; import { resolve } from "node:path"; import { discoverTestFiles } from "./discover-tests.mjs"; -import { partitionNodeTestsByPlatform } from "./node-test-groups.mjs"; +import { + partitionNodeTestsByPlatform, + selectNodeTestShard, +} from "./node-test-groups.mjs"; + +const suite = process.argv[2] ?? "all"; +const shardArgument = process.argv[3]; +if ( + process.argv.length > 4 || + !["all", "node", "ui", "node-parallel", "node-serial"].includes(suite) || + (shardArgument !== undefined && + (suite !== "node-parallel" || !shardArgument.startsWith("--shard="))) +) { + throw new Error( + "Usage: node scripts/run-tests.mjs [all|node|ui|node-parallel [--shard=index/total]|node-serial]", + ); +} const files = discoverTestFiles(resolve("tests")); const nodeTests = files.filter((file) => file.endsWith(".test.ts")); @@ -23,21 +39,47 @@ function runNodeTests(files, options = []) { return result.status ?? 1; } -const nodeTestGroups = partitionNodeTestsByPlatform(nodeTests); -const parallelNodeResult = runNodeTests(nodeTestGroups.parallel); -if (parallelNodeResult !== 0) { - process.exit(parallelNodeResult); -} +if (suite !== "ui") { + // Explicit groups use the Windows isolation boundary on every platform so + // their coverage can also be verified locally. Each CI group owns a VM. + const grouped = suite === "node-parallel" || suite === "node-serial"; + const nodeTestGroups = partitionNodeTestsByPlatform( + nodeTests, + grouped ? "win32" : process.platform, + ); + if ( + grouped && + (!nodeTestGroups.parallel.length || !nodeTestGroups.serial.length) + ) { + throw new Error("Both isolated Node test groups must be non-empty."); + } + if (suite === "node-serial") nodeTestGroups.parallel = []; + if (suite === "node-parallel") { + nodeTestGroups.serial = []; + if (shardArgument !== undefined) { + nodeTestGroups.parallel = selectNodeTestShard( + nodeTestGroups.parallel, + shardArgument.slice("--shard=".length), + ); + } + } + const parallelNodeResult = runNodeTests(nodeTestGroups.parallel); + if (parallelNodeResult !== 0) { + process.exit(parallelNodeResult); + } -// Windows process-tree tests must not overlap unrelated Node test files. -// Keep the rest of the suite on Node's default file-level concurrency. -const serialNodeResult = runNodeTests(nodeTestGroups.serial, [ - "--test-concurrency=1", -]); -if (serialNodeResult !== 0) { - process.exit(serialNodeResult); + // Windows process-tree tests must not overlap unrelated Node test files. + // Keep the rest of the suite on Node's default file-level concurrency. + const serialNodeResult = runNodeTests(nodeTestGroups.serial, [ + "--test-concurrency=1", + ]); + if (serialNodeResult !== 0) { + process.exit(serialNodeResult); + } } +if (suite.startsWith("node")) process.exit(0); + // Invoke the CLI module through Node instead of the package-manager shim. // Windows installs expose the shim as `vitest.cmd`, which cannot be launched // reliably by spawnSync without a shell. diff --git a/tests/scripts/node-test-groups.test.ts b/tests/scripts/node-test-groups.test.ts index 86c0912b5..ab2b1ca2d 100644 --- a/tests/scripts/node-test-groups.test.ts +++ b/tests/scripts/node-test-groups.test.ts @@ -1,7 +1,11 @@ import assert from "node:assert/strict"; import { resolve } from "node:path"; import test from "node:test"; -import { partitionNodeTestsByPlatform } from "../../scripts/node-test-groups.mjs"; +import { + partitionNodeTestsByPlatform, + selectNodeTestShard, +} from "../../scripts/node-test-groups.mjs"; +import { discoverTestFiles } from "../../scripts/discover-tests.mjs"; const backgroundTest = resolve( "tests", @@ -42,3 +46,17 @@ test("non-Windows keeps all Node files in the parallel group", () => { }, ); }); + +test("CI's Windows shards are disjoint and cover all discovered Node files", () => { + const files = discoverTestFiles().filter((file) => file.endsWith(".test.ts")); + const { parallel, serial } = partitionNodeTestsByPlatform(files, "win32"); + const groups = [ + selectNodeTestShard(parallel, "1/2"), + selectNodeTestShard(parallel, "2/2"), + serial, + ]; + assert.ok(groups.every((group) => group.length > 0)); + const selected = groups.flat(); + assert.equal(new Set(selected).size, selected.length); + assert.deepEqual([...selected].sort(), [...files].sort()); +}); diff --git a/tests/scripts/run-tests.test.ts b/tests/scripts/run-tests.test.ts new file mode 100644 index 000000000..881da9491 --- /dev/null +++ b/tests/scripts/run-tests.test.ts @@ -0,0 +1,207 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { + copyFileSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; + +function fixture() { + const root = mkdtempSync(join(tmpdir(), "openpi-test-runner-")); + for (const directory of [ + "scripts", + "tests/extensions/background-terminals", + "node_modules/vitest", + ]) { + mkdirSync(join(root, directory), { recursive: true }); + } + for (const script of [ + "run-tests.mjs", + "discover-tests.mjs", + "node-test-groups.mjs", + ]) { + copyFileSync(resolve("scripts", script), join(root, "scripts", script)); + } + writeFileSync( + join(root, "tests", "sample.test.ts"), + ` + import assert from "node:assert/strict"; + import test from "node:test"; + test("node suite marker", () => assert.notEqual(process.env.FAIL_SUITE, "node")); + `, + ); + writeFileSync( + join(root, "tests", "second.test.ts"), + `import test from "node:test"; test("second node marker", () => {});`, + ); + writeFileSync( + join(root, "tests/extensions/background-terminals", "process.test.ts"), + ` + import assert from "node:assert/strict"; + import test from "node:test"; + test("isolated process marker", () => assert.notEqual(process.env.FAIL_SUITE, "serial")); + `, + ); + writeFileSync( + join(root, "tests", "sample.spec.ts"), + "// UI discovery fixture\n", + ); + // A tiny CLI stand-in isolates runner selection/exit behavior from Vitest itself. + writeFileSync( + join(root, "node_modules/vitest/vitest.mjs"), + ` + import assert from "node:assert/strict"; + assert.equal(process.argv[2], "run"); + assert.ok(process.argv[3].endsWith("sample.spec.ts")); + console.log("ui suite marker"); + process.exit(process.env.FAIL_SUITE === "ui" ? 7 : 0); + `, + ); + return { + root, + run(args: string[] = [], failSuite = "") { + const env: NodeJS.ProcessEnv = { ...process.env, FAIL_SUITE: failSuite }; + // Start a fresh test runner rather than inheriting this test worker context. + delete env.NODE_TEST_CONTEXT; + return spawnSync(process.execPath, ["scripts/run-tests.mjs", ...args], { + cwd: root, + env, + encoding: "utf8", + }); + }, + dispose() { + rmSync(root, { recursive: true, force: true }); + }, + }; +} + +test("split suite commands together cover the unchanged default command", () => { + const runner = fixture(); + try { + for (const [args, nodeExpected, uiExpected] of [ + [[], true, true], + [["all"], true, true], + [["node"], true, false], + [["ui"], false, true], + ] as const) { + const result = runner.run([...args]); + assert.equal(result.status, 0, result.stderr || result.stdout); + assert.equal(result.stdout.includes("node suite marker"), nodeExpected); + assert.equal(result.stdout.includes("ui suite marker"), uiExpected); + } + } finally { + runner.dispose(); + } +}); + +test("test failures propagate and default execution stops after Node failure", () => { + const runner = fixture(); + try { + const node = runner.run(["node"], "node"); + assert.notEqual(node.status, 0); + assert.equal(node.stdout.includes("ui suite marker"), false); + assert.equal(runner.run(["ui"], "ui").status, 7); + const combined = runner.run([], "node"); + assert.notEqual(combined.status, 0); + assert.equal(combined.stdout.includes("ui suite marker"), false); + assert.equal(runner.run([], "ui").status, 7); + } finally { + runner.dispose(); + } +}); + +test("isolated shards cover every Node file exactly once and propagate failures", () => { + const runner = fixture(); + try { + const outputs = [ + runner.run(["node-parallel", "--shard=1/2"]), + runner.run(["node-parallel", "--shard=2/2"]), + runner.run(["node-serial"]), + ]; + for (const result of outputs) { + assert.equal(result.status, 0, result.stderr || result.stdout); + assert.equal(result.stdout.includes("ui suite marker"), false); + } + for (const marker of [ + "node suite marker", + "second node marker", + "isolated process marker", + ]) { + assert.equal( + outputs.filter((result) => result.stdout.includes(marker)).length, + 1, + ); + } + assert.equal(outputs[2].stdout.includes("isolated process marker"), true); + assert.notEqual( + runner.run(["node-parallel", "--shard=1/2"], "node").status, + 0, + ); + assert.notEqual(runner.run(["node-serial"], "serial").status, 0); + assert.notEqual(runner.run([], "serial").status, 0); + } finally { + runner.dispose(); + } +}); + +test("invalid and empty shards fail before executing any tests", () => { + const runner = fixture(); + try { + for (const shard of [ + "0/2", + "3/2", + "1/0", + "1/3", + "1/2junk", + "1/9007199254740992", + ]) { + const result = runner.run(["node-parallel", `--shard=${shard}`]); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Invalid Node test shard/); + assert.equal(result.stdout.includes("marker"), false); + } + for (const args of [ + ["node-serial", "--shard=1/2"], + ["node-parallel", "extra"], + ["node-parallel", "--shard=1/2", "extra"], + ]) { + assert.match(runner.run(args).stderr, /Usage:/); + } + rmSync(join(runner.root, "tests/extensions/background-terminals"), { + recursive: true, + }); + const result = runner.run(["node-parallel"]); + assert.notEqual(result.status, 0); + assert.match( + result.stderr, + /Both isolated Node test groups must be non-empty/, + ); + assert.equal(result.stdout.includes("marker"), false); + } finally { + runner.dispose(); + } +}); + +test("invalid selection and empty discovery fail before running a partial suite", () => { + const runner = fixture(); + try { + for (const args of [["unknown"], ["node", "extra"]]) { + const result = runner.run(args); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Usage:/); + assert.equal(result.stdout.includes("suite marker"), false); + } + rmSync(join(runner.root, "tests", "sample.spec.ts")); + const empty = runner.run(["node"]); + assert.notEqual(empty.status, 0); + assert.match(empty.stderr, /both suites must be non-empty/); + assert.equal(empty.stdout.includes("suite marker"), false); + } finally { + runner.dispose(); + } +}); diff --git a/tests/web/openpi-web.e2e.ts b/tests/web/openpi-web.e2e.ts index 670fe0a9f..7a8498c98 100644 --- a/tests/web/openpi-web.e2e.ts +++ b/tests/web/openpi-web.e2e.ts @@ -570,6 +570,8 @@ test("provider failures remain visible after refresh without discarding partial await expect(page.getByText("Partial answer")).toBeVisible(); await expect(page.getByText("模型请求已停止")).toBeVisible(); await expect(page.getByRole("alert")).toHaveCount(0); + // Drain snapshot handlers before the per-test request context is disposed. + await page.unrouteAll({ behavior: "wait" }); }); async function openWorkbench(page: Page) { @@ -3067,6 +3069,13 @@ test("same-named model selection sends the exact identity for an active Session" const snapshot = await response.json(); selectedSessionPath = snapshot.selectedSession.path; snapshot.runtime.status = "idle"; + // Model-selection fixtures must not inherit the Host's thinking capability. + snapshot.thinking = { + ...snapshot.thinking, + level: "off", + available: [], + supported: false, + }; snapshot.models = [ { provider: "provider-alpha", @@ -3156,6 +3165,13 @@ test("finds and selects a model omitted from the bounded snapshot", async ({ activeSessionId = snapshot.currentSessionId; activeSessionPath = snapshot.selectedSession.path; snapshot.runtime.status = "idle"; + // Model-selection fixtures must not inherit the Host's thinking capability. + snapshot.thinking = { + ...snapshot.thinking, + level: "off", + available: [], + supported: false, + }; snapshot.models = selectedIdentity === "provider-hidden/needle-251" ? [ diff --git a/tests/web/playwright.config.ts b/tests/web/playwright.config.ts index 3da0c9ef8..b90916cf5 100644 --- a/tests/web/playwright.config.ts +++ b/tests/web/playwright.config.ts @@ -41,7 +41,9 @@ export default defineConfig({ "tests/web/subagent-inspection.e2e.ts", ], outputDir: outputDirectory, - fullyParallel: false, + // Shard individual cases, including the large workbench file. Each runner + // still uses one worker so tests cannot mutate its shared server concurrently. + fullyParallel: true, workers: 1, timeout: 30_000, reporter: "list",