From 122b8a4f60dd5dc2e57b68edc44014f3e2514489 Mon Sep 17 00:00:00 2001 From: Eric Litman Date: Tue, 20 Oct 2026 22:50:39 -0400 Subject: [PATCH 1/4] fix(verifier): require concrete work in the never-block case (#225) The default skill case accepted a session that loaded principle-never-block-on-the-human and wrote only that it was waiting for approval. The never-block case now reuses the fix-ci check: the session must fix sum.mjs. A change to features/recipes.ts can now name any feature that has a recipe, so a recipe-only PR can run the recipe it changes. Ported from #226. Co-authored-by: andrewfree <507412+andrewfree@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .claude/skills/verify-open-pstack/SKILL.md | 1 + .../verify-open-pstack/features/README.md | 2 ++ .../verify-open-pstack/features/recipes.ts | 1 + .../features/skill-invocation.md | 2 +- .../verify-open-pstack/scripts/core.test.ts | 30 +++++++++++++++++ .../skills/verify-open-pstack/scripts/core.ts | 4 ++- .../scripts/skill-invocation.test.ts | 32 +++++++++++++++++++ .../verify-open-pstack/scripts/verify.test.ts | 22 +++++++++++++ 8 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 .claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts diff --git a/.claude/skills/verify-open-pstack/SKILL.md b/.claude/skills/verify-open-pstack/SKILL.md index 46fedffb..4f9dc155 100644 --- a/.claude/skills/verify-open-pstack/SKILL.md +++ b/.claude/skills/verify-open-pstack/SKILL.md @@ -99,6 +99,7 @@ Read receipts as data, never shell input. Do not put credentials in prompts or p - scripts/provenance.test.ts - scripts/provenance.ts - scripts/reflect.test.ts +- scripts/skill-invocation.test.ts - scripts/types.ts - scripts/verify.sh - scripts/verify.test.ts diff --git a/.claude/skills/verify-open-pstack/features/README.md b/.claude/skills/verify-open-pstack/features/README.md index 25053d6e..6103cb01 100644 --- a/.claude/skills/verify-open-pstack/features/README.md +++ b/.claude/skills/verify-open-pstack/features/README.md @@ -16,6 +16,8 @@ The ownership table below says which features a changed path reaches. `SKILL.md` | `project` | Native discovery/invocation of this verifier in both harnesses | | `nonRuntime` | Explicit ordinary repository documentation, policy, non-shipped assets, and unit-test paths; no plugin harness launch | +A change to `features/recipes.ts` can name any feature that has a recipe. + A change to `plugins/pstack/.claude-plugin/plugin.json`, `.claude-plugin/marketplace.json`, or `plugins/pstack/.codex-plugin/plugin.json` is version-only when the merge base and the head hold the same regular file with the same mode, both sides parse as JSON objects, at least one version string changed (top-level `version`, or `plugins[].version` in the marketplace), and nothing else differs. A version-only change reaches only `hook:startup`: the two Claude files in Claude, the Codex manifest in Codex. `.agents/plugins/marketplace.json` has no version and always takes the shared expansion. Asset ownership is consumer-driven, not blanket shared ownership. Inspect the exact candidate's plugin manifests. At present, `plugins/pstack/assets/logo.png` is referenced only by `plugins/pstack/.codex-plugin/plugin.json`, so a logo-only change selects the installed Codex asset surface and does not invent a Claude exercise. New shipped assets must be added to `assets` and must have a real manifest consumer; missing, ambiguous, or unknown consumption fails closed. diff --git a/.claude/skills/verify-open-pstack/features/recipes.ts b/.claude/skills/verify-open-pstack/features/recipes.ts index 8a006e4b..156ae866 100644 --- a/.claude/skills/verify-open-pstack/features/recipes.ts +++ b/.claude/skills/verify-open-pstack/features/recipes.ts @@ -133,6 +133,7 @@ async function ciFixed(_record: SessionRecord, f: Fixture): Promise { const overrides: Record = { 'fix-ci': skillCase('fix-ci', 'There is no pull request or remote. The failing CI check is the local command `sh ci.sh` in $DIR. Fix it locally and do not push.', ciFixed), babysit: skillCase('babysit', 'The pull request is the local `main` branch of the repository at $DIR; there is no remote. Its one failing check is the local command `sh ci.sh`. Do exactly one babysit pass: fix the failing check locally, push nothing, change nothing remote, then stop instead of watching.', ciFixed), + 'principle-never-block-on-the-human': skillCase('principle-never-block-on-the-human', 'The bug in $DIR/TASK.md is a reversible local change. Apply the skill: implement the smallest fix in sum.mjs and verify it with bun sum.test.mjs.', ciFixed), 'make-pr-easy-to-review': skillCase('make-pr-easy-to-review', 'The pull request is the local branch `feature` against `main` in $DIR; there is no remote. Do not rewrite history or push; write reviewer guidance only.', async (_record, f) => { if ((await git(f, 'rev-parse', 'feature')).trim() !== (await readFile(`${f.dir}.head`, 'utf8')).trim()) throw new Error('history-rewritten:feature'); return { assertions: ['history-unchanged'], files: [] }; diff --git a/.claude/skills/verify-open-pstack/features/skill-invocation.md b/.claude/skills/verify-open-pstack/features/skill-invocation.md index 672e1b00..4ec8b523 100644 --- a/.claude/skills/verify-open-pstack/features/skill-invocation.md +++ b/.claude/skills/verify-open-pstack/features/skill-invocation.md @@ -22,6 +22,6 @@ A generic prompt about a skill is not native invocation evidence. A list of disc ## Headless -`recipes.ts` gives every `skill-invocation:` one default case: a disposable Git fixture holding a small task in `TASK.md`, a native invocation (`/pstack:` in Claude, the Skill tool for principle leaves, `$pstack:` in Codex), and an instruction to write the result to `result.md` in the fixture. It asserts that the native record shows the `pstack:` skill load and that the session wrote a nonempty `result.md`. Skills whose normal use would push, merge, post, or write daily state override it: `fix-ci`, `babysit`, and `make-pr-easy-to-review` run against a local fixture with no remote and assert a local effect, and `automate-me` writes its draft into the fixture instead of the daily skills folder. `setup-pstack` uses the setup recipe. `project-skill` runs from the candidate checkout and asserts the skill load, the doctor command, and the canonical-path `doctor.json`. +`recipes.ts` gives every `skill-invocation:` one default case: a disposable Git fixture holding a small task in `TASK.md`, a native invocation (`/pstack:` in Claude, the Skill tool for principle leaves, `$pstack:` in Codex), and an instruction to write the result to `result.md` in the fixture. It asserts that the native record shows the `pstack:` skill load and that the session wrote a nonempty `result.md`. Skills whose case needs a concrete local effect or a disposable fixture override it: `fix-ci`, `babysit`, and `make-pr-easy-to-review` run against a local fixture with no remote and assert a local effect, `principle-never-block-on-the-human` must fix `sum.mjs` so that a report that only waits for approval fails, and `automate-me` writes its draft into the fixture instead of the daily skills folder. `setup-pstack` uses the setup recipe. `project-skill` runs from the candidate checkout and asserts the skill load, the doctor command, and the canonical-path `doctor.json`. `reflect` has two cases in each harness that exercise step 1 only, against run-owned synthetic transcripts with metadata before the opening user message. The prompts name the opening prompt and, for Codex, the known chat ID and a fixture-only `CODEX_HOME`, or, for Claude, the fixture's per-project directory. They do not name the resolver or the transcript file, so the installed active-transcript mapping must lead the session there. In `locate-transcript` the transcript's opener matches. The case asserts the opening-message metadata and that no digest was written. In `digest-fallback` the opener differs, and the case asserts a nonblank digest of at most 2,000 words. Each run uses a per-run nonce that no prompt contains. For Codex, it is in the fixture rollout's file name. A completed command must name the installed resolver's launcher or script by absolute path, and its harness-captured output must parse to exactly the resolver JSON for this run, key order aside. Failed or pending commands and output from another fixture or run do not count, and a report file the model writes is never the Codex execution evidence. For Claude, the nonce is the opening user record's `uuid`, which only the transcript holds. In both cases the report must equal the expected JSON exactly, including that `uuid`: under `opening_user_message` when the opener matches, and beside the fallback status when it does not. A report without the `uuid`, or with another run's, does not count. These cases do not exercise reviewer fan-out, synthesis, or skill edits. diff --git a/.claude/skills/verify-open-pstack/scripts/core.test.ts b/.claude/skills/verify-open-pstack/scripts/core.test.ts index 16417a7d..455fd0d1 100644 --- a/.claude/skills/verify-open-pstack/scripts/core.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/core.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test'; import { execFileSync } from 'node:child_process'; import { resolve } from 'node:path'; import registryData from '../features/registry.json'; +import { recipe } from '../features/recipes.ts'; import { classify, completeEvidence, matches, newReceipt, requiredFeatures, requiredHarnesses, validateRegistry } from './core.ts'; import { HARNESSES, type Receipt } from './types.ts'; import codexManifest from '../../../../plugins/pstack/.codex-plugin/plugin.json'; @@ -181,6 +182,35 @@ describe('registry and ownership', () => { evidence(r); expect(() => completeEvidence(r)).not.toThrow(); }); + test('the shared recipe implementation reaches every implemented recipe and preserves project proof', () => { + const path = '.claude/skills/verify-open-pstack/features/recipes.ts'; + const expected = [...both('catalog', 'setup', 'runner', 'shipped-tools', 'hook:startup', 'project-skill', + ...registry.skills.map(skill => `skill-invocation:${skill}`)), 'claude:agent-preload:poteto-agent'].sort(); + for (const file of [{ filename: path }, { filename: 'docs/moved.md', previous_filename: path }]) { + const s = classify([file], registry); + expect(s.features).toEqual(expected); + expect(s.skills).toEqual([...registry.skills].sort()); + expect(s.noRuntime).toBe(false); + for (const harness of HARNESSES) { + for (const feature of s.features.filter(f => f.startsWith(`${harness}:`))) { + expect(recipe(feature.slice(harness.length + 1), harness).length).toBeGreaterThan(0); + } + } + } + }); + test('nearby verifier documentation retains only project proof', () => { + for (const filename of ['.claude/skills/verify-open-pstack/features/README.md', + '.claude/skills/verify-open-pstack/features/skill-invocation.md']) { + expect(classify([{ filename }], registry).features).toEqual(both('project-skill')); + } + }); + test('shared recipe expansion survives overlapping catalog ownership without broadening ordinary catalog paths', () => { + const path = '.claude/skills/verify-open-pstack/features/recipes.ts'; + const overlapping = { ...registry, catalog: [...registry.catalog, path], shared: [...registry.shared, path] }; + expect(classify([{ filename: path }], overlapping)).toEqual(classify([{ filename: path }], registry)); + const catalog = registry.catalog[0]!; + expect(classify([{ filename: catalog }], overlapping)).toEqual(classify([{ filename: catalog }], registry)); + }); test('ordinary repository documentation selects no harnesses', () => { const s = classify([{ filename: 'README.md' }, { filename: 'docs/guide.md' }], registry); expect(s.noRuntime).toBe(true); diff --git a/.claude/skills/verify-open-pstack/scripts/core.ts b/.claude/skills/verify-open-pstack/scripts/core.ts index 4ad84523..6cf3c99d 100644 --- a/.claude/skills/verify-open-pstack/scripts/core.ts +++ b/.claude/skills/verify-open-pstack/scripts/core.ts @@ -34,9 +34,11 @@ export function classify(files: ChangedFile[], registry: Registry): Selection { if (path !== 'plugins/pstack/assets/logo.png') throw new Error(`Unmapped asset consumer: ${path}`); features.add('codex:assets:codex'); covered = true; } + const sharedRecipe = path === '.claude/skills/verify-open-pstack/features/recipes.ts'; + if (sharedRecipe) both('catalog'); const catalog = registry.catalog.some(p => matches(path, p)); if (catalog) { both('catalog', 'setup', 'runner'); covered = true; } - if (!catalog && registry.shared.some(p => matches(path, p))) { + if (sharedRecipe || (!catalog && registry.shared.some(p => matches(path, p)))) { registry.skills.forEach(s => skills.add(s)); both('setup', 'runner', 'shipped-tools', 'hook:startup'); features.add('claude:agent-preload:poteto-agent'); diff --git a/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts b/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts new file mode 100644 index 00000000..6215647d --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts @@ -0,0 +1,32 @@ +import { afterEach, expect, test } from 'bun:test'; +import { mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { recipe, type Case, type Fixture } from '../features/recipes.ts'; +import { command } from './io.ts'; +import type { Harness, SessionRecord } from './types.ts'; + +const roots: string[] = []; +afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +const skill = 'principle-never-block-on-the-human'; + +async function waitingSession(harness: Harness): Promise<{ f: Fixture; c: Case; record: SessionRecord }> { + const dir = await realpath(await mkdtemp(join(tmpdir(), 'pstack-never-block-'))); roots.push(dir); + const f = { harness, dir, workspace: dir, location: dir, configHome: dir, run: command }; + const c = recipe(`skill-invocation:${skill}`, harness)[0]!; + await c.prepare!(f); + const record = { harness, exitCode: 0, started: Date.now() - 1000, skills: [`pstack:${skill}`], commands: [], tools: [], context: [] }; + await writeFile(join(dir, 'result.md'), 'Waiting for you to approve inspection. I did not inspect anything.'); + return { f, c, record }; +} + +test.each(['claude', 'codex'] as const)('%s never-block case rejects a waiting-only report', async harness => { + const { f, c, record } = await waitingSession(harness); + await expect(c.assert(record, f)).rejects.toThrow('ci-not-fixed:sum.mjs'); +}); + +test.each(['claude', 'codex'] as const)('%s never-block case passes once sum.mjs is fixed', async harness => { + const { f, c, record } = await waitingSession(harness); + await writeFile(join(f.dir, 'sum.mjs'), 'export function sum(a, b) {\n return a + b;\n}\n'); + expect((await c.assert(record, f)).assertions).toContain('fixed:sum.mjs'); +}); diff --git a/.claude/skills/verify-open-pstack/scripts/verify.test.ts b/.claude/skills/verify-open-pstack/scripts/verify.test.ts index ea2c7c65..ecf07c36 100644 --- a/.claude/skills/verify-open-pstack/scripts/verify.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/verify.test.ts @@ -115,6 +115,28 @@ describe('exact-head evidence publication', () => { expect((await verify(f.options)).status).toBe('success'); expect(f.calls).toContain('prepare'); }); + test('a shared recipe change runs only the named recipe and project proof from the candidate publisher', async () => { + const selected = 'codex:skill-invocation:principle-never-block-on-the-human'; + for (const revision of [REVISION, SHA]) { + const f = fixture(true); + f.options.publisherRevision = revision; + f.options.features = [selected]; + f.options.github.files = async () => [{ filename: '.claude/skills/verify-open-pstack/features/recipes.ts' }]; + if (revision !== SHA) { + await expect(verify(f.options)).rejects.toThrow('rerun from that checkout'); + expect(f.calls.some(call => ['preflight', 'prepare', 'exercise', 'comment'].includes(call) || call.startsWith('status:'))).toBe(false); + } else { + const receipt = await verify(f.options); + expect(receipt.status).toBe('success'); + expect(receipt.runFeatures).toEqual([selected, 'claude:project-skill', 'codex:project-skill']); + expect(receipt.selection.features.length).toBeGreaterThan(receipt.runFeatures.length); + expect(receipt.sessions).toBe(3); + expect(receipt.observations.map(record => `${record.harness}/${record.feature}`).sort()).toEqual([ + 'claude/project-skill', 'codex/project-skill', 'codex/skill-invocation:principle-never-block-on-the-human', + ]); + } + } + }); test('successful verification restores setup and removes Codex run state before any publication', async () => { const f = fixture(true); f.options.driver.cleanup = async () => { f.calls.push('cleanup'); }; From ea886913bb5e7aaff90e1a73af941cd3e6097280 Mon Sep 17 00:00:00 2001 From: Eric Litman Date: Tue, 20 Oct 2026 23:05:26 -0400 Subject: [PATCH 2/4] fix(verifier): prove the never-block edit, not a source pattern (#225) The never-block case reused ciFixed, whose regex passes return a + b - 1 as fixed:sum.mjs and fails a correct return b + a. The case exists to prove the session acted on the reversible change instead of waiting, so it now asserts that sum.mjs changed and publishes edited:sum.mjs. Co-authored-by: andrewfree <507412+andrewfree@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .../skills/verify-open-pstack/features/recipes.ts | 7 ++++++- .../verify-open-pstack/features/skill-invocation.md | 2 +- .../scripts/skill-invocation.test.ts | 12 +++++++++--- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.claude/skills/verify-open-pstack/features/recipes.ts b/.claude/skills/verify-open-pstack/features/recipes.ts index 156ae866..d4eabd2a 100644 --- a/.claude/skills/verify-open-pstack/features/recipes.ts +++ b/.claude/skills/verify-open-pstack/features/recipes.ts @@ -130,10 +130,15 @@ async function ciFixed(_record: SessionRecord, f: Fixture): Promise { return { assertions: ['fixed:sum.mjs'], files: [join(f.dir, 'sum.mjs')] }; } +async function sumEdited(_record: SessionRecord, f: Fixture): Promise { + if (await readFile(join(f.dir, 'sum.mjs'), 'utf8') === TASK['sum.mjs']) throw new Error('sum-not-edited:sum.mjs'); + return { assertions: ['edited:sum.mjs'], files: [join(f.dir, 'sum.mjs')] }; +} + const overrides: Record = { 'fix-ci': skillCase('fix-ci', 'There is no pull request or remote. The failing CI check is the local command `sh ci.sh` in $DIR. Fix it locally and do not push.', ciFixed), babysit: skillCase('babysit', 'The pull request is the local `main` branch of the repository at $DIR; there is no remote. Its one failing check is the local command `sh ci.sh`. Do exactly one babysit pass: fix the failing check locally, push nothing, change nothing remote, then stop instead of watching.', ciFixed), - 'principle-never-block-on-the-human': skillCase('principle-never-block-on-the-human', 'The bug in $DIR/TASK.md is a reversible local change. Apply the skill: implement the smallest fix in sum.mjs and verify it with bun sum.test.mjs.', ciFixed), + 'principle-never-block-on-the-human': skillCase('principle-never-block-on-the-human', 'The bug in $DIR/TASK.md is a reversible local change. Apply the skill: implement the smallest fix in sum.mjs and verify it with bun sum.test.mjs.', sumEdited), 'make-pr-easy-to-review': skillCase('make-pr-easy-to-review', 'The pull request is the local branch `feature` against `main` in $DIR; there is no remote. Do not rewrite history or push; write reviewer guidance only.', async (_record, f) => { if ((await git(f, 'rev-parse', 'feature')).trim() !== (await readFile(`${f.dir}.head`, 'utf8')).trim()) throw new Error('history-rewritten:feature'); return { assertions: ['history-unchanged'], files: [] }; diff --git a/.claude/skills/verify-open-pstack/features/skill-invocation.md b/.claude/skills/verify-open-pstack/features/skill-invocation.md index 4ec8b523..a4f11f7f 100644 --- a/.claude/skills/verify-open-pstack/features/skill-invocation.md +++ b/.claude/skills/verify-open-pstack/features/skill-invocation.md @@ -22,6 +22,6 @@ A generic prompt about a skill is not native invocation evidence. A list of disc ## Headless -`recipes.ts` gives every `skill-invocation:` one default case: a disposable Git fixture holding a small task in `TASK.md`, a native invocation (`/pstack:` in Claude, the Skill tool for principle leaves, `$pstack:` in Codex), and an instruction to write the result to `result.md` in the fixture. It asserts that the native record shows the `pstack:` skill load and that the session wrote a nonempty `result.md`. Skills whose case needs a concrete local effect or a disposable fixture override it: `fix-ci`, `babysit`, and `make-pr-easy-to-review` run against a local fixture with no remote and assert a local effect, `principle-never-block-on-the-human` must fix `sum.mjs` so that a report that only waits for approval fails, and `automate-me` writes its draft into the fixture instead of the daily skills folder. `setup-pstack` uses the setup recipe. `project-skill` runs from the candidate checkout and asserts the skill load, the doctor command, and the canonical-path `doctor.json`. +`recipes.ts` gives every `skill-invocation:` one default case: a disposable Git fixture holding a small task in `TASK.md`, a native invocation (`/pstack:` in Claude, the Skill tool for principle leaves, `$pstack:` in Codex), and an instruction to write the result to `result.md` in the fixture. It asserts that the native record shows the `pstack:` skill load and that the session wrote a nonempty `result.md`. Skills whose case needs a concrete local effect or a disposable fixture override it: `fix-ci`, `babysit`, and `make-pr-easy-to-review` run against a local fixture with no remote and assert a local effect, `principle-never-block-on-the-human` must edit `sum.mjs`, so a report that only waits for approval fails, and `automate-me` writes its draft into the fixture instead of the daily skills folder. `setup-pstack` uses the setup recipe. `project-skill` runs from the candidate checkout and asserts the skill load, the doctor command, and the canonical-path `doctor.json`. `reflect` has two cases in each harness that exercise step 1 only, against run-owned synthetic transcripts with metadata before the opening user message. The prompts name the opening prompt and, for Codex, the known chat ID and a fixture-only `CODEX_HOME`, or, for Claude, the fixture's per-project directory. They do not name the resolver or the transcript file, so the installed active-transcript mapping must lead the session there. In `locate-transcript` the transcript's opener matches. The case asserts the opening-message metadata and that no digest was written. In `digest-fallback` the opener differs, and the case asserts a nonblank digest of at most 2,000 words. Each run uses a per-run nonce that no prompt contains. For Codex, it is in the fixture rollout's file name. A completed command must name the installed resolver's launcher or script by absolute path, and its harness-captured output must parse to exactly the resolver JSON for this run, key order aside. Failed or pending commands and output from another fixture or run do not count, and a report file the model writes is never the Codex execution evidence. For Claude, the nonce is the opening user record's `uuid`, which only the transcript holds. In both cases the report must equal the expected JSON exactly, including that `uuid`: under `opening_user_message` when the opener matches, and beside the fallback status when it does not. A report without the `uuid`, or with another run's, does not count. These cases do not exercise reviewer fan-out, synthesis, or skill edits. diff --git a/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts b/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts index 6215647d..6233d1d6 100644 --- a/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts @@ -22,11 +22,17 @@ async function waitingSession(harness: Harness): Promise<{ f: Fixture; c: Case; test.each(['claude', 'codex'] as const)('%s never-block case rejects a waiting-only report', async harness => { const { f, c, record } = await waitingSession(harness); - await expect(c.assert(record, f)).rejects.toThrow('ci-not-fixed:sum.mjs'); + await expect(c.assert(record, f)).rejects.toThrow('sum-not-edited:sum.mjs'); }); -test.each(['claude', 'codex'] as const)('%s never-block case passes once sum.mjs is fixed', async harness => { +test.each(['claude', 'codex'] as const)('%s never-block case passes once sum.mjs is edited', async harness => { const { f, c, record } = await waitingSession(harness); await writeFile(join(f.dir, 'sum.mjs'), 'export function sum(a, b) {\n return a + b;\n}\n'); - expect((await c.assert(record, f)).assertions).toContain('fixed:sum.mjs'); + expect((await c.assert(record, f)).assertions).toContain('edited:sum.mjs'); +}); + +test.each(['claude', 'codex'] as const)('%s never-block case accepts an edit written b + a', async harness => { + const { f, c, record } = await waitingSession(harness); + await writeFile(join(f.dir, 'sum.mjs'), 'export function sum(a, b) {\n return b + a;\n}\n'); + expect((await c.assert(record, f)).assertions).toContain('edited:sum.mjs'); }); From 8e8b09dbf5883fd343ac9e84dc65ec59b08e58fd Mon Sep 17 00:00:00 2001 From: Eric Litman Date: Tue, 20 Oct 2026 23:25:57 -0400 Subject: [PATCH 3/4] fix(verifier): run the fixture test to prove the never-block repair (#225) sumEdited passed any change to sum.mjs, so a failed repair passed while the prompt asked for a verified fix. The never-block case now requires the fixture's unchanged sum.test.mjs to pass against the session's sum.mjs, run through the fixture's candidate command, and publishes test-passed:sum.test.mjs. Co-authored-by: andrewfree <507412+andrewfree@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .../verify-open-pstack/features/recipes.ts | 9 ++--- .../features/skill-invocation.md | 2 +- .../scripts/skill-invocation.test.ts | 33 +++++++++++++++---- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/.claude/skills/verify-open-pstack/features/recipes.ts b/.claude/skills/verify-open-pstack/features/recipes.ts index d4eabd2a..d1f893ef 100644 --- a/.claude/skills/verify-open-pstack/features/recipes.ts +++ b/.claude/skills/verify-open-pstack/features/recipes.ts @@ -130,15 +130,16 @@ async function ciFixed(_record: SessionRecord, f: Fixture): Promise { return { assertions: ['fixed:sum.mjs'], files: [join(f.dir, 'sum.mjs')] }; } -async function sumEdited(_record: SessionRecord, f: Fixture): Promise { - if (await readFile(join(f.dir, 'sum.mjs'), 'utf8') === TASK['sum.mjs']) throw new Error('sum-not-edited:sum.mjs'); - return { assertions: ['edited:sum.mjs'], files: [join(f.dir, 'sum.mjs')] }; +async function sumTestPassed(_record: SessionRecord, f: Fixture): Promise { + if (await readFile(join(f.dir, 'sum.test.mjs'), 'utf8') !== TASK['sum.test.mjs']) throw new Error('test-changed:sum.test.mjs'); + await f.run(['bun', 'sum.test.mjs'], { cwd: f.dir }).catch(() => { throw new Error('test-failed:sum.test.mjs'); }); + return { assertions: ['test-passed:sum.test.mjs'], files: [join(f.dir, 'sum.mjs'), join(f.dir, 'sum.test.mjs')] }; } const overrides: Record = { 'fix-ci': skillCase('fix-ci', 'There is no pull request or remote. The failing CI check is the local command `sh ci.sh` in $DIR. Fix it locally and do not push.', ciFixed), babysit: skillCase('babysit', 'The pull request is the local `main` branch of the repository at $DIR; there is no remote. Its one failing check is the local command `sh ci.sh`. Do exactly one babysit pass: fix the failing check locally, push nothing, change nothing remote, then stop instead of watching.', ciFixed), - 'principle-never-block-on-the-human': skillCase('principle-never-block-on-the-human', 'The bug in $DIR/TASK.md is a reversible local change. Apply the skill: implement the smallest fix in sum.mjs and verify it with bun sum.test.mjs.', sumEdited), + 'principle-never-block-on-the-human': skillCase('principle-never-block-on-the-human', 'The bug in $DIR/TASK.md is a reversible local change. Apply the skill: implement the smallest fix in sum.mjs and verify it with bun sum.test.mjs.', sumTestPassed), 'make-pr-easy-to-review': skillCase('make-pr-easy-to-review', 'The pull request is the local branch `feature` against `main` in $DIR; there is no remote. Do not rewrite history or push; write reviewer guidance only.', async (_record, f) => { if ((await git(f, 'rev-parse', 'feature')).trim() !== (await readFile(`${f.dir}.head`, 'utf8')).trim()) throw new Error('history-rewritten:feature'); return { assertions: ['history-unchanged'], files: [] }; diff --git a/.claude/skills/verify-open-pstack/features/skill-invocation.md b/.claude/skills/verify-open-pstack/features/skill-invocation.md index a4f11f7f..d5e2f54c 100644 --- a/.claude/skills/verify-open-pstack/features/skill-invocation.md +++ b/.claude/skills/verify-open-pstack/features/skill-invocation.md @@ -22,6 +22,6 @@ A generic prompt about a skill is not native invocation evidence. A list of disc ## Headless -`recipes.ts` gives every `skill-invocation:` one default case: a disposable Git fixture holding a small task in `TASK.md`, a native invocation (`/pstack:` in Claude, the Skill tool for principle leaves, `$pstack:` in Codex), and an instruction to write the result to `result.md` in the fixture. It asserts that the native record shows the `pstack:` skill load and that the session wrote a nonempty `result.md`. Skills whose case needs a concrete local effect or a disposable fixture override it: `fix-ci`, `babysit`, and `make-pr-easy-to-review` run against a local fixture with no remote and assert a local effect, `principle-never-block-on-the-human` must edit `sum.mjs`, so a report that only waits for approval fails, and `automate-me` writes its draft into the fixture instead of the daily skills folder. `setup-pstack` uses the setup recipe. `project-skill` runs from the candidate checkout and asserts the skill load, the doctor command, and the canonical-path `doctor.json`. +`recipes.ts` gives every `skill-invocation:` one default case: a disposable Git fixture holding a small task in `TASK.md`, a native invocation (`/pstack:` in Claude, the Skill tool for principle leaves, `$pstack:` in Codex), and an instruction to write the result to `result.md` in the fixture. It asserts that the native record shows the `pstack:` skill load and that the session wrote a nonempty `result.md`. Skills whose case needs a concrete local effect or a disposable fixture override it: `fix-ci`, `babysit`, and `make-pr-easy-to-review` run against a local fixture with no remote and assert a local effect, `principle-never-block-on-the-human`'s repair must make the fixture's unchanged `sum.test.mjs` pass, which the verifier runs in the candidate environment, so a report that only waits for approval, or a failed repair, fails, and `automate-me` writes its draft into the fixture instead of the daily skills folder. `setup-pstack` uses the setup recipe. `project-skill` runs from the candidate checkout and asserts the skill load, the doctor command, and the canonical-path `doctor.json`. `reflect` has two cases in each harness that exercise step 1 only, against run-owned synthetic transcripts with metadata before the opening user message. The prompts name the opening prompt and, for Codex, the known chat ID and a fixture-only `CODEX_HOME`, or, for Claude, the fixture's per-project directory. They do not name the resolver or the transcript file, so the installed active-transcript mapping must lead the session there. In `locate-transcript` the transcript's opener matches. The case asserts the opening-message metadata and that no digest was written. In `digest-fallback` the opener differs, and the case asserts a nonblank digest of at most 2,000 words. Each run uses a per-run nonce that no prompt contains. For Codex, it is in the fixture rollout's file name. A completed command must name the installed resolver's launcher or script by absolute path, and its harness-captured output must parse to exactly the resolver JSON for this run, key order aside. Failed or pending commands and output from another fixture or run do not count, and a report file the model writes is never the Codex execution evidence. For Claude, the nonce is the opening user record's `uuid`, which only the transcript holds. In both cases the report must equal the expected JSON exactly, including that `uuid`: under `opening_user_message` when the opener matches, and beside the fallback status when it does not. A report without the `uuid`, or with another run's, does not count. These cases do not exercise reviewer fan-out, synthesis, or skill edits. diff --git a/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts b/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts index 6233d1d6..8880d042 100644 --- a/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts @@ -20,19 +20,40 @@ async function waitingSession(harness: Harness): Promise<{ f: Fixture; c: Case; return { f, c, record }; } +const fixed = 'export function sum(a, b) {\n return a + b;\n}\n'; + test.each(['claude', 'codex'] as const)('%s never-block case rejects a waiting-only report', async harness => { const { f, c, record } = await waitingSession(harness); - await expect(c.assert(record, f)).rejects.toThrow('sum-not-edited:sum.mjs'); + await expect(c.assert(record, f)).rejects.toThrow('test-failed:sum.test.mjs'); +}); + +test.each(['claude', 'codex'] as const)('%s never-block case rejects a repair that leaves the test failing', async harness => { + const { f, c, record } = await waitingSession(harness); + await writeFile(join(f.dir, 'sum.mjs'), 'export function sum(a, b) {\n return a + b - 1;\n}\n'); + await expect(c.assert(record, f)).rejects.toThrow('test-failed:sum.test.mjs'); +}); + +test.each(['claude', 'codex'] as const)('%s never-block case rejects a repair with a syntax error', async harness => { + const { f, c, record } = await waitingSession(harness); + await writeFile(join(f.dir, 'sum.mjs'), 'export function sum(a, b) {\n return a +;\n}\n'); + await expect(c.assert(record, f)).rejects.toThrow('test-failed:sum.test.mjs'); +}); + +test.each(['claude', 'codex'] as const)('%s never-block case rejects an edited test', async harness => { + const { f, c, record } = await waitingSession(harness); + await writeFile(join(f.dir, 'sum.mjs'), fixed); + await writeFile(join(f.dir, 'sum.test.mjs'), "import { sum } from './sum.mjs';\nsum(2, 3);\n"); + await expect(c.assert(record, f)).rejects.toThrow('test-changed:sum.test.mjs'); }); -test.each(['claude', 'codex'] as const)('%s never-block case passes once sum.mjs is edited', async harness => { +test.each(['claude', 'codex'] as const)('%s never-block case passes once the unchanged test passes', async harness => { const { f, c, record } = await waitingSession(harness); - await writeFile(join(f.dir, 'sum.mjs'), 'export function sum(a, b) {\n return a + b;\n}\n'); - expect((await c.assert(record, f)).assertions).toContain('edited:sum.mjs'); + await writeFile(join(f.dir, 'sum.mjs'), fixed); + expect((await c.assert(record, f)).assertions).toContain('test-passed:sum.test.mjs'); }); -test.each(['claude', 'codex'] as const)('%s never-block case accepts an edit written b + a', async harness => { +test.each(['claude', 'codex'] as const)('%s never-block case accepts a repair written b + a', async harness => { const { f, c, record } = await waitingSession(harness); await writeFile(join(f.dir, 'sum.mjs'), 'export function sum(a, b) {\n return b + a;\n}\n'); - expect((await c.assert(record, f)).assertions).toContain('edited:sum.mjs'); + expect((await c.assert(record, f)).assertions).toContain('test-passed:sum.test.mjs'); }); From 700fe15351aa3d6e094bbf696f9e792faf0d37b1 Mon Sep 17 00:00:00 2001 From: Eric Litman Date: Tue, 20 Oct 2026 23:46:36 -0400 Subject: [PATCH 4/4] fix(verifier): recognize the never-block repair without running it (#225) sumTestPassed ran the session's sum.mjs in the verifier parent, outside Codex's sandbox. The never-block case now recognizes the repair from the whole sum.mjs source (return a + b or b + a on the return line, optionally parenthesized), requires the fixture test unchanged, and executes neither file. Co-authored-by: andrewfree <507412+andrewfree@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .../verify-open-pstack/features/recipes.ts | 11 +-- .../features/skill-invocation.md | 2 +- .../scripts/skill-invocation.test.ts | 71 ++++++++++--------- 3 files changed, 47 insertions(+), 37 deletions(-) diff --git a/.claude/skills/verify-open-pstack/features/recipes.ts b/.claude/skills/verify-open-pstack/features/recipes.ts index d1f893ef..f9ec4361 100644 --- a/.claude/skills/verify-open-pstack/features/recipes.ts +++ b/.claude/skills/verify-open-pstack/features/recipes.ts @@ -130,16 +130,19 @@ async function ciFixed(_record: SessionRecord, f: Fixture): Promise { return { assertions: ['fixed:sum.mjs'], files: [join(f.dir, 'sum.mjs')] }; } -async function sumTestPassed(_record: SessionRecord, f: Fixture): Promise { +const SUM_OPERANDS = String.raw`(?:a[ \t]*\+[ \t]*b|b[ \t]*\+[ \t]*a)`; +const RECOGNIZED_SUM = new RegExp(String.raw`^export function sum\(a, b\) \{\n[ \t]*return[ \t]+(?:${SUM_OPERANDS}|\([ \t]*${SUM_OPERANDS}[ \t]*\))[ \t]*;?[ \t]*\n\}\n?$`); + +async function sumRepairRecognized(_record: SessionRecord, f: Fixture): Promise { if (await readFile(join(f.dir, 'sum.test.mjs'), 'utf8') !== TASK['sum.test.mjs']) throw new Error('test-changed:sum.test.mjs'); - await f.run(['bun', 'sum.test.mjs'], { cwd: f.dir }).catch(() => { throw new Error('test-failed:sum.test.mjs'); }); - return { assertions: ['test-passed:sum.test.mjs'], files: [join(f.dir, 'sum.mjs'), join(f.dir, 'sum.test.mjs')] }; + if (!RECOGNIZED_SUM.test(await readFile(join(f.dir, 'sum.mjs'), 'utf8'))) throw new Error('repair-not-recognized:sum.mjs'); + return { assertions: ['recognized-repair:sum.mjs', 'test-unchanged:sum.test.mjs'], files: [join(f.dir, 'sum.mjs'), join(f.dir, 'sum.test.mjs')] }; } const overrides: Record = { 'fix-ci': skillCase('fix-ci', 'There is no pull request or remote. The failing CI check is the local command `sh ci.sh` in $DIR. Fix it locally and do not push.', ciFixed), babysit: skillCase('babysit', 'The pull request is the local `main` branch of the repository at $DIR; there is no remote. Its one failing check is the local command `sh ci.sh`. Do exactly one babysit pass: fix the failing check locally, push nothing, change nothing remote, then stop instead of watching.', ciFixed), - 'principle-never-block-on-the-human': skillCase('principle-never-block-on-the-human', 'The bug in $DIR/TASK.md is a reversible local change. Apply the skill: implement the smallest fix in sum.mjs and verify it with bun sum.test.mjs.', sumTestPassed), + 'principle-never-block-on-the-human': skillCase('principle-never-block-on-the-human', 'The bug in $DIR/TASK.md is a reversible local change. Apply the skill: implement the smallest fix in sum.mjs and verify it with bun sum.test.mjs.', sumRepairRecognized), 'make-pr-easy-to-review': skillCase('make-pr-easy-to-review', 'The pull request is the local branch `feature` against `main` in $DIR; there is no remote. Do not rewrite history or push; write reviewer guidance only.', async (_record, f) => { if ((await git(f, 'rev-parse', 'feature')).trim() !== (await readFile(`${f.dir}.head`, 'utf8')).trim()) throw new Error('history-rewritten:feature'); return { assertions: ['history-unchanged'], files: [] }; diff --git a/.claude/skills/verify-open-pstack/features/skill-invocation.md b/.claude/skills/verify-open-pstack/features/skill-invocation.md index d5e2f54c..ecb6e541 100644 --- a/.claude/skills/verify-open-pstack/features/skill-invocation.md +++ b/.claude/skills/verify-open-pstack/features/skill-invocation.md @@ -22,6 +22,6 @@ A generic prompt about a skill is not native invocation evidence. A list of disc ## Headless -`recipes.ts` gives every `skill-invocation:` one default case: a disposable Git fixture holding a small task in `TASK.md`, a native invocation (`/pstack:` in Claude, the Skill tool for principle leaves, `$pstack:` in Codex), and an instruction to write the result to `result.md` in the fixture. It asserts that the native record shows the `pstack:` skill load and that the session wrote a nonempty `result.md`. Skills whose case needs a concrete local effect or a disposable fixture override it: `fix-ci`, `babysit`, and `make-pr-easy-to-review` run against a local fixture with no remote and assert a local effect, `principle-never-block-on-the-human`'s repair must make the fixture's unchanged `sum.test.mjs` pass, which the verifier runs in the candidate environment, so a report that only waits for approval, or a failed repair, fails, and `automate-me` writes its draft into the fixture instead of the daily skills folder. `setup-pstack` uses the setup recipe. `project-skill` runs from the candidate checkout and asserts the skill load, the doctor command, and the canonical-path `doctor.json`. +`recipes.ts` gives every `skill-invocation:` one default case: a disposable Git fixture holding a small task in `TASK.md`, a native invocation (`/pstack:` in Claude, the Skill tool for principle leaves, `$pstack:` in Codex), and an instruction to write the result to `result.md` in the fixture. It asserts that the native record shows the `pstack:` skill load and that the session wrote a nonempty `result.md`. Skills whose case needs a concrete local effect or a disposable fixture override it: `fix-ci`, `babysit`, and `make-pr-easy-to-review` run against a local fixture with no remote and assert a local effect, `principle-never-block-on-the-human`'s repair must leave `sum.mjs` as a recognized form of the add function (`return a + b` or `return b + a`, optionally parenthesized, with or without a semicolon) and the fixture's `sum.test.mjs` unchanged, and the verifier executes neither file, so a report that only waits for approval, or a failed repair, fails, and `automate-me` writes its draft into the fixture instead of the daily skills folder. `setup-pstack` uses the setup recipe. `project-skill` runs from the candidate checkout and asserts the skill load, the doctor command, and the canonical-path `doctor.json`. `reflect` has two cases in each harness that exercise step 1 only, against run-owned synthetic transcripts with metadata before the opening user message. The prompts name the opening prompt and, for Codex, the known chat ID and a fixture-only `CODEX_HOME`, or, for Claude, the fixture's per-project directory. They do not name the resolver or the transcript file, so the installed active-transcript mapping must lead the session there. In `locate-transcript` the transcript's opener matches. The case asserts the opening-message metadata and that no digest was written. In `digest-fallback` the opener differs, and the case asserts a nonblank digest of at most 2,000 words. Each run uses a per-run nonce that no prompt contains. For Codex, it is in the fixture rollout's file name. A completed command must name the installed resolver's launcher or script by absolute path, and its harness-captured output must parse to exactly the resolver JSON for this run, key order aside. Failed or pending commands and output from another fixture or run do not count, and a report file the model writes is never the Codex execution evidence. For Claude, the nonce is the opening user record's `uuid`, which only the transcript holds. In both cases the report must equal the expected JSON exactly, including that `uuid`: under `opening_user_message` when the opener matches, and beside the fallback status when it does not. A report without the `uuid`, or with another run's, does not count. These cases do not exercise reviewer fan-out, synthesis, or skill edits. diff --git a/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts b/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts index 8880d042..8a187ffb 100644 --- a/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts @@ -1,4 +1,4 @@ -import { afterEach, expect, test } from 'bun:test'; +import { afterEach, describe, expect, test } from 'bun:test'; import { mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -20,40 +20,47 @@ async function waitingSession(harness: Harness): Promise<{ f: Fixture; c: Case; return { f, c, record }; } -const fixed = 'export function sum(a, b) {\n return a + b;\n}\n'; +const harnesses: Harness[] = ['claude', 'codex']; +const sumWith = (body: string) => `export function sum(a, b) {\n${body}\n}\n`; -test.each(['claude', 'codex'] as const)('%s never-block case rejects a waiting-only report', async harness => { - const { f, c, record } = await waitingSession(harness); - await expect(c.assert(record, f)).rejects.toThrow('test-failed:sum.test.mjs'); -}); - -test.each(['claude', 'codex'] as const)('%s never-block case rejects a repair that leaves the test failing', async harness => { - const { f, c, record } = await waitingSession(harness); - await writeFile(join(f.dir, 'sum.mjs'), 'export function sum(a, b) {\n return a + b - 1;\n}\n'); - await expect(c.assert(record, f)).rejects.toThrow('test-failed:sum.test.mjs'); -}); +const recognized = [ + ' return a + b;', + ' return a + b', + ' return b + a;', + ' return (a + b);', + ' return ( b+a ) ;', + '\treturn a+b;', +]; -test.each(['claude', 'codex'] as const)('%s never-block case rejects a repair with a syntax error', async harness => { - const { f, c, record } = await waitingSession(harness); - await writeFile(join(f.dir, 'sum.mjs'), 'export function sum(a, b) {\n return a +;\n}\n'); - await expect(c.assert(record, f)).rejects.toThrow('test-failed:sum.test.mjs'); -}); +const unrecognized = [ + ' return a - b;', + ' return a + b - 1;', + ' return a +;', + ' return (a + b;', + ' return a + b; // fixed', + ' return a + b; x();', + ' return\n a + b;', + ' return a + b;\n return 0;', +]; -test.each(['claude', 'codex'] as const)('%s never-block case rejects an edited test', async harness => { - const { f, c, record } = await waitingSession(harness); - await writeFile(join(f.dir, 'sum.mjs'), fixed); - await writeFile(join(f.dir, 'sum.test.mjs'), "import { sum } from './sum.mjs';\nsum(2, 3);\n"); - await expect(c.assert(record, f)).rejects.toThrow('test-changed:sum.test.mjs'); -}); +describe.each(harnesses)('%s never-block case', harness => { + test.each(recognized)('accepts %j and the fixture test passes on it', async body => { + const { f, c, record } = await waitingSession(harness); + await writeFile(join(f.dir, 'sum.mjs'), sumWith(body)); + expect((await c.assert(record, f)).assertions).toEqual(expect.arrayContaining(['recognized-repair:sum.mjs', 'test-unchanged:sum.test.mjs'])); + await command(['bun', 'sum.test.mjs'], { cwd: f.dir }); + }); -test.each(['claude', 'codex'] as const)('%s never-block case passes once the unchanged test passes', async harness => { - const { f, c, record } = await waitingSession(harness); - await writeFile(join(f.dir, 'sum.mjs'), fixed); - expect((await c.assert(record, f)).assertions).toContain('test-passed:sum.test.mjs'); -}); + test.each(unrecognized)('rejects %j', async body => { + const { f, c, record } = await waitingSession(harness); + await writeFile(join(f.dir, 'sum.mjs'), sumWith(body)); + await expect(c.assert(record, f)).rejects.toThrow('repair-not-recognized:sum.mjs'); + }); -test.each(['claude', 'codex'] as const)('%s never-block case accepts a repair written b + a', async harness => { - const { f, c, record } = await waitingSession(harness); - await writeFile(join(f.dir, 'sum.mjs'), 'export function sum(a, b) {\n return b + a;\n}\n'); - expect((await c.assert(record, f)).assertions).toContain('test-passed:sum.test.mjs'); + test('rejects an edited test', async () => { + const { f, c, record } = await waitingSession(harness); + await writeFile(join(f.dir, 'sum.mjs'), sumWith(' return a + b;')); + await writeFile(join(f.dir, 'sum.test.mjs'), "import { sum } from './sum.mjs';\nsum(2, 3);\n"); + await expect(c.assert(record, f)).rejects.toThrow('test-changed:sum.test.mjs'); + }); });