diff --git a/.claude/skills/verify-open-pstack/SKILL.md b/.claude/skills/verify-open-pstack/SKILL.md index 46fedffb..4f9dc155 100644 --- a/.claude/skills/verify-open-pstack/SKILL.md +++ b/.claude/skills/verify-open-pstack/SKILL.md @@ -99,6 +99,7 @@ Read receipts as data, never shell input. Do not put credentials in prompts or p - scripts/provenance.test.ts - scripts/provenance.ts - scripts/reflect.test.ts +- scripts/skill-invocation.test.ts - scripts/types.ts - scripts/verify.sh - scripts/verify.test.ts diff --git a/.claude/skills/verify-open-pstack/features/README.md b/.claude/skills/verify-open-pstack/features/README.md index 25053d6e..6103cb01 100644 --- a/.claude/skills/verify-open-pstack/features/README.md +++ b/.claude/skills/verify-open-pstack/features/README.md @@ -16,6 +16,8 @@ The ownership table below says which features a changed path reaches. `SKILL.md` | `project` | Native discovery/invocation of this verifier in both harnesses | | `nonRuntime` | Explicit ordinary repository documentation, policy, non-shipped assets, and unit-test paths; no plugin harness launch | +A change to `features/recipes.ts` can name any feature that has a recipe. + A change to `plugins/pstack/.claude-plugin/plugin.json`, `.claude-plugin/marketplace.json`, or `plugins/pstack/.codex-plugin/plugin.json` is version-only when the merge base and the head hold the same regular file with the same mode, both sides parse as JSON objects, at least one version string changed (top-level `version`, or `plugins[].version` in the marketplace), and nothing else differs. A version-only change reaches only `hook:startup`: the two Claude files in Claude, the Codex manifest in Codex. `.agents/plugins/marketplace.json` has no version and always takes the shared expansion. Asset ownership is consumer-driven, not blanket shared ownership. Inspect the exact candidate's plugin manifests. At present, `plugins/pstack/assets/logo.png` is referenced only by `plugins/pstack/.codex-plugin/plugin.json`, so a logo-only change selects the installed Codex asset surface and does not invent a Claude exercise. New shipped assets must be added to `assets` and must have a real manifest consumer; missing, ambiguous, or unknown consumption fails closed. diff --git a/.claude/skills/verify-open-pstack/features/recipes.ts b/.claude/skills/verify-open-pstack/features/recipes.ts index 8a006e4b..f9ec4361 100644 --- a/.claude/skills/verify-open-pstack/features/recipes.ts +++ b/.claude/skills/verify-open-pstack/features/recipes.ts @@ -130,9 +130,19 @@ async function ciFixed(_record: SessionRecord, f: Fixture): Promise { return { assertions: ['fixed:sum.mjs'], files: [join(f.dir, 'sum.mjs')] }; } +const SUM_OPERANDS = String.raw`(?:a[ \t]*\+[ \t]*b|b[ \t]*\+[ \t]*a)`; +const RECOGNIZED_SUM = new RegExp(String.raw`^export function sum\(a, b\) \{\n[ \t]*return[ \t]+(?:${SUM_OPERANDS}|\([ \t]*${SUM_OPERANDS}[ \t]*\))[ \t]*;?[ \t]*\n\}\n?$`); + +async function sumRepairRecognized(_record: SessionRecord, f: Fixture): Promise { + if (await readFile(join(f.dir, 'sum.test.mjs'), 'utf8') !== TASK['sum.test.mjs']) throw new Error('test-changed:sum.test.mjs'); + if (!RECOGNIZED_SUM.test(await readFile(join(f.dir, 'sum.mjs'), 'utf8'))) throw new Error('repair-not-recognized:sum.mjs'); + return { assertions: ['recognized-repair:sum.mjs', 'test-unchanged:sum.test.mjs'], files: [join(f.dir, 'sum.mjs'), join(f.dir, 'sum.test.mjs')] }; +} + const overrides: Record = { 'fix-ci': skillCase('fix-ci', 'There is no pull request or remote. The failing CI check is the local command `sh ci.sh` in $DIR. Fix it locally and do not push.', ciFixed), babysit: skillCase('babysit', 'The pull request is the local `main` branch of the repository at $DIR; there is no remote. Its one failing check is the local command `sh ci.sh`. Do exactly one babysit pass: fix the failing check locally, push nothing, change nothing remote, then stop instead of watching.', ciFixed), + 'principle-never-block-on-the-human': skillCase('principle-never-block-on-the-human', 'The bug in $DIR/TASK.md is a reversible local change. Apply the skill: implement the smallest fix in sum.mjs and verify it with bun sum.test.mjs.', sumRepairRecognized), 'make-pr-easy-to-review': skillCase('make-pr-easy-to-review', 'The pull request is the local branch `feature` against `main` in $DIR; there is no remote. Do not rewrite history or push; write reviewer guidance only.', async (_record, f) => { if ((await git(f, 'rev-parse', 'feature')).trim() !== (await readFile(`${f.dir}.head`, 'utf8')).trim()) throw new Error('history-rewritten:feature'); return { assertions: ['history-unchanged'], files: [] }; diff --git a/.claude/skills/verify-open-pstack/features/skill-invocation.md b/.claude/skills/verify-open-pstack/features/skill-invocation.md index 672e1b00..ecb6e541 100644 --- a/.claude/skills/verify-open-pstack/features/skill-invocation.md +++ b/.claude/skills/verify-open-pstack/features/skill-invocation.md @@ -22,6 +22,6 @@ A generic prompt about a skill is not native invocation evidence. A list of disc ## Headless -`recipes.ts` gives every `skill-invocation:` one default case: a disposable Git fixture holding a small task in `TASK.md`, a native invocation (`/pstack:` in Claude, the Skill tool for principle leaves, `$pstack:` in Codex), and an instruction to write the result to `result.md` in the fixture. It asserts that the native record shows the `pstack:` skill load and that the session wrote a nonempty `result.md`. Skills whose normal use would push, merge, post, or write daily state override it: `fix-ci`, `babysit`, and `make-pr-easy-to-review` run against a local fixture with no remote and assert a local effect, and `automate-me` writes its draft into the fixture instead of the daily skills folder. `setup-pstack` uses the setup recipe. `project-skill` runs from the candidate checkout and asserts the skill load, the doctor command, and the canonical-path `doctor.json`. +`recipes.ts` gives every `skill-invocation:` one default case: a disposable Git fixture holding a small task in `TASK.md`, a native invocation (`/pstack:` in Claude, the Skill tool for principle leaves, `$pstack:` in Codex), and an instruction to write the result to `result.md` in the fixture. It asserts that the native record shows the `pstack:` skill load and that the session wrote a nonempty `result.md`. Skills whose case needs a concrete local effect or a disposable fixture override it: `fix-ci`, `babysit`, and `make-pr-easy-to-review` run against a local fixture with no remote and assert a local effect, `principle-never-block-on-the-human`'s repair must leave `sum.mjs` as a recognized form of the add function (`return a + b` or `return b + a`, optionally parenthesized, with or without a semicolon) and the fixture's `sum.test.mjs` unchanged, and the verifier executes neither file, so a report that only waits for approval, or a failed repair, fails, and `automate-me` writes its draft into the fixture instead of the daily skills folder. `setup-pstack` uses the setup recipe. `project-skill` runs from the candidate checkout and asserts the skill load, the doctor command, and the canonical-path `doctor.json`. `reflect` has two cases in each harness that exercise step 1 only, against run-owned synthetic transcripts with metadata before the opening user message. The prompts name the opening prompt and, for Codex, the known chat ID and a fixture-only `CODEX_HOME`, or, for Claude, the fixture's per-project directory. They do not name the resolver or the transcript file, so the installed active-transcript mapping must lead the session there. In `locate-transcript` the transcript's opener matches. The case asserts the opening-message metadata and that no digest was written. In `digest-fallback` the opener differs, and the case asserts a nonblank digest of at most 2,000 words. Each run uses a per-run nonce that no prompt contains. For Codex, it is in the fixture rollout's file name. A completed command must name the installed resolver's launcher or script by absolute path, and its harness-captured output must parse to exactly the resolver JSON for this run, key order aside. Failed or pending commands and output from another fixture or run do not count, and a report file the model writes is never the Codex execution evidence. For Claude, the nonce is the opening user record's `uuid`, which only the transcript holds. In both cases the report must equal the expected JSON exactly, including that `uuid`: under `opening_user_message` when the opener matches, and beside the fallback status when it does not. A report without the `uuid`, or with another run's, does not count. These cases do not exercise reviewer fan-out, synthesis, or skill edits. diff --git a/.claude/skills/verify-open-pstack/scripts/core.test.ts b/.claude/skills/verify-open-pstack/scripts/core.test.ts index 16417a7d..455fd0d1 100644 --- a/.claude/skills/verify-open-pstack/scripts/core.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/core.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test'; import { execFileSync } from 'node:child_process'; import { resolve } from 'node:path'; import registryData from '../features/registry.json'; +import { recipe } from '../features/recipes.ts'; import { classify, completeEvidence, matches, newReceipt, requiredFeatures, requiredHarnesses, validateRegistry } from './core.ts'; import { HARNESSES, type Receipt } from './types.ts'; import codexManifest from '../../../../plugins/pstack/.codex-plugin/plugin.json'; @@ -181,6 +182,35 @@ describe('registry and ownership', () => { evidence(r); expect(() => completeEvidence(r)).not.toThrow(); }); + test('the shared recipe implementation reaches every implemented recipe and preserves project proof', () => { + const path = '.claude/skills/verify-open-pstack/features/recipes.ts'; + const expected = [...both('catalog', 'setup', 'runner', 'shipped-tools', 'hook:startup', 'project-skill', + ...registry.skills.map(skill => `skill-invocation:${skill}`)), 'claude:agent-preload:poteto-agent'].sort(); + for (const file of [{ filename: path }, { filename: 'docs/moved.md', previous_filename: path }]) { + const s = classify([file], registry); + expect(s.features).toEqual(expected); + expect(s.skills).toEqual([...registry.skills].sort()); + expect(s.noRuntime).toBe(false); + for (const harness of HARNESSES) { + for (const feature of s.features.filter(f => f.startsWith(`${harness}:`))) { + expect(recipe(feature.slice(harness.length + 1), harness).length).toBeGreaterThan(0); + } + } + } + }); + test('nearby verifier documentation retains only project proof', () => { + for (const filename of ['.claude/skills/verify-open-pstack/features/README.md', + '.claude/skills/verify-open-pstack/features/skill-invocation.md']) { + expect(classify([{ filename }], registry).features).toEqual(both('project-skill')); + } + }); + test('shared recipe expansion survives overlapping catalog ownership without broadening ordinary catalog paths', () => { + const path = '.claude/skills/verify-open-pstack/features/recipes.ts'; + const overlapping = { ...registry, catalog: [...registry.catalog, path], shared: [...registry.shared, path] }; + expect(classify([{ filename: path }], overlapping)).toEqual(classify([{ filename: path }], registry)); + const catalog = registry.catalog[0]!; + expect(classify([{ filename: catalog }], overlapping)).toEqual(classify([{ filename: catalog }], registry)); + }); test('ordinary repository documentation selects no harnesses', () => { const s = classify([{ filename: 'README.md' }, { filename: 'docs/guide.md' }], registry); expect(s.noRuntime).toBe(true); diff --git a/.claude/skills/verify-open-pstack/scripts/core.ts b/.claude/skills/verify-open-pstack/scripts/core.ts index 4ad84523..6cf3c99d 100644 --- a/.claude/skills/verify-open-pstack/scripts/core.ts +++ b/.claude/skills/verify-open-pstack/scripts/core.ts @@ -34,9 +34,11 @@ export function classify(files: ChangedFile[], registry: Registry): Selection { if (path !== 'plugins/pstack/assets/logo.png') throw new Error(`Unmapped asset consumer: ${path}`); features.add('codex:assets:codex'); covered = true; } + const sharedRecipe = path === '.claude/skills/verify-open-pstack/features/recipes.ts'; + if (sharedRecipe) both('catalog'); const catalog = registry.catalog.some(p => matches(path, p)); if (catalog) { both('catalog', 'setup', 'runner'); covered = true; } - if (!catalog && registry.shared.some(p => matches(path, p))) { + if (sharedRecipe || (!catalog && registry.shared.some(p => matches(path, p)))) { registry.skills.forEach(s => skills.add(s)); both('setup', 'runner', 'shipped-tools', 'hook:startup'); features.add('claude:agent-preload:poteto-agent'); diff --git a/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts b/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts new file mode 100644 index 00000000..8a187ffb --- /dev/null +++ b/.claude/skills/verify-open-pstack/scripts/skill-invocation.test.ts @@ -0,0 +1,66 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { recipe, type Case, type Fixture } from '../features/recipes.ts'; +import { command } from './io.ts'; +import type { Harness, SessionRecord } from './types.ts'; + +const roots: string[] = []; +afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +const skill = 'principle-never-block-on-the-human'; + +async function waitingSession(harness: Harness): Promise<{ f: Fixture; c: Case; record: SessionRecord }> { + const dir = await realpath(await mkdtemp(join(tmpdir(), 'pstack-never-block-'))); roots.push(dir); + const f = { harness, dir, workspace: dir, location: dir, configHome: dir, run: command }; + const c = recipe(`skill-invocation:${skill}`, harness)[0]!; + await c.prepare!(f); + const record = { harness, exitCode: 0, started: Date.now() - 1000, skills: [`pstack:${skill}`], commands: [], tools: [], context: [] }; + await writeFile(join(dir, 'result.md'), 'Waiting for you to approve inspection. I did not inspect anything.'); + return { f, c, record }; +} + +const harnesses: Harness[] = ['claude', 'codex']; +const sumWith = (body: string) => `export function sum(a, b) {\n${body}\n}\n`; + +const recognized = [ + ' return a + b;', + ' return a + b', + ' return b + a;', + ' return (a + b);', + ' return ( b+a ) ;', + '\treturn a+b;', +]; + +const unrecognized = [ + ' return a - b;', + ' return a + b - 1;', + ' return a +;', + ' return (a + b;', + ' return a + b; // fixed', + ' return a + b; x();', + ' return\n a + b;', + ' return a + b;\n return 0;', +]; + +describe.each(harnesses)('%s never-block case', harness => { + test.each(recognized)('accepts %j and the fixture test passes on it', async body => { + const { f, c, record } = await waitingSession(harness); + await writeFile(join(f.dir, 'sum.mjs'), sumWith(body)); + expect((await c.assert(record, f)).assertions).toEqual(expect.arrayContaining(['recognized-repair:sum.mjs', 'test-unchanged:sum.test.mjs'])); + await command(['bun', 'sum.test.mjs'], { cwd: f.dir }); + }); + + test.each(unrecognized)('rejects %j', async body => { + const { f, c, record } = await waitingSession(harness); + await writeFile(join(f.dir, 'sum.mjs'), sumWith(body)); + await expect(c.assert(record, f)).rejects.toThrow('repair-not-recognized:sum.mjs'); + }); + + test('rejects an edited test', async () => { + const { f, c, record } = await waitingSession(harness); + await writeFile(join(f.dir, 'sum.mjs'), sumWith(' return a + b;')); + await writeFile(join(f.dir, 'sum.test.mjs'), "import { sum } from './sum.mjs';\nsum(2, 3);\n"); + await expect(c.assert(record, f)).rejects.toThrow('test-changed:sum.test.mjs'); + }); +}); diff --git a/.claude/skills/verify-open-pstack/scripts/verify.test.ts b/.claude/skills/verify-open-pstack/scripts/verify.test.ts index ea2c7c65..ecf07c36 100644 --- a/.claude/skills/verify-open-pstack/scripts/verify.test.ts +++ b/.claude/skills/verify-open-pstack/scripts/verify.test.ts @@ -115,6 +115,28 @@ describe('exact-head evidence publication', () => { expect((await verify(f.options)).status).toBe('success'); expect(f.calls).toContain('prepare'); }); + test('a shared recipe change runs only the named recipe and project proof from the candidate publisher', async () => { + const selected = 'codex:skill-invocation:principle-never-block-on-the-human'; + for (const revision of [REVISION, SHA]) { + const f = fixture(true); + f.options.publisherRevision = revision; + f.options.features = [selected]; + f.options.github.files = async () => [{ filename: '.claude/skills/verify-open-pstack/features/recipes.ts' }]; + if (revision !== SHA) { + await expect(verify(f.options)).rejects.toThrow('rerun from that checkout'); + expect(f.calls.some(call => ['preflight', 'prepare', 'exercise', 'comment'].includes(call) || call.startsWith('status:'))).toBe(false); + } else { + const receipt = await verify(f.options); + expect(receipt.status).toBe('success'); + expect(receipt.runFeatures).toEqual([selected, 'claude:project-skill', 'codex:project-skill']); + expect(receipt.selection.features.length).toBeGreaterThan(receipt.runFeatures.length); + expect(receipt.sessions).toBe(3); + expect(receipt.observations.map(record => `${record.harness}/${record.feature}`).sort()).toEqual([ + 'claude/project-skill', 'codex/project-skill', 'codex/skill-invocation:principle-never-block-on-the-human', + ]); + } + } + }); test('successful verification restores setup and removes Codex run state before any publication', async () => { const f = fixture(true); f.options.driver.cleanup = async () => { f.calls.push('cleanup'); };