Bound predict-audit ownership-walk + rule-sweep harnesses; consolidate unverified findings (#1088)

0xaslan · claude · web-flow · commit 177e62b43e7c · 2026-06-25T14:36:38.000-04:00
* Document Propbook Block Scholes feed split * Bound predict-audit ownership-walk + rule-sweep by construction; consolidate unverified findings Sibling half of the predict-audit harness rework whose orchestrator + SKILL.md half already landed in main via the reference-tick admission PR. Read-only on packages/* — audit tooling only. ownership-walk.workflow.js, rule-sweep.workflow.js: - Bound by construction: MAX_ROUNDS default 12->3 (ownership-walk) and 10->3 (rule-sweep), add verifyCap (60), severity-gated verify (cleanup-tier reported raw, no subagent). Agents <= maxRounds*units + verifyCap, so neither can run away to the agent cap. - Each now emits an `unverified` array (Info/Low/cleanup triaged out of verify). consolidate.py: - Collect the new `unverified` key from every harness: record it under its own key in findings.json (so track.py never floods the tracker with low-priority noise), keep it in the DROPPED-0 accounting (honest, no silent drop), and surface it in an Unverified report section. Re-syncs the consolidator with the already-merged orchestrator, which emits `unverified`. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * Untrack docs/superpowers spec (tracked despite /docs/ gitignore) Committed in 219a910 and left tracked, so .gitignore (/docs/, /docs/superpowers/) never applied — tracked files bypass ignore rules, which is why these specs kept reappearing in PRs. git rm --cached untracks it (local copy kept); the existing rule now keeps it out. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
diff --git a/.claude/skills/predict-audit/consolidate.py b/.claude/skills/predict-audit/consolidate.py
@@ -85,6 +85,10 @@ def collect(res, harness):
     promoted = res.get('promoted', []) or []
     for f in promoted:
         r = norm(f, harness, 'open'); r['verdict'] = 'promoted-unverified'; out.append(r)  # not panel-verified
+    # Info/Low/cleanup findings the orchestrator triaged out of the verify panel: recorded (so they are NOT
+    # silently dropped + the accounting stays honest), but kept OUT of the open tracker — raw, low-priority.
+    for f in res.get('unverified', []) or []:
+        out.append(norm(f, harness, 'unverified'))
     return out, len(promoted)
 
 def main():
@@ -136,12 +140,13 @@ def main():
             f['also'] = []; seen[k] = f; dedup_open.append(f)
     settled = [x for x in all_findings if x['status'] == 'settled']
     refuted = [x for x in all_findings if x['status'] == 'refuted']
+    unverified = [x for x in all_findings if x['status'] == 'unverified']
 
-    # ACCOUNTING: every input finding is either a unique open, a merge, a settled, or a refuted.
-    accounted = len(dedup_open) + merges + len(settled) + len(refuted)
+    # ACCOUNTING: every input finding is either a unique open, a merge, a settled, a refuted, or an unverified.
+    accounted = len(dedup_open) + merges + len(settled) + len(refuted) + len(unverified)
     dropped = total_in - accounted
     acct = (f"ACCOUNTING — parsed {total_in} | open {len(dedup_open)} (+{merges} dup-merges) | "
-            f"settled {len(settled)} | refuted {len(refuted)} | DROPPED {dropped} "
+            f"settled {len(settled)} | refuted {len(refuted)} | unverified {len(unverified)} | DROPPED {dropped} "
             + ("✅ 0 dropped" if dropped == 0 else "⚠️ MISMATCH — investigate"))
     if parse_failures:
         acct += f"\n⛔ {len(parse_failures)} INPUT FILE(S) FAILED TO PARSE (findings NOT in this report): " \
@@ -176,18 +181,25 @@ def main():
     L.append("\n## Refuted (second-guess these — a wrong refutation hides a real bug)\n")
     for f in refuted:
         L.append(f"- [{f['harness']}/{f['source']}] {f['location']} — {f['title']} → {f['why']}")
+    L.append("\n## Unverified (raw finder output — Info/Low/cleanup, NOT panel-verified; triage manually)\n")
+    if not unverified: L.append("_(none)_")
+    for f in sorted(unverified, key=lambda f: -SEV.get(f['severity'].lower(), 1)):
+        sref = f" → {f['settled_ref']}" if f.get('settled_ref') else ""
+        L.append(f"- [{f['severity'] or '?'}] {f['title']} — {f['location']} ({f['harness']}/{f['source']}){sref}")
     L.append("\n## Coverage — what was examined and (esp.) NOT examined\n")
     for h, lane, cov, top3 in coverage_blocks:
         L.append(f"- **{h}/{lane}**: {cov}")
         if top3: L.append("  - top3: " + " | ".join(top3))
 
     rpt = os.path.join(out_dir, 'consolidated-report.md')
     open(rpt, 'w', encoding='utf-8').write("\n".join(L) + "\n")
-    # findings.json (open findings with stable ids) feeds track.py, the live OPEN-ITEMS.md tracker.
-    json.dump({'open': dedup_open, 'settled': settled, 'refuted': refuted},
+    # findings.json (open findings with stable ids) feeds track.py, the live OPEN-ITEMS.md tracker. `unverified`
+    # is recorded here too but under its OWN key, so track.py (which keys on `open`) never floods the tracker
+    # with Info/Low/cleanup noise.
+    json.dump({'open': dedup_open, 'settled': settled, 'refuted': refuted, 'unverified': unverified},
               open(os.path.join(out_dir, 'findings.json'), 'w', encoding='utf-8'), indent=1)
     print(acct)
-    print(f"wrote {rpt} ({len(dedup_open)} open, {len(settled)} settled, {len(refuted)} refuted)")
+    print(f"wrote {rpt} ({len(dedup_open)} open, {len(settled)} settled, {len(refuted)} refuted, {len(unverified)} unverified)")
     sys.exit(0 if (dropped == 0 and not parse_failures and not errored_harnesses) else 1)
 
 if __name__ == '__main__':
diff --git a/.claude/skills/predict-audit/ownership-walk.workflow.js b/.claude/skills/predict-audit/ownership-walk.workflow.js
@@ -10,7 +10,15 @@
 //   units?:  string[],   // subset of MAP_UNIT keys to walk (default: all) — use to scope cost
 //   groundTruth?: string,
 //   maxViolations?: number, // cap per check unit (default 10)
+//   dryRounds?: number,     // stop after this many no-new-violation rounds (default 2)
+//   maxRounds?: number,     // hard round cap (default 3)
+//   verifyCap?: number,     // max violations sent to the verifier (default 60)
 // }
+// COST IS BOUNDED BY CONSTRUCTION: agents <= units (map) + maxRounds*checkUnits (check) + verifyCap (verify),
+// so a run cannot hit the 1000-agent cap. The `budget` global (a "+NNNm" turn directive) is only an optional
+// early-stop and often does NOT propagate into a background workflow — never rely on it. Verify is
+// SEVERITY-GATED: 'cleanup'-tier violations are reported RAW (unverified, no subagent); 'high'/'correctness'
+// get one verifier each.
 // Subagents are READ-ONLY on source; no sui build/test or localnet (watchdog) — reason from source + grep + git.
 
 export const meta = {
@@ -19,7 +27,7 @@ export const meta = {
   phases: [
     { title: 'Map', detail: 'build the responsibility map per subsystem (BARRIER — checks need the whole map)' },
     { title: 'Check', detail: 'per-module fan-out: walk every function vs R1-R7 + the map' },
-    { title: 'Verify', detail: 'adversarially refute each violation vs the map + DECISION_JOURNAL' },
+    { title: 'Verify', detail: 'severity-gated: refute each high/correctness violation vs the map + DECISION_JOURNAL; cleanup-tier reported raw' },
   ],
 }
 
@@ -151,7 +159,8 @@ function composedContext(entry) {
 // told what's already found FOR THAT MODULE so it hunts new ground), union new violations, until K dry
 // rounds or the budget floor. Auto-retries flaky units (a failed unit is an empty round that re-runs).
 const DRY_TARGET = A.dryRounds || 2
-const MAX_ROUNDS = A.maxRounds || 12
+const MAX_ROUNDS = A.maxRounds || 3
+const VERIFY_CAP = A.verifyCap || 60
 const RESERVE = (budget && budget.total) ? Math.max(4_000_000, Math.floor(budget.total * 0.3)) : 4_000_000
 function budgetLeft() { return budget && typeof budget.remaining === 'function' ? budget.remaining() : Infinity }
 // strip line numbers from node so a same violation at a shifted line still dedups across rounds (else the
@@ -193,8 +202,16 @@ while (dry < DRY_TARGET && round < MAX_ROUNDS && budgetLeft() > RESERVE) {
 }
 log(`Check converged after ${round} rounds (${dry} dry): ${candidates.length} unique candidate violations`)
 
+// SEVERITY-GATED + CAPPED: only 'high'/'correctness' violations get a verifier; 'cleanup'-tier violations are
+// reported RAW (unverified). Capped at VERIFY_CAP by severity so the agent count stays bounded.
 phase('Verify')
-const all = (await parallel(candidates.map((v, vi) => () => agent(verifyPromptV(v),
+const sevRankV = { high: 3, correctness: 2, cleanup: 1 }
+const verifyAll = candidates.filter(v => v.severity !== 'cleanup').sort((a, b) => (sevRankV[b.severity] || 0) - (sevRankV[a.severity] || 0))
+const toVerify = verifyAll.slice(0, VERIFY_CAP)
+const verifyOverflow = verifyAll.slice(VERIFY_CAP)   // beyond the cap -> reported unverified (logged, never silently dropped)
+const unverifiedV = candidates.filter(v => v.severity === 'cleanup').concat(verifyOverflow)
+if (verifyOverflow.length) log(`Verify cap ${VERIFY_CAP} hit: ${verifyOverflow.length} violation(s) reported UNVERIFIED — raise verifyCap to verify them all`)
+const all = (await parallel(toVerify.map((v, vi) => () => agent(verifyPromptV(v),
   { schema: VERDICT_SCHEMA, effort: 'high', phase: 'Verify', label: `verify:${v.module}:${vi}` })
   .then(verdict => ({ ...v, verdict }))))).filter(Boolean)
 const confirmed = all.filter(x => x.verdict && (x.verdict.verdict === 'confirmed' || x.verdict.verdict === 'uncertain'))
@@ -210,9 +227,10 @@ all.filter(x => x.verdict && (x.verdict.classification === 'update-rule' || x.ve
 log(`Modules: ${allModules.length} | check units: ${CHECK_UNITS.length} | rounds: ${round} | violations: ${all.length} | confirmed: ${confirmed.length} | settled: ${settledOut.length} | refuted: ${refutedOut.length}`)
 
 return {
-  summary: { modules: allModules.length, checkUnits: CHECK_UNITS.length, unmapped_units: unmappedUnits, rounds: round, violations: all.length, confirmed: confirmed.length, settled: settledOut.length, refuted: refutedOut.length },
+  summary: { modules: allModules.length, checkUnits: CHECK_UNITS.length, unmapped_units: unmappedUnits, rounds: round, violations: all.length, confirmed: confirmed.length, settled: settledOut.length, refuted: refutedOut.length, unverified: unverifiedV.length },
   responsibility_map: allModules.map(m => ({ module: m.module, role: m.role, owns: m.owns, composes: m.composes })),
   coverage: Object.keys(coverageByModule).map(m => ({ lane: m, coverage: coverageByModule[m] })),
+  unverified: unverifiedV.map(x => ({ rule_family: x.rule_family, node: x.node, claim: x.claim, severity: x.severity, settled_ref: x.settled_ref, recommendation: x.recommendation, status: 'unverified' })),
   confirmed: confirmed.map(x => ({ rule_family: x.rule_family, node: x.node, claim: x.claim, expected_owner: x.expected_owner, actual_owner: x.actual_owner, severity: (x.verdict && x.verdict.adjusted_severity) || x.severity, status: (x.verdict && x.verdict.verdict) || 'confirmed', classification: x.verdict && x.verdict.classification, settled_ref: x.settled_ref, recommendation: x.recommendation, data_flow: x.data_flow, proof: x.verdict && x.verdict.evidence })),
   settled: settledOut.map(x => ({ rule_family: x.rule_family, node: x.node, claim: x.claim, settled_ref: (x.verdict && x.verdict.evidence) || x.settled_ref })),
   refuted: refutedOut.map(x => ({ rule_family: x.rule_family, node: x.node, claim: x.claim, why: x.verdict && x.verdict.reasoning })),
diff --git a/.claude/skills/predict-audit/rule-sweep.workflow.js b/.claude/skills/predict-audit/rule-sweep.workflow.js
@@ -8,7 +8,12 @@
 // Before Mutate", and the leaf-self-consistency half of 9) are NOT here — they need deep per-module context
 // and live in ownership-walk.workflow.js (R1-R7). Do not duplicate them.
 //
-// args = { rules?: string[] (subset of family keys), maxFindings?: number, groundTruth?: string }
+// args = { rules?: string[] (subset of family keys), maxFindings?: number, groundTruth?: string,
+//          dryRounds?: number (default 2), maxRounds?: number (default 3), verifyCap?: number (default 60) }
+// COST IS BOUNDED BY CONSTRUCTION: agents <= maxRounds*families (sweep) + verifyCap (verify), so a run cannot
+// hit the 1000-agent cap. The `budget` global (a "+NNNm" turn directive) is only an optional early-stop and
+// often does NOT propagate into a background workflow — never rely on it. Verify is SEVERITY-GATED: low/cleanup
+// hygiene findings are reported RAW (unverified, no subagent); high/medium (or fix-code) get one verifier each.
 // Subagents READ-ONLY; no sui build/test or localnet (watchdog) — the main loop runs the compiler in the
 // parent-reconciliation pass (rule-auditor's build/test step).
 
@@ -17,7 +22,7 @@ export const meta = {
   description: 'Per-rule sweep of the mechanical/local repo rules across the Predict packages (refreshed rule-auditor): sweep -> verify/classify',
   phases: [
     { title: 'Sweep', detail: 'one agent per rule family sweeps every relevant module for that rule' },
-    { title: 'Verify', detail: 'refute / classify each violation (fix-code / update-rule / design-decision / false-positive)' },
+    { title: 'Verify', detail: 'severity-gated: refute/classify each high/medium finding; low/cleanup hygiene reported raw' },
   ],
 }
 
@@ -114,7 +119,8 @@ const VERDICT_SCHEMA = {
 // MAXIMAL MODE: loop-until-dry SWEEP. Re-sweep each rule family across rounds (each told what's already
 // found for that family so it hunts new sites), union new findings, until K dry rounds or the budget floor.
 const DRY_TARGET = A.dryRounds || 2
-const MAX_ROUNDS = A.maxRounds || 10
+const MAX_ROUNDS = A.maxRounds || 3
+const VERIFY_CAP = A.verifyCap || 60
 const RESERVE = (budget && budget.total) ? Math.max(3_000_000, Math.floor(budget.total * 0.3)) : 3_000_000
 function budgetLeft() { return budget && typeof budget.remaining === 'function' ? budget.remaining() : Infinity }
 // strip line numbers (so a shifted-line same violation dedups) but KEEP a claim digest, so two DISTINCT
@@ -151,8 +157,18 @@ while (dry < DRY_TARGET && round < MAX_ROUNDS && budgetLeft() > RESERVE) {
 }
 log(`Sweep converged after ${round} rounds (${dry} dry): ${candidates.length} unique candidate findings`)
 
+// SEVERITY-GATED + CAPPED: only high/medium (or fix-code) findings get a verifier; low/cleanup hygiene is
+// reported RAW (unverified). Capped at VERIFY_CAP by severity so the agent count stays bounded.
 phase('Verify')
-const all = (await parallel(candidates.map((f, fi) => () => agent(
+const sevRankF = { high: 4, medium: 3, low: 2, cleanup: 1 }
+const effSev = f => f.severity || (f.classification === 'fix-code' ? 'medium' : 'low')
+const ruleShouldVerify = f => { const s = effSev(f); return s === 'high' || s === 'medium' }
+const verifyAll = candidates.filter(ruleShouldVerify).sort((a, b) => (sevRankF[effSev(b)] || 0) - (sevRankF[effSev(a)] || 0))
+const toVerify = verifyAll.slice(0, VERIFY_CAP)
+const verifyOverflow = verifyAll.slice(VERIFY_CAP)   // beyond the cap -> reported unverified (logged, never silently dropped)
+const unverifiedF = candidates.filter(f => !ruleShouldVerify(f)).concat(verifyOverflow)
+if (verifyOverflow.length) log(`Verify cap ${VERIFY_CAP} hit: ${verifyOverflow.length} finding(s) reported UNVERIFIED — raise verifyCap to verify them all`)
+const all = (await parallel(toVerify.map((f, fi) => () => agent(
   `${PRELUDE}\n\nADVERSARIALLY VERIFY this single rule-sweep finding (rule family ${f.rule_family}). Read the cited code; decide: confirmed (real violation) / refuted (not a violation, or the claim is wrong) / settled (matches a DECISION_JOURNAL/AGENTS decision — cite it). Then classify: fix-code / update-rule (defensible → narrowest exception) / design-decision / false-positive. Be skeptical; mechanical rules have many intentional exceptions (getters retained by D003, public APIs needed for PTBs, disabled test suites).\n\nFINDING:\n${JSON.stringify(f, null, 2)}`,
   { schema: VERDICT_SCHEMA, effort: 'high', phase: 'Verify', label: `verify:${f.rule_family}:${fi}` })
   .then(verdict => ({ ...f, verdict }))))).filter(Boolean)
@@ -165,9 +181,10 @@ function shape(x) {
   return { rule_family: x.rule_family, severity: x.severity || (x.classification === 'fix-code' ? 'medium' : 'low'), location: x.location, claim: x.claim, classification: (x.verdict && x.verdict.classification) || x.classification, recommendation: x.recommendation, evidence: x.verdict && x.verdict.evidence }
 }
 return {
-  summary: { rule_families: FAMILIES.length, rounds: round, findings: all.length, confirmed: confirmed.length, settled: settledOut.length, refuted: refutedOut.length },
+  summary: { rule_families: FAMILIES.length, rounds: round, findings: all.length, confirmed: confirmed.length, settled: settledOut.length, refuted: refutedOut.length, unverified: unverifiedF.length },
   coverage: Object.keys(coverageByFamily).map(rf => ({ lane: rf, coverage: coverageByFamily[rf] })),
   confirmed: confirmed.map(shape),
   settled: settledOut.map(shape),
   refuted: refutedOut.map(x => ({ rule_family: x.rule_family, location: x.location, claim: x.claim, why: x.verdict && x.verdict.reasoning })),
+  unverified: unverifiedF.map(x => ({ rule_family: x.rule_family, severity: effSev(x), location: x.location, claim: x.claim, classification: x.classification, recommendation: x.recommendation, status: 'unverified' })),
 }
