From 7cc73e12c34c2cc7bcf98da85f54b212b8d09a4a Mon Sep 17 00:00:00 2001 From: Shan Tang Date: Tue, 17 Nov 2026 14:37:51 +0800 Subject: [PATCH] Add deterministic human Claim review batches --- .github/workflows/model-catalog-audit.yml | 14 + .github/workflows/validate-compatibility.yml | 1 + docs/intelligence-discovery.md | 28 ++ .../build_intelligence_claim_review_batch.py | 258 ++++++++++++++++++ tests/test_intelligence_claim_review_batch.py | 151 ++++++++++ 5 files changed, 452 insertions(+) create mode 100644 scripts/build_intelligence_claim_review_batch.py create mode 100644 tests/test_intelligence_claim_review_batch.py diff --git a/.github/workflows/model-catalog-audit.yml b/.github/workflows/model-catalog-audit.yml index 24ad13f..4d042e2 100644 --- a/.github/workflows/model-catalog-audit.yml +++ b/.github/workflows/model-catalog-audit.yml @@ -150,6 +150,14 @@ jobs: --ledger "data/intelligence-review-decisions.json" \ --output "$RUNNER_TEMP/ai-chip-intelligence-review-state.json" \ --markdown "$RUNNER_TEMP/ai-chip-intelligence-review-aging.md" + - name: Build bounded human Claim review batch + run: | + python3 scripts/build_intelligence_claim_review_batch.py \ + --review-state "$RUNNER_TEMP/ai-chip-intelligence-review-state.json" \ + --batch-size 25 \ + --output "$RUNNER_TEMP/ai-chip-intelligence-claim-review-batch.json" \ + --markdown "$RUNNER_TEMP/ai-chip-intelligence-claim-review-batch.md" \ + --decision-template "$RUNNER_TEMP/ai-chip-intelligence-claim-review-decisions-template.json" - name: Audit end-to-end stage consistency and source health run: | python3 scripts/audit_intelligence_run.py \ @@ -206,6 +214,9 @@ jobs: if [ -f "$RUNNER_TEMP/ai-chip-intelligence-review-aging.md" ]; then cat "$RUNNER_TEMP/ai-chip-intelligence-review-aging.md" >> "$GITHUB_STEP_SUMMARY" fi + if [ -f "$RUNNER_TEMP/ai-chip-intelligence-claim-review-batch.md" ]; then + cat "$RUNNER_TEMP/ai-chip-intelligence-claim-review-batch.md" >> "$GITHUB_STEP_SUMMARY" + fi if [ -f "$RUNNER_TEMP/ai-chip-intelligence-review-backlog-hydration.md" ]; then cat "$RUNNER_TEMP/ai-chip-intelligence-review-backlog-hydration.md" >> "$GITHUB_STEP_SUMMARY" fi @@ -242,6 +253,9 @@ jobs: ${{ runner.temp }}/ai-chip-intelligence-triage.json ${{ runner.temp }}/ai-chip-intelligence-triage.md ${{ runner.temp }}/ai-chip-intelligence-review-state.json + ${{ runner.temp }}/ai-chip-intelligence-claim-review-batch.json + ${{ runner.temp }}/ai-chip-intelligence-claim-review-batch.md + ${{ runner.temp }}/ai-chip-intelligence-claim-review-decisions-template.json ${{ runner.temp }}/ai-chip-intelligence-review-aging.md ${{ runner.temp }}/ai-chip-intelligence-review-backlog-hydration.json ${{ runner.temp }}/ai-chip-intelligence-review-backlog-hydration.md diff --git a/.github/workflows/validate-compatibility.yml b/.github/workflows/validate-compatibility.yml index 6ec21bb..8cab870 100644 --- a/.github/workflows/validate-compatibility.yml +++ b/.github/workflows/validate-compatibility.yml @@ -60,6 +60,7 @@ jobs: python3 -m unittest discover -s tests -p "test_intelligence_triage.py" python3 -m unittest discover -s tests -p "test_intelligence_review_state.py" python3 -m unittest discover -s tests -p "test_intelligence_backlog_hydration.py" + python3 -m unittest discover -s tests -p "test_intelligence_claim_review_batch.py" python3 -m unittest discover -s tests -p "test_intelligence_reliability.py" python3 -m unittest discover -s tests -p "test_intelligence_source_recovery.py" python3 -m unittest discover -s tests -p "test_intelligence_source_coverage.py" diff --git a/docs/intelligence-discovery.md b/docs/intelligence-discovery.md index df46563..39c0b8a 100644 --- a/docs/intelligence-discovery.md +++ b/docs/intelligence-discovery.md @@ -346,3 +346,31 @@ All restored records remain unverified extraction candidates. Priority scores are work-routing signals, not truth confidence; review decisions are not factual verification. No formal company, product, model-support or benchmark registry is modified. + + +## Bounded human Claim review batches (PR #126) + +The weekly workflow now emits a deterministic first-pass review batch from the +persistent Claim review state. By default, each batch contains up to 25 Claims +with state `pending`, ordered by the existing triage priority score (high to +low), then first-seen age (oldest first), then Claim ID. Claims already marked +`needs_more_evidence` are placed in a separate follow-up queue so they do not +repeatedly displace never-reviewed Claims. + +Each item includes the original extracted statement, source URL and excerpt, +claim/evidence classifications, scope, value/unit, qualifiers, age, priority +factors and risk flags. The batch also emits a JSON decision template with +exactly the same top-level and per-decision fields as +`data/intelligence-review-decisions.json`; blank fields must be completed by +a human before the ledger will accept them. + +The template is not imported automatically. Human decisions still require a +reviewer, a non-future timestamp and at least 30 characters of rationale. +`accepted_for_evidence_attachment` only requests preparation of a separate +evidence proposal; it does not verify the Claim or authorize a formal registry +write. Priority remains a work-routing signal, never truth confidence. + +The batch ID is deterministic for the selected Claim IDs. After decisions are +committed to the human ledger, the next discovery run removes handled Claims +from the first-pass queue and advances to the next pending set. The default +batch size is 25 and can be changed in the workflow invocation (1–100). diff --git a/scripts/build_intelligence_claim_review_batch.py b/scripts/build_intelligence_claim_review_batch.py new file mode 100644 index 0000000..d40722c --- /dev/null +++ b/scripts/build_intelligence_claim_review_batch.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +"""Build a deterministic, bounded human-review batch from persistent Claim state. + +This creates a review packet and an empty decision template. It never records +decisions, changes the decision ledger, or promotes a Claim to verified status. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +DECISIONS = ("needs_more_evidence", "rejected", "accepted_for_evidence_attachment") + + +def parse_utc(value): + if not isinstance(value, str): + raise ValueError("Claim first_seen_at must be an ISO-8601 string") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise ValueError("invalid Claim first_seen_at: " + value) from error + if parsed.tzinfo is None: + raise ValueError("Claim first_seen_at must include a timezone") + return parsed.astimezone(timezone.utc) + + +def build_review_batch(review_state, *, batch_size=25): + if type(batch_size) is not int or not 1 <= batch_size <= 100: + raise ValueError("batch_size must be 1..100") + if not isinstance(review_state, dict) or review_state.get("schema_version") != "1.0": + raise ValueError("expected review-state schema 1.0") + if review_state.get("observation_state") != "cache_persisted" or review_state.get("state_is_approval") is not False: + raise ValueError("review state must be a persisted observation, not an approval") + summary = review_state.get("summary") + rows = review_state.get("aging_backlog") + if not isinstance(summary, dict) or not isinstance(rows, list): + raise ValueError("review state requires summary and aging_backlog") + ids = [row.get("claim_id") for row in rows if isinstance(row, dict)] + if len(ids) != len(rows) or any(not isinstance(cid, str) or not cid for cid in ids) or len(set(ids)) != len(ids): + raise ValueError("aging_backlog Claim IDs must be unique and non-empty") + expected = summary.get("unresolved_backlog") + if expected is not None and expected != len(rows): + raise ValueError("unresolved backlog count does not match aging_backlog") + counts = summary.get("review_states") + if isinstance(counts, dict): + expected_unresolved = counts.get("pending", 0) + counts.get("needs_more_evidence", 0) + if expected_unresolved != len(rows): + raise ValueError("review-state counts do not match aging_backlog") + + pending, follow_up, missing = [], [], [] + for row in rows: + state = row.get("review_state") + if state not in ("pending", "needs_more_evidence"): + raise ValueError("aging_backlog may contain only unresolved review states") + snapshot = row.get("claim_snapshot") + if not isinstance(snapshot, dict): + missing.append(row["claim_id"]) + continue + if snapshot.get("claim_id") != row["claim_id"]: + raise ValueError("Claim snapshot ID does not match review-state ID") + statement = snapshot.get("statement") + source = snapshot.get("source") + if not isinstance(statement, str) or not statement.strip(): + missing.append(row["claim_id"]) + continue + if not isinstance(source, dict) or not isinstance(source.get("url"), str) or not source["url"].startswith(("https://", "http://")): + missing.append(row["claim_id"]) + continue + priority = row.get("priority") if isinstance(row.get("priority"), dict) else {} + score = priority.get("score", row.get("last_priority_score", 0)) + if type(score) not in (int, float): + score = 0 + parse_utc(row.get("first_seen_at")) + age_days = row.get("days_since_first_seen", 0) + if type(age_days) is not int or age_days < 0: + raise ValueError("days_since_first_seen must be a non-negative integer") + item = { + "claim_id": row["claim_id"], + "review_state": state, + "first_seen_at": row.get("first_seen_at"), + "days_since_first_seen": age_days, + "priority_score": score, + "priority_score_is_truth_confidence": False, + "priority_score_factors": priority.get("score_factors", {}), + "risk_flags": priority.get("risk_flags", []), + "claim": snapshot, + } + if state == "pending": + pending.append(item) + else: + follow_up.append(item) + + # A first-pass batch should move through never-reviewed items. Claims + # explicitly marked needs_more_evidence are reported separately and do + # not repeatedly displace new work from the first-pass queue. + pending.sort(key=lambda item: ( + -item["priority_score"], + -item["days_since_first_seen"], + item["claim_id"], + )) + selected = pending[:batch_size] + follow_up.sort(key=lambda item: ( + -item["days_since_first_seen"], + -item["priority_score"], + item["claim_id"], + )) + + batch_key = "\n".join(item["claim_id"] for item in selected) + batch_id = "claim-review-" + hashlib.sha256(batch_key.encode("utf-8")).hexdigest()[:12] + status = ("empty" if not rows else "unavailable" if not selected and missing + else "follow_up_only" if not selected and follow_up and not missing + else "ready" if selected and not missing else "partial") + decision_template = { + "schema_version": "1.0", + "decisions": [ + { + "claim_id": item["claim_id"], + "decision": "", + "reviewer": "", + "decided_at": "", + "notes": "", + } + for item in selected + ], + } + packet = { + "schema_version": "1.0", + "generated_at": review_state.get("generated_at"), + "status": status, + "batch_id": batch_id, + "batch_size_limit": batch_size, + "selection_method": "pending_only; priority_score_desc, first_seen_age_desc, claim_id_asc", + "summary": { + "unresolved_claims": len(rows), + "pending_claims": len(pending), + "needs_more_evidence_follow_ups": len(follow_up), + "reviewable_pending_claims": len(pending), + "selected_for_this_batch": len(selected), + "remaining_pending_after_batch": max(0, len(pending) - len(selected)), + "allowed_decisions": list(DECISIONS), + "missing_or_invalid_payloads": len(missing), + "missing_claim_ids": sorted(missing), + }, + "policy": { + "decisions_written": False, + "factual_verification_performed": False, + "priority_is_truth_confidence": False, + "review_rule": "Open the original source and verify exact product/model, configuration, units, scope, date and independent corroboration. Record uncertainty explicitly.", + "accepted_for_evidence_attachment": "Means only that a claim is suitable for preparing a proposed evidence attachment; it does not mean the claim is independently verified or approved for a formal registry.", + }, + "items": selected, + "follow_up_queue": follow_up, + "decision_template": decision_template, + } + return packet + + +def markdown_report(packet): + summary = packet["summary"] + lines = [ + "# AI Chip — Claim Review Batch", + "", + f"- Status: **{packet['status']}**", + f"- Batch: {packet['batch_id']}", + f"- Selected: **{summary['selected_for_this_batch']} / {summary['pending_claims']} pending Claims**", + f"- Remaining first-pass queue: {summary['remaining_pending_after_batch']}", + f"- Needs-more-evidence follow-ups: {summary['needs_more_evidence_follow_ups']}", + f"- Missing/invalid Claim payloads: {summary['missing_or_invalid_payloads']}", + "", + "> Priority is a work-routing score, not confidence in truth. No decision is made by this report.", + "", + "## Review protocol", + "", + "1. Open the original source; do not rely on the extracted quote alone.", + "2. Verify exact product/model, configuration, units, workload, measurement method and date.", + "3. Distinguish vendor statements, preprints and independent measurements.", + "4. Record a concise rationale (at least 30 characters) and reviewer identity.", + "5. Use accepted_for_evidence_attachment only to request preparation of a separate evidence proposal; it is not verification.", + "", + ] + for index, item in enumerate(packet["items"], 1): + claim = item["claim"] + source = claim.get("source") or {} + scope = claim.get("scope") or {} + lines.extend([ + f"## {index}. {claim.get('topic') or 'unclassified'} — {item['claim_id']}", + "", + f"- Priority: {item['priority_score']} · first seen: {item['days_since_first_seen']} days ago", + f"- Claim type: {claim.get('claim_type') or 'unknown'} · assertion: {claim.get('assertion_class') or 'unknown'}", + f"- Evidence class: {claim.get('evidence_class') or 'unknown'} · source type: {source.get('source_type') or 'unknown'}", + f"- Publisher: {source.get('publisher') or source.get('source_name') or 'unknown'}", + f"- Original source: {source.get('url') or 'unavailable'}", + "", + "**Claim candidate**", + "", + str(claim.get("statement") or "").strip(), + "", + "**Extracted source excerpt**", + "", + str(claim.get("evidence_quote") or "[No excerpt retained]").strip(), + "", + f"- Scope: {scope.get('text') or scope.get('status') or 'not stated'}", + f"- Value/unit: {claim.get('value')!r} / {claim.get('unit') or 'not stated'}", + f"- Qualifiers: {', '.join(str(x) for x in claim.get('qualifiers', [])) or 'none extracted'}", + f"- Risk flags: {', '.join(item.get('risk_flags', [])) or 'none flagged'}", + "", + "**Human review**", + "", + "- [ ] Original source opened and checked", + "- [ ] Product/model and configuration identified", + "- [ ] Units, scope and workload checked", + "- [ ] Independent corroboration sought or explicitly unavailable", + "- [ ] Decision and rationale entered in the JSON template", + "", + ]) + if summary["missing_claim_ids"]: + lines.extend(["## Missing payloads", ""]) + lines.extend(f"- {cid}" for cid in summary["missing_claim_ids"]) + lines.append("") + if packet["follow_up_queue"]: + lines.extend([ + "## Separate follow-up queue", + "", + f"{len(packet['follow_up_queue'])} Claims already marked needs_more_evidence are kept out of the first-pass batch so they do not displace never-reviewed items.", + "", + ]) + return "\n".join(lines) + "\n" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--review-state", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--markdown", type=Path, required=True) + parser.add_argument("--decision-template", type=Path, required=True) + parser.add_argument("--batch-size", type=int, default=25) + args = parser.parse_args() + try: + state = json.loads(args.review_state.read_text(encoding="utf-8")) + packet = build_review_batch(state, batch_size=args.batch_size) + for path in (args.output, args.markdown, args.decision_template): + path.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(packet, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + args.markdown.write_text(markdown_report(packet), encoding="utf-8") + args.decision_template.write_text(json.dumps(packet["decision_template"], indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + print(f"Claim review batch: {packet['status']}; {packet['summary']['selected_for_this_batch']} selected.") + return 0 + except (OSError, ValueError, TypeError, KeyError) as error: + print("FAIL: " + str(error), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_intelligence_claim_review_batch.py b/tests/test_intelligence_claim_review_batch.py new file mode 100644 index 0000000..1c45aef --- /dev/null +++ b/tests/test_intelligence_claim_review_batch.py @@ -0,0 +1,151 @@ +"""Tests for deterministic, bounded human Claim review batches.""" +import copy +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from build_intelligence_claim_review_batch import build_review_batch, markdown_report + + +def make_claim(claim_id, *, statement=None, url=None): + return { + "claim_id": claim_id, + "candidate_id": "candidate-" + claim_id, + "topic": "performance", + "claim_type": "performance_result", + "assertion_class": "research_result_candidate", + "evidence_class": "research_preprint", + "statement": statement or "Model X achieved 123 tokens/s on workload Y.", + "value": 123, + "unit": "tokens/s", + "scope": {"status": "explicit", "text": "single accelerator, batch 1"}, + "qualifiers": ["vendor-reported"], + "evidence_quote": "The measured throughput was 123 tokens/s.", + "source": { + "url": url or "https://example.org/" + claim_id, + "publisher": "Example Lab", + "source_type": "research_paper", + }, + } + + +def make_row(claim_id, *, score=10, age=5, state="pending", snapshot=True, first_seen="2026-11-01T00:00:00Z"): + row = { + "claim_id": claim_id, + "review_state": state, + "first_seen_at": first_seen, + "days_since_first_seen": age, + "last_priority_score": score, + "priority": {"score": score, "score_factors": {"high_impact_topic": score}, "risk_flags": ["research_preprint_not_independent_measurement"]}, + } + if snapshot: + row["claim_snapshot"] = make_claim(claim_id) + return row + + +def make_state(rows): + return { + "schema_version": "1.0", + "generated_at": "2026-11-17T10:00:00Z", + "observation_state": "cache_persisted", + "state_is_approval": False, + "summary": { + "tracked_backlog_claims": len(rows), + "review_states": { + "pending": sum(row["review_state"] == "pending" for row in rows), + "needs_more_evidence": sum(row["review_state"] == "needs_more_evidence" for row in rows), + "rejected": 0, + "accepted_for_evidence_attachment": 0, + }, + }, + "aging_backlog": rows, + } + + +class ClaimReviewBatchTests(unittest.TestCase): + def test_selects_highest_priority_pending_claims_deterministically(self): + state = make_state([ + make_row("low", score=5, age=40), + make_row("high-old", score=90, age=10), + make_row("high-young", score=90, age=2), + make_row("medium", score=40, age=20), + ]) + first = build_review_batch(state, batch_size=2) + second = build_review_batch(state, batch_size=2) + ids = [item["claim_id"] for item in first["items"]] + self.assertEqual(ids, ["high-old", "high-young"]) + self.assertEqual(first["batch_id"], second["batch_id"]) + self.assertEqual(first["status"], "ready") + self.assertEqual(first["summary"]["remaining_pending_after_batch"], 2) + + def test_follow_up_items_do_not_displace_never_reviewed_pending_items(self): + state = make_state([ + make_row("follow-up", score=1000, state="needs_more_evidence"), + make_row("pending", score=10), + ]) + packet = build_review_batch(state, batch_size=1) + self.assertEqual([x["claim_id"] for x in packet["items"]], ["pending"]) + self.assertEqual([x["claim_id"] for x in packet["follow_up_queue"]], ["follow-up"]) + + def test_missing_payloads_are_reported_and_status_is_partial(self): + state = make_state([make_row("good"), make_row("missing", snapshot=False)]) + packet = build_review_batch(state, batch_size=10) + self.assertEqual(packet["status"], "partial") + self.assertEqual(packet["summary"]["missing_claim_ids"], ["missing"]) + self.assertEqual(packet["summary"]["selected_for_this_batch"], 1) + + def test_decision_template_matches_human_ledger_shape_and_is_blank(self): + packet = build_review_batch(make_state([make_row("claim-1")])) + template = packet["decision_template"] + self.assertEqual(set(template), {"schema_version", "decisions"}) + self.assertEqual(template["schema_version"], "1.0") + self.assertEqual(set(template["decisions"][0]), {"claim_id", "decision", "reviewer", "decided_at", "notes"}) + self.assertTrue(all(value == "" for key, value in template["decisions"][0].items() if key != "claim_id")) + self.assertFalse(packet["policy"]["decisions_written"]) + self.assertFalse(packet["policy"]["factual_verification_performed"]) + + def test_markdown_contains_original_source_claim_and_review_checklist(self): + packet = build_review_batch(make_state([make_row("claim-1")])) + report = markdown_report(packet) + self.assertIn("Model X achieved 123 tokens/s", report) + self.assertIn("https://example.org/claim-1", report) + self.assertIn("Original source opened and checked", report) + self.assertIn("Priority is a work-routing score", report) + + def test_rejects_non_persisted_or_approval_state(self): + state = make_state([make_row("claim-1")]) + state["state_is_approval"] = True + with self.assertRaisesRegex(ValueError, "not an approval"): + build_review_batch(state) + + def test_rejects_snapshot_id_mismatch_and_duplicate_ids(self): + state = make_state([make_row("claim-1")]) + state["aging_backlog"][0]["claim_snapshot"]["claim_id"] = "other" + with self.assertRaisesRegex(ValueError, "snapshot ID"): + build_review_batch(state) + duplicate = make_state([make_row("claim-1"), make_row("claim-1")]) + with self.assertRaisesRegex(ValueError, "unique"): + build_review_batch(duplicate) + + def test_reports_follow_up_only_and_empty_states(self): + follow = make_state([make_row("follow", state="needs_more_evidence")]) + packet = build_review_batch(follow) + self.assertEqual(packet["status"], "follow_up_only") + self.assertEqual(packet["summary"]["selected_for_this_batch"], 0) + empty = make_state([]) + packet = build_review_batch(empty) + self.assertEqual(packet["status"], "empty") + + def test_rejects_bad_batch_size_and_mismatched_counts(self): + state = make_state([make_row("claim-1")]) + with self.assertRaisesRegex(ValueError, "batch_size"): + build_review_batch(state, batch_size=0) + state["summary"]["review_states"]["pending"] = 9 + with self.assertRaisesRegex(ValueError, "counts"): + build_review_batch(state) + + +if __name__ == "__main__": + unittest.main()