diff --git a/.github/workflows/validate-compatibility.yml b/.github/workflows/validate-compatibility.yml index 8cab870..0a7d4f4 100644 --- a/.github/workflows/validate-compatibility.yml +++ b/.github/workflows/validate-compatibility.yml @@ -47,6 +47,8 @@ jobs: python3 scripts/validate_model_support.py python3 scripts/validate_model_catalog.py python3 scripts/validate_intelligence_sources.py + python3 scripts/validate_intelligence_review_ledger.py --ledger data/intelligence-review-decisions.json + python3 -m unittest discover -s tests -p "test_intelligence_review_ledger.py" python3 -m unittest discover -s tests -p "test_intelligence_discovery.py" python3 -m unittest discover -s tests -p "test_intelligence_feeds.py" python3 -m unittest discover -s tests -p "test_intelligence_claims.py" diff --git a/docs/intelligence-review-state.md b/docs/intelligence-review-state.md index 3c83925..964378c 100644 --- a/docs/intelligence-review-state.md +++ b/docs/intelligence-review-state.md @@ -27,7 +27,8 @@ not a durable audit database. Optional committed \`data/intelligence-review-decisions.json\` is the human-controlled ledger. Each item explicitly specifies an extracted Claim -ID, reviewer identity, date, decision and at least 30 characters of reasoning. +ID, reviewer identity, timezone-aware ISO-8601 decision timestamp, decision +and at least 30 characters of reasoning. Allowed review outcomes are: needs_more_evidence, rejected, and accepted_for_evidence_attachment. @@ -55,3 +56,35 @@ The output summary and JSON artifact are added to the weekly Actions run. No automatic commits, notifications, external messages or data PRs are created. All individual Claim statuses remain unverified for factual content, regardless of the operational review queue state. + + +## Ledger validation (PR #128) + +The repository CI validates the committed decision ledger before merge, not +only when the next weekly discovery happens to consume it. The shared +`scripts/validate_intelligence_review_ledger.py` validator is also used by +`track_intelligence_review_state.py`, so CI and the scheduled pipeline enforce +the same rules. + +The validator rejects: +- Unknown schema fields, unsupported versions or malformed decision rows; +- Duplicate, malformed or (when a review-state file is supplied) unknown + Claim IDs; +- Unsupported outcomes, empty reviewer identities or short review notes; +- Invalid, timezone-naive or future-dated timestamps. + +Run the standalone check: + + python3 scripts/validate_intelligence_review_ledger.py \ + --ledger data/intelligence-review-decisions.json + +To additionally verify every decision against a persisted Claim-state cache: + + python3 scripts/validate_intelligence_review_ledger.py \ + --ledger data/intelligence-review-decisions.json \ + --review-state /path/to/ai-chip-intelligence-review-state.json + +A passing ledger validation checks record structure and workflow consistency; +it does not authenticate the reviewer, inspect the original source or verify +the technical truth of a Claim. Human GitHub review and the separate +Evidence-to-PR approval process remain required. diff --git a/scripts/track_intelligence_review_state.py b/scripts/track_intelligence_review_state.py index 6a6ae52..557a08d 100644 --- a/scripts/track_intelligence_review_state.py +++ b/scripts/track_intelligence_review_state.py @@ -12,8 +12,9 @@ from datetime import datetime, timezone from pathlib import Path +from validate_intelligence_review_ledger import load_decisions + REVIEW_STATUSES={"pending","needs_more_evidence","rejected","accepted_for_evidence_attachment"} -DECISION_STATUSES=REVIEW_STATUSES-{"pending"} ALERT_THRESHOLDS=(7,14,30) def parse_utc(value): @@ -27,28 +28,6 @@ def parse_utc(value): def iso(value): return value.astimezone(timezone.utc).isoformat().replace("+00:00","Z") -def load_decisions(ledger, allowed_ids, now): - if ledger is None: return {} - if not isinstance(ledger,dict) or set(ledger)!={"schema_version","decisions"} or ledger.get("schema_version")!="1.0" or not isinstance(ledger.get("decisions"),list): - raise ValueError("human review decision ledger must be schema 1.0") - result={} - for decision in ledger["decisions"]: - if not isinstance(decision,dict) or set(decision)!={"claim_id","decision","reviewer","decided_at","notes"}: - raise ValueError("human decision must have exactly claim_id, decision, reviewer, decided_at and notes") - cid=decision["claim_id"] - if not isinstance(cid,str) or cid not in allowed_ids or cid in result: - raise ValueError("human decision references unknown or duplicate Claim ID") - if decision["decision"] not in DECISION_STATUSES: - raise ValueError("invalid human decision outcome") - if not isinstance(decision["reviewer"],str) or not decision["reviewer"].strip(): - raise ValueError("human reviewer identity required") - if not isinstance(decision["notes"],str) or len(decision["notes"].strip())<30: - raise ValueError("human review notes must have at least 30 characters") - decided=parse_utc(decision["decided_at"]) - if decided>now: raise ValueError("human decision cannot be future-dated") - result[cid]={**decision,"decided_at":iso(decided)} - return result - def update_state(triage, claims, previous=None, ledger=None, *, now=None, retention_days=180): now=now or datetime.now(timezone.utc) if now.tzinfo is None: raise ValueError("now must be timezone-aware") diff --git a/scripts/validate_intelligence_review_ledger.py b/scripts/validate_intelligence_review_ledger.py new file mode 100644 index 0000000..a2d2d4b --- /dev/null +++ b/scripts/validate_intelligence_review_ledger.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Validate the human intelligence review decision ledger. + +The ledger records workflow decisions only. It never proves technical truth, +changes a company profile, or promotes a Claim to verified status. +""" +from __future__ import annotations + +import argparse +import json +import re +import sys +from datetime import datetime, timezone +from pathlib import Path + +DECISIONS = { + "needs_more_evidence", + "rejected", + "accepted_for_evidence_attachment", +} +CLAIM_ID = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") + + +def parse_timestamp(value): + if not isinstance(value, str): + raise ValueError("decided_at must be a timezone-aware ISO-8601 string") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise ValueError("decided_at must be a valid ISO-8601 timestamp") from error + if parsed.utcoffset() is None: + raise ValueError("decided_at must include a timezone") + return parsed.astimezone(timezone.utc) + + +def iso(value): + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def load_decisions(ledger, allowed_ids=None, now=None): + """Return normalized decisions keyed by Claim ID; fail closed on bad input.""" + if ledger is None: + return {} + if (not isinstance(ledger, dict) or set(ledger) != {"schema_version", "decisions"} + or ledger.get("schema_version") != "1.0" + or not isinstance(ledger.get("decisions"), list)): + raise ValueError("human review decision ledger must be schema 1.0 with exactly schema_version and decisions") + + now = now or datetime.now(timezone.utc) + if not isinstance(now, datetime) or now.utcoffset() is None: + raise ValueError("validation time must be timezone-aware") + now = now.astimezone(timezone.utc) + + result = {} + for decision in ledger["decisions"]: + required = {"claim_id", "decision", "reviewer", "decided_at", "notes"} + if not isinstance(decision, dict) or set(decision) != required: + raise ValueError("human decision must contain exactly claim_id, decision, reviewer, decided_at and notes") + claim_id = decision["claim_id"] + if not isinstance(claim_id, str) or not CLAIM_ID.fullmatch(claim_id): + raise ValueError("human decision Claim ID must be a lowercase slug") + if allowed_ids is not None and claim_id not in allowed_ids: + raise ValueError("human decision references unknown Claim ID: " + claim_id) + if claim_id in result: + raise ValueError("duplicate human decision Claim ID: " + claim_id) + if not isinstance(decision["decision"], str) or decision["decision"] not in DECISIONS: + raise ValueError("invalid human decision outcome for " + claim_id) + reviewer = decision["reviewer"] + if not isinstance(reviewer, str) or not reviewer.strip(): + raise ValueError("human reviewer identity required for " + claim_id) + notes = decision["notes"] + if not isinstance(notes, str) or len(notes.strip()) < 30: + raise ValueError("human review notes must have at least 30 characters for " + claim_id) + decided = parse_timestamp(decision["decided_at"]) + if decided > now: + raise ValueError("human decision cannot be future-dated: " + claim_id) + result[claim_id] = { + "claim_id": claim_id, + "decision": decision["decision"], + "reviewer": reviewer.strip(), + "decided_at": iso(decided), + "notes": notes.strip(), + } + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ledger", type=Path, default=Path("data/intelligence-review-decisions.json")) + parser.add_argument("--review-state", type=Path, + help="Optional persisted review-state JSON; if supplied, every Claim ID must exist in its items.") + args = parser.parse_args() + try: + ledger = json.loads(args.ledger.read_text(encoding="utf-8")) + if ledger is None: + raise ValueError("ledger file must contain a schema 1.0 object, not null") + allowed_ids = None + if args.review_state: + state = json.loads(args.review_state.read_text(encoding="utf-8")) + if (not isinstance(state, dict) or state.get("schema_version") != "1.0" + or not isinstance(state.get("items"), dict)): + raise ValueError("review state must be schema 1.0 with an items mapping") + allowed_ids = set(state["items"]) + decisions = load_decisions(ledger, allowed_ids) + print("PASS: validated " + str(len(decisions)) + " human review decisions.") + except (OSError, json.JSONDecodeError, ValueError, TypeError) as error: + print("FAIL: " + str(error), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_intelligence_review_ledger.py b/tests/test_intelligence_review_ledger.py new file mode 100644 index 0000000..4633e12 --- /dev/null +++ b/tests/test_intelligence_review_ledger.py @@ -0,0 +1,135 @@ +import json +import subprocess +import sys +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from validate_intelligence_review_ledger import load_decisions + + +NOW = datetime(2026, 11, 17, 12, tzinfo=timezone.utc) + + +def decision(claim_id="claim-abc", **overrides): + row = { + "claim_id": claim_id, + "decision": "needs_more_evidence", + "reviewer": "maintainer@example", + "decided_at": "2026-11-17T10:00:00Z", + "notes": "Inspected the original source; exact SKU and configuration still need corroboration.", + } + row.update(overrides) + return row + + +def ledger(*rows): + return {"schema_version": "1.0", "decisions": list(rows)} + + +class ReviewLedgerValidationTests(unittest.TestCase): + def test_empty_ledger_is_valid(self): + self.assertEqual(load_decisions(ledger(), set(), NOW), {}) + + def test_valid_decisions_are_normalized(self): + result = load_decisions( + ledger(decision(decided_at="2026-11-17T12:00:00+02:00")), + {"claim-abc"}, NOW + ) + self.assertEqual(result["claim-abc"]["decided_at"], "2026-11-17T10:00:00Z") + self.assertEqual(result["claim-abc"]["reviewer"], "maintainer@example") + + def test_all_allowed_outcomes_are_accepted(self): + rows = [ + decision("claim-a", decision="needs_more_evidence"), + decision("claim-b", decision="rejected"), + decision("claim-c", decision="accepted_for_evidence_attachment"), + ] + self.assertEqual(len(load_decisions(ledger(*rows), None, NOW)), 3) + + def test_unknown_claim_id_rejected_when_state_allowlist_supplied(self): + with self.assertRaisesRegex(ValueError, "unknown Claim ID"): + load_decisions(ledger(decision()), {"claim-other"}, NOW) + + def test_duplicate_claim_ids_rejected(self): + with self.assertRaisesRegex(ValueError, "duplicate"): + load_decisions(ledger(decision(), decision()), None, NOW) + + def test_invalid_or_non_slug_claim_ids_rejected(self): + for claim_id in ("", "Claim-ABC", "../claim", "claim_id", "claim:abc"): + with self.subTest(claim_id=claim_id), self.assertRaisesRegex(ValueError, "lowercase slug"): + load_decisions(ledger(decision(claim_id)), None, NOW) + + def test_unknown_or_missing_top_level_fields_rejected(self): + for payload in ( + {"schema_version": "1.0", "decisions": [], "status": "approved"}, + {"schema_version": "1.0"}, + {"schema_version": "2.0", "decisions": []}, + [], + ): + with self.subTest(payload=payload), self.assertRaisesRegex(ValueError, "schema 1.0"): + load_decisions(payload, None, NOW) + + def test_unknown_or_missing_decision_fields_rejected(self): + row = decision() + row["extra"] = "not allowed" + with self.assertRaisesRegex(ValueError, "exactly"): + load_decisions(ledger(row), None, NOW) + row = decision() + row.pop("notes") + with self.assertRaisesRegex(ValueError, "exactly"): + load_decisions(ledger(row), None, NOW) + + def test_invalid_outcome_rejected(self): + with self.assertRaisesRegex(ValueError, "outcome"): + load_decisions(ledger(decision(decision="verified")), None, NOW) + + def test_reviewer_must_be_non_empty(self): + for reviewer in ("", " ", None, 123): + with self.subTest(reviewer=reviewer), self.assertRaisesRegex(ValueError, "reviewer identity"): + load_decisions(ledger(decision(reviewer=reviewer)), None, NOW) + + def test_notes_must_have_at_least_30_non_whitespace_characters(self): + for notes in ("too short", " " * 50, None): + with self.subTest(notes=notes), self.assertRaisesRegex(ValueError, "at least 30"): + load_decisions(ledger(decision(notes=notes)), None, NOW) + + def test_invalid_naive_or_future_timestamps_rejected(self): + for stamp in ("not-a-date", "2026-11-17T10:00:00", "2026-11-18T00:00:00Z"): + with self.subTest(stamp=stamp), self.assertRaises(ValueError): + load_decisions(ledger(decision(decided_at=stamp)), None, NOW) + + def test_validation_clock_must_be_timezone_aware(self): + with self.assertRaisesRegex(ValueError, "timezone-aware"): + load_decisions(ledger(), None, datetime(2026, 11, 17)) + + def test_cli_rejects_null_ledger_file(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "ledger.json" + path.write_text("null", encoding="utf-8") + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "validate_intelligence_review_ledger.py"), + "--ledger", str(path)], + cwd=ROOT, capture_output=True, text=True, check=False + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("not null", result.stderr) + + def test_cli_validates_repository_ledger(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "ledger.json" + path.write_text(json.dumps(ledger()), encoding="utf-8") + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "validate_intelligence_review_ledger.py"), + "--ledger", str(path)], + cwd=ROOT, capture_output=True, text=True, check=False + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("PASS: validated 0", result.stdout) + + +if __name__ == "__main__": + unittest.main()