Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/validate-compatibility.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ jobs:
python3 scripts/validate_model_support.py
python3 scripts/validate_model_catalog.py
python3 scripts/validate_intelligence_sources.py
python3 scripts/validate_intelligence_review_ledger.py --ledger data/intelligence-review-decisions.json
python3 -m unittest discover -s tests -p "test_intelligence_review_ledger.py"
python3 -m unittest discover -s tests -p "test_intelligence_discovery.py"
python3 -m unittest discover -s tests -p "test_intelligence_feeds.py"
python3 -m unittest discover -s tests -p "test_intelligence_claims.py"
Expand Down
35 changes: 34 additions & 1 deletion docs/intelligence-review-state.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ not a durable audit database.

Optional committed \`data/intelligence-review-decisions.json\` is the
human-controlled ledger. Each item explicitly specifies an extracted Claim
ID, reviewer identity, date, decision and at least 30 characters of reasoning.
ID, reviewer identity, timezone-aware ISO-8601 decision timestamp, decision
and at least 30 characters of reasoning.
Allowed review outcomes are: needs_more_evidence, rejected, and
accepted_for_evidence_attachment.

Expand Down Expand Up @@ -55,3 +56,35 @@ The output summary and JSON artifact are added to the weekly Actions run.
No automatic commits, notifications, external messages or data PRs are
created. All individual Claim statuses remain unverified for factual
content, regardless of the operational review queue state.


## Ledger validation (PR #128)

The repository CI validates the committed decision ledger before merge, not
only when the next weekly discovery happens to consume it. The shared
`scripts/validate_intelligence_review_ledger.py` validator is also used by
`track_intelligence_review_state.py`, so CI and the scheduled pipeline enforce
the same rules.

The validator rejects:
- Unknown schema fields, unsupported versions or malformed decision rows;
- Duplicate, malformed or (when a review-state file is supplied) unknown
Claim IDs;
- Unsupported outcomes, empty reviewer identities or short review notes;
- Invalid, timezone-naive or future-dated timestamps.

Run the standalone check:

python3 scripts/validate_intelligence_review_ledger.py \
--ledger data/intelligence-review-decisions.json

To additionally verify every decision against a persisted Claim-state cache:

python3 scripts/validate_intelligence_review_ledger.py \
--ledger data/intelligence-review-decisions.json \
--review-state /path/to/ai-chip-intelligence-review-state.json

A passing ledger validation checks record structure and workflow consistency;
it does not authenticate the reviewer, inspect the original source or verify
the technical truth of a Claim. Human GitHub review and the separate
Evidence-to-PR approval process remain required.
25 changes: 2 additions & 23 deletions scripts/track_intelligence_review_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@
from datetime import datetime, timezone
from pathlib import Path

from validate_intelligence_review_ledger import load_decisions

REVIEW_STATUSES={"pending","needs_more_evidence","rejected","accepted_for_evidence_attachment"}
DECISION_STATUSES=REVIEW_STATUSES-{"pending"}
ALERT_THRESHOLDS=(7,14,30)

def parse_utc(value):
Expand All @@ -27,28 +28,6 @@ def parse_utc(value):
def iso(value):
return value.astimezone(timezone.utc).isoformat().replace("+00:00","Z")

def load_decisions(ledger, allowed_ids, now):
if ledger is None: return {}
if not isinstance(ledger,dict) or set(ledger)!={"schema_version","decisions"} or ledger.get("schema_version")!="1.0" or not isinstance(ledger.get("decisions"),list):
raise ValueError("human review decision ledger must be schema 1.0")
result={}
for decision in ledger["decisions"]:
if not isinstance(decision,dict) or set(decision)!={"claim_id","decision","reviewer","decided_at","notes"}:
raise ValueError("human decision must have exactly claim_id, decision, reviewer, decided_at and notes")
cid=decision["claim_id"]
if not isinstance(cid,str) or cid not in allowed_ids or cid in result:
raise ValueError("human decision references unknown or duplicate Claim ID")
if decision["decision"] not in DECISION_STATUSES:
raise ValueError("invalid human decision outcome")
if not isinstance(decision["reviewer"],str) or not decision["reviewer"].strip():
raise ValueError("human reviewer identity required")
if not isinstance(decision["notes"],str) or len(decision["notes"].strip())<30:
raise ValueError("human review notes must have at least 30 characters")
decided=parse_utc(decision["decided_at"])
if decided>now: raise ValueError("human decision cannot be future-dated")
result[cid]={**decision,"decided_at":iso(decided)}
return result

def update_state(triage, claims, previous=None, ledger=None, *, now=None, retention_days=180):
now=now or datetime.now(timezone.utc)
if now.tzinfo is None: raise ValueError("now must be timezone-aware")
Expand Down
113 changes: 113 additions & 0 deletions scripts/validate_intelligence_review_ledger.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""Validate the human intelligence review decision ledger.

The ledger records workflow decisions only. It never proves technical truth,
changes a company profile, or promotes a Claim to verified status.
"""
from __future__ import annotations

import argparse
import json
import re
import sys
from datetime import datetime, timezone
from pathlib import Path

DECISIONS = {
"needs_more_evidence",
"rejected",
"accepted_for_evidence_attachment",
}
CLAIM_ID = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")


def parse_timestamp(value):
if not isinstance(value, str):
raise ValueError("decided_at must be a timezone-aware ISO-8601 string")
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as error:
raise ValueError("decided_at must be a valid ISO-8601 timestamp") from error
if parsed.utcoffset() is None:
raise ValueError("decided_at must include a timezone")
return parsed.astimezone(timezone.utc)


def iso(value):
return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")


def load_decisions(ledger, allowed_ids=None, now=None):
"""Return normalized decisions keyed by Claim ID; fail closed on bad input."""
if ledger is None:
return {}
if (not isinstance(ledger, dict) or set(ledger) != {"schema_version", "decisions"}
or ledger.get("schema_version") != "1.0"
or not isinstance(ledger.get("decisions"), list)):
raise ValueError("human review decision ledger must be schema 1.0 with exactly schema_version and decisions")

now = now or datetime.now(timezone.utc)
if not isinstance(now, datetime) or now.utcoffset() is None:
raise ValueError("validation time must be timezone-aware")
now = now.astimezone(timezone.utc)

result = {}
for decision in ledger["decisions"]:
required = {"claim_id", "decision", "reviewer", "decided_at", "notes"}
if not isinstance(decision, dict) or set(decision) != required:
raise ValueError("human decision must contain exactly claim_id, decision, reviewer, decided_at and notes")
claim_id = decision["claim_id"]
if not isinstance(claim_id, str) or not CLAIM_ID.fullmatch(claim_id):
raise ValueError("human decision Claim ID must be a lowercase slug")
if allowed_ids is not None and claim_id not in allowed_ids:
raise ValueError("human decision references unknown Claim ID: " + claim_id)
if claim_id in result:
raise ValueError("duplicate human decision Claim ID: " + claim_id)
if not isinstance(decision["decision"], str) or decision["decision"] not in DECISIONS:
raise ValueError("invalid human decision outcome for " + claim_id)
reviewer = decision["reviewer"]
if not isinstance(reviewer, str) or not reviewer.strip():
raise ValueError("human reviewer identity required for " + claim_id)
notes = decision["notes"]
if not isinstance(notes, str) or len(notes.strip()) < 30:
raise ValueError("human review notes must have at least 30 characters for " + claim_id)
decided = parse_timestamp(decision["decided_at"])
if decided > now:
raise ValueError("human decision cannot be future-dated: " + claim_id)
result[claim_id] = {
"claim_id": claim_id,
"decision": decision["decision"],
"reviewer": reviewer.strip(),
"decided_at": iso(decided),
"notes": notes.strip(),
}
return result


def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--ledger", type=Path, default=Path("data/intelligence-review-decisions.json"))
parser.add_argument("--review-state", type=Path,
help="Optional persisted review-state JSON; if supplied, every Claim ID must exist in its items.")
args = parser.parse_args()
try:
ledger = json.loads(args.ledger.read_text(encoding="utf-8"))
if ledger is None:
raise ValueError("ledger file must contain a schema 1.0 object, not null")
allowed_ids = None
if args.review_state:
state = json.loads(args.review_state.read_text(encoding="utf-8"))
if (not isinstance(state, dict) or state.get("schema_version") != "1.0"
or not isinstance(state.get("items"), dict)):
raise ValueError("review state must be schema 1.0 with an items mapping")
allowed_ids = set(state["items"])
decisions = load_decisions(ledger, allowed_ids)
print("PASS: validated " + str(len(decisions)) + " human review decisions.")
except (OSError, json.JSONDecodeError, ValueError, TypeError) as error:
print("FAIL: " + str(error), file=sys.stderr)
return 1
return 0


if __name__ == "__main__":
raise SystemExit(main())
135 changes: 135 additions & 0 deletions tests/test_intelligence_review_ledger.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
import json
import subprocess
import sys
import tempfile
import unittest
from datetime import datetime, timezone
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from validate_intelligence_review_ledger import load_decisions


NOW = datetime(2026, 11, 17, 12, tzinfo=timezone.utc)


def decision(claim_id="claim-abc", **overrides):
row = {
"claim_id": claim_id,
"decision": "needs_more_evidence",
"reviewer": "maintainer@example",
"decided_at": "2026-11-17T10:00:00Z",
"notes": "Inspected the original source; exact SKU and configuration still need corroboration.",
}
row.update(overrides)
return row


def ledger(*rows):
return {"schema_version": "1.0", "decisions": list(rows)}


class ReviewLedgerValidationTests(unittest.TestCase):
def test_empty_ledger_is_valid(self):
self.assertEqual(load_decisions(ledger(), set(), NOW), {})

def test_valid_decisions_are_normalized(self):
result = load_decisions(
ledger(decision(decided_at="2026-11-17T12:00:00+02:00")),
{"claim-abc"}, NOW
)
self.assertEqual(result["claim-abc"]["decided_at"], "2026-11-17T10:00:00Z")
self.assertEqual(result["claim-abc"]["reviewer"], "maintainer@example")

def test_all_allowed_outcomes_are_accepted(self):
rows = [
decision("claim-a", decision="needs_more_evidence"),
decision("claim-b", decision="rejected"),
decision("claim-c", decision="accepted_for_evidence_attachment"),
]
self.assertEqual(len(load_decisions(ledger(*rows), None, NOW)), 3)

def test_unknown_claim_id_rejected_when_state_allowlist_supplied(self):
with self.assertRaisesRegex(ValueError, "unknown Claim ID"):
load_decisions(ledger(decision()), {"claim-other"}, NOW)

def test_duplicate_claim_ids_rejected(self):
with self.assertRaisesRegex(ValueError, "duplicate"):
load_decisions(ledger(decision(), decision()), None, NOW)

def test_invalid_or_non_slug_claim_ids_rejected(self):
for claim_id in ("", "Claim-ABC", "../claim", "claim_id", "claim:abc"):
with self.subTest(claim_id=claim_id), self.assertRaisesRegex(ValueError, "lowercase slug"):
load_decisions(ledger(decision(claim_id)), None, NOW)

def test_unknown_or_missing_top_level_fields_rejected(self):
for payload in (
{"schema_version": "1.0", "decisions": [], "status": "approved"},
{"schema_version": "1.0"},
{"schema_version": "2.0", "decisions": []},
[],
):
with self.subTest(payload=payload), self.assertRaisesRegex(ValueError, "schema 1.0"):
load_decisions(payload, None, NOW)

def test_unknown_or_missing_decision_fields_rejected(self):
row = decision()
row["extra"] = "not allowed"
with self.assertRaisesRegex(ValueError, "exactly"):
load_decisions(ledger(row), None, NOW)
row = decision()
row.pop("notes")
with self.assertRaisesRegex(ValueError, "exactly"):
load_decisions(ledger(row), None, NOW)

def test_invalid_outcome_rejected(self):
with self.assertRaisesRegex(ValueError, "outcome"):
load_decisions(ledger(decision(decision="verified")), None, NOW)

def test_reviewer_must_be_non_empty(self):
for reviewer in ("", " ", None, 123):
with self.subTest(reviewer=reviewer), self.assertRaisesRegex(ValueError, "reviewer identity"):
load_decisions(ledger(decision(reviewer=reviewer)), None, NOW)

def test_notes_must_have_at_least_30_non_whitespace_characters(self):
for notes in ("too short", " " * 50, None):
with self.subTest(notes=notes), self.assertRaisesRegex(ValueError, "at least 30"):
load_decisions(ledger(decision(notes=notes)), None, NOW)

def test_invalid_naive_or_future_timestamps_rejected(self):
for stamp in ("not-a-date", "2026-11-17T10:00:00", "2026-11-18T00:00:00Z"):
with self.subTest(stamp=stamp), self.assertRaises(ValueError):
load_decisions(ledger(decision(decided_at=stamp)), None, NOW)

def test_validation_clock_must_be_timezone_aware(self):
with self.assertRaisesRegex(ValueError, "timezone-aware"):
load_decisions(ledger(), None, datetime(2026, 11, 17))

def test_cli_rejects_null_ledger_file(self):
with tempfile.TemporaryDirectory() as temp:
path = Path(temp) / "ledger.json"
path.write_text("null", encoding="utf-8")
result = subprocess.run(
[sys.executable, str(ROOT / "scripts" / "validate_intelligence_review_ledger.py"),
"--ledger", str(path)],
cwd=ROOT, capture_output=True, text=True, check=False
)
self.assertNotEqual(result.returncode, 0)
self.assertIn("not null", result.stderr)

def test_cli_validates_repository_ledger(self):
with tempfile.TemporaryDirectory() as temp:
path = Path(temp) / "ledger.json"
path.write_text(json.dumps(ledger()), encoding="utf-8")
result = subprocess.run(
[sys.executable, str(ROOT / "scripts" / "validate_intelligence_review_ledger.py"),
"--ledger", str(path)],
cwd=ROOT, capture_output=True, text=True, check=False
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn("PASS: validated 0", result.stdout)


if __name__ == "__main__":
unittest.main()
Loading