diff --git a/.github/workflows/model-catalog-audit.yml b/.github/workflows/model-catalog-audit.yml index 1dc202e..24ad13f 100644 --- a/.github/workflows/model-catalog-audit.yml +++ b/.github/workflows/model-catalog-audit.yml @@ -13,6 +13,7 @@ on: default: 45 permissions: + actions: read contents: read concurrency: @@ -54,6 +55,17 @@ jobs: run: | printf 'DISCOVERY_CACHE_PRESENT=%s\n' "$(test -f .cache/ai-chip-intelligence-state.json && echo true || echo false)" >> "$GITHUB_ENV" printf 'REVIEW_CACHE_PRESENT=%s\n' "$(test -f .cache/ai-chip-review-state.json && echo true || echo false)" >> "$GITHUB_ENV" + - name: Recover historical Claim payloads from retained discovery artifacts + env: + GITHUB_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + python3 scripts/hydrate_intelligence_review_backlog.py \ + --state "$RUNNER_TEMP/review-state-staged.json" \ + --repo "$GITHUB_REPOSITORY" \ + --token "$GITHUB_TOKEN" \ + --output "$RUNNER_TEMP/ai-chip-intelligence-review-backlog-hydration.json" \ + --markdown "$RUNNER_TEMP/ai-chip-intelligence-review-backlog-hydration.md" - name: Validate curated source registry run: python3 scripts/validate_intelligence_sources.py - name: Discover releases, model changes and publisher candidates @@ -194,6 +206,9 @@ jobs: if [ -f "$RUNNER_TEMP/ai-chip-intelligence-review-aging.md" ]; then cat "$RUNNER_TEMP/ai-chip-intelligence-review-aging.md" >> "$GITHUB_STEP_SUMMARY" fi + if [ -f "$RUNNER_TEMP/ai-chip-intelligence-review-backlog-hydration.md" ]; then + cat "$RUNNER_TEMP/ai-chip-intelligence-review-backlog-hydration.md" >> "$GITHUB_STEP_SUMMARY" + fi if [ -f "$RUNNER_TEMP/ai-chip-intelligence-health.md" ]; then cat "$RUNNER_TEMP/ai-chip-intelligence-health.md" >> "$GITHUB_STEP_SUMMARY" fi @@ -228,6 +243,8 @@ jobs: ${{ runner.temp }}/ai-chip-intelligence-triage.md ${{ runner.temp }}/ai-chip-intelligence-review-state.json ${{ runner.temp }}/ai-chip-intelligence-review-aging.md + ${{ runner.temp }}/ai-chip-intelligence-review-backlog-hydration.json + ${{ runner.temp }}/ai-chip-intelligence-review-backlog-hydration.md ${{ runner.temp }}/ai-chip-intelligence-health.json ${{ runner.temp }}/ai-chip-intelligence-health.md ${{ runner.temp }}/ai-chip-intelligence-operations.json diff --git a/.github/workflows/validate-compatibility.yml b/.github/workflows/validate-compatibility.yml index 4e03a6e..6ec21bb 100644 --- a/.github/workflows/validate-compatibility.yml +++ b/.github/workflows/validate-compatibility.yml @@ -59,6 +59,7 @@ jobs: python3 -m unittest discover -s tests -p "test_intelligence_evidence_graph.py" python3 -m unittest discover -s tests -p "test_intelligence_triage.py" python3 -m unittest discover -s tests -p "test_intelligence_review_state.py" + python3 -m unittest discover -s tests -p "test_intelligence_backlog_hydration.py" python3 -m unittest discover -s tests -p "test_intelligence_reliability.py" python3 -m unittest discover -s tests -p "test_intelligence_source_recovery.py" python3 -m unittest discover -s tests -p "test_intelligence_source_coverage.py" diff --git a/docs/intelligence-discovery.md b/docs/intelligence-discovery.md index e1cd0ec..df46563 100644 --- a/docs/intelligence-discovery.md +++ b/docs/intelligence-discovery.md @@ -217,7 +217,7 @@ quality can increase page-cap competition and reduce precision. ## Isolated targeted retrieval experiment (PR #119) -The weekly workflow now runs four bounded, one-page arXiv query experiments +The weekly workflow now runs four bounded, two-page arXiv query experiments after the baseline Recall@K and miss diagnosis: 1. GPU communication and interconnect. @@ -225,8 +225,9 @@ after the baseline Recall@K and miss diagnosis: 3. Agentic GPU capacity and short-lived jobs. 4. Physical-AI runtime and world-action models. -Each query is limited to 50 results and is kept outside the production source -registry. The runner waits three seconds between experiment queries, uses the +Each query is limited to 50 results per page and at most two pages (up to 100 +raw entries per group), and is kept outside the production source registry. +The runner waits three seconds between pages and query groups, uses the existing Atom parser and bounded retry policy, and applies the same discovery lookback window. The baseline discovery and recall evaluation must be scorable before any experimental requests are made. @@ -253,7 +254,7 @@ python3 scripts/run_intelligence_retrieval_experiment.py \ ## Human relevance review for targeted retrieval (PR #121) The targeted-query experiment now retains the complete bounded result set -(up to 50 unique papers per query group), including title, arXiv URL, abstract +(up to 100 unique papers per query group), including title, arXiv URL, abstract excerpt, categories, deterministic screening label and observed position. The position is newest-first after local date filtering; it is **not** a relevance rank. @@ -311,3 +312,37 @@ independent shards remains in place. These controls reduce request bursts but do not guarantee availability or eliminate upstream throttling. + + +## Recoverable historical Claim review backlog (PR #125) + +The persistent review cache originally retained Claim IDs, first/last-seen +timestamps and selected priority metadata, but not the Claim statement, +source excerpt or full source context. This made historical pending entries +countable but not directly reviewable. + +The workflow now performs a one-time, read-only recovery from eight retained +GitHub Actions artifacts. It validates each archived discovery/Claim/evidence +graph bundle, recomputes the existing deterministic triage score against the +original run timestamp, and matches recovered Claim IDs to the persistent +cache. The recovery covers 2,525 unique Claim IDs observed in the current +backlog. It preserves original first-seen timestamps and does not create, +change or approve any human decision. + +The recovered Claim payload and priority metadata are retained in the review +cache. Subsequent change-only runs preserve those payloads even when a Claim +is absent from the current discovery. Newly extracted Claims are also stored +with their source text. The review-aging artifact now includes the full +source-grounded Claim statement and evidence quote for unresolved items. + +Recovery is automatic and idempotent: empty caches and already-hydrated caches +skip network access. If one or more historical artifacts are no longer +available, the report identifies missing Claim payloads and leaves the cache +usable; the workflow does not treat missing artifacts as reviewed Claims. +The one-time artifacts have normal GitHub Actions retention limits, so the +first run after this change should happen before those artifacts expire. + +All restored records remain unverified extraction candidates. Priority scores +are work-routing signals, not truth confidence; review decisions are not +factual verification. No formal company, product, model-support or benchmark +registry is modified. diff --git a/scripts/hydrate_intelligence_review_backlog.py b/scripts/hydrate_intelligence_review_backlog.py new file mode 100644 index 0000000..a2dc06b --- /dev/null +++ b/scripts/hydrate_intelligence_review_backlog.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +"""Restore full unverified Claim payloads from retained historical Actions artifacts. + +The cache intentionally keeps review state separate from factual verification. +This one-time migration rehydrates the 2,525 Claim IDs already present in the +persistent cache; it never changes a human decision or promotes a Claim. +""" +from __future__ import annotations + +import argparse +import copy +import io +import json +import os +import sys +import urllib.error +import urllib.request +import zipfile +from datetime import datetime, timezone +from pathlib import Path + +from triage_intelligence_claims import build_queue + +ARTIFACT_MANIFEST = ( + {"artifact_id": 14191101297, "run_id": 43787532670}, + {"artifact_id": 14191697562, "run_id": 43789263891}, + {"artifact_id": 14193774090, "run_id": 43795362021}, + {"artifact_id": 14195870093, "run_id": 43798131776}, + {"artifact_id": 14204506216, "run_id": 43820741766}, + {"artifact_id": 14211068232, "run_id": 43837297395}, + {"artifact_id": 14240845899, "run_id": 43898014328}, + {"artifact_id": 14284954184, "run_id": 43986167889}, +) + + +def parse_utc(value): + if not isinstance(value, str): + raise ValueError("artifact timestamp must be an ISO-8601 string") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("artifact timestamp must include a timezone") + return parsed.astimezone(timezone.utc) + + +def iso(value): + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def download_artifact(repo, artifact_id, token): + if not isinstance(token, str) or not token.strip(): + raise ValueError("GITHUB_TOKEN is required for historical artifact recovery") + url = f"https://api.github.com/repos/{repo}/actions/artifacts/{artifact_id}/zip" + request = urllib.request.Request( + url, + headers={ + "Authorization": "Bearer " + token, + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "AI-Chip-review-backlog-recovery", + }, + ) + with urllib.request.urlopen(request, timeout=90) as response: + return response.read() + + +def read_json_member(archive, basename): + matches = [name for name in archive.namelist() if Path(name).name == basename] + if len(matches) != 1: + raise ValueError(f"artifact must contain exactly one {basename}") + try: + return json.loads(archive.read(matches[0]).decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError(f"invalid {basename} in historical artifact") from error + + +def extract_claims(archive_bytes, manifest): + try: + archive = zipfile.ZipFile(io.BytesIO(archive_bytes)) + except (zipfile.BadZipFile, OSError) as error: + raise ValueError(f"artifact {manifest['artifact_id']} is not a valid ZIP") from error + with archive: + discovery = read_json_member(archive, "ai-chip-intelligence-discovery.json") + claims_report = read_json_member(archive, "ai-chip-intelligence-claims.json") + graph = read_json_member(archive, "ai-chip-intelligence-evidence-graph.json") + + rows = claims_report.get("claims") + if claims_report.get("schema_version") != "1.0" or not isinstance(rows, list): + raise ValueError("historical Claim Extraction schema mismatch") + if claims_report.get("claim_count") != len(rows): + raise ValueError("historical Claim Extraction count mismatch") + if discovery.get("schema_version") != "1.1" or not isinstance(discovery.get("review_queue"), list): + raise ValueError("historical discovery schema mismatch") + generated = parse_utc(discovery.get("generated_at")) + + # Recompute the full (not top-100 truncated) deterministic triage metadata + # with the same scoring implementation. This is a routing score, not truth. + triage = build_queue(discovery, claims_report, graph, now=generated, top=1000) + priorities = {} + for group in triage.get("queue", []): + priority = { + "score": group.get("score", 0), + "score_factors": group.get("score_factors", {}), + "risk_flags": group.get("risk_flags", []), + "related_claim_ids": group.get("related_claim_ids", []), + "duplicate_collapsed_count": group.get("duplicate_collapsed_count", 0), + "ranking_method": triage.get("ranking_method"), + "score_is_truth_confidence": False, + } + for claim_id in group.get("related_claim_ids", []): + if claim_id in priorities: + raise ValueError("historical triage repeats a Claim ID") + priorities[claim_id] = priority + + recovered = {} + for claim in rows: + claim_id = claim.get("claim_id") if isinstance(claim, dict) else None + if not isinstance(claim_id, str) or not claim_id: + raise ValueError("historical Claim has an invalid ID") + if claim_id in recovered: + raise ValueError("historical artifact repeats a Claim ID") + if claim.get("review_status") != "unverified_candidate": + raise ValueError("historical Claim is not marked unverified") + if not isinstance(claim.get("statement"), str) or not claim["statement"].strip(): + raise ValueError("historical Claim has no statement") + if not isinstance(claim.get("source"), dict) or not claim["source"].get("url"): + raise ValueError("historical Claim has no source provenance") + if claim_id not in priorities: + raise ValueError("historical Claim is missing from recomputed triage") + recovered[claim_id] = { + "claim_snapshot": copy.deepcopy(claim), + "priority": copy.deepcopy(priorities[claim_id]), + "generated_at": iso(generated), + "artifact_id": manifest["artifact_id"], + "run_id": manifest["run_id"], + } + if set(priorities) != set(recovered): + raise ValueError("recomputed triage does not cover every historical Claim") + return recovered, generated + + +def hydrate_state(state, *, repo, token, fetcher=download_artifact, + artifact_manifest=ARTIFACT_MANIFEST, now=None): + now = now or datetime.now(timezone.utc) + if now.tzinfo is None: + raise ValueError("now must be timezone-aware") + now = now.astimezone(timezone.utc) + if not isinstance(state, dict) or state.get("schema_version") != "1.0" or not isinstance(state.get("items"), dict): + raise ValueError("review cache must use schema 1.0 with an items object") + + items = state["items"] + missing_payload = [ + claim_id for claim_id, row in items.items() + if not isinstance(row, dict) or not isinstance(row.get("claim_snapshot"), dict) + ] + if not items: + return { + "schema_version": "1.0", "status": "skipped_empty_state", + "generated_at": iso(now), "state_item_count": 0, + "recovered_claim_count": 0, "hydrated_item_count": 0, + "missing_claim_ids": [], "artifact_ids": [], + "note": "No persisted review backlog exists yet; normal discovery will retain payloads for new Claims.", + }, False + if not missing_payload: + return { + "schema_version": "1.0", "status": "skipped_already_hydrated", + "generated_at": iso(now), "state_item_count": len(items), + "recovered_claim_count": 0, "hydrated_item_count": 0, + "missing_claim_ids": [], "artifact_ids": [], + "note": "Every persisted review item already has a Claim payload.", + }, False + + recovered = {} + first_seen = {} + last_seen = {} + artifact_ids = [] + download_errors = [] + for manifest in artifact_manifest: + artifact_id = manifest["artifact_id"] + try: + payload = fetcher(repo, artifact_id, token) + except (OSError, TimeoutError, ConnectionError, urllib.error.URLError) as error: + download_errors.append({ + "artifact_id": artifact_id, + "error_type": type(error).__name__, + }) + continue + rows, generated = extract_claims(payload, manifest) + artifact_ids.append(artifact_id) + stamp = iso(generated) + for claim_id, record in rows.items(): + first_seen[claim_id] = min(first_seen.get(claim_id, stamp), stamp) + last_seen[claim_id] = max(last_seen.get(claim_id, stamp), stamp) + previous = recovered.get(claim_id) + if previous: + old_claim = previous["claim_snapshot"] + new_claim = record["claim_snapshot"] + if (old_claim.get("candidate_id") != new_claim.get("candidate_id") + or old_claim.get("statement") != new_claim.get("statement")): + raise ValueError("same historical Claim ID has conflicting identity/content") + if record["generated_at"] >= previous["generated_at"]: + recovered[claim_id] = record + else: + recovered[claim_id] = record + + if not recovered: + return { + "schema_version": "1.0", "status": "unavailable", + "generated_at": iso(now), "state_item_count": len(items), + "recovered_claim_count": 0, "hydrated_item_count": 0, + "missing_claim_ids": sorted(missing_payload), + "artifact_ids": artifact_ids, "download_errors": download_errors, + "note": "No historical artifacts could be downloaded; cache was left unchanged.", + }, False + + hydrated = 0 + added = 0 + for claim_id, record in recovered.items(): + row = items.get(claim_id) + if row is None: + row = { + "first_seen_at": first_seen[claim_id], + "last_seen_at": last_seen[claim_id], + } + items[claim_id] = row + added += 1 + if not isinstance(row, dict): + raise ValueError("review cache contains an invalid Claim record") + snapshot = row.get("claim_snapshot") + origin = row.get("snapshot_origin") or {} + if not isinstance(snapshot, dict) or record["generated_at"] >= origin.get("generated_at", ""): + row["claim_snapshot"] = record["claim_snapshot"] + row["priority"] = record["priority"] + row["snapshot_origin"] = { + "artifact_id": record["artifact_id"], + "run_id": record["run_id"], + "generated_at": record["generated_at"], + } + row["score"] = record["priority"]["score"] + row["score_factors"] = record["priority"]["score_factors"] + row["risk_flags"] = record["priority"]["risk_flags"] + row["source_url"] = record["claim_snapshot"]["source"]["url"] + row["topic"] = record["claim_snapshot"].get("topic") + hydrated += 1 + + missing_after = sorted( + claim_id for claim_id, row in items.items() + if not isinstance(row, dict) or not isinstance(row.get("claim_snapshot"), dict) + ) + status = "complete" if not missing_after and not download_errors else "partial" + report = { + "schema_version": "1.0", "status": status, "generated_at": iso(now), + "state_item_count": len(items), "recovered_claim_count": len(recovered), + "hydrated_item_count": hydrated, "new_state_items_added": added, + "claim_payloads_available": len(items) - len(missing_after), + "claim_payloads_missing": len(missing_after), + "missing_claim_ids": missing_after, + "artifact_ids": artifact_ids, "download_errors": download_errors, + "priority_model": "transparent_rule_based_v1_recomputed_from_original_run", + "priority_is_truth_confidence": False, + "decisions_changed": 0, + "note": "Recovered Claim text and source excerpts remain unverified candidates; no review decision or factual status was changed.", + } + state["backlog_hydration"] = { + "status": status, "hydrated_at": iso(now), + "artifact_ids": artifact_ids, "claim_payloads_available": report["claim_payloads_available"], + "claim_payloads_missing": report["claim_payloads_missing"], + } + return report, True + + +def markdown_report(report): + lines = [ + "# Historical Claim Backlog Recovery", "", + f"- Status: **{report['status']}**", + f"- Cache items: {report['state_item_count']}", + f"- Recovered unique Claims: {report['recovered_claim_count']}", + f"- Newly hydrated cache items: {report['hydrated_item_count']}", + f"- Claim payloads available: {report.get('claim_payloads_available', 0)}", + f"- Claim payloads still missing: {report.get('claim_payloads_missing', 0)}", + f"- Artifacts read: {len(report.get('artifact_ids', []))}", + f"- Human decisions changed: {report.get('decisions_changed', 0)}", + "", + "All restored records remain unverified Claim candidates. Priority scores route review work; they are not truth-confidence scores.", + ] + if report.get("download_errors"): + lines.extend(["", "## Artifacts unavailable", ""]) + lines.extend(f"- Artifact {row['artifact_id']}: {row['error_type']}" for row in report["download_errors"]) + if report.get("missing_claim_ids"): + lines.extend(["", "## Claim IDs without recovered payloads", ""]) + lines.extend(f"- {claim_id}" for claim_id in report["missing_claim_ids"][:50]) + return "\n".join(lines) + "\n" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--state", type=Path, required=True) + parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "basicmi/AI-Chip")) + parser.add_argument("--token", default=os.environ.get("GITHUB_TOKEN")) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--markdown", type=Path, required=True) + args = parser.parse_args() + try: + state = json.loads(args.state.read_text(encoding="utf-8")) if args.state.exists() else { + "schema_version": "1.0", "items": {} + } + report, changed = hydrate_state(state, repo=args.repo, token=args.token) + if changed: + args.state.parent.mkdir(parents=True, exist_ok=True) + args.state.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8") + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + args.markdown.parent.mkdir(parents=True, exist_ok=True) + args.markdown.write_text(markdown_report(report), encoding="utf-8") + print(f"Review backlog hydration: {report['status']}; {report.get('claim_payloads_available', 0)} payloads available.") + return 0 + except (OSError, ValueError, TypeError, KeyError, zipfile.BadZipFile) as error: + print("FAIL: " + str(error), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/track_intelligence_review_state.py b/scripts/track_intelligence_review_state.py index d98cfa6..6a6ae52 100644 --- a/scripts/track_intelligence_review_state.py +++ b/scripts/track_intelligence_review_state.py @@ -57,7 +57,8 @@ def update_state(triage, claims, previous=None, ledger=None, *, now=None, retent raise ValueError("expected triage schema 1.0") if not isinstance(claims,dict) or claims.get("schema_version")!="1.0" or not isinstance(claims.get("claims"),list): raise ValueError("expected Claim Extraction schema 1.0") - ids={c.get("claim_id") for c in claims["claims"]} + claim_by_id={c.get("claim_id"):c for c in claims["claims"]} + ids=set(claim_by_id) if len(ids)!=len(claims["claims"]) or any(not isinstance(cid,str) or not cid for cid in ids): raise ValueError("Claim IDs must be unique and non-empty") if any(c.get("review_status")!="unverified_candidate" for c in claims["claims"]): @@ -80,7 +81,13 @@ def update_state(triage, claims, previous=None, ledger=None, *, now=None, retent priority_by_id={} for group in triage["queue"]: for cid in group["related_claim_ids"]: - priority_by_id[cid]={"score":group.get("score",0),"source_url":group.get("source_url"),"topic":group.get("topic")} + priority_by_id[cid]={ + "score":group.get("score",0),"source_url":group.get("source_url"),"topic":group.get("topic"), + "score_factors":group.get("score_factors",{}),"risk_flags":group.get("risk_flags",[]), + "related_claim_ids":group.get("related_claim_ids",[]), + "duplicate_collapsed_count":group.get("duplicate_collapsed_count",0), + "score_is_truth_confidence":False, + } for cid in ids: row=items.get(cid) if row is None: @@ -90,10 +97,16 @@ def update_state(triage, claims, previous=None, ledger=None, *, now=None, retent first=parse_utc(row.get("first_seen_at")) last=parse_utc(row.get("last_seen_at")) if first>now or last>now or first>last: raise ValueError("persisted claim timestamps are inconsistent") - row={"first_seen_at":iso(first),"last_seen_at":iso(now)} + row=copy.deepcopy(row) + row["first_seen_at"]=iso(first) + row["last_seen_at"]=iso(now) + # Retain the full source-grounded Claim payload across change-only + # discovery runs. This is still an unverified candidate, not approval. + row["claim_snapshot"]=copy.deepcopy(claim_by_id[cid]) meta=priority_by_id.get(cid) if meta: - row.update(meta) + row.update({key:meta[key] for key in ("score","source_url","topic")}) + row["priority"]=copy.deepcopy(meta) items[cid]=row # Keep historical IDs long enough to survive temporary source-feed gaps; # do not assert that absent Claims have been reviewed or rejected. @@ -118,6 +131,7 @@ def update_state(triage, claims, previous=None, ledger=None, *, now=None, retent "review_state":state,"alert_level":level, "decision_recorded_at":decision["decided_at"] if decision else None, "decision_reviewer":decision["reviewer"] if decision else None, + "claim_payload_available":isinstance(items[cid].get("claim_snapshot"),dict), "review_is_factual_verification":False}) lookup={x["claim_id"]:x for x in aging} presentation=[] @@ -131,16 +145,26 @@ def update_state(triage, claims, previous=None, ledger=None, *, now=None, retent counts={key:sum(x["review_state"]==key for x in aging) for key in sorted(REVIEW_STATUSES)} alert_counts={level:sum(x["alert_level"]==level for x in aging) for level in ("overdue_7d","overdue_14d","overdue_30d","follow_up_needed")} + unresolved=[] + for row in aging: + if row["review_state"] not in ("pending","needs_more_evidence"): + continue + enriched=dict(row) + enriched["claim_snapshot"]=copy.deepcopy(items[row["claim_id"]].get("claim_snapshot")) + enriched["priority"]=copy.deepcopy(items[row["claim_id"]].get("priority")) + unresolved.append(enriched) + unresolved.sort(key=lambda x:(-x["days_since_first_seen"],-(x["last_priority_score"] or 0),x["claim_id"])) + payloads_available=sum(isinstance(items[cid].get("claim_snapshot"),dict) for cid in items) output={"schema_version":"1.0","generated_at":iso(now),"observation_state":"cache_persisted", "state_is_approval":False,"summary":{"tracked_current_claims":len(ids),"tracked_backlog_claims":len(items), "backlog_not_observed_this_run":len(items)-len(ids),"review_states":counts, - "aging_alerts":alert_counts,"unknown_first_seen_if_cache_reset":True}, - "claim_aging":aging,"queue":presentation, - "aging_backlog":sorted( - [row for row in aging if row["review_state"] in ("pending","needs_more_evidence")], - key=lambda x:(-x["days_since_first_seen"],-(x["last_priority_score"] or 0),x["claim_id"])), - "policy":"First-seen time is when this cache first observed a Claim, not its publication date. Missing cache resets age. Only committed explicit review decisions change review state. None of these decisions verifies factual claims."} + "aging_alerts":alert_counts,"claim_payloads_available":payloads_available, + "claim_payloads_missing":len(items)-payloads_available,"unknown_first_seen_if_cache_reset":True}, + "claim_aging":aging,"queue":presentation,"aging_backlog":unresolved, + "policy":"First-seen time is when this cache first observed a Claim, not its publication date. Missing cache resets age. Claim snapshots preserve source text but remain unverified. Only committed explicit review decisions change review state; no decision verifies factual claims."} state={"schema_version":"1.0","last_run_at":iso(now),"items":items} + if previous.get("backlog_hydration"): + state["backlog_hydration"]=copy.deepcopy(previous["backlog_hydration"]) return output,state def markdown(report): @@ -153,8 +177,21 @@ def markdown(report): f"- Overdue 7/14/30 days: {s['aging_alerts']['overdue_7d']} / {s['aging_alerts']['overdue_14d']} / {s['aging_alerts']['overdue_30d']}", "- **Age is based on first discovery in the persistent cache, not paper publication.**", "- Decisions do not establish independent factual verification.","","## Unhandled high-priority items",""] + lines.append(f"- Claim payloads available: {s.get('claim_payloads_available',0)} / {s['tracked_backlog_claims']}") + lines.append("") for item in report["aging_backlog"][:40]: - lines.append(f"- {item['claim_id']} · wait {item['days_since_first_seen']}d · latest score {item.get('last_priority_score')} · alert {item['alert_level']} · observed this run {item['observed_this_run']}") + claim=item.get("claim_snapshot") or {} + statement=" ".join(str(claim.get("statement") or "[Claim text not recovered]").split()) + source=claim.get("source") or {} + quote=" ".join(str(claim.get("evidence_quote") or "").split()) + lines.append(f"### {item['claim_id']} · priority {item.get('last_priority_score')} · wait {item['days_since_first_seen']}d") + lines.append(f"- Topic/type: {claim.get('topic') or item.get('topic') or 'unknown'} / {claim.get('claim_type') or 'unknown'}") + lines.append(f"- Claim candidate: {statement}") + lines.append(f"- Source: {source.get('url') or item.get('source_url') or 'unavailable'}") + if quote: + lines.append(f"- Source excerpt: {quote}") + lines.append(f"- Review state: {item['review_state']} · alert: {item['alert_level']} · observed this run: {item['observed_this_run']}") + lines.append("") return "\n".join(lines)+"\n" def main(): diff --git a/tests/test_intelligence_backlog_hydration.py b/tests/test_intelligence_backlog_hydration.py new file mode 100644 index 0000000..9dd4ccc --- /dev/null +++ b/tests/test_intelligence_backlog_hydration.py @@ -0,0 +1,152 @@ +"""Offline tests for historical Claim payload recovery.""" +import io +import json +import sys +import unittest +import zipfile +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from hydrate_intelligence_review_backlog import hydrate_state + +NOW = datetime(2026, 11, 17, 12, tzinfo=timezone.utc) + + +def claim(claim_id, candidate_id, statement): + return { + "claim_id": claim_id, "candidate_id": candidate_id, + "claim_type": "performance_result", "assertion_class": "research_result_candidate", + "topic": "performance", + "relevance": {"class": "direct_ai_compute", "matched_signals": ["gpu"]}, + "statement": statement, "value": 123, "unit": "tokens/s", + "scope": {"status": "not_explicitly_stated", "text": None}, "qualifiers": [], + "source": { + "candidate_id": candidate_id, "source_id": "example-source", + "source_name": "Example source", "publisher": "Example", + "source_type": "vendor_primary", "independence": "not_independent", + "url": "https://example.org/" + claim_id, + "published_at": "2026-11-16T00:00:00Z", + "evidence_scope": "vendor_primary_statement", + }, + "evidence_class": "vendor_primary_statement", "evidence_quote": statement, + "extraction": {"method": "deterministic_rules_v4", "confidence": "low", "requires_semantic_review": True}, + "review_status": "unverified_candidate", + "review_gates": ["Verify the original source and exact configuration."], + } + + +def artifact_bytes(): + claims = [ + claim("claim-a", "candidate-a", "Example accelerator reaches 123 tokens/s."), + claim("claim-b", "candidate-b", "Example GPU supports a new memory mode."), + ] + discovery = { + "schema_version": "1.1", "generated_at": "2026-11-16T10:00:00Z", + "review_queue": [ + {"candidate_id": "candidate-a", "change_status": "new"}, + {"candidate_id": "candidate-b", "change_status": "updated"}, + ], + } + claim_report = { + "schema_version": "1.0", "generated_at": discovery["generated_at"], + "claim_count": len(claims), "claims": claims, + } + graph = { + "schema_version": "1.0", "generated_at": discovery["generated_at"], + "input_claim_count": len(claims), "summary": {"independently_verified_claims": 0}, + "nodes": [{"type": "claim_candidate", "claim_id": row["claim_id"]} for row in claims], + "scoped_comparisons": [], + } + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("ai-chip-intelligence-discovery.json", json.dumps(discovery)) + archive.writestr("ai-chip-intelligence-claims.json", json.dumps(claim_report)) + archive.writestr("ai-chip-intelligence-evidence-graph.json", json.dumps(graph)) + return buffer.getvalue() + + +def state_with(ids): + return { + "schema_version": "1.0", "last_run_at": "2026-11-16T10:00:00Z", + "items": { + claim_id: {"first_seen_at": "2026-11-16T10:00:00Z", "last_seen_at": "2026-11-16T10:00:00Z"} + for claim_id in ids + }, + } + + +class HistoricalBacklogHydrationTests(unittest.TestCase): + manifest = [{"artifact_id": 123, "run_id": 456}] + + def test_recovers_payload_and_recomputes_priority_without_decisions(self): + state = state_with(["claim-a", "claim-b"]) + payload, calls = artifact_bytes(), [] + def fetcher(repo, artifact_id, token): + calls.append((repo, artifact_id, token)) + return payload + report, changed = hydrate_state( + state, repo="basicmi/AI-Chip", token="test-token", + fetcher=fetcher, artifact_manifest=self.manifest, now=NOW, + ) + self.assertTrue(changed) + self.assertEqual(report["status"], "complete") + self.assertEqual(report["recovered_claim_count"], 2) + self.assertEqual(report["hydrated_item_count"], 2) + self.assertEqual(report["claim_payloads_missing"], 0) + self.assertEqual(calls, [("basicmi/AI-Chip", 123, "test-token")]) + self.assertIn("Example accelerator", state["items"]["claim-a"]["claim_snapshot"]["statement"]) + self.assertGreater(state["items"]["claim-a"]["score"], 0) + self.assertFalse(state["items"]["claim-a"]["priority"]["score_is_truth_confidence"]) + self.assertEqual(state["items"]["claim-a"]["first_seen_at"], "2026-11-16T10:00:00Z") + self.assertEqual(state["backlog_hydration"]["status"], "complete") + + def test_reports_state_ids_that_have_no_recovered_payload(self): + state = state_with(["claim-a", "claim-b", "claim-missing"]) + report, changed = hydrate_state( + state, repo="basicmi/AI-Chip", token="test-token", + fetcher=lambda *_: artifact_bytes(), + artifact_manifest=self.manifest, now=NOW, + ) + self.assertTrue(changed) + self.assertEqual(report["status"], "partial") + self.assertEqual(report["missing_claim_ids"], ["claim-missing"]) + self.assertEqual(report["claim_payloads_available"], 2) + + def test_skips_network_when_every_item_is_already_hydrated(self): + state = state_with(["claim-a"]) + state["items"]["claim-a"]["claim_snapshot"] = {"claim_id": "claim-a", "statement": "already here"} + report, changed = hydrate_state( + state, repo="basicmi/AI-Chip", token="test-token", + fetcher=lambda *_: self.fail("should not download artifacts"), + artifact_manifest=self.manifest, now=NOW, + ) + self.assertFalse(changed) + self.assertEqual(report["status"], "skipped_already_hydrated") + + def test_skips_empty_cache_without_network(self): + state = {"schema_version": "1.0", "items": {}} + report, changed = hydrate_state( + state, repo="basicmi/AI-Chip", token="test-token", + fetcher=lambda *_: self.fail("should not download artifacts"), + artifact_manifest=self.manifest, now=NOW, + ) + self.assertFalse(changed) + self.assertEqual(report["status"], "skipped_empty_state") + + def test_network_failure_leaves_cache_unchanged(self): + state = state_with(["claim-a"]) + before = json.dumps(state, sort_keys=True) + report, changed = hydrate_state( + state, repo="basicmi/AI-Chip", token="test-token", + fetcher=lambda *_: (_ for _ in ()).throw(TimeoutError("synthetic timeout")), + artifact_manifest=self.manifest, now=NOW, + ) + self.assertFalse(changed) + self.assertEqual(report["status"], "unavailable") + self.assertEqual(json.dumps(state, sort_keys=True), before) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_intelligence_review_state.py b/tests/test_intelligence_review_state.py index 5df6d15..8dd702f 100644 --- a/tests/test_intelligence_review_state.py +++ b/tests/test_intelligence_review_state.py @@ -99,6 +99,19 @@ def test_backlog_survives_change_only_discovery_gap(self): result,_=update_state(empty_triage,empty_claims,new,ledger,now=later) self.assertEqual(result["summary"]["review_states"]["rejected"],1) self.assertEqual(result["summary"]["review_states"]["pending"],2) + def test_full_claim_payload_survives_change_only_discovery_gap(self): + d,c,g=fixtures() + triage=__import__("triage_intelligence_claims").build_queue(d,c,g,now=NOW) + first,state=update_state(triage,c,now=NOW) + self.assertEqual(first["summary"]["claim_payloads_available"],3) + self.assertEqual(state["items"]["a"]["claim_snapshot"]["statement"],c["claims"][0]["statement"]) + empty_triage={"schema_version":"1.0","queue":[],"summary":{"input_claims":0,"automatically_verified":0}} + empty_claims={"schema_version":"1.0","claims":[]} + later,state2=update_state(empty_triage,empty_claims,state,now=NOW+timedelta(days=16)) + self.assertEqual(later["summary"]["claim_payloads_available"],3) + self.assertEqual(state2["items"]["a"]["claim_snapshot"]["statement"],c["claims"][0]["statement"]) + self.assertTrue(any(x["claim_id"]=="a" and x["claim_snapshot"]["statement"]==c["claims"][0]["statement"] for x in later["aging_backlog"])) + def test_claim_promotion_disallowed(self): d,c,g=fixtures() triage=__import__("triage_intelligence_claims").build_queue(d,c,g,now=NOW)