Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions data/intelligence-retrieval-experiments.json
Original file line number Diff line number Diff line change
@@ -1,32 +1,36 @@
{
"schema_version": "1.0",
"experiment_id": "arxiv-targeted-query-expansion-2026-11-v1",
"purpose": "Isolated one-page experiment to measure incremental retrieval coverage for GPU communication, inference economics and agentic infrastructure, and physical-AI runtime terms. No production query changes.",
"experiment_id": "arxiv-targeted-query-expansion-2026-11-v2-deep-pages",
"purpose": "Isolated two-page experiment to measure incremental retrieval coverage for GPU communication, inference economics and agentic infrastructure, and physical-AI runtime terms. No production query changes.",
"max_results_per_group": 50,
"groups": [
{
"id": "gpu-communication",
"name": "GPU communication and interconnect",
"max_results": 50,
"query": "cat:cs.DC AND (all:\"GPU-initiated communication\" OR all:\"GPU communication\" OR all:\"GPU networking\" OR all:NCCL OR all:NVSHMEM OR all:RDMA OR all:\"all-to-all\")"
"query": "cat:cs.DC AND (all:\"GPU-initiated communication\" OR all:\"GPU communication\" OR all:\"GPU networking\" OR all:NCCL OR all:NVSHMEM OR all:RDMA OR all:\"all-to-all\")",
"max_pages": 2
},
{
"id": "inference-energy",
"name": "LLM inference energy and KV cache",
"max_results": 50,
"query": "(cat:cs.LG OR cat:cs.AI) AND (all:\"LLM inference\" OR all:\"inference energy\" OR all:\"energy cost\" OR all:\"KV-cache\" OR all:\"agentic serving\" OR all:\"inference power\")"
"query": "(cat:cs.LG OR cat:cs.AI) AND (all:\"LLM inference\" OR all:\"inference energy\" OR all:\"energy cost\" OR all:\"KV-cache\" OR all:\"agentic serving\" OR all:\"inference power\")",
"max_pages": 2
},
{
"id": "agentic-gpu-capacity",
"name": "Agentic GPU capacity and short-lived jobs",
"max_results": 50,
"query": "(cat:cs.DC OR cat:cs.AI OR cat:cs.LG) AND (all:\"warm GPU capacity\" OR all:\"GPU job scheduling\" OR all:\"agentic workload\" OR all:\"reinforcement learning workload\" OR all:\"warm pool\" OR all:\"GPU capacity\")"
"query": "(cat:cs.DC OR cat:cs.AI OR cat:cs.LG) AND (all:\"warm GPU capacity\" OR all:\"GPU job scheduling\" OR all:\"agentic workload\" OR all:\"reinforcement learning workload\" OR all:\"warm pool\" OR all:\"GPU capacity\")",
"max_pages": 2
},
{
"id": "physical-ai-runtime",
"name": "Physical-AI runtime and world-action models",
"max_results": 50,
"query": "(cat:cs.RO OR cat:cs.CV OR cat:cs.AI) AND (all:\"world-action model\" OR all:\"robot policies\" OR all:\"runtime adaptation\" OR all:\"action-interface mismatch\" OR all:\"state-action-effect\" OR all:\"mobile robotic mission\" OR all:\"robot learning\" OR all:\"world model\")"
"query": "(cat:cs.RO OR cat:cs.CV OR cat:cs.AI) AND (all:\"world-action model\" OR all:\"robot policies\" OR all:\"runtime adaptation\" OR all:\"action-interface mismatch\" OR all:\"state-action-effect\" OR all:\"mobile robotic mission\" OR all:\"robot learning\" OR all:\"world model\")",
"max_pages": 2
}
]
}
27 changes: 27 additions & 0 deletions docs/intelligence-retrieval-deeper-experiment.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# AI Chip — deeper targeted arXiv retrieval experiment

This experiment increases the isolated targeted-query depth from one page to
at most two pages per query group. Each page remains capped at 50 raw arXiv
entries, so the experiment is bounded to at most 100 raw entries per group and
400 across the four configured groups.

The production arXiv source registry is unchanged. The experiment still uses
the same 45-day discovery window, date filtering, Atom parser, retry policy,
known-item seed set, candidate de-duplication and human-review sampling.
Pagination diagnostics record pages fetched, raw entries scanned and whether
the configured page cap was reached.

The purpose is to measure marginal coverage beyond the first 50 results:
- additional unique papers beyond the production baseline;
- additional known-item seed hits;
- overlap across groups and with baseline;
- the share of query groups that still saturate the second-page cap.

A second full page is a signal that the experiment may still be truncated; it
is not proof that more relevant papers exist. Candidate relevance remains
unverified until independent human labels are recorded. The experiment does
not modify production queries or feed its candidates into the production
review queue.

The scheduled Discovery workflow runs weekly, or this experiment can be
rerun by manually dispatching that workflow after the PR is merged.
8 changes: 6 additions & 2 deletions scripts/run_intelligence_retrieval_experiment.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ def validate_config(config):
limit = group.get("max_results", 50)
if type(limit) is not int or not 1 <= limit <= 50:
raise ValueError(prefix + ".max_results must be 1..50")
pages = group.get("max_pages", 1)
if type(pages) is not int or not 1 <= pages <= 2:
raise ValueError(prefix + ".max_pages must be 1..2")
return groups


Expand Down Expand Up @@ -110,7 +113,8 @@ def run_experiment(config, discovery, baseline, *, fetcher=fetch_text, sleep=tim
source = {
"id": "experiment-" + group["id"], "name": group["name"],
"kind": "arxiv_api", "query": group["query"],
"max_results": group.get("max_results", 50), "max_pages": 1,
"max_results": group.get("max_results", 50),
"max_pages": group.get("max_pages", 1),
"category": "retrieval-experiment",
}
try:
Expand Down Expand Up @@ -220,7 +224,7 @@ def run_experiment(config, discovery, baseline, *, fetcher=fetch_text, sleep=tim
},
"groups": group_reports,
"interpretation_limits": [
"This is a one-page, one-run query experiment, not a statistically controlled or exhaustive search.",
"This is a bounded, one-run query experiment (at most two pages per group), not a statistically controlled or exhaustive search.",
"Known-seed recall uses a curated 16-paper calibration set and is not corpus-wide recall.",
"New-result relevance precision is not measured because experiment results are not independently human-labelled.",
"Rule-screening labels are heuristic triage only and must not be treated as verified relevance.",
Expand Down
44 changes: 44 additions & 0 deletions tests/test_intelligence_retrieval_experiment.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ def test_config_bounds_group_count_query_size_and_page_size(self):
bad = {**CONFIG, "groups": [{**CONFIG["groups"][0], "query": "https://example.com"}]}
with self.assertRaisesRegex(ValueError, "not a URL"):
validate_config(bad)
bad = {**CONFIG, "groups": [{**CONFIG["groups"][0], "max_pages": 3}]}
with self.assertRaisesRegex(ValueError, "max_pages must be 1..2"):
validate_config(bad)

def test_incremental_hits_dedup_overlap_and_courtesy_delay(self):
calls, delays = [], []
Expand Down Expand Up @@ -87,6 +90,47 @@ def fetch(url):
self.assertIsNone(report["summary"]["measured_precision"])
self.assertIn("not available", markdown_report(report))

def test_second_page_adds_incremental_candidates_and_is_counted(self):
config = {
**CONFIG,
"groups": [
{**CONFIG["groups"][0], "max_pages": 2},
CONFIG["groups"][1],
],
}
calls, delays = [], []

def fetch(url):
params = parse_qs(urlsplit(url).query)
query = params["search_query"][0]
start = int(params["start"][0])
calls.append((query, start))
if "GPU" in query:
return atom(
["2611.12346v1", "2611.12347v1"]
if start == 0 else ["2611.12349v1", "2611.12350v1"]
)
return atom(["2611.12346v2", "2611.12348v1"])

report = run_experiment(config, DISCOVERY, BASELINE, fetcher=fetch, sleep=delays.append)
self.assertEqual(report["status"], "complete")
self.assertEqual(calls, [
("cat:cs.DC AND all:GPU", 0),
("cat:cs.DC AND all:GPU", 2),
("cat:cs.AI AND all:agent", 0),
])
self.assertEqual(delays, [3, 3])
first = report["groups"][0]
self.assertEqual(first["paging"]["pages_fetched"], 2)
self.assertEqual(first["paging"]["raw_items_scanned"], 4)
self.assertTrue(first["paging"]["paging_cap_reached"])
self.assertEqual(first["candidate_observation_count"], 4)
self.assertEqual(report["summary"]["request_count"], 3)
self.assertEqual(report["summary"]["api_raw_items_scanned"], 6)
self.assertEqual(report["summary"]["candidate_observation_count"], 6)
self.assertEqual(report["summary"]["unique_papers_across_groups"], 5)
self.assertEqual(report["summary"]["incremental_unique_papers"], 5)

def test_failed_query_makes_experiment_partial_not_negative(self):
calls, delays = [], []

Expand Down
Loading