From 5191a2dd3288f194cb055656047f4ea50c7ffe62 Mon Sep 17 00:00:00 2001 From: Shan Tang Date: Tue, 17 Nov 2026 09:48:28 +0800 Subject: [PATCH 1/2] Extend targeted arXiv retrieval experiment to two bounded pages --- data/intelligence-retrieval-experiments.json | 16 ++++--- ...ntelligence-retrieval-deeper-experiment.md | 27 ++++++++++++ .../run_intelligence_retrieval_experiment.py | 8 +++- .../test_intelligence_retrieval_experiment.py | 44 +++++++++++++++++++ 4 files changed, 87 insertions(+), 8 deletions(-) create mode 100644 docs/intelligence-retrieval-deeper-experiment.md diff --git a/data/intelligence-retrieval-experiments.json b/data/intelligence-retrieval-experiments.json index 6003930..7aa3eea 100644 --- a/data/intelligence-retrieval-experiments.json +++ b/data/intelligence-retrieval-experiments.json @@ -1,32 +1,36 @@ { "schema_version": "1.0", - "experiment_id": "arxiv-targeted-query-expansion-2026-11-v1", - "purpose": "Isolated one-page experiment to measure incremental retrieval coverage for GPU communication, inference economics and agentic infrastructure, and physical-AI runtime terms. No production query changes.", + "experiment_id": "arxiv-targeted-query-expansion-2026-11-v2-deep-pages", + "purpose": "Isolated two-page experiment to measure incremental retrieval coverage for GPU communication, inference economics and agentic infrastructure, and physical-AI runtime terms. No production query changes.", "max_results_per_group": 50, "groups": [ { "id": "gpu-communication", "name": "GPU communication and interconnect", "max_results": 50, - "query": "cat:cs.DC AND (all:\"GPU-initiated communication\" OR all:\"GPU communication\" OR all:\"GPU networking\" OR all:NCCL OR all:NVSHMEM OR all:RDMA OR all:\"all-to-all\")" + "query": "cat:cs.DC AND (all:\"GPU-initiated communication\" OR all:\"GPU communication\" OR all:\"GPU networking\" OR all:NCCL OR all:NVSHMEM OR all:RDMA OR all:\"all-to-all\")", + "max_pages": 2 }, { "id": "inference-energy", "name": "LLM inference energy and KV cache", "max_results": 50, - "query": "(cat:cs.LG OR cat:cs.AI) AND (all:\"LLM inference\" OR all:\"inference energy\" OR all:\"energy cost\" OR all:\"KV-cache\" OR all:\"agentic serving\" OR all:\"inference power\")" + "query": "(cat:cs.LG OR cat:cs.AI) AND (all:\"LLM inference\" OR all:\"inference energy\" OR all:\"energy cost\" OR all:\"KV-cache\" OR all:\"agentic serving\" OR all:\"inference power\")", + "max_pages": 2 }, { "id": "agentic-gpu-capacity", "name": "Agentic GPU capacity and short-lived jobs", "max_results": 50, - "query": "(cat:cs.DC OR cat:cs.AI OR cat:cs.LG) AND (all:\"warm GPU capacity\" OR all:\"GPU job scheduling\" OR all:\"agentic workload\" OR all:\"reinforcement learning workload\" OR all:\"warm pool\" OR all:\"GPU capacity\")" + "query": "(cat:cs.DC OR cat:cs.AI OR cat:cs.LG) AND (all:\"warm GPU capacity\" OR all:\"GPU job scheduling\" OR all:\"agentic workload\" OR all:\"reinforcement learning workload\" OR all:\"warm pool\" OR all:\"GPU capacity\")", + "max_pages": 2 }, { "id": "physical-ai-runtime", "name": "Physical-AI runtime and world-action models", "max_results": 50, - "query": "(cat:cs.RO OR cat:cs.CV OR cat:cs.AI) AND (all:\"world-action model\" OR all:\"robot policies\" OR all:\"runtime adaptation\" OR all:\"action-interface mismatch\" OR all:\"state-action-effect\" OR all:\"mobile robotic mission\" OR all:\"robot learning\" OR all:\"world model\")" + "query": "(cat:cs.RO OR cat:cs.CV OR cat:cs.AI) AND (all:\"world-action model\" OR all:\"robot policies\" OR all:\"runtime adaptation\" OR all:\"action-interface mismatch\" OR all:\"state-action-effect\" OR all:\"mobile robotic mission\" OR all:\"robot learning\" OR all:\"world model\")", + "max_pages": 2 } ] } diff --git a/docs/intelligence-retrieval-deeper-experiment.md b/docs/intelligence-retrieval-deeper-experiment.md new file mode 100644 index 0000000..5424111 --- /dev/null +++ b/docs/intelligence-retrieval-deeper-experiment.md @@ -0,0 +1,27 @@ +# AI Chip — deeper targeted arXiv retrieval experiment + +This experiment increases the isolated targeted-query depth from one page to +at most two pages per query group. Each page remains capped at 50 raw arXiv +entries, so the experiment is bounded to at most 100 raw entries per group and +400 across the four configured groups. + +The production arXiv source registry is unchanged. The experiment still uses +the same 45-day discovery window, date filtering, Atom parser, retry policy, +known-item seed set, candidate de-duplication and human-review sampling. +Pagination diagnostics record pages fetched, raw entries scanned and whether +the configured page cap was reached. + +The purpose is to measure marginal coverage beyond the first 50 results: +- additional unique papers beyond the production baseline; +- additional known-item seed hits; +- overlap across groups and with baseline; +- the share of query groups that still saturate the second-page cap. + +A second full page is a signal that the experiment may still be truncated; it +is not proof that more relevant papers exist. Candidate relevance remains +unverified until independent human labels are recorded. The experiment does +not modify production queries or feed its candidates into the production +review queue. + +The scheduled Discovery workflow runs weekly, or this experiment can be +rerun by manually dispatching that workflow after the PR is merged. diff --git a/scripts/run_intelligence_retrieval_experiment.py b/scripts/run_intelligence_retrieval_experiment.py index c04925d..74cce3f 100644 --- a/scripts/run_intelligence_retrieval_experiment.py +++ b/scripts/run_intelligence_retrieval_experiment.py @@ -45,6 +45,9 @@ def validate_config(config): limit = group.get("max_results", 50) if type(limit) is not int or not 1 <= limit <= 50: raise ValueError(prefix + ".max_results must be 1..50") + pages = group.get("max_pages", 1) + if type(pages) is not int or not 1 <= pages <= 2: + raise ValueError(prefix + ".max_pages must be 1..2") return groups @@ -110,7 +113,8 @@ def run_experiment(config, discovery, baseline, *, fetcher=fetch_text, sleep=tim source = { "id": "experiment-" + group["id"], "name": group["name"], "kind": "arxiv_api", "query": group["query"], - "max_results": group.get("max_results", 50), "max_pages": 1, + "max_results": group.get("max_results", 50), + "max_pages": group.get("max_pages", 1), "category": "retrieval-experiment", } try: @@ -220,7 +224,7 @@ def run_experiment(config, discovery, baseline, *, fetcher=fetch_text, sleep=tim }, "groups": group_reports, "interpretation_limits": [ - "This is a one-page, one-run query experiment, not a statistically controlled or exhaustive search.", + "This is a bounded, one-run query experiment (at most two pages per group), not a statistically controlled or exhaustive search.", "Known-seed recall uses a curated 16-paper calibration set and is not corpus-wide recall.", "New-result relevance precision is not measured because experiment results are not independently human-labelled.", "Rule-screening labels are heuristic triage only and must not be treated as verified relevance.", diff --git a/tests/test_intelligence_retrieval_experiment.py b/tests/test_intelligence_retrieval_experiment.py index a127706..33d039f 100644 --- a/tests/test_intelligence_retrieval_experiment.py +++ b/tests/test_intelligence_retrieval_experiment.py @@ -56,6 +56,9 @@ def test_config_bounds_group_count_query_size_and_page_size(self): bad = {**CONFIG, "groups": [{**CONFIG["groups"][0], "query": "https://example.com"}]} with self.assertRaisesRegex(ValueError, "not a URL"): validate_config(bad) + bad = {**CONFIG, "groups": [{**CONFIG["groups"][0], "max_pages": 3}]} + with self.assertRaisesRegex(ValueError, "max_pages must be 1..2"): + validate_config(bad) def test_incremental_hits_dedup_overlap_and_courtesy_delay(self): calls, delays = [], [] @@ -87,6 +90,47 @@ def fetch(url): self.assertIsNone(report["summary"]["measured_precision"]) self.assertIn("not available", markdown_report(report)) + def test_second_page_adds_incremental_candidates_and_is_counted(self): + config = { + **CONFIG, + "groups": [ + {**CONFIG["groups"][0], "max_pages": 2}, + CONFIG["groups"][1], + ], + } + calls, delays = [], [] + + def fetch(url): + params = parse_qs(urlsplit(url).query) + query = params["search_query"][0] + start = int(params["start"][0]) + calls.append((query, start)) + if "GPU" in query: + return atom( + ["2611.12346v1", "2611.12347v1"] + if start == 0 else ["2611.12349v1", "2611.12350v1"] + ) + return atom(["2611.12346v2", "2611.12348v1"]) + + report = run_experiment(config, DISCOVERY, BASELINE, fetcher=fetch, sleep=delays.append) + self.assertEqual(report["status"], "complete") + self.assertEqual(calls, [ + ("cat:cs.DC AND all:GPU", 0), + ("cat:cs.DC AND all:GPU", 2), + ("cat:cs.AI AND all:agent", 0), + ]) + self.assertEqual(delays, [3, 3]) + first = report["groups"][0] + self.assertEqual(first["paging"]["pages_fetched"], 2) + self.assertEqual(first["paging"]["raw_items_scanned"], 4) + self.assertTrue(first["paging"]["paging_cap_reached"]) + self.assertEqual(first["candidate_observation_count"], 4) + self.assertEqual(report["summary"]["request_count"], 3) + self.assertEqual(report["summary"]["api_raw_items_scanned"], 6) + self.assertEqual(report["summary"]["candidate_observation_count"], 6) + self.assertEqual(report["summary"]["unique_papers_across_groups"], 5) + self.assertEqual(report["summary"]["incremental_unique_papers"], 4) + def test_failed_query_makes_experiment_partial_not_negative(self): calls, delays = [], [] From 9a6549887361cfebe2aa2fb0b5dbf9556fb78c87 Mon Sep 17 00:00:00 2001 From: Shan Tang Date: Tue, 17 Nov 2026 09:48:59 +0800 Subject: [PATCH 2/2] Correct incremental unique-paper assertion for deep-page test --- tests/test_intelligence_retrieval_experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_intelligence_retrieval_experiment.py b/tests/test_intelligence_retrieval_experiment.py index 33d039f..85fc2f1 100644 --- a/tests/test_intelligence_retrieval_experiment.py +++ b/tests/test_intelligence_retrieval_experiment.py @@ -129,7 +129,7 @@ def fetch(url): self.assertEqual(report["summary"]["api_raw_items_scanned"], 6) self.assertEqual(report["summary"]["candidate_observation_count"], 6) self.assertEqual(report["summary"]["unique_papers_across_groups"], 5) - self.assertEqual(report["summary"]["incremental_unique_papers"], 4) + self.assertEqual(report["summary"]["incremental_unique_papers"], 5) def test_failed_query_makes_experiment_partial_not_negative(self): calls, delays = [], []