From 4d00313ef23c89f5f7d99821f06d9bc436e906af Mon Sep 17 00:00:00 2001
From: HackPoint <6758579+HackPoint@users.noreply.github.com>
Date: Mon, 12 Oct 2026 15:06:48 +0300
Subject: [PATCH] test(mcp): the A/B test takes files until each arm has some,
 and the timing test alternates its arms and compares medians

The A/B test read 16 files and needed three in each arm, which fails 0.4% of runs because the arm
comes from the path and the paths are new every run. It now takes files until each arm has four,
and checks that the arm a file lands in is the arm the read was answered as.

The timing test ran all reads of one arm, then all of the other, so a load spike hit one arm only,
and a single stalled read decided the result. It now alternates the arms read by read and compares
medians. A repeat path that costs 400 ms more a read still fails it.
---
 crates/lumen-mcp/tests/repeat_reads.rs | 69 +++++++++++++++++---------
 1 file changed, 45 insertions(+), 24 deletions(-)

diff --git a/crates/lumen-mcp/tests/repeat_reads.rs b/crates/lumen-mcp/tests/repeat_reads.rs
index 519c20c..8e96c93 100644
--- a/crates/lumen-mcp/tests/repeat_reads.rs
+++ b/crates/lumen-mcp/tests/repeat_reads.rs
@@ -543,20 +543,31 @@ fn fresh_is_honoured_and_recorded_when_repeat_reads_are_turned_off() {
 fn ab_withholds_the_answer_from_control_files_and_records_what_it_would_have_done() {
     let rig = rig();
     let mut s = rig.start(&[("LUMEN_REPEAT_READS", "ab")]);
+    // The arm comes from the path, new every run: take files until each arm has four.
     let (mut treated, mut control) = (0, 0);
-    for i in 0..16 {
-        let f = rig.file(&format!("f{i}.rs"), &source(30 + i, 5));
+    let mut i = 0;
+    while treated < 4 || control < 4 {
+        assert!(i < 200, "no split in {i} files: {treated}/{control}");
+        let f = rig.file(&format!("f{i}.rs"), &source(30 + i % 16, 5));
+        let in_treatment = lumen_core::repeat::treated(lumen_core::ranked::Mode::Ab, &f);
         s.read(&f);
         let second = s.read(&f);
-        if second.text.starts_with("Unchanged since") {
+        assert_eq!(
+            second.text.starts_with("Unchanged since"),
+            in_treatment,
+            "{f} is in the {} arm: {}",
+            if in_treatment { "treated" } else { "control" },
+            second.text
+        );
+        if in_treatment {
             treated += 1;
         } else {
             assert!(second.text.contains("— outline"));
             control += 1;
         }
+        i += 1;
     }
     s.end();
-    assert!(treated >= 3 && control >= 3, "split {treated}/{control}");
     let rows = rig.rows();
     let ctl: Vec<_> = rows
         .iter()
@@ -656,29 +667,39 @@ fn recall_file_after_compaction_gets_the_full_answer() {
 
 #[test]
 fn a_repeat_costs_no_more_time_than_a_first_read() {
-    let rig = rig();
     let n = 40;
-    let big = rig.file("big.rs", &source(400, 8));
-
-    let time = |env: &[(&str, &str)]| {
-        let mut s = rig.start(env);
-        let t = Instant::now();
-        let mut last = None;
-        for _ in 0..n {
-            last = Some(s.read(&big));
-        }
-        let e = t.elapsed();
-        s.end();
-        (e, last.unwrap())
-    };
+    // The arms alternate, so load lands on both alike; medians, so one stalled read decides nothing.
+    let (rig_off, rig_on) = (rig(), rig());
+    let (big_off, big_on) = (
+        rig_off.file("big.rs", &source(400, 8)),
+        rig_on.file("big.rs", &source(400, 8)),
+    );
     // Off is asked for: with nothing set repeat reads are on, and this compares the two.
-    let (off, _) = time(&[("LUMEN_REPEAT_READS", "off")]);
-    let (on, last) = time(ON);
-    assert!(last.text.starts_with("Unchanged since"));
-    eprintln!("{n} reads of a 400-function file: off {off:?}, on {on:?}");
+    let mut off = rig_off.start(&[("LUMEN_REPEAT_READS", "off")]);
+    let mut on = rig_on.start(ON);
+    let (mut off_secs, mut on_secs) = (vec![], vec![]);
+    let mut last = None;
+    for _ in 0..n {
+        let t = Instant::now();
+        off.read(&big_off);
+        off_secs.push(t.elapsed().as_secs_f64());
+        let t = Instant::now();
+        last = Some(on.read(&big_on));
+        on_secs.push(t.elapsed().as_secs_f64());
+    }
+    off.end();
+    on.end();
+    assert!(last.unwrap().text.starts_with("Unchanged since"));
+    let (off_median, on_median) = (
+        percentile(&mut off_secs, 0.5),
+        percentile(&mut on_secs, 0.5),
+    );
+    eprintln!(
+        "{n} reads of a 400-function file, median read: off {off_median:.4}s, on {on_median:.4}s"
+    );
     assert!(
-        on.as_secs_f64() < off.as_secs_f64() * 1.5 + 1.0,
-        "repeat path slower than the path it replaces: {on:?} vs {off:?}"
+        on_median < off_median * 2.0 + 0.025,
+        "repeat path slower than the path it replaces: {on_median:.4}s vs {off_median:.4}s a read"
     );
 }