Quality review + bugfix: v0.3 surface (#111)

Quality pass over the v0.3 surface (foundation forecast-residual baseline, uncertainty calibration, grown/IGSO dataset + splits) before docs and release. Leakage verified clean across all three sub-systems.

- Foundation: a non-finite forecast token no longer zeros a whole channel's residual score (median/MAD fit + standardisation run on finite residuals only), which was silently suppressing detections; adds a NaN regression test and the first cross-track (inclination-channel) detection test.
- Calibration: _as_pairs and TemperatureScaling.transform reject non-finite confidences — a NaN previously slipped past the [0, 1] range check and poisoned ECE/Brier and any fitted temperature/conformal quantile to NaN.
- docs: corrected benchmark.md's reliability-diagram publishing claim to match what the model cards actually publish (per-class operating points; reliability diagnostics are a calibration-API surface).

The latent make_temporal_split per-class running-total imbalance is left as-is (the shipped v0.3 split is non-degenerate; fixing it would regenerate splits.json and invalidate published checkpoint numbers).

Closes #63

Author: djankov

docs/benchmark.md

@@ -88,8 +88,9 @@ ece = expected_calibration_error(leo.confidences, leo.outcomes)
 ```
 
 Calibration only rescales the confidence column — it does not change which gaps a detector fires on, just how
-confident it says it is. The reliability diagrams and per-class operating points for each shipped detector are
-published with the versioned models.
+confident it says it is. The per-class operating points for each shipped detector are published with the
+versioned models; the reliability diagnostics above (`reliability_curve`, `expected_calibration_error`) are
+available from the calibration API for any detector and split.
 
 ## Scorer
 

src/maneuver_detect/calibration.py

@@ -67,6 +67,11 @@ def _as_pairs(confidences: npt.ArrayLike, outcomes: npt.ArrayLike) -> tuple[Floa
         )
     if conf.ndim != 1:
         raise ValueError(f"confidences and outcomes must be 1-D, got {conf.ndim}-D")
+    # A NaN slips past the range check below (``nan < 0`` and ``nan > 1`` are both False), so guard
+    # it explicitly — an unfiltered NaN confidence silently poisons ECE/Brier and biases a fitted
+    # temperature or conformal quantile to NaN with no error.
+    if conf.size and not np.isfinite(conf).all():
+        raise ValueError("confidences must be finite")
     if conf.size and (conf.min() < 0.0 or conf.max() > 1.0):
         raise ValueError("confidences must lie in [0, 1]")
     if out.size and not np.isin(out, (0.0, 1.0)).all():
@@ -239,6 +244,8 @@ def fit(
     def transform(self, confidences: npt.ArrayLike) -> FloatArray:
         """Rescale ``confidences`` through the fitted temperature, staying in ``[0, 1]``."""
         conf = np.asarray(confidences, dtype=np.float64)
+        if conf.size and not np.isfinite(conf).all():
+            raise ValueError("confidences must be finite")
         if conf.size and (conf.min() < 0.0 or conf.max() > 1.0):
             raise ValueError("confidences must lie in [0, 1]")
         return _sigmoid(_logit(conf) / self.temperature)

src/maneuver_detect/detectors/foundation.py

@@ -358,12 +358,21 @@ def _residual_score(self, channels: RawChannels, elements: _AlignedElements) ->
             forecast = self._forecaster.forecast(series)  # type: ignore[union-attr]
             residual = series - forecast.mean  # residual[0] is undefined (no forecast at token 0)
             body = residual[1:]
-            center = float(np.median(body))
-            mad = 1.4826 * float(np.median(np.abs(body - center)))
+            # Centre and scale on the *finite* residuals only. A single non-finite forecast value
+            # (a forecaster edge case, or a cleaned series with an interior gap) would otherwise
+            # poison the per-object median/MAD into NaN, drive the scale to NaN, and zero this
+            # channel's score for every token — silently suppressing real detections. Instead a
+            # non-finite token simply scores 0 (it cannot trigger a detection) while the rest of
+            # the channel scores normally.
+            finite = np.isfinite(body)
+            if not finite.any():
+                continue
+            center = float(np.median(body[finite]))
+            mad = 1.4826 * float(np.median(np.abs(body[finite] - center)))
             scale = max(mad, floor)
             z = np.zeros(n, dtype=np.float64)
-            z[1:] = np.abs(body - center) / scale
-            score = np.maximum(score, np.nan_to_num(z, nan=0.0, posinf=0.0, neginf=0.0))
+            z[1:][finite] = np.abs(body[finite] - center) / scale
+            score = np.maximum(score, z)
         return score
 
 

tests/test_calibration.py

@@ -62,6 +62,7 @@ def test_empty_sample_scores_zero() -> None:
         ([0.5, 0.5], [1.0], "same shape"),
         ([1.5], [1.0], r"\[0, 1\]"),
         ([0.5], [2.0], "0.0 .* or 1.0"),
+        ([float("nan"), 0.5], [1.0, 0.0], "finite"),  # a NaN must not slip past the range check
     ],
 )
 def test_input_guard_rejects_bad_pairs(
@@ -113,6 +114,12 @@ def test_temperature_fit_rejects_empty_sample() -> None:
         TemperatureScaling.fit([], [])
 
 
+def test_temperature_transform_rejects_non_finite() -> None:
+    # A NaN confidence must not pass through transform un-flagged (it would emit a NaN confidence).
+    with pytest.raises(ValueError, match="finite"):
+        TemperatureScaling(temperature=2.0).transform([0.5, float("nan")])
+
+
 # --- split conformal ---------------------------------------------------------------------------
 
 

tests/test_detectors_foundation.py

@@ -36,6 +36,21 @@ def _stand_in_detector(threshold: float | None = None) -> ChronosResidualDetecto
     )
 
 
+class _NaNInjectingForecaster:
+    """The drift stand-in with one non-finite forecast token poked in, to prove a single bad
+    forecast value cannot zero a whole channel's residual score."""
+
+    def __init__(self, bad_index: int) -> None:
+        self._inner = DriftContinuationForecaster()
+        self._bad_index = bad_index
+
+    def forecast(self, series: np.ndarray) -> Forecast:
+        base = self._inner.forecast(series)
+        mean = base.mean.copy()
+        mean[self._bad_index] = np.nan
+        return Forecast(mean=mean, scale=base.scale)
+
+
 def test_foundation_detector_is_registered() -> None:
     assert "chronos-residual" in set(available_models())
     assert isinstance(get_detector("chronos-residual"), ChronosResidualDetector)
@@ -56,6 +71,39 @@ def test_detects_an_injected_in_track_burn() -> None:
     assert strongest["delta_v_estimate"] > 0.0  # a 4 m/s in-track burn clears the LEO floor
 
 
+def test_detects_an_injected_cross_track_burn() -> None:
+    # A cross-track burn steps the inclination, so the detection comes from the *inclination*
+    # channel — every other detection assertion in this suite rides the in-track / semi-major-axis
+    # channel, so this is the one place the cross-track channel is pinned to actually fire.
+    frame = synthetic_series(norad_id=1, seed=3, n=120, burns=(Burn(45, "cross_track_ms", 4.0),))
+    out = _stand_in_detector().detect(frame)
+
+    validate_frame(out)
+    assert not out.empty
+    near_burn = out[
+        (out["elset_epoch_after"] >= frame["epoch"].iloc[44])
+        & (out["elset_epoch_after"] <= frame["epoch"].iloc[46])
+    ]
+    assert not near_burn.empty  # the inclination channel fires on the cross-track burn's gap
+    assert (near_burn["confidence"].between(0.0, 1.0)).all()
+
+
+def test_a_single_nonfinite_forecast_does_not_zero_the_channel() -> None:
+    # A non-finite forecast at an unrelated quiet token must not wipe the whole channel's score:
+    # the in-track burn at token 45 is still detected (the pre-fix median/MAD would NaN the scale
+    # and zero every token of the channel).
+    frame = synthetic_series(norad_id=1, seed=3, n=120, burns=(Burn(45, "in_track_ms", 4.0),))
+    detector = ChronosResidualDetector(
+        forecaster=_NaNInjectingForecaster(bad_index=10),
+        class_thresholds=_THRESHOLDS,
+    )
+    out = detector.detect(frame)
+
+    assert not out.empty
+    strongest = out.loc[out["confidence"].idxmax()]
+    assert frame["epoch"].iloc[44] <= strongest["elset_epoch_after"] <= frame["epoch"].iloc[46]
+
+
 def test_threshold_gates_detection_count() -> None:
     frame = synthetic_series(norad_id=1, seed=4, n=120, burns=(Burn(60, "in_track_ms", 4.0),))
     low = _stand_in_detector(threshold=2.0).detect(frame)