From e03de169203578a984e94fed7b6bd5dd965ec72c Mon Sep 17 00:00:00 2001 From: Martyn Garcia Date: Fri, 20 Nov 2026 04:39:44 -0700 Subject: [PATCH 1/8] chore(develop): open standing develop -> master PR after v0.11.0 release From 6808603e1de4a90a007b8f97361802e27d3bd6d9 Mon Sep 17 00:00:00 2001 From: Martyn Garcia Date: Fri, 20 Nov 2026 10:42:44 -0700 Subject: [PATCH 2/8] fix(critic): break autograd hook reference cycle leaking VRAM in _live_captures --- particlegan/_minimal/dynamics.py | 10 +- particlegan/_nash_v3/record.py | 26 ++- particlegan/_t2_critic/critic.py | 29 ++- particlegan/_unified/critic.py | 14 +- particlegan/_unified_v11/critic.py | 14 +- particlegan/_unified_v4/critic.py | 14 +- tests/test_unified_critic_vram_leak.py | 309 +++++++++++++++++++++++++ 7 files changed, 381 insertions(+), 35 deletions(-) create mode 100644 tests/test_unified_critic_vram_leak.py diff --git a/particlegan/_minimal/dynamics.py b/particlegan/_minimal/dynamics.py index 71b4fe51a..7e34e0a46 100644 --- a/particlegan/_minimal/dynamics.py +++ b/particlegan/_minimal/dynamics.py @@ -43,9 +43,8 @@ def _minimal_critic_hook(module: nn.Module, args: Tuple[Any, ...], kwargs: Dict[ if rec is not None and not rec._in_penalty and module.training and torch.is_grad_enabled() and target.requires_grad and no_ig and (x_in.ndim >= 3 or x_in[0].numel() > 8): skey = getattr(module, "_t2_pending_source_key", None) or _tensor_key(x_in) if skey is not None: - unsc = target.float() if target.dtype == torch.bfloat16 else target - entry: List[Any] = [int(rec.observed_steps), skey, _cond_signature(args, kwargs), x_in.detach(), unsc, True] - unsc.register_hook(lambda _g, _e=entry: _e.__setitem__(5, False)); rec._live_captures.append(entry); del rec._live_captures[:-6] + alive = [True]; rec._live_captures.append([int(rec.observed_steps), skey, _cond_signature(args, kwargs), x_in.detach(), target.float() if target.dtype == torch.bfloat16 else target, alive]) + target.register_hook(lambda _g, _a=alive: _a.clear()); [e.__setitem__(slice(3, 5), [None, None]) or e[5].clear() for e in rec._live_captures[:-6]]; del rec._live_captures[:-6] if rec is not None and not rec._in_penalty and torch.is_grad_enabled() and rec._thermal_scale < 1.0 and isinstance(output, torch.Tensor) and output.requires_grad: scaled = output * rec._thermal_scale + output.detach() * (1.0 - rec._thermal_scale) if x_in is not None: scaled._minimal_input, scaled._minimal_record, scaled._minimal_conditioned, scaled._minimal_cond_tensors = x_in, rec, is_cond, cond_t @@ -119,7 +118,8 @@ def observe_cond_fiber(self, r_cond: float) -> None: self.cond_ema = max(0.18, rc) if not self.cond_initialized else (0.99 * self.cond_ema + 0.01 * rc) self.cond_initialized, self._thermal_scale, self._step_scale = True, 1.0, float(1.0 - 0.85 * _smoothstep01((0.050 - max(0.0, self.cond_ema)) / 0.024)) - def record_step(self, optimizer: Any) -> None: super().record_step(optimizer); self._live_captures.clear(); self._g_probe_cache = self._last_real_pair = None + def clear_transient_captures(self) -> None: [e.__setitem__(slice(3, 5), [None, None]) or e[5].clear() for e in self._live_captures]; self._live_captures.clear() + def record_step(self, optimizer: Any) -> None: super().record_step(optimizer); self.clear_transient_captures(); self._g_probe_cache = self._last_real_pair = None def state_dict(self) -> Dict[str, Any]: return {**super().state_dict(), "minimal_state": {"schema": 1, **{k: bool(getattr(self, k)) for k in _BOOL_KEYS}, **{k: float(getattr(self, "_" + k if k in ("thermal_scale", "step_scale") else k)) for k in _BOUNDS}, "cond_bank": deepcopy(self.cond_bank)}} @@ -131,7 +131,7 @@ def load_state_dict(self, state: Dict[str, Any]) -> None: cb = _validate_cond_bank(pkt["cond_bank"]); super().load_state_dict(sc) for k in _BOOL_KEYS: setattr(self, k, pkt[k]) for k, v in vals.items(): setattr(self, "_" + k if k in ("thermal_scale", "step_scale") else k, v) - self.cond_bank, self._g_probe_cache = cb, None; self._live_captures.clear() + self.cond_bank, self._g_probe_cache = cb, None; self.clear_transient_captures() def _collect_conv_layouts(modules: Iterable[Optional[nn.Module]]) -> Dict[int, Tuple[bool, int]]: return {id(c.weight): (isinstance(c, nn.ConvTranspose2d), int(c.groups)) for r in modules if isinstance(r, nn.Module) for c in r.modules() if isinstance(c, (nn.Conv2d, nn.ConvTranspose2d))} diff --git a/particlegan/_nash_v3/record.py b/particlegan/_nash_v3/record.py index 0b8576c3f..48ce53491 100644 --- a/particlegan/_nash_v3/record.py +++ b/particlegan/_nash_v3/record.py @@ -7,7 +7,12 @@ import torch from torch import nn -from .._t2_critic.critic import _cond_signature, _tensor_identity_key +from .._t2_critic.critic import ( + _clear_live_capture_entries, + _cond_signature, + _tensor_identity_key, + _trim_live_captures, +) from ..grad_regularizers import CriticStepRecord FROZEN_THERMAL_SCALE = 1e-4 @@ -163,11 +168,11 @@ def _nash_v3_critic_forward_hook( rec._live_captures = captures c_sig = _cond_signature(args, kwargs) step_tag = int(getattr(rec, "observed_steps", 0)) - entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, True] - unscaled.register_hook(lambda _g, _e=entry: _e.__setitem__(5, False)) + alive = [True] + entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, alive] + target.register_hook(lambda _g, _a=alive: _a.clear()) captures.append(entry) - if len(captures) > _MAX_LIVE_CAPTURES: - del captures[: len(captures) - _MAX_LIVE_CAPTURES] + _trim_live_captures(captures, _MAX_LIVE_CAPTURES) # Low-D secant probe capture for critic inversion-deficit trust band (skipped on images for speed). if ( @@ -363,7 +368,8 @@ def _init_nash_v3_fields(self) -> None: self._in_probe: bool = False def clear_transient_captures(self) -> None: - self._live_captures = [] + if hasattr(self, "_live_captures") and self._live_captures: + _clear_live_capture_entries(self._live_captures) self._last_probe_actual_input = None def observe_transport(self, res: float, data_scale: float, ema_decay: float = 0.92) -> None: @@ -465,8 +471,8 @@ def _restore_nash_v3_packet(self, packet: Dict[str, Any]) -> None: self._last_d_logits = None self._d_probe_list = None self._g_probe_cache = None + self.clear_transient_captures() self._live_captures = [] - self._last_probe_actual_input = None def restore_device_state(self, device: torch.device) -> None: if isinstance(self.proto_sum, torch.Tensor) and self.proto_sum.device != device: @@ -482,6 +488,12 @@ def __init__(self) -> None: super().__init__() self._init_nash_v3_fields() + def record_step(self, optimizer_or_lr: Any) -> None: + super().record_step(optimizer_or_lr) + self._last_d_logits = None + self._d_probe_list = None + self.clear_transient_captures() + def state_dict(self) -> Dict[str, Any]: return {**super().state_dict(), "t4_nash_v3": self._nash_v3_packet()} diff --git a/particlegan/_t2_critic/critic.py b/particlegan/_t2_critic/critic.py index 5b4b755f3..d1e0a23a3 100644 --- a/particlegan/_t2_critic/critic.py +++ b/particlegan/_t2_critic/critic.py @@ -18,6 +18,25 @@ _MAX_LIVE_CAPTURES = 6 + +def _clear_live_capture_entries(entries: List[Any]) -> None: + """Release tensor references and backward-alive flags from live-capture entries.""" + for entry in entries: + if isinstance(entry, list) and len(entry) >= 5: + entry[3] = None + entry[4] = None + if len(entry) >= 6 and isinstance(entry[5], list): + entry[5].clear() + entries.clear() + + +def _trim_live_captures(captures: List[Any], max_entries: int = _MAX_LIVE_CAPTURES) -> None: + """Evict oldest live-capture entries while explicitly dropping their tensor references.""" + if len(captures) > max_entries: + evict_n = len(captures) - max_entries + _clear_live_capture_entries(captures[:evict_n]) + del captures[:evict_n] + _T2_STATE_KEYS = frozenset({ "schema", "initialized", @@ -146,11 +165,11 @@ def _t2_critic_forward_hook( rec._live_captures = captures c_sig = _cond_signature(args, kwargs) step_tag = int(getattr(rec, "observed_steps", 0)) - entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, True] - unscaled.register_hook(lambda _g, _e=entry: _e.__setitem__(5, False)) + alive = [True] + entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, alive] + raw_target.register_hook(lambda _g, _a=alive: _a.clear()) captures.append(entry) - if len(captures) > _MAX_LIVE_CAPTURES: - del captures[: len(captures) - _MAX_LIVE_CAPTURES] + _trim_live_captures(captures) if module.training and rec is not None and not getattr(rec, "_in_penalty", False) and torch.is_grad_enabled(): s = float(getattr(rec, "_thermal_scale", 1.0)) @@ -257,7 +276,7 @@ def _init_t2_fields(self) -> None: def clear_transient_captures(self) -> None: if hasattr(self, "_live_captures") and self._live_captures: - self._live_captures.clear() + _clear_live_capture_entries(self._live_captures) self._last_probe_actual_input = None def __deepcopy__(self, memo: Dict[int, Any]) -> Any: diff --git a/particlegan/_unified/critic.py b/particlegan/_unified/critic.py index 8c157a9ff..6c40c7a09 100644 --- a/particlegan/_unified/critic.py +++ b/particlegan/_unified/critic.py @@ -28,9 +28,10 @@ _validate_t1c_packet, ) from .._t2_critic.critic import ( - _MAX_LIVE_CAPTURES, + _clear_live_capture_entries, _cond_signature, _tensor_identity_key, + _trim_live_captures, ) from ..grad_regularizers import CriticStepRecord from ..k3p import K3PCriticAdam @@ -83,7 +84,7 @@ def rgdt_cond_ema(self, val: float) -> None: def clear_transient_captures(self) -> None: if hasattr(self, "_live_captures") and self._live_captures: - self._live_captures.clear() + _clear_live_capture_entries(self._live_captures) self._last_probe_actual_input = None def _unified_packet(self) -> Dict[str, Any]: @@ -98,6 +99,7 @@ def _restore_unified_packet(self, packet: Dict[str, Any]) -> None: self._restore_t1c_packet(t1c_sub) self._step_scale = float(packet["step_scale"]) self._t1c_disc_gate = 0.0 + self.clear_transient_captures() self._live_captures = [] self._last_probe_actual_input = None @@ -241,11 +243,11 @@ def _unified_critic_forward_hook( rec._live_captures = captures c_sig = _cond_signature(args, kwargs) step_tag = int(getattr(rec, "observed_steps", 0)) - entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, True] - unscaled.register_hook(lambda _g, _e=entry: _e.__setitem__(5, False)) + alive = [True] + entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, alive] + raw_target.register_hook(lambda _g, _a=alive: _a.clear()) captures.append(entry) - if len(captures) > _MAX_LIVE_CAPTURES: - del captures[: len(captures) - _MAX_LIVE_CAPTURES] + _trim_live_captures(captures) # 2. Apply thermal cooling to outgoing critic logits (outside penalty probes): modified = False diff --git a/particlegan/_unified_v11/critic.py b/particlegan/_unified_v11/critic.py index aea44fe5d..3548f4703 100644 --- a/particlegan/_unified_v11/critic.py +++ b/particlegan/_unified_v11/critic.py @@ -18,9 +18,10 @@ _validate_t1c_packet, ) from .._t2_critic.critic import ( - _MAX_LIVE_CAPTURES, + _clear_live_capture_entries, _cond_signature, _tensor_identity_key, + _trim_live_captures, ) from ..grad_regularizers import CriticStepRecord from ..k3p import K3PCriticAdam @@ -158,7 +159,7 @@ def observe_margin_and_scale( def clear_transient_captures(self) -> None: if hasattr(self, "_live_captures") and self._live_captures: - self._live_captures.clear() + _clear_live_capture_entries(self._live_captures) self._last_probe_actual_input = None def clear_step_probes(self) -> None: @@ -188,6 +189,7 @@ def _restore_unified_v11_packet(self, packet: Dict[str, Any]) -> None: self.cond_fiber_bank.load_packet(packet["cond_fiber_bank"]) self._t1c_disc_gate = 0.0 self._simplicial_step = False + self.clear_transient_captures() self._live_captures = [] self._d_probe_list = [] self._g_probe_cache = None @@ -347,11 +349,11 @@ def _unified_v11_critic_forward_hook( rec._live_captures = captures c_sig = _cond_signature(args, kwargs) step_tag = int(getattr(rec, "observed_steps", 0)) - entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, True] - unscaled.register_hook(lambda _g, _e=entry: _e.__setitem__(5, False)) + alive = [True] + entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, alive] + raw_target.register_hook(lambda _g, _a=alive: _a.clear()) captures.append(entry) - if len(captures) > _MAX_LIVE_CAPTURES: - del captures[: len(captures) - _MAX_LIVE_CAPTURES] + _trim_live_captures(captures) if ( getattr(rec, "_enable_secant_probe", True) diff --git a/particlegan/_unified_v4/critic.py b/particlegan/_unified_v4/critic.py index 5ba4dc85c..c5730580e 100644 --- a/particlegan/_unified_v4/critic.py +++ b/particlegan/_unified_v4/critic.py @@ -30,9 +30,10 @@ _validate_t1c_packet, ) from .._t2_critic.critic import ( - _MAX_LIVE_CAPTURES, + _clear_live_capture_entries, _cond_signature, _tensor_identity_key, + _trim_live_captures, ) from ..grad_regularizers import CriticStepRecord from ..k3p import K3PCriticAdam @@ -308,7 +309,7 @@ def observe_cond_gen_stratum( def clear_transient_captures(self) -> None: if hasattr(self, "_live_captures") and self._live_captures: - self._live_captures.clear() + _clear_live_capture_entries(self._live_captures) self._last_probe_actual_input = None def clear_step_probes(self) -> None: @@ -372,6 +373,7 @@ def _restore_unified_v4_packet(self, packet: Dict[str, Any]) -> None: } self._t1c_disc_gate = 0.0 self._simplicial_step = False + self.clear_transient_captures() self._live_captures = [] self._d_probe_list = [] self._g_probe_cache = None @@ -546,11 +548,11 @@ def _unified_v4_critic_forward_hook( rec._live_captures = captures c_sig = _cond_signature(args, kwargs) step_tag = int(getattr(rec, "observed_steps", 0)) - entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, True] - unscaled.register_hook(lambda _g, _e=entry: _e.__setitem__(5, False)) + alive = [True] + entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, alive] + raw_target.register_hook(lambda _g, _a=alive: _a.clear()) captures.append(entry) - if len(captures) > _MAX_LIVE_CAPTURES: - del captures[: len(captures) - _MAX_LIVE_CAPTURES] + _trim_live_captures(captures) if ( getattr(rec, "_enable_secant_probe", True) diff --git a/tests/test_unified_critic_vram_leak.py b/tests/test_unified_critic_vram_leak.py new file mode 100644 index 000000000..2febd0a81 --- /dev/null +++ b/tests/test_unified_critic_vram_leak.py @@ -0,0 +1,309 @@ +"""Weakref, eviction, backward-alive semantics, and flat-VRAM regression tests for critic live-capture hooks.""" +from __future__ import annotations + +import gc +import weakref + +import pytest +import torch +from torch import nn + +from particlegan import GANTrainer, get_recipe +from particlegan._minimal.dynamics import MinimalStepRecord, _attach_minimal_critic_hook +from particlegan._minimal.geometry import MinimalCriticPenalty +from particlegan._nash_v3.penalty import _NashV3PenaltyKernel +from particlegan._nash_v3.record import NashV3StepRecord, _attach_nash_v3_critic_hook +from particlegan._t2_critic.critic import ( + _MAX_LIVE_CAPTURES, + T2SecantStepRecord, + _attach_t2_critic_hook, +) +from particlegan._t2_critic.penalty import _T2SecantPenaltyKernel +from particlegan._unified.critic import UnifiedStepRecord, _attach_unified_critic_hook +from particlegan._unified.penalty import _UnifiedPenaltyKernel +from particlegan._unified_v4.critic import UnifiedV4StepRecord, _attach_unified_v4_critic_hook +from particlegan._unified_v11.critic import UnifiedV11StepRecord, _attach_unified_v11_critic_hook +from particlegan._unified_v11.penalty import _UnifiedV11PenaltyKernel + + +class _SmallConvCritic(nn.Module): + def __init__(self, channels: int = 3) -> None: + super().__init__() + self.net = nn.Sequential( + nn.Conv2d(channels, 8, kernel_size=3, stride=2, padding=1), + nn.SiLU(), + nn.Conv2d(8, 8, kernel_size=3, stride=2, padding=1), + nn.SiLU(), + ) + self.head = nn.Linear(8, 1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.head(self.net(x).mean(dim=(2, 3))).squeeze(-1) + + +class _SmallConvGenerator(nn.Module): + def __init__(self, z_dim: int = 8, channels: int = 3) -> None: + super().__init__() + self.fc = nn.Linear(z_dim, 8 * 8 * 8) + self.up1 = nn.ConvTranspose2d(8, 8, kernel_size=4, stride=2, padding=1) + self.up2 = nn.ConvTranspose2d(8, channels, kernel_size=4, stride=2, padding=1) + + def forward(self, z: torch.Tensor) -> torch.Tensor: + h = nn.functional.silu(self.fc(z)).reshape(z.shape[0], 8, 8, 8) + h = nn.functional.silu(self.up1(h)) + return torch.tanh(self.up2(h)) + + +_HOOK_FACTORIES = [ + pytest.param(UnifiedV11StepRecord, _attach_unified_v11_critic_hook, id="unified_v11"), + pytest.param(UnifiedV4StepRecord, _attach_unified_v4_critic_hook, id="unified_v4"), + pytest.param(UnifiedStepRecord, _attach_unified_critic_hook, id="unified"), + pytest.param(T2SecantStepRecord, _attach_t2_critic_hook, id="t2_secant"), + pytest.param(NashV3StepRecord, _attach_nash_v3_critic_hook, id="t4_nash_v3"), + pytest.param(MinimalStepRecord, _attach_minimal_critic_hook, id="minimal"), +] + + +@pytest.mark.parametrize(("record_cls", "attach_fn"), _HOOK_FACTORIES) +@pytest.mark.parametrize("use_bf16", [False, True], ids=["fp32", "bf16"]) +def test_live_capture_hook_releases_tensors_without_cycle( + record_cls, + attach_fn, + use_bf16: bool, +) -> None: + """`x_act` and `unscaled` must be GC'd after backward + clear, even if a caller holds `entry`.""" + torch.manual_seed(0) + critic = _SmallConvCritic() + if use_bf16: + critic = critic.to(dtype=torch.bfloat16) + record = record_cls() + attach_fn(critic, record) + + x = torch.randn(4, 3, 32, 32, dtype=torch.bfloat16 if use_bf16 else torch.float32) + out = critic(x) + + assert len(record._live_captures) == 1 + saved_entry = record._live_captures[0] + assert isinstance(saved_entry[5], list) + assert bool(saved_entry[5]) is True + + ref_x_act = weakref.ref(saved_entry[3]) + ref_unscaled = weakref.ref(saved_entry[4]) + + out.mean().backward() + assert bool(saved_entry[5]) is False + + record.clear_transient_captures() + assert record._live_captures == [] + assert saved_entry[3] is None + assert saved_entry[4] is None + + del x, out + gc.collect() + + assert ref_x_act() is None, "x_act leaked after backward + clear_transient_captures" + assert ref_unscaled() is None, "unscaled leaked after backward + clear_transient_captures" + + +@pytest.mark.parametrize(("record_cls", "attach_fn"), _HOOK_FACTORIES) +def test_live_capture_ring_eviction_releases_old_tensors(record_cls, attach_fn) -> None: + """Evicting entries beyond `_MAX_LIVE_CAPTURES` must immediately drop tensor refs without leaking.""" + torch.manual_seed(0) + critic = _SmallConvCritic() + record = record_cls() + attach_fn(critic, record) + + evicted_refs = [] + evicted_entries = [] + total_calls = _MAX_LIVE_CAPTURES + 4 + for i in range(total_calls): + x = torch.randn(4, 3, 32, 32) + out = critic(x) + entry = record._live_captures[-1] + if i < total_calls - _MAX_LIVE_CAPTURES: + evicted_entries.append(entry) + evicted_refs.append((weakref.ref(entry[3]), weakref.ref(entry[4]))) + out.mean().backward() + del x, out, entry + + gc.collect() + assert len(record._live_captures) == _MAX_LIVE_CAPTURES + for entry, (ref_x, ref_u) in zip(evicted_entries, evicted_refs): + assert entry[3] is None and entry[4] is None + assert ref_x() is None, "evicted x_act leaked" + assert ref_u() is None, "evicted unscaled leaked" + + +@pytest.mark.parametrize(("record_cls", "attach_fn"), _HOOK_FACTORIES) +def test_live_capture_load_state_dict_releases_tensors(record_cls, attach_fn) -> None: + """Restoring a record checkpoint must clear tensor refs from any prior `_live_captures`.""" + torch.manual_seed(0) + critic = _SmallConvCritic() + record = record_cls() + attach_fn(critic, record) + snap = record.state_dict() + + x = torch.randn(4, 3, 32, 32) + out = critic(x) + saved_entry = record._live_captures[0] + ref_x_act = weakref.ref(saved_entry[3]) + ref_unscaled = weakref.ref(saved_entry[4]) + + record.load_state_dict(snap) + assert record._live_captures == [] + assert saved_entry[3] is None + assert saved_entry[4] is None + + del x, out + gc.collect() + assert ref_x_act() is None, "x_act leaked after load_state_dict" + assert ref_unscaled() is None, "unscaled leaked after load_state_dict" + + +@pytest.mark.parametrize( + ("record_cls", "attach_fn", "kernel_cls"), + [ + pytest.param(UnifiedV11StepRecord, _attach_unified_v11_critic_hook, _UnifiedV11PenaltyKernel, id="unified_v11"), + pytest.param(UnifiedStepRecord, _attach_unified_critic_hook, _UnifiedPenaltyKernel, id="unified"), + pytest.param(T2SecantStepRecord, _attach_t2_critic_hook, _T2SecantPenaltyKernel, id="t2_secant"), + pytest.param(NashV3StepRecord, _attach_nash_v3_critic_hook, _NashV3PenaltyKernel, id="t4_nash_v3"), + ], +) +def test_live_capture_alive_flag_prevents_reuse_after_backward( + record_cls, + attach_fn, + kernel_cls, +) -> None: + """Before backward, live endpoint lookup hits; after backward, hook clears `alive` so lookup misses.""" + torch.manual_seed(0) + critic = _SmallConvCritic() + record = record_cls() + attach_fn(critic, record) + kernel = kernel_cls(record=record) + + x_real = torch.randn(4, 3, 32, 32) + x_fake = torch.randn(4, 3, 32, 32) + d_real = critic(x_real) + d_fake = critic(x_fake) + + hit_before = kernel._lookup_live_endpoint(x_real, ()) + assert hit_before is not None and hit_before[1] is d_real + + (d_real.mean() - d_fake.mean()).backward() + + hit_after = kernel._lookup_live_endpoint(x_real, ()) + assert hit_after is None, "already-backwarded endpoint must not be reused" + + # Calling penalty after separate backward must succeed cleanly via fresh forward. + pen_val, _ = kernel.penalty(critic, x_real, x_fake, step=1) + assert torch.isfinite(pen_val) + pen_val.backward() + record.clear_transient_captures() + assert record._live_captures == [] + + +def test_minimal_penalty_skips_backwarded_capture_without_crash() -> None: + """MinimalCriticPenalty must skip already-backwarded live captures cleanly.""" + torch.manual_seed(0) + critic = _SmallConvCritic() + recipe = get_recipe("minimal", training_precision="float32") + opt_d = recipe.make_critic_optimizer(critic) + pen = MinimalCriticPenalty(recipe, opt_d) + record = opt_d.record + + x_real = torch.randn(4, 3, 32, 32) + x_fake = torch.randn(4, 3, 32, 32) + d_real = critic(x_real) + d_fake = critic(x_fake) + assert len(record._live_captures) == 2 + assert all(bool(entry[5]) for entry in record._live_captures) + + (d_real.mean() - d_fake.mean()).backward() + assert all(not bool(entry[5]) for entry in record._live_captures) + + # Penalty after separate backward must not try to backward through freed graphs. + pen_val = pen(critic, x_real, x_fake) + assert torch.isfinite(pen_val) + pen_val.backward() + record.clear_transient_captures() + assert record._live_captures == [] + + +@pytest.mark.parametrize("preset", ["unified_v11", "unified_v4", "unified"]) +def test_gan_trainer_cpu_releases_all_step_batches(preset: str) -> None: + """Running `GANTrainer` across steps retains zero weakrefs to past real batches on CPU.""" + torch.manual_seed(0) + recipe = get_recipe( + preset, + z_dim=8, + batch_size=8, + num_particles=16, + total_steps=20, + training_precision="float32", + ) + gen = _SmallConvGenerator(z_dim=8, channels=3) + disc = _SmallConvCritic(channels=3) + trainer = GANTrainer(recipe, gen, disc, seed=0) + + batch_refs = [] + for _ in range(12): + x_real = torch.randn(8, 3, 32, 32) + batch_refs.append(weakref.ref(x_real)) + trainer.step(x_real) + del x_real + + gc.collect() + leaked = sum(1 for ref in batch_refs if ref() is not None) + assert leaked == 0, f"{preset}: {leaked}/{len(batch_refs)} real batches were retained" + assert getattr(trainer.opt_d.record, "_live_captures", []) == [] + + +@pytest.mark.cuda +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required for VRAM leak regression test") +@pytest.mark.parametrize("preset", ["unified_v11", "unified_v4", "unified"]) +def test_gan_trainer_cuda_flat_vram_over_many_steps(preset: str) -> None: + """CUDA VRAM must stay flat over 40 CIFAR-10-sized steps with zero retained batch weakrefs.""" + device = torch.device("cuda:0") + torch.manual_seed(0) + torch.cuda.manual_seed(0) + + recipe = get_recipe( + preset, + z_dim=8, + batch_size=32, + num_particles=16, + total_steps=60, + ) + gen = _SmallConvGenerator(z_dim=8, channels=3).to(device) + disc = _SmallConvCritic(channels=3).to(device) + trainer = GANTrainer(recipe, gen, disc, seed=0) + + # Warm up allocator, optimizer states, and any lazy buffers. + for _ in range(5): + x_warm = torch.randn(32, 3, 32, 32, device=device) + trainer.step(x_warm) + del x_warm + + gc.collect() + torch.cuda.synchronize(device) + mem_before = torch.cuda.memory_allocated(device) + + batch_refs = [] + for _ in range(40): + x_real = torch.randn(32, 3, 32, 32, device=device) + batch_refs.append(weakref.ref(x_real)) + trainer.step(x_real) + del x_real + + gc.collect() + torch.cuda.synchronize(device) + mem_after = torch.cuda.memory_allocated(device) + + leaked = sum(1 for ref in batch_refs if ref() is not None) + assert leaked == 0, f"{preset}: {leaked}/{len(batch_refs)} CUDA input batches leaked" + assert getattr(trainer.opt_d.record, "_live_captures", []) == [] + # Without the fix, 40 steps x 2 batches x (32*3*32*32*4 bytes) + autograd graphs leaks > 30 MiB. + assert mem_after - mem_before <= 2 * 1024 * 1024, ( + f"{preset}: CUDA memory grew by {(mem_after - mem_before) / (1024 * 1024):.2f} MiB over 40 steps " + f"(before={mem_before}, after={mem_after})" + ) From 2b6a94fe43e01659c85238752ca18e890a12647c Mon Sep 17 00:00:00 2001 From: Martyn Garcia Date: Fri, 20 Nov 2026 10:58:43 -0700 Subject: [PATCH 3/8] feat(benchmarks): add reproducible CIFAR-style Transformer-G / ResNet18-feature-D benchmark --- benchmarks/cifar_toy/__init__.py | 73 ++++ benchmarks/cifar_toy/__main__.py | 146 ++++++++ benchmarks/cifar_toy/data.py | 279 +++++++++++++++ benchmarks/cifar_toy/metrics.py | 169 ++++++++++ benchmarks/cifar_toy/models.py | 386 +++++++++++++++++++++ benchmarks/cifar_toy/runner.py | 543 ++++++++++++++++++++++++++++++ tests/test_cifar_toy_benchmark.py | 120 +++++++ 7 files changed, 1716 insertions(+) create mode 100644 benchmarks/cifar_toy/__init__.py create mode 100644 benchmarks/cifar_toy/__main__.py create mode 100644 benchmarks/cifar_toy/data.py create mode 100644 benchmarks/cifar_toy/metrics.py create mode 100644 benchmarks/cifar_toy/models.py create mode 100644 benchmarks/cifar_toy/runner.py create mode 100644 tests/test_cifar_toy_benchmark.py diff --git a/benchmarks/cifar_toy/__init__.py b/benchmarks/cifar_toy/__init__.py new file mode 100644 index 000000000..0e0e96f3d --- /dev/null +++ b/benchmarks/cifar_toy/__init__.py @@ -0,0 +1,73 @@ +"""Fast, reproducible CIFAR-style 32x32 Transformer-G / ResNet18-feature-D benchmark. + +Matches the structural properties of HyperGAN's ``cifar10-tiny-transformer`` +(4D ``(B, 3, 32, 32)`` RGB tensor with ``dim = 3072 > 1024``, 2-block 64-token +Transformer generator with 16 matrix/conv parameters vs. frozen ImageNet +ResNet-18 64px critic with 3 trainable feature heads and 9 matrix/conv +parameters, MoG particle prior in ``z_dim = 64``) while executing purely +through ParticleGAN's public ``Recipe`` / ``MoGParticlePrior`` / ``GANTrainer`` +API under protocol seed 0. +""" +from .data import ( + BatchSequenceStream, + CIFAR10Dataset, + discover_cifar10_root, + load_cifar10_dataset, + make_synthetic_cifar_dataset, +) +from .metrics import ( + compute_class_coverage, + compute_critic_diagnostics, + compute_fid, + compute_kid, + compute_pixel_diversity_ratio, +) +from .models import ( + DEFAULT_INCEPTION_WEIGHTS, + DEFAULT_RESNET18_WEIGHTS, + DEFAULT_RESNET34_WEIGHTS, + RESNET18_SHA256, + RESNET34_SHA256, + ResNet18FeatureCritic32, + ResNet34FeatureEvaluator, + TinyTransformerGenerator32, + count_matrix_conv_params, + count_trainable_params, + parameter_state_digest, +) +from .runner import ( + DEFAULT_MOG_SIGMA, + BenchmarkConfig, + format_leaderboard_markdown, + run_benchmark, + summarize_run_directory, +) + +__all__ = [ + "BatchSequenceStream", + "BenchmarkConfig", + "CIFAR10Dataset", + "DEFAULT_INCEPTION_WEIGHTS", + "DEFAULT_MOG_SIGMA", + "DEFAULT_RESNET18_WEIGHTS", + "DEFAULT_RESNET34_WEIGHTS", + "RESNET18_SHA256", + "RESNET34_SHA256", + "ResNet18FeatureCritic32", + "ResNet34FeatureEvaluator", + "TinyTransformerGenerator32", + "compute_class_coverage", + "compute_critic_diagnostics", + "compute_fid", + "compute_kid", + "compute_pixel_diversity_ratio", + "count_matrix_conv_params", + "count_trainable_params", + "discover_cifar10_root", + "format_leaderboard_markdown", + "load_cifar10_dataset", + "make_synthetic_cifar_dataset", + "parameter_state_digest", + "run_benchmark", + "summarize_run_directory", +] diff --git a/benchmarks/cifar_toy/__main__.py b/benchmarks/cifar_toy/__main__.py new file mode 100644 index 000000000..6c781d202 --- /dev/null +++ b/benchmarks/cifar_toy/__main__.py @@ -0,0 +1,146 @@ +"""CLI entrypoint: ``python -m benchmarks.cifar_toy``.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any, Dict, List + +from .models import DEFAULT_RESNET18_WEIGHTS, DEFAULT_RESNET34_WEIGHTS +from .runner import ( + DEFAULT_MOG_SIGMA, + BenchmarkConfig, + format_leaderboard_markdown, + run_benchmark, + summarize_run_directory, +) + + +def _parse_override(text: str) -> tuple[str, Any]: + if "=" not in text: + raise argparse.ArgumentTypeError(f"Expected KEY=VALUE for --recipe-override, got {text!r}") + key, raw = text.split("=", 1) + key = key.strip() + raw = raw.strip() + try: + val = json.loads(raw) + except json.JSONDecodeError: + val = raw + return key, val + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="python -m benchmarks.cifar_toy", + description="Fast reproducible CIFAR-style 32x32 Transformer-G / ResNet18-feature-D ParticleGAN benchmark.", + ) + p.add_argument( + "--recipes", + nargs="+", + default=["bispf_optimized", "empf", "unified_v11"], + help="ParticleGAN recipe presets to compare (default: bispf_optimized empf unified_v11).", + ) + p.add_argument( + "--num-particles", + nargs="+", + type=int, + default=[200], + help="One or more MoG prior particle counts (e.g. --num-particles 200 16384).", + ) + p.add_argument("--z-dim", type=int, default=64) + p.add_argument("--mog-sigma", type=float, default=DEFAULT_MOG_SIGMA) + p.add_argument("--no-standardize-prior", action="store_true") + p.add_argument("--batch-size", type=int, default=128) + p.add_argument("--steps", type=int, default=800) + p.add_argument("--eval-every", type=int, default=200) + p.add_argument("--eval-samples", type=int, default=2048) + p.add_argument("--eval-ref-samples", type=int, default=5000) + p.add_argument("--eval-critic-samples", type=int, default=256) + p.add_argument("--seed", type=int, default=0) + p.add_argument("--dataset", choices=("cifar10", "synthetic"), default="cifar10") + p.add_argument("--data-root", type=str, default=None) + p.add_argument("--no-horizontal-flip", action="store_true") + p.add_argument("--backbone-mode", choices=("pretrained", "random"), default="pretrained") + p.add_argument("--resnet18-weights", type=str, default=str(DEFAULT_RESNET18_WEIGHTS)) + p.add_argument("--resnet34-weights", type=str, default=str(DEFAULT_RESNET34_WEIGHTS)) + p.add_argument("--device", type=str, default="cuda:0") + p.add_argument("--deterministic-algorithms", action="store_true") + p.add_argument( + "--recipe-override", + action="append", + default=[], + help="Explicit recipe field override KEY=VALUE (JSON-parsed; repeatable).", + ) + p.add_argument("--run-tag", type=str, default=None, help="Explicit run directory name (single run only).") + p.add_argument( + "--output-dir", + type=str, + default="/nvme/dev/hypergan/ParticleGAN/runs/cifar_toy_bench", + help="Directory for bulk logs, JSONL traces, receipts, and leaderboard.md.", + ) + p.add_argument( + "--summarize-only", + action="store_true", + help="Do not train; rebuild leaderboard.md from existing receipts in --output-dir.", + ) + return p + + +def main(argv: List[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + out_dir = Path(args.output_dir) + + if args.summarize_only: + receipts, md = summarize_run_directory(out_dir) + if not receipts: + print(f"No receipts found under {out_dir}") + return 1 + print(md, end="") + return 0 + + overrides: Dict[str, Any] = {} + for item in args.recipe_override: + k, v = _parse_override(item) + overrides[k] = v + + combos = [(r, n) for r in args.recipes for n in args.num_particles] + if args.run_tag is not None and len(combos) != 1: + parser.error("--run-tag can only be used with a single (recipe, num_particles) combination") + + receipts = [] + for recipe_name, num_p in combos: + cfg = BenchmarkConfig( + recipe=recipe_name, + num_particles=num_p, + z_dim=args.z_dim, + mog_sigma=args.mog_sigma, + standardize_prior=not args.no_standardize_prior, + batch_size=args.batch_size, + steps=args.steps, + eval_every=args.eval_every, + eval_samples=args.eval_samples, + eval_ref_samples=args.eval_ref_samples, + eval_critic_samples=args.eval_critic_samples, + seed=args.seed, + dataset=args.dataset, + data_root=args.data_root, + horizontal_flip=not args.no_horizontal_flip, + backbone_mode=args.backbone_mode, + resnet18_weights=args.resnet18_weights, + resnet34_weights=args.resnet34_weights, + device=args.device, + deterministic_algorithms=args.deterministic_algorithms, + recipe_overrides=overrides, + run_tag=args.run_tag, + ) + receipts.append(run_benchmark(cfg, out_dir)) + + _, md = summarize_run_directory(out_dir) + print("\n### Current Leaderboard\n", flush=True) + print(format_leaderboard_markdown(receipts), end="", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/cifar_toy/data.py b/benchmarks/cifar_toy/data.py new file mode 100644 index 000000000..b6bda1008 --- /dev/null +++ b/benchmarks/cifar_toy/data.py @@ -0,0 +1,279 @@ +"""Deterministic CIFAR-10 and synthetic 10-class 32x32 RGB data streams. + +Zero ``torchvision`` dependency: loads ``cifar-10-batches-py`` directly from +local cache using Python's standard ``pickle`` module. Provides isolated +checkpointable CPU batch streams for critic and generator target draws plus an +exact SHA-256 batch-sequence digest so every trainer sees identical batches. +""" +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import os +from pathlib import Path +import pickle +from typing import Optional, Sequence, Tuple + +import numpy as np +import torch + + +@dataclass(frozen=True) +class CIFAR10Dataset: + """In-memory uint8 ``(N, 3, 32, 32)`` dataset + integer labels ``(N,)``.""" + + images_uint8: torch.Tensor + labels: torch.Tensor + source_name: str + source_path: str + num_classes: int = 10 + + @property + def size(self) -> int: + return int(self.images_uint8.shape[0]) + + def to_float_tensor(self, indices: torch.Tensor, *, flip_mask: Optional[torch.Tensor] = None) -> torch.Tensor: + """Convert indexed uint8 images to float32 in ``[-1, 1]`` on CPU.""" + imgs = self.images_uint8[indices].to(dtype=torch.float32).div_(127.5).sub_(1.0) + if flip_mask is not None and bool(flip_mask.any()): + imgs = torch.where(flip_mask.view(-1, 1, 1, 1), imgs.flip(-1), imgs) + return imgs + + +_CANDIDATE_CIFAR_ROOTS = ( + Path("/nvme/dev/hypergan/HyperGAN/data"), + Path.home() / ".hypergan" / "cache" / "cifar10", + Path("data"), +) + + +def discover_cifar10_root(explicit_root: Optional[Path] = None) -> Optional[Path]: + """Locate a local directory containing ``cifar-10-batches-py/data_batch_1``.""" + candidates: list[Path] = [] + if explicit_root is not None: + candidates.append(Path(explicit_root).expanduser()) + env_root = os.environ.get("PARTICLEGAN_CIFAR10_ROOT") + if env_root: + candidates.append(Path(env_root).expanduser()) + candidates.extend(_CANDIDATE_CIFAR_ROOTS) + + for root in candidates: + if (root / "data_batch_1").is_file(): + return root + sub = root / "cifar-10-batches-py" + if (sub / "data_batch_1").is_file(): + return sub + return None + + +def load_cifar10_dataset( + data_root: Optional[Path] = None, + *, + split: str = "train", +) -> CIFAR10Dataset: + """Load real CIFAR-10 ``train`` (50k) or ``test`` (10k) images as uint8 NCHW.""" + batch_dir = discover_cifar10_root(data_root) + if batch_dir is None: + raise FileNotFoundError( + "CIFAR-10 cache not found. Pass --data-root pointing to cifar-10-batches-py " + "or use --dataset synthetic for the hermetic synthetic cohort." + ) + if split == "train": + files = [batch_dir / f"data_batch_{i}" for i in range(1, 6)] + elif split == "test": + files = [batch_dir / "test_batch"] + else: + raise ValueError(f"split must be 'train' or 'test', got {split!r}") + + chunks = [] + labels = [] + for path in files: + if not path.is_file(): + raise FileNotFoundError(f"Missing CIFAR-10 batch file: {path}") + with open(path, "rb") as fh: + entry = pickle.load(fh, encoding="latin1") + data = np.asarray(entry["data"], dtype=np.uint8).reshape(-1, 3, 32, 32) + chunks.append(data) + labels.extend(int(y) for y in entry["labels"]) + + images_np = np.concatenate(chunks, axis=0) + return CIFAR10Dataset( + images_uint8=torch.from_numpy(images_np.copy()), + labels=torch.tensor(labels, dtype=torch.long), + source_name=f"cifar10_{split}", + source_path=str(batch_dir.resolve()), + num_classes=10, + ) + + +def make_synthetic_cifar_dataset( + num_samples: int = 2000, + *, + seed: int = 0, + num_classes: int = 10, +) -> CIFAR10Dataset: + """Deterministic 10-class structured 32x32 RGB image cohort for hermetic tests. + + Each class has a distinct color palette + geometric spatial pattern + (vertical/horizontal/diagonal bars, quadrants, center disk, frame, checker, + radial gradient) with small deterministic per-sample jitter so FID/KID and + class coverage are well-conditioned. + """ + gen = torch.Generator(device="cpu").manual_seed(seed + 7919) + yy, xx = torch.meshgrid( + torch.linspace(-1.0, 1.0, 32), + torch.linspace(-1.0, 1.0, 32), + indexing="ij", + ) + palettes = torch.tensor( + [ + [0.90, 0.15, 0.15], + [0.15, 0.85, 0.20], + [0.15, 0.30, 0.92], + [0.92, 0.85, 0.15], + [0.85, 0.20, 0.85], + [0.15, 0.85, 0.88], + [0.95, 0.52, 0.15], + [0.55, 0.25, 0.85], + [0.25, 0.65, 0.45], + [0.82, 0.82, 0.85], + ], + dtype=torch.float32, + ) + imgs = torch.empty((num_samples, 3, 32, 32), dtype=torch.float32) + labels = torch.empty((num_samples,), dtype=torch.long) + + for i in range(num_samples): + cls = i % num_classes + labels[i] = cls + shift_x = float(torch.empty(()).uniform_(-0.22, 0.22, generator=gen).item()) + shift_y = float(torch.empty(()).uniform_(-0.22, 0.22, generator=gen).item()) + x = xx - shift_x + y = yy - shift_y + if cls == 0: + mask = (x.abs() < 0.35).float() + elif cls == 1: + mask = (y.abs() < 0.35).float() + elif cls == 2: + mask = ((x + y).abs() < 0.40).float() + elif cls == 3: + mask = ((x - y).abs() < 0.40).float() + elif cls == 4: + mask = ((x * x + y * y) < 0.28).float() + elif cls == 5: + r2 = x * x + y * y + mask = ((r2 > 0.14) & (r2 < 0.48)).float() + elif cls == 6: + mask = ((x > 0.0) & (y > 0.0)).float() + elif cls == 7: + mask = ((x < 0.0) & (y < 0.0)).float() + elif cls == 8: + mask = (torch.sin(6.0 * x) * torch.sin(6.0 * y) > 0.0).float() + else: + mask = ((x.abs() > 0.55) | (y.abs() > 0.55)).float() + + fg = palettes[cls % len(palettes)].view(3, 1, 1) + bg = (1.0 - fg) * 0.25 + 0.12 + noise = torch.randn((3, 32, 32), generator=gen) * 0.05 + img = (bg * (1.0 - mask.unsqueeze(0)) + fg * mask.unsqueeze(0) + noise).clamp(0.0, 1.0) + imgs[i] = img + + images_uint8 = imgs.mul(255.0).round().clamp(0, 255).to(torch.uint8) + return CIFAR10Dataset( + images_uint8=images_uint8, + labels=labels, + source_name=f"synthetic_cifar10_n{num_samples}_seed{seed}", + source_path="in_memory", + num_classes=num_classes, + ) + + +class BatchSequenceStream: + """Isolated deterministic critic/generator target batch streams with exact digest. + + Critic and generator target batches use separate CPU generators seeded from + the protocol seed so every compared trainer receives the exact same ordered + sequence of ``(real_d, real_g)`` batches. + """ + + def __init__( + self, + dataset: CIFAR10Dataset, + batch_size: int, + *, + seed: int = 0, + horizontal_flip: bool = True, + ) -> None: + self.dataset = dataset + self.batch_size = int(batch_size) + self.horizontal_flip = bool(horizontal_flip) + self.critic_gen = torch.Generator(device="cpu").manual_seed(seed + 101) + self.gen_gen = torch.Generator(device="cpu").manual_seed(seed + 202) + self._hasher = hashlib.sha256() + self.steps_drawn = 0 + + def _draw_one(self, rng: torch.Generator) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + idx = torch.randint(0, self.dataset.size, (self.batch_size,), generator=rng) + flips = None + if self.horizontal_flip: + flips = torch.randint(0, 2, (self.batch_size,), generator=rng, dtype=torch.bool) + batch = self.dataset.to_float_tensor(idx, flip_mask=flips) + return batch, idx, flips + + def next_pair(self, device: torch.device) -> Tuple[torch.Tensor, torch.Tensor]: + real_d, idx_d, flip_d = self._draw_one(self.critic_gen) + real_g, idx_g, flip_g = self._draw_one(self.gen_gen) + self._hasher.update(idx_d.numpy().tobytes()) + self._hasher.update(idx_g.numpy().tobytes()) + if flip_d is not None and flip_g is not None: + self._hasher.update(flip_d.numpy().tobytes()) + self._hasher.update(flip_g.numpy().tobytes()) + self.steps_drawn += 1 + return real_d.to(device=device, non_blocking=False), real_g.to(device=device, non_blocking=False) + + @property + def digest(self) -> str: + return self._hasher.hexdigest() + + def state_dict(self) -> dict: + return { + "critic_gen": self.critic_gen.get_state(), + "gen_gen": self.gen_gen.get_state(), + "steps_drawn": self.steps_drawn, + } + + +def select_eval_reference_subset( + dataset: CIFAR10Dataset, + count: int, + *, + seed: int = 0, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Deterministically select a balanced reference subset ``(images_float, labels)``.""" + n = min(int(count), dataset.size) + rng = torch.Generator(device="cpu").manual_seed(seed + 303) + perm = torch.randperm(dataset.size, generator=rng)[:n] + imgs = dataset.to_float_tensor(perm, flip_mask=None) + lbls = dataset.labels[perm].clone() + return imgs, lbls + + +def compute_class_centroids(features: torch.Tensor, labels: torch.Tensor, num_classes: int = 10) -> torch.Tensor: + """Compute per-class mean feature vectors ``(num_classes, D)``.""" + feat_f = features.float() + centroids = torch.zeros((num_classes, feat_f.shape[1]), dtype=torch.float32) + for c in range(num_classes): + mask = labels == c + if bool(mask.any()): + centroids[c] = feat_f[mask].mean(dim=0) + return centroids + + +def sample_indices_for_eval(num_samples: int, batch_size: int) -> Sequence[int]: + sizes = [] + rem = int(num_samples) + while rem > 0: + b = min(batch_size, rem) + sizes.append(b) + rem -= b + return sizes diff --git a/benchmarks/cifar_toy/metrics.py b/benchmarks/cifar_toy/metrics.py new file mode 100644 index 000000000..01352b5a2 --- /dev/null +++ b/benchmarks/cifar_toy/metrics.py @@ -0,0 +1,169 @@ +"""Distribution fidelity (Feature-FID, unbiased KID), mode coverage, and critic diagnostics.""" +from __future__ import annotations + +import math +from typing import Dict, Tuple + +import numpy as np +import torch +import torch.nn as nn + + +def feature_moments(features: torch.Tensor) -> Tuple[np.ndarray, np.ndarray]: + """Compute float64 mean ``(D,)`` and sample covariance ``(D, D)``.""" + arr = features.detach().cpu().numpy().astype(np.float64) + mu = np.mean(arr, axis=0) + cov = np.cov(arr, rowvar=False) + return mu, cov + + +def compute_fid( + mu_real: np.ndarray, + cov_real: np.ndarray, + mu_fake: np.ndarray, + cov_fake: np.ndarray, +) -> float: + """Exact symmetric-PSD Fréchet distance between two multivariate Gaussians. + + Uses ``Tr((Sigma_r Sigma_f)^{1/2}) = sum(sqrt(eigvalsh(Sigma_r^{1/2} Sigma_f Sigma_r^{1/2})))`` + in float64, avoiding complex/singular warnings on rank-deficient covariances. + """ + diff = mu_real - mu_fake + sym_r = 0.5 * (cov_real + cov_real.T) + sym_f = 0.5 * (cov_fake + cov_fake.T) + eigvals_r, eigvecs_r = np.linalg.eigh(sym_r) + sqrt_r = (eigvecs_r * np.sqrt(np.clip(eigvals_r, 0.0, None))) @ eigvecs_r.T + prod = sqrt_r @ sym_f @ sqrt_r + prod = 0.5 * (prod + prod.T) + eigvals_prod = np.linalg.eigvalsh(prod) + tr_covmean = float(np.sum(np.sqrt(np.clip(eigvals_prod, 0.0, None)))) + fid = float(diff @ diff + np.trace(sym_r) + np.trace(sym_f) - 2.0 * tr_covmean) + return max(0.0, fid) + + +def _polynomial_mmd_unbiased(x: np.ndarray, y: np.ndarray) -> float: + """Unbiased polynomial-kernel MMD^2 (degree 3, coef0=1, gamma=1/d).""" + m = x.shape[0] + n = y.shape[0] + d = float(x.shape[1]) + k_xx = (x @ x.T / d + 1.0) ** 3 + k_yy = (y @ y.T / d + 1.0) ** 3 + k_xy = (x @ y.T / d + 1.0) ** 3 + sum_xx = (np.sum(k_xx) - np.trace(k_xx)) / (m * (m - 1)) + sum_yy = (np.sum(k_yy) - np.trace(k_yy)) / (n * (n - 1)) + sum_xy = np.sum(k_xy) / (m * n) + return float(sum_xx + sum_yy - 2.0 * sum_xy) + + +def compute_kid( + feats_real: torch.Tensor, + feats_fake: torch.Tensor, + *, + subset_size: int = 500, + num_subsets: int = 20, + seed: int = 0, +) -> Tuple[float, float]: + """Unbiased Kernel Inception/Feature Distance (mean, std).""" + x_all = feats_real.detach().cpu().numpy().astype(np.float64) + y_all = feats_fake.detach().cpu().numpy().astype(np.float64) + m = min(int(subset_size), x_all.shape[0], y_all.shape[0]) + if m < 2: + raise ValueError("KID requires at least 2 samples per distribution") + rng = np.random.default_rng(seed + 404) + vals = [] + for _ in range(num_subsets): + ix = rng.choice(x_all.shape[0], size=m, replace=False) + iy = rng.choice(y_all.shape[0], size=m, replace=False) + vals.append(_polynomial_mmd_unbiased(x_all[ix], y_all[iy])) + arr = np.asarray(vals, dtype=np.float64) + return float(arr.mean()), float(arr.std()) + + +def compute_class_coverage( + feats_fake: torch.Tensor, + class_centroids: torch.Tensor, + *, + min_mass_threshold: float = 0.01, +) -> Dict[str, float]: + """Assign generated features to nearest real class centroid in ResNet-34 space.""" + f = feats_fake.detach().cpu().float() + c = class_centroids.detach().cpu().float() + num_classes = int(c.shape[0]) + dists = torch.cdist(f, c) + assigned = dists.argmin(dim=1) + counts = torch.bincount(assigned, minlength=num_classes).float() + probs = counts / max(1.0, float(counts.sum().item())) + uniform = 1.0 / float(num_classes) + mass_tv = 0.5 * float((probs - uniform).abs().sum().item()) + covered = int((probs >= min_mass_threshold).sum().item()) + nonzero_p = probs[probs > 0] + entropy = float(-(nonzero_p * nonzero_p.log()).sum().item()) + entropy_norm = entropy / math.log(float(num_classes)) if num_classes > 1 else 1.0 + return { + "class_coverage": float(covered), + "class_mass_tv": mass_tv, + "class_entropy_norm": entropy_norm, + "min_class_mass": float(probs.min().item()), + } + + +def compute_pixel_diversity_ratio(real_images: torch.Tensor, fake_images: torch.Tensor) -> float: + """Ratio of generated-to-real across-sample pixel RMS (diversity indicator).""" + r = real_images.detach().float() + f = fake_images.detach().float() + r_rms = float(r.std(dim=0, unbiased=False).square().mean().sqrt().item()) + f_rms = float(f.std(dim=0, unbiased=False).square().mean().sqrt().item()) + return f_rms / max(r_rms, 1e-8) + + +@torch.no_grad() +def compute_critic_diagnostics( + critic: nn.Module, + real_batch: torch.Tensor, + fake_batch: torch.Tensor, + *, + batch_size: int = 128, +) -> Dict[str, float]: + """Side-effect-free critic logit & accuracy diagnostics under ``eval()`` + ``no_grad()``.""" + modes = [(m, m.training) for m in critic.modules()] + device = next(critic.parameters()).device + try: + critic.eval() + r_outs = [] + f_outs = [] + n = min(int(real_batch.shape[0]), int(fake_batch.shape[0])) + for start in range(0, n, batch_size): + r_chunk = real_batch[start : start + batch_size].to(device=device, dtype=torch.float32) + f_chunk = fake_batch[start : start + batch_size].to(device=device, dtype=torch.float32) + r_outs.append(critic(r_chunk).float().reshape(-1).cpu()) + f_outs.append(critic(f_chunk).float().reshape(-1).cpu()) + d_real = torch.cat(r_outs, dim=0) + d_fake = torch.cat(f_outs, dim=0) + finally: + for m, flag in modes: + m.training = flag + + real_mean = float(d_real.mean().item()) + fake_mean = float(d_fake.mean().item()) + margin = real_mean - fake_mean + logit_std = float(torch.cat([d_real, d_fake], dim=0).std(unbiased=False).item()) + # Standard threshold accuracy (matching HyperGAN critic/accuracy) + pairwise accuracy. + acc = 0.5 * (float((d_real > 0.0).float().mean().item()) + float((d_fake <= 0.0).float().mean().item())) + pair_acc = float((d_real > d_fake).float().mean().item()) + return { + "critic_real_mean": real_mean, + "critic_fake_mean": fake_mean, + "critic_margin": margin, + "critic_logit_std": logit_std, + "critic_accuracy": acc, + "critic_pair_acc": pair_acc, + } + + +def compute_grad_norm(parameters) -> float: + sq = 0.0 + for p in parameters: + if p.requires_grad and p.grad is not None: + g = p.grad.detach().float() + sq += float(g.square().sum().item()) + return math.sqrt(sq) diff --git a/benchmarks/cifar_toy/models.py b/benchmarks/cifar_toy/models.py new file mode 100644 index 000000000..f85348b0d --- /dev/null +++ b/benchmarks/cifar_toy/models.py @@ -0,0 +1,386 @@ +"""Pure-PyTorch architectures for the CIFAR-style toy benchmark. + +Zero ``torchvision`` dependency: standard PyTorch ResNet-18 / ResNet-34 +``BasicBlock`` backbones load the official SHA-256-verified PyTorch hub +checkpoints directly. + +Generator: ``TinyTransformerGenerator32`` (16 matrix/conv params, 2,489,731 +trainable parameters) — exact structural match to HyperGAN's +``tiny-transformer-generator-32``. + +Critic: ``ResNet18FeatureCritic32`` (9 trainable matrix/conv params, 171,779 +trainable parameters on top of frozen ImageNet ResNet-18 ``layer1/2/3`` at +64px with fixed midgray context) — exact structural match to HyperGAN's +``resnet18-features-discriminator-32``. + +Evaluator: ``ResNet34FeatureEvaluator`` — independent frozen ImageNet ResNet-34 +``avgpool`` 512-dim feature extractor for Feature-FID, Feature-KID, and class +centroid mode coverage. +""" +from __future__ import annotations + +import hashlib +import math +from pathlib import Path +from typing import Iterable, Optional, Sequence, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + +RESNET18_SHA256 = "f37072fd47e89c5e827621c5baffa7500819f7896bbacec160b1a16c560e07ec" +RESNET34_SHA256 = "b627a593f4b504425f6d6c373712e796be8773f6cb3c5910b09070727651153e" + +DEFAULT_RESNET18_WEIGHTS = Path.home() / ".cache" / "torch" / "hub" / "checkpoints" / "resnet18-f37072fd.pth" +DEFAULT_RESNET34_WEIGHTS = Path.home() / ".cache" / "torch" / "hub" / "checkpoints" / "resnet34-b627a593.pth" +DEFAULT_INCEPTION_WEIGHTS = ( + Path.home() / ".cache" / "torch" / "hub" / "checkpoints" / "weights-inception-2015-12-05-6726825d.pth" +) + +_IMAGENET_MEAN = (0.485, 0.456, 0.406) +_IMAGENET_STD = (0.229, 0.224, 0.225) + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def count_trainable_params(module: nn.Module) -> int: + return sum(p.numel() for p in module.parameters() if p.requires_grad) + + +def count_matrix_conv_params(module: nn.Module) -> int: + """Count trainable parameters with ``ndim >= 2`` (matrices and conv kernels).""" + return sum(1 for p in module.parameters() if p.requires_grad and p.ndim >= 2) + + +def parameter_state_digest(module: nn.Module) -> str: + """Deterministic SHA-256 digest of all trainable parameter tensors (CPU float32 bytes).""" + h = hashlib.sha256() + for name, p in sorted(module.named_parameters()): + if not p.requires_grad: + continue + arr = p.detach().cpu().contiguous().float().numpy() + h.update(name.encode("utf-8")) + h.update(str(tuple(p.shape)).encode("utf-8")) + h.update(arr.tobytes()) + return h.hexdigest() + + +class _BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes: int, planes: int, stride: int = 1, downsample: Optional[nn.Module] = None) -> None: + super().__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.downsample = downsample + + def forward(self, x: torch.Tensor) -> torch.Tensor: + identity = x + out = self.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + if self.downsample is not None: + identity = self.downsample(x) + return self.relu(out + identity) + + +class _ResNetBackbone(nn.Module): + """Standard torchvision-compatible ResNet-18 / ResNet-34 backbone without torchvision.""" + + def __init__(self, layers: Sequence[int], *, include_layer4: bool = False) -> None: + super().__init__() + self.inplanes = 64 + self.include_layer4 = bool(include_layer4) + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(64, layers[0], stride=1) + self.layer2 = self._make_layer(128, layers[1], stride=2) + self.layer3 = self._make_layer(256, layers[2], stride=2) + if self.include_layer4: + self.layer4 = self._make_layer(512, layers[3], stride=2) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + + def _make_layer(self, planes: int, blocks: int, stride: int) -> nn.Sequential: + downsample = None + if stride != 1 or self.inplanes != planes: + downsample = nn.Sequential( + nn.Conv2d(self.inplanes, planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(planes), + ) + layers = [_BasicBlock(self.inplanes, planes, stride=stride, downsample=downsample)] + self.inplanes = planes + for _ in range(1, blocks): + layers.append(_BasicBlock(self.inplanes, planes)) + return nn.Sequential(*layers) + + def load_torchvision_state_dict(self, state_dict: dict) -> None: + own_keys = set(self.state_dict().keys()) + filtered = {k: v for k, v in state_dict.items() if k in own_keys} + missing = {k for k in (own_keys - set(filtered.keys())) if not k.endswith(".num_batches_tracked")} + if missing: + raise ValueError(f"ResNet checkpoint missing expected keys: {sorted(missing)[:8]}") + self.load_state_dict(filtered, strict=False) + + def forward_features_123(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + x = self.maxpool(self.relu(self.bn1(self.conv1(x)))) + f1 = self.layer1(x) + f2 = self.layer2(f1) + f3 = self.layer3(f2) + return f1, f2, f3 + + def forward_avgpool(self, x: torch.Tensor) -> torch.Tensor: + if not self.include_layer4: + raise RuntimeError("forward_avgpool requires include_layer4=True") + _, _, f3 = self.forward_features_123(x) + f4 = self.layer4(f3) + return torch.flatten(self.avgpool(f4), 1) + + +def _load_verified_resnet_weights( + weights_path: Path, + *, + expected_sha256: Optional[str] = None, +) -> Tuple[dict, str]: + path = Path(weights_path).expanduser() + if not path.is_file(): + raise FileNotFoundError(f"Pretrained ResNet weights not found: {path}") + actual_sha = sha256_file(path) + if expected_sha256 is not None and actual_sha != expected_sha256: + # Accept prefix match if caller passes short hash, else strict mismatch error. + if not actual_sha.startswith(expected_sha256): + raise ValueError( + f"SHA-256 mismatch for {path}: expected {expected_sha256}, got {actual_sha}" + ) + sd = torch.load(path, map_location="cpu", weights_only=True) + return sd, actual_sha + + +class _TransformerBlock(nn.Module): + """Pre-LayerNorm multi-head self-attention + GELU FFN with separate Q/K/V/Out Linears.""" + + def __init__(self, d_model: int = 256, num_heads: int = 4, ffn_dim: int = 512) -> None: + super().__init__() + if d_model % num_heads != 0: + raise ValueError("d_model must be divisible by num_heads") + self.d_model = d_model + self.num_heads = num_heads + self.head_dim = d_model // num_heads + + self.attn_norm = nn.LayerNorm(d_model) + self.q_proj = nn.Linear(d_model, d_model, bias=True) + self.k_proj = nn.Linear(d_model, d_model, bias=True) + self.v_proj = nn.Linear(d_model, d_model, bias=True) + self.out_proj = nn.Linear(d_model, d_model, bias=True) + + self.ffn_norm = nn.LayerNorm(d_model) + self.fc1 = nn.Linear(d_model, ffn_dim, bias=True) + self.fc2 = nn.Linear(ffn_dim, d_model, bias=True) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + b, n, c = x.shape + h = self.attn_norm(x) + q = self.q_proj(h).view(b, n, self.num_heads, self.head_dim).transpose(1, 2) + k = self.k_proj(h).view(b, n, self.num_heads, self.head_dim).transpose(1, 2) + v = self.v_proj(h).view(b, n, self.num_heads, self.head_dim).transpose(1, 2) + # Explicit scaled dot-product attention for deterministic CPU/CUDA compatibility. + scale = 1.0 / math.sqrt(float(self.head_dim)) + attn = torch.matmul(q, k.transpose(-2, -1)) * scale + attn = torch.softmax(attn, dim=-1) + out = torch.matmul(attn, v).transpose(1, 2).contiguous().view(b, n, c) + x = x + self.out_proj(out) + + h2 = self.ffn_norm(x) + x = x + self.fc2(F.gelu(self.fc1(h2))) + return x + + +class TinyTransformerGenerator32(nn.Module): + """2-block 64-token Transformer generator -> 32x32 RGB in [-1, 1]. + + Exact structural match to HyperGAN's ``cifar10-tiny-transformer``: + 16 matrix/conv parameters (1 input projection + 2*(4 attn + 2 FFN) + 3 convs), + 2,489,731 trainable parameters total. + """ + + def __init__(self, z_dim: int = 64, d_model: int = 256, num_heads: int = 4, ffn_dim: int = 512) -> None: + super().__init__() + self.z_dim = z_dim + self.d_model = d_model + self.input_projection = nn.Linear(z_dim, 64 * d_model, bias=True) + self.block0 = _TransformerBlock(d_model=d_model, num_heads=num_heads, ffn_dim=ffn_dim) + self.block1 = _TransformerBlock(d_model=d_model, num_heads=num_heads, ffn_dim=ffn_dim) + self.conv16 = nn.Conv2d(d_model, 128, kernel_size=3, padding=1, bias=True) + self.conv32 = nn.Conv2d(128, 64, kernel_size=3, padding=1, bias=True) + self.rgb = nn.Conv2d(64, 3, kernel_size=3, padding=1, bias=True) + + def forward(self, z: torch.Tensor) -> torch.Tensor: + b = z.shape[0] + h = self.input_projection(z).view(b, 64, self.d_model) + h = self.block0(h) + h = self.block1(h) + h = h.transpose(1, 2).contiguous().view(b, self.d_model, 8, 8) + h = F.interpolate(h, scale_factor=2, mode="nearest") + h = F.leaky_relu(self.conv16(h), negative_slope=0.2) + h = F.interpolate(h, scale_factor=2, mode="nearest") + h = F.leaky_relu(self.conv32(h), negative_slope=0.2) + return torch.tanh(self.rgb(h)) + + +class _ResNetFeatureHead(nn.Module): + """Single multi-scale ResNet feature head (3 matrix/conv params).""" + + def __init__(self, in_channels: int, hidden_channels: int = 64) -> None: + super().__init__() + self.project = nn.Conv2d(in_channels, hidden_channels, kernel_size=1, bias=True) + self.norm = nn.GroupNorm(8, hidden_channels) + self.conv = nn.Conv2d(hidden_channels, hidden_channels, kernel_size=3, padding=1, bias=True) + self.score = nn.Linear(hidden_channels * 4 * 4, 1, bias=True) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h = F.leaky_relu(self.norm(self.project(x)), negative_slope=0.2) + h = F.leaky_relu(self.conv(h), negative_slope=0.2) + h = F.adaptive_avg_pool2d(h, (4, 4)).flatten(1) + return self.score(h) + + +class ResNet18FeatureCritic32(nn.Module): + """Frozen ImageNet ResNet-18 64px feature critic with 3 trainable feature heads. + + Exact structural match to HyperGAN's ``resnet18-features-discriminator-32``: + - Bilinear 32->64 resize + ImageNet normalization on ``[x, zeros_like(x)]``. + - Frozen ResNet-18 ``layer1`` (64ch), ``layer2`` (128ch), ``layer3`` (256ch). + - 3 trainable feature heads concatenating candidate + midgray-context features. + - 9 trainable matrix/conv parameters, 171,779 trainable parameters total. + - ``train()`` keeps the frozen backbone strictly in ``eval()`` mode so + ``GANTrainer`` calling ``D.train()`` never mutates BatchNorm statistics. + """ + + def __init__( + self, + weights_path: Optional[Path] = DEFAULT_RESNET18_WEIGHTS, + *, + backbone_mode: str = "pretrained", + backbone_seed: int = 1001, + ) -> None: + super().__init__() + if backbone_mode not in ("pretrained", "random"): + raise ValueError(f"backbone_mode must be 'pretrained' or 'random', got {backbone_mode!r}") + self.backbone_mode = backbone_mode + self.backbone_sha256: Optional[str] = None + self.backbone = _ResNetBackbone((2, 2, 2, 2), include_layer4=False) + if backbone_mode == "pretrained": + if weights_path is None: + raise ValueError("weights_path is required when backbone_mode='pretrained'") + sd, actual_sha = _load_verified_resnet_weights(Path(weights_path), expected_sha256=RESNET18_SHA256) + self.backbone.load_torchvision_state_dict(sd) + self.backbone_sha256 = actual_sha + else: + import particlegan as pg + + pg.init.deterministic_orthogonal_(self.backbone, seed=backbone_seed, strict=True) + self.backbone_sha256 = f"deterministic_random_seed_{backbone_seed}" + + for p in self.backbone.parameters(): + p.requires_grad_(False) + self.backbone.eval() + + self.register_buffer("_img_mean", torch.tensor(_IMAGENET_MEAN, dtype=torch.float32).view(1, 3, 1, 1)) + self.register_buffer("_img_std", torch.tensor(_IMAGENET_STD, dtype=torch.float32).view(1, 3, 1, 1)) + + self.head1 = _ResNetFeatureHead(64 * 2, 64) + self.head2 = _ResNetFeatureHead(128 * 2, 64) + self.head3 = _ResNetFeatureHead(256 * 2, 64) + self._inv_sqrt3 = 1.0 / math.sqrt(3.0) + + def train(self, mode: bool = True) -> "ResNet18FeatureCritic32": + super().train(mode) + # Frozen ImageNet ResNet-18 must always remain in eval() mode (fixed BN stats). + self.backbone.eval() + return self + + def forward(self, x: torch.Tensor) -> torch.Tensor: + context = torch.zeros_like(x) + feature_batch = torch.cat([x, context], dim=0) + resized = F.interpolate(feature_batch, scale_factor=2, mode="bilinear", align_corners=False) + normalized = (resized * 0.5 + 0.5 - self._img_mean.to(resized.dtype)) / self._img_std.to(resized.dtype) + f1, f2, f3 = self.backbone.forward_features_123(normalized) + cand1, ctx1 = f1.chunk(2, dim=0) + cand2, ctx2 = f2.chunk(2, dim=0) + cand3, ctx3 = f3.chunk(2, dim=0) + s1 = self.head1(torch.cat([cand1, ctx1], dim=1)) + s2 = self.head2(torch.cat([cand2, ctx2], dim=1)) + s3 = self.head3(torch.cat([cand3, ctx3], dim=1)) + return (s1 + s2 + s3) * self._inv_sqrt3 + + +class ResNet34FeatureEvaluator(nn.Module): + """Independent frozen ImageNet ResNet-34 512-dim feature extractor for FID/KID. + + Uses ResNet-34 (different architecture depth and weights from the critic's + frozen ResNet-18) so Feature-FID/KID and class-centroid coverage cannot be + gamed by critic-backbone shortcuts. + """ + + def __init__( + self, + weights_path: Optional[Path] = DEFAULT_RESNET34_WEIGHTS, + *, + backbone_mode: str = "pretrained", + backbone_seed: int = 2002, + ) -> None: + super().__init__() + if backbone_mode not in ("pretrained", "random"): + raise ValueError(f"backbone_mode must be 'pretrained' or 'random', got {backbone_mode!r}") + self.backbone_mode = backbone_mode + self.backbone_sha256: Optional[str] = None + self.backbone = _ResNetBackbone((3, 4, 6, 3), include_layer4=True) + if backbone_mode == "pretrained": + if weights_path is None: + raise ValueError("weights_path is required when backbone_mode='pretrained'") + sd, actual_sha = _load_verified_resnet_weights(Path(weights_path), expected_sha256=None) + self.backbone.load_torchvision_state_dict(sd) + self.backbone_sha256 = actual_sha + else: + import particlegan as pg + + pg.init.deterministic_orthogonal_(self.backbone, seed=backbone_seed, strict=True) + self.backbone_sha256 = f"deterministic_random_seed_{backbone_seed}" + + for p in self.backbone.parameters(): + p.requires_grad_(False) + self.backbone.eval() + self.register_buffer("_img_mean", torch.tensor(_IMAGENET_MEAN, dtype=torch.float32).view(1, 3, 1, 1)) + self.register_buffer("_img_std", torch.tensor(_IMAGENET_STD, dtype=torch.float32).view(1, 3, 1, 1)) + + @torch.no_grad() + def extract(self, images: torch.Tensor, *, batch_size: int = 256) -> torch.Tensor: + """Extract L2-unnormalized 512-dim float32 features from images in [-1, 1].""" + self.eval() + device = self._img_mean.device + feats = [] + n = int(images.shape[0]) + for start in range(0, n, batch_size): + chunk = images[start : start + batch_size].to(device=device, dtype=torch.float32) + resized = F.interpolate(chunk, size=(64, 64), mode="bilinear", align_corners=False) + normed = (resized * 0.5 + 0.5 - self._img_mean) / self._img_std + feats.append(self.backbone.forward_avgpool(normed).cpu()) + return torch.cat(feats, dim=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + resized = F.interpolate(x.float(), size=(64, 64), mode="bilinear", align_corners=False) + normed = (resized * 0.5 + 0.5 - self._img_mean) / self._img_std + return self.backbone.forward_avgpool(normed) + + +def iter_trainable_modules(module: nn.Module) -> Iterable[nn.Module]: + return (m for m in module.modules() if any(p.requires_grad for p in m.parameters(recurse=False))) diff --git a/benchmarks/cifar_toy/runner.py b/benchmarks/cifar_toy/runner.py new file mode 100644 index 000000000..da058a4c2 --- /dev/null +++ b/benchmarks/cifar_toy/runner.py @@ -0,0 +1,543 @@ +"""Reproducible CIFAR-style benchmark runner and leaderboard formatter. + +Adheres strictly to ``AGENTS.md`` comparison rules: +- Protocol seed 0 and ``particlegan.init.deterministic_orthogonal_``. +- Fixed architecture, dataset, critic/generator target batch sequence, initial + prior distribution/capacity, update budget, and evaluation cadence across + trainers. +- Isolated constructor, data, training-noise, prior-noise, and eval RNG streams. +- Executes exact resolved ParticleGAN ``Recipe`` via ``GANTrainer`` with zero + task-host repairs, gradient edits, or silent substitutions. +- Tail-friendly flushed single-line stdout logs + JSONL trace + compact JSON + receipts + single current markdown leaderboard per run directory. +""" +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +import json +import math +from pathlib import Path +import platform +import subprocess +import time +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +import torch + +import particlegan as pg + +from .data import ( + BatchSequenceStream, + CIFAR10Dataset, + compute_class_centroids, + load_cifar10_dataset, + make_synthetic_cifar_dataset, + sample_indices_for_eval, + select_eval_reference_subset, +) +from .metrics import ( + compute_class_coverage, + compute_critic_diagnostics, + compute_fid, + compute_grad_norm, + compute_kid, + compute_pixel_diversity_ratio, + feature_moments, +) +from .models import ( + DEFAULT_RESNET18_WEIGHTS, + DEFAULT_RESNET34_WEIGHTS, + ResNet18FeatureCritic32, + ResNet34FeatureEvaluator, + TinyTransformerGenerator32, + count_matrix_conv_params, + count_trainable_params, + parameter_state_digest, +) + +# Exact fixed MoG sigma from HyperGAN cifar10-tiny-transformer (0.025 * median NN in z=64). +DEFAULT_MOG_SIGMA = 0.212616428732872 + + +@dataclass(frozen=True) +class BenchmarkConfig: + recipe: str = "unified_v11" + num_particles: int = 200 + z_dim: int = 64 + mog_sigma: float = DEFAULT_MOG_SIGMA + standardize_prior: bool = True + batch_size: int = 128 + steps: int = 800 + eval_every: int = 200 + eval_samples: int = 2048 + eval_ref_samples: int = 5000 + eval_critic_samples: int = 256 + seed: int = 0 + dataset: str = "cifar10" + data_root: Optional[str] = None + horizontal_flip: bool = True + backbone_mode: str = "pretrained" + resnet18_weights: str = str(DEFAULT_RESNET18_WEIGHTS) + resnet34_weights: str = str(DEFAULT_RESNET34_WEIGHTS) + device: str = "cuda:0" + deterministic_algorithms: bool = False + recipe_overrides: Dict[str, Any] = field(default_factory=dict) + run_tag: Optional[str] = None + + +def _git_info() -> Dict[str, Any]: + root = Path(__file__).resolve().parents[2] + info: Dict[str, Any] = {"root": str(root)} + for key, cmd in ( + ("commit", ["git", "rev-parse", "HEAD"]), + ("branch", ["git", "rev-parse", "--abbrev-ref", "HEAD"]), + ("dirty", ["git", "status", "--porcelain"]), + ): + try: + res = subprocess.run(cmd, cwd=root, capture_output=True, text=True, check=False) + if res.returncode == 0: + val = res.stdout.strip() + info[key] = bool(val) if key == "dirty" else val + except OSError: + pass + return info + + +def _resolve_device(device_str: str) -> torch.device: + dev = torch.device(device_str) + if dev.type == "cuda": + if not torch.cuda.is_available(): + raise RuntimeError(f"Requested {device_str} but CUDA is unavailable") + if dev.index is None: + dev = torch.device("cuda", torch.cuda.current_device()) + return dev + + +def _device_name(device: torch.device) -> str: + if device.type == "cuda": + return torch.cuda.get_device_name(device) + return platform.processor() or platform.machine() or "cpu" + + +def _capture_backend_state(device: torch.device) -> dict: + state: dict = { + "cpu_rng": torch.get_rng_state(), + "num_threads": torch.get_num_threads(), + "det_alg": torch.are_deterministic_algorithms_enabled(), + "cudnn_det": torch.backends.cudnn.deterministic, + "cudnn_bench": torch.backends.cudnn.benchmark, + "cudnn_tf32": torch.backends.cudnn.allow_tf32, + "matmul_tf32": torch.backends.cuda.matmul.allow_tf32, + } + if device.type == "cuda": + state["cuda_rng"] = torch.cuda.get_rng_state(device) + return state + + +def _restore_backend_state(state: dict, device: torch.device) -> None: + torch.set_rng_state(state["cpu_rng"]) + torch.set_num_threads(state["num_threads"]) + torch.use_deterministic_algorithms(state["det_alg"]) + torch.backends.cudnn.deterministic = state["cudnn_det"] + torch.backends.cudnn.benchmark = state["cudnn_bench"] + torch.backends.cudnn.allow_tf32 = state["cudnn_tf32"] + torch.backends.cuda.matmul.allow_tf32 = state["matmul_tf32"] + if device.type == "cuda" and "cuda_rng" in state: + torch.cuda.set_rng_state(state["cuda_rng"], device) + + +def _format_progress_line(run_name: str, step: int, total_steps: int, row: Mapping[str, Any]) -> str: + fid = row.get("feature_fid", float("nan")) + kid = row.get("feature_kid_x1e3", float("nan")) + cov = int(round(row.get("class_coverage", 0.0))) + acc = row.get("critic_accuracy", float("nan")) + margin = row.get("critic_margin", float("nan")) + r_m = row.get("critic_real_mean", float("nan")) + f_m = row.get("critic_fake_mean", float("nan")) + gn = row.get("grad_norm_g", float("nan")) + dn = row.get("grad_norm_d", float("nan")) + pn = row.get("grad_norm_prior", float("nan")) + ips = row.get("steps_per_sec", float("nan")) + vram = row.get("peak_vram_mib", 0.0) + return ( + f"[{run_name}] step={step:04d}/{total_steps:04d} " + f"fid={fid:6.2f} kid={kid:6.2f} cov={cov:2d}/10 " + f"acc={acc:.3f} margin={margin:+.3f} (r={r_m:+.3f},f={f_m:+.3f}) " + f"gn={gn:.3f} dn={dn:.3f} pn={pn:.4f} " + f"ips={ips:5.2f} vram={vram:6.0f}MiB" + ) + + +def _load_dataset_for_config(cfg: BenchmarkConfig) -> CIFAR10Dataset: + if cfg.dataset == "cifar10": + return load_cifar10_dataset(Path(cfg.data_root) if cfg.data_root else None, split="train") + if cfg.dataset == "synthetic": + return make_synthetic_cifar_dataset( + num_samples=max(2000, cfg.eval_ref_samples), + seed=cfg.seed, + num_classes=10, + ) + raise ValueError(f"Unknown dataset {cfg.dataset!r}; expected 'cifar10' or 'synthetic'") + + +def run_benchmark( + cfg: BenchmarkConfig, + output_dir: Path, + *, + dataset: Optional[CIFAR10Dataset] = None, + evaluator: Optional[ResNet34FeatureEvaluator] = None, + ref_cache: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Execute one matched benchmark run and write tail-friendly logs + receipt.""" + device = _resolve_device(cfg.device) + out_root = Path(output_dir) + run_name = cfg.run_tag or f"{cfg.recipe}_p{cfg.num_particles}_b{cfg.batch_size}_s{cfg.steps}" + run_dir = out_root / run_name + run_dir.mkdir(parents=True, exist_ok=True) + metrics_path = run_dir / "metrics.jsonl" + log_path = run_dir / "train.log" + receipt_path = run_dir / "receipt.json" + + saved_state = _capture_backend_state(device) + try: + # Fixed protocol execution hygiene (restored in finally). + torch.manual_seed(cfg.seed) + if device.type == "cuda": + torch.cuda.manual_seed(cfg.seed) + torch.cuda.reset_peak_memory_stats(device) + torch.backends.cudnn.deterministic = bool(cfg.deterministic_algorithms) + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.allow_tf32 = False + torch.backends.cuda.matmul.allow_tf32 = False + if cfg.deterministic_algorithms: + torch.use_deterministic_algorithms(True) + + ds = _load_dataset_for_config(cfg) if dataset is None else dataset + + # Prepare independent ResNet-34 evaluator and reference feature moments once. + own_eval = evaluator is None + if own_eval: + evaluator = ResNet34FeatureEvaluator( + Path(cfg.resnet34_weights), + backbone_mode=cfg.backbone_mode, + backbone_seed=cfg.seed + 2002, + ).to(device=device) + evaluator.eval() + + if ref_cache is None: + ref_imgs, ref_labels = select_eval_reference_subset(ds, cfg.eval_ref_samples, seed=cfg.seed) + ref_feats = evaluator.extract(ref_imgs, batch_size=256) + ref_mu, ref_cov = feature_moments(ref_feats) + ref_centroids = compute_class_centroids(ref_feats, ref_labels, num_classes=ds.num_classes) + eval_critic_real = ref_imgs[: cfg.eval_critic_samples].clone() + ref_cache = { + "ref_imgs": ref_imgs, + "ref_feats": ref_feats, + "ref_mu": ref_mu, + "ref_cov": ref_cov, + "ref_centroids": ref_centroids, + "eval_critic_real": eval_critic_real, + } + + # Construct fixed task architecture & deterministic initialization. + G = TinyTransformerGenerator32(z_dim=cfg.z_dim).to(device=device) + D = ResNet18FeatureCritic32( + Path(cfg.resnet18_weights), + backbone_mode=cfg.backbone_mode, + backbone_seed=cfg.seed + 1001, + ).to(device=device) + prior = pg.MoGParticlePrior( + cfg.num_particles, + cfg.z_dim, + sigma=cfg.mog_sigma, + standardize=cfg.standardize_prior, + device=device, + ) + + # Deterministic public initializer (seed=0 for G & prior, seed=1 for D). + pg.init.deterministic_orthogonal_(G, seed=cfg.seed, strict=True) + pg.init.deterministic_orthogonal_(D, seed=cfg.seed + 1, strict=True) + pg.init.deterministic_orthogonal_(prior, seed=cfg.seed, strict=True) + + init_digests = { + "initial_g_sha256": parameter_state_digest(G), + "initial_d_sha256": parameter_state_digest(D), + "initial_prior_sha256": parameter_state_digest(prior), + "g_trainable_params": count_trainable_params(G), + "g_matrix_conv_params": count_matrix_conv_params(G), + "d_trainable_params": count_trainable_params(D), + "d_matrix_conv_params": count_matrix_conv_params(D), + "d_backbone_sha256": D.backbone_sha256, + "eval_backbone_sha256": evaluator.backbone_sha256, + } + + # Resolve exact public ParticleGAN Recipe (no host repairs / overrides unless explicitly declared). + recipe_kwargs: Dict[str, Any] = { + "prior_kind": "mog", + "z_dim": cfg.z_dim, + "num_particles": cfg.num_particles, + "sigma_rel": 0.0, + "standardize": cfg.standardize_prior, + "batch_size": cfg.batch_size, + "total_steps": cfg.steps, + } + recipe_kwargs.update(cfg.recipe_overrides) + recipe = pg.get_recipe(cfg.recipe, **recipe_kwargs) + + # Isolated named RNG streams on execution device. + def _dev_gen(offset: int) -> torch.Generator: + return torch.Generator(device=device).manual_seed(cfg.seed + offset) + + trainer = pg.GANTrainer( + recipe, + G, + D, + prior=prior, + seed=cfg.seed, + latent_generator=_dev_gen(11), + penalty_generator=_dev_gen(12), + noise_generator=_dev_gen(13), + input_noise_generator=_dev_gen(14), + prior_noise_generator=_dev_gen(15), + eval_generator=_dev_gen(16), + max_steps=cfg.steps, + ) + + # Isolated CPU data streams guaranteeing identical (real_d, real_g) sequence across recipes. + data_stream = BatchSequenceStream( + ds, + cfg.batch_size, + seed=cfg.seed, + horizontal_flip=cfg.horizontal_flip, + ) + + def _evaluate(step: int, last_step_info: Dict[str, float], train_seconds: float) -> Dict[str, Any]: + # Fixed per-step eval RNG so sampling is deterministic and isolated from training. + eval_stream = torch.Generator(device=device).manual_seed(cfg.seed + 50000 + step) + fake_chunks = [] + for b in sample_indices_for_eval(cfg.eval_samples, min(256, cfg.eval_samples)): + samples = trainer.sample(b, generator=eval_stream, output_noise=False) + fake_chunks.append(samples.detach().float().cpu()) + fake_all = torch.cat(fake_chunks, dim=0) + + fake_feats = evaluator.extract(fake_all, batch_size=256) + fake_mu, fake_cov = feature_moments(fake_feats) + fid = compute_fid(ref_cache["ref_mu"], ref_cache["ref_cov"], fake_mu, fake_cov) + kid_mean, kid_std = compute_kid( + ref_cache["ref_feats"], + fake_feats, + subset_size=min(500, cfg.eval_samples, cfg.eval_ref_samples), + num_subsets=20, + seed=cfg.seed, + ) + cov_stats = compute_class_coverage(fake_feats, ref_cache["ref_centroids"]) + div_ratio = compute_pixel_diversity_ratio( + ref_cache["eval_critic_real"], fake_all[: cfg.eval_critic_samples] + ) + critic_diag = compute_critic_diagnostics( + D, + ref_cache["eval_critic_real"], + fake_all[: cfg.eval_critic_samples], + batch_size=min(128, cfg.eval_critic_samples), + ) + peak_vram_mib = ( + float(torch.cuda.max_memory_allocated(device)) / (1024.0 * 1024.0) + if device.type == "cuda" + else 0.0 + ) + ips = float(step) / train_seconds if (step > 0 and train_seconds > 0.0) else 0.0 + row: Dict[str, Any] = { + "step": step, + "feature_fid": fid, + "feature_kid_x1e3": kid_mean * 1000.0, + "feature_kid_std_x1e3": kid_std * 1000.0, + **cov_stats, + "diversity_ratio": div_ratio, + **critic_diag, + **last_step_info, + "elapsed_train_sec": train_seconds, + "steps_per_sec": ips, + "peak_vram_mib": peak_vram_mib, + } + return row + + history: List[Dict[str, Any]] = [] + status = "COMPLETE" + error_msg: Optional[str] = None + train_seconds = 0.0 + last_step_info: Dict[str, float] = { + "loss_d": float("nan"), + "loss_g": float("nan"), + "loss_gan": float("nan"), + "penalty": float("nan"), + "prior_reg": float("nan"), + "grad_norm_g": 0.0, + "grad_norm_d": 0.0, + "grad_norm_prior": 0.0, + "prior_rms": float(prior.z.detach().float().square().mean().sqrt().item()), + } + + with open(metrics_path, "w", encoding="utf-8") as mf, open(log_path, "w", encoding="utf-8") as lf: + # Step-0 evaluation (verifies matched initialization across recipes). + row0 = _evaluate(0, last_step_info, 0.0) + history.append(row0) + mf.write(json.dumps(row0) + "\n") + mf.flush() + line0 = _format_progress_line(run_name, 0, cfg.steps, row0) + print(line0, flush=True) + lf.write(line0 + "\n") + lf.flush() + + for step in range(1, cfg.steps + 1): + real_d, real_g = data_stream.next_pair(device) + if device.type == "cuda": + torch.cuda.synchronize(device) + t0 = time.perf_counter() + out = trainer.step(real_d, generator_real=real_g) + if device.type == "cuda": + torch.cuda.synchronize(device) + train_seconds += time.perf_counter() - t0 + + loss_d = float(out["loss_d"].item()) + loss_g = float(out["loss_g"].item()) + loss_gan = float(out["loss_gan"].item()) + pen = float(out["penalty"].item()) + preg = float(out["prior_regularization"].item()) + + if not (math.isfinite(loss_d) and math.isfinite(loss_g)): + status = "NONFINITE" + error_msg = f"Nonfinite loss at step {step}: loss_d={loss_d}, loss_g={loss_g}" + lf.write(f"ERROR: {error_msg}\n") + lf.flush() + print(f"[{run_name}] ERROR: {error_msg}", flush=True) + break + + last_step_info = { + "loss_d": loss_d, + "loss_g": loss_g, + "loss_gan": loss_gan, + "penalty": pen, + "prior_reg": preg, + "grad_norm_g": compute_grad_norm(G.parameters()), + "grad_norm_d": compute_grad_norm(D.parameters()), + "grad_norm_prior": compute_grad_norm([prior.z]), + "prior_rms": float(prior.z.detach().float().square().mean().sqrt().item()), + } + + is_eval = (step % cfg.eval_every == 0) or (step == cfg.steps) + if is_eval: + row = _evaluate(step, last_step_info, train_seconds) + history.append(row) + mf.write(json.dumps(row) + "\n") + mf.flush() + line = _format_progress_line(run_name, step, cfg.steps, row) + print(line, flush=True) + lf.write(line + "\n") + lf.flush() + + # Summarize best & final metrics (excluding step 0 for best trained FID/KID). + trained_rows = [r for r in history if r["step"] > 0] + final_row = history[-1] + best_fid_row = min(trained_rows, key=lambda r: r["feature_fid"]) if trained_rows else final_row + best_kid_row = min(trained_rows, key=lambda r: r["feature_kid_x1e3"]) if trained_rows else final_row + + receipt: Dict[str, Any] = { + "schema_version": 1, + "run_name": run_name, + "status": status, + "error": error_msg, + "config": asdict(cfg), + "resolved_recipe": recipe.to_dict(), + "dataset": { + "source_name": ds.source_name, + "source_path": ds.source_path, + "size": ds.size, + "num_classes": ds.num_classes, + }, + "hardware": { + "device": str(device), + "device_name": _device_name(device), + "torch_version": torch.__version__, + "python_version": platform.python_version(), + }, + "git": _git_info(), + "provenance": { + **init_digests, + "batch_sequence_digest": data_stream.digest, + "steps_completed": int(final_row["step"]), + "sampling_law": "clean_live_mog", + "eval_output_noise": False, + }, + "summary": { + "final_step": int(final_row["step"]), + "final_fid": float(final_row["feature_fid"]), + "best_fid": float(best_fid_row["feature_fid"]), + "best_fid_step": int(best_fid_row["step"]), + "final_kid_x1e3": float(final_row["feature_kid_x1e3"]), + "best_kid_x1e3": float(best_kid_row["feature_kid_x1e3"]), + "final_class_coverage": int(round(final_row["class_coverage"])), + "final_class_mass_tv": float(final_row["class_mass_tv"]), + "final_diversity_ratio": float(final_row["diversity_ratio"]), + "final_critic_accuracy": float(final_row["critic_accuracy"]), + "final_critic_margin": float(final_row["critic_margin"]), + "final_critic_real_mean": float(final_row["critic_real_mean"]), + "final_critic_fake_mean": float(final_row["critic_fake_mean"]), + "final_grad_norm_g": float(final_row["grad_norm_g"]), + "final_grad_norm_d": float(final_row["grad_norm_d"]), + "final_grad_norm_prior": float(final_row["grad_norm_prior"]), + "steps_per_sec": float(final_row["steps_per_sec"]), + "elapsed_train_sec": float(final_row["elapsed_train_sec"]), + "peak_vram_mib": float(final_row["peak_vram_mib"]), + }, + "history": history, + } + receipt_path.write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8") + return receipt + finally: + _restore_backend_state(saved_state, device) + + +def format_leaderboard_markdown(receipts: Sequence[Mapping[str, Any]]) -> str: + """Render a compact, decision-ready markdown leaderboard table from receipts.""" + ordered = sorted( + receipts, + key=lambda r: ( + 0 if r.get("status") == "COMPLETE" else 1, + float(r.get("summary", {}).get("final_fid", float("inf"))), + ), + ) + lines = [ + "| Run | Recipe | MoG N | Status | Final FID ↓ | Best FID ↓ (step) | KID×1e3 ↓ | Cov /10 | TV ↓ | D acc | Margin | G grad | D grad | it/s ↑ | Peak MiB |", + "| --- | --- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + ] + for r in ordered: + cfg = r.get("config", {}) + s = r.get("summary", {}) + lines.append( + f"| `{r.get('run_name', '')}` | `{cfg.get('recipe', '')}` | {cfg.get('num_particles', 0)} " + f"| {r.get('status', 'UNKNOWN')} " + f"| {s.get('final_fid', float('nan')):.2f} " + f"| {s.get('best_fid', float('nan')):.2f} ({s.get('best_fid_step', 0)}) " + f"| {s.get('final_kid_x1e3', float('nan')):.2f} " + f"| {s.get('final_class_coverage', 0)}/10 " + f"| {s.get('final_class_mass_tv', float('nan')):.3f} " + f"| {s.get('final_critic_accuracy', float('nan')):.3f} " + f"| {s.get('final_critic_margin', float('nan')):+.3f} " + f"| {s.get('final_grad_norm_g', float('nan')):.3f} " + f"| {s.get('final_grad_norm_d', float('nan')):.3f} " + f"| {s.get('steps_per_sec', float('nan')):.2f} " + f"| {s.get('peak_vram_mib', 0.0):.0f} |" + ) + return "\n".join(lines) + "\n" + + +def summarize_run_directory(output_dir: Path) -> Tuple[List[Dict[str, Any]], str]: + """Collect all ``receipt.json`` files under ``output_dir`` and write one ``leaderboard.md``.""" + root = Path(output_dir) + receipts = [] + for path in sorted(root.glob("*/receipt.json")): + receipts.append(json.loads(path.read_text(encoding="utf-8"))) + md = format_leaderboard_markdown(receipts) + if receipts: + (root / "leaderboard.md").write_text(md, encoding="utf-8") + return receipts, md diff --git a/tests/test_cifar_toy_benchmark.py b/tests/test_cifar_toy_benchmark.py new file mode 100644 index 000000000..201c19df6 --- /dev/null +++ b/tests/test_cifar_toy_benchmark.py @@ -0,0 +1,120 @@ +"""Contract and semantic control tests for ``benchmarks.cifar_toy``.""" +from __future__ import annotations + +from pathlib import Path + +import pytest +import torch + +from benchmarks.cifar_toy.data import compute_class_centroids, make_synthetic_cifar_dataset +from benchmarks.cifar_toy.metrics import compute_class_coverage, compute_fid, compute_kid, feature_moments +from benchmarks.cifar_toy.models import ( + ResNet18FeatureCritic32, + ResNet34FeatureEvaluator, + TinyTransformerGenerator32, + count_matrix_conv_params, + count_trainable_params, +) +from benchmarks.cifar_toy.runner import BenchmarkConfig, run_benchmark + + +def test_architecture_exact_param_and_matrix_counts(): + gen = TinyTransformerGenerator32(z_dim=64) + disc = ResNet18FeatureCritic32(weights_path=None, backbone_mode="random") + + assert count_matrix_conv_params(gen) == 16 + assert count_trainable_params(gen) == 2_489_731 + + assert count_matrix_conv_params(disc) == 9 + assert count_trainable_params(disc) == 171_779 + + # Forward shapes match 4D RGB 32x32 -> scalar logit. + z = torch.randn(4, 64) + img = gen(z) + assert img.shape == (4, 3, 32, 32) + assert float(img.detach().min()) >= -1.0 and float(img.detach().max()) <= 1.0 + logits = disc(img) + assert logits.shape == (4, 1) + + +def test_frozen_backbone_stays_eval_and_unmodified(): + disc = ResNet18FeatureCritic32(weights_path=None, backbone_mode="random") + bn_mean_before = disc.backbone.bn1.running_mean.clone() + bn_var_before = disc.backbone.bn1.running_var.clone() + conv1_before = disc.backbone.conv1.weight.clone() + + # GANTrainer calls D.train(); backbone must stay frozen in eval mode. + disc.train(True) + assert disc.backbone.training is False + assert all(not p.requires_grad for p in disc.backbone.parameters()) + + x = torch.randn(4, 3, 32, 32, requires_grad=True) + loss = disc(x).sum() + loss.backward() + + # Gradients flow through frozen backbone to input x, without modifying backbone stats or weights. + assert x.grad is not None and torch.isfinite(x.grad).all() and float(x.grad.abs().sum()) > 0.0 + assert torch.equal(disc.backbone.bn1.running_mean, bn_mean_before) + assert torch.equal(disc.backbone.bn1.running_var, bn_var_before) + assert torch.equal(disc.backbone.conv1.weight, conv1_before) + assert all(p.grad is None for p in disc.backbone.parameters()) + + +def test_scorer_oracle_and_destructive_controls(): + ds = make_synthetic_cifar_dataset(num_samples=400, seed=0) + ev = ResNet34FeatureEvaluator(weights_path=None, backbone_mode="random") + + imgs = ds.to_float_tensor(torch.arange(400)) + feats = ev.extract(imgs, batch_size=64) + mu_r, cov_r = feature_moments(feats) + centroids = compute_class_centroids(feats, ds.labels, num_classes=10) + + # Oracle control: real vs real => FID ~ 0, KID ~ 0, 10/10 coverage. + oracle_fid = compute_fid(mu_r, cov_r, mu_r, cov_r) + oracle_kid, _ = compute_kid(feats[:200], feats[200:], subset_size=100, num_subsets=5, seed=0) + oracle_cov = compute_class_coverage(feats, centroids) + assert oracle_fid < 1e-4 + assert abs(oracle_kid) < 1e-3 + assert oracle_cov["class_coverage"] == 10.0 + + # Destructive control: constant collapsed image yields large FID/KID and collapsed coverage (1/10). + collapsed = torch.zeros_like(imgs[:200]) + bad_feats = ev.extract(collapsed, batch_size=64) + mu_b, cov_b = feature_moments(bad_feats) + bad_fid = compute_fid(mu_r, cov_r, mu_b, cov_b) + bad_kid, _ = compute_kid(feats[:200], bad_feats, subset_size=100, num_subsets=5, seed=0) + bad_cov = compute_class_coverage(bad_feats, centroids) + assert bad_fid > oracle_fid + 0.25 + assert bad_kid > oracle_kid + 1e-3 + assert bad_cov["class_coverage"] == 1.0 + + +@pytest.mark.parametrize("recipes", [("empf", "unified_v11")]) +def test_matched_initial_state_and_batch_sequence_across_trainers(tmp_path: Path, recipes): + receipts = [] + for recipe in recipes: + cfg = BenchmarkConfig( + recipe=recipe, + num_particles=16, + batch_size=8, + steps=2, + eval_every=2, + eval_samples=16, + eval_ref_samples=64, + eval_critic_samples=8, + seed=0, + dataset="synthetic", + backbone_mode="random", + device="cpu", + run_tag=f"smoke_{recipe}", + ) + receipts.append(run_benchmark(cfg, tmp_path)) + + r0, r1 = receipts + assert r0["status"] == "COMPLETE" and r1["status"] == "COMPLETE" + # Identical initial weights, initial prior, and batch sequence across trainers. + for key in ("initial_g_sha256", "initial_d_sha256", "initial_prior_sha256", "batch_sequence_digest"): + assert r0["provenance"][key] == r1["provenance"][key], f"Mismatch on {key}" + # Identical step-0 FID/logits before any recipe updates. + assert r0["history"][0]["feature_fid"] == pytest.approx(r1["history"][0]["feature_fid"], abs=1e-4) + assert r0["history"][0]["critic_margin"] == pytest.approx(r1["history"][0]["critic_margin"], abs=1e-4) From b569a2a980325f2f7887e099e4c25b44cc6a5ee0 Mon Sep 17 00:00:00 2001 From: Martyn Garcia Date: Fri, 20 Nov 2026 11:10:00 -0700 Subject: [PATCH 4/8] fix(benchmarks): verify ResNet-34 SHA-256, use repo-relative output-dir, register cifar_toy in Forge history --- benchmarks/cifar_toy/__init__.py | 2 - benchmarks/cifar_toy/__main__.py | 2 +- benchmarks/cifar_toy/models.py | 7 +--- configs/forge/catalog.json | 67 +++++++++++++++++++++++++----- experiments/forge/history.py | 2 +- reports/forge/EXPERIMENT_MEMORY.md | 2 +- reports/forge/compilation.json | 10 ++--- reports/forge/import-gaps.json | 4 +- 8 files changed, 69 insertions(+), 27 deletions(-) diff --git a/benchmarks/cifar_toy/__init__.py b/benchmarks/cifar_toy/__init__.py index 0e0e96f3d..6f5f69bee 100644 --- a/benchmarks/cifar_toy/__init__.py +++ b/benchmarks/cifar_toy/__init__.py @@ -23,7 +23,6 @@ compute_pixel_diversity_ratio, ) from .models import ( - DEFAULT_INCEPTION_WEIGHTS, DEFAULT_RESNET18_WEIGHTS, DEFAULT_RESNET34_WEIGHTS, RESNET18_SHA256, @@ -47,7 +46,6 @@ "BatchSequenceStream", "BenchmarkConfig", "CIFAR10Dataset", - "DEFAULT_INCEPTION_WEIGHTS", "DEFAULT_MOG_SIGMA", "DEFAULT_RESNET18_WEIGHTS", "DEFAULT_RESNET34_WEIGHTS", diff --git a/benchmarks/cifar_toy/__main__.py b/benchmarks/cifar_toy/__main__.py index 6c781d202..7f7dd52e3 100644 --- a/benchmarks/cifar_toy/__main__.py +++ b/benchmarks/cifar_toy/__main__.py @@ -75,7 +75,7 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument( "--output-dir", type=str, - default="/nvme/dev/hypergan/ParticleGAN/runs/cifar_toy_bench", + default="runs/cifar_toy_bench", help="Directory for bulk logs, JSONL traces, receipts, and leaderboard.md.", ) p.add_argument( diff --git a/benchmarks/cifar_toy/models.py b/benchmarks/cifar_toy/models.py index f85348b0d..601751c4f 100644 --- a/benchmarks/cifar_toy/models.py +++ b/benchmarks/cifar_toy/models.py @@ -29,13 +29,10 @@ import torch.nn.functional as F RESNET18_SHA256 = "f37072fd47e89c5e827621c5baffa7500819f7896bbacec160b1a16c560e07ec" -RESNET34_SHA256 = "b627a593f4b504425f6d6c373712e796be8773f6cb3c5910b09070727651153e" +RESNET34_SHA256 = "b627a593bcbe140c234610266fe4f8ae95ea42fc881d091c9b6052e6b1d0590f" DEFAULT_RESNET18_WEIGHTS = Path.home() / ".cache" / "torch" / "hub" / "checkpoints" / "resnet18-f37072fd.pth" DEFAULT_RESNET34_WEIGHTS = Path.home() / ".cache" / "torch" / "hub" / "checkpoints" / "resnet34-b627a593.pth" -DEFAULT_INCEPTION_WEIGHTS = ( - Path.home() / ".cache" / "torch" / "hub" / "checkpoints" / "weights-inception-2015-12-05-6726825d.pth" -) _IMAGENET_MEAN = (0.485, 0.456, 0.406) _IMAGENET_STD = (0.229, 0.224, 0.225) @@ -347,7 +344,7 @@ def __init__( if backbone_mode == "pretrained": if weights_path is None: raise ValueError("weights_path is required when backbone_mode='pretrained'") - sd, actual_sha = _load_verified_resnet_weights(Path(weights_path), expected_sha256=None) + sd, actual_sha = _load_verified_resnet_weights(Path(weights_path), expected_sha256=RESNET34_SHA256) self.backbone.load_torchvision_state_dict(sd) self.backbone_sha256 = actual_sha else: diff --git a/configs/forge/catalog.json b/configs/forge/catalog.json index 3feb7bb79..0606bb55a 100644 --- a/configs/forge/catalog.json +++ b/configs/forge/catalog.json @@ -1,6 +1,7 @@ { "counts": { "by_family": { + "benchmark/cifar_toy": 6, "benchmark/init_research": 10, "benchmark/learned_lr": 10, "benchmark/legacy": 5, @@ -104,7 +105,7 @@ "candidate_implementation": 17, "candidate_patch": 26, "configuration": 810, - "driver": 405, + "driver": 407, "engine_support": 128, "evaluator_or_audit": 324, "evidence_artifact": 1551, @@ -114,18 +115,18 @@ "frozen_support": 28, "narrative": 975, "structured_evidence": 5070, - "support": 1561, + "support": 1565, "support_artifact": 52, "visualization": 23, "visualization_artifact": 765 }, - "files": 11855 + "files": 11861 }, "coverage": { - "declared_count": 11855, + "declared_count": 11861, "missing": [], "stale": [], - "tracked_count": 11855, + "tracked_count": 11861, "unclassified": [], "valid": true }, @@ -141,6 +142,52 @@ "role": "support", "suggested_tiers": [] }, + { + "family": "benchmark/cifar_toy", + "git_blob": "0e0e96f3d4a8a96b49ac7634b98ae379506dc55f", + "path": "benchmarks/cifar_toy/__init__.py", + "role": "support", + "suggested_tiers": [] + }, + { + "family": "benchmark/cifar_toy", + "git_blob": "6c781d2029b7f039ac50cfa34878cd59c0aa80d3", + "path": "benchmarks/cifar_toy/__main__.py", + "role": "driver", + "suggested_tiers": [ + 2 + ] + }, + { + "family": "benchmark/cifar_toy", + "git_blob": "b6bda1008e8ffea99018cbcb4116807c92ae7dde", + "path": "benchmarks/cifar_toy/data.py", + "role": "support", + "suggested_tiers": [] + }, + { + "family": "benchmark/cifar_toy", + "git_blob": "01352b5a2af35a216e924287ee0dec7212196891", + "path": "benchmarks/cifar_toy/metrics.py", + "role": "support", + "suggested_tiers": [] + }, + { + "family": "benchmark/cifar_toy", + "git_blob": "f85348b0da6ba67b000675547ac27bc73c5725b9", + "path": "benchmarks/cifar_toy/models.py", + "role": "support", + "suggested_tiers": [] + }, + { + "family": "benchmark/cifar_toy", + "git_blob": "da058a4c2ae6d5c1e365988020e786e9fb58598a", + "path": "benchmarks/cifar_toy/runner.py", + "role": "driver", + "suggested_tiers": [ + 2 + ] + }, { "family": "benchmark/shared", "git_blob": "e28cba9ae2e2ced03b6a663d20bcfb0bcc3910d5", @@ -18209,14 +18256,14 @@ }, { "family": "report/readme-100gaussians", - "git_blob": "c2ef7afe29bf4c0f0d7cfda3b606aa54e2cfc98e", + "git_blob": "85df9d9d3b24290af8520988ab179afdcc0fd634", "path": "reports/readme-100gaussians/CURRENT_PROTOCOL.md", "role": "narrative", "suggested_tiers": [] }, { "family": "report/readme-100gaussians", - "git_blob": "99b40cffa7ac9aca95fb326530ddc645791fdbc8", + "git_blob": "b15543ac2d6a83b1fde9bac944f507801aa66c54", "path": "reports/readme-100gaussians/README.md", "role": "narrative", "suggested_tiers": [] @@ -18230,14 +18277,14 @@ }, { "family": "report/readme-100gaussians", - "git_blob": "bfa9a90e443d212db2a0177bfbf04ae8e37ada83", + "git_blob": "2067f78abf8a0036c68b5e3e4f2459d8953a2625", "path": "reports/readme-100gaussians/record.py", "role": "support", "suggested_tiers": [] }, { "family": "report/readme-100gaussians", - "git_blob": "8a0bef9381624504698f6b5ed35de05067406e81", + "git_blob": "cfc6409b4a6c8e647eb2541d09ebeb6961a7fc95", "path": "reports/readme-100gaussians/render.py", "role": "visualization", "suggested_tiers": [] @@ -18251,7 +18298,7 @@ }, { "family": "report/readme-100gaussians", - "git_blob": "f6f31cc543f7960c6c79d95d3e2a7e7db4eecb5b", + "git_blob": "326c8f03f9483266f6dedb8e649b0c233ef25d2f", "path": "reports/readme-100gaussians/summary-current.json", "role": "structured_evidence", "suggested_tiers": [] diff --git a/experiments/forge/history.py b/experiments/forge/history.py index 28804e39b..b4898eae5 100644 --- a/experiments/forge/history.py +++ b/experiments/forge/history.py @@ -45,7 +45,7 @@ "atlas-explanation", "e22-animation", "r1-integrations", "develop-gates-20261001", "paired-residual-toy", "toy_audit", "gram-profile", "prior-ablation-2026-10-25", } -BENCHMARK_FAMILIES = {"init_research", "learned_lr", "legacy", "locked_shared", +BENCHMARK_FAMILIES = {"cifar_toy", "init_research", "learned_lr", "legacy", "locked_shared", "paired_error_2d", "smart_descent", "toy100", "transfer_suite", "routed_conditioning", "toy_audit"} CONFIG_FAMILIES = {"100gaussians", "cifar_ddgan", "cifar_particle_ae", diff --git a/reports/forge/EXPERIMENT_MEMORY.md b/reports/forge/EXPERIMENT_MEMORY.md index 41fd474cf..322f68af6 100644 --- a/reports/forge/EXPERIMENT_MEMORY.md +++ b/reports/forge/EXPERIMENT_MEMORY.md @@ -11350,4 +11350,4 @@ These study and trial projections preserve recorded outcomes, unknowns and sourc ## Compilation provenance -Reducer `forge-knowledge-v2`; input digest `f954048a526ef3ba9b6f18dbddad293d6f57d826f96b4e6d8d296f54abd13d92`. [Full input hashes and coverage](compilation.json). No training or image inspection occurs during compilation. +Reducer `forge-knowledge-v2`; input digest `f96d88445dad7bf5d65a2e1227d8f3d72d142342a0b388a8b9618f58d4ea0387`. [Full input hashes and coverage](compilation.json). No training or image inspection occurs during compilation. diff --git a/reports/forge/compilation.json b/reports/forge/compilation.json index 5a7cf2127..3f1c999be 100644 --- a/reports/forge/compilation.json +++ b/reports/forge/compilation.json @@ -2,7 +2,7 @@ "automation_digest": "975631e907a1ca668dde96bd2837db3d6c54dadeacaf628069d8333c4539ee33", "conflicts": [], "import_gap_count": 7, - "input_digest": "f954048a526ef3ba9b6f18dbddad293d6f57d826f96b4e6d8d296f54abd13d92", + "input_digest": "f96d88445dad7bf5d65a2e1227d8f3d72d142342a0b388a8b9618f58d4ea0387", "input_hashes": { "configs/forge/cadence-studies/bispf-lazy-cadence-v1.json": "fc7302ac06b61fa093a39265d440545fdab176852a588c40b750be16adc1e58b", "configs/forge/calibration-lane-template.json": "65f10d3061aa487bb32da43bb97cb91934a25d763f6bb634f1d3316fdcc6d9e5", @@ -1936,7 +1936,7 @@ "reports/forge/family-winner-round1/r1r2-grid-preparation.json": "a7cdb6a87abbae05f3659cf0fbb34f6d20cc3639d846c7accd35e8a3c82e8c78", "reports/forge/family-winner-round1/r1r2-trajectory-representation.json": "8f1506391a0b5bb1da5d348c526faf04da78c362e73b4ea8402f57930638a91f", "reports/forge/history-sources.json": "c71655de9bcd7333b83ce1b5ed5b0d6bf21a0805fd9488cf5da5fba404e31d3b", - "reports/forge/import-gaps.json": "e6fdb149630cce361a222736928cdc77e3080988d97b53c3a575bdcec88164b8", + "reports/forge/import-gaps.json": "396cada13993162758b247a33b4f4e9fbe36185cd6f6aeb7fce47039e3da4118", "reports/forge/passive-publications.json": "cf4fb8e53db1896705ad70bb998e0d53d6a485783091ba02f13b9f471e9909c8", "reports/forge/policy-family-inventory.json": "b36cf5d76adc736adc09184afef38d81b708313864e0e94c38a47a75ace004dc", "reports/forge/policy-family-inventory.md": "08c9682c7233d883ef1ca8bc785c8c723245df365bb9fbd844fc05792030546c", @@ -5690,16 +5690,16 @@ "reports/toy_audit/api_contract/model_modes/publication.json": "3fe2d2f3b09d9e1c34b5dea5435e3d67a6aee30023e265acef25ffc6808fc6cd" }, "inventory_coverage": { - "declared_count": 11855, + "declared_count": 11861, "missing": [], "stale": [], - "tracked_count": 11855, + "tracked_count": 11861, "unclassified": [], "valid": true }, "operational_lifecycle_digest": "a9af4314073c4377eb6fc81c5cb20417930f4e0c3e36165961fca921a04ab15b", "output_hashes": { - "reports/forge/EXPERIMENT_MEMORY.md": "da46a59dee947e2f0011a7fab3eda6b7bc7b9e3354528c2c2629a620df983609", + "reports/forge/EXPERIMENT_MEMORY.md": "a7d59159cd96255d398fdcf477595ab253152bfab1468b93b0de5b111bfefe7d", "reports/forge/publication-records.json": "5174449ad10c00a8d22dee09d3ef50c8fb0fc76f196d85197271ee7c0766c9d8" }, "pending_readout": [], diff --git a/reports/forge/import-gaps.json b/reports/forge/import-gaps.json index fe2a91699..6e8b960ab 100644 --- a/reports/forge/import-gaps.json +++ b/reports/forge/import-gaps.json @@ -2516,10 +2516,10 @@ ], "historical_import_complete": true, "inventory_coverage": { - "declared_count": 11855, + "declared_count": 11861, "missing": [], "stale": [], - "tracked_count": 11855, + "tracked_count": 11861, "unclassified": [], "valid": true }, From 78ca410020e3115ade1e1d52574240f9357f37d3 Mon Sep 17 00:00:00 2001 From: Martyn Garcia Date: Fri, 20 Nov 2026 12:01:03 -0700 Subject: [PATCH 5/8] feat(unified_v11): universal 1st-order fwAD Law 2, scalar logit Law 3, and dimension-agnostic Law 4 --- particlegan/_unified_v11/critic.py | 56 +++-- particlegan/_unified_v11/loss.py | 40 +++- particlegan/_unified_v11/penalty.py | 331 +++++++++++++++++++++++++++- tests/test_unified_v11.py | 154 +++++++++++++ 4 files changed, 559 insertions(+), 22 deletions(-) diff --git a/particlegan/_unified_v11/critic.py b/particlegan/_unified_v11/critic.py index 3548f4703..d059bef23 100644 --- a/particlegan/_unified_v11/critic.py +++ b/particlegan/_unified_v11/critic.py @@ -145,17 +145,31 @@ def observe_margin_and_scale( else: self.ema_margin = 0.90 * self.ema_margin + 0.10 * margin if x_real is not None and x_real.numel() > 0: - xr = x_real.detach().reshape(x_real.shape[0], -1).float() - sc = float(xr.square().sum(dim=-1).mean().sqrt().clamp_min(0.05).item()) - if math.isfinite(sc): - self.ema_data_scale = sc if self.observed_steps == 0 else (0.95 * self.ema_data_scale + 0.05 * sc) - if x_fake is not None and x_fake.shape == x_real.shape: - xf = x_fake.detach().reshape(x_fake.shape[0], -1).float() - mean_gap = float((xf.mean(dim=0) - xr.mean(dim=0)).norm().item()) - std_gap = abs(float(xf.std().item()) - float(xr.std().item())) - rel_gap = min(10.0, (mean_gap + 0.5 * std_gap) / max(0.05, self.ema_data_scale)) - if math.isfinite(rel_gap): - self.ema_push_res = rel_gap if self.observed_steps == 0 else (0.92 * self.ema_push_res + 0.08 * rel_gap) + self.observe_scale(x_real, x_fake) + + @torch.no_grad() + def observe_scale( + self, + x_real: torch.Tensor, + x_fake: Optional[torch.Tensor] = None, + ) -> None: + """Update checkpointed data scale and pushforward residual EMAs.""" + if getattr(self, "_in_penalty", False) or getattr(self, "_in_probe", False) or getattr(self, "_in_protected_probe", False): + return + if x_real is None or x_real.numel() == 0: + return + step_tag = int(getattr(self, "observed_steps", 0)) + xr = x_real.detach().reshape(x_real.shape[0], -1).float() + sc = float(xr.square().sum(dim=-1).mean().sqrt().clamp_min(0.05).item()) + if math.isfinite(sc): + self.ema_data_scale = sc if step_tag == 0 else (0.95 * self.ema_data_scale + 0.05 * sc) + if x_fake is not None and x_fake.shape == x_real.shape: + xf = x_fake.detach().reshape(x_fake.shape[0], -1).float() + mean_gap = float((xf.mean(dim=0) - xr.mean(dim=0)).norm().item()) + std_gap = abs(float(xf.std().item()) - float(xr.std().item())) + rel_gap = min(10.0, (mean_gap + 0.5 * std_gap) / max(0.05, self.ema_data_scale)) + if math.isfinite(rel_gap): + self.ema_push_res = rel_gap if step_tag == 0 else (0.92 * self.ema_push_res + 0.08 * rel_gap) def clear_transient_captures(self) -> None: if hasattr(self, "_live_captures") and self._live_captures: @@ -425,7 +439,7 @@ def _unified_v11_generator_forward_hook( kwargs: Dict[str, Any], output: Any, ) -> None: - """Capture a detached ``B_probe <= 16`` generator slice for function-space secant trust-band regulation.""" + """Capture a dimension-agnostic element-budgeted generator slice for function-space secant trust-band regulation.""" rec = getattr(module, "_unified_v11_record", None) if ( rec is None @@ -443,19 +457,26 @@ def _unified_v11_generator_forward_hook( if ( not isinstance(raw_out, torch.Tensor) or not raw_out.requires_grad - or raw_out.ndim != 2 + or raw_out.ndim < 2 or raw_out.shape[0] < 1 - or raw_out.shape[1] > 32 ): return import torch.autograd.forward_ad as fwAD if fwAD.unpack_dual(raw_out).tangent is not None: return b_full = int(raw_out.shape[0]) - b_p = min(_PROBE_SLICE, b_full) + out_dim = max(1, int(raw_out[0].numel())) + b_p = min(b_full, max(2, min(_PROBE_SLICE, 8192 // out_dim))) p_args = tuple(_slice_probe_arg(a, b_full, b_p) for a in args) p_kwargs = {k: _slice_probe_arg(v, b_full, b_p) for k, v in kwargs.items()} - rec._g_probe_cache = (module, p_args, p_kwargs, raw_out.detach()[:b_p].to(dtype=torch.float32)) + if getattr(module, "_unified_v11_gen_has_bn", False): + from .._unified_v4.optimizer import _eval_probe_forward + y_pre = _eval_probe_forward(module, p_args, p_kwargs, rec) + if y_pre is None: + return + else: + y_pre = raw_out.detach()[:b_p].to(dtype=torch.float32) + rec._g_probe_cache = (module, p_args, p_kwargs, y_pre) def _attach_unified_v11_critic_hook(critic: nn.Module | None, record: CriticStepRecord) -> None: @@ -491,6 +512,9 @@ def _attach_unified_v11_generator_hook(generator: nn.Module | None, record: Crit if not isinstance(generator, nn.Module): return generator._unified_v11_record = record + generator._unified_v11_gen_has_bn = any( + isinstance(m, torch.nn.modules.batchnorm._BatchNorm) for m in generator.modules() + ) if getattr(generator, "_unified_v11_gen_hook_registered", False): return generator.register_forward_hook(_unified_v11_generator_forward_hook, with_kwargs=True) diff --git a/particlegan/_unified_v11/loss.py b/particlegan/_unified_v11/loss.py index 475b45f74..7be14c336 100644 --- a/particlegan/_unified_v11/loss.py +++ b/particlegan/_unified_v11/loss.py @@ -615,16 +615,38 @@ def _resolve_pair_parts( parts_r = None return x_f, x_r, cond_f, cond_r, rec, has_cont_cond, parts_f, parts_r, captured + @staticmethod + def _scalar_pair_relativistic( + pos_logits: torch.Tensor, + neg_logits: torch.Tensor, + *, + detach_neg: bool = False, + ) -> Optional[torch.Tensor]: + b_p, b_n = int(pos_logits.shape[0]), int(neg_logits.shape[0]) + if b_p != b_n or b_p < 1: + return None + p_flat = pos_logits.reshape(b_p, -1).mean(dim=1) if pos_logits.ndim > 1 else pos_logits.reshape(b_p) + n_src = neg_logits.detach() if detach_neg else neg_logits + n_flat = n_src.reshape(b_n, -1).mean(dim=1) if n_src.ndim > 1 else n_src.reshape(b_n) + return F.softplus(n_flat - p_flat).mean() + def d_loss(self, real_logits: torch.Tensor, fake_logits: torch.Tensor) -> torch.Tensor: base = super().d_loss(real_logits, fake_logits) x_f, x_r, cond_f, cond_r, rec, has_cont_cond, _, _, captured = self._resolve_pair_parts( fake_logits, real_logits ) - if captured and rec is not None and hasattr(rec, "observe_margin_and_scale") and not has_cont_cond: - rec.observe_margin_and_scale(real_logits, fake_logits, x_r, x_f) + if rec is not None and hasattr(rec, "observe_margin_and_scale") and not has_cont_cond: + if captured: + rec.observe_margin_and_scale(real_logits, fake_logits, x_r, x_f) + else: + rec.observe_margin_and_scale(real_logits, fake_logits) if self.critic_weight == 0.0 and self.fibre_rel_blend == 0.0: return base if not captured: + if self.loss == "non_saturating" and self.t1c.weight > 0.0 and self.fibre_rel_blend > 0.0: + pair_d = self._scalar_pair_relativistic(real_logits, fake_logits) + if pair_d is not None: + return (1.0 - self.fibre_rel_blend) * base + self.fibre_rel_blend * pair_d return base if self.conditional_fibre == "slack" and has_cont_cond: if self.fibre_rel_blend > 0.0: @@ -672,7 +694,12 @@ def g_loss(self, fake_logits: torch.Tensor, real_logits: torch.Tensor | None = N fake_logits, real_logits ) if not captured: - return self.t1c.weight * super().g_loss(fake_logits, real_logits) + base_g = super().g_loss(fake_logits, real_logits) + if self.loss == "non_saturating" and self.fibre_rel_blend > 0.0: + pair_g = self._scalar_pair_relativistic(fake_logits, real_logits, detach_neg=True) + if pair_g is not None: + return self.t1c.weight * ((1.0 - self.fibre_rel_blend) * base_g + self.fibre_rel_blend * pair_g) + return self.t1c.weight * base_g b_f = fake_logits.shape[0] b_r = real_logits.shape[0] @@ -733,7 +760,12 @@ def g_loss(self, fake_logits: torch.Tensor, real_logits: torch.Tensor | None = N rec._t1c_disc_gate = 0.0 if rec is not None and hasattr(rec, "_simplicial_step"): rec._simplicial_step = False - return super().g_loss(fake_logits, real_logits) + base_g = super().g_loss(fake_logits, real_logits) + if self.loss == "non_saturating" and self.fibre_rel_blend > 0.0: + pair_g = self._scalar_pair_relativistic(fake_logits, real_logits, detach_neg=True) + if pair_g is not None: + return self.t1c.weight * ((1.0 - self.fibre_rel_blend) * base_g + self.fibre_rel_blend * pair_g) + return base_g # 3. H_0-connectedness-gated chart coupling + zero-eigh orphan macro-transport: allow_ell = torch.is_grad_enabled() and not getattr(rec, "_in_protected_probe", False) diff --git a/particlegan/_unified_v11/penalty.py b/particlegan/_unified_v11/penalty.py index 3c7624f9b..1bb5bc8c7 100644 --- a/particlegan/_unified_v11/penalty.py +++ b/particlegan/_unified_v11/penalty.py @@ -1,13 +1,22 @@ """Unified-v11 1st-Order Spatial-Frame Forward-Tangent & Multi-Probe Secant Born-Infeld Critic Penalty.""" from __future__ import annotations -from typing import Any, Dict +from typing import Any, Callable, Dict, Optional, Sequence, Tuple import torch from torch import nn +import torch.autograd.forward_ad as fwAD +import torch.nn.functional as F +from .._t2_critic.penalty import ( + _format_logits_2d, + _murmur32_signs, + _separable_image_unit_perturbation, +) from .._unified.penalty import ( + _GOLDEN_CONJ, _UnifiedPenaltyKernel, + _adaptive_fd_delta, _eval_lowd_forward_grad_sq, ) from ..k3p import CriticPenalty as _K3PCriticPenalty, K3PCriticAdam, _first_output @@ -20,14 +29,330 @@ from .optimizer import UnifiedV11EquilibratedOptimizer +def _universal_image_unit_tangent(x: torch.Tensor, phase: float, salt: int) -> torch.Tensor: + """Zero-RNG isotropic unit tangent ``||u_i||_2 = 1`` for any spatial tensor rank >= 3.""" + b = int(x.shape[0]) + d = max(1, int(x[0].numel())) + if d <= 8192: + ranks = torch.arange(b, device=x.device, dtype=torch.int64) + u = _murmur32_signs(ranks, d, phase, salt, x.device) + u = u / u.norm(dim=1, keepdim=True).clamp_min(1e-8) + return u.to(dtype=x.dtype).reshape_as(x) + return _separable_image_unit_perturbation(x, phase, salt=salt) + + +def _bn_dual_bf16_hook( + _module: nn.Module, + _inputs: Tuple[Any, ...], + output: Any, +) -> Any: + """Keep BatchNorm dual primal + tangent aligned in BF16 during autocast fwAD passes.""" + if isinstance(output, torch.Tensor): + primal, tangent = fwAD.unpack_dual(output) + if tangent is not None and (primal.dtype != torch.bfloat16 or tangent.dtype != torch.bfloat16): + return fwAD.make_dual(primal.to(dtype=torch.bfloat16), tangent.to(dtype=torch.bfloat16)) + return output + + class _UnifiedV11PenaltyKernel(_UnifiedPenaltyKernel): - """Unified-v11 1st-order forward-tangent & multi-probe secant Born-Infeld kernel (`0` `autograd.grad`).""" + """Unified-v11 universal 1st-order forward-AD Born-Infeld critic kernel (`0` `autograd.grad`).""" def __init__(self, *, record: Any = None, **kwargs: Any) -> None: rec = UnifiedV11StepRecord() if record is None else record super().__init__(record=rec, **kwargs) self.arm = "unified_v11" + def _eval_image_fwad_sq( + self, + live: Callable[[torch.Tensor], torch.Tensor], + x_probe: torch.Tensor, + phi: float, + *, + salt: int, + dim: int, + has_batchnorm: bool, + critic_module: Optional[nn.Module], + quantized_fprop: bool, + ) -> torch.Tensor: + """Evaluate ``dim * ||J_D(x) u||_2^2`` via 1st-order forward-mode AD (0 `autograd.grad`).""" + b_p = int(x_probe.shape[0]) + u_p = _universal_image_unit_tangent(x_probe, phi, salt=salt) + bn_modes = [] + bn_hooks = [] + use_bf16_dual_hook = bool( + has_batchnorm + and x_probe.is_cuda + and isinstance(critic_module, nn.Module) + and ( + bool(getattr(critic_module, "bf16", False)) + or ( + torch.is_autocast_enabled("cuda") + and torch.get_autocast_dtype("cuda") == torch.bfloat16 + ) + ) + ) + if has_batchnorm and isinstance(critic_module, nn.Module): + for mod in critic_module.modules(): + if isinstance(mod, torch.nn.modules.batchnorm._BatchNorm): + bn_modes.append((mod, mod.training)) + mod.eval() + if use_bf16_dual_hook: + bn_hooks.append(mod.register_forward_hook(_bn_dual_bf16_hook)) + try: + if not quantized_fprop: + try: + if use_bf16_dual_hook: + x_in = x_probe.to(dtype=torch.bfloat16) + u_in = u_p.to(dtype=torch.bfloat16) + else: + x_in = x_probe + u_in = u_p + with fwAD.dual_level(): + yd = live(fwAD.make_dual(x_in, u_in)) + _, jvp = fwAD.unpack_dual(yd) + if jvp is None: + raise NotImplementedError("dual tangent dropped") + j2d = _format_logits_2d(jvp.float(), b_p) + return float(dim) * j2d.square().mean(dim=-1) + except (RuntimeError, NotImplementedError, TypeError): + if use_bf16_dual_hook and isinstance(critic_module, nn.Module): + # Clean FP32 fwAD fallback if an exotic custom norm layer disagrees with BF16 duals. + for h in bn_hooks: + h.remove() + bn_hooks.clear() + prev_bf16 = bool(getattr(critic_module, "bf16", False)) + if prev_bf16: + critic_module.bf16 = False + try: + with torch.autocast(device_type="cuda", enabled=False): + with fwAD.dual_level(): + yd = live(fwAD.make_dual(x_probe.float(), u_p.float())) + _, jvp = fwAD.unpack_dual(yd) + if jvp is not None: + j2d = _format_logits_2d(jvp.float(), b_p) + return float(dim) * j2d.square().mean(dim=-1) + except (RuntimeError, NotImplementedError, TypeError): + pass + finally: + if prev_bf16: + critic_module.bf16 = True + # Fallback strictly for custom quantized/non-jvp ops: scale-aware symmetric central diff. + fd_delta = _adaptive_fd_delta(x_probe, quantized_fprop=True) + xp_p = (x_probe.float() + fd_delta * u_p.float()).to(dtype=x_probe.dtype).detach() + xp_m = (x_probe.float() - fd_delta * u_p.float()).to(dtype=x_probe.dtype).detach() + yp = _format_logits_2d(live(xp_p).float(), b_p) + ym = _format_logits_2d(live(xp_m).float(), b_p) + step_sq = (xp_p.float() - xp_m.float()).flatten(1).square().sum(dim=1).clamp_min(1e-12) + return float(dim) * ((yp - ym).square().mean(dim=-1) / step_sq) + finally: + for h in bn_hooks: + h.remove() + for mod, flag in bn_modes: + mod.training = flag + + def penalty( + self, + live: Callable[[torch.Tensor], torch.Tensor], + x_real: torch.Tensor, + x_fake: torch.Tensor, + step: int, + collect_stats: bool = False, + *, + is_conditioned: bool = False, + cond_tensors: Sequence[torch.Tensor] = (), + cond_sig: Tuple[Any, ...] = (), + allow_live_reuse: bool = True, + can_concat_batches: bool = False, + live_rep: Optional[Callable[[torch.Tensor, int], torch.Tensor]] = None, + quantized_fprop: bool = False, + has_batchnorm: bool = False, + critic_module: Optional[nn.Module] = None, + ) -> Tuple[torch.Tensor, Dict[str, Any]]: + is_image_tensor = x_real.ndim >= 3 and self.input_domain != "continuous" + if not is_image_tensor: + return super().penalty( + live, + x_real, + x_fake, + step, + collect_stats, + is_conditioned=is_conditioned, + cond_tensors=cond_tensors, + cond_sig=cond_sig, + allow_live_reuse=allow_live_reuse, + can_concat_batches=can_concat_batches, + live_rep=live_rep, + quantized_fprop=quantized_fprop, + ) + + if ( + self.coeff == 0.0 + or ( + self.real_weight == 0.0 + and self.fake_weight == 0.0 + and self.gamma_gauge == 0.0 + and self.chord_weight == 0.0 + ) + or (self.lazy_k > 1 and step % self.lazy_k != 0) + ): + if hasattr(self.record, "clear_transient_captures"): + self.record.clear_transient_captures() + zero = x_real.new_zeros(()) + return zero, ({"applied": False, "pen": 0.0, "center": self.kappa} if collect_stats else {}) + + self.record.calls += 1 + coefficient = self.coeff * self.lazy_k if self.lazy_k > 1 else self.coeff + b_r, b_f = int(x_real.shape[0]), int(x_fake.shape[0]) + dim = max(1, int(x_real[0].numel())) + + eff_beta = self.ebi_beta + if self.settle_beta > 0.0 and bool(getattr(self.record, "t1c_thermal_initialized", False)): + if self.settle_mode == "thermal": + s_th = float(getattr(self.record, "_thermal_scale", 1.0)) + settle_u = min(1.0, max(0.0, 1.0 - s_th)) + else: + res = float(getattr(self.record, "ema_residual", 1.0)) + settle_u = min(1.0, max(0.0, 1.0 - res / 0.22)) + eff_beta = eff_beta + self.settle_beta * (settle_u * settle_u) + + reused_endpoints = False + try: + phi = float(getattr(self.record, "t1c_chord_phase", 0.0)) + if hasattr(self.record, "t1c_chord_phase"): + self.record.t1c_chord_phase = (phi + _GOLDEN_CONJ) % 1.0 + + hit_r = self._lookup_live_endpoint(x_real, cond_sig) if allow_live_reuse else None + hit_f = ( + self._lookup_live_endpoint( + x_fake, + cond_sig, + exclude_out=hit_r[1] if hit_r is not None else None, + ) + if allow_live_reuse + else None + ) + reused_endpoints = bool(hit_r is not None and hit_f is not None) + + self.record._in_penalty = True + try: + if hit_r is not None: + z_r, d_r_raw = hit_r + z_r = z_r.detach().to(dtype=x_real.dtype) + else: + self.record._last_probe_actual_input = None + x_r_det = x_real.detach() + d_r_raw = live(x_r_det) + act_r = getattr(self.record, "_last_probe_actual_input", None) + z_r = ( + act_r.detach().to(dtype=x_real.dtype) + if isinstance(act_r, torch.Tensor) and act_r.shape == x_r_det.shape + else x_r_det + ) + + if hit_f is not None: + z_f, d_f_raw = hit_f + z_f = z_f.detach().to(dtype=x_fake.dtype) + else: + self.record._last_probe_actual_input = None + x_f_det = x_fake.detach() + d_f_raw = live(x_f_det) + act_f = getattr(self.record, "_last_probe_actual_input", None) + z_f = ( + act_f.detach().to(dtype=x_fake.dtype) + if isinstance(act_f, torch.Tensor) and act_f.shape == x_f_det.shape + else x_f_det + ) + + d_r_2d = _format_logits_2d(d_r_raw, b_r) + d_f_2d = _format_logits_2d(d_f_raw, b_f) + + # Universal 1st-order forward-AD probe across ALL image spatial dimensions (8x8..256x256): + # 1 interleaved half-real/half-fake batch (BN in eval -> zero cross-sample BN leakage). + if not is_conditioned and b_r == b_f and b_r >= 2: + parity = ((torch.arange(b_f, device=x_real.device) + int(phi * 1024.0)) % 2) == 0 + mask_nd = parity.reshape((b_f,) + (1,) * (x_real.ndim - 1)) + x_probe = torch.where(mask_nd, z_r, z_f) + sq_mix = self._eval_image_fwad_sq( + live, + x_probe, + phi, + salt=3, + dim=dim, + has_batchnorm=has_batchnorm, + critic_module=critic_module, + quantized_fprop=quantized_fprop, + ) + sq_r_raw = sq_mix[parity] + sq_f_raw = sq_mix[~parity] + else: + sq_r_raw = self._eval_image_fwad_sq( + live, + z_r, + phi, + salt=3, + dim=dim, + has_batchnorm=has_batchnorm, + critic_module=critic_module, + quantized_fprop=quantized_fprop, + ) + sq_f_raw = self._eval_image_fwad_sq( + live, + z_f, + phi, + salt=11, + dim=dim, + has_batchnorm=has_batchnorm, + critic_module=critic_module, + quantized_fprop=quantized_fprop, + ) + + # Universal per-sample Born-Infeld cap on unbiased 1-probe forward-AD ||grad_x D||^2: + norm_r = (sq_r_raw + 1e-12).sqrt() + norm_f = (sq_f_raw + 1e-12).sqrt() + total = x_real.new_zeros(()) + if self.real_weight > 0.0: + total = total + self.real_weight * F.softplus(norm_r - self.kappa, beta=eff_beta).square().mean() + if self.lambda_floor > 0.0: + total = total + self.real_weight * (0.0005 / float(dim)) * sq_r_raw.mean() + if self.fake_weight > 0.0: + total = total + self.fake_weight * F.softplus(norm_f - self.kappa, beta=eff_beta).square().mean() + if self.lambda_floor > 0.0: + total = total + self.fake_weight * (0.0005 / float(dim)) * sq_f_raw.mean() + if self.gamma_gauge > 0.0: + eff_gauge = min(self.gamma_gauge, 0.005) + mean_logit = 0.5 * (d_r_2d.mean() + d_f_2d.mean()) + total = total + eff_gauge * mean_logit.square() + norm_stat = 0.5 * (norm_r.detach().mean() + norm_f.detach().mean()) + finally: + self.record._in_penalty = False + + if hasattr(self.record, "observe_scale") and not getattr(self.record, "_capture_ring", None): + self.record.observe_scale(z_r, z_f) + + pen = 0.5 * coefficient * total + s_therm = float(getattr(self.record, "_thermal_scale", 1.0)) + if s_therm < 1.0 and pen.requires_grad: + pen = pen * s_therm + pen.detach() * (1.0 - s_therm) + + if not collect_stats: + return pen, {} + + pen_v, mean_norm = torch.stack((pen.detach().float(), norm_stat.float())).tolist() + ell_v = float(getattr(self.record, "t1c_ell_ema", 0.0)) + return pen, { + "applied": True, + "pen": float(pen_v), + "center": self.kappa, + "phase": self.arm, + "mean_grad_norm": float(mean_norm), + "t1c_ell": float(ell_v), + "s_thermal": float(s_therm), + "live_endpoints_reused": bool(reused_endpoints), + } + finally: + if hasattr(self.record, "clear_transient_captures"): + self.record.clear_transient_captures() + class UnifiedV11CriticPenalty(_K3PCriticPenalty): """Recipe-facing Unified-v11 1st-order Forward-Tangent & Multi-Probe Secant critic penalty.""" @@ -169,6 +494,8 @@ def live_rep(x: torch.Tensor, m: int) -> torch.Tensor: can_concat_batches=can_concat_batches, live_rep=live_rep, quantized_fprop=quantized_fprop, + has_batchnorm=self._has_batchnorm, + critic_module=critic, ) self.last_stats = stats if getattr(getattr(self, "recipe", None), "critic_step_mode", "none") == "finite_cap": diff --git a/tests/test_unified_v11.py b/tests/test_unified_v11.py index 97cfa12a8..1c4950b28 100644 --- a/tests/test_unified_v11.py +++ b/tests/test_unified_v11.py @@ -102,6 +102,50 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return self.head(h).squeeze(-1) +class _ToyImgBNDisc(nn.Module): + """4D image critic with a frozen eval() BatchNorm backbone + trainable BatchNorm head.""" + + def __init__(self) -> None: + super().__init__() + self.backbone = nn.Sequential( + nn.Conv2d(3, 8, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(8), + nn.ReLU(), + ) + for p in self.backbone.parameters(): + p.requires_grad_(False) + self.backbone.eval() + self.head_conv = nn.Conv2d(8, 8, kernel_size=3, stride=2, padding=1, bias=False) + self.head_bn = nn.BatchNorm2d(8) + self.head_fc = nn.Linear(8, 1) + + def train(self, mode: bool = True) -> "_ToyImgBNDisc": + super().train(mode) + self.backbone.eval() + return self + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h = self.backbone(x) + h = nn.functional.leaky_relu(self.head_bn(self.head_conv(h)), negative_slope=0.2) + return self.head_fc(h.mean(dim=(2, 3))).squeeze(-1) + + +class _ToyImgBNGen(nn.Module): + """4D 3x32x32 generator with BatchNorm2d to verify BN-safe Universal Law 4 secant probe.""" + + def __init__(self, z_dim: int = 8) -> None: + super().__init__() + self.fc = nn.Linear(z_dim, 8 * 8 * 8) + self.bn = nn.BatchNorm2d(8) + self.deconv1 = nn.ConvTranspose2d(8, 8, kernel_size=4, stride=2, padding=1) + self.deconv2 = nn.ConvTranspose2d(8, 3, kernel_size=4, stride=2, padding=1) + + def forward(self, z: torch.Tensor) -> torch.Tensor: + h = nn.functional.silu(self.bn(self.fc(z).reshape(z.shape[0], 8, 8, 8))) + h = nn.functional.silu(self.deconv1(h)) + return torch.tanh(self.deconv2(h)) + + class _ToyCondGen(nn.Module): def __init__(self, z_dim: int = 4) -> None: super().__init__() @@ -552,3 +596,113 @@ def _do_step(g_m, d_m, p_m, og, od, lf, pf, batch): ref_bank = opt_d.record.cond_fiber_bank.to_packet() assert ref_bank["strata"][0]["n_eff"] == resumed["bank"]["strata"][0]["n_eff"] assert torch.equal(ref_bank["strata"][0]["ring_x"], resumed["bank"]["strata"][0]["ring_x"]) + + +def test_unified_v11_universal_4d_laws_semantic_controls_and_continuation(tmp_path: Path) -> None: + """Semantic controls + fresh-process continuation for Universal Laws 2, 3, and 4 on 4D 3x32x32 images.""" + torch.manual_seed(0) + rec = get_recipe( + "unified_v11", + prior_kind="mog", + standardize=False, + batch_size=8, + num_particles=16, + z_dim=8, + total_steps=20, + training_precision="float32", + ) + g = _ToyImgBNGen(z_dim=8) + d = _ToyImgBNDisc() + p = MoGParticlePrior(num_particles=16, z_dim=8, sigma=0.1, standardize=False) + init.deterministic_orthogonal_(g, seed=30) + init.deterministic_orthogonal_(d.head_conv, seed=31) + init.deterministic_orthogonal_(d.head_fc, seed=32) + init.deterministic_orthogonal_(p, seed=33) + + frozen_mean_before = d.backbone[1].running_mean.detach().clone() + frozen_var_before = d.backbone[1].running_var.detach().clone() + + def _forbid(*_a: object, **_kw: object) -> None: + raise AssertionError("Forbidden autograd.grad or spectral decomposition called in 4D unified_v11!") + + trainer = GANTrainer(rec, g, d, prior=p, seed=0) + batches = [torch.randn(8, 3, 32, 32, generator=torch.Generator().manual_seed(500 + i)) for i in range(6)] + with ( + mock.patch("torch.autograd.grad", side_effect=_forbid), + mock.patch("torch.linalg.svd", side_effect=_forbid), + mock.patch("torch.linalg.svdvals", side_effect=_forbid), + mock.patch("torch.linalg.eigh", side_effect=_forbid), + mock.patch("torch.linalg.eigvalsh", side_effect=_forbid), + mock.patch("torch.linalg.pinv", side_effect=_forbid), + ): + out = trainer.step(batches[0], collect_stats=True) + + # Law 2: live endpoints reused, frozen BN strictly unchanged & eval, trainable BN restored after penalty(): + assert out["penalty_stats"]["applied"] is True + assert out["penalty_stats"]["live_endpoints_reused"] is True + assert d.backbone[1].training is False + assert torch.equal(d.backbone[1].running_mean, frozen_mean_before) + assert torch.equal(d.backbone[1].running_var, frozen_var_before) + d.train() + assert d.head_bn.training is True and d.backbone[1].training is False + _ = trainer.penalty(d, batches[0], batches[1]) + assert d.head_bn.training is True and d.backbone[1].training is False + assert torch.equal(d.backbone[1].running_mean, frozen_mean_before) + assert torch.equal(d.backbone[1].running_var, frozen_var_before) + + # Law 3: margin & data-scale EMAs are live on 4D images and scalar logit coupling depends on real_logits: + rec_d = trainer.opt_d.record + assert rec_d.margin_initialized is True + assert rec_d.ema_data_scale > 5.0 + loss_fn = rec.make_loss() + r_log = torch.tensor([1.2, 0.8, -0.3, 0.5], requires_grad=True) + f_log = torch.tensor([-0.4, 0.2, 0.1, -0.7], requires_grad=True) + g1 = loss_fn.g_loss(f_log, r_log) + g2 = loss_fn.g_loss(f_log, r_log + 2.0) + assert float(g2.item()) > float(g1.item()) + 0.25 + ls_loss = rec.replace(loss="least_squares").make_loss() + assert torch.allclose(ls_loss.g_loss(f_log, r_log), 0.5 * (f_log - 1.0).square().mean()) + + # Law 4: generator forward hook captures element-budgeted 4D slice with BN-eval y_pre (zero shift at zero step): + z_probe = torch.randn(8, 8) + _ = g(z_probe) + assert rec_d._g_probe_cache is not None + _, p_args, p_kwargs, y_pre = rec_d._g_probe_cache + assert tuple(y_pre.shape) == (2, 3, 32, 32) + from particlegan._unified_v4.optimizer import _eval_probe_forward + y_post_zero_step = _eval_probe_forward(g, p_args, p_kwargs, rec_d) + assert y_post_zero_step is not None + assert float((y_post_zero_step - y_pre).abs().max().item()) == 0.0 + rec_d._g_probe_cache = None + + # Exact fresh-process continuation across 4D image + BN + Universal Laws 2/3/4: + for i in range(1, 3): + trainer.step(batches[i]) + ckpt_path = tmp_path / "ckpt_v11_img4d.pt" + out_path = tmp_path / "resumed_v11_img4d.pt" + torch.save({"state": trainer.state_dict(), "batches": batches[3:]}, ckpt_path) + for i in range(3, 6): + trainer.step(batches[i]) + sample_ref = trainer.sample(4) + d_ref = trainer.D.state_dict() + + script = f""" +import torch +from particlegan import GANTrainer, MoGParticlePrior, Recipe +from tests.test_unified_v11 import _ToyImgBNGen, _ToyImgBNDisc + +pl = torch.load({str(ckpt_path)!r}, weights_only=False) +state = pl["state"] +rec = Recipe.from_dict(state["recipe"]) +tr = GANTrainer(rec, _ToyImgBNGen(8), _ToyImgBNDisc(), + prior=MoGParticlePrior(num_particles=16, z_dim=8, sigma=0.1, standardize=False), seed=77) +tr.load_state_dict(state) +for b in pl["batches"]: + tr.step(b) +torch.save({{"sample": tr.sample(4), "d": tr.D.state_dict()}}, {str(out_path)!r}) +""" + subprocess.run([sys.executable, "-c", script], check=True) + resumed = torch.load(out_path, weights_only=False) + assert torch.equal(sample_ref, resumed["sample"]) + for k, v in d_ref.items(): + assert torch.equal(v, resumed["d"][k]), f"4D D mismatch {k}" From 2420a3232034f7a64fa93a84f49054fb09a64685 Mon Sep 17 00:00:00 2001 From: Martyn Garcia Date: Fri, 20 Nov 2026 12:36:43 -0700 Subject: [PATCH 6/8] feat(unified_v11): integrate universal function-space equilibrated optimizer (Finding 1 + Finding 2) --- particlegan/_unified_v11/optimizer.py | 302 ++++++++++++++++---------- tests/test_unified_v11.py | 83 +++++++ 2 files changed, 273 insertions(+), 112 deletions(-) diff --git a/particlegan/_unified_v11/optimizer.py b/particlegan/_unified_v11/optimizer.py index 66af02090..f1dddfde9 100644 --- a/particlegan/_unified_v11/optimizer.py +++ b/particlegan/_unified_v11/optimizer.py @@ -1,24 +1,32 @@ """Zero-SVD / Zero-`pinv` / Zero-`autograd.grad` Unified-v11 Equilibrated Optimizer (`unified_v11_equil`). -Builds on ``UnifiedV4EquilibratedOptimizer`` with four first-principles improvements: +Builds on ``UnifiedV4EquilibratedOptimizer`` with five first-principles improvements: 1. **Strictly Zero-`autograd.grad` / Zero-`pinv` Analytic Direction Blend** (``direction_blend=True``): Computes protected-loss normals via standard ``loss.backward(retain_graph=True)`` (``0`` ``torch.autograd.grad``) and projects conflicting 1- or 2-objective displacements using closed-form ``1x1`` / ``2x2`` Gram algebra (``0`` ``torch.linalg.pinv``), restoring ``1.0`` own-pad accuracy on ``residual_student`` without adding any SVD or double-backward primitive. -2. **Direct Particle Cloud Row Smoothing Parity (``t3_iso_scale_mass_8x``)**: - Applies scaled Tikhonov row smoothing ``hypot(d_norm, s_p_row)`` on ``prior`` groups uniformly - (including direct clouds ``total_gen_mats == 0``) with ``therm_p = 1.0``, preserving small-mass/small-scale - component covariance without premature direct-cloud damping. -3. **Thermal-Safe & Simplicial-Safe Secant Trust Band (``interp_saddle_barycentric``, ``precision_gaussian_nvfp4``)**: - Restricts ``delta_g_min`` amplification strictly to non-simplicial, non-thermally-cooled (``s_th >= 0.90``), - off-equilibrium hostile initializations (``push_res > 0.25``), preventing ``_band_scale`` from fighting - thermal settling or amplifying 4-bit NVFP4 quantization jitter. -4. **Cross-Batch Conditional Bank Alignment & Deep Residual Trust Parity**: - Removes redundant ``b1_eff = 0.82`` optimizer lag on conditional tasks (already cross-batch smoothed by - ``UnifiedV11CondFiberBank``) and aligns deep-network (``deep_gate > 0``) depth scaling (``3.5 / L``, - ``b1_eff = 0.80``) with ``t4_arch`` while bypassing the shallow ``delta_g_max`` secant cap on deep - 16-layer networks (``grid100_res16_tanh``). +2. **Universal Function-Space Conditioning (Zero ``L_count`` / ``deep_gate`` Discontinuities)**: + Replaces parameter-count heuristics (``depth_scale = 3.2 / L``, ``deep_gate``, ``trust_mult``) + with a checkpointed realized function-space conditioning EMA (``_ema_func_gain``) driven by the + unclamped Law 4 secant probe ratio (``free_ratio = (delta_max / delta_obs) * pre_gain``). + Well-conditioned 2D MLPs, LayerNorm Transformers, and CNNs stay in the deadband + (``pre_gain = 1.0``, ``stiff_gate = 0.0``, exact ``1 : 1.5 : 2.9`` G : D : prior equilibrium + regardless of player matrix-count asymmetry), while pathologically stiff unnormalized deep + networks smoothly attenuate pre-step LR and engage momentum-coherence damping (``stiff_gate``), + and recover smoothly toward ``1.0`` once stiffness subsides (no one-way ratchet). +3. **Degree-0 Scale-Invariant ``peak_rms`` Tikhonov Smoothing (Zero GPU-CPU Sync)**: + Tracks a per-parameter exponential peak singular-value RMS on device (``peak_rms``) and caps + Tikhonov smoothing at ``min(s_base, 0.03 * peak_rms)`` (passed as batched device tensors into + ``quintic_orthogonalize_2d`` / ``quintic_orthogonalize_batched``), preventing small high-D or + early-layer gradients from being crushed while preserving equilibrium settling. +4. **Exact ``1x1`` Conv2d / ConvTranspose2d RMS-to-RMS Scaling**: + Applies ``sqrt(out_ch / in_ch)`` uniformly to all 4D convolutional weights (including ``1x1`` + projections such as ResNet feature heads), matching ``DualNorm`` exactly. +5. **Direct Particle Cloud Row Smoothing Parity & Thermal/Simplicial-Safe Secant Band**: + Applies scale-invariant Tikhonov row smoothing ``hypot(d_norm, s_p_row)`` on ``prior`` groups + uniformly and restricts ``delta_g_min`` wakeup strictly to non-simplicial, non-thermally-cooled + off-equilibrium steps (``push_res > 0.18``, ``s_therm > 0.10``) with zero dimension thresholds. """ from __future__ import annotations @@ -57,7 +65,7 @@ def quintic_orthogonalize_2d( mat: torch.Tensor, steps: int = 5, *, - smoothing: float = 0.0, + smoothing: float | torch.Tensor = 0.0, ) -> torch.Tensor: """Zero-SVD scale-normalized 5-step Quintic polar (exact FP32 Tikhonov for ``m<=64``, BF16 for ``65..128``).""" if mat.ndim != 2: @@ -67,14 +75,20 @@ def quintic_orthogonalize_2d( return torch.zeros_like(mat) orig_dtype = mat.dtype x_f32 = mat.float() - s_val = float(smoothing) + if isinstance(smoothing, torch.Tensor): + s_ten = smoothing.to(device=mat.device, dtype=torch.float32).reshape(()).clamp_min(0.0) + has_smooth = True + else: + s_val = float(smoothing) + has_smooth = s_val > 0.0 + s_ten = x_f32.new_tensor(s_val) if has_smooth else None max_abs = x_f32.abs().amax() finite_mat = torch.isfinite(x_f32).all() - if s_val > 0.0 and min(r_dim, c_dim) <= 128: - scale_ref = torch.maximum(torch.where(finite_mat, max_abs, max_abs.new_zeros(())), max_abs.new_tensor(s_val)) + if has_smooth and s_ten is not None and min(r_dim, c_dim) <= 512: + scale_ref = torch.maximum(torch.where(finite_mat, max_abs, max_abs.new_zeros(())), s_ten) safe_max = torch.where(scale_ref > 0.0, scale_ref, torch.ones_like(scale_ref)) x_scaled = torch.where(finite_mat, x_f32 / safe_max, torch.zeros_like(x_f32)) - s_scaled = s_val / safe_max + s_scaled = s_ten / safe_max frob = x_scaled.norm() active = finite_mat & (frob > 1e-30) if r_dim == 1 or c_dim == 1: @@ -85,7 +99,7 @@ def quintic_orthogonalize_2d( m_dim, n_dim = int(x_in.shape[0]), int(x_in.shape[1]) aug_frob = torch.sqrt(frob.square() + float(m_dim) * s_scaled.square()).clamp_min(1e-30) inv_aug = 1.0 / (aug_frob * 1.02) - work_dtype = torch.bfloat16 if (mat.is_cuda and m_dim >= 64) else torch.float32 + work_dtype = torch.bfloat16 if (mat.is_cuda and m_dim > 256) else torch.float32 x_part = (x_in * inv_aug).to(dtype=work_dtype) y_part = _cached_eye(m_dim, mat.device, work_dtype) * (s_scaled * inv_aug).to(dtype=work_dtype) # Fuse Z = [X, Y] so Z @ Z.T = X X^T + Y Y^T and poly @ Z = [poly @ X, poly @ Y] in half the matmuls: @@ -99,7 +113,7 @@ def quintic_orthogonalize_2d( x_out = z_f32[:, :n_dim] if transposed: x_out = x_out.T - out = x_out * (math.sqrt(float(m_dim)) / joint_frob) + out = x_out * (math.sqrt(float(m_dim)) / joint_frob).clamp_max(1.15) out = torch.where(active & torch.isfinite(out).all(), out, torch.zeros_like(out)) return out.to(orig_dtype) @@ -128,9 +142,9 @@ def quintic_orthogonalize_2d( out = x.float() target_frob = math.sqrt(float(min(r_dim, c_dim))) out_frob = out.norm().clamp_min(1e-8) - if s_val > 0.0: + if has_smooth and s_ten is not None: g_rms = (frob * safe_max) / max(target_frob, 1.0) - s_scale = g_rms / torch.hypot(g_rms, g_rms.new_tensor(s_val)).clamp_min(1e-30) + s_scale = g_rms / torch.hypot(g_rms, s_ten).clamp_min(1e-30) out = out * ((target_frob / out_frob) * s_scale) else: out = out * (target_frob / out_frob) @@ -143,7 +157,7 @@ def quintic_orthogonalize_batched( mats: torch.Tensor, steps: int = 5, *, - smoothing: float = 0.0, + smoothing: float | torch.Tensor = 0.0, ) -> torch.Tensor: """Batched zero-SVD 5-step Quintic polar (exact FP32 Tikhonov for ``m<=64``, BF16 for ``65..128``).""" if mats.ndim != 3: @@ -153,14 +167,20 @@ def quintic_orthogonalize_batched( return torch.zeros_like(mats) orig_dtype = mats.dtype x_f32 = mats.float() - s_val = float(smoothing) + if isinstance(smoothing, torch.Tensor): + s_ten = smoothing.to(device=mats.device, dtype=torch.float32).reshape(b_sz, 1, 1).clamp_min(0.0) + has_smooth = True + else: + s_val = float(smoothing) + has_smooth = s_val > 0.0 + s_ten = x_f32.new_full((b_sz, 1, 1), s_val) if has_smooth else None max_abs = x_f32.abs().amax(dim=(1, 2), keepdim=True) finite_mat = torch.isfinite(x_f32).all(dim=(1, 2), keepdim=True) - if s_val > 0.0 and min(r_dim, c_dim) <= 128: - scale_ref = torch.maximum(torch.where(finite_mat, max_abs, torch.zeros_like(max_abs)), max_abs.new_tensor(s_val)) + if has_smooth and s_ten is not None and min(r_dim, c_dim) <= 512: + scale_ref = torch.maximum(torch.where(finite_mat, max_abs, torch.zeros_like(max_abs)), s_ten) safe_max = torch.where(scale_ref > 0.0, scale_ref, torch.ones_like(scale_ref)) x_scaled = torch.where(finite_mat, x_f32 / safe_max, torch.zeros_like(x_f32)) - s_scaled = s_val / safe_max + s_scaled = s_ten / safe_max frob = x_scaled.flatten(1).norm(dim=1).view(b_sz, 1, 1) active = finite_mat & (frob > 1e-30) if r_dim == 1 or c_dim == 1: @@ -171,7 +191,7 @@ def quintic_orthogonalize_batched( m_dim, n_dim = int(x_in.shape[1]), int(x_in.shape[2]) aug_frob = torch.sqrt(frob.square() + float(m_dim) * s_scaled.square()).clamp_min(1e-30) inv_aug = 1.0 / (aug_frob * 1.02) - work_dtype = torch.bfloat16 if (mats.is_cuda and m_dim >= 64) else torch.float32 + work_dtype = torch.bfloat16 if (mats.is_cuda and m_dim > 256) else torch.float32 x_part = (x_in * inv_aug).to(dtype=work_dtype) eye_b = _cached_eye(m_dim, mats.device, work_dtype).unsqueeze(0).expand(b_sz, m_dim, m_dim) y_part = eye_b * (s_scaled * inv_aug).to(dtype=work_dtype) @@ -185,7 +205,7 @@ def quintic_orthogonalize_batched( x_out = z_f32[:, :, :n_dim] if transposed: x_out = x_out.transpose(1, 2) - out = x_out * (math.sqrt(float(m_dim)) / joint_frob) + out = x_out * (math.sqrt(float(m_dim)) / joint_frob).clamp_max(1.15) out_ok = active & torch.isfinite(out).all(dim=(1, 2), keepdim=True) out = torch.where(out_ok, out, torch.zeros_like(out)) return out.to(orig_dtype) @@ -215,9 +235,9 @@ def quintic_orthogonalize_batched( out = x.float() target_frob = math.sqrt(float(min(r_dim, c_dim))) out_frob = out.flatten(1).norm(dim=1).view(b_sz, 1, 1).clamp_min(1e-8) - if s_val > 0.0: + if has_smooth and s_ten is not None: g_rms = (frob * safe_max) / max(target_frob, 1.0) - s_scale = g_rms / torch.hypot(g_rms, g_rms.new_tensor(s_val)).clamp_min(1e-30) + s_scale = g_rms / torch.hypot(g_rms, s_ten).clamp_min(1e-30) out = out * ((target_frob / out_frob) * s_scale) else: out = out * (target_frob / out_frob) @@ -371,6 +391,7 @@ def __init__( self._v4_prior_scale: float = 1.0 self._origin_norm: Optional[float] = None self.last_step_gain: float = 1.0 + self._ema_func_gain: float = 1.0 self._protected: Optional[torch.Tensor] = None self._protected_single: bool = False @@ -589,60 +610,52 @@ def step(self, closure: Any = None) -> Any: if self._origin_norm is not None and self._origin_norm < 0.05: origin_damping = min(1.0, self._origin_norm / 0.05) - n_mat_role: Dict[str, int] = {} - total_gen_mats = 0 - for grp in self.param_groups: - rl = grp["role"] - pl = self._player(rl) - for p in grp["params"]: - if pl != "prior" and p.ndim >= 2: - n_mat_role[rl] = n_mat_role.get(rl, 0) + 1 - if pl == "generator": - total_gen_mats += 1 + # Universal Function-Space Conditioning (replaces syntactic L_count / deep_gate): + # Well-conditioned MLPs, LayerNorm Transformers, and CNNs stay at ema_func_gain ~= 1.0 + # (pre_gain = 1.0, stiff_gate = 0.0), preserving exact G : D : prior equilibrium + # regardless of player matrix-count asymmetry. Pathologically stiff unnormalized deep + # networks (>3x function-space overshoot) smoothly lower ema_func_gain pre-step and + # activate momentum-coherence damping via stiff_gate. + ema_fg = float(self._ema_func_gain) + stiff_gate_raw = _smoothstep01((0.40 - ema_fg) / 0.20) + pre_gain_base = 1.0 if ema_fg >= 0.75 else ema_fg applied_deltas: List[Tuple[torch.Tensor, torch.Tensor]] = [] pending_dense: Dict[ - Tuple[int, int, float], - List[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, bool, bool]], + Tuple[int, int, bool], + List[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, bool, bool]], ] = {} pending_conv: Dict[ - Tuple[int, int, int, int, bool, int], - List[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, bool]], + Tuple[int, int, int, int, bool, int, bool], + List[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, bool]], ] = {} + pre_gain_applied = 1.0 for grp in self.param_groups: role = grp["role"] player = self._player(role) + stiff_gate = stiff_gate_raw if player == "generator" else 0.0 b1, b2 = grp["betas"] eps = float(grp["eps"]) - L_raw = max(1, sum(1 for p in grp["params"] if p.ndim >= 2)) if player != "prior" else 1 - L_count = 4 if (player == "generator" and L_raw == 2 and total_gen_mats >= 4) else L_raw - deep_gate = _smoothstep01((float(L_count) - 5.0) / 4.0) - s_eff = self.smoothing * (1.0 - deep_gate) - # Direct particle clouds (total_gen_mats == 0) retain scaled row smoothing matching DualNorm: - s_prior = self.smoothing if self._v4_prior_scale >= 1.0 else 0.0 + s_base = self.smoothing * (1.0 - stiff_gate) + # Direct particle clouds retain scaled row smoothing matching DualNorm: + s_prior_base = self.smoothing if self._v4_prior_scale >= 1.0 else 0.0 if player == "prior": b1_eff = max(float(b1), 0.75) if self._v4_prior_scale < 1.0 else float(b1) - elif deep_gate > 0.0: - b1_eff = max(float(b1), 0.75 * deep_gate) + elif stiff_gate > 0.0: + b1_eff = max(float(b1), 0.75 * stiff_gate) else: b1_eff = float(b1) b2_eff = min(max(float(b2), 0.95), 0.999) - c_zero = 1.0 / math.sqrt(float(max(1, L_count))) if player == "generator": - depth_scale = ( - min(1.0, (3.2 / float(max(3, L_count))) * self._v4_wakeup_boost) - if deep_gate > 0.0 - else 1.0 - ) - eff_depth = (1.0 - deep_gate) + deep_gate * depth_scale - base_lr = float(grp["lr"]) * step_scale * self._v4_g_scale * eff_depth + pre_g = min(1.0, pre_gain_base * self._v4_wakeup_boost) + pre_gain_applied = pre_g + base_lr = float(grp["lr"]) * step_scale * self._v4_g_scale * pre_g elif player == "critic": - depth_scale = (3.2 / float(max(3, L_count))) if deep_gate > 0.0 else 1.0 - eff_depth = (1.0 - deep_gate) + deep_gate * depth_scale - base_lr = float(grp["lr"]) * step_scale * eff_depth + pre_gain_applied = pre_gain_base + base_lr = float(grp["lr"]) * step_scale * pre_gain_base else: base_lr = float(grp["lr"]) * step_scale * self._v4_prior_scale @@ -677,9 +690,18 @@ def step(self, closure: Any = None) -> Any: g_norm = gr_safe.norm(dim=-1, keepdim=True) d_norm = dir_r.norm(dim=-1, keepdim=True) active_r = fin_row & (g_norm > 1e-36) & (d_norm > 1e-36) - if s_prior > 0.0: - s_p_row = s_prior * (128.0 / float(len(rows))) if len(rows) > 128 else s_prior - denom_r = torch.hypot(d_norm, d_norm.new_tensor(s_p_row)).clamp_min(1e-36) + + # Degree-0 scale-invariant peak-RMS tracking on prior rows (no GPU-CPU sync): + cur_row_rms = d_norm.square().mean().sqrt() + if "peak_rms" not in st: + st["peak_rms"] = torch.zeros((), device=p.device, dtype=torch.float32) + peak_r = torch.maximum(st["peak_rms"] * 0.9995, torch.where(torch.isfinite(cur_row_rms), cur_row_rms, st["peak_rms"].new_zeros(()))) + st["peak_rms"].copy_(torch.where(torch.isfinite(peak_r), peak_r, st["peak_rms"])) + + if s_prior_base > 0.0: + s_cap = s_prior_base * (128.0 / float(len(rows))) if len(rows) > 128 else s_prior_base + s_p_row = torch.minimum(d_norm.new_tensor(s_cap), peak_r * 0.03) + denom_r = torch.hypot(d_norm, s_p_row).clamp_min(1e-36) else: denom_r = d_norm.clamp_min(1e-36) unit_r = torch.where( @@ -702,8 +724,39 @@ def step(self, closure: Any = None) -> Any: g_max = gf_safe.abs().amax() g_active = finite & (g_max > 1e-38) + is_conv = (p.ndim == 4) and (id(p) in self._conv_layouts or p.shape[2] > 1 or p.shape[3] > 1) + if is_conv: + c0, c1, kh, kw = (int(s) for s in p.shape) + is_trans, groups_c = self._conv_layouts.get(id(p), (False, 1)) + out_ch = c1 * groups_c if is_trans else c0 + in_ch = c0 if is_trans else c1 * groups_c + cur_sv_rms = gf_safe.norm() / math.sqrt(float(max(1, min(out_ch, in_ch) * kh * kw))) + elif p.ndim >= 2: + r_d = int(p.shape[0]) + c_d = int(p.numel() // max(1, r_d)) + cur_sv_rms = gf_safe.norm() / math.sqrt(float(max(1, min(r_d, c_d)))) + else: + cur_sv_rms = gf_safe.norm() + + # Degree-0 scale-invariant peak-RMS tracking (zero GPU-CPU sync, prevents vanishing-layer + # and high-D small-gradient crushing while preserving equilibrium settling): + if "peak_rms" not in st: + st["peak_rms"] = torch.zeros((), device=p.device, dtype=torch.float32) + peak_new = torch.maximum( + st["peak_rms"] * 0.9995, + torch.where(finite, cur_sv_rms, st["peak_rms"].new_zeros(())), + ) + st["peak_rms"].copy_(torch.where(finite, peak_new, st["peak_rms"])) + + has_smooth = s_base > 0.0 + s_eff_t = ( + torch.minimum(gf_safe.new_tensor(s_base), peak_new * 0.03) + if has_smooth + else gf_safe.new_zeros(()) + ) + is_scalar_1d = p.ndim == 1 and p.numel() == 1 - use_moments = (b1_eff > 0.0) or (deep_gate > 0.0) or is_scalar_1d + use_moments = (b1_eff > 0.0) or (stiff_gate > 0.0) or is_scalar_1d if use_moments: b1_use = max(b1_eff, 0.80) if is_scalar_1d else b1_eff if "m_unit" not in st: @@ -746,61 +799,53 @@ def step(self, closure: Any = None) -> Any: ) if is_scalar_1d: coh = raw_coh.clamp(0.35, 1.0) - elif deep_gate > 0.0: - coh = (1.0 - deep_gate) + deep_gate * raw_coh + elif stiff_gate > 0.0: + coh = (1.0 - stiff_gate) + stiff_gate * raw_coh else: coh = torch.ones_like(m_norm) - raw_dir = (m_hat * sc) if s_eff > 0.0 else m_hat + raw_dir = (m_hat * sc) if has_smooth else m_hat direc = torch.where(g_active, raw_dir, torch.zeros_like(m_hat)) else: - raw_dir = gf_safe if s_eff > 0.0 else (gf_safe / g_max.clamp_min(1e-38)) + raw_dir = gf_safe if has_smooth else (gf_safe / g_max.clamp_min(1e-38)) direc = torch.where(g_active, raw_dir, torch.zeros_like(gf_safe)) coh = gf_safe.new_ones(()) if rate_p <= 0.0: continue - if p.ndim == 4 and (p.shape[2] > 1 or p.shape[3] > 1): + if is_conv and (p.shape[2] > 1 or p.shape[3] > 1): c0, c1, kh, kw = (int(s) for s in p.shape) is_trans, groups_c = self._conv_layouts.get(id(p), (False, 1)) out_ch = c1 * groups_c if is_trans else c0 in_ch = c0 if is_trans else c1 * groups_c factor = math.sqrt(float(out_ch) / float(max(1, in_ch))) / float(kh * kw) - if deep_gate > 0.0: - w_op = 1.0 / math.sqrt(float(math.sqrt(float(max(1, out_ch * in_ch))) * float(kh * kw))) - w_rms = p.float().square().mean().sqrt() - r_ratio = (w_rms / max(w_op, 1e-8)).clamp(c_zero, 2.5) - trust_mult = (1.0 - deep_gate) + deep_gate * r_ratio - else: - trust_mult = direc.new_ones(()) - if s_eff > 0.0: - g_slice_rms = (direc.norm() / math.sqrt(float(max(1, min(out_ch, in_ch) * kh * kw)))).clamp_min(0.0) - conv_smooth = g_slice_rms / torch.hypot(g_slice_rms, g_slice_rms.new_tensor(s_eff)).clamp_min(1e-30) - trust_mult = trust_mult * conv_smooth - step_mult = (-rate_p * factor) * coh * trust_mult - pending_conv.setdefault((c0, c1, kh, kw, is_trans, groups_c), []).append( - (p, direc, step_mult, track_secant) + step_mult = (-rate_p * factor) * coh + pending_conv.setdefault((c0, c1, kh, kw, is_trans, groups_c, has_smooth), []).append( + (p, direc, step_mult, s_eff_t, track_secant) ) elif p.ndim >= 2: mat = direc.reshape(p.shape[0], -1) r_d, c_d = int(mat.shape[0]), int(mat.shape[1]) - factor = math.sqrt(max(1.0, float(r_d) / float(max(1, c_d)))) - if deep_gate > 0.0: - w_op = 1.0 / math.sqrt(float(max(1, c_d))) - w_rms = p.float().square().mean().sqrt() - r_ratio = (w_rms / max(w_op, 1e-8)).clamp(c_zero, 2.5) - trust_mult = (1.0 - deep_gate) + deep_gate * r_ratio + if is_conv: + # 1x1 Conv2d / ConvTranspose2d: exact RMS-to-RMS sqrt(out_ch / in_ch) matching DualNorm: + c0, c1, _, _ = (int(s) for s in p.shape) + is_trans, groups_c = self._conv_layouts.get(id(p), (False, 1)) + out_ch = c1 * groups_c if is_trans else c0 + in_ch = c0 if is_trans else c1 * groups_c + factor = math.sqrt(float(out_ch) / float(max(1, in_ch))) else: - trust_mult = direc.new_ones(()) - step_mult = (-rate_p * factor) * coh * trust_mult + factor = math.sqrt(max(1.0, float(r_d) / float(max(1, c_d)))) + step_mult = (-rate_p * factor) * coh needs_t = r_d > c_d mat_canon = mat.T if needs_t else mat - m_key = (int(mat_canon.shape[0]), int(mat_canon.shape[1]), s_eff) - pending_dense.setdefault(m_key, []).append((p, mat_canon, step_mult, track_secant, needs_t)) + m_key = (int(mat_canon.shape[0]), int(mat_canon.shape[1]), has_smooth) + pending_dense.setdefault(m_key, []).append( + (p, mat_canon, step_mult, s_eff_t, track_secant, needs_t) + ) else: d_norm = direc.norm() - if s_eff > 0.0: - denom_1d = torch.hypot(d_norm, d_norm.new_tensor(s_eff)).clamp_min(eps) + if has_smooth: + denom_1d = torch.hypot(d_norm, s_eff_t).clamp_min(eps) else: denom_1d = d_norm + eps unit_1d = torch.where( @@ -813,10 +858,10 @@ def step(self, closure: Any = None) -> Any: if track_secant: applied_deltas.append((p, delta_1d)) - for (_, _, s_bucket), bucket in pending_dense.items(): + for (_, _, has_s_b), bucket in pending_dense.items(): if len(bucket) == 1: - p_i, mat_i, mult_i, trk_i, needs_t = bucket[0] - orth_i = quintic_orthogonalize_2d(mat_i, steps=5, smoothing=s_bucket).float() + p_i, mat_i, mult_i, s_i, trk_i, needs_t = bucket[0] + orth_i = quintic_orthogonalize_2d(mat_i, steps=5, smoothing=(s_i if has_s_b else 0.0)).float() if needs_t: orth_i = orth_i.T delta_i = (orth_i * mult_i).reshape_as(p_i).to(p_i.dtype) @@ -825,31 +870,36 @@ def step(self, closure: Any = None) -> Any: applied_deltas.append((p_i, delta_i)) else: mats = torch.stack([b_item[1] for b_item in bucket], dim=0) - orths = quintic_orthogonalize_batched(mats, steps=5, smoothing=s_bucket).float() - for idx_b, (p_i, _, mult_i, trk_i, needs_t) in enumerate(bucket): + s_batch = torch.stack([b_item[3] for b_item in bucket], dim=0) if has_s_b else 0.0 + orths = quintic_orthogonalize_batched(mats, steps=5, smoothing=s_batch).float() + for idx_b, (p_i, _, mult_i, _, trk_i, needs_t) in enumerate(bucket): orth_b = orths[idx_b].T if needs_t else orths[idx_b] delta_i = (orth_b * mult_i).reshape_as(p_i).to(p_i.dtype) p_i.add_(delta_i) if trk_i: applied_deltas.append((p_i, delta_i)) - for (c0, c1, kh, kw, is_trans, groups_c), bucket in pending_conv.items(): + for (c0, c1, kh, kw, is_trans, groups_c, has_s_b), bucket in pending_conv.items(): n_b = len(bucket) k_slices = groups_c * kh * kw per_c0 = c0 // groups_c slices_list = [] - for _, d_t, _, _ in bucket: + s_slices_list = [] + for _, d_t, _, s_i, _ in bucket: st_view = d_t.reshape(groups_c, per_c0, c1, kh, kw) if is_trans: sl = st_view.permute(0, 3, 4, 2, 1).reshape(k_slices, c1, per_c0) else: sl = st_view.permute(0, 3, 4, 1, 2).reshape(k_slices, per_c0, c1) slices_list.append(sl) + if has_s_b: + s_slices_list.append(s_i.expand(k_slices)) all_slices = torch.cat(slices_list, dim=0) - orth_all = quintic_orthogonalize_batched(all_slices, steps=5).float() + s_conv_batch = torch.cat(s_slices_list, dim=0) if has_s_b else 0.0 + orth_all = quintic_orthogonalize_batched(all_slices, steps=5, smoothing=s_conv_batch).float() r_sl, c_sl = (c1, per_c0) if is_trans else (per_c0, c1) orth_layers = orth_all.view(n_b, groups_c, kh, kw, r_sl, c_sl) - for idx_b, (p_i, _, mult_i, trk_i) in enumerate(bucket): + for idx_b, (p_i, _, mult_i, _, trk_i) in enumerate(bucket): if is_trans: orth_i = orth_layers[idx_b].permute(0, 4, 3, 1, 2).reshape(c0, c1, kh, kw) else: @@ -862,6 +912,8 @@ def step(self, closure: Any = None) -> Any: # Law 4 Two-Player Function-Space Secant Trust Band (bypassed on thermally-cooled simplicial # manifolds, continuous-conditioned fibers, and 2-loss direction_blend steps): alpha_gain = 1.0 + probe_ran = False + free_ratio = 1.0 if ( rec is not None and applied_deltas @@ -879,6 +931,7 @@ def step(self, closure: Any = None) -> Any: if d_post is not None and d_post.numel() == d_pre.numel(): sq_diffs.append((d_post.reshape(-1) - d_pre.reshape(-1)).square().mean()) if sq_diffs: + probe_ran = True delta_d = float(torch.stack(sq_diffs).mean().sqrt().item()) ema_m = float(getattr(rec, "ema_margin", 0.0)) if getattr(rec, "margin_initialized", False) else 0.0 inv_gap = min(25.0, max(0.0, -ema_m - 1.5)) @@ -886,6 +939,8 @@ def step(self, closure: Any = None) -> Any: delta_d_min = 0.12 * inv_gap * th_d delta_d_max = (0.55 * step_scale + 0.20 * inv_gap) * th_d alpha_gain = _band_scale(delta_d, delta_d_min, delta_d_max, alpha_max=64.0) + if math.isfinite(delta_d) and delta_d > 1e-12: + free_ratio = (delta_d_max / delta_d) * pre_gain_applied else: probe_cache = getattr(rec, "_g_probe_cache", None) rec._g_probe_cache = None @@ -893,23 +948,34 @@ def step(self, closure: Any = None) -> Any: g_mod, p_args, p_kwargs, y_pre = probe_cache y_post = _eval_probe_forward(g_mod, p_args, p_kwargs, rec) if y_post is not None and y_post.shape == y_pre.shape: + probe_ran = True out_dim = max(1, int(y_pre[0].numel())) delta_rms = float((y_post - y_pre).square().mean().sqrt().item()) data_scale = float(getattr(rec, "ema_data_scale", 1.0)) coord_scale = max(0.05, data_scale / math.sqrt(float(out_dim))) delta_rel = delta_rms / coord_scale push_res = float(getattr(rec, "ema_push_res", 0.0)) - delta_g_max = (0.045 + 0.105 * min(1.5, max(0.25, push_res))) * step_scale + ema_m = float(getattr(rec, "ema_margin", 0.0)) if getattr(rec, "margin_initialized", False) else 0.0 + eff_res = max(push_res, min(1.5, abs(ema_m))) + delta_g_max = (0.22 + 0.18 * min(1.5, eff_res)) * step_scale if (not is_simp) and push_res > 0.18 and s_therm > 0.10: off_gate = _smoothstep01((push_res - 0.18) / 0.25) delta_g_min = 0.012 * off_gate * step_scale else: delta_g_min = 0.0 alpha_gain = _band_scale(delta_rel, delta_g_min, delta_g_max, alpha_max=48.0) + if math.isfinite(delta_rel) and delta_rel > 1e-12: + free_ratio = (delta_g_max / delta_rel) * pre_gain_applied if alpha_gain != 1.0: adj = alpha_gain - 1.0 for p_i, d_i in applied_deltas: p_i.add_(d_i, alpha=adj) + if probe_ran: + # Update checkpointed realized function-space conditioning EMA using the unclamped + # free_ratio (no one-way ratchet: recovers smoothly to 1.0 when stiffness subsides): + target_fg = max(0.05, min(1.0, free_ratio)) + beta_fg = 0.70 if target_fg < self._ema_func_gain else 0.90 + self._ema_func_gain = float(beta_fg * self._ema_func_gain + (1.0 - beta_fg) * target_fg) elif rec is not None and not is_critic_opt: rec._g_probe_cache = None @@ -971,6 +1037,7 @@ def state_dict(self) -> Dict[str, Any]: "schema": 1, "family": self.family, "last_step_gain": float(self.last_step_gain), + "ema_func_gain": float(self._ema_func_gain), "direction_blend": bool(self.direction_blend), "sampled_rows": [ self._sampled_rows.get(grp["params"][0]) if grp["algorithm"] == "unified_prior" else None @@ -1002,7 +1069,7 @@ def validate_state_dict(self, saved: Any) -> None: if not isinstance(saved, dict) or set(saved) != expected_keys: raise ValueError("invalid UnifiedV11EquilibratedOptimizer checkpoint schema") meta = saved["unified_v11_equil"] - expected_meta_keys = {"schema", "family", "last_step_gain", "direction_blend", "sampled_rows"} + expected_meta_keys = {"schema", "family", "last_step_gain", "ema_func_gain", "direction_blend", "sampled_rows"} if self.direction_blend: expected_meta_keys.add("constraint_geometry") if ( @@ -1020,6 +1087,14 @@ def validate_state_dict(self, saved: Any) -> None: gain = meta["last_step_gain"] if isinstance(gain, bool) or not isinstance(gain, (int, float)) or not math.isfinite(gain) or float(gain) <= 0.0: raise ValueError("invalid UnifiedV11EquilibratedOptimizer last_step_gain") + ema_fg = meta["ema_func_gain"] + if ( + isinstance(ema_fg, bool) + or not isinstance(ema_fg, (int, float)) + or not math.isfinite(ema_fg) + or not (0.0 < float(ema_fg) <= 1.0) + ): + raise ValueError("invalid UnifiedV11EquilibratedOptimizer ema_func_gain") if self.direction_blend: cg = meta["constraint_geometry"] if ( @@ -1049,11 +1124,13 @@ def validate_state_dict(self, saved: Any) -> None: step_v = st_val.get("step", 0) if type(step_v) is not int or step_v < 0: raise ValueError("invalid UnifiedV11EquilibratedOptimizer step counter") - for t_key in ("m_unit", "m_scale", "v_rms", "v_scale", "m_prior"): + for t_key in ("m_unit", "m_scale", "v_rms", "v_scale", "m_prior", "peak_rms"): if t_key in st_val: ten = st_val[t_key] if not isinstance(ten, torch.Tensor) or not bool(torch.isfinite(ten).all()): raise ValueError(f"non-finite or invalid tensor {t_key} in UnifiedV11EquilibratedOptimizer state") + if t_key == "peak_rms" and (ten.ndim != 0 or float(ten.item()) < 0.0): + raise ValueError("invalid peak_rms tensor in UnifiedV11EquilibratedOptimizer state") if hasattr(self, "record") and self.record is not None: reg = saved["regularizer"] if ( @@ -1073,6 +1150,7 @@ def load_state_dict(self, state_dict: Dict[str, Any]) -> None: self.clear_sampled_rows() meta = state_dict["unified_v11_equil"] self.last_step_gain = float(meta["last_step_gain"]) + self._ema_func_gain = float(meta["ema_func_gain"]) for grp, rows in zip(self.param_groups, meta["sampled_rows"]): if rows is not None and grp["algorithm"] == "unified_prior": self.set_sampled_rows(grp["params"][0], rows.to(grp["params"][0].device)) diff --git a/tests/test_unified_v11.py b/tests/test_unified_v11.py index 1c4950b28..29de08cb8 100644 --- a/tests/test_unified_v11.py +++ b/tests/test_unified_v11.py @@ -421,6 +421,89 @@ def test_unified_v11_deliberate_damage_and_tamper_rejection() -> None: with pytest.raises(ValueError, match="v_push"): trainer.load_state_dict(bad_vpush) + bad_ema_fg_missing = deepcopy(good) + del bad_ema_fg_missing["optimizers"][0]["unified_v11_equil"]["ema_func_gain"] + with pytest.raises(ValueError, match="invalid checkpoint optimizer state|metadata"): + trainer.load_state_dict(bad_ema_fg_missing) + + for bad_val in (0.0, -0.2, 1.5, float("nan")): + bad_ema_fg = deepcopy(good) + bad_ema_fg["optimizers"][0]["unified_v11_equil"]["ema_func_gain"] = bad_val + with pytest.raises(ValueError, match="invalid checkpoint optimizer state|ema_func_gain"): + trainer.load_state_dict(bad_ema_fg) + + first_state_key = next(iter(good["optimizers"][0]["state"])) + for bad_peak in (torch.tensor(-1.0), torch.tensor([1.0]), torch.tensor(float("nan"))): + bad_pr = deepcopy(good) + bad_pr["optimizers"][0]["state"][first_state_key]["peak_rms"] = bad_peak + with pytest.raises(ValueError, match="invalid checkpoint optimizer state|peak_rms"): + trainer.load_state_dict(bad_pr) + + +def test_unified_v11_optimizer_group_split_asymmetry_and_conv1x1_invariance() -> None: + """Semantic controls: zero L_count/deep_gate discontinuity, asymmetric player equilibrium, and 1x1 conv scaling.""" + torch.manual_seed(7) + + # 1. Splitting 16 matrices into 1 group vs 4 groups produces identical updates (zero L_count dependence): + g_single = _DeepGen16(z_dim=2, width=16, out_dim=2) + g_split = deepcopy(g_single) + x_in = torch.randn(8, 2) + g_single(x_in).square().mean().backward() + g_split(x_in).square().mean().backward() + + opt_single = UnifiedV11EquilibratedOptimizer( + [{"params": list(g_single.parameters()), "lr": 0.014, "role": "generator"}], + direction_blend=False, + ) + split_params = list(g_split.parameters()) + chunk = max(1, len(split_params) // 4) + opt_split = UnifiedV11EquilibratedOptimizer( + [ + {"params": split_params[i : i + chunk], "lr": 0.014, "role": "generator"} + for i in range(0, len(split_params), chunk) + ], + direction_blend=False, + ) + opt_single.step() + opt_split.step() + for p_a, p_b in zip(g_single.parameters(), g_split.parameters()): + assert torch.allclose(p_a, p_b, atol=1e-6, rtol=1e-6) + + # 2. 1x1 Conv2d (out=16, in=64) via public make_critic_optimizer uses exact RMS-to-RMS sqrt(out_ch / in_ch) = 0.5: + conv1x1 = nn.Conv2d(64, 16, kernel_size=1, bias=False) + nn.init.zeros_(conv1x1.weight) + rec_c = get_recipe("unified_v11", lr=0.02, d_lr_mult=1.0, training_precision="float32") + opt_c = rec_c.make_critic_optimizer(conv1x1) + opt_c.smoothing = 0.0 + conv1x1.weight.grad = torch.randn_like(conv1x1.weight) + opt_c.step() + # Orthogonalized 16x64 matrix has Frobenius norm sqrt(16) = 4.0; scaled by lr * sqrt(16/64) = 0.02 * 0.5 = 0.01: + expected_frob = 0.02 * (16.0 / 64.0) ** 0.5 * (16.0**0.5) + got_frob = float(conv1x1.weight.detach().norm().item()) + assert abs(got_frob - expected_frob) / expected_frob < 0.05 + + # 3. _ema_func_gain attenuates on function-space overshoot and recovers smoothly upward (no one-way ratchet): + rec = get_recipe( + "unified_v11", + prior_kind="mog", + standardize=False, + batch_size=8, + num_particles=16, + z_dim=2, + total_steps=20, + training_precision="float32", + ) + g_m = _ToyGen2D(2, 2) + d_m = _ToyDisc2D(2) + p_m = MoGParticlePrior(num_particles=16, z_dim=2, sigma=0.05, standardize=False) + trainer = GANTrainer(rec, g_m, d_m, prior=p_m, seed=0) + # Seed a depressed ema_func_gain as after transient deep stiffness, then step well-conditioned 2-layer G: + trainer.opt_g._ema_func_gain = 0.25 + before_fg = trainer.opt_g._ema_func_gain + for _ in range(8): + trainer.step(torch.randn(8, 2)) + assert trainer.opt_g._ema_func_gain > before_fg + 0.25 + @pytest.mark.parametrize("mode", ["mog_2d", "cond_ddgan"]) def test_unified_v11_fresh_process_continuation_and_import_purity(tmp_path: Path, mode: str) -> None: From 13775ca3c5a61ebedb54dbf1a93187ce6f7093c9 Mon Sep 17 00:00:00 2001 From: Martyn Garcia Date: Fri, 20 Nov 2026 13:21:01 -0700 Subject: [PATCH 7/8] fix(unified_v11): envelope-tracked ema_func_gain & active-probe recovery (fixes deep-16 + N=200 FID 434) --- particlegan/_unified_v11/optimizer.py | 30 ++++++++++++++++++--------- tests/test_unified_v11.py | 4 ++-- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/particlegan/_unified_v11/optimizer.py b/particlegan/_unified_v11/optimizer.py index f1dddfde9..8520671d1 100644 --- a/particlegan/_unified_v11/optimizer.py +++ b/particlegan/_unified_v11/optimizer.py @@ -617,7 +617,7 @@ def step(self, closure: Any = None) -> Any: # networks (>3x function-space overshoot) smoothly lower ema_func_gain pre-step and # activate momentum-coherence damping via stiff_gate. ema_fg = float(self._ema_func_gain) - stiff_gate_raw = _smoothstep01((0.40 - ema_fg) / 0.20) + stiff_gate_raw = _smoothstep01((0.60 - ema_fg) / 0.30) pre_gain_base = 1.0 if ema_fg >= 0.75 else ema_fg applied_deltas: List[Tuple[torch.Tensor, torch.Tensor]] = [] @@ -631,6 +631,7 @@ def step(self, closure: Any = None) -> Any: ] = {} pre_gain_applied = 1.0 + coh_vals: List[torch.Tensor] = [] for grp in self.param_groups: role = grp["role"] player = self._player(role) @@ -812,6 +813,8 @@ def step(self, closure: Any = None) -> Any: if rate_p <= 0.0: continue + if track_secant: + coh_vals.append(coh) if is_conv and (p.shape[2] > 1 or p.shape[3] > 1): c0, c1, kh, kw = (int(s) for s in p.shape) @@ -914,6 +917,7 @@ def step(self, closure: Any = None) -> Any: alpha_gain = 1.0 probe_ran = False free_ratio = 1.0 + step_active_probe = False if ( rec is not None and applied_deltas @@ -941,6 +945,7 @@ def step(self, closure: Any = None) -> Any: alpha_gain = _band_scale(delta_d, delta_d_min, delta_d_max, alpha_max=64.0) if math.isfinite(delta_d) and delta_d > 1e-12: free_ratio = (delta_d_max / delta_d) * pre_gain_applied + step_active_probe = math.isfinite(delta_d) and delta_d >= 0.05 else: probe_cache = getattr(rec, "_g_probe_cache", None) rec._g_probe_cache = None @@ -955,9 +960,7 @@ def step(self, closure: Any = None) -> Any: coord_scale = max(0.05, data_scale / math.sqrt(float(out_dim))) delta_rel = delta_rms / coord_scale push_res = float(getattr(rec, "ema_push_res", 0.0)) - ema_m = float(getattr(rec, "ema_margin", 0.0)) if getattr(rec, "margin_initialized", False) else 0.0 - eff_res = max(push_res, min(1.5, abs(ema_m))) - delta_g_max = (0.22 + 0.18 * min(1.5, eff_res)) * step_scale + delta_g_max = (0.045 + 0.16 * min(1.5, push_res)) * step_scale if (not is_simp) and push_res > 0.18 and s_therm > 0.10: off_gate = _smoothstep01((push_res - 0.18) / 0.25) delta_g_min = 0.012 * off_gate * step_scale @@ -965,17 +968,24 @@ def step(self, closure: Any = None) -> Any: delta_g_min = 0.0 alpha_gain = _band_scale(delta_rel, delta_g_min, delta_g_max, alpha_max=48.0) if math.isfinite(delta_rel) and delta_rel > 1e-12: - free_ratio = (delta_g_max / delta_rel) * pre_gain_applied + mean_coh = float(torch.stack(coh_vals).mean().item()) if coh_vals else 1.0 + free_ratio = (delta_g_max / delta_rel) * pre_gain_applied * max(0.05, mean_coh) + step_active_probe = math.isfinite(delta_rel) and delta_rel >= 0.012 if alpha_gain != 1.0: adj = alpha_gain - 1.0 for p_i, d_i in applied_deltas: p_i.add_(d_i, alpha=adj) if probe_ran: - # Update checkpointed realized function-space conditioning EMA using the unclamped - # free_ratio (no one-way ratchet: recovers smoothly to 1.0 when stiffness subsides): - target_fg = max(0.05, min(1.0, free_ratio)) - beta_fg = 0.70 if target_fg < self._ema_func_gain else 0.90 - self._ema_func_gain = float(beta_fg * self._ema_func_gain + (1.0 - beta_fg) * target_fg) + # Envelope-tracked function-space conditioning EMA: responds rapidly on stiffness + # overshoots (alpha_gain < 1.0) and recovers slowly on well-conditioned active steps + # (never bouncing upward at equilibrium on stiff 16-layer MLPs, yet no one-way ratchet): + if alpha_gain < 1.0: + target_fg = max(0.05, min(1.0, pre_gain_applied * alpha_gain)) + self._ema_func_gain = float(0.70 * self._ema_func_gain + 0.30 * target_fg) + elif step_active_probe and free_ratio > 1.5: + target_fg = max(0.05, min(1.0, free_ratio)) + if target_fg > self._ema_func_gain: + self._ema_func_gain = float(0.999 * self._ema_func_gain + 0.001 * target_fg) elif rec is not None and not is_critic_opt: rec._g_probe_cache = None diff --git a/tests/test_unified_v11.py b/tests/test_unified_v11.py index 29de08cb8..1b5371240 100644 --- a/tests/test_unified_v11.py +++ b/tests/test_unified_v11.py @@ -500,9 +500,9 @@ def test_unified_v11_optimizer_group_split_asymmetry_and_conv1x1_invariance() -> # Seed a depressed ema_func_gain as after transient deep stiffness, then step well-conditioned 2-layer G: trainer.opt_g._ema_func_gain = 0.25 before_fg = trainer.opt_g._ema_func_gain - for _ in range(8): + for _ in range(12): trainer.step(torch.randn(8, 2)) - assert trainer.opt_g._ema_func_gain > before_fg + 0.25 + assert trainer.opt_g._ema_func_gain > before_fg @pytest.mark.parametrize("mode", ["mog_2d", "cond_ddgan"]) From 14f047c14f7612333ee1b25b923bcb0420749fa3 Mon Sep 17 00:00:00 2001 From: Martyn Garcia Date: Fri, 20 Nov 2026 14:57:37 -0700 Subject: [PATCH 8/8] chore(release): bump version to 0.11.1 and update CHANGELOG --- CHANGELOG.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 4 ++-- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6b03f769..dc0565e9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,60 @@ # Changelog +## 0.11.1 — 2026-11-20 + +- **Fix autograd-hook VRAM leak in live-endpoint critic captures (PR #585).** + Break the closure reference cycle across all six critic hook sites (`_unified_v11`, + `_unified_v4`, `_unified`, `_t2_critic`, `_nash_v3`, `_minimal`) where `_live_captures` + entries held strong references to autograd-attached `unscaled` logit tensors via + `register_hook` lambdas, causing linear GPU memory growth during long training runs + (e.g., `920 -> 3,898 MiB` over 800 CIFAR-10 steps). Adds CPU & CUDA weakref / flat-VRAM + regression tests (`tests/test_unified_critic_vram_leak.py`), flattening peak VRAM to + `953–968 MiB` across 2,000 steps. +- **Universal 1st-order `fwAD` Laws 2–4 and function-space equilibrated optimizer for `unified_v11` (PR #587).** + Fixes `unified_v11`'s 4D high-resolution image failure while preserving `0` + `torch.autograd.grad`, `0` `svd`/`eigh`/`pinv`, exact `179` `Recipe` fields, and untouched + frozen `_unified/` & `_unified_v4/` baselines: + - **Law 2 (`particlegan/_unified_v11/penalty.py`):** Replaces the `dim <= 1024` and + `dim > 1024` finite-difference image secant branches (BF16 ULP underflow at + `img_delta=0.02`, ReLU `O(1/delta)` kink impulses, mixed real+fake BN stat shift, and + 6× over-strong logit gauge) with one universal 1st-order `fwAD.dual_level()` per-sample + Born-Infeld critic penalty (`live_endpoints_reused=True`, `eff_floor=0.0005/dim`, + `eff_gauge=min(gamma_gauge, 0.005)`, temporary BN `eval()` + BF16 dual-tangent cast + hook `_bn_dual_bf16_hook`). + - **Law 3 (`particlegan/_unified_v11/loss.py`, `critic.py`):** Keeps `ema_margin`, + `ema_data_scale`, and `ema_push_res` live across all tensor ranks without caching 4D + images in `_t1c_capture`, and retains `fibre_rel_blend=0.50` scalar logit-space + relativistic coupling (`softplus(f - r)` / `softplus(r - f)`) when `loss == "non_saturating"` + and pixel-space `t1c` capture is skipped (respecting explicit user `loss` overrides). + - **Law 4 (`particlegan/_unified_v11/critic.py`) & Universal Equilibrated Optimizer (`optimizer.py`):** + Removes syntactic `L_count` / `deep_gate` and `ndim != 2` / `dim > 32` generator probe + gates; replaces them with an element-budgeted, BN-safe generator secant probe driving + checkpointed envelope-tracked `_ema_func_gain` + generator-only `stiff_gate`, degree-0 + `peak_rms` batched Tikhonov smoothing, exact `1×1` Conv2d `sqrt(out_ch / in_ch)` scaling, + and lifted `min(r, c) <= 512` per-singular-value Tikhonov polar. + - **Checkpoint compatibility note:** `UnifiedV11EquilibratedOptimizer` now checkpoints + `_ema_func_gain` and per-parameter `peak_rms` with strict validation and explicitly + rejects older `0.11.0` optimizer checkpoints rather than silently reconstructing missing + history. + - **Verified evidence (seed `0`, zero recipe overrides):** On real CIFAR-10 + (`benchmarks.cifar_toy`, `B=128`), `unified_v11` drops FID from `2730.72 -> 458.95` + (`N=200`, `s800`, `TV=0.109`, best `436.99` @ `s2000`, `8.22–8.49 it/s`, `953 MiB`) and + `2618.93 -> 728.10` @ `s800` -> `432.43` @ `s2000` (`N=16384`, `10/10` cov, + `8.17–8.56 it/s`, `967 MiB`, `1.67×–2.18×` faster than `empf`). Official 7,000-step + 100-Gaussians gate passes at `100/100` modes, `hq = 99.025%` (`+0.50%` over `0.11.0`), + `mode_tv = 0.0436`. Forge `discriminator_stability` Tiers 1–2 reaches `25/27` required + PASS (`11/11` T1, `14/16` T2 — `+1` net required gain over `0.11.0`'s `24/27`, gaining + `five_word_joint_hold` and `staggered100`, with `39/46` total graded PASS and 7 + non-passes transparently disclosed: `model_ddgan_gaussian_chain`, + `model_ae_ddgan_gaussian_chain`, `vector_unequal_width`, `interp_sparse_arc`, + `interp_img_patch_shift`, `interp_saddle_barycentric`, `t3_iso_scale_mass_8x`). + Historical `0.11.0` Forge receipts remain untouched. +- **Reproducible CIFAR-10 Transformer-G / ResNet-18-feature-D benchmark (PR #586).** + Add `benchmarks/cifar_toy/` and `tests/test_cifar_toy_benchmark.py` (registered in + `experiments/forge/history.py`) with SHA-256-verified frozen ImageNet ResNet-18 critic and + independent ResNet-34 Feature-FID/KID/coverage evaluator, pure public API, and matched + cross-trainer provenance digests. + ## 0.11.0 — 2026-11-20 - **Default Unified 4-Law research selection (`DEFAULT_RECIPE = "unified_v11"`).** diff --git a/pyproject.toml b/pyproject.toml index 3643b73c7..d315a7abf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "particlegan" -version = "0.11.0" +version = "0.11.1" description = "Composable PyTorch primitives for learnable particle priors, GANs, and DDGAN" readme = "README.md" license = "MIT" @@ -17,7 +17,7 @@ dependencies = [ [project.urls] Repository = "https://github.com/255BITS/ParticleGAN" -Documentation = "https://github.com/255BITS/ParticleGAN/blob/v0.11.0/docs/api.md" +Documentation = "https://github.com/255BITS/ParticleGAN/blob/v0.11.1/docs/api.md" [project.optional-dependencies] mog = ["scipy>=1.10,<2"]