From 6808603e1de4a90a007b8f97361802e27d3bd6d9 Mon Sep 17 00:00:00 2001 From: Martyn Garcia Date: Fri, 20 Nov 2026 10:42:44 -0700 Subject: [PATCH] fix(critic): break autograd hook reference cycle leaking VRAM in _live_captures --- particlegan/_minimal/dynamics.py | 10 +- particlegan/_nash_v3/record.py | 26 ++- particlegan/_t2_critic/critic.py | 29 ++- particlegan/_unified/critic.py | 14 +- particlegan/_unified_v11/critic.py | 14 +- particlegan/_unified_v4/critic.py | 14 +- tests/test_unified_critic_vram_leak.py | 309 +++++++++++++++++++++++++ 7 files changed, 381 insertions(+), 35 deletions(-) create mode 100644 tests/test_unified_critic_vram_leak.py diff --git a/particlegan/_minimal/dynamics.py b/particlegan/_minimal/dynamics.py index 71b4fe51a..7e34e0a46 100644 --- a/particlegan/_minimal/dynamics.py +++ b/particlegan/_minimal/dynamics.py @@ -43,9 +43,8 @@ def _minimal_critic_hook(module: nn.Module, args: Tuple[Any, ...], kwargs: Dict[ if rec is not None and not rec._in_penalty and module.training and torch.is_grad_enabled() and target.requires_grad and no_ig and (x_in.ndim >= 3 or x_in[0].numel() > 8): skey = getattr(module, "_t2_pending_source_key", None) or _tensor_key(x_in) if skey is not None: - unsc = target.float() if target.dtype == torch.bfloat16 else target - entry: List[Any] = [int(rec.observed_steps), skey, _cond_signature(args, kwargs), x_in.detach(), unsc, True] - unsc.register_hook(lambda _g, _e=entry: _e.__setitem__(5, False)); rec._live_captures.append(entry); del rec._live_captures[:-6] + alive = [True]; rec._live_captures.append([int(rec.observed_steps), skey, _cond_signature(args, kwargs), x_in.detach(), target.float() if target.dtype == torch.bfloat16 else target, alive]) + target.register_hook(lambda _g, _a=alive: _a.clear()); [e.__setitem__(slice(3, 5), [None, None]) or e[5].clear() for e in rec._live_captures[:-6]]; del rec._live_captures[:-6] if rec is not None and not rec._in_penalty and torch.is_grad_enabled() and rec._thermal_scale < 1.0 and isinstance(output, torch.Tensor) and output.requires_grad: scaled = output * rec._thermal_scale + output.detach() * (1.0 - rec._thermal_scale) if x_in is not None: scaled._minimal_input, scaled._minimal_record, scaled._minimal_conditioned, scaled._minimal_cond_tensors = x_in, rec, is_cond, cond_t @@ -119,7 +118,8 @@ def observe_cond_fiber(self, r_cond: float) -> None: self.cond_ema = max(0.18, rc) if not self.cond_initialized else (0.99 * self.cond_ema + 0.01 * rc) self.cond_initialized, self._thermal_scale, self._step_scale = True, 1.0, float(1.0 - 0.85 * _smoothstep01((0.050 - max(0.0, self.cond_ema)) / 0.024)) - def record_step(self, optimizer: Any) -> None: super().record_step(optimizer); self._live_captures.clear(); self._g_probe_cache = self._last_real_pair = None + def clear_transient_captures(self) -> None: [e.__setitem__(slice(3, 5), [None, None]) or e[5].clear() for e in self._live_captures]; self._live_captures.clear() + def record_step(self, optimizer: Any) -> None: super().record_step(optimizer); self.clear_transient_captures(); self._g_probe_cache = self._last_real_pair = None def state_dict(self) -> Dict[str, Any]: return {**super().state_dict(), "minimal_state": {"schema": 1, **{k: bool(getattr(self, k)) for k in _BOOL_KEYS}, **{k: float(getattr(self, "_" + k if k in ("thermal_scale", "step_scale") else k)) for k in _BOUNDS}, "cond_bank": deepcopy(self.cond_bank)}} @@ -131,7 +131,7 @@ def load_state_dict(self, state: Dict[str, Any]) -> None: cb = _validate_cond_bank(pkt["cond_bank"]); super().load_state_dict(sc) for k in _BOOL_KEYS: setattr(self, k, pkt[k]) for k, v in vals.items(): setattr(self, "_" + k if k in ("thermal_scale", "step_scale") else k, v) - self.cond_bank, self._g_probe_cache = cb, None; self._live_captures.clear() + self.cond_bank, self._g_probe_cache = cb, None; self.clear_transient_captures() def _collect_conv_layouts(modules: Iterable[Optional[nn.Module]]) -> Dict[int, Tuple[bool, int]]: return {id(c.weight): (isinstance(c, nn.ConvTranspose2d), int(c.groups)) for r in modules if isinstance(r, nn.Module) for c in r.modules() if isinstance(c, (nn.Conv2d, nn.ConvTranspose2d))} diff --git a/particlegan/_nash_v3/record.py b/particlegan/_nash_v3/record.py index 0b8576c3f..48ce53491 100644 --- a/particlegan/_nash_v3/record.py +++ b/particlegan/_nash_v3/record.py @@ -7,7 +7,12 @@ import torch from torch import nn -from .._t2_critic.critic import _cond_signature, _tensor_identity_key +from .._t2_critic.critic import ( + _clear_live_capture_entries, + _cond_signature, + _tensor_identity_key, + _trim_live_captures, +) from ..grad_regularizers import CriticStepRecord FROZEN_THERMAL_SCALE = 1e-4 @@ -163,11 +168,11 @@ def _nash_v3_critic_forward_hook( rec._live_captures = captures c_sig = _cond_signature(args, kwargs) step_tag = int(getattr(rec, "observed_steps", 0)) - entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, True] - unscaled.register_hook(lambda _g, _e=entry: _e.__setitem__(5, False)) + alive = [True] + entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, alive] + target.register_hook(lambda _g, _a=alive: _a.clear()) captures.append(entry) - if len(captures) > _MAX_LIVE_CAPTURES: - del captures[: len(captures) - _MAX_LIVE_CAPTURES] + _trim_live_captures(captures, _MAX_LIVE_CAPTURES) # Low-D secant probe capture for critic inversion-deficit trust band (skipped on images for speed). if ( @@ -363,7 +368,8 @@ def _init_nash_v3_fields(self) -> None: self._in_probe: bool = False def clear_transient_captures(self) -> None: - self._live_captures = [] + if hasattr(self, "_live_captures") and self._live_captures: + _clear_live_capture_entries(self._live_captures) self._last_probe_actual_input = None def observe_transport(self, res: float, data_scale: float, ema_decay: float = 0.92) -> None: @@ -465,8 +471,8 @@ def _restore_nash_v3_packet(self, packet: Dict[str, Any]) -> None: self._last_d_logits = None self._d_probe_list = None self._g_probe_cache = None + self.clear_transient_captures() self._live_captures = [] - self._last_probe_actual_input = None def restore_device_state(self, device: torch.device) -> None: if isinstance(self.proto_sum, torch.Tensor) and self.proto_sum.device != device: @@ -482,6 +488,12 @@ def __init__(self) -> None: super().__init__() self._init_nash_v3_fields() + def record_step(self, optimizer_or_lr: Any) -> None: + super().record_step(optimizer_or_lr) + self._last_d_logits = None + self._d_probe_list = None + self.clear_transient_captures() + def state_dict(self) -> Dict[str, Any]: return {**super().state_dict(), "t4_nash_v3": self._nash_v3_packet()} diff --git a/particlegan/_t2_critic/critic.py b/particlegan/_t2_critic/critic.py index 5b4b755f3..d1e0a23a3 100644 --- a/particlegan/_t2_critic/critic.py +++ b/particlegan/_t2_critic/critic.py @@ -18,6 +18,25 @@ _MAX_LIVE_CAPTURES = 6 + +def _clear_live_capture_entries(entries: List[Any]) -> None: + """Release tensor references and backward-alive flags from live-capture entries.""" + for entry in entries: + if isinstance(entry, list) and len(entry) >= 5: + entry[3] = None + entry[4] = None + if len(entry) >= 6 and isinstance(entry[5], list): + entry[5].clear() + entries.clear() + + +def _trim_live_captures(captures: List[Any], max_entries: int = _MAX_LIVE_CAPTURES) -> None: + """Evict oldest live-capture entries while explicitly dropping their tensor references.""" + if len(captures) > max_entries: + evict_n = len(captures) - max_entries + _clear_live_capture_entries(captures[:evict_n]) + del captures[:evict_n] + _T2_STATE_KEYS = frozenset({ "schema", "initialized", @@ -146,11 +165,11 @@ def _t2_critic_forward_hook( rec._live_captures = captures c_sig = _cond_signature(args, kwargs) step_tag = int(getattr(rec, "observed_steps", 0)) - entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, True] - unscaled.register_hook(lambda _g, _e=entry: _e.__setitem__(5, False)) + alive = [True] + entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, alive] + raw_target.register_hook(lambda _g, _a=alive: _a.clear()) captures.append(entry) - if len(captures) > _MAX_LIVE_CAPTURES: - del captures[: len(captures) - _MAX_LIVE_CAPTURES] + _trim_live_captures(captures) if module.training and rec is not None and not getattr(rec, "_in_penalty", False) and torch.is_grad_enabled(): s = float(getattr(rec, "_thermal_scale", 1.0)) @@ -257,7 +276,7 @@ def _init_t2_fields(self) -> None: def clear_transient_captures(self) -> None: if hasattr(self, "_live_captures") and self._live_captures: - self._live_captures.clear() + _clear_live_capture_entries(self._live_captures) self._last_probe_actual_input = None def __deepcopy__(self, memo: Dict[int, Any]) -> Any: diff --git a/particlegan/_unified/critic.py b/particlegan/_unified/critic.py index 8c157a9ff..6c40c7a09 100644 --- a/particlegan/_unified/critic.py +++ b/particlegan/_unified/critic.py @@ -28,9 +28,10 @@ _validate_t1c_packet, ) from .._t2_critic.critic import ( - _MAX_LIVE_CAPTURES, + _clear_live_capture_entries, _cond_signature, _tensor_identity_key, + _trim_live_captures, ) from ..grad_regularizers import CriticStepRecord from ..k3p import K3PCriticAdam @@ -83,7 +84,7 @@ def rgdt_cond_ema(self, val: float) -> None: def clear_transient_captures(self) -> None: if hasattr(self, "_live_captures") and self._live_captures: - self._live_captures.clear() + _clear_live_capture_entries(self._live_captures) self._last_probe_actual_input = None def _unified_packet(self) -> Dict[str, Any]: @@ -98,6 +99,7 @@ def _restore_unified_packet(self, packet: Dict[str, Any]) -> None: self._restore_t1c_packet(t1c_sub) self._step_scale = float(packet["step_scale"]) self._t1c_disc_gate = 0.0 + self.clear_transient_captures() self._live_captures = [] self._last_probe_actual_input = None @@ -241,11 +243,11 @@ def _unified_critic_forward_hook( rec._live_captures = captures c_sig = _cond_signature(args, kwargs) step_tag = int(getattr(rec, "observed_steps", 0)) - entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, True] - unscaled.register_hook(lambda _g, _e=entry: _e.__setitem__(5, False)) + alive = [True] + entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, alive] + raw_target.register_hook(lambda _g, _a=alive: _a.clear()) captures.append(entry) - if len(captures) > _MAX_LIVE_CAPTURES: - del captures[: len(captures) - _MAX_LIVE_CAPTURES] + _trim_live_captures(captures) # 2. Apply thermal cooling to outgoing critic logits (outside penalty probes): modified = False diff --git a/particlegan/_unified_v11/critic.py b/particlegan/_unified_v11/critic.py index aea44fe5d..3548f4703 100644 --- a/particlegan/_unified_v11/critic.py +++ b/particlegan/_unified_v11/critic.py @@ -18,9 +18,10 @@ _validate_t1c_packet, ) from .._t2_critic.critic import ( - _MAX_LIVE_CAPTURES, + _clear_live_capture_entries, _cond_signature, _tensor_identity_key, + _trim_live_captures, ) from ..grad_regularizers import CriticStepRecord from ..k3p import K3PCriticAdam @@ -158,7 +159,7 @@ def observe_margin_and_scale( def clear_transient_captures(self) -> None: if hasattr(self, "_live_captures") and self._live_captures: - self._live_captures.clear() + _clear_live_capture_entries(self._live_captures) self._last_probe_actual_input = None def clear_step_probes(self) -> None: @@ -188,6 +189,7 @@ def _restore_unified_v11_packet(self, packet: Dict[str, Any]) -> None: self.cond_fiber_bank.load_packet(packet["cond_fiber_bank"]) self._t1c_disc_gate = 0.0 self._simplicial_step = False + self.clear_transient_captures() self._live_captures = [] self._d_probe_list = [] self._g_probe_cache = None @@ -347,11 +349,11 @@ def _unified_v11_critic_forward_hook( rec._live_captures = captures c_sig = _cond_signature(args, kwargs) step_tag = int(getattr(rec, "observed_steps", 0)) - entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, True] - unscaled.register_hook(lambda _g, _e=entry: _e.__setitem__(5, False)) + alive = [True] + entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, alive] + raw_target.register_hook(lambda _g, _a=alive: _a.clear()) captures.append(entry) - if len(captures) > _MAX_LIVE_CAPTURES: - del captures[: len(captures) - _MAX_LIVE_CAPTURES] + _trim_live_captures(captures) if ( getattr(rec, "_enable_secant_probe", True) diff --git a/particlegan/_unified_v4/critic.py b/particlegan/_unified_v4/critic.py index 5ba4dc85c..c5730580e 100644 --- a/particlegan/_unified_v4/critic.py +++ b/particlegan/_unified_v4/critic.py @@ -30,9 +30,10 @@ _validate_t1c_packet, ) from .._t2_critic.critic import ( - _MAX_LIVE_CAPTURES, + _clear_live_capture_entries, _cond_signature, _tensor_identity_key, + _trim_live_captures, ) from ..grad_regularizers import CriticStepRecord from ..k3p import K3PCriticAdam @@ -308,7 +309,7 @@ def observe_cond_gen_stratum( def clear_transient_captures(self) -> None: if hasattr(self, "_live_captures") and self._live_captures: - self._live_captures.clear() + _clear_live_capture_entries(self._live_captures) self._last_probe_actual_input = None def clear_step_probes(self) -> None: @@ -372,6 +373,7 @@ def _restore_unified_v4_packet(self, packet: Dict[str, Any]) -> None: } self._t1c_disc_gate = 0.0 self._simplicial_step = False + self.clear_transient_captures() self._live_captures = [] self._d_probe_list = [] self._g_probe_cache = None @@ -546,11 +548,11 @@ def _unified_v4_critic_forward_hook( rec._live_captures = captures c_sig = _cond_signature(args, kwargs) step_tag = int(getattr(rec, "observed_steps", 0)) - entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, True] - unscaled.register_hook(lambda _g, _e=entry: _e.__setitem__(5, False)) + alive = [True] + entry: List[Any] = [step_tag, src_key, c_sig, x_act, unscaled, alive] + raw_target.register_hook(lambda _g, _a=alive: _a.clear()) captures.append(entry) - if len(captures) > _MAX_LIVE_CAPTURES: - del captures[: len(captures) - _MAX_LIVE_CAPTURES] + _trim_live_captures(captures) if ( getattr(rec, "_enable_secant_probe", True) diff --git a/tests/test_unified_critic_vram_leak.py b/tests/test_unified_critic_vram_leak.py new file mode 100644 index 000000000..2febd0a81 --- /dev/null +++ b/tests/test_unified_critic_vram_leak.py @@ -0,0 +1,309 @@ +"""Weakref, eviction, backward-alive semantics, and flat-VRAM regression tests for critic live-capture hooks.""" +from __future__ import annotations + +import gc +import weakref + +import pytest +import torch +from torch import nn + +from particlegan import GANTrainer, get_recipe +from particlegan._minimal.dynamics import MinimalStepRecord, _attach_minimal_critic_hook +from particlegan._minimal.geometry import MinimalCriticPenalty +from particlegan._nash_v3.penalty import _NashV3PenaltyKernel +from particlegan._nash_v3.record import NashV3StepRecord, _attach_nash_v3_critic_hook +from particlegan._t2_critic.critic import ( + _MAX_LIVE_CAPTURES, + T2SecantStepRecord, + _attach_t2_critic_hook, +) +from particlegan._t2_critic.penalty import _T2SecantPenaltyKernel +from particlegan._unified.critic import UnifiedStepRecord, _attach_unified_critic_hook +from particlegan._unified.penalty import _UnifiedPenaltyKernel +from particlegan._unified_v4.critic import UnifiedV4StepRecord, _attach_unified_v4_critic_hook +from particlegan._unified_v11.critic import UnifiedV11StepRecord, _attach_unified_v11_critic_hook +from particlegan._unified_v11.penalty import _UnifiedV11PenaltyKernel + + +class _SmallConvCritic(nn.Module): + def __init__(self, channels: int = 3) -> None: + super().__init__() + self.net = nn.Sequential( + nn.Conv2d(channels, 8, kernel_size=3, stride=2, padding=1), + nn.SiLU(), + nn.Conv2d(8, 8, kernel_size=3, stride=2, padding=1), + nn.SiLU(), + ) + self.head = nn.Linear(8, 1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.head(self.net(x).mean(dim=(2, 3))).squeeze(-1) + + +class _SmallConvGenerator(nn.Module): + def __init__(self, z_dim: int = 8, channels: int = 3) -> None: + super().__init__() + self.fc = nn.Linear(z_dim, 8 * 8 * 8) + self.up1 = nn.ConvTranspose2d(8, 8, kernel_size=4, stride=2, padding=1) + self.up2 = nn.ConvTranspose2d(8, channels, kernel_size=4, stride=2, padding=1) + + def forward(self, z: torch.Tensor) -> torch.Tensor: + h = nn.functional.silu(self.fc(z)).reshape(z.shape[0], 8, 8, 8) + h = nn.functional.silu(self.up1(h)) + return torch.tanh(self.up2(h)) + + +_HOOK_FACTORIES = [ + pytest.param(UnifiedV11StepRecord, _attach_unified_v11_critic_hook, id="unified_v11"), + pytest.param(UnifiedV4StepRecord, _attach_unified_v4_critic_hook, id="unified_v4"), + pytest.param(UnifiedStepRecord, _attach_unified_critic_hook, id="unified"), + pytest.param(T2SecantStepRecord, _attach_t2_critic_hook, id="t2_secant"), + pytest.param(NashV3StepRecord, _attach_nash_v3_critic_hook, id="t4_nash_v3"), + pytest.param(MinimalStepRecord, _attach_minimal_critic_hook, id="minimal"), +] + + +@pytest.mark.parametrize(("record_cls", "attach_fn"), _HOOK_FACTORIES) +@pytest.mark.parametrize("use_bf16", [False, True], ids=["fp32", "bf16"]) +def test_live_capture_hook_releases_tensors_without_cycle( + record_cls, + attach_fn, + use_bf16: bool, +) -> None: + """`x_act` and `unscaled` must be GC'd after backward + clear, even if a caller holds `entry`.""" + torch.manual_seed(0) + critic = _SmallConvCritic() + if use_bf16: + critic = critic.to(dtype=torch.bfloat16) + record = record_cls() + attach_fn(critic, record) + + x = torch.randn(4, 3, 32, 32, dtype=torch.bfloat16 if use_bf16 else torch.float32) + out = critic(x) + + assert len(record._live_captures) == 1 + saved_entry = record._live_captures[0] + assert isinstance(saved_entry[5], list) + assert bool(saved_entry[5]) is True + + ref_x_act = weakref.ref(saved_entry[3]) + ref_unscaled = weakref.ref(saved_entry[4]) + + out.mean().backward() + assert bool(saved_entry[5]) is False + + record.clear_transient_captures() + assert record._live_captures == [] + assert saved_entry[3] is None + assert saved_entry[4] is None + + del x, out + gc.collect() + + assert ref_x_act() is None, "x_act leaked after backward + clear_transient_captures" + assert ref_unscaled() is None, "unscaled leaked after backward + clear_transient_captures" + + +@pytest.mark.parametrize(("record_cls", "attach_fn"), _HOOK_FACTORIES) +def test_live_capture_ring_eviction_releases_old_tensors(record_cls, attach_fn) -> None: + """Evicting entries beyond `_MAX_LIVE_CAPTURES` must immediately drop tensor refs without leaking.""" + torch.manual_seed(0) + critic = _SmallConvCritic() + record = record_cls() + attach_fn(critic, record) + + evicted_refs = [] + evicted_entries = [] + total_calls = _MAX_LIVE_CAPTURES + 4 + for i in range(total_calls): + x = torch.randn(4, 3, 32, 32) + out = critic(x) + entry = record._live_captures[-1] + if i < total_calls - _MAX_LIVE_CAPTURES: + evicted_entries.append(entry) + evicted_refs.append((weakref.ref(entry[3]), weakref.ref(entry[4]))) + out.mean().backward() + del x, out, entry + + gc.collect() + assert len(record._live_captures) == _MAX_LIVE_CAPTURES + for entry, (ref_x, ref_u) in zip(evicted_entries, evicted_refs): + assert entry[3] is None and entry[4] is None + assert ref_x() is None, "evicted x_act leaked" + assert ref_u() is None, "evicted unscaled leaked" + + +@pytest.mark.parametrize(("record_cls", "attach_fn"), _HOOK_FACTORIES) +def test_live_capture_load_state_dict_releases_tensors(record_cls, attach_fn) -> None: + """Restoring a record checkpoint must clear tensor refs from any prior `_live_captures`.""" + torch.manual_seed(0) + critic = _SmallConvCritic() + record = record_cls() + attach_fn(critic, record) + snap = record.state_dict() + + x = torch.randn(4, 3, 32, 32) + out = critic(x) + saved_entry = record._live_captures[0] + ref_x_act = weakref.ref(saved_entry[3]) + ref_unscaled = weakref.ref(saved_entry[4]) + + record.load_state_dict(snap) + assert record._live_captures == [] + assert saved_entry[3] is None + assert saved_entry[4] is None + + del x, out + gc.collect() + assert ref_x_act() is None, "x_act leaked after load_state_dict" + assert ref_unscaled() is None, "unscaled leaked after load_state_dict" + + +@pytest.mark.parametrize( + ("record_cls", "attach_fn", "kernel_cls"), + [ + pytest.param(UnifiedV11StepRecord, _attach_unified_v11_critic_hook, _UnifiedV11PenaltyKernel, id="unified_v11"), + pytest.param(UnifiedStepRecord, _attach_unified_critic_hook, _UnifiedPenaltyKernel, id="unified"), + pytest.param(T2SecantStepRecord, _attach_t2_critic_hook, _T2SecantPenaltyKernel, id="t2_secant"), + pytest.param(NashV3StepRecord, _attach_nash_v3_critic_hook, _NashV3PenaltyKernel, id="t4_nash_v3"), + ], +) +def test_live_capture_alive_flag_prevents_reuse_after_backward( + record_cls, + attach_fn, + kernel_cls, +) -> None: + """Before backward, live endpoint lookup hits; after backward, hook clears `alive` so lookup misses.""" + torch.manual_seed(0) + critic = _SmallConvCritic() + record = record_cls() + attach_fn(critic, record) + kernel = kernel_cls(record=record) + + x_real = torch.randn(4, 3, 32, 32) + x_fake = torch.randn(4, 3, 32, 32) + d_real = critic(x_real) + d_fake = critic(x_fake) + + hit_before = kernel._lookup_live_endpoint(x_real, ()) + assert hit_before is not None and hit_before[1] is d_real + + (d_real.mean() - d_fake.mean()).backward() + + hit_after = kernel._lookup_live_endpoint(x_real, ()) + assert hit_after is None, "already-backwarded endpoint must not be reused" + + # Calling penalty after separate backward must succeed cleanly via fresh forward. + pen_val, _ = kernel.penalty(critic, x_real, x_fake, step=1) + assert torch.isfinite(pen_val) + pen_val.backward() + record.clear_transient_captures() + assert record._live_captures == [] + + +def test_minimal_penalty_skips_backwarded_capture_without_crash() -> None: + """MinimalCriticPenalty must skip already-backwarded live captures cleanly.""" + torch.manual_seed(0) + critic = _SmallConvCritic() + recipe = get_recipe("minimal", training_precision="float32") + opt_d = recipe.make_critic_optimizer(critic) + pen = MinimalCriticPenalty(recipe, opt_d) + record = opt_d.record + + x_real = torch.randn(4, 3, 32, 32) + x_fake = torch.randn(4, 3, 32, 32) + d_real = critic(x_real) + d_fake = critic(x_fake) + assert len(record._live_captures) == 2 + assert all(bool(entry[5]) for entry in record._live_captures) + + (d_real.mean() - d_fake.mean()).backward() + assert all(not bool(entry[5]) for entry in record._live_captures) + + # Penalty after separate backward must not try to backward through freed graphs. + pen_val = pen(critic, x_real, x_fake) + assert torch.isfinite(pen_val) + pen_val.backward() + record.clear_transient_captures() + assert record._live_captures == [] + + +@pytest.mark.parametrize("preset", ["unified_v11", "unified_v4", "unified"]) +def test_gan_trainer_cpu_releases_all_step_batches(preset: str) -> None: + """Running `GANTrainer` across steps retains zero weakrefs to past real batches on CPU.""" + torch.manual_seed(0) + recipe = get_recipe( + preset, + z_dim=8, + batch_size=8, + num_particles=16, + total_steps=20, + training_precision="float32", + ) + gen = _SmallConvGenerator(z_dim=8, channels=3) + disc = _SmallConvCritic(channels=3) + trainer = GANTrainer(recipe, gen, disc, seed=0) + + batch_refs = [] + for _ in range(12): + x_real = torch.randn(8, 3, 32, 32) + batch_refs.append(weakref.ref(x_real)) + trainer.step(x_real) + del x_real + + gc.collect() + leaked = sum(1 for ref in batch_refs if ref() is not None) + assert leaked == 0, f"{preset}: {leaked}/{len(batch_refs)} real batches were retained" + assert getattr(trainer.opt_d.record, "_live_captures", []) == [] + + +@pytest.mark.cuda +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required for VRAM leak regression test") +@pytest.mark.parametrize("preset", ["unified_v11", "unified_v4", "unified"]) +def test_gan_trainer_cuda_flat_vram_over_many_steps(preset: str) -> None: + """CUDA VRAM must stay flat over 40 CIFAR-10-sized steps with zero retained batch weakrefs.""" + device = torch.device("cuda:0") + torch.manual_seed(0) + torch.cuda.manual_seed(0) + + recipe = get_recipe( + preset, + z_dim=8, + batch_size=32, + num_particles=16, + total_steps=60, + ) + gen = _SmallConvGenerator(z_dim=8, channels=3).to(device) + disc = _SmallConvCritic(channels=3).to(device) + trainer = GANTrainer(recipe, gen, disc, seed=0) + + # Warm up allocator, optimizer states, and any lazy buffers. + for _ in range(5): + x_warm = torch.randn(32, 3, 32, 32, device=device) + trainer.step(x_warm) + del x_warm + + gc.collect() + torch.cuda.synchronize(device) + mem_before = torch.cuda.memory_allocated(device) + + batch_refs = [] + for _ in range(40): + x_real = torch.randn(32, 3, 32, 32, device=device) + batch_refs.append(weakref.ref(x_real)) + trainer.step(x_real) + del x_real + + gc.collect() + torch.cuda.synchronize(device) + mem_after = torch.cuda.memory_allocated(device) + + leaked = sum(1 for ref in batch_refs if ref() is not None) + assert leaked == 0, f"{preset}: {leaked}/{len(batch_refs)} CUDA input batches leaked" + assert getattr(trainer.opt_d.record, "_live_captures", []) == [] + # Without the fix, 40 steps x 2 batches x (32*3*32*32*4 bytes) + autograd graphs leaks > 30 MiB. + assert mem_after - mem_before <= 2 * 1024 * 1024, ( + f"{preset}: CUDA memory grew by {(mem_after - mem_before) / (1024 * 1024):.2f} MiB over 40 steps " + f"(before={mem_before}, after={mem_after})" + )