From 78ca410020e3115ade1e1d52574240f9357f37d3 Mon Sep 17 00:00:00 2001 From: Martyn Garcia Date: Fri, 20 Nov 2026 12:01:03 -0700 Subject: [PATCH 1/3] feat(unified_v11): universal 1st-order fwAD Law 2, scalar logit Law 3, and dimension-agnostic Law 4 --- particlegan/_unified_v11/critic.py | 56 +++-- particlegan/_unified_v11/loss.py | 40 +++- particlegan/_unified_v11/penalty.py | 331 +++++++++++++++++++++++++++- tests/test_unified_v11.py | 154 +++++++++++++ 4 files changed, 559 insertions(+), 22 deletions(-) diff --git a/particlegan/_unified_v11/critic.py b/particlegan/_unified_v11/critic.py index 3548f4703..d059bef23 100644 --- a/particlegan/_unified_v11/critic.py +++ b/particlegan/_unified_v11/critic.py @@ -145,17 +145,31 @@ def observe_margin_and_scale( else: self.ema_margin = 0.90 * self.ema_margin + 0.10 * margin if x_real is not None and x_real.numel() > 0: - xr = x_real.detach().reshape(x_real.shape[0], -1).float() - sc = float(xr.square().sum(dim=-1).mean().sqrt().clamp_min(0.05).item()) - if math.isfinite(sc): - self.ema_data_scale = sc if self.observed_steps == 0 else (0.95 * self.ema_data_scale + 0.05 * sc) - if x_fake is not None and x_fake.shape == x_real.shape: - xf = x_fake.detach().reshape(x_fake.shape[0], -1).float() - mean_gap = float((xf.mean(dim=0) - xr.mean(dim=0)).norm().item()) - std_gap = abs(float(xf.std().item()) - float(xr.std().item())) - rel_gap = min(10.0, (mean_gap + 0.5 * std_gap) / max(0.05, self.ema_data_scale)) - if math.isfinite(rel_gap): - self.ema_push_res = rel_gap if self.observed_steps == 0 else (0.92 * self.ema_push_res + 0.08 * rel_gap) + self.observe_scale(x_real, x_fake) + + @torch.no_grad() + def observe_scale( + self, + x_real: torch.Tensor, + x_fake: Optional[torch.Tensor] = None, + ) -> None: + """Update checkpointed data scale and pushforward residual EMAs.""" + if getattr(self, "_in_penalty", False) or getattr(self, "_in_probe", False) or getattr(self, "_in_protected_probe", False): + return + if x_real is None or x_real.numel() == 0: + return + step_tag = int(getattr(self, "observed_steps", 0)) + xr = x_real.detach().reshape(x_real.shape[0], -1).float() + sc = float(xr.square().sum(dim=-1).mean().sqrt().clamp_min(0.05).item()) + if math.isfinite(sc): + self.ema_data_scale = sc if step_tag == 0 else (0.95 * self.ema_data_scale + 0.05 * sc) + if x_fake is not None and x_fake.shape == x_real.shape: + xf = x_fake.detach().reshape(x_fake.shape[0], -1).float() + mean_gap = float((xf.mean(dim=0) - xr.mean(dim=0)).norm().item()) + std_gap = abs(float(xf.std().item()) - float(xr.std().item())) + rel_gap = min(10.0, (mean_gap + 0.5 * std_gap) / max(0.05, self.ema_data_scale)) + if math.isfinite(rel_gap): + self.ema_push_res = rel_gap if step_tag == 0 else (0.92 * self.ema_push_res + 0.08 * rel_gap) def clear_transient_captures(self) -> None: if hasattr(self, "_live_captures") and self._live_captures: @@ -425,7 +439,7 @@ def _unified_v11_generator_forward_hook( kwargs: Dict[str, Any], output: Any, ) -> None: - """Capture a detached ``B_probe <= 16`` generator slice for function-space secant trust-band regulation.""" + """Capture a dimension-agnostic element-budgeted generator slice for function-space secant trust-band regulation.""" rec = getattr(module, "_unified_v11_record", None) if ( rec is None @@ -443,19 +457,26 @@ def _unified_v11_generator_forward_hook( if ( not isinstance(raw_out, torch.Tensor) or not raw_out.requires_grad - or raw_out.ndim != 2 + or raw_out.ndim < 2 or raw_out.shape[0] < 1 - or raw_out.shape[1] > 32 ): return import torch.autograd.forward_ad as fwAD if fwAD.unpack_dual(raw_out).tangent is not None: return b_full = int(raw_out.shape[0]) - b_p = min(_PROBE_SLICE, b_full) + out_dim = max(1, int(raw_out[0].numel())) + b_p = min(b_full, max(2, min(_PROBE_SLICE, 8192 // out_dim))) p_args = tuple(_slice_probe_arg(a, b_full, b_p) for a in args) p_kwargs = {k: _slice_probe_arg(v, b_full, b_p) for k, v in kwargs.items()} - rec._g_probe_cache = (module, p_args, p_kwargs, raw_out.detach()[:b_p].to(dtype=torch.float32)) + if getattr(module, "_unified_v11_gen_has_bn", False): + from .._unified_v4.optimizer import _eval_probe_forward + y_pre = _eval_probe_forward(module, p_args, p_kwargs, rec) + if y_pre is None: + return + else: + y_pre = raw_out.detach()[:b_p].to(dtype=torch.float32) + rec._g_probe_cache = (module, p_args, p_kwargs, y_pre) def _attach_unified_v11_critic_hook(critic: nn.Module | None, record: CriticStepRecord) -> None: @@ -491,6 +512,9 @@ def _attach_unified_v11_generator_hook(generator: nn.Module | None, record: Crit if not isinstance(generator, nn.Module): return generator._unified_v11_record = record + generator._unified_v11_gen_has_bn = any( + isinstance(m, torch.nn.modules.batchnorm._BatchNorm) for m in generator.modules() + ) if getattr(generator, "_unified_v11_gen_hook_registered", False): return generator.register_forward_hook(_unified_v11_generator_forward_hook, with_kwargs=True) diff --git a/particlegan/_unified_v11/loss.py b/particlegan/_unified_v11/loss.py index 475b45f74..7be14c336 100644 --- a/particlegan/_unified_v11/loss.py +++ b/particlegan/_unified_v11/loss.py @@ -615,16 +615,38 @@ def _resolve_pair_parts( parts_r = None return x_f, x_r, cond_f, cond_r, rec, has_cont_cond, parts_f, parts_r, captured + @staticmethod + def _scalar_pair_relativistic( + pos_logits: torch.Tensor, + neg_logits: torch.Tensor, + *, + detach_neg: bool = False, + ) -> Optional[torch.Tensor]: + b_p, b_n = int(pos_logits.shape[0]), int(neg_logits.shape[0]) + if b_p != b_n or b_p < 1: + return None + p_flat = pos_logits.reshape(b_p, -1).mean(dim=1) if pos_logits.ndim > 1 else pos_logits.reshape(b_p) + n_src = neg_logits.detach() if detach_neg else neg_logits + n_flat = n_src.reshape(b_n, -1).mean(dim=1) if n_src.ndim > 1 else n_src.reshape(b_n) + return F.softplus(n_flat - p_flat).mean() + def d_loss(self, real_logits: torch.Tensor, fake_logits: torch.Tensor) -> torch.Tensor: base = super().d_loss(real_logits, fake_logits) x_f, x_r, cond_f, cond_r, rec, has_cont_cond, _, _, captured = self._resolve_pair_parts( fake_logits, real_logits ) - if captured and rec is not None and hasattr(rec, "observe_margin_and_scale") and not has_cont_cond: - rec.observe_margin_and_scale(real_logits, fake_logits, x_r, x_f) + if rec is not None and hasattr(rec, "observe_margin_and_scale") and not has_cont_cond: + if captured: + rec.observe_margin_and_scale(real_logits, fake_logits, x_r, x_f) + else: + rec.observe_margin_and_scale(real_logits, fake_logits) if self.critic_weight == 0.0 and self.fibre_rel_blend == 0.0: return base if not captured: + if self.loss == "non_saturating" and self.t1c.weight > 0.0 and self.fibre_rel_blend > 0.0: + pair_d = self._scalar_pair_relativistic(real_logits, fake_logits) + if pair_d is not None: + return (1.0 - self.fibre_rel_blend) * base + self.fibre_rel_blend * pair_d return base if self.conditional_fibre == "slack" and has_cont_cond: if self.fibre_rel_blend > 0.0: @@ -672,7 +694,12 @@ def g_loss(self, fake_logits: torch.Tensor, real_logits: torch.Tensor | None = N fake_logits, real_logits ) if not captured: - return self.t1c.weight * super().g_loss(fake_logits, real_logits) + base_g = super().g_loss(fake_logits, real_logits) + if self.loss == "non_saturating" and self.fibre_rel_blend > 0.0: + pair_g = self._scalar_pair_relativistic(fake_logits, real_logits, detach_neg=True) + if pair_g is not None: + return self.t1c.weight * ((1.0 - self.fibre_rel_blend) * base_g + self.fibre_rel_blend * pair_g) + return self.t1c.weight * base_g b_f = fake_logits.shape[0] b_r = real_logits.shape[0] @@ -733,7 +760,12 @@ def g_loss(self, fake_logits: torch.Tensor, real_logits: torch.Tensor | None = N rec._t1c_disc_gate = 0.0 if rec is not None and hasattr(rec, "_simplicial_step"): rec._simplicial_step = False - return super().g_loss(fake_logits, real_logits) + base_g = super().g_loss(fake_logits, real_logits) + if self.loss == "non_saturating" and self.fibre_rel_blend > 0.0: + pair_g = self._scalar_pair_relativistic(fake_logits, real_logits, detach_neg=True) + if pair_g is not None: + return self.t1c.weight * ((1.0 - self.fibre_rel_blend) * base_g + self.fibre_rel_blend * pair_g) + return base_g # 3. H_0-connectedness-gated chart coupling + zero-eigh orphan macro-transport: allow_ell = torch.is_grad_enabled() and not getattr(rec, "_in_protected_probe", False) diff --git a/particlegan/_unified_v11/penalty.py b/particlegan/_unified_v11/penalty.py index 3c7624f9b..1bb5bc8c7 100644 --- a/particlegan/_unified_v11/penalty.py +++ b/particlegan/_unified_v11/penalty.py @@ -1,13 +1,22 @@ """Unified-v11 1st-Order Spatial-Frame Forward-Tangent & Multi-Probe Secant Born-Infeld Critic Penalty.""" from __future__ import annotations -from typing import Any, Dict +from typing import Any, Callable, Dict, Optional, Sequence, Tuple import torch from torch import nn +import torch.autograd.forward_ad as fwAD +import torch.nn.functional as F +from .._t2_critic.penalty import ( + _format_logits_2d, + _murmur32_signs, + _separable_image_unit_perturbation, +) from .._unified.penalty import ( + _GOLDEN_CONJ, _UnifiedPenaltyKernel, + _adaptive_fd_delta, _eval_lowd_forward_grad_sq, ) from ..k3p import CriticPenalty as _K3PCriticPenalty, K3PCriticAdam, _first_output @@ -20,14 +29,330 @@ from .optimizer import UnifiedV11EquilibratedOptimizer +def _universal_image_unit_tangent(x: torch.Tensor, phase: float, salt: int) -> torch.Tensor: + """Zero-RNG isotropic unit tangent ``||u_i||_2 = 1`` for any spatial tensor rank >= 3.""" + b = int(x.shape[0]) + d = max(1, int(x[0].numel())) + if d <= 8192: + ranks = torch.arange(b, device=x.device, dtype=torch.int64) + u = _murmur32_signs(ranks, d, phase, salt, x.device) + u = u / u.norm(dim=1, keepdim=True).clamp_min(1e-8) + return u.to(dtype=x.dtype).reshape_as(x) + return _separable_image_unit_perturbation(x, phase, salt=salt) + + +def _bn_dual_bf16_hook( + _module: nn.Module, + _inputs: Tuple[Any, ...], + output: Any, +) -> Any: + """Keep BatchNorm dual primal + tangent aligned in BF16 during autocast fwAD passes.""" + if isinstance(output, torch.Tensor): + primal, tangent = fwAD.unpack_dual(output) + if tangent is not None and (primal.dtype != torch.bfloat16 or tangent.dtype != torch.bfloat16): + return fwAD.make_dual(primal.to(dtype=torch.bfloat16), tangent.to(dtype=torch.bfloat16)) + return output + + class _UnifiedV11PenaltyKernel(_UnifiedPenaltyKernel): - """Unified-v11 1st-order forward-tangent & multi-probe secant Born-Infeld kernel (`0` `autograd.grad`).""" + """Unified-v11 universal 1st-order forward-AD Born-Infeld critic kernel (`0` `autograd.grad`).""" def __init__(self, *, record: Any = None, **kwargs: Any) -> None: rec = UnifiedV11StepRecord() if record is None else record super().__init__(record=rec, **kwargs) self.arm = "unified_v11" + def _eval_image_fwad_sq( + self, + live: Callable[[torch.Tensor], torch.Tensor], + x_probe: torch.Tensor, + phi: float, + *, + salt: int, + dim: int, + has_batchnorm: bool, + critic_module: Optional[nn.Module], + quantized_fprop: bool, + ) -> torch.Tensor: + """Evaluate ``dim * ||J_D(x) u||_2^2`` via 1st-order forward-mode AD (0 `autograd.grad`).""" + b_p = int(x_probe.shape[0]) + u_p = _universal_image_unit_tangent(x_probe, phi, salt=salt) + bn_modes = [] + bn_hooks = [] + use_bf16_dual_hook = bool( + has_batchnorm + and x_probe.is_cuda + and isinstance(critic_module, nn.Module) + and ( + bool(getattr(critic_module, "bf16", False)) + or ( + torch.is_autocast_enabled("cuda") + and torch.get_autocast_dtype("cuda") == torch.bfloat16 + ) + ) + ) + if has_batchnorm and isinstance(critic_module, nn.Module): + for mod in critic_module.modules(): + if isinstance(mod, torch.nn.modules.batchnorm._BatchNorm): + bn_modes.append((mod, mod.training)) + mod.eval() + if use_bf16_dual_hook: + bn_hooks.append(mod.register_forward_hook(_bn_dual_bf16_hook)) + try: + if not quantized_fprop: + try: + if use_bf16_dual_hook: + x_in = x_probe.to(dtype=torch.bfloat16) + u_in = u_p.to(dtype=torch.bfloat16) + else: + x_in = x_probe + u_in = u_p + with fwAD.dual_level(): + yd = live(fwAD.make_dual(x_in, u_in)) + _, jvp = fwAD.unpack_dual(yd) + if jvp is None: + raise NotImplementedError("dual tangent dropped") + j2d = _format_logits_2d(jvp.float(), b_p) + return float(dim) * j2d.square().mean(dim=-1) + except (RuntimeError, NotImplementedError, TypeError): + if use_bf16_dual_hook and isinstance(critic_module, nn.Module): + # Clean FP32 fwAD fallback if an exotic custom norm layer disagrees with BF16 duals. + for h in bn_hooks: + h.remove() + bn_hooks.clear() + prev_bf16 = bool(getattr(critic_module, "bf16", False)) + if prev_bf16: + critic_module.bf16 = False + try: + with torch.autocast(device_type="cuda", enabled=False): + with fwAD.dual_level(): + yd = live(fwAD.make_dual(x_probe.float(), u_p.float())) + _, jvp = fwAD.unpack_dual(yd) + if jvp is not None: + j2d = _format_logits_2d(jvp.float(), b_p) + return float(dim) * j2d.square().mean(dim=-1) + except (RuntimeError, NotImplementedError, TypeError): + pass + finally: + if prev_bf16: + critic_module.bf16 = True + # Fallback strictly for custom quantized/non-jvp ops: scale-aware symmetric central diff. + fd_delta = _adaptive_fd_delta(x_probe, quantized_fprop=True) + xp_p = (x_probe.float() + fd_delta * u_p.float()).to(dtype=x_probe.dtype).detach() + xp_m = (x_probe.float() - fd_delta * u_p.float()).to(dtype=x_probe.dtype).detach() + yp = _format_logits_2d(live(xp_p).float(), b_p) + ym = _format_logits_2d(live(xp_m).float(), b_p) + step_sq = (xp_p.float() - xp_m.float()).flatten(1).square().sum(dim=1).clamp_min(1e-12) + return float(dim) * ((yp - ym).square().mean(dim=-1) / step_sq) + finally: + for h in bn_hooks: + h.remove() + for mod, flag in bn_modes: + mod.training = flag + + def penalty( + self, + live: Callable[[torch.Tensor], torch.Tensor], + x_real: torch.Tensor, + x_fake: torch.Tensor, + step: int, + collect_stats: bool = False, + *, + is_conditioned: bool = False, + cond_tensors: Sequence[torch.Tensor] = (), + cond_sig: Tuple[Any, ...] = (), + allow_live_reuse: bool = True, + can_concat_batches: bool = False, + live_rep: Optional[Callable[[torch.Tensor, int], torch.Tensor]] = None, + quantized_fprop: bool = False, + has_batchnorm: bool = False, + critic_module: Optional[nn.Module] = None, + ) -> Tuple[torch.Tensor, Dict[str, Any]]: + is_image_tensor = x_real.ndim >= 3 and self.input_domain != "continuous" + if not is_image_tensor: + return super().penalty( + live, + x_real, + x_fake, + step, + collect_stats, + is_conditioned=is_conditioned, + cond_tensors=cond_tensors, + cond_sig=cond_sig, + allow_live_reuse=allow_live_reuse, + can_concat_batches=can_concat_batches, + live_rep=live_rep, + quantized_fprop=quantized_fprop, + ) + + if ( + self.coeff == 0.0 + or ( + self.real_weight == 0.0 + and self.fake_weight == 0.0 + and self.gamma_gauge == 0.0 + and self.chord_weight == 0.0 + ) + or (self.lazy_k > 1 and step % self.lazy_k != 0) + ): + if hasattr(self.record, "clear_transient_captures"): + self.record.clear_transient_captures() + zero = x_real.new_zeros(()) + return zero, ({"applied": False, "pen": 0.0, "center": self.kappa} if collect_stats else {}) + + self.record.calls += 1 + coefficient = self.coeff * self.lazy_k if self.lazy_k > 1 else self.coeff + b_r, b_f = int(x_real.shape[0]), int(x_fake.shape[0]) + dim = max(1, int(x_real[0].numel())) + + eff_beta = self.ebi_beta + if self.settle_beta > 0.0 and bool(getattr(self.record, "t1c_thermal_initialized", False)): + if self.settle_mode == "thermal": + s_th = float(getattr(self.record, "_thermal_scale", 1.0)) + settle_u = min(1.0, max(0.0, 1.0 - s_th)) + else: + res = float(getattr(self.record, "ema_residual", 1.0)) + settle_u = min(1.0, max(0.0, 1.0 - res / 0.22)) + eff_beta = eff_beta + self.settle_beta * (settle_u * settle_u) + + reused_endpoints = False + try: + phi = float(getattr(self.record, "t1c_chord_phase", 0.0)) + if hasattr(self.record, "t1c_chord_phase"): + self.record.t1c_chord_phase = (phi + _GOLDEN_CONJ) % 1.0 + + hit_r = self._lookup_live_endpoint(x_real, cond_sig) if allow_live_reuse else None + hit_f = ( + self._lookup_live_endpoint( + x_fake, + cond_sig, + exclude_out=hit_r[1] if hit_r is not None else None, + ) + if allow_live_reuse + else None + ) + reused_endpoints = bool(hit_r is not None and hit_f is not None) + + self.record._in_penalty = True + try: + if hit_r is not None: + z_r, d_r_raw = hit_r + z_r = z_r.detach().to(dtype=x_real.dtype) + else: + self.record._last_probe_actual_input = None + x_r_det = x_real.detach() + d_r_raw = live(x_r_det) + act_r = getattr(self.record, "_last_probe_actual_input", None) + z_r = ( + act_r.detach().to(dtype=x_real.dtype) + if isinstance(act_r, torch.Tensor) and act_r.shape == x_r_det.shape + else x_r_det + ) + + if hit_f is not None: + z_f, d_f_raw = hit_f + z_f = z_f.detach().to(dtype=x_fake.dtype) + else: + self.record._last_probe_actual_input = None + x_f_det = x_fake.detach() + d_f_raw = live(x_f_det) + act_f = getattr(self.record, "_last_probe_actual_input", None) + z_f = ( + act_f.detach().to(dtype=x_fake.dtype) + if isinstance(act_f, torch.Tensor) and act_f.shape == x_f_det.shape + else x_f_det + ) + + d_r_2d = _format_logits_2d(d_r_raw, b_r) + d_f_2d = _format_logits_2d(d_f_raw, b_f) + + # Universal 1st-order forward-AD probe across ALL image spatial dimensions (8x8..256x256): + # 1 interleaved half-real/half-fake batch (BN in eval -> zero cross-sample BN leakage). + if not is_conditioned and b_r == b_f and b_r >= 2: + parity = ((torch.arange(b_f, device=x_real.device) + int(phi * 1024.0)) % 2) == 0 + mask_nd = parity.reshape((b_f,) + (1,) * (x_real.ndim - 1)) + x_probe = torch.where(mask_nd, z_r, z_f) + sq_mix = self._eval_image_fwad_sq( + live, + x_probe, + phi, + salt=3, + dim=dim, + has_batchnorm=has_batchnorm, + critic_module=critic_module, + quantized_fprop=quantized_fprop, + ) + sq_r_raw = sq_mix[parity] + sq_f_raw = sq_mix[~parity] + else: + sq_r_raw = self._eval_image_fwad_sq( + live, + z_r, + phi, + salt=3, + dim=dim, + has_batchnorm=has_batchnorm, + critic_module=critic_module, + quantized_fprop=quantized_fprop, + ) + sq_f_raw = self._eval_image_fwad_sq( + live, + z_f, + phi, + salt=11, + dim=dim, + has_batchnorm=has_batchnorm, + critic_module=critic_module, + quantized_fprop=quantized_fprop, + ) + + # Universal per-sample Born-Infeld cap on unbiased 1-probe forward-AD ||grad_x D||^2: + norm_r = (sq_r_raw + 1e-12).sqrt() + norm_f = (sq_f_raw + 1e-12).sqrt() + total = x_real.new_zeros(()) + if self.real_weight > 0.0: + total = total + self.real_weight * F.softplus(norm_r - self.kappa, beta=eff_beta).square().mean() + if self.lambda_floor > 0.0: + total = total + self.real_weight * (0.0005 / float(dim)) * sq_r_raw.mean() + if self.fake_weight > 0.0: + total = total + self.fake_weight * F.softplus(norm_f - self.kappa, beta=eff_beta).square().mean() + if self.lambda_floor > 0.0: + total = total + self.fake_weight * (0.0005 / float(dim)) * sq_f_raw.mean() + if self.gamma_gauge > 0.0: + eff_gauge = min(self.gamma_gauge, 0.005) + mean_logit = 0.5 * (d_r_2d.mean() + d_f_2d.mean()) + total = total + eff_gauge * mean_logit.square() + norm_stat = 0.5 * (norm_r.detach().mean() + norm_f.detach().mean()) + finally: + self.record._in_penalty = False + + if hasattr(self.record, "observe_scale") and not getattr(self.record, "_capture_ring", None): + self.record.observe_scale(z_r, z_f) + + pen = 0.5 * coefficient * total + s_therm = float(getattr(self.record, "_thermal_scale", 1.0)) + if s_therm < 1.0 and pen.requires_grad: + pen = pen * s_therm + pen.detach() * (1.0 - s_therm) + + if not collect_stats: + return pen, {} + + pen_v, mean_norm = torch.stack((pen.detach().float(), norm_stat.float())).tolist() + ell_v = float(getattr(self.record, "t1c_ell_ema", 0.0)) + return pen, { + "applied": True, + "pen": float(pen_v), + "center": self.kappa, + "phase": self.arm, + "mean_grad_norm": float(mean_norm), + "t1c_ell": float(ell_v), + "s_thermal": float(s_therm), + "live_endpoints_reused": bool(reused_endpoints), + } + finally: + if hasattr(self.record, "clear_transient_captures"): + self.record.clear_transient_captures() + class UnifiedV11CriticPenalty(_K3PCriticPenalty): """Recipe-facing Unified-v11 1st-order Forward-Tangent & Multi-Probe Secant critic penalty.""" @@ -169,6 +494,8 @@ def live_rep(x: torch.Tensor, m: int) -> torch.Tensor: can_concat_batches=can_concat_batches, live_rep=live_rep, quantized_fprop=quantized_fprop, + has_batchnorm=self._has_batchnorm, + critic_module=critic, ) self.last_stats = stats if getattr(getattr(self, "recipe", None), "critic_step_mode", "none") == "finite_cap": diff --git a/tests/test_unified_v11.py b/tests/test_unified_v11.py index 97cfa12a8..1c4950b28 100644 --- a/tests/test_unified_v11.py +++ b/tests/test_unified_v11.py @@ -102,6 +102,50 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return self.head(h).squeeze(-1) +class _ToyImgBNDisc(nn.Module): + """4D image critic with a frozen eval() BatchNorm backbone + trainable BatchNorm head.""" + + def __init__(self) -> None: + super().__init__() + self.backbone = nn.Sequential( + nn.Conv2d(3, 8, kernel_size=3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(8), + nn.ReLU(), + ) + for p in self.backbone.parameters(): + p.requires_grad_(False) + self.backbone.eval() + self.head_conv = nn.Conv2d(8, 8, kernel_size=3, stride=2, padding=1, bias=False) + self.head_bn = nn.BatchNorm2d(8) + self.head_fc = nn.Linear(8, 1) + + def train(self, mode: bool = True) -> "_ToyImgBNDisc": + super().train(mode) + self.backbone.eval() + return self + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h = self.backbone(x) + h = nn.functional.leaky_relu(self.head_bn(self.head_conv(h)), negative_slope=0.2) + return self.head_fc(h.mean(dim=(2, 3))).squeeze(-1) + + +class _ToyImgBNGen(nn.Module): + """4D 3x32x32 generator with BatchNorm2d to verify BN-safe Universal Law 4 secant probe.""" + + def __init__(self, z_dim: int = 8) -> None: + super().__init__() + self.fc = nn.Linear(z_dim, 8 * 8 * 8) + self.bn = nn.BatchNorm2d(8) + self.deconv1 = nn.ConvTranspose2d(8, 8, kernel_size=4, stride=2, padding=1) + self.deconv2 = nn.ConvTranspose2d(8, 3, kernel_size=4, stride=2, padding=1) + + def forward(self, z: torch.Tensor) -> torch.Tensor: + h = nn.functional.silu(self.bn(self.fc(z).reshape(z.shape[0], 8, 8, 8))) + h = nn.functional.silu(self.deconv1(h)) + return torch.tanh(self.deconv2(h)) + + class _ToyCondGen(nn.Module): def __init__(self, z_dim: int = 4) -> None: super().__init__() @@ -552,3 +596,113 @@ def _do_step(g_m, d_m, p_m, og, od, lf, pf, batch): ref_bank = opt_d.record.cond_fiber_bank.to_packet() assert ref_bank["strata"][0]["n_eff"] == resumed["bank"]["strata"][0]["n_eff"] assert torch.equal(ref_bank["strata"][0]["ring_x"], resumed["bank"]["strata"][0]["ring_x"]) + + +def test_unified_v11_universal_4d_laws_semantic_controls_and_continuation(tmp_path: Path) -> None: + """Semantic controls + fresh-process continuation for Universal Laws 2, 3, and 4 on 4D 3x32x32 images.""" + torch.manual_seed(0) + rec = get_recipe( + "unified_v11", + prior_kind="mog", + standardize=False, + batch_size=8, + num_particles=16, + z_dim=8, + total_steps=20, + training_precision="float32", + ) + g = _ToyImgBNGen(z_dim=8) + d = _ToyImgBNDisc() + p = MoGParticlePrior(num_particles=16, z_dim=8, sigma=0.1, standardize=False) + init.deterministic_orthogonal_(g, seed=30) + init.deterministic_orthogonal_(d.head_conv, seed=31) + init.deterministic_orthogonal_(d.head_fc, seed=32) + init.deterministic_orthogonal_(p, seed=33) + + frozen_mean_before = d.backbone[1].running_mean.detach().clone() + frozen_var_before = d.backbone[1].running_var.detach().clone() + + def _forbid(*_a: object, **_kw: object) -> None: + raise AssertionError("Forbidden autograd.grad or spectral decomposition called in 4D unified_v11!") + + trainer = GANTrainer(rec, g, d, prior=p, seed=0) + batches = [torch.randn(8, 3, 32, 32, generator=torch.Generator().manual_seed(500 + i)) for i in range(6)] + with ( + mock.patch("torch.autograd.grad", side_effect=_forbid), + mock.patch("torch.linalg.svd", side_effect=_forbid), + mock.patch("torch.linalg.svdvals", side_effect=_forbid), + mock.patch("torch.linalg.eigh", side_effect=_forbid), + mock.patch("torch.linalg.eigvalsh", side_effect=_forbid), + mock.patch("torch.linalg.pinv", side_effect=_forbid), + ): + out = trainer.step(batches[0], collect_stats=True) + + # Law 2: live endpoints reused, frozen BN strictly unchanged & eval, trainable BN restored after penalty(): + assert out["penalty_stats"]["applied"] is True + assert out["penalty_stats"]["live_endpoints_reused"] is True + assert d.backbone[1].training is False + assert torch.equal(d.backbone[1].running_mean, frozen_mean_before) + assert torch.equal(d.backbone[1].running_var, frozen_var_before) + d.train() + assert d.head_bn.training is True and d.backbone[1].training is False + _ = trainer.penalty(d, batches[0], batches[1]) + assert d.head_bn.training is True and d.backbone[1].training is False + assert torch.equal(d.backbone[1].running_mean, frozen_mean_before) + assert torch.equal(d.backbone[1].running_var, frozen_var_before) + + # Law 3: margin & data-scale EMAs are live on 4D images and scalar logit coupling depends on real_logits: + rec_d = trainer.opt_d.record + assert rec_d.margin_initialized is True + assert rec_d.ema_data_scale > 5.0 + loss_fn = rec.make_loss() + r_log = torch.tensor([1.2, 0.8, -0.3, 0.5], requires_grad=True) + f_log = torch.tensor([-0.4, 0.2, 0.1, -0.7], requires_grad=True) + g1 = loss_fn.g_loss(f_log, r_log) + g2 = loss_fn.g_loss(f_log, r_log + 2.0) + assert float(g2.item()) > float(g1.item()) + 0.25 + ls_loss = rec.replace(loss="least_squares").make_loss() + assert torch.allclose(ls_loss.g_loss(f_log, r_log), 0.5 * (f_log - 1.0).square().mean()) + + # Law 4: generator forward hook captures element-budgeted 4D slice with BN-eval y_pre (zero shift at zero step): + z_probe = torch.randn(8, 8) + _ = g(z_probe) + assert rec_d._g_probe_cache is not None + _, p_args, p_kwargs, y_pre = rec_d._g_probe_cache + assert tuple(y_pre.shape) == (2, 3, 32, 32) + from particlegan._unified_v4.optimizer import _eval_probe_forward + y_post_zero_step = _eval_probe_forward(g, p_args, p_kwargs, rec_d) + assert y_post_zero_step is not None + assert float((y_post_zero_step - y_pre).abs().max().item()) == 0.0 + rec_d._g_probe_cache = None + + # Exact fresh-process continuation across 4D image + BN + Universal Laws 2/3/4: + for i in range(1, 3): + trainer.step(batches[i]) + ckpt_path = tmp_path / "ckpt_v11_img4d.pt" + out_path = tmp_path / "resumed_v11_img4d.pt" + torch.save({"state": trainer.state_dict(), "batches": batches[3:]}, ckpt_path) + for i in range(3, 6): + trainer.step(batches[i]) + sample_ref = trainer.sample(4) + d_ref = trainer.D.state_dict() + + script = f""" +import torch +from particlegan import GANTrainer, MoGParticlePrior, Recipe +from tests.test_unified_v11 import _ToyImgBNGen, _ToyImgBNDisc + +pl = torch.load({str(ckpt_path)!r}, weights_only=False) +state = pl["state"] +rec = Recipe.from_dict(state["recipe"]) +tr = GANTrainer(rec, _ToyImgBNGen(8), _ToyImgBNDisc(), + prior=MoGParticlePrior(num_particles=16, z_dim=8, sigma=0.1, standardize=False), seed=77) +tr.load_state_dict(state) +for b in pl["batches"]: + tr.step(b) +torch.save({{"sample": tr.sample(4), "d": tr.D.state_dict()}}, {str(out_path)!r}) +""" + subprocess.run([sys.executable, "-c", script], check=True) + resumed = torch.load(out_path, weights_only=False) + assert torch.equal(sample_ref, resumed["sample"]) + for k, v in d_ref.items(): + assert torch.equal(v, resumed["d"][k]), f"4D D mismatch {k}" From 2420a3232034f7a64fa93a84f49054fb09a64685 Mon Sep 17 00:00:00 2001 From: Martyn Garcia Date: Fri, 20 Nov 2026 12:36:43 -0700 Subject: [PATCH 2/3] feat(unified_v11): integrate universal function-space equilibrated optimizer (Finding 1 + Finding 2) --- particlegan/_unified_v11/optimizer.py | 302 ++++++++++++++++---------- tests/test_unified_v11.py | 83 +++++++ 2 files changed, 273 insertions(+), 112 deletions(-) diff --git a/particlegan/_unified_v11/optimizer.py b/particlegan/_unified_v11/optimizer.py index 66af02090..f1dddfde9 100644 --- a/particlegan/_unified_v11/optimizer.py +++ b/particlegan/_unified_v11/optimizer.py @@ -1,24 +1,32 @@ """Zero-SVD / Zero-`pinv` / Zero-`autograd.grad` Unified-v11 Equilibrated Optimizer (`unified_v11_equil`). -Builds on ``UnifiedV4EquilibratedOptimizer`` with four first-principles improvements: +Builds on ``UnifiedV4EquilibratedOptimizer`` with five first-principles improvements: 1. **Strictly Zero-`autograd.grad` / Zero-`pinv` Analytic Direction Blend** (``direction_blend=True``): Computes protected-loss normals via standard ``loss.backward(retain_graph=True)`` (``0`` ``torch.autograd.grad``) and projects conflicting 1- or 2-objective displacements using closed-form ``1x1`` / ``2x2`` Gram algebra (``0`` ``torch.linalg.pinv``), restoring ``1.0`` own-pad accuracy on ``residual_student`` without adding any SVD or double-backward primitive. -2. **Direct Particle Cloud Row Smoothing Parity (``t3_iso_scale_mass_8x``)**: - Applies scaled Tikhonov row smoothing ``hypot(d_norm, s_p_row)`` on ``prior`` groups uniformly - (including direct clouds ``total_gen_mats == 0``) with ``therm_p = 1.0``, preserving small-mass/small-scale - component covariance without premature direct-cloud damping. -3. **Thermal-Safe & Simplicial-Safe Secant Trust Band (``interp_saddle_barycentric``, ``precision_gaussian_nvfp4``)**: - Restricts ``delta_g_min`` amplification strictly to non-simplicial, non-thermally-cooled (``s_th >= 0.90``), - off-equilibrium hostile initializations (``push_res > 0.25``), preventing ``_band_scale`` from fighting - thermal settling or amplifying 4-bit NVFP4 quantization jitter. -4. **Cross-Batch Conditional Bank Alignment & Deep Residual Trust Parity**: - Removes redundant ``b1_eff = 0.82`` optimizer lag on conditional tasks (already cross-batch smoothed by - ``UnifiedV11CondFiberBank``) and aligns deep-network (``deep_gate > 0``) depth scaling (``3.5 / L``, - ``b1_eff = 0.80``) with ``t4_arch`` while bypassing the shallow ``delta_g_max`` secant cap on deep - 16-layer networks (``grid100_res16_tanh``). +2. **Universal Function-Space Conditioning (Zero ``L_count`` / ``deep_gate`` Discontinuities)**: + Replaces parameter-count heuristics (``depth_scale = 3.2 / L``, ``deep_gate``, ``trust_mult``) + with a checkpointed realized function-space conditioning EMA (``_ema_func_gain``) driven by the + unclamped Law 4 secant probe ratio (``free_ratio = (delta_max / delta_obs) * pre_gain``). + Well-conditioned 2D MLPs, LayerNorm Transformers, and CNNs stay in the deadband + (``pre_gain = 1.0``, ``stiff_gate = 0.0``, exact ``1 : 1.5 : 2.9`` G : D : prior equilibrium + regardless of player matrix-count asymmetry), while pathologically stiff unnormalized deep + networks smoothly attenuate pre-step LR and engage momentum-coherence damping (``stiff_gate``), + and recover smoothly toward ``1.0`` once stiffness subsides (no one-way ratchet). +3. **Degree-0 Scale-Invariant ``peak_rms`` Tikhonov Smoothing (Zero GPU-CPU Sync)**: + Tracks a per-parameter exponential peak singular-value RMS on device (``peak_rms``) and caps + Tikhonov smoothing at ``min(s_base, 0.03 * peak_rms)`` (passed as batched device tensors into + ``quintic_orthogonalize_2d`` / ``quintic_orthogonalize_batched``), preventing small high-D or + early-layer gradients from being crushed while preserving equilibrium settling. +4. **Exact ``1x1`` Conv2d / ConvTranspose2d RMS-to-RMS Scaling**: + Applies ``sqrt(out_ch / in_ch)`` uniformly to all 4D convolutional weights (including ``1x1`` + projections such as ResNet feature heads), matching ``DualNorm`` exactly. +5. **Direct Particle Cloud Row Smoothing Parity & Thermal/Simplicial-Safe Secant Band**: + Applies scale-invariant Tikhonov row smoothing ``hypot(d_norm, s_p_row)`` on ``prior`` groups + uniformly and restricts ``delta_g_min`` wakeup strictly to non-simplicial, non-thermally-cooled + off-equilibrium steps (``push_res > 0.18``, ``s_therm > 0.10``) with zero dimension thresholds. """ from __future__ import annotations @@ -57,7 +65,7 @@ def quintic_orthogonalize_2d( mat: torch.Tensor, steps: int = 5, *, - smoothing: float = 0.0, + smoothing: float | torch.Tensor = 0.0, ) -> torch.Tensor: """Zero-SVD scale-normalized 5-step Quintic polar (exact FP32 Tikhonov for ``m<=64``, BF16 for ``65..128``).""" if mat.ndim != 2: @@ -67,14 +75,20 @@ def quintic_orthogonalize_2d( return torch.zeros_like(mat) orig_dtype = mat.dtype x_f32 = mat.float() - s_val = float(smoothing) + if isinstance(smoothing, torch.Tensor): + s_ten = smoothing.to(device=mat.device, dtype=torch.float32).reshape(()).clamp_min(0.0) + has_smooth = True + else: + s_val = float(smoothing) + has_smooth = s_val > 0.0 + s_ten = x_f32.new_tensor(s_val) if has_smooth else None max_abs = x_f32.abs().amax() finite_mat = torch.isfinite(x_f32).all() - if s_val > 0.0 and min(r_dim, c_dim) <= 128: - scale_ref = torch.maximum(torch.where(finite_mat, max_abs, max_abs.new_zeros(())), max_abs.new_tensor(s_val)) + if has_smooth and s_ten is not None and min(r_dim, c_dim) <= 512: + scale_ref = torch.maximum(torch.where(finite_mat, max_abs, max_abs.new_zeros(())), s_ten) safe_max = torch.where(scale_ref > 0.0, scale_ref, torch.ones_like(scale_ref)) x_scaled = torch.where(finite_mat, x_f32 / safe_max, torch.zeros_like(x_f32)) - s_scaled = s_val / safe_max + s_scaled = s_ten / safe_max frob = x_scaled.norm() active = finite_mat & (frob > 1e-30) if r_dim == 1 or c_dim == 1: @@ -85,7 +99,7 @@ def quintic_orthogonalize_2d( m_dim, n_dim = int(x_in.shape[0]), int(x_in.shape[1]) aug_frob = torch.sqrt(frob.square() + float(m_dim) * s_scaled.square()).clamp_min(1e-30) inv_aug = 1.0 / (aug_frob * 1.02) - work_dtype = torch.bfloat16 if (mat.is_cuda and m_dim >= 64) else torch.float32 + work_dtype = torch.bfloat16 if (mat.is_cuda and m_dim > 256) else torch.float32 x_part = (x_in * inv_aug).to(dtype=work_dtype) y_part = _cached_eye(m_dim, mat.device, work_dtype) * (s_scaled * inv_aug).to(dtype=work_dtype) # Fuse Z = [X, Y] so Z @ Z.T = X X^T + Y Y^T and poly @ Z = [poly @ X, poly @ Y] in half the matmuls: @@ -99,7 +113,7 @@ def quintic_orthogonalize_2d( x_out = z_f32[:, :n_dim] if transposed: x_out = x_out.T - out = x_out * (math.sqrt(float(m_dim)) / joint_frob) + out = x_out * (math.sqrt(float(m_dim)) / joint_frob).clamp_max(1.15) out = torch.where(active & torch.isfinite(out).all(), out, torch.zeros_like(out)) return out.to(orig_dtype) @@ -128,9 +142,9 @@ def quintic_orthogonalize_2d( out = x.float() target_frob = math.sqrt(float(min(r_dim, c_dim))) out_frob = out.norm().clamp_min(1e-8) - if s_val > 0.0: + if has_smooth and s_ten is not None: g_rms = (frob * safe_max) / max(target_frob, 1.0) - s_scale = g_rms / torch.hypot(g_rms, g_rms.new_tensor(s_val)).clamp_min(1e-30) + s_scale = g_rms / torch.hypot(g_rms, s_ten).clamp_min(1e-30) out = out * ((target_frob / out_frob) * s_scale) else: out = out * (target_frob / out_frob) @@ -143,7 +157,7 @@ def quintic_orthogonalize_batched( mats: torch.Tensor, steps: int = 5, *, - smoothing: float = 0.0, + smoothing: float | torch.Tensor = 0.0, ) -> torch.Tensor: """Batched zero-SVD 5-step Quintic polar (exact FP32 Tikhonov for ``m<=64``, BF16 for ``65..128``).""" if mats.ndim != 3: @@ -153,14 +167,20 @@ def quintic_orthogonalize_batched( return torch.zeros_like(mats) orig_dtype = mats.dtype x_f32 = mats.float() - s_val = float(smoothing) + if isinstance(smoothing, torch.Tensor): + s_ten = smoothing.to(device=mats.device, dtype=torch.float32).reshape(b_sz, 1, 1).clamp_min(0.0) + has_smooth = True + else: + s_val = float(smoothing) + has_smooth = s_val > 0.0 + s_ten = x_f32.new_full((b_sz, 1, 1), s_val) if has_smooth else None max_abs = x_f32.abs().amax(dim=(1, 2), keepdim=True) finite_mat = torch.isfinite(x_f32).all(dim=(1, 2), keepdim=True) - if s_val > 0.0 and min(r_dim, c_dim) <= 128: - scale_ref = torch.maximum(torch.where(finite_mat, max_abs, torch.zeros_like(max_abs)), max_abs.new_tensor(s_val)) + if has_smooth and s_ten is not None and min(r_dim, c_dim) <= 512: + scale_ref = torch.maximum(torch.where(finite_mat, max_abs, torch.zeros_like(max_abs)), s_ten) safe_max = torch.where(scale_ref > 0.0, scale_ref, torch.ones_like(scale_ref)) x_scaled = torch.where(finite_mat, x_f32 / safe_max, torch.zeros_like(x_f32)) - s_scaled = s_val / safe_max + s_scaled = s_ten / safe_max frob = x_scaled.flatten(1).norm(dim=1).view(b_sz, 1, 1) active = finite_mat & (frob > 1e-30) if r_dim == 1 or c_dim == 1: @@ -171,7 +191,7 @@ def quintic_orthogonalize_batched( m_dim, n_dim = int(x_in.shape[1]), int(x_in.shape[2]) aug_frob = torch.sqrt(frob.square() + float(m_dim) * s_scaled.square()).clamp_min(1e-30) inv_aug = 1.0 / (aug_frob * 1.02) - work_dtype = torch.bfloat16 if (mats.is_cuda and m_dim >= 64) else torch.float32 + work_dtype = torch.bfloat16 if (mats.is_cuda and m_dim > 256) else torch.float32 x_part = (x_in * inv_aug).to(dtype=work_dtype) eye_b = _cached_eye(m_dim, mats.device, work_dtype).unsqueeze(0).expand(b_sz, m_dim, m_dim) y_part = eye_b * (s_scaled * inv_aug).to(dtype=work_dtype) @@ -185,7 +205,7 @@ def quintic_orthogonalize_batched( x_out = z_f32[:, :, :n_dim] if transposed: x_out = x_out.transpose(1, 2) - out = x_out * (math.sqrt(float(m_dim)) / joint_frob) + out = x_out * (math.sqrt(float(m_dim)) / joint_frob).clamp_max(1.15) out_ok = active & torch.isfinite(out).all(dim=(1, 2), keepdim=True) out = torch.where(out_ok, out, torch.zeros_like(out)) return out.to(orig_dtype) @@ -215,9 +235,9 @@ def quintic_orthogonalize_batched( out = x.float() target_frob = math.sqrt(float(min(r_dim, c_dim))) out_frob = out.flatten(1).norm(dim=1).view(b_sz, 1, 1).clamp_min(1e-8) - if s_val > 0.0: + if has_smooth and s_ten is not None: g_rms = (frob * safe_max) / max(target_frob, 1.0) - s_scale = g_rms / torch.hypot(g_rms, g_rms.new_tensor(s_val)).clamp_min(1e-30) + s_scale = g_rms / torch.hypot(g_rms, s_ten).clamp_min(1e-30) out = out * ((target_frob / out_frob) * s_scale) else: out = out * (target_frob / out_frob) @@ -371,6 +391,7 @@ def __init__( self._v4_prior_scale: float = 1.0 self._origin_norm: Optional[float] = None self.last_step_gain: float = 1.0 + self._ema_func_gain: float = 1.0 self._protected: Optional[torch.Tensor] = None self._protected_single: bool = False @@ -589,60 +610,52 @@ def step(self, closure: Any = None) -> Any: if self._origin_norm is not None and self._origin_norm < 0.05: origin_damping = min(1.0, self._origin_norm / 0.05) - n_mat_role: Dict[str, int] = {} - total_gen_mats = 0 - for grp in self.param_groups: - rl = grp["role"] - pl = self._player(rl) - for p in grp["params"]: - if pl != "prior" and p.ndim >= 2: - n_mat_role[rl] = n_mat_role.get(rl, 0) + 1 - if pl == "generator": - total_gen_mats += 1 + # Universal Function-Space Conditioning (replaces syntactic L_count / deep_gate): + # Well-conditioned MLPs, LayerNorm Transformers, and CNNs stay at ema_func_gain ~= 1.0 + # (pre_gain = 1.0, stiff_gate = 0.0), preserving exact G : D : prior equilibrium + # regardless of player matrix-count asymmetry. Pathologically stiff unnormalized deep + # networks (>3x function-space overshoot) smoothly lower ema_func_gain pre-step and + # activate momentum-coherence damping via stiff_gate. + ema_fg = float(self._ema_func_gain) + stiff_gate_raw = _smoothstep01((0.40 - ema_fg) / 0.20) + pre_gain_base = 1.0 if ema_fg >= 0.75 else ema_fg applied_deltas: List[Tuple[torch.Tensor, torch.Tensor]] = [] pending_dense: Dict[ - Tuple[int, int, float], - List[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, bool, bool]], + Tuple[int, int, bool], + List[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, bool, bool]], ] = {} pending_conv: Dict[ - Tuple[int, int, int, int, bool, int], - List[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, bool]], + Tuple[int, int, int, int, bool, int, bool], + List[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, bool]], ] = {} + pre_gain_applied = 1.0 for grp in self.param_groups: role = grp["role"] player = self._player(role) + stiff_gate = stiff_gate_raw if player == "generator" else 0.0 b1, b2 = grp["betas"] eps = float(grp["eps"]) - L_raw = max(1, sum(1 for p in grp["params"] if p.ndim >= 2)) if player != "prior" else 1 - L_count = 4 if (player == "generator" and L_raw == 2 and total_gen_mats >= 4) else L_raw - deep_gate = _smoothstep01((float(L_count) - 5.0) / 4.0) - s_eff = self.smoothing * (1.0 - deep_gate) - # Direct particle clouds (total_gen_mats == 0) retain scaled row smoothing matching DualNorm: - s_prior = self.smoothing if self._v4_prior_scale >= 1.0 else 0.0 + s_base = self.smoothing * (1.0 - stiff_gate) + # Direct particle clouds retain scaled row smoothing matching DualNorm: + s_prior_base = self.smoothing if self._v4_prior_scale >= 1.0 else 0.0 if player == "prior": b1_eff = max(float(b1), 0.75) if self._v4_prior_scale < 1.0 else float(b1) - elif deep_gate > 0.0: - b1_eff = max(float(b1), 0.75 * deep_gate) + elif stiff_gate > 0.0: + b1_eff = max(float(b1), 0.75 * stiff_gate) else: b1_eff = float(b1) b2_eff = min(max(float(b2), 0.95), 0.999) - c_zero = 1.0 / math.sqrt(float(max(1, L_count))) if player == "generator": - depth_scale = ( - min(1.0, (3.2 / float(max(3, L_count))) * self._v4_wakeup_boost) - if deep_gate > 0.0 - else 1.0 - ) - eff_depth = (1.0 - deep_gate) + deep_gate * depth_scale - base_lr = float(grp["lr"]) * step_scale * self._v4_g_scale * eff_depth + pre_g = min(1.0, pre_gain_base * self._v4_wakeup_boost) + pre_gain_applied = pre_g + base_lr = float(grp["lr"]) * step_scale * self._v4_g_scale * pre_g elif player == "critic": - depth_scale = (3.2 / float(max(3, L_count))) if deep_gate > 0.0 else 1.0 - eff_depth = (1.0 - deep_gate) + deep_gate * depth_scale - base_lr = float(grp["lr"]) * step_scale * eff_depth + pre_gain_applied = pre_gain_base + base_lr = float(grp["lr"]) * step_scale * pre_gain_base else: base_lr = float(grp["lr"]) * step_scale * self._v4_prior_scale @@ -677,9 +690,18 @@ def step(self, closure: Any = None) -> Any: g_norm = gr_safe.norm(dim=-1, keepdim=True) d_norm = dir_r.norm(dim=-1, keepdim=True) active_r = fin_row & (g_norm > 1e-36) & (d_norm > 1e-36) - if s_prior > 0.0: - s_p_row = s_prior * (128.0 / float(len(rows))) if len(rows) > 128 else s_prior - denom_r = torch.hypot(d_norm, d_norm.new_tensor(s_p_row)).clamp_min(1e-36) + + # Degree-0 scale-invariant peak-RMS tracking on prior rows (no GPU-CPU sync): + cur_row_rms = d_norm.square().mean().sqrt() + if "peak_rms" not in st: + st["peak_rms"] = torch.zeros((), device=p.device, dtype=torch.float32) + peak_r = torch.maximum(st["peak_rms"] * 0.9995, torch.where(torch.isfinite(cur_row_rms), cur_row_rms, st["peak_rms"].new_zeros(()))) + st["peak_rms"].copy_(torch.where(torch.isfinite(peak_r), peak_r, st["peak_rms"])) + + if s_prior_base > 0.0: + s_cap = s_prior_base * (128.0 / float(len(rows))) if len(rows) > 128 else s_prior_base + s_p_row = torch.minimum(d_norm.new_tensor(s_cap), peak_r * 0.03) + denom_r = torch.hypot(d_norm, s_p_row).clamp_min(1e-36) else: denom_r = d_norm.clamp_min(1e-36) unit_r = torch.where( @@ -702,8 +724,39 @@ def step(self, closure: Any = None) -> Any: g_max = gf_safe.abs().amax() g_active = finite & (g_max > 1e-38) + is_conv = (p.ndim == 4) and (id(p) in self._conv_layouts or p.shape[2] > 1 or p.shape[3] > 1) + if is_conv: + c0, c1, kh, kw = (int(s) for s in p.shape) + is_trans, groups_c = self._conv_layouts.get(id(p), (False, 1)) + out_ch = c1 * groups_c if is_trans else c0 + in_ch = c0 if is_trans else c1 * groups_c + cur_sv_rms = gf_safe.norm() / math.sqrt(float(max(1, min(out_ch, in_ch) * kh * kw))) + elif p.ndim >= 2: + r_d = int(p.shape[0]) + c_d = int(p.numel() // max(1, r_d)) + cur_sv_rms = gf_safe.norm() / math.sqrt(float(max(1, min(r_d, c_d)))) + else: + cur_sv_rms = gf_safe.norm() + + # Degree-0 scale-invariant peak-RMS tracking (zero GPU-CPU sync, prevents vanishing-layer + # and high-D small-gradient crushing while preserving equilibrium settling): + if "peak_rms" not in st: + st["peak_rms"] = torch.zeros((), device=p.device, dtype=torch.float32) + peak_new = torch.maximum( + st["peak_rms"] * 0.9995, + torch.where(finite, cur_sv_rms, st["peak_rms"].new_zeros(())), + ) + st["peak_rms"].copy_(torch.where(finite, peak_new, st["peak_rms"])) + + has_smooth = s_base > 0.0 + s_eff_t = ( + torch.minimum(gf_safe.new_tensor(s_base), peak_new * 0.03) + if has_smooth + else gf_safe.new_zeros(()) + ) + is_scalar_1d = p.ndim == 1 and p.numel() == 1 - use_moments = (b1_eff > 0.0) or (deep_gate > 0.0) or is_scalar_1d + use_moments = (b1_eff > 0.0) or (stiff_gate > 0.0) or is_scalar_1d if use_moments: b1_use = max(b1_eff, 0.80) if is_scalar_1d else b1_eff if "m_unit" not in st: @@ -746,61 +799,53 @@ def step(self, closure: Any = None) -> Any: ) if is_scalar_1d: coh = raw_coh.clamp(0.35, 1.0) - elif deep_gate > 0.0: - coh = (1.0 - deep_gate) + deep_gate * raw_coh + elif stiff_gate > 0.0: + coh = (1.0 - stiff_gate) + stiff_gate * raw_coh else: coh = torch.ones_like(m_norm) - raw_dir = (m_hat * sc) if s_eff > 0.0 else m_hat + raw_dir = (m_hat * sc) if has_smooth else m_hat direc = torch.where(g_active, raw_dir, torch.zeros_like(m_hat)) else: - raw_dir = gf_safe if s_eff > 0.0 else (gf_safe / g_max.clamp_min(1e-38)) + raw_dir = gf_safe if has_smooth else (gf_safe / g_max.clamp_min(1e-38)) direc = torch.where(g_active, raw_dir, torch.zeros_like(gf_safe)) coh = gf_safe.new_ones(()) if rate_p <= 0.0: continue - if p.ndim == 4 and (p.shape[2] > 1 or p.shape[3] > 1): + if is_conv and (p.shape[2] > 1 or p.shape[3] > 1): c0, c1, kh, kw = (int(s) for s in p.shape) is_trans, groups_c = self._conv_layouts.get(id(p), (False, 1)) out_ch = c1 * groups_c if is_trans else c0 in_ch = c0 if is_trans else c1 * groups_c factor = math.sqrt(float(out_ch) / float(max(1, in_ch))) / float(kh * kw) - if deep_gate > 0.0: - w_op = 1.0 / math.sqrt(float(math.sqrt(float(max(1, out_ch * in_ch))) * float(kh * kw))) - w_rms = p.float().square().mean().sqrt() - r_ratio = (w_rms / max(w_op, 1e-8)).clamp(c_zero, 2.5) - trust_mult = (1.0 - deep_gate) + deep_gate * r_ratio - else: - trust_mult = direc.new_ones(()) - if s_eff > 0.0: - g_slice_rms = (direc.norm() / math.sqrt(float(max(1, min(out_ch, in_ch) * kh * kw)))).clamp_min(0.0) - conv_smooth = g_slice_rms / torch.hypot(g_slice_rms, g_slice_rms.new_tensor(s_eff)).clamp_min(1e-30) - trust_mult = trust_mult * conv_smooth - step_mult = (-rate_p * factor) * coh * trust_mult - pending_conv.setdefault((c0, c1, kh, kw, is_trans, groups_c), []).append( - (p, direc, step_mult, track_secant) + step_mult = (-rate_p * factor) * coh + pending_conv.setdefault((c0, c1, kh, kw, is_trans, groups_c, has_smooth), []).append( + (p, direc, step_mult, s_eff_t, track_secant) ) elif p.ndim >= 2: mat = direc.reshape(p.shape[0], -1) r_d, c_d = int(mat.shape[0]), int(mat.shape[1]) - factor = math.sqrt(max(1.0, float(r_d) / float(max(1, c_d)))) - if deep_gate > 0.0: - w_op = 1.0 / math.sqrt(float(max(1, c_d))) - w_rms = p.float().square().mean().sqrt() - r_ratio = (w_rms / max(w_op, 1e-8)).clamp(c_zero, 2.5) - trust_mult = (1.0 - deep_gate) + deep_gate * r_ratio + if is_conv: + # 1x1 Conv2d / ConvTranspose2d: exact RMS-to-RMS sqrt(out_ch / in_ch) matching DualNorm: + c0, c1, _, _ = (int(s) for s in p.shape) + is_trans, groups_c = self._conv_layouts.get(id(p), (False, 1)) + out_ch = c1 * groups_c if is_trans else c0 + in_ch = c0 if is_trans else c1 * groups_c + factor = math.sqrt(float(out_ch) / float(max(1, in_ch))) else: - trust_mult = direc.new_ones(()) - step_mult = (-rate_p * factor) * coh * trust_mult + factor = math.sqrt(max(1.0, float(r_d) / float(max(1, c_d)))) + step_mult = (-rate_p * factor) * coh needs_t = r_d > c_d mat_canon = mat.T if needs_t else mat - m_key = (int(mat_canon.shape[0]), int(mat_canon.shape[1]), s_eff) - pending_dense.setdefault(m_key, []).append((p, mat_canon, step_mult, track_secant, needs_t)) + m_key = (int(mat_canon.shape[0]), int(mat_canon.shape[1]), has_smooth) + pending_dense.setdefault(m_key, []).append( + (p, mat_canon, step_mult, s_eff_t, track_secant, needs_t) + ) else: d_norm = direc.norm() - if s_eff > 0.0: - denom_1d = torch.hypot(d_norm, d_norm.new_tensor(s_eff)).clamp_min(eps) + if has_smooth: + denom_1d = torch.hypot(d_norm, s_eff_t).clamp_min(eps) else: denom_1d = d_norm + eps unit_1d = torch.where( @@ -813,10 +858,10 @@ def step(self, closure: Any = None) -> Any: if track_secant: applied_deltas.append((p, delta_1d)) - for (_, _, s_bucket), bucket in pending_dense.items(): + for (_, _, has_s_b), bucket in pending_dense.items(): if len(bucket) == 1: - p_i, mat_i, mult_i, trk_i, needs_t = bucket[0] - orth_i = quintic_orthogonalize_2d(mat_i, steps=5, smoothing=s_bucket).float() + p_i, mat_i, mult_i, s_i, trk_i, needs_t = bucket[0] + orth_i = quintic_orthogonalize_2d(mat_i, steps=5, smoothing=(s_i if has_s_b else 0.0)).float() if needs_t: orth_i = orth_i.T delta_i = (orth_i * mult_i).reshape_as(p_i).to(p_i.dtype) @@ -825,31 +870,36 @@ def step(self, closure: Any = None) -> Any: applied_deltas.append((p_i, delta_i)) else: mats = torch.stack([b_item[1] for b_item in bucket], dim=0) - orths = quintic_orthogonalize_batched(mats, steps=5, smoothing=s_bucket).float() - for idx_b, (p_i, _, mult_i, trk_i, needs_t) in enumerate(bucket): + s_batch = torch.stack([b_item[3] for b_item in bucket], dim=0) if has_s_b else 0.0 + orths = quintic_orthogonalize_batched(mats, steps=5, smoothing=s_batch).float() + for idx_b, (p_i, _, mult_i, _, trk_i, needs_t) in enumerate(bucket): orth_b = orths[idx_b].T if needs_t else orths[idx_b] delta_i = (orth_b * mult_i).reshape_as(p_i).to(p_i.dtype) p_i.add_(delta_i) if trk_i: applied_deltas.append((p_i, delta_i)) - for (c0, c1, kh, kw, is_trans, groups_c), bucket in pending_conv.items(): + for (c0, c1, kh, kw, is_trans, groups_c, has_s_b), bucket in pending_conv.items(): n_b = len(bucket) k_slices = groups_c * kh * kw per_c0 = c0 // groups_c slices_list = [] - for _, d_t, _, _ in bucket: + s_slices_list = [] + for _, d_t, _, s_i, _ in bucket: st_view = d_t.reshape(groups_c, per_c0, c1, kh, kw) if is_trans: sl = st_view.permute(0, 3, 4, 2, 1).reshape(k_slices, c1, per_c0) else: sl = st_view.permute(0, 3, 4, 1, 2).reshape(k_slices, per_c0, c1) slices_list.append(sl) + if has_s_b: + s_slices_list.append(s_i.expand(k_slices)) all_slices = torch.cat(slices_list, dim=0) - orth_all = quintic_orthogonalize_batched(all_slices, steps=5).float() + s_conv_batch = torch.cat(s_slices_list, dim=0) if has_s_b else 0.0 + orth_all = quintic_orthogonalize_batched(all_slices, steps=5, smoothing=s_conv_batch).float() r_sl, c_sl = (c1, per_c0) if is_trans else (per_c0, c1) orth_layers = orth_all.view(n_b, groups_c, kh, kw, r_sl, c_sl) - for idx_b, (p_i, _, mult_i, trk_i) in enumerate(bucket): + for idx_b, (p_i, _, mult_i, _, trk_i) in enumerate(bucket): if is_trans: orth_i = orth_layers[idx_b].permute(0, 4, 3, 1, 2).reshape(c0, c1, kh, kw) else: @@ -862,6 +912,8 @@ def step(self, closure: Any = None) -> Any: # Law 4 Two-Player Function-Space Secant Trust Band (bypassed on thermally-cooled simplicial # manifolds, continuous-conditioned fibers, and 2-loss direction_blend steps): alpha_gain = 1.0 + probe_ran = False + free_ratio = 1.0 if ( rec is not None and applied_deltas @@ -879,6 +931,7 @@ def step(self, closure: Any = None) -> Any: if d_post is not None and d_post.numel() == d_pre.numel(): sq_diffs.append((d_post.reshape(-1) - d_pre.reshape(-1)).square().mean()) if sq_diffs: + probe_ran = True delta_d = float(torch.stack(sq_diffs).mean().sqrt().item()) ema_m = float(getattr(rec, "ema_margin", 0.0)) if getattr(rec, "margin_initialized", False) else 0.0 inv_gap = min(25.0, max(0.0, -ema_m - 1.5)) @@ -886,6 +939,8 @@ def step(self, closure: Any = None) -> Any: delta_d_min = 0.12 * inv_gap * th_d delta_d_max = (0.55 * step_scale + 0.20 * inv_gap) * th_d alpha_gain = _band_scale(delta_d, delta_d_min, delta_d_max, alpha_max=64.0) + if math.isfinite(delta_d) and delta_d > 1e-12: + free_ratio = (delta_d_max / delta_d) * pre_gain_applied else: probe_cache = getattr(rec, "_g_probe_cache", None) rec._g_probe_cache = None @@ -893,23 +948,34 @@ def step(self, closure: Any = None) -> Any: g_mod, p_args, p_kwargs, y_pre = probe_cache y_post = _eval_probe_forward(g_mod, p_args, p_kwargs, rec) if y_post is not None and y_post.shape == y_pre.shape: + probe_ran = True out_dim = max(1, int(y_pre[0].numel())) delta_rms = float((y_post - y_pre).square().mean().sqrt().item()) data_scale = float(getattr(rec, "ema_data_scale", 1.0)) coord_scale = max(0.05, data_scale / math.sqrt(float(out_dim))) delta_rel = delta_rms / coord_scale push_res = float(getattr(rec, "ema_push_res", 0.0)) - delta_g_max = (0.045 + 0.105 * min(1.5, max(0.25, push_res))) * step_scale + ema_m = float(getattr(rec, "ema_margin", 0.0)) if getattr(rec, "margin_initialized", False) else 0.0 + eff_res = max(push_res, min(1.5, abs(ema_m))) + delta_g_max = (0.22 + 0.18 * min(1.5, eff_res)) * step_scale if (not is_simp) and push_res > 0.18 and s_therm > 0.10: off_gate = _smoothstep01((push_res - 0.18) / 0.25) delta_g_min = 0.012 * off_gate * step_scale else: delta_g_min = 0.0 alpha_gain = _band_scale(delta_rel, delta_g_min, delta_g_max, alpha_max=48.0) + if math.isfinite(delta_rel) and delta_rel > 1e-12: + free_ratio = (delta_g_max / delta_rel) * pre_gain_applied if alpha_gain != 1.0: adj = alpha_gain - 1.0 for p_i, d_i in applied_deltas: p_i.add_(d_i, alpha=adj) + if probe_ran: + # Update checkpointed realized function-space conditioning EMA using the unclamped + # free_ratio (no one-way ratchet: recovers smoothly to 1.0 when stiffness subsides): + target_fg = max(0.05, min(1.0, free_ratio)) + beta_fg = 0.70 if target_fg < self._ema_func_gain else 0.90 + self._ema_func_gain = float(beta_fg * self._ema_func_gain + (1.0 - beta_fg) * target_fg) elif rec is not None and not is_critic_opt: rec._g_probe_cache = None @@ -971,6 +1037,7 @@ def state_dict(self) -> Dict[str, Any]: "schema": 1, "family": self.family, "last_step_gain": float(self.last_step_gain), + "ema_func_gain": float(self._ema_func_gain), "direction_blend": bool(self.direction_blend), "sampled_rows": [ self._sampled_rows.get(grp["params"][0]) if grp["algorithm"] == "unified_prior" else None @@ -1002,7 +1069,7 @@ def validate_state_dict(self, saved: Any) -> None: if not isinstance(saved, dict) or set(saved) != expected_keys: raise ValueError("invalid UnifiedV11EquilibratedOptimizer checkpoint schema") meta = saved["unified_v11_equil"] - expected_meta_keys = {"schema", "family", "last_step_gain", "direction_blend", "sampled_rows"} + expected_meta_keys = {"schema", "family", "last_step_gain", "ema_func_gain", "direction_blend", "sampled_rows"} if self.direction_blend: expected_meta_keys.add("constraint_geometry") if ( @@ -1020,6 +1087,14 @@ def validate_state_dict(self, saved: Any) -> None: gain = meta["last_step_gain"] if isinstance(gain, bool) or not isinstance(gain, (int, float)) or not math.isfinite(gain) or float(gain) <= 0.0: raise ValueError("invalid UnifiedV11EquilibratedOptimizer last_step_gain") + ema_fg = meta["ema_func_gain"] + if ( + isinstance(ema_fg, bool) + or not isinstance(ema_fg, (int, float)) + or not math.isfinite(ema_fg) + or not (0.0 < float(ema_fg) <= 1.0) + ): + raise ValueError("invalid UnifiedV11EquilibratedOptimizer ema_func_gain") if self.direction_blend: cg = meta["constraint_geometry"] if ( @@ -1049,11 +1124,13 @@ def validate_state_dict(self, saved: Any) -> None: step_v = st_val.get("step", 0) if type(step_v) is not int or step_v < 0: raise ValueError("invalid UnifiedV11EquilibratedOptimizer step counter") - for t_key in ("m_unit", "m_scale", "v_rms", "v_scale", "m_prior"): + for t_key in ("m_unit", "m_scale", "v_rms", "v_scale", "m_prior", "peak_rms"): if t_key in st_val: ten = st_val[t_key] if not isinstance(ten, torch.Tensor) or not bool(torch.isfinite(ten).all()): raise ValueError(f"non-finite or invalid tensor {t_key} in UnifiedV11EquilibratedOptimizer state") + if t_key == "peak_rms" and (ten.ndim != 0 or float(ten.item()) < 0.0): + raise ValueError("invalid peak_rms tensor in UnifiedV11EquilibratedOptimizer state") if hasattr(self, "record") and self.record is not None: reg = saved["regularizer"] if ( @@ -1073,6 +1150,7 @@ def load_state_dict(self, state_dict: Dict[str, Any]) -> None: self.clear_sampled_rows() meta = state_dict["unified_v11_equil"] self.last_step_gain = float(meta["last_step_gain"]) + self._ema_func_gain = float(meta["ema_func_gain"]) for grp, rows in zip(self.param_groups, meta["sampled_rows"]): if rows is not None and grp["algorithm"] == "unified_prior": self.set_sampled_rows(grp["params"][0], rows.to(grp["params"][0].device)) diff --git a/tests/test_unified_v11.py b/tests/test_unified_v11.py index 1c4950b28..29de08cb8 100644 --- a/tests/test_unified_v11.py +++ b/tests/test_unified_v11.py @@ -421,6 +421,89 @@ def test_unified_v11_deliberate_damage_and_tamper_rejection() -> None: with pytest.raises(ValueError, match="v_push"): trainer.load_state_dict(bad_vpush) + bad_ema_fg_missing = deepcopy(good) + del bad_ema_fg_missing["optimizers"][0]["unified_v11_equil"]["ema_func_gain"] + with pytest.raises(ValueError, match="invalid checkpoint optimizer state|metadata"): + trainer.load_state_dict(bad_ema_fg_missing) + + for bad_val in (0.0, -0.2, 1.5, float("nan")): + bad_ema_fg = deepcopy(good) + bad_ema_fg["optimizers"][0]["unified_v11_equil"]["ema_func_gain"] = bad_val + with pytest.raises(ValueError, match="invalid checkpoint optimizer state|ema_func_gain"): + trainer.load_state_dict(bad_ema_fg) + + first_state_key = next(iter(good["optimizers"][0]["state"])) + for bad_peak in (torch.tensor(-1.0), torch.tensor([1.0]), torch.tensor(float("nan"))): + bad_pr = deepcopy(good) + bad_pr["optimizers"][0]["state"][first_state_key]["peak_rms"] = bad_peak + with pytest.raises(ValueError, match="invalid checkpoint optimizer state|peak_rms"): + trainer.load_state_dict(bad_pr) + + +def test_unified_v11_optimizer_group_split_asymmetry_and_conv1x1_invariance() -> None: + """Semantic controls: zero L_count/deep_gate discontinuity, asymmetric player equilibrium, and 1x1 conv scaling.""" + torch.manual_seed(7) + + # 1. Splitting 16 matrices into 1 group vs 4 groups produces identical updates (zero L_count dependence): + g_single = _DeepGen16(z_dim=2, width=16, out_dim=2) + g_split = deepcopy(g_single) + x_in = torch.randn(8, 2) + g_single(x_in).square().mean().backward() + g_split(x_in).square().mean().backward() + + opt_single = UnifiedV11EquilibratedOptimizer( + [{"params": list(g_single.parameters()), "lr": 0.014, "role": "generator"}], + direction_blend=False, + ) + split_params = list(g_split.parameters()) + chunk = max(1, len(split_params) // 4) + opt_split = UnifiedV11EquilibratedOptimizer( + [ + {"params": split_params[i : i + chunk], "lr": 0.014, "role": "generator"} + for i in range(0, len(split_params), chunk) + ], + direction_blend=False, + ) + opt_single.step() + opt_split.step() + for p_a, p_b in zip(g_single.parameters(), g_split.parameters()): + assert torch.allclose(p_a, p_b, atol=1e-6, rtol=1e-6) + + # 2. 1x1 Conv2d (out=16, in=64) via public make_critic_optimizer uses exact RMS-to-RMS sqrt(out_ch / in_ch) = 0.5: + conv1x1 = nn.Conv2d(64, 16, kernel_size=1, bias=False) + nn.init.zeros_(conv1x1.weight) + rec_c = get_recipe("unified_v11", lr=0.02, d_lr_mult=1.0, training_precision="float32") + opt_c = rec_c.make_critic_optimizer(conv1x1) + opt_c.smoothing = 0.0 + conv1x1.weight.grad = torch.randn_like(conv1x1.weight) + opt_c.step() + # Orthogonalized 16x64 matrix has Frobenius norm sqrt(16) = 4.0; scaled by lr * sqrt(16/64) = 0.02 * 0.5 = 0.01: + expected_frob = 0.02 * (16.0 / 64.0) ** 0.5 * (16.0**0.5) + got_frob = float(conv1x1.weight.detach().norm().item()) + assert abs(got_frob - expected_frob) / expected_frob < 0.05 + + # 3. _ema_func_gain attenuates on function-space overshoot and recovers smoothly upward (no one-way ratchet): + rec = get_recipe( + "unified_v11", + prior_kind="mog", + standardize=False, + batch_size=8, + num_particles=16, + z_dim=2, + total_steps=20, + training_precision="float32", + ) + g_m = _ToyGen2D(2, 2) + d_m = _ToyDisc2D(2) + p_m = MoGParticlePrior(num_particles=16, z_dim=2, sigma=0.05, standardize=False) + trainer = GANTrainer(rec, g_m, d_m, prior=p_m, seed=0) + # Seed a depressed ema_func_gain as after transient deep stiffness, then step well-conditioned 2-layer G: + trainer.opt_g._ema_func_gain = 0.25 + before_fg = trainer.opt_g._ema_func_gain + for _ in range(8): + trainer.step(torch.randn(8, 2)) + assert trainer.opt_g._ema_func_gain > before_fg + 0.25 + @pytest.mark.parametrize("mode", ["mog_2d", "cond_ddgan"]) def test_unified_v11_fresh_process_continuation_and_import_purity(tmp_path: Path, mode: str) -> None: From 13775ca3c5a61ebedb54dbf1a93187ce6f7093c9 Mon Sep 17 00:00:00 2001 From: Martyn Garcia Date: Fri, 20 Nov 2026 13:21:01 -0700 Subject: [PATCH 3/3] fix(unified_v11): envelope-tracked ema_func_gain & active-probe recovery (fixes deep-16 + N=200 FID 434) --- particlegan/_unified_v11/optimizer.py | 30 ++++++++++++++++++--------- tests/test_unified_v11.py | 4 ++-- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/particlegan/_unified_v11/optimizer.py b/particlegan/_unified_v11/optimizer.py index f1dddfde9..8520671d1 100644 --- a/particlegan/_unified_v11/optimizer.py +++ b/particlegan/_unified_v11/optimizer.py @@ -617,7 +617,7 @@ def step(self, closure: Any = None) -> Any: # networks (>3x function-space overshoot) smoothly lower ema_func_gain pre-step and # activate momentum-coherence damping via stiff_gate. ema_fg = float(self._ema_func_gain) - stiff_gate_raw = _smoothstep01((0.40 - ema_fg) / 0.20) + stiff_gate_raw = _smoothstep01((0.60 - ema_fg) / 0.30) pre_gain_base = 1.0 if ema_fg >= 0.75 else ema_fg applied_deltas: List[Tuple[torch.Tensor, torch.Tensor]] = [] @@ -631,6 +631,7 @@ def step(self, closure: Any = None) -> Any: ] = {} pre_gain_applied = 1.0 + coh_vals: List[torch.Tensor] = [] for grp in self.param_groups: role = grp["role"] player = self._player(role) @@ -812,6 +813,8 @@ def step(self, closure: Any = None) -> Any: if rate_p <= 0.0: continue + if track_secant: + coh_vals.append(coh) if is_conv and (p.shape[2] > 1 or p.shape[3] > 1): c0, c1, kh, kw = (int(s) for s in p.shape) @@ -914,6 +917,7 @@ def step(self, closure: Any = None) -> Any: alpha_gain = 1.0 probe_ran = False free_ratio = 1.0 + step_active_probe = False if ( rec is not None and applied_deltas @@ -941,6 +945,7 @@ def step(self, closure: Any = None) -> Any: alpha_gain = _band_scale(delta_d, delta_d_min, delta_d_max, alpha_max=64.0) if math.isfinite(delta_d) and delta_d > 1e-12: free_ratio = (delta_d_max / delta_d) * pre_gain_applied + step_active_probe = math.isfinite(delta_d) and delta_d >= 0.05 else: probe_cache = getattr(rec, "_g_probe_cache", None) rec._g_probe_cache = None @@ -955,9 +960,7 @@ def step(self, closure: Any = None) -> Any: coord_scale = max(0.05, data_scale / math.sqrt(float(out_dim))) delta_rel = delta_rms / coord_scale push_res = float(getattr(rec, "ema_push_res", 0.0)) - ema_m = float(getattr(rec, "ema_margin", 0.0)) if getattr(rec, "margin_initialized", False) else 0.0 - eff_res = max(push_res, min(1.5, abs(ema_m))) - delta_g_max = (0.22 + 0.18 * min(1.5, eff_res)) * step_scale + delta_g_max = (0.045 + 0.16 * min(1.5, push_res)) * step_scale if (not is_simp) and push_res > 0.18 and s_therm > 0.10: off_gate = _smoothstep01((push_res - 0.18) / 0.25) delta_g_min = 0.012 * off_gate * step_scale @@ -965,17 +968,24 @@ def step(self, closure: Any = None) -> Any: delta_g_min = 0.0 alpha_gain = _band_scale(delta_rel, delta_g_min, delta_g_max, alpha_max=48.0) if math.isfinite(delta_rel) and delta_rel > 1e-12: - free_ratio = (delta_g_max / delta_rel) * pre_gain_applied + mean_coh = float(torch.stack(coh_vals).mean().item()) if coh_vals else 1.0 + free_ratio = (delta_g_max / delta_rel) * pre_gain_applied * max(0.05, mean_coh) + step_active_probe = math.isfinite(delta_rel) and delta_rel >= 0.012 if alpha_gain != 1.0: adj = alpha_gain - 1.0 for p_i, d_i in applied_deltas: p_i.add_(d_i, alpha=adj) if probe_ran: - # Update checkpointed realized function-space conditioning EMA using the unclamped - # free_ratio (no one-way ratchet: recovers smoothly to 1.0 when stiffness subsides): - target_fg = max(0.05, min(1.0, free_ratio)) - beta_fg = 0.70 if target_fg < self._ema_func_gain else 0.90 - self._ema_func_gain = float(beta_fg * self._ema_func_gain + (1.0 - beta_fg) * target_fg) + # Envelope-tracked function-space conditioning EMA: responds rapidly on stiffness + # overshoots (alpha_gain < 1.0) and recovers slowly on well-conditioned active steps + # (never bouncing upward at equilibrium on stiff 16-layer MLPs, yet no one-way ratchet): + if alpha_gain < 1.0: + target_fg = max(0.05, min(1.0, pre_gain_applied * alpha_gain)) + self._ema_func_gain = float(0.70 * self._ema_func_gain + 0.30 * target_fg) + elif step_active_probe and free_ratio > 1.5: + target_fg = max(0.05, min(1.0, free_ratio)) + if target_fg > self._ema_func_gain: + self._ema_func_gain = float(0.999 * self._ema_func_gain + 0.001 * target_fg) elif rec is not None and not is_critic_opt: rec._g_probe_cache = None diff --git a/tests/test_unified_v11.py b/tests/test_unified_v11.py index 29de08cb8..1b5371240 100644 --- a/tests/test_unified_v11.py +++ b/tests/test_unified_v11.py @@ -500,9 +500,9 @@ def test_unified_v11_optimizer_group_split_asymmetry_and_conv1x1_invariance() -> # Seed a depressed ema_func_gain as after transient deep stiffness, then step well-conditioned 2-layer G: trainer.opt_g._ema_func_gain = 0.25 before_fg = trainer.opt_g._ema_func_gain - for _ in range(8): + for _ in range(12): trainer.step(torch.randn(8, 2)) - assert trainer.opt_g._ema_func_gain > before_fg + 0.25 + assert trainer.opt_g._ema_func_gain > before_fg @pytest.mark.parametrize("mode", ["mog_2d", "cond_ddgan"])