diff --git a/benchmarks/cifar_toy/__init__.py b/benchmarks/cifar_toy/__init__.py new file mode 100644 index 000000000..6f5f69bee --- /dev/null +++ b/benchmarks/cifar_toy/__init__.py @@ -0,0 +1,71 @@ +"""Fast, reproducible CIFAR-style 32x32 Transformer-G / ResNet18-feature-D benchmark. + +Matches the structural properties of HyperGAN's ``cifar10-tiny-transformer`` +(4D ``(B, 3, 32, 32)`` RGB tensor with ``dim = 3072 > 1024``, 2-block 64-token +Transformer generator with 16 matrix/conv parameters vs. frozen ImageNet +ResNet-18 64px critic with 3 trainable feature heads and 9 matrix/conv +parameters, MoG particle prior in ``z_dim = 64``) while executing purely +through ParticleGAN's public ``Recipe`` / ``MoGParticlePrior`` / ``GANTrainer`` +API under protocol seed 0. +""" +from .data import ( + BatchSequenceStream, + CIFAR10Dataset, + discover_cifar10_root, + load_cifar10_dataset, + make_synthetic_cifar_dataset, +) +from .metrics import ( + compute_class_coverage, + compute_critic_diagnostics, + compute_fid, + compute_kid, + compute_pixel_diversity_ratio, +) +from .models import ( + DEFAULT_RESNET18_WEIGHTS, + DEFAULT_RESNET34_WEIGHTS, + RESNET18_SHA256, + RESNET34_SHA256, + ResNet18FeatureCritic32, + ResNet34FeatureEvaluator, + TinyTransformerGenerator32, + count_matrix_conv_params, + count_trainable_params, + parameter_state_digest, +) +from .runner import ( + DEFAULT_MOG_SIGMA, + BenchmarkConfig, + format_leaderboard_markdown, + run_benchmark, + summarize_run_directory, +) + +__all__ = [ + "BatchSequenceStream", + "BenchmarkConfig", + "CIFAR10Dataset", + "DEFAULT_MOG_SIGMA", + "DEFAULT_RESNET18_WEIGHTS", + "DEFAULT_RESNET34_WEIGHTS", + "RESNET18_SHA256", + "RESNET34_SHA256", + "ResNet18FeatureCritic32", + "ResNet34FeatureEvaluator", + "TinyTransformerGenerator32", + "compute_class_coverage", + "compute_critic_diagnostics", + "compute_fid", + "compute_kid", + "compute_pixel_diversity_ratio", + "count_matrix_conv_params", + "count_trainable_params", + "discover_cifar10_root", + "format_leaderboard_markdown", + "load_cifar10_dataset", + "make_synthetic_cifar_dataset", + "parameter_state_digest", + "run_benchmark", + "summarize_run_directory", +] diff --git a/benchmarks/cifar_toy/__main__.py b/benchmarks/cifar_toy/__main__.py new file mode 100644 index 000000000..7f7dd52e3 --- /dev/null +++ b/benchmarks/cifar_toy/__main__.py @@ -0,0 +1,146 @@ +"""CLI entrypoint: ``python -m benchmarks.cifar_toy``.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any, Dict, List + +from .models import DEFAULT_RESNET18_WEIGHTS, DEFAULT_RESNET34_WEIGHTS +from .runner import ( + DEFAULT_MOG_SIGMA, + BenchmarkConfig, + format_leaderboard_markdown, + run_benchmark, + summarize_run_directory, +) + + +def _parse_override(text: str) -> tuple[str, Any]: + if "=" not in text: + raise argparse.ArgumentTypeError(f"Expected KEY=VALUE for --recipe-override, got {text!r}") + key, raw = text.split("=", 1) + key = key.strip() + raw = raw.strip() + try: + val = json.loads(raw) + except json.JSONDecodeError: + val = raw + return key, val + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="python -m benchmarks.cifar_toy", + description="Fast reproducible CIFAR-style 32x32 Transformer-G / ResNet18-feature-D ParticleGAN benchmark.", + ) + p.add_argument( + "--recipes", + nargs="+", + default=["bispf_optimized", "empf", "unified_v11"], + help="ParticleGAN recipe presets to compare (default: bispf_optimized empf unified_v11).", + ) + p.add_argument( + "--num-particles", + nargs="+", + type=int, + default=[200], + help="One or more MoG prior particle counts (e.g. --num-particles 200 16384).", + ) + p.add_argument("--z-dim", type=int, default=64) + p.add_argument("--mog-sigma", type=float, default=DEFAULT_MOG_SIGMA) + p.add_argument("--no-standardize-prior", action="store_true") + p.add_argument("--batch-size", type=int, default=128) + p.add_argument("--steps", type=int, default=800) + p.add_argument("--eval-every", type=int, default=200) + p.add_argument("--eval-samples", type=int, default=2048) + p.add_argument("--eval-ref-samples", type=int, default=5000) + p.add_argument("--eval-critic-samples", type=int, default=256) + p.add_argument("--seed", type=int, default=0) + p.add_argument("--dataset", choices=("cifar10", "synthetic"), default="cifar10") + p.add_argument("--data-root", type=str, default=None) + p.add_argument("--no-horizontal-flip", action="store_true") + p.add_argument("--backbone-mode", choices=("pretrained", "random"), default="pretrained") + p.add_argument("--resnet18-weights", type=str, default=str(DEFAULT_RESNET18_WEIGHTS)) + p.add_argument("--resnet34-weights", type=str, default=str(DEFAULT_RESNET34_WEIGHTS)) + p.add_argument("--device", type=str, default="cuda:0") + p.add_argument("--deterministic-algorithms", action="store_true") + p.add_argument( + "--recipe-override", + action="append", + default=[], + help="Explicit recipe field override KEY=VALUE (JSON-parsed; repeatable).", + ) + p.add_argument("--run-tag", type=str, default=None, help="Explicit run directory name (single run only).") + p.add_argument( + "--output-dir", + type=str, + default="runs/cifar_toy_bench", + help="Directory for bulk logs, JSONL traces, receipts, and leaderboard.md.", + ) + p.add_argument( + "--summarize-only", + action="store_true", + help="Do not train; rebuild leaderboard.md from existing receipts in --output-dir.", + ) + return p + + +def main(argv: List[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + out_dir = Path(args.output_dir) + + if args.summarize_only: + receipts, md = summarize_run_directory(out_dir) + if not receipts: + print(f"No receipts found under {out_dir}") + return 1 + print(md, end="") + return 0 + + overrides: Dict[str, Any] = {} + for item in args.recipe_override: + k, v = _parse_override(item) + overrides[k] = v + + combos = [(r, n) for r in args.recipes for n in args.num_particles] + if args.run_tag is not None and len(combos) != 1: + parser.error("--run-tag can only be used with a single (recipe, num_particles) combination") + + receipts = [] + for recipe_name, num_p in combos: + cfg = BenchmarkConfig( + recipe=recipe_name, + num_particles=num_p, + z_dim=args.z_dim, + mog_sigma=args.mog_sigma, + standardize_prior=not args.no_standardize_prior, + batch_size=args.batch_size, + steps=args.steps, + eval_every=args.eval_every, + eval_samples=args.eval_samples, + eval_ref_samples=args.eval_ref_samples, + eval_critic_samples=args.eval_critic_samples, + seed=args.seed, + dataset=args.dataset, + data_root=args.data_root, + horizontal_flip=not args.no_horizontal_flip, + backbone_mode=args.backbone_mode, + resnet18_weights=args.resnet18_weights, + resnet34_weights=args.resnet34_weights, + device=args.device, + deterministic_algorithms=args.deterministic_algorithms, + recipe_overrides=overrides, + run_tag=args.run_tag, + ) + receipts.append(run_benchmark(cfg, out_dir)) + + _, md = summarize_run_directory(out_dir) + print("\n### Current Leaderboard\n", flush=True) + print(format_leaderboard_markdown(receipts), end="", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/cifar_toy/data.py b/benchmarks/cifar_toy/data.py new file mode 100644 index 000000000..b6bda1008 --- /dev/null +++ b/benchmarks/cifar_toy/data.py @@ -0,0 +1,279 @@ +"""Deterministic CIFAR-10 and synthetic 10-class 32x32 RGB data streams. + +Zero ``torchvision`` dependency: loads ``cifar-10-batches-py`` directly from +local cache using Python's standard ``pickle`` module. Provides isolated +checkpointable CPU batch streams for critic and generator target draws plus an +exact SHA-256 batch-sequence digest so every trainer sees identical batches. +""" +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import os +from pathlib import Path +import pickle +from typing import Optional, Sequence, Tuple + +import numpy as np +import torch + + +@dataclass(frozen=True) +class CIFAR10Dataset: + """In-memory uint8 ``(N, 3, 32, 32)`` dataset + integer labels ``(N,)``.""" + + images_uint8: torch.Tensor + labels: torch.Tensor + source_name: str + source_path: str + num_classes: int = 10 + + @property + def size(self) -> int: + return int(self.images_uint8.shape[0]) + + def to_float_tensor(self, indices: torch.Tensor, *, flip_mask: Optional[torch.Tensor] = None) -> torch.Tensor: + """Convert indexed uint8 images to float32 in ``[-1, 1]`` on CPU.""" + imgs = self.images_uint8[indices].to(dtype=torch.float32).div_(127.5).sub_(1.0) + if flip_mask is not None and bool(flip_mask.any()): + imgs = torch.where(flip_mask.view(-1, 1, 1, 1), imgs.flip(-1), imgs) + return imgs + + +_CANDIDATE_CIFAR_ROOTS = ( + Path("/nvme/dev/hypergan/HyperGAN/data"), + Path.home() / ".hypergan" / "cache" / "cifar10", + Path("data"), +) + + +def discover_cifar10_root(explicit_root: Optional[Path] = None) -> Optional[Path]: + """Locate a local directory containing ``cifar-10-batches-py/data_batch_1``.""" + candidates: list[Path] = [] + if explicit_root is not None: + candidates.append(Path(explicit_root).expanduser()) + env_root = os.environ.get("PARTICLEGAN_CIFAR10_ROOT") + if env_root: + candidates.append(Path(env_root).expanduser()) + candidates.extend(_CANDIDATE_CIFAR_ROOTS) + + for root in candidates: + if (root / "data_batch_1").is_file(): + return root + sub = root / "cifar-10-batches-py" + if (sub / "data_batch_1").is_file(): + return sub + return None + + +def load_cifar10_dataset( + data_root: Optional[Path] = None, + *, + split: str = "train", +) -> CIFAR10Dataset: + """Load real CIFAR-10 ``train`` (50k) or ``test`` (10k) images as uint8 NCHW.""" + batch_dir = discover_cifar10_root(data_root) + if batch_dir is None: + raise FileNotFoundError( + "CIFAR-10 cache not found. Pass --data-root pointing to cifar-10-batches-py " + "or use --dataset synthetic for the hermetic synthetic cohort." + ) + if split == "train": + files = [batch_dir / f"data_batch_{i}" for i in range(1, 6)] + elif split == "test": + files = [batch_dir / "test_batch"] + else: + raise ValueError(f"split must be 'train' or 'test', got {split!r}") + + chunks = [] + labels = [] + for path in files: + if not path.is_file(): + raise FileNotFoundError(f"Missing CIFAR-10 batch file: {path}") + with open(path, "rb") as fh: + entry = pickle.load(fh, encoding="latin1") + data = np.asarray(entry["data"], dtype=np.uint8).reshape(-1, 3, 32, 32) + chunks.append(data) + labels.extend(int(y) for y in entry["labels"]) + + images_np = np.concatenate(chunks, axis=0) + return CIFAR10Dataset( + images_uint8=torch.from_numpy(images_np.copy()), + labels=torch.tensor(labels, dtype=torch.long), + source_name=f"cifar10_{split}", + source_path=str(batch_dir.resolve()), + num_classes=10, + ) + + +def make_synthetic_cifar_dataset( + num_samples: int = 2000, + *, + seed: int = 0, + num_classes: int = 10, +) -> CIFAR10Dataset: + """Deterministic 10-class structured 32x32 RGB image cohort for hermetic tests. + + Each class has a distinct color palette + geometric spatial pattern + (vertical/horizontal/diagonal bars, quadrants, center disk, frame, checker, + radial gradient) with small deterministic per-sample jitter so FID/KID and + class coverage are well-conditioned. + """ + gen = torch.Generator(device="cpu").manual_seed(seed + 7919) + yy, xx = torch.meshgrid( + torch.linspace(-1.0, 1.0, 32), + torch.linspace(-1.0, 1.0, 32), + indexing="ij", + ) + palettes = torch.tensor( + [ + [0.90, 0.15, 0.15], + [0.15, 0.85, 0.20], + [0.15, 0.30, 0.92], + [0.92, 0.85, 0.15], + [0.85, 0.20, 0.85], + [0.15, 0.85, 0.88], + [0.95, 0.52, 0.15], + [0.55, 0.25, 0.85], + [0.25, 0.65, 0.45], + [0.82, 0.82, 0.85], + ], + dtype=torch.float32, + ) + imgs = torch.empty((num_samples, 3, 32, 32), dtype=torch.float32) + labels = torch.empty((num_samples,), dtype=torch.long) + + for i in range(num_samples): + cls = i % num_classes + labels[i] = cls + shift_x = float(torch.empty(()).uniform_(-0.22, 0.22, generator=gen).item()) + shift_y = float(torch.empty(()).uniform_(-0.22, 0.22, generator=gen).item()) + x = xx - shift_x + y = yy - shift_y + if cls == 0: + mask = (x.abs() < 0.35).float() + elif cls == 1: + mask = (y.abs() < 0.35).float() + elif cls == 2: + mask = ((x + y).abs() < 0.40).float() + elif cls == 3: + mask = ((x - y).abs() < 0.40).float() + elif cls == 4: + mask = ((x * x + y * y) < 0.28).float() + elif cls == 5: + r2 = x * x + y * y + mask = ((r2 > 0.14) & (r2 < 0.48)).float() + elif cls == 6: + mask = ((x > 0.0) & (y > 0.0)).float() + elif cls == 7: + mask = ((x < 0.0) & (y < 0.0)).float() + elif cls == 8: + mask = (torch.sin(6.0 * x) * torch.sin(6.0 * y) > 0.0).float() + else: + mask = ((x.abs() > 0.55) | (y.abs() > 0.55)).float() + + fg = palettes[cls % len(palettes)].view(3, 1, 1) + bg = (1.0 - fg) * 0.25 + 0.12 + noise = torch.randn((3, 32, 32), generator=gen) * 0.05 + img = (bg * (1.0 - mask.unsqueeze(0)) + fg * mask.unsqueeze(0) + noise).clamp(0.0, 1.0) + imgs[i] = img + + images_uint8 = imgs.mul(255.0).round().clamp(0, 255).to(torch.uint8) + return CIFAR10Dataset( + images_uint8=images_uint8, + labels=labels, + source_name=f"synthetic_cifar10_n{num_samples}_seed{seed}", + source_path="in_memory", + num_classes=num_classes, + ) + + +class BatchSequenceStream: + """Isolated deterministic critic/generator target batch streams with exact digest. + + Critic and generator target batches use separate CPU generators seeded from + the protocol seed so every compared trainer receives the exact same ordered + sequence of ``(real_d, real_g)`` batches. + """ + + def __init__( + self, + dataset: CIFAR10Dataset, + batch_size: int, + *, + seed: int = 0, + horizontal_flip: bool = True, + ) -> None: + self.dataset = dataset + self.batch_size = int(batch_size) + self.horizontal_flip = bool(horizontal_flip) + self.critic_gen = torch.Generator(device="cpu").manual_seed(seed + 101) + self.gen_gen = torch.Generator(device="cpu").manual_seed(seed + 202) + self._hasher = hashlib.sha256() + self.steps_drawn = 0 + + def _draw_one(self, rng: torch.Generator) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + idx = torch.randint(0, self.dataset.size, (self.batch_size,), generator=rng) + flips = None + if self.horizontal_flip: + flips = torch.randint(0, 2, (self.batch_size,), generator=rng, dtype=torch.bool) + batch = self.dataset.to_float_tensor(idx, flip_mask=flips) + return batch, idx, flips + + def next_pair(self, device: torch.device) -> Tuple[torch.Tensor, torch.Tensor]: + real_d, idx_d, flip_d = self._draw_one(self.critic_gen) + real_g, idx_g, flip_g = self._draw_one(self.gen_gen) + self._hasher.update(idx_d.numpy().tobytes()) + self._hasher.update(idx_g.numpy().tobytes()) + if flip_d is not None and flip_g is not None: + self._hasher.update(flip_d.numpy().tobytes()) + self._hasher.update(flip_g.numpy().tobytes()) + self.steps_drawn += 1 + return real_d.to(device=device, non_blocking=False), real_g.to(device=device, non_blocking=False) + + @property + def digest(self) -> str: + return self._hasher.hexdigest() + + def state_dict(self) -> dict: + return { + "critic_gen": self.critic_gen.get_state(), + "gen_gen": self.gen_gen.get_state(), + "steps_drawn": self.steps_drawn, + } + + +def select_eval_reference_subset( + dataset: CIFAR10Dataset, + count: int, + *, + seed: int = 0, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Deterministically select a balanced reference subset ``(images_float, labels)``.""" + n = min(int(count), dataset.size) + rng = torch.Generator(device="cpu").manual_seed(seed + 303) + perm = torch.randperm(dataset.size, generator=rng)[:n] + imgs = dataset.to_float_tensor(perm, flip_mask=None) + lbls = dataset.labels[perm].clone() + return imgs, lbls + + +def compute_class_centroids(features: torch.Tensor, labels: torch.Tensor, num_classes: int = 10) -> torch.Tensor: + """Compute per-class mean feature vectors ``(num_classes, D)``.""" + feat_f = features.float() + centroids = torch.zeros((num_classes, feat_f.shape[1]), dtype=torch.float32) + for c in range(num_classes): + mask = labels == c + if bool(mask.any()): + centroids[c] = feat_f[mask].mean(dim=0) + return centroids + + +def sample_indices_for_eval(num_samples: int, batch_size: int) -> Sequence[int]: + sizes = [] + rem = int(num_samples) + while rem > 0: + b = min(batch_size, rem) + sizes.append(b) + rem -= b + return sizes diff --git a/benchmarks/cifar_toy/metrics.py b/benchmarks/cifar_toy/metrics.py new file mode 100644 index 000000000..01352b5a2 --- /dev/null +++ b/benchmarks/cifar_toy/metrics.py @@ -0,0 +1,169 @@ +"""Distribution fidelity (Feature-FID, unbiased KID), mode coverage, and critic diagnostics.""" +from __future__ import annotations + +import math +from typing import Dict, Tuple + +import numpy as np +import torch +import torch.nn as nn + + +def feature_moments(features: torch.Tensor) -> Tuple[np.ndarray, np.ndarray]: + """Compute float64 mean ``(D,)`` and sample covariance ``(D, D)``.""" + arr = features.detach().cpu().numpy().astype(np.float64) + mu = np.mean(arr, axis=0) + cov = np.cov(arr, rowvar=False) + return mu, cov + + +def compute_fid( + mu_real: np.ndarray, + cov_real: np.ndarray, + mu_fake: np.ndarray, + cov_fake: np.ndarray, +) -> float: + """Exact symmetric-PSD Fréchet distance between two multivariate Gaussians. + + Uses ``Tr((Sigma_r Sigma_f)^{1/2}) = sum(sqrt(eigvalsh(Sigma_r^{1/2} Sigma_f Sigma_r^{1/2})))`` + in float64, avoiding complex/singular warnings on rank-deficient covariances. + """ + diff = mu_real - mu_fake + sym_r = 0.5 * (cov_real + cov_real.T) + sym_f = 0.5 * (cov_fake + cov_fake.T) + eigvals_r, eigvecs_r = np.linalg.eigh(sym_r) + sqrt_r = (eigvecs_r * np.sqrt(np.clip(eigvals_r, 0.0, None))) @ eigvecs_r.T + prod = sqrt_r @ sym_f @ sqrt_r + prod = 0.5 * (prod + prod.T) + eigvals_prod = np.linalg.eigvalsh(prod) + tr_covmean = float(np.sum(np.sqrt(np.clip(eigvals_prod, 0.0, None)))) + fid = float(diff @ diff + np.trace(sym_r) + np.trace(sym_f) - 2.0 * tr_covmean) + return max(0.0, fid) + + +def _polynomial_mmd_unbiased(x: np.ndarray, y: np.ndarray) -> float: + """Unbiased polynomial-kernel MMD^2 (degree 3, coef0=1, gamma=1/d).""" + m = x.shape[0] + n = y.shape[0] + d = float(x.shape[1]) + k_xx = (x @ x.T / d + 1.0) ** 3 + k_yy = (y @ y.T / d + 1.0) ** 3 + k_xy = (x @ y.T / d + 1.0) ** 3 + sum_xx = (np.sum(k_xx) - np.trace(k_xx)) / (m * (m - 1)) + sum_yy = (np.sum(k_yy) - np.trace(k_yy)) / (n * (n - 1)) + sum_xy = np.sum(k_xy) / (m * n) + return float(sum_xx + sum_yy - 2.0 * sum_xy) + + +def compute_kid( + feats_real: torch.Tensor, + feats_fake: torch.Tensor, + *, + subset_size: int = 500, + num_subsets: int = 20, + seed: int = 0, +) -> Tuple[float, float]: + """Unbiased Kernel Inception/Feature Distance (mean, std).""" + x_all = feats_real.detach().cpu().numpy().astype(np.float64) + y_all = feats_fake.detach().cpu().numpy().astype(np.float64) + m = min(int(subset_size), x_all.shape[0], y_all.shape[0]) + if m < 2: + raise ValueError("KID requires at least 2 samples per distribution") + rng = np.random.default_rng(seed + 404) + vals = [] + for _ in range(num_subsets): + ix = rng.choice(x_all.shape[0], size=m, replace=False) + iy = rng.choice(y_all.shape[0], size=m, replace=False) + vals.append(_polynomial_mmd_unbiased(x_all[ix], y_all[iy])) + arr = np.asarray(vals, dtype=np.float64) + return float(arr.mean()), float(arr.std()) + + +def compute_class_coverage( + feats_fake: torch.Tensor, + class_centroids: torch.Tensor, + *, + min_mass_threshold: float = 0.01, +) -> Dict[str, float]: + """Assign generated features to nearest real class centroid in ResNet-34 space.""" + f = feats_fake.detach().cpu().float() + c = class_centroids.detach().cpu().float() + num_classes = int(c.shape[0]) + dists = torch.cdist(f, c) + assigned = dists.argmin(dim=1) + counts = torch.bincount(assigned, minlength=num_classes).float() + probs = counts / max(1.0, float(counts.sum().item())) + uniform = 1.0 / float(num_classes) + mass_tv = 0.5 * float((probs - uniform).abs().sum().item()) + covered = int((probs >= min_mass_threshold).sum().item()) + nonzero_p = probs[probs > 0] + entropy = float(-(nonzero_p * nonzero_p.log()).sum().item()) + entropy_norm = entropy / math.log(float(num_classes)) if num_classes > 1 else 1.0 + return { + "class_coverage": float(covered), + "class_mass_tv": mass_tv, + "class_entropy_norm": entropy_norm, + "min_class_mass": float(probs.min().item()), + } + + +def compute_pixel_diversity_ratio(real_images: torch.Tensor, fake_images: torch.Tensor) -> float: + """Ratio of generated-to-real across-sample pixel RMS (diversity indicator).""" + r = real_images.detach().float() + f = fake_images.detach().float() + r_rms = float(r.std(dim=0, unbiased=False).square().mean().sqrt().item()) + f_rms = float(f.std(dim=0, unbiased=False).square().mean().sqrt().item()) + return f_rms / max(r_rms, 1e-8) + + +@torch.no_grad() +def compute_critic_diagnostics( + critic: nn.Module, + real_batch: torch.Tensor, + fake_batch: torch.Tensor, + *, + batch_size: int = 128, +) -> Dict[str, float]: + """Side-effect-free critic logit & accuracy diagnostics under ``eval()`` + ``no_grad()``.""" + modes = [(m, m.training) for m in critic.modules()] + device = next(critic.parameters()).device + try: + critic.eval() + r_outs = [] + f_outs = [] + n = min(int(real_batch.shape[0]), int(fake_batch.shape[0])) + for start in range(0, n, batch_size): + r_chunk = real_batch[start : start + batch_size].to(device=device, dtype=torch.float32) + f_chunk = fake_batch[start : start + batch_size].to(device=device, dtype=torch.float32) + r_outs.append(critic(r_chunk).float().reshape(-1).cpu()) + f_outs.append(critic(f_chunk).float().reshape(-1).cpu()) + d_real = torch.cat(r_outs, dim=0) + d_fake = torch.cat(f_outs, dim=0) + finally: + for m, flag in modes: + m.training = flag + + real_mean = float(d_real.mean().item()) + fake_mean = float(d_fake.mean().item()) + margin = real_mean - fake_mean + logit_std = float(torch.cat([d_real, d_fake], dim=0).std(unbiased=False).item()) + # Standard threshold accuracy (matching HyperGAN critic/accuracy) + pairwise accuracy. + acc = 0.5 * (float((d_real > 0.0).float().mean().item()) + float((d_fake <= 0.0).float().mean().item())) + pair_acc = float((d_real > d_fake).float().mean().item()) + return { + "critic_real_mean": real_mean, + "critic_fake_mean": fake_mean, + "critic_margin": margin, + "critic_logit_std": logit_std, + "critic_accuracy": acc, + "critic_pair_acc": pair_acc, + } + + +def compute_grad_norm(parameters) -> float: + sq = 0.0 + for p in parameters: + if p.requires_grad and p.grad is not None: + g = p.grad.detach().float() + sq += float(g.square().sum().item()) + return math.sqrt(sq) diff --git a/benchmarks/cifar_toy/models.py b/benchmarks/cifar_toy/models.py new file mode 100644 index 000000000..601751c4f --- /dev/null +++ b/benchmarks/cifar_toy/models.py @@ -0,0 +1,383 @@ +"""Pure-PyTorch architectures for the CIFAR-style toy benchmark. + +Zero ``torchvision`` dependency: standard PyTorch ResNet-18 / ResNet-34 +``BasicBlock`` backbones load the official SHA-256-verified PyTorch hub +checkpoints directly. + +Generator: ``TinyTransformerGenerator32`` (16 matrix/conv params, 2,489,731 +trainable parameters) — exact structural match to HyperGAN's +``tiny-transformer-generator-32``. + +Critic: ``ResNet18FeatureCritic32`` (9 trainable matrix/conv params, 171,779 +trainable parameters on top of frozen ImageNet ResNet-18 ``layer1/2/3`` at +64px with fixed midgray context) — exact structural match to HyperGAN's +``resnet18-features-discriminator-32``. + +Evaluator: ``ResNet34FeatureEvaluator`` — independent frozen ImageNet ResNet-34 +``avgpool`` 512-dim feature extractor for Feature-FID, Feature-KID, and class +centroid mode coverage. +""" +from __future__ import annotations + +import hashlib +import math +from pathlib import Path +from typing import Iterable, Optional, Sequence, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + +RESNET18_SHA256 = "f37072fd47e89c5e827621c5baffa7500819f7896bbacec160b1a16c560e07ec" +RESNET34_SHA256 = "b627a593bcbe140c234610266fe4f8ae95ea42fc881d091c9b6052e6b1d0590f" + +DEFAULT_RESNET18_WEIGHTS = Path.home() / ".cache" / "torch" / "hub" / "checkpoints" / "resnet18-f37072fd.pth" +DEFAULT_RESNET34_WEIGHTS = Path.home() / ".cache" / "torch" / "hub" / "checkpoints" / "resnet34-b627a593.pth" + +_IMAGENET_MEAN = (0.485, 0.456, 0.406) +_IMAGENET_STD = (0.229, 0.224, 0.225) + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def count_trainable_params(module: nn.Module) -> int: + return sum(p.numel() for p in module.parameters() if p.requires_grad) + + +def count_matrix_conv_params(module: nn.Module) -> int: + """Count trainable parameters with ``ndim >= 2`` (matrices and conv kernels).""" + return sum(1 for p in module.parameters() if p.requires_grad and p.ndim >= 2) + + +def parameter_state_digest(module: nn.Module) -> str: + """Deterministic SHA-256 digest of all trainable parameter tensors (CPU float32 bytes).""" + h = hashlib.sha256() + for name, p in sorted(module.named_parameters()): + if not p.requires_grad: + continue + arr = p.detach().cpu().contiguous().float().numpy() + h.update(name.encode("utf-8")) + h.update(str(tuple(p.shape)).encode("utf-8")) + h.update(arr.tobytes()) + return h.hexdigest() + + +class _BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes: int, planes: int, stride: int = 1, downsample: Optional[nn.Module] = None) -> None: + super().__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.downsample = downsample + + def forward(self, x: torch.Tensor) -> torch.Tensor: + identity = x + out = self.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + if self.downsample is not None: + identity = self.downsample(x) + return self.relu(out + identity) + + +class _ResNetBackbone(nn.Module): + """Standard torchvision-compatible ResNet-18 / ResNet-34 backbone without torchvision.""" + + def __init__(self, layers: Sequence[int], *, include_layer4: bool = False) -> None: + super().__init__() + self.inplanes = 64 + self.include_layer4 = bool(include_layer4) + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(64, layers[0], stride=1) + self.layer2 = self._make_layer(128, layers[1], stride=2) + self.layer3 = self._make_layer(256, layers[2], stride=2) + if self.include_layer4: + self.layer4 = self._make_layer(512, layers[3], stride=2) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + + def _make_layer(self, planes: int, blocks: int, stride: int) -> nn.Sequential: + downsample = None + if stride != 1 or self.inplanes != planes: + downsample = nn.Sequential( + nn.Conv2d(self.inplanes, planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(planes), + ) + layers = [_BasicBlock(self.inplanes, planes, stride=stride, downsample=downsample)] + self.inplanes = planes + for _ in range(1, blocks): + layers.append(_BasicBlock(self.inplanes, planes)) + return nn.Sequential(*layers) + + def load_torchvision_state_dict(self, state_dict: dict) -> None: + own_keys = set(self.state_dict().keys()) + filtered = {k: v for k, v in state_dict.items() if k in own_keys} + missing = {k for k in (own_keys - set(filtered.keys())) if not k.endswith(".num_batches_tracked")} + if missing: + raise ValueError(f"ResNet checkpoint missing expected keys: {sorted(missing)[:8]}") + self.load_state_dict(filtered, strict=False) + + def forward_features_123(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + x = self.maxpool(self.relu(self.bn1(self.conv1(x)))) + f1 = self.layer1(x) + f2 = self.layer2(f1) + f3 = self.layer3(f2) + return f1, f2, f3 + + def forward_avgpool(self, x: torch.Tensor) -> torch.Tensor: + if not self.include_layer4: + raise RuntimeError("forward_avgpool requires include_layer4=True") + _, _, f3 = self.forward_features_123(x) + f4 = self.layer4(f3) + return torch.flatten(self.avgpool(f4), 1) + + +def _load_verified_resnet_weights( + weights_path: Path, + *, + expected_sha256: Optional[str] = None, +) -> Tuple[dict, str]: + path = Path(weights_path).expanduser() + if not path.is_file(): + raise FileNotFoundError(f"Pretrained ResNet weights not found: {path}") + actual_sha = sha256_file(path) + if expected_sha256 is not None and actual_sha != expected_sha256: + # Accept prefix match if caller passes short hash, else strict mismatch error. + if not actual_sha.startswith(expected_sha256): + raise ValueError( + f"SHA-256 mismatch for {path}: expected {expected_sha256}, got {actual_sha}" + ) + sd = torch.load(path, map_location="cpu", weights_only=True) + return sd, actual_sha + + +class _TransformerBlock(nn.Module): + """Pre-LayerNorm multi-head self-attention + GELU FFN with separate Q/K/V/Out Linears.""" + + def __init__(self, d_model: int = 256, num_heads: int = 4, ffn_dim: int = 512) -> None: + super().__init__() + if d_model % num_heads != 0: + raise ValueError("d_model must be divisible by num_heads") + self.d_model = d_model + self.num_heads = num_heads + self.head_dim = d_model // num_heads + + self.attn_norm = nn.LayerNorm(d_model) + self.q_proj = nn.Linear(d_model, d_model, bias=True) + self.k_proj = nn.Linear(d_model, d_model, bias=True) + self.v_proj = nn.Linear(d_model, d_model, bias=True) + self.out_proj = nn.Linear(d_model, d_model, bias=True) + + self.ffn_norm = nn.LayerNorm(d_model) + self.fc1 = nn.Linear(d_model, ffn_dim, bias=True) + self.fc2 = nn.Linear(ffn_dim, d_model, bias=True) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + b, n, c = x.shape + h = self.attn_norm(x) + q = self.q_proj(h).view(b, n, self.num_heads, self.head_dim).transpose(1, 2) + k = self.k_proj(h).view(b, n, self.num_heads, self.head_dim).transpose(1, 2) + v = self.v_proj(h).view(b, n, self.num_heads, self.head_dim).transpose(1, 2) + # Explicit scaled dot-product attention for deterministic CPU/CUDA compatibility. + scale = 1.0 / math.sqrt(float(self.head_dim)) + attn = torch.matmul(q, k.transpose(-2, -1)) * scale + attn = torch.softmax(attn, dim=-1) + out = torch.matmul(attn, v).transpose(1, 2).contiguous().view(b, n, c) + x = x + self.out_proj(out) + + h2 = self.ffn_norm(x) + x = x + self.fc2(F.gelu(self.fc1(h2))) + return x + + +class TinyTransformerGenerator32(nn.Module): + """2-block 64-token Transformer generator -> 32x32 RGB in [-1, 1]. + + Exact structural match to HyperGAN's ``cifar10-tiny-transformer``: + 16 matrix/conv parameters (1 input projection + 2*(4 attn + 2 FFN) + 3 convs), + 2,489,731 trainable parameters total. + """ + + def __init__(self, z_dim: int = 64, d_model: int = 256, num_heads: int = 4, ffn_dim: int = 512) -> None: + super().__init__() + self.z_dim = z_dim + self.d_model = d_model + self.input_projection = nn.Linear(z_dim, 64 * d_model, bias=True) + self.block0 = _TransformerBlock(d_model=d_model, num_heads=num_heads, ffn_dim=ffn_dim) + self.block1 = _TransformerBlock(d_model=d_model, num_heads=num_heads, ffn_dim=ffn_dim) + self.conv16 = nn.Conv2d(d_model, 128, kernel_size=3, padding=1, bias=True) + self.conv32 = nn.Conv2d(128, 64, kernel_size=3, padding=1, bias=True) + self.rgb = nn.Conv2d(64, 3, kernel_size=3, padding=1, bias=True) + + def forward(self, z: torch.Tensor) -> torch.Tensor: + b = z.shape[0] + h = self.input_projection(z).view(b, 64, self.d_model) + h = self.block0(h) + h = self.block1(h) + h = h.transpose(1, 2).contiguous().view(b, self.d_model, 8, 8) + h = F.interpolate(h, scale_factor=2, mode="nearest") + h = F.leaky_relu(self.conv16(h), negative_slope=0.2) + h = F.interpolate(h, scale_factor=2, mode="nearest") + h = F.leaky_relu(self.conv32(h), negative_slope=0.2) + return torch.tanh(self.rgb(h)) + + +class _ResNetFeatureHead(nn.Module): + """Single multi-scale ResNet feature head (3 matrix/conv params).""" + + def __init__(self, in_channels: int, hidden_channels: int = 64) -> None: + super().__init__() + self.project = nn.Conv2d(in_channels, hidden_channels, kernel_size=1, bias=True) + self.norm = nn.GroupNorm(8, hidden_channels) + self.conv = nn.Conv2d(hidden_channels, hidden_channels, kernel_size=3, padding=1, bias=True) + self.score = nn.Linear(hidden_channels * 4 * 4, 1, bias=True) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + h = F.leaky_relu(self.norm(self.project(x)), negative_slope=0.2) + h = F.leaky_relu(self.conv(h), negative_slope=0.2) + h = F.adaptive_avg_pool2d(h, (4, 4)).flatten(1) + return self.score(h) + + +class ResNet18FeatureCritic32(nn.Module): + """Frozen ImageNet ResNet-18 64px feature critic with 3 trainable feature heads. + + Exact structural match to HyperGAN's ``resnet18-features-discriminator-32``: + - Bilinear 32->64 resize + ImageNet normalization on ``[x, zeros_like(x)]``. + - Frozen ResNet-18 ``layer1`` (64ch), ``layer2`` (128ch), ``layer3`` (256ch). + - 3 trainable feature heads concatenating candidate + midgray-context features. + - 9 trainable matrix/conv parameters, 171,779 trainable parameters total. + - ``train()`` keeps the frozen backbone strictly in ``eval()`` mode so + ``GANTrainer`` calling ``D.train()`` never mutates BatchNorm statistics. + """ + + def __init__( + self, + weights_path: Optional[Path] = DEFAULT_RESNET18_WEIGHTS, + *, + backbone_mode: str = "pretrained", + backbone_seed: int = 1001, + ) -> None: + super().__init__() + if backbone_mode not in ("pretrained", "random"): + raise ValueError(f"backbone_mode must be 'pretrained' or 'random', got {backbone_mode!r}") + self.backbone_mode = backbone_mode + self.backbone_sha256: Optional[str] = None + self.backbone = _ResNetBackbone((2, 2, 2, 2), include_layer4=False) + if backbone_mode == "pretrained": + if weights_path is None: + raise ValueError("weights_path is required when backbone_mode='pretrained'") + sd, actual_sha = _load_verified_resnet_weights(Path(weights_path), expected_sha256=RESNET18_SHA256) + self.backbone.load_torchvision_state_dict(sd) + self.backbone_sha256 = actual_sha + else: + import particlegan as pg + + pg.init.deterministic_orthogonal_(self.backbone, seed=backbone_seed, strict=True) + self.backbone_sha256 = f"deterministic_random_seed_{backbone_seed}" + + for p in self.backbone.parameters(): + p.requires_grad_(False) + self.backbone.eval() + + self.register_buffer("_img_mean", torch.tensor(_IMAGENET_MEAN, dtype=torch.float32).view(1, 3, 1, 1)) + self.register_buffer("_img_std", torch.tensor(_IMAGENET_STD, dtype=torch.float32).view(1, 3, 1, 1)) + + self.head1 = _ResNetFeatureHead(64 * 2, 64) + self.head2 = _ResNetFeatureHead(128 * 2, 64) + self.head3 = _ResNetFeatureHead(256 * 2, 64) + self._inv_sqrt3 = 1.0 / math.sqrt(3.0) + + def train(self, mode: bool = True) -> "ResNet18FeatureCritic32": + super().train(mode) + # Frozen ImageNet ResNet-18 must always remain in eval() mode (fixed BN stats). + self.backbone.eval() + return self + + def forward(self, x: torch.Tensor) -> torch.Tensor: + context = torch.zeros_like(x) + feature_batch = torch.cat([x, context], dim=0) + resized = F.interpolate(feature_batch, scale_factor=2, mode="bilinear", align_corners=False) + normalized = (resized * 0.5 + 0.5 - self._img_mean.to(resized.dtype)) / self._img_std.to(resized.dtype) + f1, f2, f3 = self.backbone.forward_features_123(normalized) + cand1, ctx1 = f1.chunk(2, dim=0) + cand2, ctx2 = f2.chunk(2, dim=0) + cand3, ctx3 = f3.chunk(2, dim=0) + s1 = self.head1(torch.cat([cand1, ctx1], dim=1)) + s2 = self.head2(torch.cat([cand2, ctx2], dim=1)) + s3 = self.head3(torch.cat([cand3, ctx3], dim=1)) + return (s1 + s2 + s3) * self._inv_sqrt3 + + +class ResNet34FeatureEvaluator(nn.Module): + """Independent frozen ImageNet ResNet-34 512-dim feature extractor for FID/KID. + + Uses ResNet-34 (different architecture depth and weights from the critic's + frozen ResNet-18) so Feature-FID/KID and class-centroid coverage cannot be + gamed by critic-backbone shortcuts. + """ + + def __init__( + self, + weights_path: Optional[Path] = DEFAULT_RESNET34_WEIGHTS, + *, + backbone_mode: str = "pretrained", + backbone_seed: int = 2002, + ) -> None: + super().__init__() + if backbone_mode not in ("pretrained", "random"): + raise ValueError(f"backbone_mode must be 'pretrained' or 'random', got {backbone_mode!r}") + self.backbone_mode = backbone_mode + self.backbone_sha256: Optional[str] = None + self.backbone = _ResNetBackbone((3, 4, 6, 3), include_layer4=True) + if backbone_mode == "pretrained": + if weights_path is None: + raise ValueError("weights_path is required when backbone_mode='pretrained'") + sd, actual_sha = _load_verified_resnet_weights(Path(weights_path), expected_sha256=RESNET34_SHA256) + self.backbone.load_torchvision_state_dict(sd) + self.backbone_sha256 = actual_sha + else: + import particlegan as pg + + pg.init.deterministic_orthogonal_(self.backbone, seed=backbone_seed, strict=True) + self.backbone_sha256 = f"deterministic_random_seed_{backbone_seed}" + + for p in self.backbone.parameters(): + p.requires_grad_(False) + self.backbone.eval() + self.register_buffer("_img_mean", torch.tensor(_IMAGENET_MEAN, dtype=torch.float32).view(1, 3, 1, 1)) + self.register_buffer("_img_std", torch.tensor(_IMAGENET_STD, dtype=torch.float32).view(1, 3, 1, 1)) + + @torch.no_grad() + def extract(self, images: torch.Tensor, *, batch_size: int = 256) -> torch.Tensor: + """Extract L2-unnormalized 512-dim float32 features from images in [-1, 1].""" + self.eval() + device = self._img_mean.device + feats = [] + n = int(images.shape[0]) + for start in range(0, n, batch_size): + chunk = images[start : start + batch_size].to(device=device, dtype=torch.float32) + resized = F.interpolate(chunk, size=(64, 64), mode="bilinear", align_corners=False) + normed = (resized * 0.5 + 0.5 - self._img_mean) / self._img_std + feats.append(self.backbone.forward_avgpool(normed).cpu()) + return torch.cat(feats, dim=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + resized = F.interpolate(x.float(), size=(64, 64), mode="bilinear", align_corners=False) + normed = (resized * 0.5 + 0.5 - self._img_mean) / self._img_std + return self.backbone.forward_avgpool(normed) + + +def iter_trainable_modules(module: nn.Module) -> Iterable[nn.Module]: + return (m for m in module.modules() if any(p.requires_grad for p in m.parameters(recurse=False))) diff --git a/benchmarks/cifar_toy/runner.py b/benchmarks/cifar_toy/runner.py new file mode 100644 index 000000000..da058a4c2 --- /dev/null +++ b/benchmarks/cifar_toy/runner.py @@ -0,0 +1,543 @@ +"""Reproducible CIFAR-style benchmark runner and leaderboard formatter. + +Adheres strictly to ``AGENTS.md`` comparison rules: +- Protocol seed 0 and ``particlegan.init.deterministic_orthogonal_``. +- Fixed architecture, dataset, critic/generator target batch sequence, initial + prior distribution/capacity, update budget, and evaluation cadence across + trainers. +- Isolated constructor, data, training-noise, prior-noise, and eval RNG streams. +- Executes exact resolved ParticleGAN ``Recipe`` via ``GANTrainer`` with zero + task-host repairs, gradient edits, or silent substitutions. +- Tail-friendly flushed single-line stdout logs + JSONL trace + compact JSON + receipts + single current markdown leaderboard per run directory. +""" +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +import json +import math +from pathlib import Path +import platform +import subprocess +import time +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +import torch + +import particlegan as pg + +from .data import ( + BatchSequenceStream, + CIFAR10Dataset, + compute_class_centroids, + load_cifar10_dataset, + make_synthetic_cifar_dataset, + sample_indices_for_eval, + select_eval_reference_subset, +) +from .metrics import ( + compute_class_coverage, + compute_critic_diagnostics, + compute_fid, + compute_grad_norm, + compute_kid, + compute_pixel_diversity_ratio, + feature_moments, +) +from .models import ( + DEFAULT_RESNET18_WEIGHTS, + DEFAULT_RESNET34_WEIGHTS, + ResNet18FeatureCritic32, + ResNet34FeatureEvaluator, + TinyTransformerGenerator32, + count_matrix_conv_params, + count_trainable_params, + parameter_state_digest, +) + +# Exact fixed MoG sigma from HyperGAN cifar10-tiny-transformer (0.025 * median NN in z=64). +DEFAULT_MOG_SIGMA = 0.212616428732872 + + +@dataclass(frozen=True) +class BenchmarkConfig: + recipe: str = "unified_v11" + num_particles: int = 200 + z_dim: int = 64 + mog_sigma: float = DEFAULT_MOG_SIGMA + standardize_prior: bool = True + batch_size: int = 128 + steps: int = 800 + eval_every: int = 200 + eval_samples: int = 2048 + eval_ref_samples: int = 5000 + eval_critic_samples: int = 256 + seed: int = 0 + dataset: str = "cifar10" + data_root: Optional[str] = None + horizontal_flip: bool = True + backbone_mode: str = "pretrained" + resnet18_weights: str = str(DEFAULT_RESNET18_WEIGHTS) + resnet34_weights: str = str(DEFAULT_RESNET34_WEIGHTS) + device: str = "cuda:0" + deterministic_algorithms: bool = False + recipe_overrides: Dict[str, Any] = field(default_factory=dict) + run_tag: Optional[str] = None + + +def _git_info() -> Dict[str, Any]: + root = Path(__file__).resolve().parents[2] + info: Dict[str, Any] = {"root": str(root)} + for key, cmd in ( + ("commit", ["git", "rev-parse", "HEAD"]), + ("branch", ["git", "rev-parse", "--abbrev-ref", "HEAD"]), + ("dirty", ["git", "status", "--porcelain"]), + ): + try: + res = subprocess.run(cmd, cwd=root, capture_output=True, text=True, check=False) + if res.returncode == 0: + val = res.stdout.strip() + info[key] = bool(val) if key == "dirty" else val + except OSError: + pass + return info + + +def _resolve_device(device_str: str) -> torch.device: + dev = torch.device(device_str) + if dev.type == "cuda": + if not torch.cuda.is_available(): + raise RuntimeError(f"Requested {device_str} but CUDA is unavailable") + if dev.index is None: + dev = torch.device("cuda", torch.cuda.current_device()) + return dev + + +def _device_name(device: torch.device) -> str: + if device.type == "cuda": + return torch.cuda.get_device_name(device) + return platform.processor() or platform.machine() or "cpu" + + +def _capture_backend_state(device: torch.device) -> dict: + state: dict = { + "cpu_rng": torch.get_rng_state(), + "num_threads": torch.get_num_threads(), + "det_alg": torch.are_deterministic_algorithms_enabled(), + "cudnn_det": torch.backends.cudnn.deterministic, + "cudnn_bench": torch.backends.cudnn.benchmark, + "cudnn_tf32": torch.backends.cudnn.allow_tf32, + "matmul_tf32": torch.backends.cuda.matmul.allow_tf32, + } + if device.type == "cuda": + state["cuda_rng"] = torch.cuda.get_rng_state(device) + return state + + +def _restore_backend_state(state: dict, device: torch.device) -> None: + torch.set_rng_state(state["cpu_rng"]) + torch.set_num_threads(state["num_threads"]) + torch.use_deterministic_algorithms(state["det_alg"]) + torch.backends.cudnn.deterministic = state["cudnn_det"] + torch.backends.cudnn.benchmark = state["cudnn_bench"] + torch.backends.cudnn.allow_tf32 = state["cudnn_tf32"] + torch.backends.cuda.matmul.allow_tf32 = state["matmul_tf32"] + if device.type == "cuda" and "cuda_rng" in state: + torch.cuda.set_rng_state(state["cuda_rng"], device) + + +def _format_progress_line(run_name: str, step: int, total_steps: int, row: Mapping[str, Any]) -> str: + fid = row.get("feature_fid", float("nan")) + kid = row.get("feature_kid_x1e3", float("nan")) + cov = int(round(row.get("class_coverage", 0.0))) + acc = row.get("critic_accuracy", float("nan")) + margin = row.get("critic_margin", float("nan")) + r_m = row.get("critic_real_mean", float("nan")) + f_m = row.get("critic_fake_mean", float("nan")) + gn = row.get("grad_norm_g", float("nan")) + dn = row.get("grad_norm_d", float("nan")) + pn = row.get("grad_norm_prior", float("nan")) + ips = row.get("steps_per_sec", float("nan")) + vram = row.get("peak_vram_mib", 0.0) + return ( + f"[{run_name}] step={step:04d}/{total_steps:04d} " + f"fid={fid:6.2f} kid={kid:6.2f} cov={cov:2d}/10 " + f"acc={acc:.3f} margin={margin:+.3f} (r={r_m:+.3f},f={f_m:+.3f}) " + f"gn={gn:.3f} dn={dn:.3f} pn={pn:.4f} " + f"ips={ips:5.2f} vram={vram:6.0f}MiB" + ) + + +def _load_dataset_for_config(cfg: BenchmarkConfig) -> CIFAR10Dataset: + if cfg.dataset == "cifar10": + return load_cifar10_dataset(Path(cfg.data_root) if cfg.data_root else None, split="train") + if cfg.dataset == "synthetic": + return make_synthetic_cifar_dataset( + num_samples=max(2000, cfg.eval_ref_samples), + seed=cfg.seed, + num_classes=10, + ) + raise ValueError(f"Unknown dataset {cfg.dataset!r}; expected 'cifar10' or 'synthetic'") + + +def run_benchmark( + cfg: BenchmarkConfig, + output_dir: Path, + *, + dataset: Optional[CIFAR10Dataset] = None, + evaluator: Optional[ResNet34FeatureEvaluator] = None, + ref_cache: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Execute one matched benchmark run and write tail-friendly logs + receipt.""" + device = _resolve_device(cfg.device) + out_root = Path(output_dir) + run_name = cfg.run_tag or f"{cfg.recipe}_p{cfg.num_particles}_b{cfg.batch_size}_s{cfg.steps}" + run_dir = out_root / run_name + run_dir.mkdir(parents=True, exist_ok=True) + metrics_path = run_dir / "metrics.jsonl" + log_path = run_dir / "train.log" + receipt_path = run_dir / "receipt.json" + + saved_state = _capture_backend_state(device) + try: + # Fixed protocol execution hygiene (restored in finally). + torch.manual_seed(cfg.seed) + if device.type == "cuda": + torch.cuda.manual_seed(cfg.seed) + torch.cuda.reset_peak_memory_stats(device) + torch.backends.cudnn.deterministic = bool(cfg.deterministic_algorithms) + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.allow_tf32 = False + torch.backends.cuda.matmul.allow_tf32 = False + if cfg.deterministic_algorithms: + torch.use_deterministic_algorithms(True) + + ds = _load_dataset_for_config(cfg) if dataset is None else dataset + + # Prepare independent ResNet-34 evaluator and reference feature moments once. + own_eval = evaluator is None + if own_eval: + evaluator = ResNet34FeatureEvaluator( + Path(cfg.resnet34_weights), + backbone_mode=cfg.backbone_mode, + backbone_seed=cfg.seed + 2002, + ).to(device=device) + evaluator.eval() + + if ref_cache is None: + ref_imgs, ref_labels = select_eval_reference_subset(ds, cfg.eval_ref_samples, seed=cfg.seed) + ref_feats = evaluator.extract(ref_imgs, batch_size=256) + ref_mu, ref_cov = feature_moments(ref_feats) + ref_centroids = compute_class_centroids(ref_feats, ref_labels, num_classes=ds.num_classes) + eval_critic_real = ref_imgs[: cfg.eval_critic_samples].clone() + ref_cache = { + "ref_imgs": ref_imgs, + "ref_feats": ref_feats, + "ref_mu": ref_mu, + "ref_cov": ref_cov, + "ref_centroids": ref_centroids, + "eval_critic_real": eval_critic_real, + } + + # Construct fixed task architecture & deterministic initialization. + G = TinyTransformerGenerator32(z_dim=cfg.z_dim).to(device=device) + D = ResNet18FeatureCritic32( + Path(cfg.resnet18_weights), + backbone_mode=cfg.backbone_mode, + backbone_seed=cfg.seed + 1001, + ).to(device=device) + prior = pg.MoGParticlePrior( + cfg.num_particles, + cfg.z_dim, + sigma=cfg.mog_sigma, + standardize=cfg.standardize_prior, + device=device, + ) + + # Deterministic public initializer (seed=0 for G & prior, seed=1 for D). + pg.init.deterministic_orthogonal_(G, seed=cfg.seed, strict=True) + pg.init.deterministic_orthogonal_(D, seed=cfg.seed + 1, strict=True) + pg.init.deterministic_orthogonal_(prior, seed=cfg.seed, strict=True) + + init_digests = { + "initial_g_sha256": parameter_state_digest(G), + "initial_d_sha256": parameter_state_digest(D), + "initial_prior_sha256": parameter_state_digest(prior), + "g_trainable_params": count_trainable_params(G), + "g_matrix_conv_params": count_matrix_conv_params(G), + "d_trainable_params": count_trainable_params(D), + "d_matrix_conv_params": count_matrix_conv_params(D), + "d_backbone_sha256": D.backbone_sha256, + "eval_backbone_sha256": evaluator.backbone_sha256, + } + + # Resolve exact public ParticleGAN Recipe (no host repairs / overrides unless explicitly declared). + recipe_kwargs: Dict[str, Any] = { + "prior_kind": "mog", + "z_dim": cfg.z_dim, + "num_particles": cfg.num_particles, + "sigma_rel": 0.0, + "standardize": cfg.standardize_prior, + "batch_size": cfg.batch_size, + "total_steps": cfg.steps, + } + recipe_kwargs.update(cfg.recipe_overrides) + recipe = pg.get_recipe(cfg.recipe, **recipe_kwargs) + + # Isolated named RNG streams on execution device. + def _dev_gen(offset: int) -> torch.Generator: + return torch.Generator(device=device).manual_seed(cfg.seed + offset) + + trainer = pg.GANTrainer( + recipe, + G, + D, + prior=prior, + seed=cfg.seed, + latent_generator=_dev_gen(11), + penalty_generator=_dev_gen(12), + noise_generator=_dev_gen(13), + input_noise_generator=_dev_gen(14), + prior_noise_generator=_dev_gen(15), + eval_generator=_dev_gen(16), + max_steps=cfg.steps, + ) + + # Isolated CPU data streams guaranteeing identical (real_d, real_g) sequence across recipes. + data_stream = BatchSequenceStream( + ds, + cfg.batch_size, + seed=cfg.seed, + horizontal_flip=cfg.horizontal_flip, + ) + + def _evaluate(step: int, last_step_info: Dict[str, float], train_seconds: float) -> Dict[str, Any]: + # Fixed per-step eval RNG so sampling is deterministic and isolated from training. + eval_stream = torch.Generator(device=device).manual_seed(cfg.seed + 50000 + step) + fake_chunks = [] + for b in sample_indices_for_eval(cfg.eval_samples, min(256, cfg.eval_samples)): + samples = trainer.sample(b, generator=eval_stream, output_noise=False) + fake_chunks.append(samples.detach().float().cpu()) + fake_all = torch.cat(fake_chunks, dim=0) + + fake_feats = evaluator.extract(fake_all, batch_size=256) + fake_mu, fake_cov = feature_moments(fake_feats) + fid = compute_fid(ref_cache["ref_mu"], ref_cache["ref_cov"], fake_mu, fake_cov) + kid_mean, kid_std = compute_kid( + ref_cache["ref_feats"], + fake_feats, + subset_size=min(500, cfg.eval_samples, cfg.eval_ref_samples), + num_subsets=20, + seed=cfg.seed, + ) + cov_stats = compute_class_coverage(fake_feats, ref_cache["ref_centroids"]) + div_ratio = compute_pixel_diversity_ratio( + ref_cache["eval_critic_real"], fake_all[: cfg.eval_critic_samples] + ) + critic_diag = compute_critic_diagnostics( + D, + ref_cache["eval_critic_real"], + fake_all[: cfg.eval_critic_samples], + batch_size=min(128, cfg.eval_critic_samples), + ) + peak_vram_mib = ( + float(torch.cuda.max_memory_allocated(device)) / (1024.0 * 1024.0) + if device.type == "cuda" + else 0.0 + ) + ips = float(step) / train_seconds if (step > 0 and train_seconds > 0.0) else 0.0 + row: Dict[str, Any] = { + "step": step, + "feature_fid": fid, + "feature_kid_x1e3": kid_mean * 1000.0, + "feature_kid_std_x1e3": kid_std * 1000.0, + **cov_stats, + "diversity_ratio": div_ratio, + **critic_diag, + **last_step_info, + "elapsed_train_sec": train_seconds, + "steps_per_sec": ips, + "peak_vram_mib": peak_vram_mib, + } + return row + + history: List[Dict[str, Any]] = [] + status = "COMPLETE" + error_msg: Optional[str] = None + train_seconds = 0.0 + last_step_info: Dict[str, float] = { + "loss_d": float("nan"), + "loss_g": float("nan"), + "loss_gan": float("nan"), + "penalty": float("nan"), + "prior_reg": float("nan"), + "grad_norm_g": 0.0, + "grad_norm_d": 0.0, + "grad_norm_prior": 0.0, + "prior_rms": float(prior.z.detach().float().square().mean().sqrt().item()), + } + + with open(metrics_path, "w", encoding="utf-8") as mf, open(log_path, "w", encoding="utf-8") as lf: + # Step-0 evaluation (verifies matched initialization across recipes). + row0 = _evaluate(0, last_step_info, 0.0) + history.append(row0) + mf.write(json.dumps(row0) + "\n") + mf.flush() + line0 = _format_progress_line(run_name, 0, cfg.steps, row0) + print(line0, flush=True) + lf.write(line0 + "\n") + lf.flush() + + for step in range(1, cfg.steps + 1): + real_d, real_g = data_stream.next_pair(device) + if device.type == "cuda": + torch.cuda.synchronize(device) + t0 = time.perf_counter() + out = trainer.step(real_d, generator_real=real_g) + if device.type == "cuda": + torch.cuda.synchronize(device) + train_seconds += time.perf_counter() - t0 + + loss_d = float(out["loss_d"].item()) + loss_g = float(out["loss_g"].item()) + loss_gan = float(out["loss_gan"].item()) + pen = float(out["penalty"].item()) + preg = float(out["prior_regularization"].item()) + + if not (math.isfinite(loss_d) and math.isfinite(loss_g)): + status = "NONFINITE" + error_msg = f"Nonfinite loss at step {step}: loss_d={loss_d}, loss_g={loss_g}" + lf.write(f"ERROR: {error_msg}\n") + lf.flush() + print(f"[{run_name}] ERROR: {error_msg}", flush=True) + break + + last_step_info = { + "loss_d": loss_d, + "loss_g": loss_g, + "loss_gan": loss_gan, + "penalty": pen, + "prior_reg": preg, + "grad_norm_g": compute_grad_norm(G.parameters()), + "grad_norm_d": compute_grad_norm(D.parameters()), + "grad_norm_prior": compute_grad_norm([prior.z]), + "prior_rms": float(prior.z.detach().float().square().mean().sqrt().item()), + } + + is_eval = (step % cfg.eval_every == 0) or (step == cfg.steps) + if is_eval: + row = _evaluate(step, last_step_info, train_seconds) + history.append(row) + mf.write(json.dumps(row) + "\n") + mf.flush() + line = _format_progress_line(run_name, step, cfg.steps, row) + print(line, flush=True) + lf.write(line + "\n") + lf.flush() + + # Summarize best & final metrics (excluding step 0 for best trained FID/KID). + trained_rows = [r for r in history if r["step"] > 0] + final_row = history[-1] + best_fid_row = min(trained_rows, key=lambda r: r["feature_fid"]) if trained_rows else final_row + best_kid_row = min(trained_rows, key=lambda r: r["feature_kid_x1e3"]) if trained_rows else final_row + + receipt: Dict[str, Any] = { + "schema_version": 1, + "run_name": run_name, + "status": status, + "error": error_msg, + "config": asdict(cfg), + "resolved_recipe": recipe.to_dict(), + "dataset": { + "source_name": ds.source_name, + "source_path": ds.source_path, + "size": ds.size, + "num_classes": ds.num_classes, + }, + "hardware": { + "device": str(device), + "device_name": _device_name(device), + "torch_version": torch.__version__, + "python_version": platform.python_version(), + }, + "git": _git_info(), + "provenance": { + **init_digests, + "batch_sequence_digest": data_stream.digest, + "steps_completed": int(final_row["step"]), + "sampling_law": "clean_live_mog", + "eval_output_noise": False, + }, + "summary": { + "final_step": int(final_row["step"]), + "final_fid": float(final_row["feature_fid"]), + "best_fid": float(best_fid_row["feature_fid"]), + "best_fid_step": int(best_fid_row["step"]), + "final_kid_x1e3": float(final_row["feature_kid_x1e3"]), + "best_kid_x1e3": float(best_kid_row["feature_kid_x1e3"]), + "final_class_coverage": int(round(final_row["class_coverage"])), + "final_class_mass_tv": float(final_row["class_mass_tv"]), + "final_diversity_ratio": float(final_row["diversity_ratio"]), + "final_critic_accuracy": float(final_row["critic_accuracy"]), + "final_critic_margin": float(final_row["critic_margin"]), + "final_critic_real_mean": float(final_row["critic_real_mean"]), + "final_critic_fake_mean": float(final_row["critic_fake_mean"]), + "final_grad_norm_g": float(final_row["grad_norm_g"]), + "final_grad_norm_d": float(final_row["grad_norm_d"]), + "final_grad_norm_prior": float(final_row["grad_norm_prior"]), + "steps_per_sec": float(final_row["steps_per_sec"]), + "elapsed_train_sec": float(final_row["elapsed_train_sec"]), + "peak_vram_mib": float(final_row["peak_vram_mib"]), + }, + "history": history, + } + receipt_path.write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8") + return receipt + finally: + _restore_backend_state(saved_state, device) + + +def format_leaderboard_markdown(receipts: Sequence[Mapping[str, Any]]) -> str: + """Render a compact, decision-ready markdown leaderboard table from receipts.""" + ordered = sorted( + receipts, + key=lambda r: ( + 0 if r.get("status") == "COMPLETE" else 1, + float(r.get("summary", {}).get("final_fid", float("inf"))), + ), + ) + lines = [ + "| Run | Recipe | MoG N | Status | Final FID ↓ | Best FID ↓ (step) | KID×1e3 ↓ | Cov /10 | TV ↓ | D acc | Margin | G grad | D grad | it/s ↑ | Peak MiB |", + "| --- | --- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + ] + for r in ordered: + cfg = r.get("config", {}) + s = r.get("summary", {}) + lines.append( + f"| `{r.get('run_name', '')}` | `{cfg.get('recipe', '')}` | {cfg.get('num_particles', 0)} " + f"| {r.get('status', 'UNKNOWN')} " + f"| {s.get('final_fid', float('nan')):.2f} " + f"| {s.get('best_fid', float('nan')):.2f} ({s.get('best_fid_step', 0)}) " + f"| {s.get('final_kid_x1e3', float('nan')):.2f} " + f"| {s.get('final_class_coverage', 0)}/10 " + f"| {s.get('final_class_mass_tv', float('nan')):.3f} " + f"| {s.get('final_critic_accuracy', float('nan')):.3f} " + f"| {s.get('final_critic_margin', float('nan')):+.3f} " + f"| {s.get('final_grad_norm_g', float('nan')):.3f} " + f"| {s.get('final_grad_norm_d', float('nan')):.3f} " + f"| {s.get('steps_per_sec', float('nan')):.2f} " + f"| {s.get('peak_vram_mib', 0.0):.0f} |" + ) + return "\n".join(lines) + "\n" + + +def summarize_run_directory(output_dir: Path) -> Tuple[List[Dict[str, Any]], str]: + """Collect all ``receipt.json`` files under ``output_dir`` and write one ``leaderboard.md``.""" + root = Path(output_dir) + receipts = [] + for path in sorted(root.glob("*/receipt.json")): + receipts.append(json.loads(path.read_text(encoding="utf-8"))) + md = format_leaderboard_markdown(receipts) + if receipts: + (root / "leaderboard.md").write_text(md, encoding="utf-8") + return receipts, md diff --git a/configs/forge/catalog.json b/configs/forge/catalog.json index 3feb7bb79..0606bb55a 100644 --- a/configs/forge/catalog.json +++ b/configs/forge/catalog.json @@ -1,6 +1,7 @@ { "counts": { "by_family": { + "benchmark/cifar_toy": 6, "benchmark/init_research": 10, "benchmark/learned_lr": 10, "benchmark/legacy": 5, @@ -104,7 +105,7 @@ "candidate_implementation": 17, "candidate_patch": 26, "configuration": 810, - "driver": 405, + "driver": 407, "engine_support": 128, "evaluator_or_audit": 324, "evidence_artifact": 1551, @@ -114,18 +115,18 @@ "frozen_support": 28, "narrative": 975, "structured_evidence": 5070, - "support": 1561, + "support": 1565, "support_artifact": 52, "visualization": 23, "visualization_artifact": 765 }, - "files": 11855 + "files": 11861 }, "coverage": { - "declared_count": 11855, + "declared_count": 11861, "missing": [], "stale": [], - "tracked_count": 11855, + "tracked_count": 11861, "unclassified": [], "valid": true }, @@ -141,6 +142,52 @@ "role": "support", "suggested_tiers": [] }, + { + "family": "benchmark/cifar_toy", + "git_blob": "0e0e96f3d4a8a96b49ac7634b98ae379506dc55f", + "path": "benchmarks/cifar_toy/__init__.py", + "role": "support", + "suggested_tiers": [] + }, + { + "family": "benchmark/cifar_toy", + "git_blob": "6c781d2029b7f039ac50cfa34878cd59c0aa80d3", + "path": "benchmarks/cifar_toy/__main__.py", + "role": "driver", + "suggested_tiers": [ + 2 + ] + }, + { + "family": "benchmark/cifar_toy", + "git_blob": "b6bda1008e8ffea99018cbcb4116807c92ae7dde", + "path": "benchmarks/cifar_toy/data.py", + "role": "support", + "suggested_tiers": [] + }, + { + "family": "benchmark/cifar_toy", + "git_blob": "01352b5a2af35a216e924287ee0dec7212196891", + "path": "benchmarks/cifar_toy/metrics.py", + "role": "support", + "suggested_tiers": [] + }, + { + "family": "benchmark/cifar_toy", + "git_blob": "f85348b0da6ba67b000675547ac27bc73c5725b9", + "path": "benchmarks/cifar_toy/models.py", + "role": "support", + "suggested_tiers": [] + }, + { + "family": "benchmark/cifar_toy", + "git_blob": "da058a4c2ae6d5c1e365988020e786e9fb58598a", + "path": "benchmarks/cifar_toy/runner.py", + "role": "driver", + "suggested_tiers": [ + 2 + ] + }, { "family": "benchmark/shared", "git_blob": "e28cba9ae2e2ced03b6a663d20bcfb0bcc3910d5", @@ -18209,14 +18256,14 @@ }, { "family": "report/readme-100gaussians", - "git_blob": "c2ef7afe29bf4c0f0d7cfda3b606aa54e2cfc98e", + "git_blob": "85df9d9d3b24290af8520988ab179afdcc0fd634", "path": "reports/readme-100gaussians/CURRENT_PROTOCOL.md", "role": "narrative", "suggested_tiers": [] }, { "family": "report/readme-100gaussians", - "git_blob": "99b40cffa7ac9aca95fb326530ddc645791fdbc8", + "git_blob": "b15543ac2d6a83b1fde9bac944f507801aa66c54", "path": "reports/readme-100gaussians/README.md", "role": "narrative", "suggested_tiers": [] @@ -18230,14 +18277,14 @@ }, { "family": "report/readme-100gaussians", - "git_blob": "bfa9a90e443d212db2a0177bfbf04ae8e37ada83", + "git_blob": "2067f78abf8a0036c68b5e3e4f2459d8953a2625", "path": "reports/readme-100gaussians/record.py", "role": "support", "suggested_tiers": [] }, { "family": "report/readme-100gaussians", - "git_blob": "8a0bef9381624504698f6b5ed35de05067406e81", + "git_blob": "cfc6409b4a6c8e647eb2541d09ebeb6961a7fc95", "path": "reports/readme-100gaussians/render.py", "role": "visualization", "suggested_tiers": [] @@ -18251,7 +18298,7 @@ }, { "family": "report/readme-100gaussians", - "git_blob": "f6f31cc543f7960c6c79d95d3e2a7e7db4eecb5b", + "git_blob": "326c8f03f9483266f6dedb8e649b0c233ef25d2f", "path": "reports/readme-100gaussians/summary-current.json", "role": "structured_evidence", "suggested_tiers": [] diff --git a/experiments/forge/history.py b/experiments/forge/history.py index 28804e39b..b4898eae5 100644 --- a/experiments/forge/history.py +++ b/experiments/forge/history.py @@ -45,7 +45,7 @@ "atlas-explanation", "e22-animation", "r1-integrations", "develop-gates-20261001", "paired-residual-toy", "toy_audit", "gram-profile", "prior-ablation-2026-10-25", } -BENCHMARK_FAMILIES = {"init_research", "learned_lr", "legacy", "locked_shared", +BENCHMARK_FAMILIES = {"cifar_toy", "init_research", "learned_lr", "legacy", "locked_shared", "paired_error_2d", "smart_descent", "toy100", "transfer_suite", "routed_conditioning", "toy_audit"} CONFIG_FAMILIES = {"100gaussians", "cifar_ddgan", "cifar_particle_ae", diff --git a/reports/forge/EXPERIMENT_MEMORY.md b/reports/forge/EXPERIMENT_MEMORY.md index 41fd474cf..322f68af6 100644 --- a/reports/forge/EXPERIMENT_MEMORY.md +++ b/reports/forge/EXPERIMENT_MEMORY.md @@ -11350,4 +11350,4 @@ These study and trial projections preserve recorded outcomes, unknowns and sourc ## Compilation provenance -Reducer `forge-knowledge-v2`; input digest `f954048a526ef3ba9b6f18dbddad293d6f57d826f96b4e6d8d296f54abd13d92`. [Full input hashes and coverage](compilation.json). No training or image inspection occurs during compilation. +Reducer `forge-knowledge-v2`; input digest `f96d88445dad7bf5d65a2e1227d8f3d72d142342a0b388a8b9618f58d4ea0387`. [Full input hashes and coverage](compilation.json). No training or image inspection occurs during compilation. diff --git a/reports/forge/compilation.json b/reports/forge/compilation.json index 5a7cf2127..3f1c999be 100644 --- a/reports/forge/compilation.json +++ b/reports/forge/compilation.json @@ -2,7 +2,7 @@ "automation_digest": "975631e907a1ca668dde96bd2837db3d6c54dadeacaf628069d8333c4539ee33", "conflicts": [], "import_gap_count": 7, - "input_digest": "f954048a526ef3ba9b6f18dbddad293d6f57d826f96b4e6d8d296f54abd13d92", + "input_digest": "f96d88445dad7bf5d65a2e1227d8f3d72d142342a0b388a8b9618f58d4ea0387", "input_hashes": { "configs/forge/cadence-studies/bispf-lazy-cadence-v1.json": "fc7302ac06b61fa093a39265d440545fdab176852a588c40b750be16adc1e58b", "configs/forge/calibration-lane-template.json": "65f10d3061aa487bb32da43bb97cb91934a25d763f6bb634f1d3316fdcc6d9e5", @@ -1936,7 +1936,7 @@ "reports/forge/family-winner-round1/r1r2-grid-preparation.json": "a7cdb6a87abbae05f3659cf0fbb34f6d20cc3639d846c7accd35e8a3c82e8c78", "reports/forge/family-winner-round1/r1r2-trajectory-representation.json": "8f1506391a0b5bb1da5d348c526faf04da78c362e73b4ea8402f57930638a91f", "reports/forge/history-sources.json": "c71655de9bcd7333b83ce1b5ed5b0d6bf21a0805fd9488cf5da5fba404e31d3b", - "reports/forge/import-gaps.json": "e6fdb149630cce361a222736928cdc77e3080988d97b53c3a575bdcec88164b8", + "reports/forge/import-gaps.json": "396cada13993162758b247a33b4f4e9fbe36185cd6f6aeb7fce47039e3da4118", "reports/forge/passive-publications.json": "cf4fb8e53db1896705ad70bb998e0d53d6a485783091ba02f13b9f471e9909c8", "reports/forge/policy-family-inventory.json": "b36cf5d76adc736adc09184afef38d81b708313864e0e94c38a47a75ace004dc", "reports/forge/policy-family-inventory.md": "08c9682c7233d883ef1ca8bc785c8c723245df365bb9fbd844fc05792030546c", @@ -5690,16 +5690,16 @@ "reports/toy_audit/api_contract/model_modes/publication.json": "3fe2d2f3b09d9e1c34b5dea5435e3d67a6aee30023e265acef25ffc6808fc6cd" }, "inventory_coverage": { - "declared_count": 11855, + "declared_count": 11861, "missing": [], "stale": [], - "tracked_count": 11855, + "tracked_count": 11861, "unclassified": [], "valid": true }, "operational_lifecycle_digest": "a9af4314073c4377eb6fc81c5cb20417930f4e0c3e36165961fca921a04ab15b", "output_hashes": { - "reports/forge/EXPERIMENT_MEMORY.md": "da46a59dee947e2f0011a7fab3eda6b7bc7b9e3354528c2c2629a620df983609", + "reports/forge/EXPERIMENT_MEMORY.md": "a7d59159cd96255d398fdcf477595ab253152bfab1468b93b0de5b111bfefe7d", "reports/forge/publication-records.json": "5174449ad10c00a8d22dee09d3ef50c8fb0fc76f196d85197271ee7c0766c9d8" }, "pending_readout": [], diff --git a/reports/forge/import-gaps.json b/reports/forge/import-gaps.json index fe2a91699..6e8b960ab 100644 --- a/reports/forge/import-gaps.json +++ b/reports/forge/import-gaps.json @@ -2516,10 +2516,10 @@ ], "historical_import_complete": true, "inventory_coverage": { - "declared_count": 11855, + "declared_count": 11861, "missing": [], "stale": [], - "tracked_count": 11855, + "tracked_count": 11861, "unclassified": [], "valid": true }, diff --git a/tests/test_cifar_toy_benchmark.py b/tests/test_cifar_toy_benchmark.py new file mode 100644 index 000000000..201c19df6 --- /dev/null +++ b/tests/test_cifar_toy_benchmark.py @@ -0,0 +1,120 @@ +"""Contract and semantic control tests for ``benchmarks.cifar_toy``.""" +from __future__ import annotations + +from pathlib import Path + +import pytest +import torch + +from benchmarks.cifar_toy.data import compute_class_centroids, make_synthetic_cifar_dataset +from benchmarks.cifar_toy.metrics import compute_class_coverage, compute_fid, compute_kid, feature_moments +from benchmarks.cifar_toy.models import ( + ResNet18FeatureCritic32, + ResNet34FeatureEvaluator, + TinyTransformerGenerator32, + count_matrix_conv_params, + count_trainable_params, +) +from benchmarks.cifar_toy.runner import BenchmarkConfig, run_benchmark + + +def test_architecture_exact_param_and_matrix_counts(): + gen = TinyTransformerGenerator32(z_dim=64) + disc = ResNet18FeatureCritic32(weights_path=None, backbone_mode="random") + + assert count_matrix_conv_params(gen) == 16 + assert count_trainable_params(gen) == 2_489_731 + + assert count_matrix_conv_params(disc) == 9 + assert count_trainable_params(disc) == 171_779 + + # Forward shapes match 4D RGB 32x32 -> scalar logit. + z = torch.randn(4, 64) + img = gen(z) + assert img.shape == (4, 3, 32, 32) + assert float(img.detach().min()) >= -1.0 and float(img.detach().max()) <= 1.0 + logits = disc(img) + assert logits.shape == (4, 1) + + +def test_frozen_backbone_stays_eval_and_unmodified(): + disc = ResNet18FeatureCritic32(weights_path=None, backbone_mode="random") + bn_mean_before = disc.backbone.bn1.running_mean.clone() + bn_var_before = disc.backbone.bn1.running_var.clone() + conv1_before = disc.backbone.conv1.weight.clone() + + # GANTrainer calls D.train(); backbone must stay frozen in eval mode. + disc.train(True) + assert disc.backbone.training is False + assert all(not p.requires_grad for p in disc.backbone.parameters()) + + x = torch.randn(4, 3, 32, 32, requires_grad=True) + loss = disc(x).sum() + loss.backward() + + # Gradients flow through frozen backbone to input x, without modifying backbone stats or weights. + assert x.grad is not None and torch.isfinite(x.grad).all() and float(x.grad.abs().sum()) > 0.0 + assert torch.equal(disc.backbone.bn1.running_mean, bn_mean_before) + assert torch.equal(disc.backbone.bn1.running_var, bn_var_before) + assert torch.equal(disc.backbone.conv1.weight, conv1_before) + assert all(p.grad is None for p in disc.backbone.parameters()) + + +def test_scorer_oracle_and_destructive_controls(): + ds = make_synthetic_cifar_dataset(num_samples=400, seed=0) + ev = ResNet34FeatureEvaluator(weights_path=None, backbone_mode="random") + + imgs = ds.to_float_tensor(torch.arange(400)) + feats = ev.extract(imgs, batch_size=64) + mu_r, cov_r = feature_moments(feats) + centroids = compute_class_centroids(feats, ds.labels, num_classes=10) + + # Oracle control: real vs real => FID ~ 0, KID ~ 0, 10/10 coverage. + oracle_fid = compute_fid(mu_r, cov_r, mu_r, cov_r) + oracle_kid, _ = compute_kid(feats[:200], feats[200:], subset_size=100, num_subsets=5, seed=0) + oracle_cov = compute_class_coverage(feats, centroids) + assert oracle_fid < 1e-4 + assert abs(oracle_kid) < 1e-3 + assert oracle_cov["class_coverage"] == 10.0 + + # Destructive control: constant collapsed image yields large FID/KID and collapsed coverage (1/10). + collapsed = torch.zeros_like(imgs[:200]) + bad_feats = ev.extract(collapsed, batch_size=64) + mu_b, cov_b = feature_moments(bad_feats) + bad_fid = compute_fid(mu_r, cov_r, mu_b, cov_b) + bad_kid, _ = compute_kid(feats[:200], bad_feats, subset_size=100, num_subsets=5, seed=0) + bad_cov = compute_class_coverage(bad_feats, centroids) + assert bad_fid > oracle_fid + 0.25 + assert bad_kid > oracle_kid + 1e-3 + assert bad_cov["class_coverage"] == 1.0 + + +@pytest.mark.parametrize("recipes", [("empf", "unified_v11")]) +def test_matched_initial_state_and_batch_sequence_across_trainers(tmp_path: Path, recipes): + receipts = [] + for recipe in recipes: + cfg = BenchmarkConfig( + recipe=recipe, + num_particles=16, + batch_size=8, + steps=2, + eval_every=2, + eval_samples=16, + eval_ref_samples=64, + eval_critic_samples=8, + seed=0, + dataset="synthetic", + backbone_mode="random", + device="cpu", + run_tag=f"smoke_{recipe}", + ) + receipts.append(run_benchmark(cfg, tmp_path)) + + r0, r1 = receipts + assert r0["status"] == "COMPLETE" and r1["status"] == "COMPLETE" + # Identical initial weights, initial prior, and batch sequence across trainers. + for key in ("initial_g_sha256", "initial_d_sha256", "initial_prior_sha256", "batch_sequence_digest"): + assert r0["provenance"][key] == r1["provenance"][key], f"Mismatch on {key}" + # Identical step-0 FID/logits before any recipe updates. + assert r0["history"][0]["feature_fid"] == pytest.approx(r1["history"][0]["feature_fid"], abs=1e-4) + assert r0["history"][0]["critic_margin"] == pytest.approx(r1["history"][0]["critic_margin"], abs=1e-4)