fix: bound DTLS handshake maintenance

bench/dtls-maintenance.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Compare original and fixed DTLS maintenance with identical benchmark clients."""
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import platform
+import re
+import signal
+import statistics
+import subprocess
+
+ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
+ROW = re.compile(r"\b([14])T\s+(0|100|1000)B\s+([\d,]+)\s+([\d.]+|-)\s+([\d.]+|-)\s+([\d.]+|-)\s+([\d,]+)")
+
+
+def run(command):
+    with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                          text=True, start_new_session=True) as process:
+        try:
+            stdout, stderr = process.communicate(timeout=120)
+        except BaseException:
+            try:
+                os.killpg(process.pid, signal.SIGTERM)
+                process.communicate(timeout=5)
+            except subprocess.TimeoutExpired:
+                os.killpg(process.pid, signal.SIGKILL)
+                process.communicate()
+            except ProcessLookupError:
+                pass
+            raise
+        if process.returncode:
+            raise RuntimeError(stdout + stderr)
+        return ANSI.sub("", stdout + stderr)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("baseline", type=Path)
+    parser.add_argument("fixed", type=Path)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--count", type=int, default=300000)
+    parser.add_argument("--repetitions", type=int, default=5)
+    parser.add_argument("--plain", action="store_true", help="Plain CON control instead of DTLS")
+    args = parser.parse_args()
+    if min(args.count, args.repetitions) <= 0:
+        parser.error("counts must be positive")
+    binaries = {key: value.resolve() for key, value in (("baseline", args.baseline), ("fixed", args.fixed))}
+    data = {"machine": platform.platform(), "zig": subprocess.check_output(["zig", "version"], text=True).strip(),
+            "binary_sha256": {key: hashlib.sha256(path.read_bytes()).hexdigest() for key, path in binaries.items()},
+            "count": args.count, "warmup": 1000, "single_window": 32, "multi_window_per_worker": 64,
+            "transport": "plain" if args.plain else "dtls", "repetitions": args.repetitions, "samples": []}
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    for threads in (1, 4):
+        for payload in (0, 100, 1000):
+            for repeat in range(args.repetitions):
+                order = ("baseline", "fixed") if repeat % 2 == 0 else ("fixed", "baseline")
+                for backend in order:
+                    command = [str(binaries[backend]), "--plain-only" if args.plain else "--dtls-only", "--con-only",
+                               "--single-only" if threads == 1 else "--multi-only", "--threads", "4",
+                               "--window", "32", "--payload-only", str(payload), "--count", str(args.count), "--warmup", "1000"]
+                    output = run(command)
+                    matches = list(ROW.finditer(output))
+                    if len(matches) != 1:
+                        raise RuntimeError(output)
+                    values = matches[0].groups()
+                    rps, errors = int(values[2].replace(",", "")), int(values[6].replace(",", ""))
+                    if (int(values[0]), int(values[1])) != (threads, payload) or not rps or errors:
+                        raise RuntimeError(f"Invalid or failed benchmark: {output}")
+                    sample = dict(backend=backend, threads=threads, payload=payload, repeat=repeat,
+                                  rps=rps, p50_us=float(values[3].replace("-", "0")),
+                                  p99_us=float(values[4].replace("-", "0")), p999_us=float(values[5].replace("-", "0")),
+                                  errors=errors, command=command, output=output)
+                    data["samples"].append(sample)
+                    args.output.write_text(json.dumps(data, indent=2) + "\n")
+                    print(f"{backend:8} {threads}T {payload:4}B run {repeat + 1}: {rps:,} req/s", flush=True)
+    print("\nArithmetic means:")
+    for threads in (1, 4):
+        for payload in (0, 100, 1000):
+            means = {key: statistics.mean(s["rps"] for s in data["samples"]
+                     if (s["backend"], s["threads"], s["payload"]) == (key, threads, payload)) for key in binaries}
+            print(f"{threads}T {payload:4}B: {means['baseline']:,.0f} -> {means['fixed']:,.0f} ({means['fixed']/means['baseline']:.3f}x)")
+
+
+if __name__ == "__main__":
+    main()

docs/DTLS_MAINTENANCE.md

@@ -0,0 +1,89 @@
+# DTLS handshake maintenance fix
+
+Baseline: `39c42dc` (unmodified original implementation). Fixed version uses the
+same benchmark, crypto, client pipeline, capacity and socket settings. This is
+not the Zio experiment and changes no public API or configured limits.
+
+## Cause and fix
+
+`Server.tick()` called `checkDtlsRetransmits()` after every completion batch.
+That loop visited all 65,536 allocated capacity slots even with one established
+session and no pending handshake. A one-million-request profile attributed
+70.2% of combined client/server sampled user CPU to that state check.
+
+The fix checks at most once per 50ms using the existing idle tick wakeups and
+walks the existing live-session LRU chain, not free capacity. Only handshaking
+sessions with cached flights can retransmit. The successor is saved before
+release so timeouts can unlink head/interior/tail nodes safely. The cadence
+adds up to one check interval plus existing tick processing/wakeup latency;
+handshake retry counts, cached bytes and exponential-backoff limits are unchanged.
+
+No capacity reduction, new collection, allocation or thread was introduced.
+
+## Tests
+
+183/183 tests pass in Debug and ReleaseSafe. ReleaseFast library, examples and
+benchmark build. New regressions cover expired head/tail removal, established
+sessions with cached flights, slot reuse, empty live lists, cadence boundaries,
+uncached handshakes, retransmission bytes and capped exponential backoff.
+
+## Throughput
+
+AMD Ryzen AI MAX+ 395, Linux 7.1.11-arch1-1, Zig 0.16.0, ReleaseFast.
+Each DTLS scenario: 300,000 measured requests, 1,000 warmup, five repetitions,
+alternating original/fixed execution order, one new embedded server per run.
+Single-worker window 32; four-worker window 64 per worker. All runs report
+nonzero throughput and zero errors. Arithmetic means, requests/second:
+
+| Workers | Payload | Original | Fixed | Change |
+|---:|---:|---:|---:|---:|
+| 1 | 0 | 183,498 | 182,031 | -0.8% |
+| 1 | 100 | 176,570 | 177,729 | +0.7% |
+| 1 | 1000 | 144,206 | 147,581 | +2.3% |
+| 4 | 0 | 139,873 | 158,603 | +13.4% |
+| 4 | 100 | 140,281 | 150,623 | +7.4% |
+| 4 | 1000 | 154,925 | 178,363 | +15.1% |
+
+Across all payloads, means improve **0.6% single-worker** and **12.1%
+four-worker**. There is no demonstrated 2x throughput gain from this fix alone.
+
+An additional 36-run plain-CON control (100,000 requests, three repetitions)
+changes aggregate mean throughput by +0.3% single-worker and -0.3% four-worker.
+Individual short multi-worker measurements vary substantially; no statistical
+significance is claimed for those small aggregate differences.
+
+## Follow-up profile
+
+Same 100-byte, window-32 workload, one million measured requests. Sampling uses
+`perf record -m 1 -e cpu-clock -F 499 -g`; small perf buffers avoid exhausting
+locked-memory resources needed by io_uring. Invalid zero-throughput profiling
+attempts were discarded, never treated as successful runs.
+
+After the fix, `Server.tick` accounts for 8.7% of sampled user CPU; the old
+capacity-scan hotspot is absent. Combined sampled user CPU event count fell
+from approximately 4.81 to 2.39 CPU-seconds. A separate `perf stat` pair reported
+user CPU 4.48 -> 2.24 seconds, while wall time was 6.20 -> 5.98 seconds. These
+single profiling pairs include setup/warmup and are diagnostic, not repeated
+performance estimates. System CPU remained substantial.
+
+The next large user-space hotspot is `memset` (34.8%), with CCM authentication
+14.4%. Removing the full-capacity scan saves CPU but does not eliminate the
+original client/I/O/remaining tick costs. The previous Zio whole-system speed
+gap must not be attributed exclusively to this loop.
+
+## Reproduction
+
+Build the original and fixed checkouts with `zig build -Doptimize=ReleaseFast`.
+From the fixed checkout:
+
+```sh
+python3 bench/dtls-maintenance.py ../zio-baseline/zig-out/bin/bench \
+  zig-out/bin/bench --output docs/DTLS_MAINTENANCE_RESULTS.json
+python3 bench/dtls-maintenance.py ../zio-baseline/zig-out/bin/bench \
+  zig-out/bin/bench --plain --count 100000 --repetitions 3 \
+  --output docs/DTLS_MAINTENANCE_PLAIN_RESULTS.json
+```
+
+Raw JSON includes executable SHA-256, every command/output and latency
+percentiles. The harness rejects zero throughput, errors or wrong scenarios
+and cleans isolated process groups on interruption/timeout.