From 4ab0fc0672fb71a85444563f61947d5600a812a6 Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Sat, 17 Oct 2026 12:20:30 +0300 Subject: [PATCH 1/5] fix: explain per-event-loop construction in the async resilience cross-loop error --- src/httpware/middleware/resilience/bulkhead.py | 7 ++++--- .../middleware/resilience/circuit_breaker.py | 14 ++++++++------ tests/test_bulkhead.py | 7 ++++++- tests/test_circuit_breaker.py | 7 ++++++- tests/test_keyed_circuit_breaker.py | 7 ++++++- 5 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/httpware/middleware/resilience/bulkhead.py b/src/httpware/middleware/resilience/bulkhead.py index 58bb8f4..09111fd 100644 --- a/src/httpware/middleware/resilience/bulkhead.py +++ b/src/httpware/middleware/resilience/bulkhead.py @@ -31,9 +31,10 @@ _MAX_CONCURRENT_INVALID = "max_concurrent must be >= 1" _ACQUIRE_TIMEOUT_INVALID = "acquire_timeout must be >= 0" _ASYNCBULKHEAD_CROSS_LOOP_MSG = ( - "AsyncBulkhead is bound to a single event loop. First seen on {first!r}; " - "current request is on {current!r}. Use one AsyncBulkhead per loop; " - "cross-thread sharing requires the sync Bulkhead primitive." + "AsyncBulkhead is bound to the event loop it first ran on ({first!r}) and was called from {current!r}. " + "Create async resilience middleware, and the AsyncClient holding it, once per event loop, " + "for example inside the coroutine passed to asyncio.run(). " + "Sharing one instance across pytest-asyncio's function-scoped loops is the usual way to hit this." ) _LOGGER = logging.getLogger("httpware.bulkhead") diff --git a/src/httpware/middleware/resilience/circuit_breaker.py b/src/httpware/middleware/resilience/circuit_breaker.py index 46e4331..6ccfb51 100644 --- a/src/httpware/middleware/resilience/circuit_breaker.py +++ b/src/httpware/middleware/resilience/circuit_breaker.py @@ -75,14 +75,16 @@ _WINDOW_SECONDS_INVALID = "window_seconds must be > 0" _MINIMUM_CALLS_INVALID = "minimum_calls must be >= 1" _CROSS_LOOP_MSG = ( - "AsyncCircuitBreaker is bound to a single event loop. First seen on {first!r}; " - "current request is on {current!r}. Use one AsyncCircuitBreaker per loop; " - "cross-thread sharing requires the sync CircuitBreaker primitive." + "AsyncCircuitBreaker is bound to the event loop it first ran on ({first!r}) and was called from {current!r}. " + "Create async resilience middleware, and the AsyncClient holding it, once per event loop, " + "for example inside the coroutine passed to asyncio.run(). " + "Sharing one instance across pytest-asyncio's function-scoped loops is the usual way to hit this." ) _KEYED_CROSS_LOOP_MSG = ( - "AsyncKeyedCircuitBreaker is bound to a single event loop. First seen on {first!r}; " - "current request is on {current!r}. Use one AsyncKeyedCircuitBreaker per loop; " - "cross-thread sharing requires the sync KeyedCircuitBreaker primitive." + "AsyncKeyedCircuitBreaker is bound to the event loop it first ran on ({first!r}) and was called from {current!r}. " + "Create async resilience middleware, and the AsyncClient holding it, once per event loop, " + "for example inside the coroutine passed to asyncio.run(). " + "Sharing one instance across pytest-asyncio's function-scoped loops is the usual way to hit this." ) _DEFAULT_FAILURE_STATUS_CODES = frozenset(range(500, 600)) diff --git a/tests/test_bulkhead.py b/tests/test_bulkhead.py index 383cda6..cc054eb 100644 --- a/tests/test_bulkhead.py +++ b/tests/test_bulkhead.py @@ -495,5 +495,10 @@ async def _run_once() -> None: await client.get("https://example.test/x") asyncio.run(_run_once()) # captures loop L1, then L1 closes - with pytest.raises(RuntimeError, match="AsyncBulkhead is bound to a single event loop"): + with pytest.raises(RuntimeError) as info: asyncio.run(_run_once()) + message = str(info.value) + assert message.startswith("AsyncBulkhead is bound to the event loop it first ran on (<") + assert ") and was called from <" in message + assert "once per event loop, for example inside the coroutine passed to asyncio.run()" in message + assert "pytest-asyncio" in message diff --git a/tests/test_circuit_breaker.py b/tests/test_circuit_breaker.py index 476d6b7..56b789a 100644 --- a/tests/test_circuit_breaker.py +++ b/tests/test_circuit_breaker.py @@ -512,8 +512,13 @@ async def _run_once() -> None: await client.get("https://example.test/x") asyncio.run(_run_once()) # binds to loop L1 - with pytest.raises(RuntimeError, match="bound to a single event loop"): + with pytest.raises(RuntimeError) as info: asyncio.run(_run_once()) + message = str(info.value) + assert message.startswith("AsyncCircuitBreaker is bound to the event loop it first ran on (<") + assert ") and was called from <" in message + assert "once per event loop, for example inside the coroutine passed to asyncio.run()" in message + assert "pytest-asyncio" in message # ── rate-mode config validation ── diff --git a/tests/test_keyed_circuit_breaker.py b/tests/test_keyed_circuit_breaker.py index a08dfc2..4857ce6 100644 --- a/tests/test_keyed_circuit_breaker.py +++ b/tests/test_keyed_circuit_breaker.py @@ -214,8 +214,13 @@ async def _run_once(url: str) -> None: await client.get(url) asyncio.run(_run_once("https://a.test/x")) - with pytest.raises(RuntimeError, match="AsyncKeyedCircuitBreaker is bound to a single event loop"): + with pytest.raises(RuntimeError) as info: asyncio.run(_run_once("https://b.test/x")) + message = str(info.value) + assert message.startswith("AsyncKeyedCircuitBreaker is bound to the event loop it first ran on (<") + assert ") and was called from <" in message + assert "once per event loop, for example inside the coroutine passed to asyncio.run()" in message + assert "pytest-asyncio" in message class _Held: From 5491c981bf7fe5a2a93e7135fc3a9204d32b6a8e Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Sat, 17 Oct 2026 12:22:53 +0300 Subject: [PATCH 2/5] docs: recommend the circuit breaker before AsyncTimeout so timeouts count as failures --- docs/adr/0004-circuit-breaker-scope.md | 20 ++++++++----- docs/resilience.md | 13 ++++---- src/httpware/middleware/resilience/timeout.py | 10 ++++--- tests/test_circuit_breaker.py | 30 +++++++++++++++++++ 4 files changed, 57 insertions(+), 16 deletions(-) diff --git a/docs/adr/0004-circuit-breaker-scope.md b/docs/adr/0004-circuit-breaker-scope.md index 8af311c..21cd639 100644 --- a/docs/adr/0004-circuit-breaker-scope.md +++ b/docs/adr/0004-circuit-breaker-scope.md @@ -2,10 +2,16 @@ Rate mode uses a rolling `window_seconds` gated by `minimum_calls`. A count-based "last N calls" window was rejected because on low traffic it reaches back past deploys and recoveries, and -`minimum_calls` already supplies the sample-size guarantee. A slow-call rate is not added because -`AsyncTimeout` already converts a slow call into a counted `TimeoutError`. `force_open()` / -`force_closed()` in the style of Polly's `ManualControl` were declined in -[#118](https://github.com/modern-python/httpware/issues/118): breaker state is per process, so -pinning one replica sheds nothing fleet-wide, and the fixes that do work, a config flag read by every -replica or state in Redis, reduce to a check the caller makes before sending or to I/O inside a -transition. +`minimum_calls` already supplies the sample-size guarantee. A slow-call rate is not added because a +deadline already turns a slow call into a counted failure, provided the breaker sits outside the +deadline. That holds in the recommended order, `AsyncCircuitBreaker → AsyncTimeout → AsyncBulkhead +→ AsyncRetry`: an expired `AsyncTimeout` raises httpware `TimeoutError` through the breaker, which +counts it. With `AsyncTimeout` outside the breaker, the deadline cancels the breaker's call, the +breaker sees `CancelledError`, which it does not count, and a hanging upstream never opens the +circuit. httpware documents this order but does not enforce it, so the argument covers only callers +who follow it. httpx2's per-request timeouts are raised inside the breaker in any order, and they +are the only deadline on the sync side. `force_open()` / `force_closed()` in the style of Polly's +`ManualControl` were declined in [#118](https://github.com/modern-python/httpware/issues/118): +breaker state is per process, so pinning one replica sheds nothing fleet-wide, and the fixes that do +work, a config flag read by every replica or state in Redis, reduce to a check the caller makes +before sending or to I/O inside a transition. diff --git a/docs/resilience.md b/docs/resilience.md index a42edf2..cd0a2c8 100644 --- a/docs/resilience.md +++ b/docs/resilience.md @@ -180,6 +180,8 @@ An outcome only counts against the state its request was admitted under. A reque A counted failure is a `NetworkError`, an httpware `TimeoutError`, or a `StatusError` whose status is in `failure_status_codes`. Other exceptions pass through without changing the circuit. +An expired `AsyncTimeout` deadline counts only when the breaker comes before `AsyncTimeout`, as in the [composition order](#composition). With the breaker after it, the deadline cancels the breaker's call, and a cancellation is not a counted failure. + A `RetryBudgetExhaustedError` from a retry placed after the breaker is classified by its `last_exception`. If the last attempt failed with a counted failure, the breaker counts one; if it failed with a 4xx, that is a success. 4xx responses, 429 included, count as successes. A 429 means the service is up and throttling you; opening the circuit on it would add circuit-open rejections on top of the throttling. @@ -309,7 +311,7 @@ In the [composition order](#composition), the keyed breaker takes the circuit br from httpware.middleware.resilience import AsyncTimeout ``` -Bounds the total time of everything after it in the chain. Put it first so a call, with all its retries and backoff, finishes within `timeout` seconds. When time runs out it raises `httpware.TimeoutError`. +Bounds the total time of everything after it in the chain. Put it right after the circuit breaker and before `AsyncBulkhead` and `AsyncRetry`, so a call, with all its retries and backoff, finishes within `timeout` seconds and the breaker counts an expired deadline as a failure. When time runs out it raises `httpware.TimeoutError`. | Parameter | Default | Effect | |---|---|---| @@ -319,17 +321,18 @@ Bounds the total time of everything after it in the chain. Put it first so a cal There is no sync `Timeout`, because sync Python cannot interrupt a blocking httpx2 call midway. In sync code, set `httpx2.Timeout` on the client or pass `timeout=` per request. -On expiry it emits `timeout.exceeded` on `httpware.timeout`. [Composition](#composition) has an example with `AsyncTimeout` first in the chain. +On expiry it emits `timeout.exceeded` on `httpware.timeout`. [Composition](#composition) has an example with `AsyncTimeout` in the recommended place. ## Composition httpware doesn't enforce an order, but this one is recommended: ``` -AsyncTimeout → AsyncCircuitBreaker → AsyncBulkhead → AsyncRetry → terminal +AsyncCircuitBreaker → AsyncTimeout → AsyncBulkhead → AsyncRetry → terminal ``` -- `AsyncTimeout` comes first, so the deadline covers every retry and backoff. +- `AsyncCircuitBreaker` comes before `AsyncTimeout`, so an expired deadline reaches the breaker as an httpware `TimeoutError`, which it counts as a failure, and an upstream that hangs opens the circuit. In the reverse order the deadline fires while the breaker is still waiting on its inner call: the breaker sees a `CancelledError`, which it doesn't count, and the circuit stays closed however long the upstream hangs. The breaker itself never waits, so the deadline loses nothing by starting after it. An open circuit still rejects at once, and the half-open probe runs under the deadline like any other call. +- `AsyncTimeout` comes before `AsyncBulkhead` and `AsyncRetry`, so the deadline covers the wait for a slot, every retry and every backoff. - `AsyncCircuitBreaker` comes before `AsyncRetry`, so an open circuit rejects the call without starting the retry loop, and the breaker counts one outcome per retry sequence instead of one per attempt. Putting it before `AsyncBulkhead` too means a rejected call never takes a slot. - `AsyncBulkhead` comes before `AsyncRetry`, so one slot covers every attempt of a call. In the reverse order each retry takes a new slot, and under load the bulkhead admits more work than its cap intends. @@ -347,8 +350,8 @@ async def main() -> None: async with AsyncClient( base_url="https://api.example.com", middleware=[ - AsyncTimeout(timeout=30.0), AsyncCircuitBreaker(), + AsyncTimeout(timeout=30.0), AsyncBulkhead(max_concurrent=10), AsyncRetry(), ], diff --git a/src/httpware/middleware/resilience/timeout.py b/src/httpware/middleware/resilience/timeout.py index 17730e0..574201e 100644 --- a/src/httpware/middleware/resilience/timeout.py +++ b/src/httpware/middleware/resilience/timeout.py @@ -4,8 +4,9 @@ right tool for bounding a single outbound call, and AsyncTimeout does not duplicate them. What httpx2 cannot bound is the total wall-clock across the whole middleware pipeline (most importantly across an AsyncRetry loop, whose attempts and backoff -sleeps it knows nothing about). Place AsyncTimeout outermost to enforce -"this whole operation must finish within `timeout` seconds, even across retries." +sleeps it knows nothing about). Place AsyncTimeout right after the circuit breaker, +before bulkhead and retry, to enforce "this whole operation must finish within +`timeout` seconds, even across retries" and let the breaker count an expired deadline. Async-only by design: a sync total-deadline cannot interrupt a blocking httpx2 call mid-flight (sync Python has no cancellation), and httpx2 already covers sync per-call @@ -38,8 +39,9 @@ class AsyncTimeout: including everything it wraps (retries, backoff sleeps, the call itself). Must be ``> 0``. On expiry the middleware raises ``httpware.TimeoutError``. - Place outermost in the chain for an overall-operation deadline. For bounding a - single outbound call (connect/read/write/pool), configure ``httpx2`` instead. + Place right after the circuit breaker, before bulkhead and retry, so the breaker + counts an expired deadline. For bounding a single outbound call + (connect/read/write/pool), configure ``httpx2`` instead. """ diff --git a/tests/test_circuit_breaker.py b/tests/test_circuit_breaker.py index 56b789a..b4c9b02 100644 --- a/tests/test_circuit_breaker.py +++ b/tests/test_circuit_breaker.py @@ -18,6 +18,7 @@ from httpware import ( AsyncClient, AsyncRetry, + AsyncTimeout, CircuitOpenError, CircuitState, InternalServerError, @@ -500,6 +501,35 @@ async def _handler_async(request: httpx2.Request) -> httpx2.Response: await probe_task +# ── composition: breaker outside AsyncTimeout ── + + +async def test_breaker_before_timeout_counts_expired_deadlines() -> None: + """In the recommended order a hanging upstream opens the circuit. + + With AsyncTimeout outside the breaker, the deadline cancels the breaker's call and + the CancelledError is not a counted failure, so the circuit would never open. + """ + sent: list[httpx2.Request] = [] + + async def _hang(request: httpx2.Request) -> httpx2.Response: + sent.append(request) + return await asyncio.get_running_loop().create_future() + + breaker = AsyncCircuitBreaker(failure_threshold=2, _now=_Clock()) + async with AsyncClient( + httpx2_client=httpx2.AsyncClient(transport=httpx2.MockTransport(_hang)), + middleware=[breaker, AsyncTimeout(timeout=0.01)], + ) as client: + for _ in range(2): + with pytest.raises(TimeoutError): + await client.get("https://example.test/x") + with pytest.raises(CircuitOpenError): + await client.get("https://example.test/x") + assert breaker.state is CircuitState.OPEN + assert len(sent) == 2 # noqa: PLR2004 + + # ── single-event-loop guard ── From b66067bcc48d5705e8defbff953f902a2ef2a939 Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Sat, 17 Oct 2026 12:22:53 +0300 Subject: [PATCH 3/5] docs: explain that a success can open the circuit in rate mode --- docs/resilience.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/resilience.md b/docs/resilience.md index cd0a2c8..0860b2d 100644 --- a/docs/resilience.md +++ b/docs/resilience.md @@ -222,6 +222,8 @@ breaker = AsyncCircuitBreaker( ) ``` +Once the window holds at least `minimum_calls` outcomes, the rate is checked after every outcome, successes included. A success can therefore be the call that opens the circuit. With `minimum_calls=4` and a threshold of `0.5`, three failures leave the circuit closed because the window holds only three outcomes; a fourth call that succeeds brings it to four, the rate is 3/4, and the circuit opens. That call still returns its response to the caller, and the next request is rejected with `CircuitOpenError`. + In rate mode `failure_threshold` is ignored. Recovery through HALF_OPEN works the same in both modes, and the sync `CircuitBreaker` takes the same parameters. ### State introspection From aae2cee34ae58a3e58dd23557ab03d29ae77b869 Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Sat, 17 Oct 2026 12:22:53 +0300 Subject: [PATCH 4/5] docs: describe the one-event-loop-per-instance rule for async resilience middleware --- docs/resilience.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/resilience.md b/docs/resilience.md index 0860b2d..db6229d 100644 --- a/docs/resilience.md +++ b/docs/resilience.md @@ -363,6 +363,32 @@ async def main() -> None: Your own middleware that sets something per call, like the request-ID middleware in [Middleware](middleware.md), also belongs before `AsyncRetry`, so every attempt of a call shares one ID. +## One event loop per instance + +`AsyncBulkhead`, `AsyncCircuitBreaker` and `AsyncKeyedCircuitBreaker` are bound to the event loop they first run on. Using the same instance from another loop raises `RuntimeError`, and so does a later `asyncio.run()` in the same thread, because each `asyncio.run()` starts a new loop. `httpx2.AsyncClient` can't be reused across `asyncio.run()` calls either. Create the async resilience middleware, and the `AsyncClient` holding it, once per event loop, for example inside the coroutine you pass to `asyncio.run()`: + +```python +import asyncio + +from httpware import AsyncClient +from httpware.middleware.resilience import AsyncCircuitBreaker, AsyncRetry + + +async def main() -> None: + async with AsyncClient( + base_url="https://api.example.com", + middleware=[AsyncCircuitBreaker(), AsyncRetry()], + ) as client: + await client.get("/users/1") + + +asyncio.run(main()) +``` + +The usual way to hit the error is in tests: pytest-asyncio gives each test function its own loop by default, so an instance created at module level or in a session-scoped fixture fails in the second test that uses it. Create it in a function-scoped fixture or in the test. + +To share a cap or a circuit across threads, use the sync classes with `Client`. + ## Sync Retry and Bulkhead Use these with the sync `Client`. From a6730fbf51cca9cb8a5bf4fd790a459552acd11f Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Sat, 17 Oct 2026 12:28:54 +0300 Subject: [PATCH 5/5] docs: align the full-stack demo with the breaker-before-timeout order --- docs/demos/engine.js | 26 ++++++++++++-------------- docs/demos/full-stack.md | 26 ++++++++++++++------------ 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/demos/engine.js b/docs/demos/engine.js index 01d5ecd..be8a084 100644 --- a/docs/demos/engine.js +++ b/docs/demos/engine.js @@ -793,11 +793,10 @@ window.HttpwareDemo = (function () { if (L.pend[i].land <= tick) { const p = L.pend[i]; L.pend.splice(i, 1); let retried = false; - // AsyncTimeout bounds the WHOLE operation (outermost: timeout -> - // circuitBreaker -> bulkhead -> retry -> terminal). An entry clamped - // to its deadline (see clampToDeadline) lands as a TimeoutError - // failure here, regardless of what the untimed outcome would have - // been, and is never retried. + // AsyncTimeout bounds the WHOLE operation (circuitBreaker -> timeout -> + // bulkhead -> retry -> terminal). An entry clamped to its deadline (see + // clampToDeadline) lands as a TimeoutError failure here, regardless of + // what the untimed outcome would have been, and is never retried. if (L === B && p.timedOut) { timedOutCount++; // Retry decision happens at landing time: a failed lane-B attempt gets @@ -811,7 +810,7 @@ window.HttpwareDemo = (function () { const backoffSec = rnd() * Math.min(retryCfg.maxDelay, retryCfg.baseDelay * Math.pow(2, p.attempt)); const f2 = scenario.fault(now + backoffSec, rnd); const landTicks = Math.max(1, Math.round((backoffSec + f2.ms) * 1000 / TICK)); - // Composition order is timeout -> circuitBreaker -> bulkhead -> retry -> + // Composition order is circuitBreaker -> timeout -> bulkhead -> retry -> // terminal: the bulkhead sits OUTSIDE retry, so one slot is acquired // once and held for the WHOLE operation (all attempts), released only // when the operation finally lands. Carry `bh` forward on every retry @@ -833,14 +832,12 @@ window.HttpwareDemo = (function () { // so a null/absent bulkhead or a rejected (never-pushed) request is a no-op. // Breaker records ONCE per fully-exhausted retry sequence (not per attempt): // brk.res is called here, only on the terminal (non-retried) landing. A - // timed-out entry is an outer-timeout cancellation: AsyncTimeout is - // OUTERMOST, so the inner breaker's `except BaseException` clause does see - // the CancelledError (it releases the in-flight probe slot) but does not - // count it as a success/failure outcome — only the outer TimeoutError - // surfaces to the caller. + // timed-out entry counts as a failure: the breaker sits OUTSIDE + // AsyncTimeout, so the TimeoutError raised on expiry passes through it + // as a counted failure (a CancelledError would not be counted). if (!retried) { L.if--; - if (L === B && p.timedOut) { L.bad++; } + if (L === B && p.timedOut) { if (brk) brk.res(false, now); L.bad++; } else { if (L === B && brk) brk.res(p.ok, now); if (p.ok) L.ok++; else L.bad++; } if (L === B && bulk && p.bh) bulk.release(); } @@ -864,8 +861,9 @@ window.HttpwareDemo = (function () { B.rej++; spawnDot(dotsB, els.trackB, 'rej'); } else { if (budget) budget.deposit(now); - // Total deadline set once at the first attempt (Infinity when no - // AsyncTimeout is configured, so the clamp below is always a no-op). + // Total deadline set once at the first attempt, after the breaker admitted + // the request, so an open circuit rejects before any deadline starts + // (Infinity when no AsyncTimeout is configured: the clamp is a no-op). const deadline = tmoCfg ? now + tmoCfg.timeout : Infinity; const { land, timedOut } = clampToDeadline(tick, landTicks, deadline); B.if++; B.pend.push({ land, ok: f.ok, attempt: 0, bh: !!bulk, start: now, deadline, timedOut }); diff --git a/docs/demos/full-stack.md b/docs/demos/full-stack.md index d2f4fb7..502a022 100644 --- a/docs/demos/full-stack.md +++ b/docs/demos/full-stack.md @@ -1,7 +1,7 @@ # Full stack: combining the patterns Real clients combine these patterns, in the recommended order -`AsyncTimeout -> AsyncCircuitBreaker -> AsyncBulkhead -> AsyncRetry -> terminal`. +`AsyncCircuitBreaker -> AsyncTimeout -> AsyncBulkhead -> AsyncRetry -> terminal`. Here both clients go through an incident in three phases, and each layer handles a different phase. @@ -13,30 +13,32 @@ document.addEventListener('DOMContentLoaded', function () { scenarios: [ { id: 'incident', label: 'Multi-phase incident', dur: 16.0, fault: (now, rnd) => { - if (now >= 2.0 && now < 5.0) return { ok: true, ms: 4.0, label: 'latency spike' }; - if (now >= 5.0 && now < 8.0) return rnd() < 0.6 + if (now >= 2.0 && now < 5.0) return rnd() < 0.6 ? { ok: false, ms: 0.6, label: 'brownout' } : { ok: true, ms: 0.6 }; + if (now >= 5.0 && now < 8.0) return { ok: true, ms: 4.0, label: 'latency spike' }; if (now >= 8.0 && now < 12.0) return { ok: false, ms: 0.3, label: 'hard down' }; return { ok: true, ms: 0.05 }; }, - chainB: { timeout: { timeout: 2.0 }, - circuitBreaker: { failureThreshold: 5, resetTimeout: 2.0, successThreshold: 1 }, + chainB: { circuitBreaker: { failureThreshold: 5, resetTimeout: 2.0, successThreshold: 1 }, + timeout: { timeout: 2.0 }, bulkhead: { maxConcurrent: 10, acquireTimeout: 0 }, retry: { maxAttempts: 3, baseDelay: 0.1, maxDelay: 5.0 }, budget: { ttl: 10.0, minRetriesPerSec: 10.0, percentCanRetry: 0.2 } } }, ], macroStrip: true, stageLabel: (now) => now < 2 ? 'healthy' - : now < 5 ? 'phase 1: latency spike' - : now < 8 ? 'phase 2: brownout' + : now < 5 ? 'phase 1: brownout' + : now < 8 ? 'phase 2: latency spike' : now < 12 ? 'phase 3: hard down' : 'recovered', buildStops: () => [ - { when: (s) => s.now >= 2.4, spot: ['poolB', 'elapsedB'], title: 'Phase 1: latency spike', - body: 'The bulkhead caps concurrency so the slow phase cannot exhaust the client, and the timeout ends each operation after 2s.' }, - { when: (s) => s.now >= 5.4, spot: ['ifB'], title: 'Phase 2: brownout', + { when: (s) => s.now >= 2.4, spot: ['ifB'], title: 'Phase 1: brownout', body: 'Retry recovers many of the transient errors, and the budget stops it from adding much load.' }, - { when: (s) => s.mw.cb && s.mw.cb.state === 'OPEN', spot: ['brkB'], title: 'Phase 3: hard down', - body: 'Repeated failures open the breaker. It sits before the retry layer, so it rejects whole retry sequences and counts one outcome per sequence, not per attempt.' }, + { when: (s) => s.now >= 5.4, spot: ['poolB', 'elapsedB'], title: 'Phase 2: latency spike', + body: 'The bulkhead caps concurrency so the slow upstream cannot exhaust the client, and the timeout ends each operation after 2s.' }, + { when: (s) => s.mw.cb && s.mw.cb.state === 'OPEN', spot: ['brkB'], title: 'Timeouts open the breaker', + body: 'The breaker sits before the timeout, so each expired deadline counts as a failure. After five in a row the circuit opens, and new calls fail at once instead of waiting 2s on the slow upstream.' }, + { when: (s) => s.now >= 8.5, spot: ['brkB'], title: 'Phase 3: hard down', + body: 'The circuit stays open through the outage, because each half-open probe fails and reopens it. It sits before the retry layer, so it rejects whole retry sequences and counts one outcome per sequence, not per attempt.' }, { when: (s) => s.now >= 13.5, spot: ['ifA', 'latA', 'ifB', 'latB'], title: 'Full stack vs none', body: 'The plain client fails through every phase. In the httpware client, each layer handles the phase it is built for.' }, ],