From 8dde1de4fa5b78d811124449f9297f6580356d15 Mon Sep 17 00:00:00 2001 From: ai-ag2026 <261867348+ai-ag2026@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:22:31 +0200 Subject: [PATCH 1/5] fix(bg): back off drain loop instead of silent tight-spin (#2476, #4633) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _drain_loop read process_registry.completion_queue directly under a bare `except Exception: continue`. A registry missing that attribute raised AttributeError that was swallowed and retried with no backoff — a 100%-CPU tight loop with no log line. The loop now reads the queue via getattr(..., None) and backs off on the stop event when it is absent (mirroring streaming.py), catches queue.Empty explicitly for the normal idle path, and logs a warning + backs off (_DRAIN_STOP.wait(1.0)) on any other queue error so a persistent fault can neither spin the CPU nor stay invisible. Co-Authored-By: Claude Opus 4.8 (1M context) --- api/background_process.py | 22 +++- tests/test_bg_task_drain_loop_backoff.py | 129 +++++++++++++++++++++++ 2 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 tests/test_bg_task_drain_loop_backoff.py diff --git a/api/background_process.py b/api/background_process.py index 7147291d1..754c31c3b 100644 --- a/api/background_process.py +++ b/api/background_process.py @@ -1325,10 +1325,28 @@ def _drain_loop() -> None: return logger.info("bg_task_complete drain thread started") while not _DRAIN_STOP.is_set(): + # Read the queue defensively: a rebuilt/partially-initialized registry + # may not expose ``completion_queue`` (mirrors streaming.py's + # ``getattr(process_registry, 'completion_queue', None)`` guard). Direct + # attribute access here would raise AttributeError, which the old broad + # ``except Exception: continue`` swallowed silently and re-tried with no + # backoff — a 100%-CPU tight loop. Back off on the stop event instead. + q = getattr(process_registry, "completion_queue", None) + if q is None: + _DRAIN_STOP.wait(1.0) + continue try: - evt = process_registry.completion_queue.get(timeout=1.0) + evt = q.get(timeout=1.0) + except queue.Empty: + # Nothing to drain this second — re-check the stop flag and loop. + continue except Exception: - # queue.Empty or transient — re-check stop flag and continue. + # Unexpected queue failure: log it (not silent) and back off on the + # stop event so a persistent error can't spin the thread hot. + logger.warning( + "bg_task_complete drain queue read failed", exc_info=True + ) + _DRAIN_STOP.wait(1.0) continue if not isinstance(evt, dict): continue diff --git a/tests/test_bg_task_drain_loop_backoff.py b/tests/test_bg_task_drain_loop_backoff.py new file mode 100644 index 000000000..f8f927b85 --- /dev/null +++ b/tests/test_bg_task_drain_loop_backoff.py @@ -0,0 +1,129 @@ +"""Regression test for the bg_task_complete drain-loop tight-spin (#2476, #4633). + +``_drain_loop`` read ``process_registry.completion_queue`` directly and wrapped +it in a bare ``except Exception: continue``. A registry missing that attribute +raised ``AttributeError`` that was swallowed and retried with no backoff — a +100%-CPU tight loop with no log line. The loop now: + + * reads the queue via ``getattr(process_registry, 'completion_queue', None)`` + and backs off on the stop event when it is absent (mirrors streaming.py), + * catches ``queue.Empty`` explicitly (the normal idle path → plain continue), + * logs + backs off on any other queue error instead of tight-looping. + +Uses the shared ``install_fake_registry`` stub helper (see ``_wakeup_helpers``). +""" + +from __future__ import annotations + +import queue +import threading + +from _wakeup_helpers import install_fake_registry + + +class _CountingStop: + """Stand-in for ``_DRAIN_STOP`` that records ``wait()`` and self-terminates. + + In the OLD (buggy) code the missing-queue path hit ``continue`` and never + touched the stop event's ``wait`` — it just re-spun on ``is_set()``. So a + recorded ``wait(1.0)`` call is exactly the proof the backoff path was taken. + ``wait`` sets the flag after ``exit_after`` calls so the loop exits promptly + without any real sleep, keeping the test deterministic and fast. + """ + + def __init__(self, exit_after: int = 1): + self._flag = threading.Event() + self.wait_calls: list[float | None] = [] + self._exit_after = exit_after + + def is_set(self) -> bool: + return self._flag.is_set() + + def set(self) -> None: + self._flag.set() + + def clear(self) -> None: + self._flag.clear() + + def wait(self, timeout=None): + self.wait_calls.append(timeout) + if len(self.wait_calls) >= self._exit_after: + self._flag.set() + return self._flag.is_set() + + +def _run_drain_with_stop(monkeypatch, fake_registry, stop) -> threading.Thread: + from api import background_process as bp + + install_fake_registry(monkeypatch, fake_registry) + monkeypatch.setattr(bp, "_DRAIN_STOP", stop, raising=True) + t = threading.Thread(target=bp._drain_loop, name="test-drain", daemon=True) + t.start() + return t + + +def test_missing_completion_queue_backs_off_instead_of_spinning(monkeypatch): + """A registry with no completion_queue must back off, not tight-loop.""" + + class _NoQueueRegistry: + pass # deliberately no completion_queue attribute + + stop = _CountingStop(exit_after=1) + t = _run_drain_with_stop(monkeypatch, _NoQueueRegistry(), stop) + t.join(timeout=3.0) + + assert not t.is_alive(), "drain loop did not terminate on stop" + # The backoff path ran: wait() was called with the 1.0s backoff timeout. + assert stop.wait_calls, "missing-queue path did not back off (tight loop)" + assert stop.wait_calls[0] == 1.0 + + +def test_non_empty_queue_error_logs_and_backs_off(monkeypatch): + """A non-Empty queue error is logged and backed off, not swallowed silently.""" + + class _BoomQueue: + def get(self, timeout=None): + raise RuntimeError("queue exploded") + + class _BoomRegistry: + completion_queue = _BoomQueue() + + stop = _CountingStop(exit_after=1) + t = _run_drain_with_stop(monkeypatch, _BoomRegistry(), stop) + t.join(timeout=3.0) + + assert not t.is_alive() + assert stop.wait_calls and stop.wait_calls[0] == 1.0 + + +def test_empty_queue_is_the_plain_continue_path(monkeypatch): + """queue.Empty is the normal idle path: continue, no backoff wait().""" + + class _EmptyQueue: + def __init__(self, stop): + self._stop = stop + + def get(self, timeout=None): + # Stop the loop after being polled once so the test terminates, + # then raise Empty like a real idle queue. + self._stop.set() + raise queue.Empty + + class _EmptyRegistry: + pass + + stop = _CountingStop(exit_after=99) # effectively never self-terminates + reg = _EmptyRegistry() + reg.completion_queue = _EmptyQueue(stop) + t = _run_drain_with_stop(monkeypatch, reg, stop) + t.join(timeout=3.0) + + assert not t.is_alive() + # Empty took the explicit `continue` path — no backoff wait() was recorded. + assert stop.wait_calls == [] + + +if __name__ == "__main__": # pragma: no cover - manual invocation + import pytest + + raise SystemExit(pytest.main([__file__, "-v"])) From 141b49b74c5a56883759c61b1186c57eb3d37fe3 Mon Sep 17 00:00:00 2001 From: ai-ag2026 Date: Wed, 8 Jul 2026 23:04:43 +0200 Subject: [PATCH 2/5] test(bg): fix drain-loop backoff test import so it actually collects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The regression test used a bare `from _wakeup_helpers import install_fake_registry`, which raises ModuleNotFoundError under the repo test runner (rootdir = repo root, tests is a package). pytest treats that as a COLLECTION error for the file, not a failure — so the whole suite stayed green while this file's tests silently never ran, leaving the backoff fix with zero effective coverage. Use the package-qualified `from tests._wakeup_helpers import install_fake_registry`, matching the four sibling bg_task_complete suites. Confirmed the import now resolves from the repo root (the bare form fails). Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_bg_task_drain_loop_backoff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bg_task_drain_loop_backoff.py b/tests/test_bg_task_drain_loop_backoff.py index f8f927b85..383414247 100644 --- a/tests/test_bg_task_drain_loop_backoff.py +++ b/tests/test_bg_task_drain_loop_backoff.py @@ -18,7 +18,7 @@ from __future__ import annotations import queue import threading -from _wakeup_helpers import install_fake_registry +from tests._wakeup_helpers import install_fake_registry class _CountingStop: From a1aca345e15c9f532454691e4f9f4bcd32591e38 Mon Sep 17 00:00:00 2001 From: nesquena-hermes Date: Thu, 9 Jul 2026 01:18:50 +0000 Subject: [PATCH 3/5] docs(changelog): #5782 drain-loop backoff --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bb944357..fc691eb8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ ### Fixed +- **The background-task completion drain thread can no longer spin at 100% CPU when the process registry is (re)initializing.** If the registry didn't yet expose its `completion_queue`, the drain loop's broad `except Exception: continue` swallowed the `AttributeError` and immediately retried with no delay — a hot loop. The loop now reads the queue defensively (`getattr(..., None)`) and backs off on the stop event when it's missing, catches `queue.Empty` separately as the normal idle path, and logs (no longer silently swallows) any unexpected queue error before backing off. No delivery latency is added — real completions are still drained on the 1 Hz blocking `get`. Thanks @ai-ag2026. (#5782, #2476, #4633) + - **Passkey login no longer risks a hung or mis-framed response on keep-alive connections.** The passkey-login success `200` was written without a `Content-Length` header, so a client on a persistent connection could wait for a body boundary that never came or mis-frame the next response. The response now sends an explicit `Content-Length` matching the body's byte length. Thanks @ai-ag2026. (#5785) - **Zero-token turns (a pre-flight cancel or setup error) no longer leak a streaming-meter session.** The streaming meter's `begin_session()` and its 1 Hz ticker were started before the outer `try`, so a raise or cancel before the first token left the `_sessions[stream_id]` entry unreclaimed (`get_stats()` only prunes sessions that emitted a token), inflating the SSE `active` count over a long-lived server. `begin_session()` + the ticker start now live inside the outer `try` so the paired `end_session()` / ticker-stop teardown in the `finally` always runs. Thanks @ai-ag2026. (#5787, #4633, #2476) From 68d406f7ffe6e5735ab3d684fe22ae6e3db7fffa Mon Sep 17 00:00:00 2001 From: sbe27 <283218367+sbe27@users.noreply.github.com> Date: Wed, 8 Jul 2026 02:04:21 +0200 Subject: [PATCH 4/5] fix(models): refresh OpenCode Go static catalog --- api/config.py | 21 ++++-- ...est_issue5311_opencode_go_static_models.py | 70 +++++++++++++++---- 2 files changed, 70 insertions(+), 21 deletions(-) diff --git a/api/config.py b/api/config.py index b2f80efc6..3d14e7bb2 100644 --- a/api/config.py +++ b/api/config.py @@ -1766,22 +1766,29 @@ _PROVIDER_MODELS = { {"id": "nemotron-3-super-free", "label": "Nemotron 3 Super Free"}, {"id": "big-pickle", "label": "Big Pickle"}, ], - # OpenCode Go — flat-rate models via opencode.ai/go ($10/month) + # OpenCode Go — flat-rate models via opencode.ai/go ($10/month). + # Synced 2026-07-08 from the public Go docs and documented models endpoint. + # Keep preview/free-only Zen models out of this Go picker snapshot. "opencode-go": [ + {"id": "minimax-m3", "label": "MiniMax M3"}, + {"id": "minimax-m2.7", "label": "MiniMax M2.7"}, + {"id": "minimax-m2.5", "label": "MiniMax M2.5"}, + {"id": "kimi-k2.7-code", "label": "Kimi K2.7 Code"}, + {"id": "kimi-k2.6", "label": "Kimi K2.6"}, + {"id": "kimi-k2.5", "label": "Kimi K2.5"}, + {"id": "glm-5.2", "label": "GLM-5.2"}, {"id": "glm-5.1", "label": "GLM-5.1"}, {"id": "glm-5", "label": "GLM-5"}, - {"id": "kimi-k2.5", "label": "Kimi K2.5"}, - {"id": "kimi-k2.6", "label": "Kimi K2.6"}, {"id": "deepseek-v4-pro", "label": "DeepSeek V4 Pro"}, {"id": "deepseek-v4-flash","label": "DeepSeek V4 Flash"}, + {"id": "qwen3.7-max", "label": "Qwen3.7 Max"}, + {"id": "qwen3.7-plus", "label": "Qwen3.7 Plus"}, + {"id": "qwen3.6-plus", "label": "Qwen3.6 Plus"}, + {"id": "qwen3.5-plus", "label": "Qwen3.5 Plus"}, {"id": "mimo-v2-pro", "label": "MiMo V2 Pro"}, {"id": "mimo-v2-omni", "label": "MiMo V2 Omni"}, {"id": "mimo-v2.5-pro", "label": "MiMo V2.5 Pro"}, {"id": "mimo-v2.5", "label": "MiMo V2.5"}, - {"id": "minimax-m2.7", "label": "MiniMax M2.7"}, - {"id": "minimax-m2.5", "label": "MiniMax M2.5"}, - {"id": "qwen3.6-plus", "label": "Qwen3.6 Plus"}, - {"id": "qwen3.5-plus", "label": "Qwen3.5 Plus"}, ], # 'gemini' is the hermes_cli provider ID for Google AI Studio # Model IDs are bare — sent directly to: diff --git a/tests/test_issue5311_opencode_go_static_models.py b/tests/test_issue5311_opencode_go_static_models.py index 679fa3e46..7539619dc 100644 --- a/tests/test_issue5311_opencode_go_static_models.py +++ b/tests/test_issue5311_opencode_go_static_models.py @@ -1,30 +1,72 @@ -"""#5311 / #5611: OpenCode Go's live /v1/models probe returns public-catalog -models not enabled on the Go tier, so selecting one 404s on send. get_available_models -must skip the live probe for opencode-go and fall through to the curated static -_PROVIDER_MODELS list instead. +"""#5311 / #5611: OpenCode Go's generic live probe used to return public-catalog +models not enabled on the Go tier, so the picker intentionally falls back to a curated +static _PROVIDER_MODELS list. Keep that curated list in sync with the public Go docs +and documented Go models endpoint while excluding preview/free-only Zen models. """ +import ast from pathlib import Path ROOT = Path(__file__).resolve().parent.parent -CONFIG = (ROOT / "api" / "config.py").read_text(encoding="utf-8") +CONFIG_PATH = ROOT / "api" / "config.py" +CONFIG = CONFIG_PATH.read_text(encoding="utf-8") + +EXPECTED_OPENCODE_GO_MODEL_IDS = [ + "minimax-m3", + "minimax-m2.7", + "minimax-m2.5", + "kimi-k2.7-code", + "kimi-k2.6", + "kimi-k2.5", + "glm-5.2", + "glm-5.1", + "glm-5", + "deepseek-v4-pro", + "deepseek-v4-flash", + "qwen3.7-max", + "qwen3.7-plus", + "qwen3.6-plus", + "qwen3.5-plus", + "mimo-v2-pro", + "mimo-v2-omni", + "mimo-v2.5-pro", + "mimo-v2.5", +] + + +def _opencode_go_static_models(): + tree = ast.parse(CONFIG, filename=str(CONFIG_PATH)) + for node in tree.body: + if not isinstance(node, ast.Assign): + continue + if not any(isinstance(target, ast.Name) and target.id == "_PROVIDER_MODELS" for target in node.targets): + continue + provider_models = ast.literal_eval(node.value) + return provider_models["opencode-go"] + raise AssertionError("_PROVIDER_MODELS assignment not found") def test_opencode_go_skips_live_models_probe(): - # The provider-loop must special-case opencode-go to skip the live probe. + # The provider-loop must special-case opencode-go to skip the old generic + # live probe and fall through to this curated static list. body = CONFIG[CONFIG.index("def get_available_models"):] body = body[: body.index("\ndef ", 1)] assert 'elif pid == "opencode-go":' in body - # It must NOT call the live-provider probe on the opencode-go branch — the - # branch is a bare skip so the next `if not raw_models` falls back to static. idx = body.index('elif pid == "opencode-go":') branch = body[idx: idx + 400] assert "_models_from_live_provider_ids" not in branch.split("else:")[0] -def test_opencode_go_has_curated_static_models(): - # The static fallback the skip relies on must exist and be non-empty. - assert '"opencode-go": [' in CONFIG - block = CONFIG[CONFIG.index('"opencode-go": ['):] - block = block[: block.index("],") + 1] - assert block.count('"id":') >= 5 +def test_opencode_go_static_models_match_documented_endpoint_snapshot(): + # Snapshot from https://opencode.ai/zen/go/v1/models on 2026-07-07. + # This catches stale-list regressions such as missing GLM-5.2 / MiniMax M3. + models = _opencode_go_static_models() + assert [model["id"] for model in models] == EXPECTED_OPENCODE_GO_MODEL_IDS + + +def test_opencode_go_recent_additions_have_human_labels(): + labels = {model["id"]: model["label"] for model in _opencode_go_static_models()} + assert labels["glm-5.2"] == "GLM-5.2" + assert labels["minimax-m3"] == "MiniMax M3" + assert labels["kimi-k2.7-code"] == "Kimi K2.7 Code" + assert labels["qwen3.7-max"] == "Qwen3.7 Max" From 5f2ec9e1f0fb989d14d483fd608a6108abccd18f Mon Sep 17 00:00:00 2001 From: nesquena-hermes Date: Thu, 9 Jul 2026 01:23:33 +0000 Subject: [PATCH 5/5] docs(changelog): #5761 OpenCode Go catalog refresh --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc691eb8f..102a077f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ ### Fixed +- **The OpenCode Go model picker now lists the current flat-rate catalog.** The static `opencode-go` model snapshot was stale; it's been re-synced from the public opencode.ai/go docs + models endpoint (adds MiniMax M3, Kimi K2.7 Code, GLM-5.2, Qwen 3.7 Max/Plus, and more), keeping preview/free-only Zen models out of the Go picker. Thanks @ai-ag2026. (#5761) + - **The background-task completion drain thread can no longer spin at 100% CPU when the process registry is (re)initializing.** If the registry didn't yet expose its `completion_queue`, the drain loop's broad `except Exception: continue` swallowed the `AttributeError` and immediately retried with no delay — a hot loop. The loop now reads the queue defensively (`getattr(..., None)`) and backs off on the stop event when it's missing, catches `queue.Empty` separately as the normal idle path, and logs (no longer silently swallows) any unexpected queue error before backing off. No delivery latency is added — real completions are still drained on the 1 Hz blocking `get`. Thanks @ai-ag2026. (#5782, #2476, #4633) - **Passkey login no longer risks a hung or mis-framed response on keep-alive connections.** The passkey-login success `200` was written without a `Content-Length` header, so a client on a persistent connection could wait for a body boundary that never came or mis-frame the next response. The response now sends an explicit `Content-Length` matching the body's byte length. Thanks @ai-ag2026. (#5785)