mirror of
https://github.com/nesquena/hermes-webui.git
synced 2026-07-12 18:50:35 +00:00
nesquena-hermes
81e748b455
## Release v0.51.247 — Release HO (stage-q19) Backend correctness fix. ### Fixed | Issue | Author | Fix | |-------|--------|-----| | #3505 | @franksong2702 | **Reasoning effort is coerced to a level the active model/provider actually supports** before each request, instead of being sent verbatim and rejected. `openai-codex` `gpt-5` no longer gets `max` (→ `xhigh`); `o1`/`o3`/`o4` clamp to `low`/`medium`/`high`. Coercion only steps *down* (never escalates); `none`/unset preserved. The capability filter is applied across heuristic / models.dev / Copilot / LM Studio paths. | This is the narrow, correct fix for the detection gap that #3431 tried to address by removing the chip-visibility gate (which we shelved). The chip-visibility gate is **untouched** (Codex confirmed) — `get_reasoning_status`/`_applyReasoningChip` still hide the chip for unconfirmed models. ### Review fix absorbed (Codex + self-flagged) The first cut **dropped** a configured effort for *unrecognized* models, because capability detection returns `[]` for both "known-unsupported" and "simply-unknown" (custom providers, aggregator-rewritten ids, new releases) — that's a behavior change vs master (which sent it verbatim) and would silently disable reasoning. Fixed: an **empty** capability set now **preserves** the configured effort (provider stays the final authority; worst case = the same rejected request master already produces, i.e. no regression). Known-bad clamps return *non-empty* filtered sets, so they still degrade correctly. Nathan chose this "preserve-for-unknown" behavior. + regression test. ### Gate - Full pytest suite: **7548 passed, 0 failed** - ruff: CLEAN · 48 reasoning tests pass (incl. preserve-for-unknown + codex-clamp + never-escalate) - Codex (regression): SHIP-ONLY-WITH-FIXES (unknown-model drop) → fixed → **SAFE TO SHIP** - Verified empirically: gpt-5/codex max→xhigh, o3 max/xhigh→high, unknown high→high (preserved), none/unset preserved Co-authored-by: franksong2702 <franksong2702@users.noreply.github.com>
155 lines
4.3 KiB
Python
155 lines
4.3 KiB
Python
"""Regression coverage for Agent model-metadata reasoning capability lookup."""
|
|
|
|
import sys
|
|
import types
|
|
|
|
import pytest
|
|
from types import SimpleNamespace
|
|
|
|
|
|
def _install_fake_models_dev(monkeypatch, fake_fn):
|
|
fake_agent = types.ModuleType("agent")
|
|
fake_models_dev = types.ModuleType("agent.models_dev")
|
|
setattr(fake_models_dev, "get_model_capabilities", fake_fn)
|
|
setattr(fake_agent, "models_dev", fake_models_dev)
|
|
monkeypatch.setitem(sys.modules, "agent", fake_agent)
|
|
monkeypatch.setitem(sys.modules, "agent.models_dev", fake_models_dev)
|
|
|
|
|
|
def test_models_dev_true_returns_full_efforts(monkeypatch):
|
|
_install_fake_models_dev(
|
|
monkeypatch,
|
|
lambda provider, model: SimpleNamespace(supports_reasoning=True),
|
|
)
|
|
|
|
import api.config as cfg
|
|
|
|
assert cfg._models_dev_reasoning_efforts("grok-4.3", "xai-oauth") == list(
|
|
cfg.VALID_REASONING_EFFORTS
|
|
)
|
|
|
|
|
|
def test_models_dev_false_returns_authoritative_empty(monkeypatch):
|
|
_install_fake_models_dev(
|
|
monkeypatch,
|
|
lambda provider, model: SimpleNamespace(supports_reasoning=False),
|
|
)
|
|
|
|
import api.config as cfg
|
|
|
|
assert cfg._models_dev_reasoning_efforts("grok-4.20-non-reasoning", "xai-oauth") == []
|
|
|
|
|
|
def test_models_dev_unknown_allows_compatibility_fallback(monkeypatch):
|
|
_install_fake_models_dev(monkeypatch, lambda provider, model: None)
|
|
|
|
import api.config as cfg
|
|
|
|
assert cfg.resolve_model_reasoning_efforts(
|
|
"x-ai/grok-4", provider_id="openrouter"
|
|
) == list(cfg.VALID_REASONING_EFFORTS)
|
|
|
|
|
|
def test_xai_oauth_grok_uses_agent_metadata(monkeypatch):
|
|
seen = []
|
|
|
|
def fake_capabilities(provider, model):
|
|
seen.append((provider, model))
|
|
return SimpleNamespace(supports_reasoning=True)
|
|
|
|
_install_fake_models_dev(monkeypatch, fake_capabilities)
|
|
|
|
import api.config as cfg
|
|
|
|
assert cfg.resolve_model_reasoning_efforts(
|
|
"@xai-oauth:grok-4.3", provider_id="xai-oauth"
|
|
) == list(cfg.VALID_REASONING_EFFORTS)
|
|
assert seen == [("xai-oauth", "grok-4.3")]
|
|
|
|
|
|
def test_models_dev_false_suppresses_prefix_heuristic(monkeypatch):
|
|
_install_fake_models_dev(
|
|
monkeypatch,
|
|
lambda provider, model: SimpleNamespace(supports_reasoning=False),
|
|
)
|
|
|
|
import api.config as cfg
|
|
|
|
assert cfg.resolve_model_reasoning_efforts(
|
|
"x-ai/grok-4-non-reasoning", provider_id="openrouter"
|
|
) == []
|
|
|
|
|
|
def test_codex_gpt55_uses_models_dev_excluding_unsupported_max(monkeypatch):
|
|
_install_fake_models_dev(
|
|
monkeypatch,
|
|
lambda provider, model: SimpleNamespace(supports_reasoning=True),
|
|
)
|
|
|
|
import api.config as cfg
|
|
|
|
result = cfg.resolve_model_reasoning_efforts(
|
|
"gpt-5.5", provider_id="openai-codex"
|
|
)
|
|
assert "xhigh" in result
|
|
assert "max" not in result
|
|
|
|
|
|
def test_codex_metadata_false_returns_empty(monkeypatch):
|
|
_install_fake_models_dev(
|
|
monkeypatch,
|
|
lambda provider, model: SimpleNamespace(supports_reasoning=False),
|
|
)
|
|
|
|
import api.config as cfg
|
|
|
|
assert cfg.resolve_model_reasoning_efforts(
|
|
"gpt-5.5", provider_id="openai-codex"
|
|
) == []
|
|
|
|
|
|
def test_copilot_gpt55_caps_at_high(monkeypatch):
|
|
import api.config as cfg
|
|
|
|
try:
|
|
from hermes_cli.models import github_model_reasoning_efforts
|
|
except ImportError:
|
|
pytest.skip("hermes_cli not available")
|
|
|
|
result = cfg.resolve_model_reasoning_efforts(
|
|
"gpt-5.5", provider_id="copilot"
|
|
)
|
|
assert "xhigh" not in result
|
|
|
|
|
|
def test_get_reasoning_status_uses_config_default_model(monkeypatch, tmp_path):
|
|
_install_fake_models_dev(
|
|
monkeypatch,
|
|
lambda provider, model: SimpleNamespace(supports_reasoning=True)
|
|
if (provider, model) == ("xai-oauth", "grok-4.3")
|
|
else None,
|
|
)
|
|
|
|
import api.config as cfg
|
|
|
|
config_path = tmp_path / "config.yaml"
|
|
config_path.write_text(
|
|
"""
|
|
model:
|
|
default: grok-4.3
|
|
provider: xai-oauth
|
|
agent:
|
|
reasoning_effort: medium
|
|
display:
|
|
show_reasoning: true
|
|
""".strip(),
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setattr(cfg, "_get_config_path", lambda: config_path)
|
|
|
|
status = cfg.get_reasoning_status()
|
|
|
|
assert status["reasoning_effort"] == "medium"
|
|
assert status["supported_efforts"] == list(cfg.VALID_REASONING_EFFORTS)
|
|
assert status["supports_reasoning_effort"] is True
|