mirror of
https://github.com/nesquena/hermes-webui.git
synced 2026-05-25 03:00:23 +00:00
fix(models): structural OpenRouter free-tier visibility — live fetch + augment fallback (#1426)
Augments @bergeouss's PR #1548 v2 with the structural fix the issue actually requested. The original PR added 5 hardcoded entries to _FALLBACK_MODELS which would rot fast as OpenRouter's free-tier roster turns over monthly. Adds proper live-fetch logic to the OpenRouter group population so the free-tier list stays fresh without requiring a code release every time a new free model lands. api/config.py:2120 — replaces the static _FALLBACK_MODELS slice with: 1. Live curated catalog via hermes_cli.models.fetch_openrouter_models() — applies the tool-support filter (Kilo-Org/kilocode#9068). 2. Free-tier live fetch — direct call to https://openrouter.ai/api/v1/models, filtered to free-tier-only (pricing.prompt == 0 AND pricing.completion == 0, OR :free suffix), bypasses the tool-support filter so newly-added free variants appear even before OpenRouter annotates them with tools. Capped at 30 entries to keep the picker usable. 3. Defense-in-depth fallback to _FALLBACK_MODELS (which retains @bergeouss's hardcoded list for offline / test envs). 4. Deduplication via seen_ids — model in both surfaces appears once. 5 new tests + 1 fixed test in tests/test_minimax_provider.py (scoped the provider='MiniMax' assertion to direct-MiniMax routes by filtering for 'minimax/' prefix and excluding ':free' since the OpenRouter free-tier variant minimax/minimax-m2.5:free correctly carries provider='OpenRouter'). Co-authored-by: bergeouss <[email protected]>
This commit is contained in:
Hermes Bot
+84
-4
@@ -2117,14 +2117,94 @@ def get_available_models() -> dict:
|
||||
continue
|
||||
provider_name = _PROVIDER_DISPLAY.get(pid, pid.title())
|
||||
if pid == "openrouter":
|
||||
# OpenRouter has two model surfaces:
|
||||
# (1) curated tool-supporting catalog via hermes_cli.models.fetch_openrouter_models()
|
||||
# — the canonical agent-ready list, applies a tool-support filter
|
||||
# (Kilo-Org/kilocode#9068) that hides image/completion-only models
|
||||
# (2) free-tier `:free` variants — newly-added models OpenRouter ships
|
||||
# experimentally that may not yet advertise `tools` in supported_parameters
|
||||
# (see #1426). These get filtered out of (1) but users want them visible.
|
||||
#
|
||||
# Strategy: take the live curated list as the base, then augment with a
|
||||
# separate live-fetch of OpenRouter's /v1/models filtered to free-tier-only.
|
||||
# Free-tier entries get a "(free)" label suffix so the picker is honest about
|
||||
# what the user is selecting. Falls back to the static _FALLBACK_MODELS list
|
||||
# when both live fetches fail (offline, transient API error, test env).
|
||||
raw_models = []
|
||||
seen_ids = set()
|
||||
try:
|
||||
from hermes_cli.models import (
|
||||
fetch_openrouter_models as _fetch_or_models,
|
||||
)
|
||||
live_curated = _fetch_or_models() or []
|
||||
for mid, _desc in live_curated:
|
||||
if mid and mid not in seen_ids:
|
||||
seen_ids.add(mid)
|
||||
raw_models.append({"id": mid, "label": mid})
|
||||
except Exception:
|
||||
logger.warning("Failed to load OpenRouter curated catalog from hermes_cli")
|
||||
|
||||
# Free-tier live fetch — bypasses the tool-support filter so models
|
||||
# OpenRouter has flagged free but hasn't yet annotated with tools=[]
|
||||
# (or that have tools=[] but the user explicitly wants to try) appear.
|
||||
try:
|
||||
import urllib.request as _urlreq
|
||||
_req = _urlreq.Request(
|
||||
"https://openrouter.ai/api/v1/models",
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
with _urlreq.urlopen(_req, timeout=8.0) as _resp:
|
||||
_payload = json.loads(_resp.read().decode())
|
||||
_free_count = 0
|
||||
_free_cap = 30 # don't drown the picker — top 30 free tier
|
||||
for _item in _payload.get("data", []) or []:
|
||||
if not isinstance(_item, dict):
|
||||
continue
|
||||
_mid = str(_item.get("id") or "").strip()
|
||||
if not _mid or _mid in seen_ids:
|
||||
continue
|
||||
_pricing = _item.get("pricing") or {}
|
||||
try:
|
||||
_is_free = (
|
||||
float(_pricing.get("prompt", "0") or "0") == 0
|
||||
and float(_pricing.get("completion", "0") or "0") == 0
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
_is_free = False
|
||||
# Also include explicit `:free` suffix variants
|
||||
_is_free = _is_free or _mid.endswith(":free")
|
||||
if not _is_free:
|
||||
continue
|
||||
_name = (
|
||||
str(_item.get("name") or "").strip() or _mid
|
||||
)
|
||||
# Strip provider prefix from name for display, append (free)
|
||||
_label = _name.split("/")[-1] if "/" in _name else _name
|
||||
if "(free)" not in _label.lower():
|
||||
_label = f"{_label} (free)"
|
||||
seen_ids.add(_mid)
|
||||
raw_models.append({"id": _mid, "label": _label})
|
||||
_free_count += 1
|
||||
if _free_count >= _free_cap:
|
||||
break
|
||||
except Exception:
|
||||
logger.debug("OpenRouter free-tier live fetch unavailable; using fallback")
|
||||
|
||||
if not raw_models:
|
||||
# Both live fetches failed — fall back to the curated static list.
|
||||
# Deepcopy so dedup/prefix mutation downstream does not bleed
|
||||
# into the module-level catalog.
|
||||
raw_models = [
|
||||
{"id": m["id"], "label": m["label"]}
|
||||
for m in _FALLBACK_MODELS
|
||||
if m.get("provider") == "OpenRouter"
|
||||
]
|
||||
|
||||
groups.append(
|
||||
{
|
||||
"provider": "OpenRouter",
|
||||
"provider_id": "openrouter",
|
||||
"models": [
|
||||
{"id": m["id"], "label": m["label"]}
|
||||
for m in _FALLBACK_MODELS
|
||||
],
|
||||
"models": raw_models,
|
||||
}
|
||||
)
|
||||
elif pid == "ollama-cloud":
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
"""Regression tests for #1426 — OpenRouter free-tier visibility (structural fix).
|
||||
|
||||
Original PR #1548 added 6 hardcoded `_FALLBACK_MODELS` entries. This is the
|
||||
structural augmentation: WebUI now does TWO live fetches when populating the
|
||||
OpenRouter group:
|
||||
|
||||
(1) `hermes_cli.models.fetch_openrouter_models()` — the curated tool-supporting
|
||||
list, which goes through the tool-support filter (Kilo-Org/kilocode#9068).
|
||||
(2) Direct `https://openrouter.ai/api/v1/models` — filtered to free-tier-only,
|
||||
bypassing the tool-support filter so newly-added free variants appear.
|
||||
|
||||
Both fall back to `_FALLBACK_MODELS` (which retains @bergeouss's hardcoded list
|
||||
as a defense-in-depth fallback) when the API is unreachable.
|
||||
|
||||
These tests verify the structural fix without depending on real network access:
|
||||
the urllib.request layer is monkeypatched.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.request
|
||||
|
||||
import pytest
|
||||
|
||||
import api.config as config
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload: dict):
|
||||
self._buf = json.dumps(payload).encode()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def read(self) -> bytes:
|
||||
return self._buf
|
||||
|
||||
|
||||
def _make_or_payload(*items: dict) -> dict:
|
||||
return {"data": list(items)}
|
||||
|
||||
|
||||
def _get_grouped_models() -> list[dict]:
|
||||
"""Helper: return the `groups` field from get_available_models()."""
|
||||
# Reset internal cache so each call re-runs the live-fetch path
|
||||
try:
|
||||
config.invalidate_models_cache()
|
||||
except Exception:
|
||||
pass
|
||||
result = config.get_available_models()
|
||||
return result.get("groups", [])
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_openrouter_cache(monkeypatch):
|
||||
"""Reset the curated cache before each test so the live-fetch path runs.
|
||||
Also force `openrouter` as the active provider so the openrouter branch
|
||||
in get_available_models() actually runs."""
|
||||
try:
|
||||
from hermes_cli import models as _hm
|
||||
|
||||
monkeypatch.setattr(_hm, "_openrouter_catalog_cache", None, raising=False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Force openrouter to be detected by injecting it into config
|
||||
monkeypatch.setattr(
|
||||
config,
|
||||
"cfg",
|
||||
{
|
||||
"model": {"provider": "openrouter", "default": "anthropic/claude-sonnet-4.6"},
|
||||
"providers": {"openrouter": {"api_key": "sk-or-test-key"}},
|
||||
},
|
||||
raising=False,
|
||||
)
|
||||
# Reset module-level cache
|
||||
try:
|
||||
config.invalidate_models_cache()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def test_fallback_list_contains_free_tier_entries():
|
||||
"""The hardcoded fallback list (defense-in-depth) still contains the
|
||||
contributor's free-tier entries so offline / test envs see them."""
|
||||
or_entries = [m for m in config._FALLBACK_MODELS if m.get("provider") == "OpenRouter"]
|
||||
assert len(or_entries) >= 5, "fallback list should include at least 5 free-tier entries"
|
||||
free_labels = [m["label"] for m in or_entries if "free" in m["label"].lower()]
|
||||
assert len(free_labels) >= 5, f"expected ≥5 free-tier entries in fallback, got {len(free_labels)}"
|
||||
|
||||
|
||||
def test_openrouter_group_uses_live_fetch_when_available(monkeypatch):
|
||||
"""When OpenRouter /v1/models is reachable, the picker shows live data,
|
||||
not just the fallback list. Free-tier entries get a (free) suffix."""
|
||||
fake_payload = _make_or_payload(
|
||||
# Tool-supporting paid model
|
||||
{"id": "anthropic/claude-sonnet-4.6", "name": "Claude Sonnet 4.6",
|
||||
"supported_parameters": ["tools"], "pricing": {"prompt": "0.000003", "completion": "0.000015"}},
|
||||
# Free-tier model NOT advertising tools — the bug from #1426
|
||||
{"id": "minimax/minimax-m2.5:free", "name": "MiniMax M2.5",
|
||||
"supported_parameters": [], "pricing": {"prompt": "0", "completion": "0"}},
|
||||
# Free model without :free suffix but pricing shows free
|
||||
{"id": "openrouter/elephant-alpha", "name": "Elephant Alpha",
|
||||
"supported_parameters": ["tools"], "pricing": {"prompt": "0", "completion": "0"}},
|
||||
)
|
||||
|
||||
def _fake_urlopen(req, timeout=None):
|
||||
return _FakeResponse(fake_payload)
|
||||
|
||||
monkeypatch.setattr(urllib.request, "urlopen", _fake_urlopen)
|
||||
try:
|
||||
from hermes_cli import models as _hm
|
||||
monkeypatch.setattr(_hm, "_openrouter_catalog_cache", None, raising=False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
grouped = _get_grouped_models()
|
||||
or_group = next((g for g in grouped if g.get("provider_id") == "openrouter"), None)
|
||||
assert or_group is not None, "openrouter group must be present"
|
||||
|
||||
model_ids = [m["id"] for m in or_group["models"]]
|
||||
# Free-tier variants must be visible despite not advertising tool support
|
||||
assert "minimax/minimax-m2.5:free" in model_ids, \
|
||||
"free-tier minimax/minimax-m2.5:free must surface in the picker even without tools support"
|
||||
assert "openrouter/elephant-alpha" in model_ids, \
|
||||
"free pricing model must surface even without :free suffix"
|
||||
|
||||
|
||||
def test_openrouter_falls_back_to_static_when_live_fails(monkeypatch):
|
||||
"""If both hermes_cli.fetch and the direct urlopen raise, the picker
|
||||
must fall back to the hardcoded `_FALLBACK_MODELS` list — never empty."""
|
||||
def _fake_urlopen(req, timeout=None):
|
||||
raise OSError("simulated network outage")
|
||||
|
||||
monkeypatch.setattr(urllib.request, "urlopen", _fake_urlopen)
|
||||
|
||||
# Force hermes_cli to fail too
|
||||
import sys
|
||||
fake_module = type(sys)("hermes_cli.models")
|
||||
|
||||
def _raise(*args, **kwargs):
|
||||
raise RuntimeError("simulated import failure")
|
||||
|
||||
fake_module.fetch_openrouter_models = _raise
|
||||
fake_module.provider_model_ids = lambda *a, **k: []
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli.models", fake_module)
|
||||
|
||||
grouped = _get_grouped_models()
|
||||
or_group = next((g for g in grouped if g.get("provider_id") == "openrouter"), None)
|
||||
assert or_group is not None, "openrouter group must still be present in fallback path"
|
||||
assert len(or_group["models"]) > 0, "fallback must produce a non-empty model list"
|
||||
# The hardcoded free-tier entries MUST be in the fallback
|
||||
fallback_ids = {m["id"] for m in or_group["models"]}
|
||||
# At least one of the contributor's hardcoded free-tier entries must be present
|
||||
expected_free_ids = {
|
||||
"openrouter/elephant-alpha",
|
||||
"openrouter/owl-alpha",
|
||||
"tencent/hy3-preview:free",
|
||||
"nvidia/nemotron-3-super-120b-a12b:free",
|
||||
"arcee-ai/trinity-large-preview:free",
|
||||
}
|
||||
overlap = fallback_ids & expected_free_ids
|
||||
assert len(overlap) >= 3, \
|
||||
f"static fallback must include the contributor's hardcoded free-tier entries; got overlap={overlap}"
|
||||
|
||||
|
||||
def test_free_tier_cap_prevents_picker_drowning(monkeypatch):
|
||||
"""OpenRouter may return hundreds of free-tier variants — the implementation
|
||||
caps the live-fetch additions at 30 to keep the picker usable."""
|
||||
items = []
|
||||
for i in range(50):
|
||||
items.append({
|
||||
"id": f"vendor{i}/model-{i}:free",
|
||||
"name": f"Model {i}",
|
||||
"supported_parameters": [],
|
||||
"pricing": {"prompt": "0", "completion": "0"},
|
||||
})
|
||||
fake_payload = _make_or_payload(*items)
|
||||
|
||||
def _fake_urlopen(req, timeout=None):
|
||||
return _FakeResponse(fake_payload)
|
||||
|
||||
monkeypatch.setattr(urllib.request, "urlopen", _fake_urlopen)
|
||||
|
||||
try:
|
||||
from hermes_cli import models as _hm
|
||||
monkeypatch.setattr(_hm, "_openrouter_catalog_cache", None, raising=False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
grouped = _get_grouped_models()
|
||||
or_group = next((g for g in grouped if g.get("provider_id") == "openrouter"), None)
|
||||
assert or_group is not None
|
||||
free_added_ids = {m["id"] for m in or_group["models"] if ":free" in m["id"]}
|
||||
assert len(free_added_ids) <= 50, "should not exceed the items provided"
|
||||
assert len(free_added_ids) > 0, "free-tier live fetch should add at least some entries"
|
||||
|
||||
|
||||
def test_openrouter_dedupe_curated_and_free_tier(monkeypatch):
|
||||
"""If a model appears in both the curated catalog AND the free-tier fetch,
|
||||
it must appear exactly once in the picker (via `seen_ids` deduplication)."""
|
||||
fake_payload = _make_or_payload(
|
||||
{"id": "anthropic/claude-sonnet-4.6", "name": "Claude Sonnet 4.6",
|
||||
"supported_parameters": ["tools"], "pricing": {"prompt": "0", "completion": "0"}},
|
||||
)
|
||||
|
||||
def _fake_urlopen(req, timeout=None):
|
||||
return _FakeResponse(fake_payload)
|
||||
|
||||
monkeypatch.setattr(urllib.request, "urlopen", _fake_urlopen)
|
||||
|
||||
import sys
|
||||
fake_module = type(sys)("hermes_cli.models")
|
||||
fake_module.fetch_openrouter_models = lambda **k: [("anthropic/claude-sonnet-4.6", "")]
|
||||
fake_module.provider_model_ids = lambda *a, **k: ["anthropic/claude-sonnet-4.6"]
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli.models", fake_module)
|
||||
|
||||
grouped = _get_grouped_models()
|
||||
or_group = next((g for g in grouped if g.get("provider_id") == "openrouter"), None)
|
||||
assert or_group is not None
|
||||
matching = [m for m in or_group["models"] if m["id"] == "anthropic/claude-sonnet-4.6"]
|
||||
assert len(matching) == 1, \
|
||||
f"model present in both surfaces should appear once, got {len(matching)}"
|
||||
@@ -92,10 +92,22 @@ def test_minimax_m2_7_highspeed_in_fallback_models():
|
||||
|
||||
|
||||
def test_minimax_fallback_provider_label():
|
||||
"""MiniMax fallback entries must use 'MiniMax' as the provider label."""
|
||||
minimax_entries = [m for m in config._FALLBACK_MODELS if 'minimax' in m['id'].lower()]
|
||||
assert minimax_entries, "No MiniMax entries found in _FALLBACK_MODELS"
|
||||
for entry in minimax_entries:
|
||||
"""MiniMax fallback entries (direct API routing) must use 'MiniMax' as
|
||||
the provider label.
|
||||
|
||||
NOTE: This filters by `minimax/` ID prefix to scope strictly to the
|
||||
direct MiniMax provider routes — `minimax-X` is the canonical pattern
|
||||
for hermes-agent routing to api.minimax.io. OpenRouter free-tier variants
|
||||
that happen to contain 'minimax' in their ID (e.g.
|
||||
`minimax/minimax-m2.5:free`) are routed via OpenRouter, not direct
|
||||
MiniMax, and correctly carry provider='OpenRouter'. See #1426.
|
||||
"""
|
||||
direct_minimax = [
|
||||
m for m in config._FALLBACK_MODELS
|
||||
if m['id'].startswith('minimax/') and ':free' not in m['id']
|
||||
]
|
||||
assert direct_minimax, "No direct-MiniMax entries found in _FALLBACK_MODELS"
|
||||
for entry in direct_minimax:
|
||||
assert entry['provider'] == 'MiniMax', (
|
||||
f"Expected provider='MiniMax', got '{entry['provider']}' for {entry['id']}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user