diff --git a/api/config.py b/api/config.py index 9d4fa457..52490d56 100644 --- a/api/config.py +++ b/api/config.py @@ -2117,14 +2117,94 @@ def get_available_models() -> dict: continue provider_name = _PROVIDER_DISPLAY.get(pid, pid.title()) if pid == "openrouter": + # OpenRouter has two model surfaces: + # (1) curated tool-supporting catalog via hermes_cli.models.fetch_openrouter_models() + # — the canonical agent-ready list, applies a tool-support filter + # (Kilo-Org/kilocode#9068) that hides image/completion-only models + # (2) free-tier `:free` variants — newly-added models OpenRouter ships + # experimentally that may not yet advertise `tools` in supported_parameters + # (see #1426). These get filtered out of (1) but users want them visible. + # + # Strategy: take the live curated list as the base, then augment with a + # separate live-fetch of OpenRouter's /v1/models filtered to free-tier-only. + # Free-tier entries get a "(free)" label suffix so the picker is honest about + # what the user is selecting. Falls back to the static _FALLBACK_MODELS list + # when both live fetches fail (offline, transient API error, test env). + raw_models = [] + seen_ids = set() + try: + from hermes_cli.models import ( + fetch_openrouter_models as _fetch_or_models, + ) + live_curated = _fetch_or_models() or [] + for mid, _desc in live_curated: + if mid and mid not in seen_ids: + seen_ids.add(mid) + raw_models.append({"id": mid, "label": mid}) + except Exception: + logger.warning("Failed to load OpenRouter curated catalog from hermes_cli") + + # Free-tier live fetch — bypasses the tool-support filter so models + # OpenRouter has flagged free but hasn't yet annotated with tools=[] + # (or that have tools=[] but the user explicitly wants to try) appear. + try: + import urllib.request as _urlreq + _req = _urlreq.Request( + "https://openrouter.ai/api/v1/models", + headers={"Accept": "application/json"}, + ) + with _urlreq.urlopen(_req, timeout=8.0) as _resp: + _payload = json.loads(_resp.read().decode()) + _free_count = 0 + _free_cap = 30 # don't drown the picker — top 30 free tier + for _item in _payload.get("data", []) or []: + if not isinstance(_item, dict): + continue + _mid = str(_item.get("id") or "").strip() + if not _mid or _mid in seen_ids: + continue + _pricing = _item.get("pricing") or {} + try: + _is_free = ( + float(_pricing.get("prompt", "0") or "0") == 0 + and float(_pricing.get("completion", "0") or "0") == 0 + ) + except (TypeError, ValueError): + _is_free = False + # Also include explicit `:free` suffix variants + _is_free = _is_free or _mid.endswith(":free") + if not _is_free: + continue + _name = ( + str(_item.get("name") or "").strip() or _mid + ) + # Strip provider prefix from name for display, append (free) + _label = _name.split("/")[-1] if "/" in _name else _name + if "(free)" not in _label.lower(): + _label = f"{_label} (free)" + seen_ids.add(_mid) + raw_models.append({"id": _mid, "label": _label}) + _free_count += 1 + if _free_count >= _free_cap: + break + except Exception: + logger.debug("OpenRouter free-tier live fetch unavailable; using fallback") + + if not raw_models: + # Both live fetches failed — fall back to the curated static list. + # Deepcopy so dedup/prefix mutation downstream does not bleed + # into the module-level catalog. + raw_models = [ + {"id": m["id"], "label": m["label"]} + for m in _FALLBACK_MODELS + if m.get("provider") == "OpenRouter" + ] + groups.append( { "provider": "OpenRouter", "provider_id": "openrouter", - "models": [ - {"id": m["id"], "label": m["label"]} - for m in _FALLBACK_MODELS - ], + "models": raw_models, } ) elif pid == "ollama-cloud": diff --git a/tests/test_issue1426_openrouter_free_tier_live_fetch.py b/tests/test_issue1426_openrouter_free_tier_live_fetch.py new file mode 100644 index 00000000..c5f6175f --- /dev/null +++ b/tests/test_issue1426_openrouter_free_tier_live_fetch.py @@ -0,0 +1,226 @@ +"""Regression tests for #1426 — OpenRouter free-tier visibility (structural fix). + +Original PR #1548 added 6 hardcoded `_FALLBACK_MODELS` entries. This is the +structural augmentation: WebUI now does TWO live fetches when populating the +OpenRouter group: + + (1) `hermes_cli.models.fetch_openrouter_models()` — the curated tool-supporting + list, which goes through the tool-support filter (Kilo-Org/kilocode#9068). + (2) Direct `https://openrouter.ai/api/v1/models` — filtered to free-tier-only, + bypassing the tool-support filter so newly-added free variants appear. + +Both fall back to `_FALLBACK_MODELS` (which retains @bergeouss's hardcoded list +as a defense-in-depth fallback) when the API is unreachable. + +These tests verify the structural fix without depending on real network access: +the urllib.request layer is monkeypatched. +""" +from __future__ import annotations + +import json +import urllib.request + +import pytest + +import api.config as config + + +class _FakeResponse: + def __init__(self, payload: dict): + self._buf = json.dumps(payload).encode() + + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self) -> bytes: + return self._buf + + +def _make_or_payload(*items: dict) -> dict: + return {"data": list(items)} + + +def _get_grouped_models() -> list[dict]: + """Helper: return the `groups` field from get_available_models().""" + # Reset internal cache so each call re-runs the live-fetch path + try: + config.invalidate_models_cache() + except Exception: + pass + result = config.get_available_models() + return result.get("groups", []) + + +@pytest.fixture(autouse=True) +def _isolate_openrouter_cache(monkeypatch): + """Reset the curated cache before each test so the live-fetch path runs. + Also force `openrouter` as the active provider so the openrouter branch + in get_available_models() actually runs.""" + try: + from hermes_cli import models as _hm + + monkeypatch.setattr(_hm, "_openrouter_catalog_cache", None, raising=False) + except Exception: + pass + + # Force openrouter to be detected by injecting it into config + monkeypatch.setattr( + config, + "cfg", + { + "model": {"provider": "openrouter", "default": "anthropic/claude-sonnet-4.6"}, + "providers": {"openrouter": {"api_key": "sk-or-test-key"}}, + }, + raising=False, + ) + # Reset module-level cache + try: + config.invalidate_models_cache() + except Exception: + pass + + +def test_fallback_list_contains_free_tier_entries(): + """The hardcoded fallback list (defense-in-depth) still contains the + contributor's free-tier entries so offline / test envs see them.""" + or_entries = [m for m in config._FALLBACK_MODELS if m.get("provider") == "OpenRouter"] + assert len(or_entries) >= 5, "fallback list should include at least 5 free-tier entries" + free_labels = [m["label"] for m in or_entries if "free" in m["label"].lower()] + assert len(free_labels) >= 5, f"expected ≥5 free-tier entries in fallback, got {len(free_labels)}" + + +def test_openrouter_group_uses_live_fetch_when_available(monkeypatch): + """When OpenRouter /v1/models is reachable, the picker shows live data, + not just the fallback list. Free-tier entries get a (free) suffix.""" + fake_payload = _make_or_payload( + # Tool-supporting paid model + {"id": "anthropic/claude-sonnet-4.6", "name": "Claude Sonnet 4.6", + "supported_parameters": ["tools"], "pricing": {"prompt": "0.000003", "completion": "0.000015"}}, + # Free-tier model NOT advertising tools — the bug from #1426 + {"id": "minimax/minimax-m2.5:free", "name": "MiniMax M2.5", + "supported_parameters": [], "pricing": {"prompt": "0", "completion": "0"}}, + # Free model without :free suffix but pricing shows free + {"id": "openrouter/elephant-alpha", "name": "Elephant Alpha", + "supported_parameters": ["tools"], "pricing": {"prompt": "0", "completion": "0"}}, + ) + + def _fake_urlopen(req, timeout=None): + return _FakeResponse(fake_payload) + + monkeypatch.setattr(urllib.request, "urlopen", _fake_urlopen) + try: + from hermes_cli import models as _hm + monkeypatch.setattr(_hm, "_openrouter_catalog_cache", None, raising=False) + except Exception: + pass + + grouped = _get_grouped_models() + or_group = next((g for g in grouped if g.get("provider_id") == "openrouter"), None) + assert or_group is not None, "openrouter group must be present" + + model_ids = [m["id"] for m in or_group["models"]] + # Free-tier variants must be visible despite not advertising tool support + assert "minimax/minimax-m2.5:free" in model_ids, \ + "free-tier minimax/minimax-m2.5:free must surface in the picker even without tools support" + assert "openrouter/elephant-alpha" in model_ids, \ + "free pricing model must surface even without :free suffix" + + +def test_openrouter_falls_back_to_static_when_live_fails(monkeypatch): + """If both hermes_cli.fetch and the direct urlopen raise, the picker + must fall back to the hardcoded `_FALLBACK_MODELS` list — never empty.""" + def _fake_urlopen(req, timeout=None): + raise OSError("simulated network outage") + + monkeypatch.setattr(urllib.request, "urlopen", _fake_urlopen) + + # Force hermes_cli to fail too + import sys + fake_module = type(sys)("hermes_cli.models") + + def _raise(*args, **kwargs): + raise RuntimeError("simulated import failure") + + fake_module.fetch_openrouter_models = _raise + fake_module.provider_model_ids = lambda *a, **k: [] + monkeypatch.setitem(sys.modules, "hermes_cli.models", fake_module) + + grouped = _get_grouped_models() + or_group = next((g for g in grouped if g.get("provider_id") == "openrouter"), None) + assert or_group is not None, "openrouter group must still be present in fallback path" + assert len(or_group["models"]) > 0, "fallback must produce a non-empty model list" + # The hardcoded free-tier entries MUST be in the fallback + fallback_ids = {m["id"] for m in or_group["models"]} + # At least one of the contributor's hardcoded free-tier entries must be present + expected_free_ids = { + "openrouter/elephant-alpha", + "openrouter/owl-alpha", + "tencent/hy3-preview:free", + "nvidia/nemotron-3-super-120b-a12b:free", + "arcee-ai/trinity-large-preview:free", + } + overlap = fallback_ids & expected_free_ids + assert len(overlap) >= 3, \ + f"static fallback must include the contributor's hardcoded free-tier entries; got overlap={overlap}" + + +def test_free_tier_cap_prevents_picker_drowning(monkeypatch): + """OpenRouter may return hundreds of free-tier variants — the implementation + caps the live-fetch additions at 30 to keep the picker usable.""" + items = [] + for i in range(50): + items.append({ + "id": f"vendor{i}/model-{i}:free", + "name": f"Model {i}", + "supported_parameters": [], + "pricing": {"prompt": "0", "completion": "0"}, + }) + fake_payload = _make_or_payload(*items) + + def _fake_urlopen(req, timeout=None): + return _FakeResponse(fake_payload) + + monkeypatch.setattr(urllib.request, "urlopen", _fake_urlopen) + + try: + from hermes_cli import models as _hm + monkeypatch.setattr(_hm, "_openrouter_catalog_cache", None, raising=False) + except Exception: + pass + + grouped = _get_grouped_models() + or_group = next((g for g in grouped if g.get("provider_id") == "openrouter"), None) + assert or_group is not None + free_added_ids = {m["id"] for m in or_group["models"] if ":free" in m["id"]} + assert len(free_added_ids) <= 50, "should not exceed the items provided" + assert len(free_added_ids) > 0, "free-tier live fetch should add at least some entries" + + +def test_openrouter_dedupe_curated_and_free_tier(monkeypatch): + """If a model appears in both the curated catalog AND the free-tier fetch, + it must appear exactly once in the picker (via `seen_ids` deduplication).""" + fake_payload = _make_or_payload( + {"id": "anthropic/claude-sonnet-4.6", "name": "Claude Sonnet 4.6", + "supported_parameters": ["tools"], "pricing": {"prompt": "0", "completion": "0"}}, + ) + + def _fake_urlopen(req, timeout=None): + return _FakeResponse(fake_payload) + + monkeypatch.setattr(urllib.request, "urlopen", _fake_urlopen) + + import sys + fake_module = type(sys)("hermes_cli.models") + fake_module.fetch_openrouter_models = lambda **k: [("anthropic/claude-sonnet-4.6", "")] + fake_module.provider_model_ids = lambda *a, **k: ["anthropic/claude-sonnet-4.6"] + monkeypatch.setitem(sys.modules, "hermes_cli.models", fake_module) + + grouped = _get_grouped_models() + or_group = next((g for g in grouped if g.get("provider_id") == "openrouter"), None) + assert or_group is not None + matching = [m for m in or_group["models"] if m["id"] == "anthropic/claude-sonnet-4.6"] + assert len(matching) == 1, \ + f"model present in both surfaces should appear once, got {len(matching)}" diff --git a/tests/test_minimax_provider.py b/tests/test_minimax_provider.py index 692b06cf..9893c7e2 100644 --- a/tests/test_minimax_provider.py +++ b/tests/test_minimax_provider.py @@ -92,10 +92,22 @@ def test_minimax_m2_7_highspeed_in_fallback_models(): def test_minimax_fallback_provider_label(): - """MiniMax fallback entries must use 'MiniMax' as the provider label.""" - minimax_entries = [m for m in config._FALLBACK_MODELS if 'minimax' in m['id'].lower()] - assert minimax_entries, "No MiniMax entries found in _FALLBACK_MODELS" - for entry in minimax_entries: + """MiniMax fallback entries (direct API routing) must use 'MiniMax' as + the provider label. + + NOTE: This filters by `minimax/` ID prefix to scope strictly to the + direct MiniMax provider routes — `minimax-X` is the canonical pattern + for hermes-agent routing to api.minimax.io. OpenRouter free-tier variants + that happen to contain 'minimax' in their ID (e.g. + `minimax/minimax-m2.5:free`) are routed via OpenRouter, not direct + MiniMax, and correctly carry provider='OpenRouter'. See #1426. + """ + direct_minimax = [ + m for m in config._FALLBACK_MODELS + if m['id'].startswith('minimax/') and ':free' not in m['id'] + ] + assert direct_minimax, "No direct-MiniMax entries found in _FALLBACK_MODELS" + for entry in direct_minimax: assert entry['provider'] == 'MiniMax', ( f"Expected provider='MiniMax', got '{entry['provider']}' for {entry['id']}" )