diff --git a/devplacepy/services/openai_gateway/gateway.py b/devplacepy/services/openai_gateway/gateway.py index bc2c4da..5ad3456 100644 --- a/devplacepy/services/openai_gateway/gateway.py +++ b/devplacepy/services/openai_gateway/gateway.py @@ -259,6 +259,14 @@ class GatewayRuntime: else: model = cfg["gateway_model"] log(f"requested model {requested!r} has no route, falling back to {model!r}") + # Fallback lookup is keyed on whichever route (if any) actually matched + # the client's raw request, never on the *effective* model: overlay + # unconditionally forces gateway_force_model, so "was a specific route + # matched" cannot be read off that flag - it is always true once any + # route (including "molodetz" itself) has matched. An unmatched request + # normalizes to "molodetz" so its own configured fallback still protects + # the default path, instead of silently never being consulted. + fallback_key = requested if overlay is not None else "molodetz" stream = bool(body.get("stream")) include_usage = bool((body.get("stream_options") or {}).get("include_usage")) @@ -310,7 +318,7 @@ class GatewayRuntime: requested_model = requested if _call_failed(resp, exc, timing): - fallback_route = resolve_fallback(requested, "chat") + fallback_route = resolve_fallback(fallback_key, "chat") fallback_overlay = ( chat_overlay(fallback_route.source_model, base_cfg) if fallback_route is not None @@ -612,6 +620,10 @@ class GatewayRuntime: else: model = cfg["gateway_embed_model"] log(f"requested embed model {requested!r} has no route, falling back to {model!r}") + # See handle_chat's matching comment: fallback lookup must key off + # whether a route actually matched, not off gateway_force_model, which + # a matched overlay always forces true. + fallback_key = requested if overlay is not None else "molodetz~embed" payload = dict(body) payload["model"] = model @@ -637,7 +649,7 @@ class GatewayRuntime: requested_model = requested if _call_failed(resp, exc, timing): - fallback_route = resolve_fallback(requested, "embed") + fallback_route = resolve_fallback(fallback_key, "embed") fallback_overlay = ( embed_overlay(fallback_route.source_model, base_cfg) if fallback_route is not None @@ -818,6 +830,10 @@ class GatewayRuntime: else: model = cfg["gateway_image_model"] log(f"requested image model {requested!r} has no route, falling back to {model!r}") + # See handle_chat's matching comment: fallback lookup must key off + # whether a route actually matched, not off gateway_force_model, which + # a matched overlay always forces true. + fallback_key = requested if overlay is not None else "molodetz-img-small" payload = dict(body) payload["model"] = model @@ -843,7 +859,7 @@ class GatewayRuntime: requested_model = requested if _call_failed(resp, exc, timing): - fallback_route = resolve_fallback(requested, "image") + fallback_route = resolve_fallback(fallback_key, "image") fallback_overlay = ( image_overlay(fallback_route.source_model, base_cfg) if fallback_route is not None diff --git a/tests/unit/services/openai_gateway/gateway.py b/tests/unit/services/openai_gateway/gateway.py index 97ec66d..6d64899 100644 --- a/tests/unit/services/openai_gateway/gateway.py +++ b/tests/unit/services/openai_gateway/gateway.py @@ -728,6 +728,101 @@ def test_chat_falls_back_when_the_primary_model_fails(local_db, monkeypatch): routing.model_store.remove("fb-backup-route") +def test_chat_falls_back_via_molodetz_when_the_client_sends_an_unrouted_model_name( + local_db, monkeypatch +): + from devplacepy.services.openai_gateway import routing + + original = routing.model_store.get("molodetz") + routing.model_store.set( + routing.ModelRouteIn(source_model="fb-molodetz-backup", target_model="backup-target") + ) + routing.model_store.set( + routing.ModelRouteIn( + source_model="molodetz", + target_model="primary-target", + fallback_model="fb-molodetz-backup", + ) + ) + try: + svc = GatewayService() + cfg = svc.effective_config() + cfg["gateway_vision_enabled"] = False + cfg["gateway_max_retries"] = 0 + cfg["gateway_force_model"] = False + base_default_model = cfg["gateway_model"] + + class FakeUnroutedPrimaryFailsClient: + def __init__(self, *a, **k): + self.calls = [] + + def build_request(self, method, url, headers=None, json=None, content=None): + return FakeRequest(method, url, json) + + async def send(self, request, stream=False): + self.calls.append((request.url, request.json_body)) + body = request.json_body or {} + model = body.get("model") + if model == base_default_model: + return FakeResp_openai_gateway( + status=402, payload={"error": {"message": "Insufficient Balance"}} + ) + return FakeResp_openai_gateway( + payload={ + "id": "x", + "model": model, + "choices": [{"message": {"content": "fallback ok"}}], + } + ) + + async def aclose(self): + pass + + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeUnroutedPrimaryFailsClient) + rt = svc.runtime() + # The client sends a model name that matches no configured route at all + # (as a caller like dpc does, which is never told the exact "molodetz" + # routing alias) - the primary call falls through to the flat/global + # default model (no route overlay applies), but a failure there must + # still consult molodetz's own configured fallback, not silently skip + # it just because the client's literal string never matched anything. + response = run_async( + rt.handle_chat( + { + "model": "some-client-side-model-name", + "messages": [{"role": "user", "content": "hi"}], + }, + cfg, + ("guest", "fallback_chat_unrouted_client_model"), + "test", + "default", + ) + ) + assert response.status_code == 200 + assert rt._client.calls[0][1]["model"] == base_default_model + assert rt._client.calls[-1][1]["model"] == "backup-target" + row = get_table("gateway_usage_ledger").find_one( + owner_id="fallback_chat_unrouted_client_model" + ) + assert row is not None + assert row["requested_model"] == "some-client-side-model-name" + assert row["model"] == "backup-target" + assert row["success"] == 1 + finally: + routing.model_store.remove("fb-molodetz-backup") + if original is not None: + routing.model_store.set( + routing.ModelRouteIn( + **{ + field: getattr(original, field) + for field in routing.ModelRouteIn.model_fields + } + ) + ) + else: + routing.model_store.remove("molodetz") + + def test_chat_returns_the_fallback_failure_when_both_models_fail(local_db, monkeypatch): from devplacepy.services.openai_gateway import routing