From 67c85e4184203da2a9cc2ede414b00826bd5aa13 Mon Sep 17 00:00:00 2001 From: retoor Date: Thu, 3 Sep 2026 18:56:05 +0000 Subject: [PATCH] Fix gateway model fallback silently skipped for unrouted client model names The per-route fallback_model mechanism (used to fail over to a different provider when the primary upstream errors, e.g. insufficient balance) looked up the fallback keyed on the client's raw, literal "model" string. That string only matches a configured route when the caller sends the exact alias ("molodetz"/"molodetz~embed"/"molodetz-img-small") or another exact route name - any other value (the common case for external agents, which rarely echo DevPlace's own alias) resolves no route at all, so resolve_fallback() returned None and a real 402/5xx from the primary upstream propagated straight to the caller even with a fallback configured on the default route. Fixed in all three handlers (chat/embed/image): the fallback lookup key now reflects whether a route actually matched the raw request (chat_overlay result), independent of gateway_force_model - which is always forced true by a successful overlay and therefore cannot be used to tell "matched a specific route" apart from "used the default". An unmatched request now normalizes to the default alias for fallback purposes, so its configured fallback_model is consulted instead of silently skipped. Added a regression test that reproduces the exact failure (an unrouted client model name, primary upstream returns 402, fallback configured on the default route) and confirms it now recovers instead of surfacing the 402 to the caller. Co-Authored-By: Claude Sonnet 5 --- devplacepy/services/openai_gateway/gateway.py | 22 ++++- tests/unit/services/openai_gateway/gateway.py | 95 +++++++++++++++++++ 2 files changed, 114 insertions(+), 3 deletions(-) diff --git a/devplacepy/services/openai_gateway/gateway.py b/devplacepy/services/openai_gateway/gateway.py index bc2c4da..5ad3456 100644 --- a/devplacepy/services/openai_gateway/gateway.py +++ b/devplacepy/services/openai_gateway/gateway.py @@ -259,6 +259,14 @@ class GatewayRuntime: else: model = cfg["gateway_model"] log(f"requested model {requested!r} has no route, falling back to {model!r}") + # Fallback lookup is keyed on whichever route (if any) actually matched + # the client's raw request, never on the *effective* model: overlay + # unconditionally forces gateway_force_model, so "was a specific route + # matched" cannot be read off that flag - it is always true once any + # route (including "molodetz" itself) has matched. An unmatched request + # normalizes to "molodetz" so its own configured fallback still protects + # the default path, instead of silently never being consulted. + fallback_key = requested if overlay is not None else "molodetz" stream = bool(body.get("stream")) include_usage = bool((body.get("stream_options") or {}).get("include_usage")) @@ -310,7 +318,7 @@ class GatewayRuntime: requested_model = requested if _call_failed(resp, exc, timing): - fallback_route = resolve_fallback(requested, "chat") + fallback_route = resolve_fallback(fallback_key, "chat") fallback_overlay = ( chat_overlay(fallback_route.source_model, base_cfg) if fallback_route is not None @@ -612,6 +620,10 @@ class GatewayRuntime: else: model = cfg["gateway_embed_model"] log(f"requested embed model {requested!r} has no route, falling back to {model!r}") + # See handle_chat's matching comment: fallback lookup must key off + # whether a route actually matched, not off gateway_force_model, which + # a matched overlay always forces true. + fallback_key = requested if overlay is not None else "molodetz~embed" payload = dict(body) payload["model"] = model @@ -637,7 +649,7 @@ class GatewayRuntime: requested_model = requested if _call_failed(resp, exc, timing): - fallback_route = resolve_fallback(requested, "embed") + fallback_route = resolve_fallback(fallback_key, "embed") fallback_overlay = ( embed_overlay(fallback_route.source_model, base_cfg) if fallback_route is not None @@ -818,6 +830,10 @@ class GatewayRuntime: else: model = cfg["gateway_image_model"] log(f"requested image model {requested!r} has no route, falling back to {model!r}") + # See handle_chat's matching comment: fallback lookup must key off + # whether a route actually matched, not off gateway_force_model, which + # a matched overlay always forces true. + fallback_key = requested if overlay is not None else "molodetz-img-small" payload = dict(body) payload["model"] = model @@ -843,7 +859,7 @@ class GatewayRuntime: requested_model = requested if _call_failed(resp, exc, timing): - fallback_route = resolve_fallback(requested, "image") + fallback_route = resolve_fallback(fallback_key, "image") fallback_overlay = ( image_overlay(fallback_route.source_model, base_cfg) if fallback_route is not None diff --git a/tests/unit/services/openai_gateway/gateway.py b/tests/unit/services/openai_gateway/gateway.py index 97ec66d..6d64899 100644 --- a/tests/unit/services/openai_gateway/gateway.py +++ b/tests/unit/services/openai_gateway/gateway.py @@ -728,6 +728,101 @@ def test_chat_falls_back_when_the_primary_model_fails(local_db, monkeypatch): routing.model_store.remove("fb-backup-route") +def test_chat_falls_back_via_molodetz_when_the_client_sends_an_unrouted_model_name( + local_db, monkeypatch +): + from devplacepy.services.openai_gateway import routing + + original = routing.model_store.get("molodetz") + routing.model_store.set( + routing.ModelRouteIn(source_model="fb-molodetz-backup", target_model="backup-target") + ) + routing.model_store.set( + routing.ModelRouteIn( + source_model="molodetz", + target_model="primary-target", + fallback_model="fb-molodetz-backup", + ) + ) + try: + svc = GatewayService() + cfg = svc.effective_config() + cfg["gateway_vision_enabled"] = False + cfg["gateway_max_retries"] = 0 + cfg["gateway_force_model"] = False + base_default_model = cfg["gateway_model"] + + class FakeUnroutedPrimaryFailsClient: + def __init__(self, *a, **k): + self.calls = [] + + def build_request(self, method, url, headers=None, json=None, content=None): + return FakeRequest(method, url, json) + + async def send(self, request, stream=False): + self.calls.append((request.url, request.json_body)) + body = request.json_body or {} + model = body.get("model") + if model == base_default_model: + return FakeResp_openai_gateway( + status=402, payload={"error": {"message": "Insufficient Balance"}} + ) + return FakeResp_openai_gateway( + payload={ + "id": "x", + "model": model, + "choices": [{"message": {"content": "fallback ok"}}], + } + ) + + async def aclose(self): + pass + + monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeUnroutedPrimaryFailsClient) + rt = svc.runtime() + # The client sends a model name that matches no configured route at all + # (as a caller like dpc does, which is never told the exact "molodetz" + # routing alias) - the primary call falls through to the flat/global + # default model (no route overlay applies), but a failure there must + # still consult molodetz's own configured fallback, not silently skip + # it just because the client's literal string never matched anything. + response = run_async( + rt.handle_chat( + { + "model": "some-client-side-model-name", + "messages": [{"role": "user", "content": "hi"}], + }, + cfg, + ("guest", "fallback_chat_unrouted_client_model"), + "test", + "default", + ) + ) + assert response.status_code == 200 + assert rt._client.calls[0][1]["model"] == base_default_model + assert rt._client.calls[-1][1]["model"] == "backup-target" + row = get_table("gateway_usage_ledger").find_one( + owner_id="fallback_chat_unrouted_client_model" + ) + assert row is not None + assert row["requested_model"] == "some-client-side-model-name" + assert row["model"] == "backup-target" + assert row["success"] == 1 + finally: + routing.model_store.remove("fb-molodetz-backup") + if original is not None: + routing.model_store.set( + routing.ModelRouteIn( + **{ + field: getattr(original, field) + for field in routing.ModelRouteIn.model_fields + } + ) + ) + else: + routing.model_store.remove("molodetz") + + def test_chat_returns_the_fallback_failure_when_both_models_fail(local_db, monkeypatch): from devplacepy.services.openai_gateway import routing