Fix gateway model fallback silently skipped for unrouted client model names
DevPlace CI / test (push) Failing after 26m32s
DevPlace CI / test (push) Failing after 26m32s
The per-route fallback_model mechanism (used to fail over to a different
provider when the primary upstream errors, e.g. insufficient balance)
looked up the fallback keyed on the client's raw, literal "model" string.
That string only matches a configured route when the caller sends the
exact alias ("molodetz"/"molodetz~embed"/"molodetz-img-small") or another
exact route name - any other value (the common case for external agents,
which rarely echo DevPlace's own alias) resolves no route at all, so
resolve_fallback() returned None and a real 402/5xx from the primary
upstream propagated straight to the caller even with a fallback configured
on the default route.
Fixed in all three handlers (chat/embed/image): the fallback lookup key
now reflects whether a route actually matched the raw request (chat_overlay
result), independent of gateway_force_model - which is always forced true
by a successful overlay and therefore cannot be used to tell "matched a
specific route" apart from "used the default". An unmatched request now
normalizes to the default alias for fallback purposes, so its configured
fallback_model is consulted instead of silently skipped.
Added a regression test that reproduces the exact failure (an unrouted
client model name, primary upstream returns 402, fallback configured on
the default route) and confirms it now recovers instead of surfacing the
402 to the caller.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -728,6 +728,101 @@ def test_chat_falls_back_when_the_primary_model_fails(local_db, monkeypatch):
|
||||
routing.model_store.remove("fb-backup-route")
|
||||
|
||||
|
||||
def test_chat_falls_back_via_molodetz_when_the_client_sends_an_unrouted_model_name(
|
||||
local_db, monkeypatch
|
||||
):
|
||||
from devplacepy.services.openai_gateway import routing
|
||||
|
||||
original = routing.model_store.get("molodetz")
|
||||
routing.model_store.set(
|
||||
routing.ModelRouteIn(source_model="fb-molodetz-backup", target_model="backup-target")
|
||||
)
|
||||
routing.model_store.set(
|
||||
routing.ModelRouteIn(
|
||||
source_model="molodetz",
|
||||
target_model="primary-target",
|
||||
fallback_model="fb-molodetz-backup",
|
||||
)
|
||||
)
|
||||
try:
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
cfg["gateway_max_retries"] = 0
|
||||
cfg["gateway_force_model"] = False
|
||||
base_default_model = cfg["gateway_model"]
|
||||
|
||||
class FakeUnroutedPrimaryFailsClient:
|
||||
def __init__(self, *a, **k):
|
||||
self.calls = []
|
||||
|
||||
def build_request(self, method, url, headers=None, json=None, content=None):
|
||||
return FakeRequest(method, url, json)
|
||||
|
||||
async def send(self, request, stream=False):
|
||||
self.calls.append((request.url, request.json_body))
|
||||
body = request.json_body or {}
|
||||
model = body.get("model")
|
||||
if model == base_default_model:
|
||||
return FakeResp_openai_gateway(
|
||||
status=402, payload={"error": {"message": "Insufficient Balance"}}
|
||||
)
|
||||
return FakeResp_openai_gateway(
|
||||
payload={
|
||||
"id": "x",
|
||||
"model": model,
|
||||
"choices": [{"message": {"content": "fallback ok"}}],
|
||||
}
|
||||
)
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeUnroutedPrimaryFailsClient)
|
||||
rt = svc.runtime()
|
||||
# The client sends a model name that matches no configured route at all
|
||||
# (as a caller like dpc does, which is never told the exact "molodetz"
|
||||
# routing alias) - the primary call falls through to the flat/global
|
||||
# default model (no route overlay applies), but a failure there must
|
||||
# still consult molodetz's own configured fallback, not silently skip
|
||||
# it just because the client's literal string never matched anything.
|
||||
response = run_async(
|
||||
rt.handle_chat(
|
||||
{
|
||||
"model": "some-client-side-model-name",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
},
|
||||
cfg,
|
||||
("guest", "fallback_chat_unrouted_client_model"),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert rt._client.calls[0][1]["model"] == base_default_model
|
||||
assert rt._client.calls[-1][1]["model"] == "backup-target"
|
||||
row = get_table("gateway_usage_ledger").find_one(
|
||||
owner_id="fallback_chat_unrouted_client_model"
|
||||
)
|
||||
assert row is not None
|
||||
assert row["requested_model"] == "some-client-side-model-name"
|
||||
assert row["model"] == "backup-target"
|
||||
assert row["success"] == 1
|
||||
finally:
|
||||
routing.model_store.remove("fb-molodetz-backup")
|
||||
if original is not None:
|
||||
routing.model_store.set(
|
||||
routing.ModelRouteIn(
|
||||
**{
|
||||
field: getattr(original, field)
|
||||
for field in routing.ModelRouteIn.model_fields
|
||||
}
|
||||
)
|
||||
)
|
||||
else:
|
||||
routing.model_store.remove("molodetz")
|
||||
|
||||
|
||||
def test_chat_returns_the_fallback_failure_when_both_models_fail(local_db, monkeypatch):
|
||||
from devplacepy.services.openai_gateway import routing
|
||||
|
||||
|
||||
Reference in New Issue
Block a user