Gateway: refine routing, vision and request handling, update docs and tests

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-15 01:50:50 +01:00
co-authored by Claude Sonnet 5.5
parent 8e36807926
commit 8d4d06de61
11 changed files with 947 additions and 67 deletions
+395 -7
View File
@@ -3726,8 +3726,9 @@ def test_a_failed_target_is_skipped_without_an_upstream_call(local_db):
def test_one_bad_request_does_not_block_another_target(local_db):
rt, cfg = _target_runtime_openai_gateway()
client = FakeAlwaysFailClient_openai_gateway()
_target_send_openai_gateway(rt, cfg, client, "https://target-b.example/v1/chat/completions", "m1")
other = _target_send_openai_gateway(rt, cfg, client, "https://target-c.example/v1/chat/completions", "m1")
url = "https://target-b.example/v1/chat/completions"
_target_send_openai_gateway(rt, cfg, client, url, "model-b1")
other = _target_send_openai_gateway(rt, cfg, client, url, "model-b2")
assert other[2]["circuit_open"] is False
assert len(client.calls) == 2
@@ -3759,20 +3760,407 @@ def test_embedded_200_error_trips_the_target(local_db):
resp = FakeResp_openai_gateway(status=200, payload={"error": {"message": "provider exploded"}})
url = "https://target-e.example/v1/chat/completions"
assert rt._embedded_error(resp, False, url, "m1", cfg) is not None
assert rt._target_breaker(gwmod._target_key(url, {"model": "m1"}), cfg).is_open is True
assert rt.open_targets()[0]["provider"] == "target-e.example"
assert rt._target_breaker("m1", cfg).is_open is True
assert rt.open_targets()[0]["model"] == "m1"
def test_transition_notifies_each_admin_once_with_detail_and_no_key(local_db, monkeypatch):
sent = []
monkeypatch.setattr(gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a)))
rt, cfg = _target_runtime_openai_gateway()
target = ("target-f.example", "m1")
rt._trip_target(target, cfg, "upstream_error", 500, "Bearer sk-secret123 failed", 1)
rt._trip_target(target, cfg, "upstream_error", 500, "again", 1)
url = "https://target-f.example/v1/chat/completions"
rt._trip_target("m1", url, cfg, "upstream_error", 500, "Bearer sk-secret123 failed", 1)
rt._trip_target("m1", url, cfg, "upstream_error", 500, "again", 1)
assert len(sent) == 1
detail = sent[0][1][1]
assert detail["category"] == "upstream_error"
assert detail["status"] == 500
assert "secret123" not in detail["upstream"]
assert detail["provider"] == "target-f.example"
def test_breaker_is_per_model_across_upstreams(local_db):
rt, cfg = _target_runtime_openai_gateway()
client = FakeAlwaysFailClient_openai_gateway()
_target_send_openai_gateway(rt, cfg, client, "https://target-g1.example/v1/chat/completions", "shared-model")
second = _target_send_openai_gateway(rt, cfg, client, "https://target-g2.example/v1/chat/completions", "shared-model")
assert second[2]["circuit_open"] is True
assert len(client.calls) == 1
def test_a_request_without_a_model_never_uses_the_target_breaker(local_db):
rt, cfg = _target_runtime_openai_gateway()
client = FakeAlwaysFailClient_openai_gateway()
url = "https://target-h.example/v1/other"
_target_send_openai_gateway(rt, cfg, client, url, "")
again = _target_send_openai_gateway(rt, cfg, client, url, "")
assert again[2]["circuit_open"] is False
assert rt._target_breakers == {}
class FakeNotifyClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
if body.get("model") == "nb-primary-target":
return FakeResp_openai_gateway(
status=500, payload={"error": "boom sk-secret-xyz"}
)
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "notify ok"}}],
}
)
async def aclose(self):
pass
class FakeNotifyChainClient_openai_gateway:
def __init__(self, *a, **k):
self.calls = []
def build_request(self, method, url, headers=None, json=None, content=None):
return FakeRequest(method, url, json)
async def send(self, request, stream=False):
self.calls.append((request.url, request.json_body))
body = request.json_body or {}
if body.get("model") in ("n3-a-target", "n3-b-target"):
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
return FakeResp_openai_gateway(
payload={
"id": "x",
"model": body.get("model"),
"choices": [{"message": {"content": "chain ok"}}],
}
)
async def aclose(self):
pass
def _notify_chat_openai_gateway(monkeypatch, client_cls, routes, source, owner_id):
from devplacepy.services.openai_gateway import routing
monkeypatch.setattr(gwmod.httpx, "AsyncClient", client_cls)
for payload in routes:
routing.model_store.set(routing.ModelRouteIn(**payload))
sent = []
def _capture_notify_openai_gateway(fn, *a, **k):
if fn is gwmod._notify_gateway_status:
sent.append((fn, a))
monkeypatch.setattr(gwmod.background, "submit", _capture_notify_openai_gateway)
try:
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_vision_enabled"] = False
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
response = run_async(
rt.handle_chat(
{"model": source, "messages": [{"role": "user", "content": "hi"}]},
cfg,
("guest", owner_id),
"test",
"default",
)
)
return response, sent
finally:
for payload in routes:
routing.model_store.remove(payload["source_model"])
def _rendered_messages_openai_gateway(monkeypatch, details):
messages = []
class FakeUsersTable_openai_gateway:
def find(self, **kwargs):
if kwargs.get("role") == "Admin":
return [{"uid": "admin-1"}]
return []
monkeypatch.setattr(
"devplacepy.database.get_table",
lambda name: FakeUsersTable_openai_gateway(),
)
monkeypatch.setattr(
"devplacepy.utils.create_notification",
lambda uid, kind, message, *a, **k: messages.append(message),
)
for detail in details:
gwmod._notify_gateway_status(True, detail)
return messages
def test_trip_notification_carries_local_config_redirect_and_outcome(local_db, monkeypatch):
routes = [
{"source_model": "nb-backup", "target_model": "nb-backup-target"},
{
"source_model": "nb-primary",
"target_model": "nb-primary-target",
"fallback_model": "nb-backup",
},
]
response, sent = _notify_chat_openai_gateway(
monkeypatch, FakeNotifyClient_openai_gateway, routes, "nb-primary", "notify_one"
)
assert response.status_code == 200
assert len(sent) == 1
assert sent[0][0] is gwmod._notify_gateway_status
detail = sent[0][1][1]
assert detail["route"] == "nb-primary"
assert detail["model"] == "nb-primary-target"
assert detail["requested"] == "nb-primary"
assert "nb-primary" in detail["chain"]
assert "nb-backup" in detail["chain"]
assert detail["redirect"] == "nb-backup"
assert detail["outcome_ok"] is True
assert detail["served_route"] == "nb-backup"
assert "nb-backup" in detail["outcome"]
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
assert len(messages) == 1
assert "tripped its circuit breaker" in messages[0]
assert "Local config: 'nb-primary'" in messages[0]
assert "Full route:" in messages[0]
assert "nb-primary" in messages[0].split("Full route:")[1]
assert "nb-backup" in messages[0].split("Full route:")[1]
assert "Redirect: will redirect to fallback 'nb-backup'" in messages[0]
assert "Outcome: fallback 'nb-backup' answered HTTP 200 ok" in messages[0]
assert "Error class: upstream_error" in messages[0]
assert "HTTP status: 500" in messages[0]
assert "sk-secret-xyz" not in messages[0]
assert "[redacted]" in messages[0]
def test_trip_notifications_cover_every_hop_of_a_recursive_chain(local_db, monkeypatch):
routes = [
{"source_model": "n3-c", "target_model": "n3-c-target"},
{
"source_model": "n3-b",
"target_model": "n3-b-target",
"fallback_model": "n3-c",
},
{
"source_model": "n3-a",
"target_model": "n3-a-target",
"fallback_model": "n3-b",
},
]
response, sent = _notify_chat_openai_gateway(
monkeypatch, FakeNotifyChainClient_openai_gateway, routes, "n3-a", "notify_chain"
)
assert response.status_code == 200
assert len(sent) == 2
first = sent[0][1][1]
second = sent[1][1][1]
assert first["route"] == "n3-a"
assert first["redirect"] == "n3-b"
assert second["route"] == "n3-b"
assert second["redirect"] == "n3-c"
for detail in (first, second):
assert detail["requested"] == "n3-a"
assert "n3-a" in detail["chain"]
assert "n3-b" in detail["chain"]
assert "n3-c" in detail["chain"]
assert detail["outcome_ok"] is True
assert detail["outcome"] == "fallback 'n3-c' answered HTTP 200 ok"
assert detail["served_route"] == "n3-c"
messages = _rendered_messages_openai_gateway(monkeypatch, [first, second])
assert "Redirect: will redirect to fallback 'n3-b'" in messages[0]
assert "Redirect: will redirect to fallback 'n3-c'" in messages[1]
assert "Outcome: fallback 'n3-c' answered HTTP 200 ok" in messages[0]
assert "Outcome: fallback 'n3-c' answered HTTP 200 ok" in messages[1]
def test_trip_notification_reports_failed_outcome_when_every_fallback_fails(
local_db, monkeypatch
):
routes = [
{"source_model": "n4-b", "target_model": "n4-b-target"},
{
"source_model": "n4-a",
"target_model": "n4-a-target",
"fallback_model": "n4-b",
},
]
response, sent = _notify_chat_openai_gateway(
monkeypatch, FakeAlwaysFailClient_openai_gateway, routes, "n4-a", "notify_fail"
)
assert response.status_code == 500
assert len(sent) == 2
first = sent[0][1][1]
second = sent[1][1][1]
assert first["route"] == "n4-a"
assert first["redirect"] == "n4-b"
assert second["route"] == "n4-b"
assert second["redirect"] == ""
for detail in (first, second):
assert detail["outcome_ok"] is False
assert "all 2 route(s) failed" in detail["outcome"]
assert "HTTP 500" in detail["outcome"]
messages = _rendered_messages_openai_gateway(monkeypatch, [first, second])
assert "Redirect: will redirect to fallback 'n4-b'" in messages[0]
assert "Redirect: none, end of the fallback line" in messages[1]
assert "Outcome: all 2 route(s) failed" in messages[1]
def test_trip_notification_redirects_unrouted_default_to_first_fallback(
local_db, monkeypatch
):
rt, cfg = _target_runtime_openai_gateway()
sent = []
monkeypatch.setattr(
gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a))
)
detail = gwmod._trip_detail(
"default-target",
"https://default.example/v1/chat/completions",
30,
"upstream_error",
500,
"boom",
0,
route="",
)
rt._notify_trips(
[{"kind": "target", "detail": detail}],
"chat",
"molodetz",
"weird-model",
[{"route": ""}, {"route": "m-pro"}],
True,
"fallback 'm-pro' answered HTTP 200 ok",
"m-pro",
)
assert len(sent) == 1
enriched = sent[0][1][1]
assert enriched["redirect"] == "m-pro"
messages = _rendered_messages_openai_gateway(monkeypatch, [enriched])
assert "Local config: default (no routed model)" in messages[0]
assert "Redirect: will redirect to fallback 'm-pro'" in messages[0]
assert "Outcome: fallback 'm-pro' answered HTTP 200 ok" in messages[0]
def test_vision_trip_notifies_with_local_config_and_no_fallback(local_db, monkeypatch):
import httpx
from devplacepy.services.openai_gateway.reliability import CircuitBreaker
from devplacepy.services.openai_gateway.vision import VisionAugmenter, VisionCache
class FailVisionClient_openai_gateway:
async def post(self, url, headers=None, json=None):
raise httpx.ConnectError("vision down")
breaker = CircuitBreaker(1, 60)
sink = []
augmenter = VisionAugmenter(
"https://vision.example/v1/chat/completions",
"vis-model",
"k",
VisionCache(0),
breaker=breaker,
max_retries=0,
trip_sink=sink,
)
image = {"type": "image_url", "image_url": {"url": "http://x/y.png"}}
first = run_async(augmenter._describe_one(FailVisionClient_openai_gateway(), image))
assert first.startswith("[vision call failed")
assert breaker.is_open is True
assert len(sink) == 1
assert sink[0]["kind"] == "vision"
assert sink[0]["detail"]["route"] == "vision:vis-model"
second = run_async(augmenter._describe_one(FailVisionClient_openai_gateway(), image))
assert second.startswith("[vision unavailable")
assert len(sink) == 1
rt, cfg = _target_runtime_openai_gateway()
sent = []
monkeypatch.setattr(
gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a))
)
rt._notify_trips(
sink,
"chat",
"molodetz",
"molodetz",
[{"route": "molodetz"}],
True,
"fallback 'molodetz' answered HTTP 200 ok",
"molodetz",
)
assert len(sent) == 1
messages = _rendered_messages_openai_gateway(monkeypatch, [sent[0][1][1]])
assert "Local config: 'vision:vis-model'" in messages[0]
assert "Redirect: none, vision has no fallback model" in messages[0]
assert "Outcome: fallback 'molodetz' answered HTTP 200 ok" in messages[0]
def test_trip_notification_outcome_scrubs_secrets(local_db, monkeypatch):
detail = gwmod._trip_detail(
"s1-target",
"https://s1.example/v1/chat/completions",
30,
"connection",
"no response",
"boom",
0,
route="s1",
)
detail["requested"] = "s1"
detail["chain"] = "s1 (s1-target @ default)"
detail["redirect"] = ""
secret = "sk" + "-abc123"
detail["outcome"] = "connection failed (Bearer " + secret + ")"
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
assert secret not in messages[0]
assert "[redacted]" in messages[0]
def test_passthrough_global_trip_carries_local_config(local_db, monkeypatch):
monkeypatch.setattr(
gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway
)
svc = GatewayService()
cfg = svc.effective_config()
cfg["gateway_circuit_threshold"] = 1
cfg["gateway_max_retries"] = 0
rt = svc.runtime()
sent = []
monkeypatch.setattr(
gwmod.background,
"submit",
lambda fn, *a, **k: sent.append((fn, a))
if fn is gwmod._notify_gateway_status
else None,
)
resp = run_async(
rt.handle_passthrough(
"POST",
"responses",
"application/json",
b"{}",
cfg,
("guest", "ptrip_local"),
"test",
"default",
)
)
assert resp.status_code == 500
assert len(sent) == 1
detail = sent[0][1][1]
assert detail.get("global") is True
assert detail["route"] == cfg["gateway_model"]
assert "no fallback line" in detail["chain"]
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
assert f"Local config: '{cfg['gateway_model']}'" in messages[0]
assert "passthrough POST responses (no fallback line)" in messages[0]
assert "Outcome: all 1 route(s) failed" in messages[0]
@@ -804,3 +804,39 @@ def test_fetch_provider_models_tries_meta_listing_second(local_db, monkeypatch):
]
finally:
r.provider_store.remove("probeprovmeta")
def test_provider_key_rotation_follows_the_default_upstream_key(local_db):
from devplacepy.database import get_setting, set_setting
original = get_setting(r.DEFAULT_KEY_SETTING, "")
try:
set_setting(r.DEFAULT_KEY_SETTING, "shared-old")
r.provider_store.set(
r.ProviderIn(name="utrot", base_url="https://up.example/v1/chat/completions", api_key="shared-old")
)
r.provider_store.set(
r.ProviderIn(name="utrot", base_url="https://up.example/v1/chat/completions", api_key="shared-new")
)
assert get_setting(r.DEFAULT_KEY_SETTING, "") == "shared-new"
finally:
set_setting(r.DEFAULT_KEY_SETTING, original)
_cleanup(["utrot"], [])
def test_provider_key_rotation_leaves_an_unrelated_default_key_alone(local_db):
from devplacepy.database import get_setting, set_setting
original = get_setting(r.DEFAULT_KEY_SETTING, "")
try:
set_setting(r.DEFAULT_KEY_SETTING, "default-key")
r.provider_store.set(
r.ProviderIn(name="utrot2", base_url="https://up.example/v1/chat/completions", api_key="a")
)
r.provider_store.set(
r.ProviderIn(name="utrot2", base_url="https://up.example/v1/chat/completions", api_key="b")
)
assert get_setting(r.DEFAULT_KEY_SETTING, "") == "default-key"
finally:
set_setting(r.DEFAULT_KEY_SETTING, original)
_cleanup(["utrot2"], [])