|
|
|
@@ -3726,8 +3726,9 @@ def test_a_failed_target_is_skipped_without_an_upstream_call(local_db):
|
|
|
|
|
def test_one_bad_request_does_not_block_another_target(local_db):
|
|
|
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
|
|
|
client = FakeAlwaysFailClient_openai_gateway()
|
|
|
|
|
_target_send_openai_gateway(rt, cfg, client, "https://target-b.example/v1/chat/completions", "m1")
|
|
|
|
|
other = _target_send_openai_gateway(rt, cfg, client, "https://target-c.example/v1/chat/completions", "m1")
|
|
|
|
|
url = "https://target-b.example/v1/chat/completions"
|
|
|
|
|
_target_send_openai_gateway(rt, cfg, client, url, "model-b1")
|
|
|
|
|
other = _target_send_openai_gateway(rt, cfg, client, url, "model-b2")
|
|
|
|
|
assert other[2]["circuit_open"] is False
|
|
|
|
|
assert len(client.calls) == 2
|
|
|
|
|
|
|
|
|
@@ -3759,20 +3760,407 @@ def test_embedded_200_error_trips_the_target(local_db):
|
|
|
|
|
resp = FakeResp_openai_gateway(status=200, payload={"error": {"message": "provider exploded"}})
|
|
|
|
|
url = "https://target-e.example/v1/chat/completions"
|
|
|
|
|
assert rt._embedded_error(resp, False, url, "m1", cfg) is not None
|
|
|
|
|
assert rt._target_breaker(gwmod._target_key(url, {"model": "m1"}), cfg).is_open is True
|
|
|
|
|
assert rt.open_targets()[0]["provider"] == "target-e.example"
|
|
|
|
|
assert rt._target_breaker("m1", cfg).is_open is True
|
|
|
|
|
assert rt.open_targets()[0]["model"] == "m1"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_transition_notifies_each_admin_once_with_detail_and_no_key(local_db, monkeypatch):
|
|
|
|
|
sent = []
|
|
|
|
|
monkeypatch.setattr(gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a)))
|
|
|
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
|
|
|
target = ("target-f.example", "m1")
|
|
|
|
|
rt._trip_target(target, cfg, "upstream_error", 500, "Bearer sk-secret123 failed", 1)
|
|
|
|
|
rt._trip_target(target, cfg, "upstream_error", 500, "again", 1)
|
|
|
|
|
url = "https://target-f.example/v1/chat/completions"
|
|
|
|
|
rt._trip_target("m1", url, cfg, "upstream_error", 500, "Bearer sk-secret123 failed", 1)
|
|
|
|
|
rt._trip_target("m1", url, cfg, "upstream_error", 500, "again", 1)
|
|
|
|
|
assert len(sent) == 1
|
|
|
|
|
detail = sent[0][1][1]
|
|
|
|
|
assert detail["category"] == "upstream_error"
|
|
|
|
|
assert detail["status"] == 500
|
|
|
|
|
assert "secret123" not in detail["upstream"]
|
|
|
|
|
assert detail["provider"] == "target-f.example"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_breaker_is_per_model_across_upstreams(local_db):
|
|
|
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
|
|
|
client = FakeAlwaysFailClient_openai_gateway()
|
|
|
|
|
_target_send_openai_gateway(rt, cfg, client, "https://target-g1.example/v1/chat/completions", "shared-model")
|
|
|
|
|
second = _target_send_openai_gateway(rt, cfg, client, "https://target-g2.example/v1/chat/completions", "shared-model")
|
|
|
|
|
assert second[2]["circuit_open"] is True
|
|
|
|
|
assert len(client.calls) == 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_a_request_without_a_model_never_uses_the_target_breaker(local_db):
|
|
|
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
|
|
|
client = FakeAlwaysFailClient_openai_gateway()
|
|
|
|
|
url = "https://target-h.example/v1/other"
|
|
|
|
|
_target_send_openai_gateway(rt, cfg, client, url, "")
|
|
|
|
|
again = _target_send_openai_gateway(rt, cfg, client, url, "")
|
|
|
|
|
assert again[2]["circuit_open"] is False
|
|
|
|
|
assert rt._target_breakers == {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class FakeNotifyClient_openai_gateway:
|
|
|
|
|
def __init__(self, *a, **k):
|
|
|
|
|
self.calls = []
|
|
|
|
|
|
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
|
|
|
return FakeRequest(method, url, json)
|
|
|
|
|
|
|
|
|
|
async def send(self, request, stream=False):
|
|
|
|
|
self.calls.append((request.url, request.json_body))
|
|
|
|
|
body = request.json_body or {}
|
|
|
|
|
if body.get("model") == "nb-primary-target":
|
|
|
|
|
return FakeResp_openai_gateway(
|
|
|
|
|
status=500, payload={"error": "boom sk-secret-xyz"}
|
|
|
|
|
)
|
|
|
|
|
return FakeResp_openai_gateway(
|
|
|
|
|
payload={
|
|
|
|
|
"id": "x",
|
|
|
|
|
"model": body.get("model"),
|
|
|
|
|
"choices": [{"message": {"content": "notify ok"}}],
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
async def aclose(self):
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class FakeNotifyChainClient_openai_gateway:
|
|
|
|
|
def __init__(self, *a, **k):
|
|
|
|
|
self.calls = []
|
|
|
|
|
|
|
|
|
|
def build_request(self, method, url, headers=None, json=None, content=None):
|
|
|
|
|
return FakeRequest(method, url, json)
|
|
|
|
|
|
|
|
|
|
async def send(self, request, stream=False):
|
|
|
|
|
self.calls.append((request.url, request.json_body))
|
|
|
|
|
body = request.json_body or {}
|
|
|
|
|
if body.get("model") in ("n3-a-target", "n3-b-target"):
|
|
|
|
|
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
|
|
|
|
|
return FakeResp_openai_gateway(
|
|
|
|
|
payload={
|
|
|
|
|
"id": "x",
|
|
|
|
|
"model": body.get("model"),
|
|
|
|
|
"choices": [{"message": {"content": "chain ok"}}],
|
|
|
|
|
}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
async def aclose(self):
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _notify_chat_openai_gateway(monkeypatch, client_cls, routes, source, owner_id):
|
|
|
|
|
from devplacepy.services.openai_gateway import routing
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(gwmod.httpx, "AsyncClient", client_cls)
|
|
|
|
|
for payload in routes:
|
|
|
|
|
routing.model_store.set(routing.ModelRouteIn(**payload))
|
|
|
|
|
sent = []
|
|
|
|
|
def _capture_notify_openai_gateway(fn, *a, **k):
|
|
|
|
|
if fn is gwmod._notify_gateway_status:
|
|
|
|
|
sent.append((fn, a))
|
|
|
|
|
monkeypatch.setattr(gwmod.background, "submit", _capture_notify_openai_gateway)
|
|
|
|
|
try:
|
|
|
|
|
svc = GatewayService()
|
|
|
|
|
cfg = svc.effective_config()
|
|
|
|
|
cfg["gateway_vision_enabled"] = False
|
|
|
|
|
cfg["gateway_max_retries"] = 0
|
|
|
|
|
rt = svc.runtime()
|
|
|
|
|
response = run_async(
|
|
|
|
|
rt.handle_chat(
|
|
|
|
|
{"model": source, "messages": [{"role": "user", "content": "hi"}]},
|
|
|
|
|
cfg,
|
|
|
|
|
("guest", owner_id),
|
|
|
|
|
"test",
|
|
|
|
|
"default",
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
return response, sent
|
|
|
|
|
finally:
|
|
|
|
|
for payload in routes:
|
|
|
|
|
routing.model_store.remove(payload["source_model"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _rendered_messages_openai_gateway(monkeypatch, details):
|
|
|
|
|
messages = []
|
|
|
|
|
|
|
|
|
|
class FakeUsersTable_openai_gateway:
|
|
|
|
|
def find(self, **kwargs):
|
|
|
|
|
if kwargs.get("role") == "Admin":
|
|
|
|
|
return [{"uid": "admin-1"}]
|
|
|
|
|
return []
|
|
|
|
|
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"devplacepy.database.get_table",
|
|
|
|
|
lambda name: FakeUsersTable_openai_gateway(),
|
|
|
|
|
)
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
"devplacepy.utils.create_notification",
|
|
|
|
|
lambda uid, kind, message, *a, **k: messages.append(message),
|
|
|
|
|
)
|
|
|
|
|
for detail in details:
|
|
|
|
|
gwmod._notify_gateway_status(True, detail)
|
|
|
|
|
return messages
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_trip_notification_carries_local_config_redirect_and_outcome(local_db, monkeypatch):
|
|
|
|
|
routes = [
|
|
|
|
|
{"source_model": "nb-backup", "target_model": "nb-backup-target"},
|
|
|
|
|
{
|
|
|
|
|
"source_model": "nb-primary",
|
|
|
|
|
"target_model": "nb-primary-target",
|
|
|
|
|
"fallback_model": "nb-backup",
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
response, sent = _notify_chat_openai_gateway(
|
|
|
|
|
monkeypatch, FakeNotifyClient_openai_gateway, routes, "nb-primary", "notify_one"
|
|
|
|
|
)
|
|
|
|
|
assert response.status_code == 200
|
|
|
|
|
assert len(sent) == 1
|
|
|
|
|
assert sent[0][0] is gwmod._notify_gateway_status
|
|
|
|
|
detail = sent[0][1][1]
|
|
|
|
|
assert detail["route"] == "nb-primary"
|
|
|
|
|
assert detail["model"] == "nb-primary-target"
|
|
|
|
|
assert detail["requested"] == "nb-primary"
|
|
|
|
|
assert "nb-primary" in detail["chain"]
|
|
|
|
|
assert "nb-backup" in detail["chain"]
|
|
|
|
|
assert detail["redirect"] == "nb-backup"
|
|
|
|
|
assert detail["outcome_ok"] is True
|
|
|
|
|
assert detail["served_route"] == "nb-backup"
|
|
|
|
|
assert "nb-backup" in detail["outcome"]
|
|
|
|
|
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
|
|
|
|
|
assert len(messages) == 1
|
|
|
|
|
assert "tripped its circuit breaker" in messages[0]
|
|
|
|
|
assert "Local config: 'nb-primary'" in messages[0]
|
|
|
|
|
assert "Full route:" in messages[0]
|
|
|
|
|
assert "nb-primary" in messages[0].split("Full route:")[1]
|
|
|
|
|
assert "nb-backup" in messages[0].split("Full route:")[1]
|
|
|
|
|
assert "Redirect: will redirect to fallback 'nb-backup'" in messages[0]
|
|
|
|
|
assert "Outcome: fallback 'nb-backup' answered HTTP 200 ok" in messages[0]
|
|
|
|
|
assert "Error class: upstream_error" in messages[0]
|
|
|
|
|
assert "HTTP status: 500" in messages[0]
|
|
|
|
|
assert "sk-secret-xyz" not in messages[0]
|
|
|
|
|
assert "[redacted]" in messages[0]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_trip_notifications_cover_every_hop_of_a_recursive_chain(local_db, monkeypatch):
|
|
|
|
|
routes = [
|
|
|
|
|
{"source_model": "n3-c", "target_model": "n3-c-target"},
|
|
|
|
|
{
|
|
|
|
|
"source_model": "n3-b",
|
|
|
|
|
"target_model": "n3-b-target",
|
|
|
|
|
"fallback_model": "n3-c",
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"source_model": "n3-a",
|
|
|
|
|
"target_model": "n3-a-target",
|
|
|
|
|
"fallback_model": "n3-b",
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
response, sent = _notify_chat_openai_gateway(
|
|
|
|
|
monkeypatch, FakeNotifyChainClient_openai_gateway, routes, "n3-a", "notify_chain"
|
|
|
|
|
)
|
|
|
|
|
assert response.status_code == 200
|
|
|
|
|
assert len(sent) == 2
|
|
|
|
|
first = sent[0][1][1]
|
|
|
|
|
second = sent[1][1][1]
|
|
|
|
|
assert first["route"] == "n3-a"
|
|
|
|
|
assert first["redirect"] == "n3-b"
|
|
|
|
|
assert second["route"] == "n3-b"
|
|
|
|
|
assert second["redirect"] == "n3-c"
|
|
|
|
|
for detail in (first, second):
|
|
|
|
|
assert detail["requested"] == "n3-a"
|
|
|
|
|
assert "n3-a" in detail["chain"]
|
|
|
|
|
assert "n3-b" in detail["chain"]
|
|
|
|
|
assert "n3-c" in detail["chain"]
|
|
|
|
|
assert detail["outcome_ok"] is True
|
|
|
|
|
assert detail["outcome"] == "fallback 'n3-c' answered HTTP 200 ok"
|
|
|
|
|
assert detail["served_route"] == "n3-c"
|
|
|
|
|
messages = _rendered_messages_openai_gateway(monkeypatch, [first, second])
|
|
|
|
|
assert "Redirect: will redirect to fallback 'n3-b'" in messages[0]
|
|
|
|
|
assert "Redirect: will redirect to fallback 'n3-c'" in messages[1]
|
|
|
|
|
assert "Outcome: fallback 'n3-c' answered HTTP 200 ok" in messages[0]
|
|
|
|
|
assert "Outcome: fallback 'n3-c' answered HTTP 200 ok" in messages[1]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_trip_notification_reports_failed_outcome_when_every_fallback_fails(
|
|
|
|
|
local_db, monkeypatch
|
|
|
|
|
):
|
|
|
|
|
routes = [
|
|
|
|
|
{"source_model": "n4-b", "target_model": "n4-b-target"},
|
|
|
|
|
{
|
|
|
|
|
"source_model": "n4-a",
|
|
|
|
|
"target_model": "n4-a-target",
|
|
|
|
|
"fallback_model": "n4-b",
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
response, sent = _notify_chat_openai_gateway(
|
|
|
|
|
monkeypatch, FakeAlwaysFailClient_openai_gateway, routes, "n4-a", "notify_fail"
|
|
|
|
|
)
|
|
|
|
|
assert response.status_code == 500
|
|
|
|
|
assert len(sent) == 2
|
|
|
|
|
first = sent[0][1][1]
|
|
|
|
|
second = sent[1][1][1]
|
|
|
|
|
assert first["route"] == "n4-a"
|
|
|
|
|
assert first["redirect"] == "n4-b"
|
|
|
|
|
assert second["route"] == "n4-b"
|
|
|
|
|
assert second["redirect"] == ""
|
|
|
|
|
for detail in (first, second):
|
|
|
|
|
assert detail["outcome_ok"] is False
|
|
|
|
|
assert "all 2 route(s) failed" in detail["outcome"]
|
|
|
|
|
assert "HTTP 500" in detail["outcome"]
|
|
|
|
|
messages = _rendered_messages_openai_gateway(monkeypatch, [first, second])
|
|
|
|
|
assert "Redirect: will redirect to fallback 'n4-b'" in messages[0]
|
|
|
|
|
assert "Redirect: none, end of the fallback line" in messages[1]
|
|
|
|
|
assert "Outcome: all 2 route(s) failed" in messages[1]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_trip_notification_redirects_unrouted_default_to_first_fallback(
|
|
|
|
|
local_db, monkeypatch
|
|
|
|
|
):
|
|
|
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
|
|
|
sent = []
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a))
|
|
|
|
|
)
|
|
|
|
|
detail = gwmod._trip_detail(
|
|
|
|
|
"default-target",
|
|
|
|
|
"https://default.example/v1/chat/completions",
|
|
|
|
|
30,
|
|
|
|
|
"upstream_error",
|
|
|
|
|
500,
|
|
|
|
|
"boom",
|
|
|
|
|
0,
|
|
|
|
|
route="",
|
|
|
|
|
)
|
|
|
|
|
rt._notify_trips(
|
|
|
|
|
[{"kind": "target", "detail": detail}],
|
|
|
|
|
"chat",
|
|
|
|
|
"molodetz",
|
|
|
|
|
"weird-model",
|
|
|
|
|
[{"route": ""}, {"route": "m-pro"}],
|
|
|
|
|
True,
|
|
|
|
|
"fallback 'm-pro' answered HTTP 200 ok",
|
|
|
|
|
"m-pro",
|
|
|
|
|
)
|
|
|
|
|
assert len(sent) == 1
|
|
|
|
|
enriched = sent[0][1][1]
|
|
|
|
|
assert enriched["redirect"] == "m-pro"
|
|
|
|
|
messages = _rendered_messages_openai_gateway(monkeypatch, [enriched])
|
|
|
|
|
assert "Local config: default (no routed model)" in messages[0]
|
|
|
|
|
assert "Redirect: will redirect to fallback 'm-pro'" in messages[0]
|
|
|
|
|
assert "Outcome: fallback 'm-pro' answered HTTP 200 ok" in messages[0]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_vision_trip_notifies_with_local_config_and_no_fallback(local_db, monkeypatch):
|
|
|
|
|
import httpx
|
|
|
|
|
|
|
|
|
|
from devplacepy.services.openai_gateway.reliability import CircuitBreaker
|
|
|
|
|
from devplacepy.services.openai_gateway.vision import VisionAugmenter, VisionCache
|
|
|
|
|
|
|
|
|
|
class FailVisionClient_openai_gateway:
|
|
|
|
|
async def post(self, url, headers=None, json=None):
|
|
|
|
|
raise httpx.ConnectError("vision down")
|
|
|
|
|
|
|
|
|
|
breaker = CircuitBreaker(1, 60)
|
|
|
|
|
sink = []
|
|
|
|
|
augmenter = VisionAugmenter(
|
|
|
|
|
"https://vision.example/v1/chat/completions",
|
|
|
|
|
"vis-model",
|
|
|
|
|
"k",
|
|
|
|
|
VisionCache(0),
|
|
|
|
|
breaker=breaker,
|
|
|
|
|
max_retries=0,
|
|
|
|
|
trip_sink=sink,
|
|
|
|
|
)
|
|
|
|
|
image = {"type": "image_url", "image_url": {"url": "http://x/y.png"}}
|
|
|
|
|
first = run_async(augmenter._describe_one(FailVisionClient_openai_gateway(), image))
|
|
|
|
|
assert first.startswith("[vision call failed")
|
|
|
|
|
assert breaker.is_open is True
|
|
|
|
|
assert len(sink) == 1
|
|
|
|
|
assert sink[0]["kind"] == "vision"
|
|
|
|
|
assert sink[0]["detail"]["route"] == "vision:vis-model"
|
|
|
|
|
second = run_async(augmenter._describe_one(FailVisionClient_openai_gateway(), image))
|
|
|
|
|
assert second.startswith("[vision unavailable")
|
|
|
|
|
assert len(sink) == 1
|
|
|
|
|
rt, cfg = _target_runtime_openai_gateway()
|
|
|
|
|
sent = []
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a))
|
|
|
|
|
)
|
|
|
|
|
rt._notify_trips(
|
|
|
|
|
sink,
|
|
|
|
|
"chat",
|
|
|
|
|
"molodetz",
|
|
|
|
|
"molodetz",
|
|
|
|
|
[{"route": "molodetz"}],
|
|
|
|
|
True,
|
|
|
|
|
"fallback 'molodetz' answered HTTP 200 ok",
|
|
|
|
|
"molodetz",
|
|
|
|
|
)
|
|
|
|
|
assert len(sent) == 1
|
|
|
|
|
messages = _rendered_messages_openai_gateway(monkeypatch, [sent[0][1][1]])
|
|
|
|
|
assert "Local config: 'vision:vis-model'" in messages[0]
|
|
|
|
|
assert "Redirect: none, vision has no fallback model" in messages[0]
|
|
|
|
|
assert "Outcome: fallback 'molodetz' answered HTTP 200 ok" in messages[0]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_trip_notification_outcome_scrubs_secrets(local_db, monkeypatch):
|
|
|
|
|
detail = gwmod._trip_detail(
|
|
|
|
|
"s1-target",
|
|
|
|
|
"https://s1.example/v1/chat/completions",
|
|
|
|
|
30,
|
|
|
|
|
"connection",
|
|
|
|
|
"no response",
|
|
|
|
|
"boom",
|
|
|
|
|
0,
|
|
|
|
|
route="s1",
|
|
|
|
|
)
|
|
|
|
|
detail["requested"] = "s1"
|
|
|
|
|
detail["chain"] = "s1 (s1-target @ default)"
|
|
|
|
|
detail["redirect"] = ""
|
|
|
|
|
secret = "sk" + "-abc123"
|
|
|
|
|
detail["outcome"] = "connection failed (Bearer " + secret + ")"
|
|
|
|
|
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
|
|
|
|
|
assert secret not in messages[0]
|
|
|
|
|
assert "[redacted]" in messages[0]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_passthrough_global_trip_carries_local_config(local_db, monkeypatch):
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway
|
|
|
|
|
)
|
|
|
|
|
svc = GatewayService()
|
|
|
|
|
cfg = svc.effective_config()
|
|
|
|
|
cfg["gateway_circuit_threshold"] = 1
|
|
|
|
|
cfg["gateway_max_retries"] = 0
|
|
|
|
|
rt = svc.runtime()
|
|
|
|
|
sent = []
|
|
|
|
|
monkeypatch.setattr(
|
|
|
|
|
gwmod.background,
|
|
|
|
|
"submit",
|
|
|
|
|
lambda fn, *a, **k: sent.append((fn, a))
|
|
|
|
|
if fn is gwmod._notify_gateway_status
|
|
|
|
|
else None,
|
|
|
|
|
)
|
|
|
|
|
resp = run_async(
|
|
|
|
|
rt.handle_passthrough(
|
|
|
|
|
"POST",
|
|
|
|
|
"responses",
|
|
|
|
|
"application/json",
|
|
|
|
|
b"{}",
|
|
|
|
|
cfg,
|
|
|
|
|
("guest", "ptrip_local"),
|
|
|
|
|
"test",
|
|
|
|
|
"default",
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
assert resp.status_code == 500
|
|
|
|
|
assert len(sent) == 1
|
|
|
|
|
detail = sent[0][1][1]
|
|
|
|
|
assert detail.get("global") is True
|
|
|
|
|
assert detail["route"] == cfg["gateway_model"]
|
|
|
|
|
assert "no fallback line" in detail["chain"]
|
|
|
|
|
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
|
|
|
|
|
assert f"Local config: '{cfg['gateway_model']}'" in messages[0]
|
|
|
|
|
assert "passthrough POST responses (no fallback line)" in messages[0]
|
|
|
|
|
assert "Outcome: all 1 route(s) failed" in messages[0]
|
|
|
|
|