Gateway: per-target circuit breaker trips on first AI error and notifies all admins

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-14 01:07:47 +01:00
co-authored by Claude Sonnet 5.5
parent 151476125f
commit 8e36807926
6 changed files with 241 additions and 16 deletions
@@ -3678,3 +3678,101 @@ def test_fallback_attempt_is_shaped_with_its_own_learned_method(local_db, monkey
learned_store.forget("default", "shape-backup-target")
routing.model_store.remove("shape-primary")
routing.model_store.remove("shape-backup")
def _target_send_openai_gateway(rt, cfg, client, url, model):
import asyncio
async def go():
return await rt._send(
client,
asyncio.Semaphore(4),
"POST",
url,
{},
{**cfg, "gateway_max_retries": 0},
lambda message: None,
json_body={"model": model},
)
return run_async(go())
def _target_runtime_openai_gateway():
svc = GatewayService()
return svc.runtime(), svc.effective_config()
def test_trip_opens_below_threshold_and_ignores_threshold():
from devplacepy.services.openai_gateway.reliability import CircuitBreaker
breaker = CircuitBreaker(5, 30)
breaker.trip()
assert breaker.is_open is True
assert breaker.allow() is False
def test_a_failed_target_is_skipped_without_an_upstream_call(local_db):
rt, cfg = _target_runtime_openai_gateway()
client = FakeAlwaysFailClient_openai_gateway()
url = "https://target-a.example/v1/chat/completions"
first = _target_send_openai_gateway(rt, cfg, client, url, "m1")
assert first[0].status_code == 500
second = _target_send_openai_gateway(rt, cfg, client, url, "m1")
assert second[2]["circuit_open"] is True
assert len(client.calls) == 1
def test_one_bad_request_does_not_block_another_target(local_db):
rt, cfg = _target_runtime_openai_gateway()
client = FakeAlwaysFailClient_openai_gateway()
_target_send_openai_gateway(rt, cfg, client, "https://target-b.example/v1/chat/completions", "m1")
other = _target_send_openai_gateway(rt, cfg, client, "https://target-c.example/v1/chat/completions", "m1")
assert other[2]["circuit_open"] is False
assert len(client.calls) == 2
def test_target_breaker_kill_switch_keeps_global_behaviour(local_db):
rt, cfg = _target_runtime_openai_gateway()
cfg = {**cfg, "gateway_target_circuit_enabled": False}
client = FakeAlwaysFailClient_openai_gateway()
url = "https://target-d.example/v1/chat/completions"
_target_send_openai_gateway(rt, cfg, client, url, "m1")
again = _target_send_openai_gateway(rt, cfg, client, url, "m1")
assert again[2]["circuit_open"] is False
assert len(client.calls) == 2
def test_shape_fixable_400_does_not_trip_the_target(local_db):
rt, cfg = _target_runtime_openai_gateway()
resp = FakeResp_openai_gateway(
status=400,
payload={"issues": [{"code": "unrecognized_keys", "keys": ["thinking"]}]},
)
assert gwmod._trip_worthy(resp) is False
plain = FakeResp_openai_gateway(status=400, payload={"error": "bad schema"})
assert gwmod._trip_worthy(plain) is True
def test_embedded_200_error_trips_the_target(local_db):
rt, cfg = _target_runtime_openai_gateway()
resp = FakeResp_openai_gateway(status=200, payload={"error": {"message": "provider exploded"}})
url = "https://target-e.example/v1/chat/completions"
assert rt._embedded_error(resp, False, url, "m1", cfg) is not None
assert rt._target_breaker(gwmod._target_key(url, {"model": "m1"}), cfg).is_open is True
assert rt.open_targets()[0]["provider"] == "target-e.example"
def test_transition_notifies_each_admin_once_with_detail_and_no_key(local_db, monkeypatch):
sent = []
monkeypatch.setattr(gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a)))
rt, cfg = _target_runtime_openai_gateway()
target = ("target-f.example", "m1")
rt._trip_target(target, cfg, "upstream_error", 500, "Bearer sk-secret123 failed", 1)
rt._trip_target(target, cfg, "upstream_error", 500, "again", 1)
assert len(sent) == 1
detail = sent[0][1][1]
assert detail["category"] == "upstream_error"
assert detail["status"] == 500
assert "secret123" not in detail["upstream"]
assert detail["provider"] == "target-f.example"