Gateway: per-target circuit breaker trips on first AI error and notifies all admins
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3678,3 +3678,101 @@ def test_fallback_attempt_is_shaped_with_its_own_learned_method(local_db, monkey
|
||||
learned_store.forget("default", "shape-backup-target")
|
||||
routing.model_store.remove("shape-primary")
|
||||
routing.model_store.remove("shape-backup")
|
||||
|
||||
|
||||
def _target_send_openai_gateway(rt, cfg, client, url, model):
|
||||
import asyncio
|
||||
|
||||
async def go():
|
||||
return await rt._send(
|
||||
client,
|
||||
asyncio.Semaphore(4),
|
||||
"POST",
|
||||
url,
|
||||
{},
|
||||
{**cfg, "gateway_max_retries": 0},
|
||||
lambda message: None,
|
||||
json_body={"model": model},
|
||||
)
|
||||
|
||||
return run_async(go())
|
||||
|
||||
|
||||
def _target_runtime_openai_gateway():
|
||||
svc = GatewayService()
|
||||
return svc.runtime(), svc.effective_config()
|
||||
|
||||
|
||||
def test_trip_opens_below_threshold_and_ignores_threshold():
|
||||
from devplacepy.services.openai_gateway.reliability import CircuitBreaker
|
||||
|
||||
breaker = CircuitBreaker(5, 30)
|
||||
breaker.trip()
|
||||
assert breaker.is_open is True
|
||||
assert breaker.allow() is False
|
||||
|
||||
|
||||
def test_a_failed_target_is_skipped_without_an_upstream_call(local_db):
|
||||
rt, cfg = _target_runtime_openai_gateway()
|
||||
client = FakeAlwaysFailClient_openai_gateway()
|
||||
url = "https://target-a.example/v1/chat/completions"
|
||||
first = _target_send_openai_gateway(rt, cfg, client, url, "m1")
|
||||
assert first[0].status_code == 500
|
||||
second = _target_send_openai_gateway(rt, cfg, client, url, "m1")
|
||||
assert second[2]["circuit_open"] is True
|
||||
assert len(client.calls) == 1
|
||||
|
||||
|
||||
def test_one_bad_request_does_not_block_another_target(local_db):
|
||||
rt, cfg = _target_runtime_openai_gateway()
|
||||
client = FakeAlwaysFailClient_openai_gateway()
|
||||
_target_send_openai_gateway(rt, cfg, client, "https://target-b.example/v1/chat/completions", "m1")
|
||||
other = _target_send_openai_gateway(rt, cfg, client, "https://target-c.example/v1/chat/completions", "m1")
|
||||
assert other[2]["circuit_open"] is False
|
||||
assert len(client.calls) == 2
|
||||
|
||||
|
||||
def test_target_breaker_kill_switch_keeps_global_behaviour(local_db):
|
||||
rt, cfg = _target_runtime_openai_gateway()
|
||||
cfg = {**cfg, "gateway_target_circuit_enabled": False}
|
||||
client = FakeAlwaysFailClient_openai_gateway()
|
||||
url = "https://target-d.example/v1/chat/completions"
|
||||
_target_send_openai_gateway(rt, cfg, client, url, "m1")
|
||||
again = _target_send_openai_gateway(rt, cfg, client, url, "m1")
|
||||
assert again[2]["circuit_open"] is False
|
||||
assert len(client.calls) == 2
|
||||
|
||||
|
||||
def test_shape_fixable_400_does_not_trip_the_target(local_db):
|
||||
rt, cfg = _target_runtime_openai_gateway()
|
||||
resp = FakeResp_openai_gateway(
|
||||
status=400,
|
||||
payload={"issues": [{"code": "unrecognized_keys", "keys": ["thinking"]}]},
|
||||
)
|
||||
assert gwmod._trip_worthy(resp) is False
|
||||
plain = FakeResp_openai_gateway(status=400, payload={"error": "bad schema"})
|
||||
assert gwmod._trip_worthy(plain) is True
|
||||
|
||||
|
||||
def test_embedded_200_error_trips_the_target(local_db):
|
||||
rt, cfg = _target_runtime_openai_gateway()
|
||||
resp = FakeResp_openai_gateway(status=200, payload={"error": {"message": "provider exploded"}})
|
||||
url = "https://target-e.example/v1/chat/completions"
|
||||
assert rt._embedded_error(resp, False, url, "m1", cfg) is not None
|
||||
assert rt._target_breaker(gwmod._target_key(url, {"model": "m1"}), cfg).is_open is True
|
||||
assert rt.open_targets()[0]["provider"] == "target-e.example"
|
||||
|
||||
|
||||
def test_transition_notifies_each_admin_once_with_detail_and_no_key(local_db, monkeypatch):
|
||||
sent = []
|
||||
monkeypatch.setattr(gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a)))
|
||||
rt, cfg = _target_runtime_openai_gateway()
|
||||
target = ("target-f.example", "m1")
|
||||
rt._trip_target(target, cfg, "upstream_error", 500, "Bearer sk-secret123 failed", 1)
|
||||
rt._trip_target(target, cfg, "upstream_error", 500, "again", 1)
|
||||
assert len(sent) == 1
|
||||
detail = sent[0][1][1]
|
||||
assert detail["category"] == "upstream_error"
|
||||
assert detail["status"] == 500
|
||||
assert "secret123" not in detail["upstream"]
|
||||
assert detail["provider"] == "target-f.example"
|
||||
|
||||
Reference in New Issue
Block a user