From 8e368079262cf8eb96aa8981c8ba00ab1c1fc066 Mon Sep 17 00:00:00 2001 From: retoor Date: Sat, 14 Nov 2026 01:07:47 +0100 Subject: [PATCH] Gateway: per-target circuit breaker trips on first AI error and notifies all admins Co-Authored-By: Claude Sonnet 5.5 --- devplacepy/services/openai_gateway/CLAUDE.md | 2 +- devplacepy/services/openai_gateway/gateway.py | 142 ++++++++++++++++-- .../services/openai_gateway/reliability.py | 4 + devplacepy/services/openai_gateway/service.py | 9 ++ pyproject.toml | 2 +- tests/unit/services/openai_gateway/gateway.py | 98 ++++++++++++ 6 files changed, 241 insertions(+), 16 deletions(-) diff --git a/devplacepy/services/openai_gateway/CLAUDE.md b/devplacepy/services/openai_gateway/CLAUDE.md index 04e7af24..efa32509 100644 --- a/devplacepy/services/openai_gateway/CLAUDE.md +++ b/devplacepy/services/openai_gateway/CLAUDE.md @@ -253,7 +253,7 @@ Any `gateway_models` route (chat, embed, or image) can optionally name `fallback - **Rebuilt like a fresh routed call, not retried in place.** On failure, `handle_chat`/`handle_embeddings`/`handle_images` recompute the overlay from `base_cfg` (the pre-primary-overlay config) via `chat_overlay(fallback_route.source_model, base_cfg)`/`embed_overlay`/`image_overlay`, so the fallback gets its OWN provider, URL, key, and pricing - never the primary route's. `pricing`/`context_map` for the ledger are (re)computed AFTER the fallback decision so the recorded cost always reflects whichever model actually served the request. - **One ledger row per client call, not one per attempt.** The primary failure never writes a `gateway_usage_ledger` row by itself (only `_send`'s own internal retries and the circuit breaker counters observe it); `finalize()`/`self._ledger.record(...)` still runs exactly once, after the fallback decision, with `model` = whichever model ultimately answered and `requested_model` = the ORIGINALLY requested model name (an existing-but-previously-unpopulated ledger column, now populated by all three handlers) - so a fallback shows up in `/admin/ai-usage` as "requested X, served Y" rather than as two calls. - **Streaming falls back before any byte reaches the client.** `client.send(request, stream=True)` returns as soon as the response headers arrive, so `resp.status_code` is already known before the SSE body is ever touched - `handle_chat` decides whether to fall back at that same point, before it ever opens `_stream_chat_response` to the caller. A stream is therefore never abandoned mid-flight in favor of a fallback; the caller either gets the fallback's own stream from byte one, or the fallback's own definitive failure. -- **Known limitation, inherited from the existing provider routing design, not introduced by this feature: the circuit breaker (`self._breaker`) is one instance per `GatewayRuntime`, shared across every route.** A primary attempt that trips the breaker can make the immediately-following fallback attempt itself see `circuit_open` too. This mirrors the pre-existing behavior for the multi-provider routing overlay in general (a struggling route's failures already could open the breaker in front of an unrelated route before fallback existed) and was left as-is rather than redesigning per-provider breakers, which is a materially larger change than "add a fallback model." +- **Per-target breakers (`GatewayRuntime._target_breakers`).** Besides the global `self._breaker`, `_send` keeps one `CircuitBreaker(1, gateway_circuit_cooldown_seconds)` per `(upstream host, request model)` key (`_target_key`). One upstream error opens it at once (`CircuitBreaker.trip()`): exception, 5xx, 429, and any 4xx except a 400/422 whose body names rejected keys (the shape ladder still has to retry those; `_trip_worthy`), plus a 200 with an embedded error body (`_embedded_error`, called by the chat, embed and image handlers). An open target returns `timing["circuit_open"]` with no upstream call, so `_should_fallback` walks the fallback chain. The opening transition logs `ai.gateway.circuit.target_opened` and sends ONE detailed `system` notification to every admin (`_notify_gateway_status(True, detail)`: provider, model, error class, status, scrubbed upstream text capped at 300 chars, retries, cooldown). Kill switch `gateway_target_circuit_enabled` (Reliability, default on). Open targets appear in `metrics()["targets_open"]`. Not persisted across restarts, per worker, no single-probe half-open guard (bounded by the semaphore). Vision uses its own `_vision_breaker` outside `_send`. - **A 400 is terminal UNLESS its body names rejected keys** the auto-resolve ladder can fix (see "Auto-resolve of request shape"): that case is retried on the SAME route with another shape before any fallback is considered. - **Deliberately NOT covered:** `gateway_max_retries`/`gateway_retry_backoff_ms`, the circuit breaker, and hard docker-adjacent settings are untouched - fallback is additive on top of them, not a replacement retry mechanism. diff --git a/devplacepy/services/openai_gateway/gateway.py b/devplacepy/services/openai_gateway/gateway.py index c10e4d04..ffa432ad 100644 --- a/devplacepy/services/openai_gateway/gateway.py +++ b/devplacepy/services/openai_gateway/gateway.py @@ -3,9 +3,11 @@ import asyncio import json import logging +import re import time from dataclasses import dataclass from typing import Any, Optional +from urllib.parse import urlparse import httpx from fastapi.responses import JSONResponse, Response, StreamingResponse @@ -43,6 +45,7 @@ from devplacepy.services.openai_gateway.tool_emulation import ( from devplacepy.services.openai_gateway.learned import learned_store from devplacepy.services.openai_gateway.shape import ( NATIVE_METHOD, + RejectedKeys, ShapeMethod, ShapeRetrier, ) @@ -65,6 +68,8 @@ from devplacepy.services.openai_gateway.vision import VisionAugmenter, VisionCac logger = logging.getLogger(__name__) +MAX_OPEN_TARGETS_LISTED = 20 + async def _close_quietly(resp) -> None: if resp is None: @@ -87,7 +92,7 @@ def _call_failed(resp, exc, timing: dict) -> bool: # context_length is deliberately NOT in this set: a fallback model commonly # has a different (often larger) context window than the primary, so it can # genuinely recover a request the primary couldn't fit - see -# _fits_context_window, which additionally skips a doomed primary attempt +# _route_context_window, which additionally skips a doomed primary attempt # outright when its own declared window is already known to be too small. # model_not_found is excluded for the same reason: an upstream 400/404 saying # the model is unavailable names the MODEL, not the request shape, so trying @@ -337,15 +342,63 @@ def _route_context_window(cfg: dict, model: str) -> Optional[int]: return int(window) if window else None -def _notify_gateway_status(is_open: bool) -> None: +UPSTREAM_DETAIL_MAX_CHARS = 300 +SECRET_PATTERN = re.compile(r"(Bearer\s+\S+|sk-[A-Za-z0-9_-]+)") + + +def _scrub_upstream(text: str) -> str: + return SECRET_PATTERN.sub("[redacted]", text or "")[:UPSTREAM_DETAIL_MAX_CHARS] + + +def _target_key(url: str, json_body: Optional[dict]) -> tuple[str, str]: + model = json_body.get("model") if isinstance(json_body, dict) else "" + return (urlparse(url or "").netloc, str(model or "")) + + +def _trip_detail(target, cooldown, category, status, upstream, retries) -> dict: + return { + "provider": target[0], + "model": target[1] or "-", + "cooldown": cooldown, + "category": category, + "status": status, + "upstream": _scrub_upstream(upstream), + "retries": retries, + } + + +def _resp_text(resp) -> str: + try: + return resp.text or "" + except Exception: # noqa: BLE001 - an unread streamed body has no text yet + return "" + + +def _trip_worthy(resp) -> bool: + if resp.status_code < 400: + return False + if resp.status_code in (400, 422): + return not RejectedKeys.parse(_resp_text(resp)) + return True + + +def _notify_gateway_status(is_open: bool, detail: Optional[dict] = None) -> None: from devplacepy.database import get_table from devplacepy.utils import create_notification - message = ( - "AI gateway circuit breaker opened - upstream calls are being rejected" - if is_open - else "AI gateway circuit breaker closed - upstream calls have resumed" - ) + if detail: + message = ( + f"AI gateway target {detail['provider']}/{detail['model']} tripped its circuit " + f"breaker after 1 error - calls are rejected for {detail['cooldown']}s. " + f"Error class: {detail['category']}. HTTP status: {detail['status']}. " + f"Upstream said: {detail['upstream']}. Retries used: {detail['retries']}." + ) + else: + message = ( + "AI gateway circuit breaker opened - upstream calls are being rejected" + if is_open + else "AI gateway circuit breaker closed - upstream calls have resumed" + ) for admin in get_table("users").find(role="Admin"): create_notification(admin["uid"], "system", message, "gateway", "/admin/services/openai") @@ -400,6 +453,7 @@ class GatewayRuntime: self._vision_breaker = CircuitBreaker( config.CIRCUIT_THRESHOLD_DEFAULT, config.CIRCUIT_COOLDOWN_SECONDS_DEFAULT ) + self._target_breakers: dict[tuple[str, str], CircuitBreaker] = {} self._catalog_cache = CatalogCache() self.requests = 0 self.errors = 0 @@ -452,6 +506,48 @@ class GatewayRuntime: ) return self._client, self._sem + def _target_breaker(self, target, cfg) -> CircuitBreaker: + breaker = self._target_breakers.get(target) + if breaker is None: + breaker = CircuitBreaker(1, cfg["gateway_circuit_cooldown_seconds"]) + self._target_breakers[target] = breaker + else: + breaker.configure(1, cfg["gateway_circuit_cooldown_seconds"]) + return breaker + + def _trip_target(self, target, cfg, category, status, upstream, retries) -> None: + breaker = self._target_breaker(target, cfg) + was_open = breaker.is_open + breaker.trip() + if was_open: + return + detail = _trip_detail( + target, cfg["gateway_circuit_cooldown_seconds"], category, status, upstream, retries + ) + logger.warning( + "ai.gateway.circuit.target_opened provider=%s model=%s category=%s status=%s retries=%s", + detail["provider"], detail["model"], category, status, retries, + ) + background.submit(_notify_gateway_status, True, detail) + + def _embedded_error(self, resp, stream, url, model, cfg, protocol="chat_completions"): + message = _embedded_failure_message(resp, stream, protocol) + if message and cfg.get("gateway_target_circuit_enabled", True): + self._trip_target(_target_key(url, {"model": model}), cfg, "embedded_error", 200, message, 0) + return message + + def open_targets(self) -> list[dict]: + now = time.monotonic() + return [ + { + "provider": key[0], + "model": key[1], + "seconds_left": max(0, round(b.cooldown_seconds - (now - b.opened_at))), + } + for key, b in self._target_breakers.items() + if b.is_open + ][:MAX_OPEN_TARGETS_LISTED] + async def aclose(self) -> None: if self._client is not None: await self._client.aclose() @@ -479,7 +575,11 @@ class GatewayRuntime: "retry_succeeded": False, "circuit_open": False, } - if not self._breaker.allow(): + target = _target_key(url, json_body) + target_enabled = cfg.get("gateway_target_circuit_enabled", True) + if not self._breaker.allow() or ( + target_enabled and not self._target_breaker(target, cfg).allow() + ): timing["circuit_open"] = True log("circuit breaker open, rejecting upstream call") return None, None, timing @@ -523,6 +623,11 @@ class GatewayRuntime: self._breaker.record_failure() if not was_open and self._breaker.is_open: background.submit(_notify_gateway_status, True) + if target_enabled: + self._trip_target( + target, cfg, classify_error(0, exc), "no response", str(exc), + timing["retries_attempted"], + ) log(f"{method} {url} connection failed after {attempts} attempt(s): {exc}") return None, exc, timing self.last_status = resp.status_code @@ -543,6 +648,14 @@ class GatewayRuntime: if was_open and not self._breaker.is_open: background.submit(_notify_gateway_status, False) timing["retry_succeeded"] = attempts > 1 + if target_enabled: + if _trip_worthy(resp): + self._trip_target( + target, cfg, classify_error(resp.status_code, None, _resp_text(resp)), + resp.status_code, _resp_text(resp), timing["retries_attempted"], + ) + elif resp.status_code < 400: + self._target_breaker(target, cfg).record_success() return resp, None, timing async def _shaped_send( @@ -809,7 +922,7 @@ class GatewayRuntime: send_start = attempt.send_start requested_model = requested - body_error = _embedded_failure_message(resp, stream, protocol) + body_error = self._embedded_error(resp, stream, cfg["gateway_upstream_url"], model, cfg, protocol) for fallback_route in resolve_fallback_chain(chain_start, "chat"): if fallback_route.source_model in attempted_routes: continue @@ -842,7 +955,7 @@ class GatewayRuntime: resp, exc, timing = attempt.resp, attempt.exc, attempt.timing payload = attempt.shaped send_start = attempt.send_start - body_error = _embedded_failure_message(resp, stream, protocol) + body_error = self._embedded_error(resp, stream, cfg["gateway_upstream_url"], model, cfg, protocol) pricing = pricing_from_cfg(cfg) context_map = parse_context_map(cfg.get("gateway_model_context_map")) @@ -1417,7 +1530,7 @@ class GatewayRuntime: attempted_routes = ( {entry_route.source_model} if entry_route is not None else set() ) - body_error = _embedded_failure_message(resp, False) + body_error = self._embedded_error(resp, False, cfg["gateway_embed_url"], model, cfg) for fallback_route in resolve_fallback_chain(chain_start, "embed"): if fallback_route.source_model in attempted_routes: continue @@ -1449,7 +1562,7 @@ class GatewayRuntime: log, json_body=payload, ) - body_error = _embedded_failure_message(resp, False) + body_error = self._embedded_error(resp, False, cfg["gateway_embed_url"], model, cfg) pricing = pricing_from_cfg(cfg) context_map = parse_context_map(cfg.get("gateway_model_context_map")) @@ -1674,7 +1787,7 @@ class GatewayRuntime: attempted_routes = ( {entry_route.source_model} if entry_route is not None else set() ) - body_error = _embedded_failure_message(resp, False) + body_error = self._embedded_error(resp, False, cfg["gateway_image_url"], model, cfg) for fallback_route in resolve_fallback_chain(chain_start, "image"): if fallback_route.source_model in attempted_routes: continue @@ -1706,7 +1819,7 @@ class GatewayRuntime: log, json_body=payload, ) - body_error = _embedded_failure_message(resp, False) + body_error = self._embedded_error(resp, False, cfg["gateway_image_url"], model, cfg) pricing = pricing_from_cfg(cfg) context_map = parse_context_map(cfg.get("gateway_model_context_map")) @@ -1953,4 +2066,5 @@ class GatewayRuntime: "pool": self._instances, "circuit_open": self._breaker.is_open, "vision_circuit_open": self._vision_breaker.is_open, + "targets_open": self.open_targets(), } diff --git a/devplacepy/services/openai_gateway/reliability.py b/devplacepy/services/openai_gateway/reliability.py index 1548a658..4ed2b52e 100644 --- a/devplacepy/services/openai_gateway/reliability.py +++ b/devplacepy/services/openai_gateway/reliability.py @@ -58,6 +58,10 @@ class CircuitBreaker: if self.threshold > 0 and self.failures >= self.threshold: self.opened_at = time.monotonic() + def trip(self) -> None: + self.failures = max(self.threshold, 1) + self.opened_at = time.monotonic() + MAX_RETRY_AFTER_SECONDS = 30.0 diff --git a/devplacepy/services/openai_gateway/service.py b/devplacepy/services/openai_gateway/service.py index 8ccf9b66..84498af9 100644 --- a/devplacepy/services/openai_gateway/service.py +++ b/devplacepy/services/openai_gateway/service.py @@ -524,6 +524,15 @@ class GatewayService(BaseService): minimum=1, group="Reliability", ), + ConfigField( + "gateway_target_circuit_enabled", + "Per-target circuit breaker", + type="bool", + default=True, + help="Open a breaker for one provider/model after a single upstream error " + "and notify every admin. Off leaves only the global breaker.", + group="Reliability", + ), ConfigField( "gateway_usage_retention_hours", "Usage retention (hours)", diff --git a/pyproject.toml b/pyproject.toml index 22af6ce6..3b8be00f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "devplacepy" -version = "1.1.23" +version = "1.1.24" description = "DevPlace - The Developer Social Network" requires-python = ">=3.12" dependencies = [ diff --git a/tests/unit/services/openai_gateway/gateway.py b/tests/unit/services/openai_gateway/gateway.py index 35f6f95d..e3f06643 100644 --- a/tests/unit/services/openai_gateway/gateway.py +++ b/tests/unit/services/openai_gateway/gateway.py @@ -3678,3 +3678,101 @@ def test_fallback_attempt_is_shaped_with_its_own_learned_method(local_db, monkey learned_store.forget("default", "shape-backup-target") routing.model_store.remove("shape-primary") routing.model_store.remove("shape-backup") + + +def _target_send_openai_gateway(rt, cfg, client, url, model): + import asyncio + + async def go(): + return await rt._send( + client, + asyncio.Semaphore(4), + "POST", + url, + {}, + {**cfg, "gateway_max_retries": 0}, + lambda message: None, + json_body={"model": model}, + ) + + return run_async(go()) + + +def _target_runtime_openai_gateway(): + svc = GatewayService() + return svc.runtime(), svc.effective_config() + + +def test_trip_opens_below_threshold_and_ignores_threshold(): + from devplacepy.services.openai_gateway.reliability import CircuitBreaker + + breaker = CircuitBreaker(5, 30) + breaker.trip() + assert breaker.is_open is True + assert breaker.allow() is False + + +def test_a_failed_target_is_skipped_without_an_upstream_call(local_db): + rt, cfg = _target_runtime_openai_gateway() + client = FakeAlwaysFailClient_openai_gateway() + url = "https://target-a.example/v1/chat/completions" + first = _target_send_openai_gateway(rt, cfg, client, url, "m1") + assert first[0].status_code == 500 + second = _target_send_openai_gateway(rt, cfg, client, url, "m1") + assert second[2]["circuit_open"] is True + assert len(client.calls) == 1 + + +def test_one_bad_request_does_not_block_another_target(local_db): + rt, cfg = _target_runtime_openai_gateway() + client = FakeAlwaysFailClient_openai_gateway() + _target_send_openai_gateway(rt, cfg, client, "https://target-b.example/v1/chat/completions", "m1") + other = _target_send_openai_gateway(rt, cfg, client, "https://target-c.example/v1/chat/completions", "m1") + assert other[2]["circuit_open"] is False + assert len(client.calls) == 2 + + +def test_target_breaker_kill_switch_keeps_global_behaviour(local_db): + rt, cfg = _target_runtime_openai_gateway() + cfg = {**cfg, "gateway_target_circuit_enabled": False} + client = FakeAlwaysFailClient_openai_gateway() + url = "https://target-d.example/v1/chat/completions" + _target_send_openai_gateway(rt, cfg, client, url, "m1") + again = _target_send_openai_gateway(rt, cfg, client, url, "m1") + assert again[2]["circuit_open"] is False + assert len(client.calls) == 2 + + +def test_shape_fixable_400_does_not_trip_the_target(local_db): + rt, cfg = _target_runtime_openai_gateway() + resp = FakeResp_openai_gateway( + status=400, + payload={"issues": [{"code": "unrecognized_keys", "keys": ["thinking"]}]}, + ) + assert gwmod._trip_worthy(resp) is False + plain = FakeResp_openai_gateway(status=400, payload={"error": "bad schema"}) + assert gwmod._trip_worthy(plain) is True + + +def test_embedded_200_error_trips_the_target(local_db): + rt, cfg = _target_runtime_openai_gateway() + resp = FakeResp_openai_gateway(status=200, payload={"error": {"message": "provider exploded"}}) + url = "https://target-e.example/v1/chat/completions" + assert rt._embedded_error(resp, False, url, "m1", cfg) is not None + assert rt._target_breaker(gwmod._target_key(url, {"model": "m1"}), cfg).is_open is True + assert rt.open_targets()[0]["provider"] == "target-e.example" + + +def test_transition_notifies_each_admin_once_with_detail_and_no_key(local_db, monkeypatch): + sent = [] + monkeypatch.setattr(gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a))) + rt, cfg = _target_runtime_openai_gateway() + target = ("target-f.example", "m1") + rt._trip_target(target, cfg, "upstream_error", 500, "Bearer sk-secret123 failed", 1) + rt._trip_target(target, cfg, "upstream_error", 500, "again", 1) + assert len(sent) == 1 + detail = sent[0][1][1] + assert detail["category"] == "upstream_error" + assert detail["status"] == 500 + assert "secret123" not in detail["upstream"] + assert detail["provider"] == "target-f.example"