forked from retoor/devplacepy
Gateway: refine routing, vision and request handling, update docs and tests
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -50,6 +50,15 @@ class AdminGatewayOut(_Out):
|
||||
default_provider: dict = {}
|
||||
quota_defaults: dict = {}
|
||||
stats_ranges: list = []
|
||||
providers_total: int = 0
|
||||
models_total: int = 0
|
||||
quota_rules_total: int = 0
|
||||
stats: dict = {}
|
||||
stats_range: str = "24h"
|
||||
stats_generated_iso: Optional[str] = None
|
||||
stats_models: list = []
|
||||
stats_table_models: list = []
|
||||
stats_table_models_total: int = 0
|
||||
admin_section: Optional[str] = None
|
||||
tested: str = ""
|
||||
test_image: str = ""
|
||||
|
||||
@@ -253,7 +253,7 @@ Any `gateway_models` route (chat, embed, or image) can optionally name `fallback
|
||||
- **Rebuilt like a fresh routed call, not retried in place.** On failure, `handle_chat`/`handle_embeddings`/`handle_images` recompute the overlay from `base_cfg` (the pre-primary-overlay config) via `chat_overlay(fallback_route.source_model, base_cfg)`/`embed_overlay`/`image_overlay`, so the fallback gets its OWN provider, URL, key, and pricing - never the primary route's. `pricing`/`context_map` for the ledger are (re)computed AFTER the fallback decision so the recorded cost always reflects whichever model actually served the request.
|
||||
- **One ledger row per client call, not one per attempt.** The primary failure never writes a `gateway_usage_ledger` row by itself (only `_send`'s own internal retries and the circuit breaker counters observe it); `finalize()`/`self._ledger.record(...)` still runs exactly once, after the fallback decision, with `model` = whichever model ultimately answered and `requested_model` = the ORIGINALLY requested model name (an existing-but-previously-unpopulated ledger column, now populated by all three handlers) - so a fallback shows up in `/admin/ai-usage` as "requested X, served Y" rather than as two calls.
|
||||
- **Streaming falls back before any byte reaches the client.** `client.send(request, stream=True)` returns as soon as the response headers arrive, so `resp.status_code` is already known before the SSE body is ever touched - `handle_chat` decides whether to fall back at that same point, before it ever opens `_stream_chat_response` to the caller. A stream is therefore never abandoned mid-flight in favor of a fallback; the caller either gets the fallback's own stream from byte one, or the fallback's own definitive failure.
|
||||
- **Per-target breakers (`GatewayRuntime._target_breakers`).** Besides the global `self._breaker`, `_send` keeps one `CircuitBreaker(1, gateway_circuit_cooldown_seconds)` per `(upstream host, request model)` key (`_target_key`). One upstream error opens it at once (`CircuitBreaker.trip()`): exception, 5xx, 429, and any 4xx except a 400/422 whose body names rejected keys (the shape ladder still has to retry those; `_trip_worthy`), plus a 200 with an embedded error body (`_embedded_error`, called by the chat, embed and image handlers). An open target returns `timing["circuit_open"]` with no upstream call, so `_should_fallback` walks the fallback chain. The opening transition logs `ai.gateway.circuit.target_opened` and sends ONE detailed `system` notification to every admin (`_notify_gateway_status(True, detail)`: provider, model, error class, status, scrubbed upstream text capped at 300 chars, retries, cooldown). Kill switch `gateway_target_circuit_enabled` (Reliability, default on). Open targets appear in `metrics()["targets_open"]`. Not persisted across restarts, per worker, no single-probe half-open guard (bounded by the semaphore). Vision uses its own `_vision_breaker` outside `_send`.
|
||||
- **Per-model breakers (`GatewayRuntime._target_breakers`).** Besides the global `self._breaker`, `_send` keeps one `CircuitBreaker(1, gateway_circuit_cooldown_seconds)` per request MODEL name (`_target_key`; the model name alone, so the same model on two providers shares one breaker, and a call with no model, such as passthrough, never uses it). One upstream error opens it at once (`CircuitBreaker.trip()`): exception, 5xx, 429, and any 4xx except a 400/422 whose body names rejected keys (the shape ladder still has to retry those; `_trip_worthy`), plus a 200 with an embedded error body (`_embedded_error`, called by the chat, embed and image handlers). An open model returns `timing["circuit_open"]` with no upstream call, so `_should_fallback` walks the fallback chain. The opening transition logs `ai.gateway.circuit.target_opened` (now with the routed `route=` name) and is reported to every admin with ONE detailed `system` notification per tripped breaker (`_notify_gateway_status(True, detail)`). Chat, embed and image handlers defer every trip of their request into a per-request `trip_sink` (`route_name` threaded through `_chat_attempt`/`_shaped_send`/`_send`/`_embedded_error`) and, once the fallback line has resolved, emit one notification per tripped breaker via `_notify_trips`: each names the local config (`route`, the `source_model` that tripped, plus the requested model), the full recursive route (`_chain_records`/`_format_chain` over `resolve_fallback_chain`, every hop with its upstream target and provider), the redirect (`will redirect to fallback '<next>'`, or `none, end of the fallback line`), and the final outcome (`_request_outcome`: which fallback answered HTTP 200 ok, or that all N routes failed with the last error). The pre-existing fields are unchanged (model, upstream host, error class, status, scrubbed upstream text capped at 300 chars, retries, cooldown), so every breaker in a multi-layer line notifies exactly once, with the complete route and the eventual result. A request that trips nothing notifies nothing; an already-open breaker re-notifies nothing (transition-only, no per-rejection spam). Passthrough has no route or fallback line, so its global trips notify with the default model as the local config and an explicit no-fallback line. Kill switch `gateway_target_circuit_enabled` (Reliability, default on). Open models appear in `metrics()["targets_open"]`. Not persisted across restarts, per worker, no single-probe half-open guard (bounded by the semaphore). Vision uses its own `_vision_breaker` outside `_send`, but its opening transition joins the same per-request `trip_sink` (local config `vision:<model>`, no fallback line) and notifies with the request final outcome. Tests: the `test_trip_*`, `test_a_failed_target_*`, `test_one_bad_request_*`, `test_target_breaker_kill_switch_*`, `test_shape_fixable_400_*`, `test_embedded_200_error_*`, `test_breaker_is_per_model_across_upstreams`, `test_a_request_without_a_model_*`, `test_transition_notifies_*`, `test_trip_notification_*`, `test_trip_notification_redirects_unrouted_default_to_first_fallback`, `test_vision_trip_notifies_with_local_config_and_no_fallback`, `test_trip_notification_outcome_scrubs_secrets` and `test_passthrough_global_trip_carries_local_config` unit tests in `tests/unit/services/openai_gateway/gateway.py`, and the real-server integration tests in `tests/api/admin/gateway/circuit.py` (a local fake upstream behind a routed provider: trip on first error, no second upstream call, one notification per admin with no key leaked, isolation between models, one shared breaker per model across providers, bad request not re-sent, shape-fixable 400 left closed, healthy target silent).
|
||||
- **A 400 is terminal UNLESS its body names rejected keys** the auto-resolve ladder can fix (see "Auto-resolve of request shape"): that case is retried on the SAME route with another shape before any fallback is considered.
|
||||
- **Deliberately NOT covered:** `gateway_max_retries`/`gateway_retry_backoff_ms`, the circuit breaker, and hard docker-adjacent settings are untouched - fallback is additive on top of them, not a replacement retry mechanism.
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ from devplacepy.services.openai_gateway.routing import (
|
||||
chat_overlay,
|
||||
embed_overlay,
|
||||
image_overlay,
|
||||
model_store,
|
||||
resolve_entry_route,
|
||||
resolve_fallback_chain,
|
||||
)
|
||||
@@ -350,15 +351,16 @@ def _scrub_upstream(text: str) -> str:
|
||||
return SECRET_PATTERN.sub("[redacted]", text or "")[:UPSTREAM_DETAIL_MAX_CHARS]
|
||||
|
||||
|
||||
def _target_key(url: str, json_body: Optional[dict]) -> tuple[str, str]:
|
||||
def _target_key(json_body: Optional[dict]) -> str:
|
||||
model = json_body.get("model") if isinstance(json_body, dict) else ""
|
||||
return (urlparse(url or "").netloc, str(model or ""))
|
||||
return str(model or "")
|
||||
|
||||
|
||||
def _trip_detail(target, cooldown, category, status, upstream, retries) -> dict:
|
||||
def _trip_detail(model, url, cooldown, category, status, upstream, retries, route=None) -> dict:
|
||||
return {
|
||||
"provider": target[0],
|
||||
"model": target[1] or "-",
|
||||
"provider": urlparse(url or "").netloc,
|
||||
"model": model,
|
||||
"route": route or "",
|
||||
"cooldown": cooldown,
|
||||
"category": category,
|
||||
"status": status,
|
||||
@@ -367,6 +369,62 @@ def _trip_detail(target, cooldown, category, status, upstream, retries) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _chain_records(chain_start, kind) -> list:
|
||||
try:
|
||||
records = []
|
||||
start = model_store.get(chain_start)
|
||||
if start is not None and start.kind == kind:
|
||||
records.append(
|
||||
{
|
||||
"route": start.source_model,
|
||||
"target": start.target_model,
|
||||
"provider": start.provider or "default",
|
||||
}
|
||||
)
|
||||
for member in resolve_fallback_chain(chain_start, kind):
|
||||
records.append(
|
||||
{
|
||||
"route": member.source_model,
|
||||
"target": member.target_model,
|
||||
"provider": member.provider or "default",
|
||||
}
|
||||
)
|
||||
return records
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _format_chain(records, chain_start) -> str:
|
||||
if not records:
|
||||
return chain_start or "unrouted default"
|
||||
return " -> ".join(
|
||||
f"{record['route']} ({record['target']} @ {record['provider']})"
|
||||
for record in records
|
||||
)
|
||||
|
||||
|
||||
def _request_outcome(attempted, resp, exc, body_error, fallback_used_route, primary_route) -> tuple:
|
||||
count = len(attempted)
|
||||
if exc is not None:
|
||||
return False, f"all {count} route(s) failed, last error: connection failed ({_scrub_upstream(str(exc))})"
|
||||
if body_error:
|
||||
return False, f"all {count} route(s) failed, last error: embedded upstream error ({_scrub_upstream(str(body_error))})"
|
||||
if resp is None:
|
||||
return False, f"all {count} route(s) failed, last error: rejected by an open circuit, no upstream call"
|
||||
if resp.status_code != 200:
|
||||
try:
|
||||
text = resp.text
|
||||
except Exception:
|
||||
text = ""
|
||||
category = classify_error(resp.status_code, None, text)
|
||||
return False, f"all {count} route(s) failed, last error: HTTP {resp.status_code} ({category})"
|
||||
if fallback_used_route:
|
||||
return True, f"fallback '{fallback_used_route}' answered HTTP 200 ok"
|
||||
if primary_route:
|
||||
return True, f"local config '{primary_route}' answered HTTP 200 ok"
|
||||
return True, "default upstream answered HTTP 200 ok"
|
||||
|
||||
|
||||
def _resp_text(resp) -> str:
|
||||
try:
|
||||
return resp.text or ""
|
||||
@@ -383,24 +441,67 @@ def _trip_worthy(resp) -> bool:
|
||||
|
||||
|
||||
def _notify_gateway_status(is_open: bool, detail: Optional[dict] = None) -> None:
|
||||
from devplacepy.database import get_table
|
||||
from devplacepy.utils import create_notification
|
||||
try:
|
||||
from devplacepy.database import get_table
|
||||
from devplacepy.utils import create_notification
|
||||
|
||||
if detail:
|
||||
message = (
|
||||
f"AI gateway target {detail['provider']}/{detail['model']} tripped its circuit "
|
||||
f"breaker after 1 error - calls are rejected for {detail['cooldown']}s. "
|
||||
f"Error class: {detail['category']}. HTTP status: {detail['status']}. "
|
||||
f"Upstream said: {detail['upstream']}. Retries used: {detail['retries']}."
|
||||
)
|
||||
else:
|
||||
message = (
|
||||
"AI gateway circuit breaker opened - upstream calls are being rejected"
|
||||
if is_open
|
||||
else "AI gateway circuit breaker closed - upstream calls have resumed"
|
||||
)
|
||||
for admin in get_table("users").find(role="Admin"):
|
||||
create_notification(admin["uid"], "system", message, "gateway", "/admin/services/openai")
|
||||
detail = dict(detail) if isinstance(detail, dict) else {}
|
||||
if detail.get("global"):
|
||||
message = (
|
||||
"AI gateway global circuit breaker opened - upstream calls are being rejected. "
|
||||
f"Local config: '{detail.get('route') or detail.get('requested') or 'default'}'. "
|
||||
f"Requested model: '{detail.get('requested')}'. "
|
||||
f"Full route: {detail.get('chain')}. "
|
||||
f"Outcome: {_scrub_upstream(str(detail.get('outcome') or ''))}."
|
||||
)
|
||||
elif detail:
|
||||
model = detail.get("model") or "-"
|
||||
provider = detail.get("provider") or "-"
|
||||
message = (
|
||||
f"AI gateway model {model} (upstream {provider}) tripped its circuit "
|
||||
f"breaker after 1 error - calls are rejected for {detail.get('cooldown')}s. "
|
||||
)
|
||||
route = str(detail.get("route") or "")
|
||||
if route:
|
||||
message += f"Local config: '{route}'. "
|
||||
else:
|
||||
message += "Local config: default (no routed model). "
|
||||
if detail.get("requested"):
|
||||
message += f"Requested model: '{detail.get('requested')}'. "
|
||||
if detail.get("chain"):
|
||||
message += f"Full route: {detail.get('chain')}. "
|
||||
if detail.get("redirect"):
|
||||
message += f"Redirect: will redirect to fallback '{detail['redirect']}'. "
|
||||
elif "redirect" in detail:
|
||||
if route.startswith("vision:"):
|
||||
message += "Redirect: none, vision has no fallback model. "
|
||||
else:
|
||||
message += "Redirect: none, end of the fallback line. "
|
||||
outcome = _scrub_upstream(str(detail.get("outcome") or ""))
|
||||
if outcome:
|
||||
message += f"Outcome: {outcome}. "
|
||||
message += (
|
||||
f"Error class: {detail.get('category')}. HTTP status: {detail.get('status')}. "
|
||||
f"Upstream said: {detail.get('upstream')}. Retries used: {detail.get('retries')}."
|
||||
)
|
||||
else:
|
||||
message = (
|
||||
"AI gateway circuit breaker opened - upstream calls are being rejected"
|
||||
if is_open
|
||||
else "AI gateway circuit breaker closed - upstream calls have resumed"
|
||||
)
|
||||
try:
|
||||
admins = list(get_table("users").find(role="Admin"))
|
||||
except Exception as exc:
|
||||
logger.warning("ai.gateway.circuit.notify skipped, admin lookup failed: %s", exc)
|
||||
return
|
||||
for admin in admins:
|
||||
try:
|
||||
create_notification(admin["uid"], "system", message, "gateway", "/admin/services/openai")
|
||||
except Exception as exc:
|
||||
logger.warning("ai.gateway.circuit.notify failed for one admin: %s", exc)
|
||||
except Exception as exc:
|
||||
logger.warning("ai.gateway.circuit.notify failed: %s", exc)
|
||||
|
||||
|
||||
def _connect_tracer(holder: dict):
|
||||
@@ -453,7 +554,7 @@ class GatewayRuntime:
|
||||
self._vision_breaker = CircuitBreaker(
|
||||
config.CIRCUIT_THRESHOLD_DEFAULT, config.CIRCUIT_COOLDOWN_SECONDS_DEFAULT
|
||||
)
|
||||
self._target_breakers: dict[tuple[str, str], CircuitBreaker] = {}
|
||||
self._target_breakers: dict[str, CircuitBreaker] = {}
|
||||
self._catalog_cache = CatalogCache()
|
||||
self.requests = 0
|
||||
self.errors = 0
|
||||
@@ -515,36 +616,124 @@ class GatewayRuntime:
|
||||
breaker.configure(1, cfg["gateway_circuit_cooldown_seconds"])
|
||||
return breaker
|
||||
|
||||
def _trip_target(self, target, cfg, category, status, upstream, retries) -> None:
|
||||
breaker = self._target_breaker(target, cfg)
|
||||
def _trip_target(
|
||||
self, model, url, cfg, category, status, upstream, retries,
|
||||
route_name=None, trip_sink=None,
|
||||
) -> None:
|
||||
breaker = self._target_breaker(model, cfg)
|
||||
was_open = breaker.is_open
|
||||
breaker.trip()
|
||||
if was_open:
|
||||
return
|
||||
detail = _trip_detail(
|
||||
target, cfg["gateway_circuit_cooldown_seconds"], category, status, upstream, retries
|
||||
model, url, cfg["gateway_circuit_cooldown_seconds"], category, status, upstream, retries,
|
||||
route=route_name,
|
||||
)
|
||||
logger.warning(
|
||||
"ai.gateway.circuit.target_opened provider=%s model=%s category=%s status=%s retries=%s",
|
||||
detail["provider"], detail["model"], category, status, retries,
|
||||
"ai.gateway.circuit.target_opened provider=%s model=%s route=%s category=%s status=%s retries=%s",
|
||||
detail["provider"], detail["model"], detail["route"], category, status, retries,
|
||||
)
|
||||
background.submit(_notify_gateway_status, True, detail)
|
||||
if trip_sink is not None:
|
||||
trip_sink.append({"kind": "target", "detail": detail})
|
||||
else:
|
||||
background.submit(_notify_gateway_status, True, detail)
|
||||
|
||||
def _embedded_error(self, resp, stream, url, model, cfg, protocol="chat_completions"):
|
||||
def _notify_trips(
|
||||
self, trip_sink, kind, chain_start, requested, attempted,
|
||||
outcome_ok, outcome, served_route,
|
||||
) -> None:
|
||||
try:
|
||||
if not trip_sink:
|
||||
return
|
||||
try:
|
||||
records = _chain_records(chain_start, kind)
|
||||
chain = _format_chain(records, chain_start)
|
||||
order = [entry.get("route") or "" for entry in attempted]
|
||||
names = [record["route"] for record in records]
|
||||
except Exception:
|
||||
chain = chain_start or "unrouted default"
|
||||
order = []
|
||||
names = []
|
||||
primary = order[0] if order else (requested or "")
|
||||
for event in trip_sink:
|
||||
if event.get("kind") == "global_open":
|
||||
background.submit(
|
||||
_notify_gateway_status,
|
||||
True,
|
||||
{
|
||||
"global": True,
|
||||
"route": primary,
|
||||
"requested": requested or "",
|
||||
"chain": chain,
|
||||
"outcome": outcome,
|
||||
},
|
||||
)
|
||||
continue
|
||||
if event.get("kind") == "global_closed":
|
||||
background.submit(_notify_gateway_status, False)
|
||||
continue
|
||||
if event.get("kind") == "vision":
|
||||
vision_detail = dict(event.get("detail") or {})
|
||||
vision_route = str(
|
||||
vision_detail.get("route") or vision_detail.get("model") or "vision"
|
||||
)
|
||||
vision_detail["chain"] = (
|
||||
f"{vision_route} (vision model {vision_detail.get('model')} "
|
||||
f"@ {vision_detail.get('provider')}, no fallback line)"
|
||||
)
|
||||
vision_detail["redirect"] = ""
|
||||
vision_detail["requested"] = ""
|
||||
vision_detail["served_route"] = served_route or ""
|
||||
vision_detail["outcome_ok"] = outcome_ok
|
||||
vision_detail["outcome"] = outcome
|
||||
background.submit(_notify_gateway_status, True, vision_detail)
|
||||
continue
|
||||
detail = dict(event.get("detail") or {})
|
||||
tripped = detail.get("route") or ""
|
||||
redirect = ""
|
||||
if tripped and tripped in order:
|
||||
pos = order.index(tripped)
|
||||
if pos + 1 < len(order):
|
||||
redirect = order[pos + 1]
|
||||
if not redirect and tripped and tripped in names:
|
||||
pos = names.index(tripped)
|
||||
if pos + 1 < len(names):
|
||||
redirect = names[pos + 1]
|
||||
if not redirect and not tripped:
|
||||
if len(order) > 1:
|
||||
redirect = order[1]
|
||||
elif names:
|
||||
redirect = names[0]
|
||||
detail["requested"] = requested or ""
|
||||
detail["chain"] = chain
|
||||
detail["redirect"] = redirect
|
||||
detail["served_route"] = served_route or ""
|
||||
detail["outcome_ok"] = outcome_ok
|
||||
detail["outcome"] = outcome
|
||||
background.submit(_notify_gateway_status, True, detail)
|
||||
except Exception as exc:
|
||||
logger.warning("ai.gateway.circuit.notify_trips failed: %s", exc)
|
||||
|
||||
def _embedded_error(
|
||||
self, resp, stream, url, model, cfg, protocol="chat_completions",
|
||||
route_name=None, trip_sink=None,
|
||||
):
|
||||
message = _embedded_failure_message(resp, stream, protocol)
|
||||
if message and cfg.get("gateway_target_circuit_enabled", True):
|
||||
self._trip_target(_target_key(url, {"model": model}), cfg, "embedded_error", 200, message, 0)
|
||||
if message and model and cfg.get("gateway_target_circuit_enabled", True):
|
||||
self._trip_target(
|
||||
model, url, cfg, "embedded_error", 200, message, 0,
|
||||
route_name=route_name, trip_sink=trip_sink,
|
||||
)
|
||||
return message
|
||||
|
||||
def open_targets(self) -> list[dict]:
|
||||
now = time.monotonic()
|
||||
return [
|
||||
{
|
||||
"provider": key[0],
|
||||
"model": key[1],
|
||||
"model": model,
|
||||
"seconds_left": max(0, round(b.cooldown_seconds - (now - b.opened_at))),
|
||||
}
|
||||
for key, b in self._target_breakers.items()
|
||||
for model, b in self._target_breakers.items()
|
||||
if b.is_open
|
||||
][:MAX_OPEN_TARGETS_LISTED]
|
||||
|
||||
@@ -566,6 +755,8 @@ class GatewayRuntime:
|
||||
json_body=None,
|
||||
content=None,
|
||||
stream=False,
|
||||
route_name=None,
|
||||
trip_sink=None,
|
||||
):
|
||||
timing = {
|
||||
"queue_wait_ms": 0.0,
|
||||
@@ -575,8 +766,8 @@ class GatewayRuntime:
|
||||
"retry_succeeded": False,
|
||||
"circuit_open": False,
|
||||
}
|
||||
target = _target_key(url, json_body)
|
||||
target_enabled = cfg.get("gateway_target_circuit_enabled", True)
|
||||
target = _target_key(json_body)
|
||||
target_enabled = bool(target) and cfg.get("gateway_target_circuit_enabled", True)
|
||||
if not self._breaker.allow() or (
|
||||
target_enabled and not self._target_breaker(target, cfg).allow()
|
||||
):
|
||||
@@ -622,11 +813,15 @@ class GatewayRuntime:
|
||||
was_open = self._breaker.is_open
|
||||
self._breaker.record_failure()
|
||||
if not was_open and self._breaker.is_open:
|
||||
background.submit(_notify_gateway_status, True)
|
||||
if trip_sink is not None:
|
||||
trip_sink.append({"kind": "global_open"})
|
||||
else:
|
||||
background.submit(_notify_gateway_status, True)
|
||||
if target_enabled:
|
||||
self._trip_target(
|
||||
target, cfg, classify_error(0, exc), "no response", str(exc),
|
||||
target, url, cfg, classify_error(0, exc), "no response", str(exc),
|
||||
timing["retries_attempted"],
|
||||
route_name=route_name, trip_sink=trip_sink,
|
||||
)
|
||||
log(f"{method} {url} connection failed after {attempts} attempt(s): {exc}")
|
||||
return None, exc, timing
|
||||
@@ -641,18 +836,25 @@ class GatewayRuntime:
|
||||
was_open = self._breaker.is_open
|
||||
self._breaker.record_failure()
|
||||
if not was_open and self._breaker.is_open:
|
||||
background.submit(_notify_gateway_status, True)
|
||||
if trip_sink is not None:
|
||||
trip_sink.append({"kind": "global_open"})
|
||||
else:
|
||||
background.submit(_notify_gateway_status, True)
|
||||
else:
|
||||
was_open = self._breaker.is_open
|
||||
self._breaker.record_success()
|
||||
if was_open and not self._breaker.is_open:
|
||||
background.submit(_notify_gateway_status, False)
|
||||
if trip_sink is not None:
|
||||
trip_sink.append({"kind": "global_closed"})
|
||||
else:
|
||||
background.submit(_notify_gateway_status, False)
|
||||
timing["retry_succeeded"] = attempts > 1
|
||||
if target_enabled:
|
||||
if _trip_worthy(resp):
|
||||
self._trip_target(
|
||||
target, cfg, classify_error(resp.status_code, None, _resp_text(resp)),
|
||||
target, url, cfg, classify_error(resp.status_code, None, _resp_text(resp)),
|
||||
resp.status_code, _resp_text(resp), timing["retries_attempted"],
|
||||
route_name=route_name, trip_sink=trip_sink,
|
||||
)
|
||||
elif resp.status_code < 400:
|
||||
self._target_breaker(target, cfg).record_success()
|
||||
@@ -672,6 +874,8 @@ class GatewayRuntime:
|
||||
owner: tuple,
|
||||
log,
|
||||
method: ShapeMethod,
|
||||
route_name=None,
|
||||
trip_sink=None,
|
||||
) -> ChatAttempt:
|
||||
shaped = method.apply(pristine, cfg, protocol, stream, log)
|
||||
shaped["model"] = model
|
||||
@@ -689,6 +893,8 @@ class GatewayRuntime:
|
||||
log,
|
||||
json_body=send_body,
|
||||
stream=stream,
|
||||
route_name=route_name,
|
||||
trip_sink=trip_sink,
|
||||
)
|
||||
if resp is not None and stream and resp.status_code != 200:
|
||||
# The streamed response body hasn't been read yet; the shared error
|
||||
@@ -715,6 +921,8 @@ class GatewayRuntime:
|
||||
stream: bool,
|
||||
owner: tuple,
|
||||
log,
|
||||
route_name=None,
|
||||
trip_sink=None,
|
||||
) -> ChatAttempt:
|
||||
native_dialect = upstream_capabilities(
|
||||
cfg.get("gateway_upstream_url", ""),
|
||||
@@ -730,7 +938,7 @@ class GatewayRuntime:
|
||||
)
|
||||
first = await self._shaped_send(
|
||||
client, sem, pristine, body, cfg, protocol, model, headers,
|
||||
stream, owner, log, retrier.method,
|
||||
stream, owner, log, retrier.method, route_name, trip_sink,
|
||||
)
|
||||
current = first
|
||||
while current.resp is not None and retrier.eligible(
|
||||
@@ -743,7 +951,7 @@ class GatewayRuntime:
|
||||
await _close_quietly(current.resp)
|
||||
current = await self._shaped_send(
|
||||
client, sem, pristine, body, cfg, protocol, model, headers,
|
||||
stream, owner, log, method,
|
||||
stream, owner, log, method, route_name, trip_sink,
|
||||
)
|
||||
if current is first:
|
||||
return first
|
||||
@@ -804,6 +1012,7 @@ class GatewayRuntime:
|
||||
messages = body.get("messages", []) or []
|
||||
|
||||
vision_cost = 0.0
|
||||
trip_sink: list = []
|
||||
if cfg["gateway_vision_enabled"]:
|
||||
attribution = attribution_headers(app_reference)
|
||||
augmenter = VisionAugmenter(
|
||||
@@ -824,6 +1033,7 @@ class GatewayRuntime:
|
||||
),
|
||||
referer=attribution.get("HTTP-Referer", ""),
|
||||
title=attribution.get(TITLE_HEADER, ""),
|
||||
trip_sink=trip_sink,
|
||||
)
|
||||
messages = await augmenter.augment_messages(client, messages)
|
||||
self.vision_calls += augmenter.calls
|
||||
@@ -879,6 +1089,7 @@ class GatewayRuntime:
|
||||
attempted_routes = (
|
||||
{entry_route.source_model} if entry_route is not None else set()
|
||||
)
|
||||
attempted: list = []
|
||||
context_window = _route_context_window(cfg, model)
|
||||
if context_window:
|
||||
estimated_tokens = _estimate_request_tokens(pristine)
|
||||
@@ -910,19 +1121,41 @@ class GatewayRuntime:
|
||||
headers = _fallback_headers(cfg, "gateway_api_key", app_reference)
|
||||
break
|
||||
|
||||
primary_route = fallback_used_route
|
||||
if primary_route is None and overlay is not None:
|
||||
if entry_route is not None:
|
||||
primary_route = entry_route.source_model
|
||||
else:
|
||||
primary_route = (requested or "").strip() or None
|
||||
attempted.append(
|
||||
{
|
||||
"route": primary_route,
|
||||
"target": model,
|
||||
"provider": cfg.get("gateway_provider_name") or "default",
|
||||
}
|
||||
)
|
||||
try:
|
||||
attempt = await self._chat_attempt(
|
||||
client, sem, pristine, body, cfg, protocol, model, headers,
|
||||
stream, owner, log,
|
||||
route_name=primary_route, trip_sink=trip_sink,
|
||||
)
|
||||
except ToolEmulationError as tool_error:
|
||||
self._notify_trips(
|
||||
trip_sink, "chat", chain_start, requested, attempted,
|
||||
False, f"request aborted: {tool_error}",
|
||||
fallback_used_route or primary_route or "",
|
||||
)
|
||||
return _tool_emulation_rejection(tool_error)
|
||||
resp, exc, timing = attempt.resp, attempt.exc, attempt.timing
|
||||
payload = attempt.shaped
|
||||
send_start = attempt.send_start
|
||||
|
||||
requested_model = requested
|
||||
body_error = self._embedded_error(resp, stream, cfg["gateway_upstream_url"], model, cfg, protocol)
|
||||
body_error = self._embedded_error(
|
||||
resp, stream, cfg["gateway_upstream_url"], model, cfg, protocol,
|
||||
route_name=primary_route, trip_sink=trip_sink,
|
||||
)
|
||||
for fallback_route in resolve_fallback_chain(chain_start, "chat"):
|
||||
if fallback_route.source_model in attempted_routes:
|
||||
continue
|
||||
@@ -945,17 +1178,41 @@ class GatewayRuntime:
|
||||
)
|
||||
model = cfg["gateway_model"]
|
||||
headers = _fallback_headers(cfg, "gateway_api_key", app_reference)
|
||||
attempted.append(
|
||||
{
|
||||
"route": fallback_route.source_model,
|
||||
"target": model,
|
||||
"provider": cfg.get("gateway_provider_name") or "default",
|
||||
}
|
||||
)
|
||||
try:
|
||||
attempt = await self._chat_attempt(
|
||||
client, sem, pristine, body, cfg, protocol, model, headers,
|
||||
stream, owner, log,
|
||||
route_name=fallback_route.source_model, trip_sink=trip_sink,
|
||||
)
|
||||
except ToolEmulationError as tool_error:
|
||||
self._notify_trips(
|
||||
trip_sink, "chat", chain_start, requested, attempted,
|
||||
False, f"request aborted: {tool_error}",
|
||||
fallback_used_route or primary_route or "",
|
||||
)
|
||||
return _tool_emulation_rejection(tool_error)
|
||||
resp, exc, timing = attempt.resp, attempt.exc, attempt.timing
|
||||
payload = attempt.shaped
|
||||
send_start = attempt.send_start
|
||||
body_error = self._embedded_error(resp, stream, cfg["gateway_upstream_url"], model, cfg, protocol)
|
||||
body_error = self._embedded_error(
|
||||
resp, stream, cfg["gateway_upstream_url"], model, cfg, protocol,
|
||||
route_name=fallback_route.source_model, trip_sink=trip_sink,
|
||||
)
|
||||
|
||||
outcome_ok, outcome = _request_outcome(
|
||||
attempted, resp, exc, body_error, fallback_used_route, primary_route
|
||||
)
|
||||
self._notify_trips(
|
||||
trip_sink, "chat", chain_start, requested, attempted,
|
||||
outcome_ok, outcome, fallback_used_route or primary_route or "",
|
||||
)
|
||||
|
||||
pricing = pricing_from_cfg(cfg)
|
||||
context_map = parse_context_map(cfg.get("gateway_model_context_map"))
|
||||
@@ -1515,6 +1772,20 @@ class GatewayRuntime:
|
||||
"Authorization": f"Bearer {cfg['gateway_embed_key']}",
|
||||
}
|
||||
|
||||
primary_route = fallback_used_route
|
||||
if primary_route is None and overlay is not None:
|
||||
if entry_route is not None:
|
||||
primary_route = entry_route.source_model
|
||||
else:
|
||||
primary_route = (requested or "").strip() or None
|
||||
attempted: list = [
|
||||
{
|
||||
"route": primary_route,
|
||||
"target": model,
|
||||
"provider": cfg.get("gateway_provider_name") or "default",
|
||||
}
|
||||
]
|
||||
trip_sink: list = []
|
||||
resp, exc, timing = await self._send(
|
||||
client,
|
||||
sem,
|
||||
@@ -1524,13 +1795,18 @@ class GatewayRuntime:
|
||||
cfg,
|
||||
log,
|
||||
json_body=payload,
|
||||
route_name=primary_route,
|
||||
trip_sink=trip_sink,
|
||||
)
|
||||
|
||||
requested_model = requested
|
||||
attempted_routes = (
|
||||
{entry_route.source_model} if entry_route is not None else set()
|
||||
)
|
||||
body_error = self._embedded_error(resp, False, cfg["gateway_embed_url"], model, cfg)
|
||||
body_error = self._embedded_error(
|
||||
resp, False, cfg["gateway_embed_url"], model, cfg,
|
||||
route_name=primary_route, trip_sink=trip_sink,
|
||||
)
|
||||
for fallback_route in resolve_fallback_chain(chain_start, "embed"):
|
||||
if fallback_route.source_model in attempted_routes:
|
||||
continue
|
||||
@@ -1552,6 +1828,13 @@ class GatewayRuntime:
|
||||
payload = dict(payload)
|
||||
payload["model"] = model
|
||||
headers = _fallback_headers(cfg, "gateway_embed_key", app_reference)
|
||||
attempted.append(
|
||||
{
|
||||
"route": fallback_route.source_model,
|
||||
"target": model,
|
||||
"provider": cfg.get("gateway_provider_name") or "default",
|
||||
}
|
||||
)
|
||||
resp, exc, timing = await self._send(
|
||||
client,
|
||||
sem,
|
||||
@@ -1561,8 +1844,21 @@ class GatewayRuntime:
|
||||
cfg,
|
||||
log,
|
||||
json_body=payload,
|
||||
route_name=fallback_route.source_model,
|
||||
trip_sink=trip_sink,
|
||||
)
|
||||
body_error = self._embedded_error(resp, False, cfg["gateway_embed_url"], model, cfg)
|
||||
body_error = self._embedded_error(
|
||||
resp, False, cfg["gateway_embed_url"], model, cfg,
|
||||
route_name=fallback_route.source_model, trip_sink=trip_sink,
|
||||
)
|
||||
|
||||
outcome_ok, outcome = _request_outcome(
|
||||
attempted, resp, exc, body_error, fallback_used_route, primary_route
|
||||
)
|
||||
self._notify_trips(
|
||||
trip_sink, "embed", chain_start, requested, attempted,
|
||||
outcome_ok, outcome, fallback_used_route or primary_route or "",
|
||||
)
|
||||
|
||||
pricing = pricing_from_cfg(cfg)
|
||||
context_map = parse_context_map(cfg.get("gateway_model_context_map"))
|
||||
@@ -1772,6 +2068,20 @@ class GatewayRuntime:
|
||||
"No upstream image API key configured (gateway_image_key / gateway_api_key / OPENROUTER_API_KEY); upstream will likely reject the request"
|
||||
)
|
||||
|
||||
primary_route = fallback_used_route
|
||||
if primary_route is None and overlay is not None:
|
||||
if entry_route is not None:
|
||||
primary_route = entry_route.source_model
|
||||
else:
|
||||
primary_route = (requested or "").strip() or None
|
||||
attempted: list = [
|
||||
{
|
||||
"route": primary_route,
|
||||
"target": model,
|
||||
"provider": cfg.get("gateway_provider_name") or "default",
|
||||
}
|
||||
]
|
||||
trip_sink: list = []
|
||||
resp, exc, timing = await self._send(
|
||||
client,
|
||||
sem,
|
||||
@@ -1781,13 +2091,18 @@ class GatewayRuntime:
|
||||
cfg,
|
||||
log,
|
||||
json_body=payload,
|
||||
route_name=primary_route,
|
||||
trip_sink=trip_sink,
|
||||
)
|
||||
|
||||
requested_model = requested
|
||||
attempted_routes = (
|
||||
{entry_route.source_model} if entry_route is not None else set()
|
||||
)
|
||||
body_error = self._embedded_error(resp, False, cfg["gateway_image_url"], model, cfg)
|
||||
body_error = self._embedded_error(
|
||||
resp, False, cfg["gateway_image_url"], model, cfg,
|
||||
route_name=primary_route, trip_sink=trip_sink,
|
||||
)
|
||||
for fallback_route in resolve_fallback_chain(chain_start, "image"):
|
||||
if fallback_route.source_model in attempted_routes:
|
||||
continue
|
||||
@@ -1809,6 +2124,13 @@ class GatewayRuntime:
|
||||
payload = dict(payload)
|
||||
payload["model"] = model
|
||||
headers = _fallback_headers(cfg, "gateway_image_key", app_reference)
|
||||
attempted.append(
|
||||
{
|
||||
"route": fallback_route.source_model,
|
||||
"target": model,
|
||||
"provider": cfg.get("gateway_provider_name") or "default",
|
||||
}
|
||||
)
|
||||
resp, exc, timing = await self._send(
|
||||
client,
|
||||
sem,
|
||||
@@ -1818,8 +2140,21 @@ class GatewayRuntime:
|
||||
cfg,
|
||||
log,
|
||||
json_body=payload,
|
||||
route_name=fallback_route.source_model,
|
||||
trip_sink=trip_sink,
|
||||
)
|
||||
body_error = self._embedded_error(resp, False, cfg["gateway_image_url"], model, cfg)
|
||||
body_error = self._embedded_error(
|
||||
resp, False, cfg["gateway_image_url"], model, cfg,
|
||||
route_name=fallback_route.source_model, trip_sink=trip_sink,
|
||||
)
|
||||
|
||||
outcome_ok, outcome = _request_outcome(
|
||||
attempted, resp, exc, body_error, fallback_used_route, primary_route
|
||||
)
|
||||
self._notify_trips(
|
||||
trip_sink, "image", chain_start, requested, attempted,
|
||||
outcome_ok, outcome, fallback_used_route or primary_route or "",
|
||||
)
|
||||
|
||||
pricing = pricing_from_cfg(cfg)
|
||||
context_map = parse_context_map(cfg.get("gateway_model_context_map"))
|
||||
@@ -1962,9 +2297,49 @@ class GatewayRuntime:
|
||||
if content_type:
|
||||
headers["Content-Type"] = content_type
|
||||
|
||||
trip_sink: list = []
|
||||
resp, exc, timing = await self._send(
|
||||
client, sem, method, url, headers, cfg, log, content=body
|
||||
client, sem, method, url, headers, cfg, log, content=body,
|
||||
trip_sink=trip_sink,
|
||||
)
|
||||
if trip_sink:
|
||||
try:
|
||||
outcome_ok, outcome = _request_outcome(
|
||||
[
|
||||
{
|
||||
"route": cfg["gateway_model"],
|
||||
"target": cfg["gateway_model"],
|
||||
"provider": cfg.get("gateway_provider_name") or "default",
|
||||
}
|
||||
],
|
||||
resp, exc, None, None, cfg["gateway_model"],
|
||||
)
|
||||
chain = f"passthrough {method} {subpath} (no fallback line)"
|
||||
for event in trip_sink:
|
||||
if event.get("kind") == "global_open":
|
||||
background.submit(
|
||||
_notify_gateway_status,
|
||||
True,
|
||||
{
|
||||
"global": True,
|
||||
"route": cfg["gateway_model"],
|
||||
"requested": "",
|
||||
"chain": chain,
|
||||
"outcome": outcome,
|
||||
},
|
||||
)
|
||||
elif event.get("kind") == "global_closed":
|
||||
background.submit(_notify_gateway_status, False)
|
||||
else:
|
||||
passthrough_detail = dict(event.get("detail") or {})
|
||||
passthrough_detail["requested"] = ""
|
||||
passthrough_detail["chain"] = chain
|
||||
passthrough_detail["redirect"] = ""
|
||||
passthrough_detail["outcome_ok"] = outcome_ok
|
||||
passthrough_detail["outcome"] = outcome
|
||||
background.submit(_notify_gateway_status, True, passthrough_detail)
|
||||
except Exception as notify_exc:
|
||||
logger.warning("ai.gateway.circuit.passthrough notify failed: %s", notify_exc)
|
||||
|
||||
base = {
|
||||
"owner_kind": owner[0],
|
||||
|
||||
@@ -12,7 +12,9 @@ from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
from devplacepy.database import (
|
||||
bump_cache_version,
|
||||
db,
|
||||
get_setting,
|
||||
get_table,
|
||||
set_setting,
|
||||
sync_local_cache,
|
||||
)
|
||||
from devplacepy.services.openai_gateway import muse_code, opencode_zen
|
||||
@@ -25,6 +27,7 @@ from devplacepy.services.openai_gateway.tool_emulation import (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PROVIDERS_TABLE = "gateway_providers"
|
||||
DEFAULT_KEY_SETTING = "gateway_api_key"
|
||||
MODELS_TABLE = "gateway_models"
|
||||
CACHE_NAME = "gateway_routing"
|
||||
KINDS = ("chat", "embed", "image")
|
||||
@@ -538,6 +541,7 @@ class ProviderStore:
|
||||
)
|
||||
if existing:
|
||||
table.update({**record, "id": existing["id"]}, ["id"])
|
||||
self._rotate_default_key(str(existing.get("api_key") or ""), payload)
|
||||
else:
|
||||
record["created_at"] = _now()
|
||||
table.insert(record)
|
||||
@@ -547,6 +551,18 @@ class ProviderStore:
|
||||
_ROUTING_CACHE.clear()
|
||||
return self.get(payload.name) or record
|
||||
|
||||
@staticmethod
|
||||
def _rotate_default_key(previous_key: str, payload: ProviderIn) -> None:
|
||||
if not previous_key or not payload.api_key or previous_key == payload.api_key:
|
||||
return
|
||||
if get_setting(DEFAULT_KEY_SETTING, "") != previous_key:
|
||||
return
|
||||
set_setting(DEFAULT_KEY_SETTING, payload.api_key)
|
||||
logger.info(
|
||||
"Provider %s key rotated; default upstream key shared the old value and was rotated with it",
|
||||
payload.name,
|
||||
)
|
||||
|
||||
def remove(self, name: str) -> bool:
|
||||
name = (name or "").strip().lower()
|
||||
if not name or PROVIDERS_TABLE not in db.tables:
|
||||
|
||||
@@ -109,6 +109,7 @@ class VisionAugmenter:
|
||||
breaker=None,
|
||||
max_retries: int = 2,
|
||||
retry_backoff_ms: int = 250,
|
||||
trip_sink=None,
|
||||
):
|
||||
self.vision_url = vision_url
|
||||
self.vision_model = vision_model
|
||||
@@ -125,6 +126,7 @@ class VisionAugmenter:
|
||||
self.breaker = breaker
|
||||
self.max_retries = max(0, int(max_retries or 0))
|
||||
self.retry_backoff_ms = max(0, int(retry_backoff_ms or 0))
|
||||
self.trip_sink = trip_sink
|
||||
self.calls = 0
|
||||
self.cost_usd = 0.0
|
||||
|
||||
@@ -185,7 +187,48 @@ class VisionAugmenter:
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
await _backoff(self.retry_backoff_ms, attempt, retry_after)
|
||||
return resp, exc
|
||||
return resp, exc, attempt
|
||||
|
||||
def _report_trip(self, exc, resp, tries) -> None:
|
||||
try:
|
||||
if self.breaker is None:
|
||||
return
|
||||
if exc is not None:
|
||||
category = classify_error(0, exc)
|
||||
status: Any = "no response"
|
||||
upstream = str(exc)
|
||||
else:
|
||||
try:
|
||||
text = resp.text if resp is not None else ""
|
||||
except Exception:
|
||||
text = ""
|
||||
code = resp.status_code if resp is not None else 0
|
||||
category = classify_error(code, None, text)
|
||||
status = code
|
||||
upstream = text
|
||||
detail = {
|
||||
"provider": urlparse(self.vision_url or "").netloc,
|
||||
"model": self.vision_model,
|
||||
"route": f"vision:{self.vision_model}",
|
||||
"cooldown": self.breaker.cooldown_seconds,
|
||||
"category": category,
|
||||
"status": status,
|
||||
"upstream": (upstream or "")[:300],
|
||||
"retries": max(0, int(tries or 0) - 1),
|
||||
}
|
||||
logger.warning(
|
||||
"ai.gateway.circuit.vision_opened provider=%s model=%s route=%s category=%s status=%s retries=%s",
|
||||
detail["provider"], detail["model"], detail["route"], category, status, detail["retries"],
|
||||
)
|
||||
if self.trip_sink is not None:
|
||||
self.trip_sink.append({"kind": "vision", "detail": detail})
|
||||
else:
|
||||
from devplacepy.services.background import background
|
||||
from devplacepy.services.openai_gateway import gateway as gateway_mod
|
||||
|
||||
background.submit(gateway_mod._notify_gateway_status, True, detail)
|
||||
except Exception as report_exc:
|
||||
logger.warning("ai.gateway.circuit.vision report failed: %s", report_exc)
|
||||
|
||||
async def _describe_one(self, client: httpx.AsyncClient, image_block: dict) -> str:
|
||||
if not self.vision_key:
|
||||
@@ -231,16 +274,18 @@ class VisionAugmenter:
|
||||
headers[LEGACY_TITLE_HEADER] = self.title
|
||||
start = time.monotonic()
|
||||
shaped = retrier.method.apply(payload, shape_cfg, "chat_completions", False, logger.info)
|
||||
resp, exc = await self._post_with_retries(client, shaped, headers)
|
||||
resp, exc, tries = await self._post_with_retries(client, shaped, headers)
|
||||
first = (resp, exc)
|
||||
resolving = False
|
||||
total_tries = tries
|
||||
while resp is not None and retrier.eligible(resp.status_code):
|
||||
method = retrier.next_after(resp.status_code, resp.text)
|
||||
if method is None:
|
||||
break
|
||||
resolving = True
|
||||
shaped = method.apply(payload, shape_cfg, "chat_completions", False, logger.info)
|
||||
resp, exc = await self._post_with_retries(client, shaped, headers)
|
||||
resp, exc, tries = await self._post_with_retries(client, shaped, headers)
|
||||
total_tries += tries
|
||||
if resolving:
|
||||
if resp is not None and resp.status_code < 400:
|
||||
retrier.succeeded()
|
||||
@@ -249,7 +294,10 @@ class VisionAugmenter:
|
||||
resp, exc = first
|
||||
if self.breaker is not None:
|
||||
if exc is not None or (resp is not None and resp.status_code >= 500):
|
||||
was_open = self.breaker.is_open
|
||||
self.breaker.record_failure()
|
||||
if not was_open and self.breaker.is_open:
|
||||
self._report_trip(exc, resp, total_tries)
|
||||
else:
|
||||
self.breaker.record_success()
|
||||
if exc is not None:
|
||||
|
||||
@@ -44,7 +44,7 @@ Chat requests are augmented (vision), forwarded, retried, priced, and ledgered.
|
||||
|
||||
**Pricing:** `gateway_price_cache_hit_per_m` (0.0028), `gateway_price_cache_miss_per_m` (0.14), `gateway_price_output_per_m` (0.28), `gateway_vision_price_input_per_m` (0.0), `gateway_vision_price_output_per_m` (0.0), `gateway_embed_price_input_per_m` (0.01), `gateway_image_price_per_call` (0.04). Token rates are USD per 1M tokens; the image price is a flat USD per generated image. Used only when the upstream returns no native cost.
|
||||
|
||||
**Reliability:** `gateway_max_retries` (default 2, 0 to 10), `gateway_retry_backoff_ms` (250, linear multiplied by attempt number), `gateway_circuit_threshold` (consecutive failures before opening, default 5, 0 disables), `gateway_circuit_cooldown_seconds` (default 30).
|
||||
**Reliability:** `gateway_max_retries` (default 2, 0 to 10), `gateway_retry_backoff_ms` (250, linear multiplied by attempt number), `gateway_circuit_threshold` (consecutive failures before opening, default 5, 0 disables), `gateway_circuit_cooldown_seconds` (default 30, shared by the global and the per-target breakers), `gateway_target_circuit_enabled` (default on, see below).
|
||||
|
||||
**Tracking:** `gateway_usage_retention_hours` (default 720), `gateway_model_context_map` (JSON mapping model name to context window for utilization tracking).
|
||||
|
||||
@@ -78,6 +78,8 @@ The AI content correction worker reads `X-Gateway-Cost-USD` and the token header
|
||||
|
||||
`retry_send()` retries on connection errors, timeouts, and upstream 5xx, up to `max_retries`, with a linear backoff of `backoff_ms * attempt`. The `CircuitBreaker` counts consecutive failures: at `threshold` it opens and new requests are rejected immediately with `503` and error type `circuit_open`; after `cooldown_seconds` it half-opens to test recovery and closes on the next success. Setting the threshold to 0 disables the breaker.
|
||||
|
||||
**Per-model breaker (first error opens it, admins are told):** on top of the global breaker, `GatewayRuntime._send` keeps one breaker per MODEL (the model name sent upstream) that opens on the FIRST error, so one failing model stops receiving traffic while every other model keeps working. The key is the model name alone: the same model name served by two providers shares one breaker. A call with no model (passthrough) never uses it. It opens on a connection error or timeout, a 5xx or 429 that survived the retries, any other 4xx (a malformed request is not re-sent for the cooldown), and a 200 whose body carries an embedded error. A 400/422 that names rejected request keys does not open it, because the shape auto-resolver still has to retry it. While open, calls to that target make no upstream request and walk the fallback chain, or return `503` with type `circuit_open`. The cooldown is `gateway_circuit_cooldown_seconds`, then the next call is the probe and a success closes it. Opening sends exactly one `system` notification to every admin naming the model, its upstream host, error class, HTTP status, retries used, cooldown and the upstream error text (capped at 300 characters, Bearer and `sk-` tokens removed), linking to `/admin/services/openai`. State is per worker and is not kept across restarts. `gateway_target_circuit_enabled` off restores global-only behaviour.
|
||||
|
||||
## System-message composition
|
||||
|
||||
Before a chat request is forwarded (and only for `chat/completions`, never embeddings or passthrough), the gateway composes a single system message in the fixed order: operator preamble, then a date line, then the client's original system content.
|
||||
@@ -102,7 +104,7 @@ Before forwarding a chat request, image blocks are described by the vision model
|
||||
|
||||
## Data it offers
|
||||
|
||||
**Live metrics** (in `describe()`, at `GET /admin/services/data`): runtime counters (`requests`, `errors`, `in_flight`, `peak_in_flight`, `vision_calls`, `embed_calls`, `image_calls`, `last_status`, `last_latency_ms`, `pool`, `circuit_open`) and a 24h summary (`requests`, `success_pct`, `error_pct`, `cost_hour`, `cost_24h`, `tokens_24h`, `avg_latency_ms`, `avg_tps`, `peak_concurrency`, `top_model`, `top_caller`).
|
||||
**Live metrics** (in `describe()`, at `GET /admin/services/data`): runtime counters (`requests`, `errors`, `in_flight`, `peak_in_flight`, `vision_calls`, `embed_calls`, `image_calls`, `last_status`, `last_latency_ms`, `pool`, `circuit_open`, `vision_circuit_open`, `targets_open`: the open models with seconds left) and a 24h summary (`requests`, `success_pct`, `error_pct`, `cost_hour`, `cost_24h`, `tokens_24h`, `avg_latency_ms`, `avg_tps`, `peak_concurrency`, `top_model`, `top_caller`).
|
||||
|
||||
**Ledger** `gateway_usage_ledger`: one row per upstream call with owner, backend, requested and actual model, status, success, error category, the full latency breakdown (`upstream_latency_ms`, `gateway_overhead_ms`, `queue_wait_ms`, `connect_ms`, `total_latency_ms`, and for a streamed call `ttft_ms`/`inter_token_ms`), token counts (prompt, completion, cache hit and miss, reasoning, total), `tokens_per_second`, context window and utilization, the cost fields (`cost_usd`, `input_cost_usd`, `output_cost_usd`, `native_cost`), request shape (`stream_requested`, `temperature`, `top_p`, `max_tokens`, `has_tools`), and reliability flags (`retries_attempted`, `retry_succeeded`, `circuit_open`). Pruned to `gateway_usage_retention_hours`.
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "devplacepy"
|
||||
version = "1.1.24"
|
||||
version = "1.1.25"
|
||||
description = "DevPlace - The Developer Social Network"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
|
||||
@@ -153,7 +153,10 @@ def test_gateway_auth_and_routing(alice):
|
||||
headers={"Authorization": f"Bearer {admin_key}"},
|
||||
json={"messages": [{"role": "user", "content": "hi"}]},
|
||||
)
|
||||
assert with_admin.status_code == 502
|
||||
assert with_admin.status_code == 503, (
|
||||
"auth and consent passed, but the per-model breaker tripped on the "
|
||||
"unreachable upstream above now rejects the same model without a call"
|
||||
)
|
||||
finally:
|
||||
_config(page, gateway_access_key="")
|
||||
page.request.post(f"{BASE_URL}/admin/services/openai/start")
|
||||
|
||||
@@ -67,7 +67,10 @@ def test_embeddings_auth_passes_upstream_unreachable(alice):
|
||||
headers={"Authorization": f"Bearer {admin_key}"},
|
||||
json={"model": "molodetz~embed", "input": "hello"},
|
||||
)
|
||||
assert with_admin.status_code == 502
|
||||
assert with_admin.status_code == 503, (
|
||||
"auth and consent passed, but the per-model breaker tripped on the "
|
||||
"unreachable upstream above now rejects the same model without a call"
|
||||
)
|
||||
finally:
|
||||
_config(page, gateway_access_key="", gateway_embed_key="")
|
||||
page.request.post(f"{BASE_URL}/admin/services/openai/start")
|
||||
|
||||
@@ -3726,8 +3726,9 @@ def test_a_failed_target_is_skipped_without_an_upstream_call(local_db):
|
||||
def test_one_bad_request_does_not_block_another_target(local_db):
|
||||
rt, cfg = _target_runtime_openai_gateway()
|
||||
client = FakeAlwaysFailClient_openai_gateway()
|
||||
_target_send_openai_gateway(rt, cfg, client, "https://target-b.example/v1/chat/completions", "m1")
|
||||
other = _target_send_openai_gateway(rt, cfg, client, "https://target-c.example/v1/chat/completions", "m1")
|
||||
url = "https://target-b.example/v1/chat/completions"
|
||||
_target_send_openai_gateway(rt, cfg, client, url, "model-b1")
|
||||
other = _target_send_openai_gateway(rt, cfg, client, url, "model-b2")
|
||||
assert other[2]["circuit_open"] is False
|
||||
assert len(client.calls) == 2
|
||||
|
||||
@@ -3759,20 +3760,407 @@ def test_embedded_200_error_trips_the_target(local_db):
|
||||
resp = FakeResp_openai_gateway(status=200, payload={"error": {"message": "provider exploded"}})
|
||||
url = "https://target-e.example/v1/chat/completions"
|
||||
assert rt._embedded_error(resp, False, url, "m1", cfg) is not None
|
||||
assert rt._target_breaker(gwmod._target_key(url, {"model": "m1"}), cfg).is_open is True
|
||||
assert rt.open_targets()[0]["provider"] == "target-e.example"
|
||||
assert rt._target_breaker("m1", cfg).is_open is True
|
||||
assert rt.open_targets()[0]["model"] == "m1"
|
||||
|
||||
|
||||
def test_transition_notifies_each_admin_once_with_detail_and_no_key(local_db, monkeypatch):
|
||||
sent = []
|
||||
monkeypatch.setattr(gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a)))
|
||||
rt, cfg = _target_runtime_openai_gateway()
|
||||
target = ("target-f.example", "m1")
|
||||
rt._trip_target(target, cfg, "upstream_error", 500, "Bearer sk-secret123 failed", 1)
|
||||
rt._trip_target(target, cfg, "upstream_error", 500, "again", 1)
|
||||
url = "https://target-f.example/v1/chat/completions"
|
||||
rt._trip_target("m1", url, cfg, "upstream_error", 500, "Bearer sk-secret123 failed", 1)
|
||||
rt._trip_target("m1", url, cfg, "upstream_error", 500, "again", 1)
|
||||
assert len(sent) == 1
|
||||
detail = sent[0][1][1]
|
||||
assert detail["category"] == "upstream_error"
|
||||
assert detail["status"] == 500
|
||||
assert "secret123" not in detail["upstream"]
|
||||
assert detail["provider"] == "target-f.example"
|
||||
|
||||
|
||||
def test_breaker_is_per_model_across_upstreams(local_db):
|
||||
rt, cfg = _target_runtime_openai_gateway()
|
||||
client = FakeAlwaysFailClient_openai_gateway()
|
||||
_target_send_openai_gateway(rt, cfg, client, "https://target-g1.example/v1/chat/completions", "shared-model")
|
||||
second = _target_send_openai_gateway(rt, cfg, client, "https://target-g2.example/v1/chat/completions", "shared-model")
|
||||
assert second[2]["circuit_open"] is True
|
||||
assert len(client.calls) == 1
|
||||
|
||||
|
||||
def test_a_request_without_a_model_never_uses_the_target_breaker(local_db):
|
||||
rt, cfg = _target_runtime_openai_gateway()
|
||||
client = FakeAlwaysFailClient_openai_gateway()
|
||||
url = "https://target-h.example/v1/other"
|
||||
_target_send_openai_gateway(rt, cfg, client, url, "")
|
||||
again = _target_send_openai_gateway(rt, cfg, client, url, "")
|
||||
assert again[2]["circuit_open"] is False
|
||||
assert rt._target_breakers == {}
|
||||
|
||||
|
||||
class FakeNotifyClient_openai_gateway:
|
||||
def __init__(self, *a, **k):
|
||||
self.calls = []
|
||||
|
||||
def build_request(self, method, url, headers=None, json=None, content=None):
|
||||
return FakeRequest(method, url, json)
|
||||
|
||||
async def send(self, request, stream=False):
|
||||
self.calls.append((request.url, request.json_body))
|
||||
body = request.json_body or {}
|
||||
if body.get("model") == "nb-primary-target":
|
||||
return FakeResp_openai_gateway(
|
||||
status=500, payload={"error": "boom sk-secret-xyz"}
|
||||
)
|
||||
return FakeResp_openai_gateway(
|
||||
payload={
|
||||
"id": "x",
|
||||
"model": body.get("model"),
|
||||
"choices": [{"message": {"content": "notify ok"}}],
|
||||
}
|
||||
)
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
|
||||
|
||||
class FakeNotifyChainClient_openai_gateway:
|
||||
def __init__(self, *a, **k):
|
||||
self.calls = []
|
||||
|
||||
def build_request(self, method, url, headers=None, json=None, content=None):
|
||||
return FakeRequest(method, url, json)
|
||||
|
||||
async def send(self, request, stream=False):
|
||||
self.calls.append((request.url, request.json_body))
|
||||
body = request.json_body or {}
|
||||
if body.get("model") in ("n3-a-target", "n3-b-target"):
|
||||
return FakeResp_openai_gateway(status=500, payload={"error": "boom"})
|
||||
return FakeResp_openai_gateway(
|
||||
payload={
|
||||
"id": "x",
|
||||
"model": body.get("model"),
|
||||
"choices": [{"message": {"content": "chain ok"}}],
|
||||
}
|
||||
)
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
|
||||
|
||||
def _notify_chat_openai_gateway(monkeypatch, client_cls, routes, source, owner_id):
|
||||
from devplacepy.services.openai_gateway import routing
|
||||
|
||||
monkeypatch.setattr(gwmod.httpx, "AsyncClient", client_cls)
|
||||
for payload in routes:
|
||||
routing.model_store.set(routing.ModelRouteIn(**payload))
|
||||
sent = []
|
||||
def _capture_notify_openai_gateway(fn, *a, **k):
|
||||
if fn is gwmod._notify_gateway_status:
|
||||
sent.append((fn, a))
|
||||
monkeypatch.setattr(gwmod.background, "submit", _capture_notify_openai_gateway)
|
||||
try:
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_vision_enabled"] = False
|
||||
cfg["gateway_max_retries"] = 0
|
||||
rt = svc.runtime()
|
||||
response = run_async(
|
||||
rt.handle_chat(
|
||||
{"model": source, "messages": [{"role": "user", "content": "hi"}]},
|
||||
cfg,
|
||||
("guest", owner_id),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
return response, sent
|
||||
finally:
|
||||
for payload in routes:
|
||||
routing.model_store.remove(payload["source_model"])
|
||||
|
||||
|
||||
def _rendered_messages_openai_gateway(monkeypatch, details):
|
||||
messages = []
|
||||
|
||||
class FakeUsersTable_openai_gateway:
|
||||
def find(self, **kwargs):
|
||||
if kwargs.get("role") == "Admin":
|
||||
return [{"uid": "admin-1"}]
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"devplacepy.database.get_table",
|
||||
lambda name: FakeUsersTable_openai_gateway(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"devplacepy.utils.create_notification",
|
||||
lambda uid, kind, message, *a, **k: messages.append(message),
|
||||
)
|
||||
for detail in details:
|
||||
gwmod._notify_gateway_status(True, detail)
|
||||
return messages
|
||||
|
||||
|
||||
def test_trip_notification_carries_local_config_redirect_and_outcome(local_db, monkeypatch):
|
||||
routes = [
|
||||
{"source_model": "nb-backup", "target_model": "nb-backup-target"},
|
||||
{
|
||||
"source_model": "nb-primary",
|
||||
"target_model": "nb-primary-target",
|
||||
"fallback_model": "nb-backup",
|
||||
},
|
||||
]
|
||||
response, sent = _notify_chat_openai_gateway(
|
||||
monkeypatch, FakeNotifyClient_openai_gateway, routes, "nb-primary", "notify_one"
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert len(sent) == 1
|
||||
assert sent[0][0] is gwmod._notify_gateway_status
|
||||
detail = sent[0][1][1]
|
||||
assert detail["route"] == "nb-primary"
|
||||
assert detail["model"] == "nb-primary-target"
|
||||
assert detail["requested"] == "nb-primary"
|
||||
assert "nb-primary" in detail["chain"]
|
||||
assert "nb-backup" in detail["chain"]
|
||||
assert detail["redirect"] == "nb-backup"
|
||||
assert detail["outcome_ok"] is True
|
||||
assert detail["served_route"] == "nb-backup"
|
||||
assert "nb-backup" in detail["outcome"]
|
||||
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
|
||||
assert len(messages) == 1
|
||||
assert "tripped its circuit breaker" in messages[0]
|
||||
assert "Local config: 'nb-primary'" in messages[0]
|
||||
assert "Full route:" in messages[0]
|
||||
assert "nb-primary" in messages[0].split("Full route:")[1]
|
||||
assert "nb-backup" in messages[0].split("Full route:")[1]
|
||||
assert "Redirect: will redirect to fallback 'nb-backup'" in messages[0]
|
||||
assert "Outcome: fallback 'nb-backup' answered HTTP 200 ok" in messages[0]
|
||||
assert "Error class: upstream_error" in messages[0]
|
||||
assert "HTTP status: 500" in messages[0]
|
||||
assert "sk-secret-xyz" not in messages[0]
|
||||
assert "[redacted]" in messages[0]
|
||||
|
||||
|
||||
def test_trip_notifications_cover_every_hop_of_a_recursive_chain(local_db, monkeypatch):
|
||||
routes = [
|
||||
{"source_model": "n3-c", "target_model": "n3-c-target"},
|
||||
{
|
||||
"source_model": "n3-b",
|
||||
"target_model": "n3-b-target",
|
||||
"fallback_model": "n3-c",
|
||||
},
|
||||
{
|
||||
"source_model": "n3-a",
|
||||
"target_model": "n3-a-target",
|
||||
"fallback_model": "n3-b",
|
||||
},
|
||||
]
|
||||
response, sent = _notify_chat_openai_gateway(
|
||||
monkeypatch, FakeNotifyChainClient_openai_gateway, routes, "n3-a", "notify_chain"
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert len(sent) == 2
|
||||
first = sent[0][1][1]
|
||||
second = sent[1][1][1]
|
||||
assert first["route"] == "n3-a"
|
||||
assert first["redirect"] == "n3-b"
|
||||
assert second["route"] == "n3-b"
|
||||
assert second["redirect"] == "n3-c"
|
||||
for detail in (first, second):
|
||||
assert detail["requested"] == "n3-a"
|
||||
assert "n3-a" in detail["chain"]
|
||||
assert "n3-b" in detail["chain"]
|
||||
assert "n3-c" in detail["chain"]
|
||||
assert detail["outcome_ok"] is True
|
||||
assert detail["outcome"] == "fallback 'n3-c' answered HTTP 200 ok"
|
||||
assert detail["served_route"] == "n3-c"
|
||||
messages = _rendered_messages_openai_gateway(monkeypatch, [first, second])
|
||||
assert "Redirect: will redirect to fallback 'n3-b'" in messages[0]
|
||||
assert "Redirect: will redirect to fallback 'n3-c'" in messages[1]
|
||||
assert "Outcome: fallback 'n3-c' answered HTTP 200 ok" in messages[0]
|
||||
assert "Outcome: fallback 'n3-c' answered HTTP 200 ok" in messages[1]
|
||||
|
||||
|
||||
def test_trip_notification_reports_failed_outcome_when_every_fallback_fails(
|
||||
local_db, monkeypatch
|
||||
):
|
||||
routes = [
|
||||
{"source_model": "n4-b", "target_model": "n4-b-target"},
|
||||
{
|
||||
"source_model": "n4-a",
|
||||
"target_model": "n4-a-target",
|
||||
"fallback_model": "n4-b",
|
||||
},
|
||||
]
|
||||
response, sent = _notify_chat_openai_gateway(
|
||||
monkeypatch, FakeAlwaysFailClient_openai_gateway, routes, "n4-a", "notify_fail"
|
||||
)
|
||||
assert response.status_code == 500
|
||||
assert len(sent) == 2
|
||||
first = sent[0][1][1]
|
||||
second = sent[1][1][1]
|
||||
assert first["route"] == "n4-a"
|
||||
assert first["redirect"] == "n4-b"
|
||||
assert second["route"] == "n4-b"
|
||||
assert second["redirect"] == ""
|
||||
for detail in (first, second):
|
||||
assert detail["outcome_ok"] is False
|
||||
assert "all 2 route(s) failed" in detail["outcome"]
|
||||
assert "HTTP 500" in detail["outcome"]
|
||||
messages = _rendered_messages_openai_gateway(monkeypatch, [first, second])
|
||||
assert "Redirect: will redirect to fallback 'n4-b'" in messages[0]
|
||||
assert "Redirect: none, end of the fallback line" in messages[1]
|
||||
assert "Outcome: all 2 route(s) failed" in messages[1]
|
||||
|
||||
|
||||
def test_trip_notification_redirects_unrouted_default_to_first_fallback(
|
||||
local_db, monkeypatch
|
||||
):
|
||||
rt, cfg = _target_runtime_openai_gateway()
|
||||
sent = []
|
||||
monkeypatch.setattr(
|
||||
gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a))
|
||||
)
|
||||
detail = gwmod._trip_detail(
|
||||
"default-target",
|
||||
"https://default.example/v1/chat/completions",
|
||||
30,
|
||||
"upstream_error",
|
||||
500,
|
||||
"boom",
|
||||
0,
|
||||
route="",
|
||||
)
|
||||
rt._notify_trips(
|
||||
[{"kind": "target", "detail": detail}],
|
||||
"chat",
|
||||
"molodetz",
|
||||
"weird-model",
|
||||
[{"route": ""}, {"route": "m-pro"}],
|
||||
True,
|
||||
"fallback 'm-pro' answered HTTP 200 ok",
|
||||
"m-pro",
|
||||
)
|
||||
assert len(sent) == 1
|
||||
enriched = sent[0][1][1]
|
||||
assert enriched["redirect"] == "m-pro"
|
||||
messages = _rendered_messages_openai_gateway(monkeypatch, [enriched])
|
||||
assert "Local config: default (no routed model)" in messages[0]
|
||||
assert "Redirect: will redirect to fallback 'm-pro'" in messages[0]
|
||||
assert "Outcome: fallback 'm-pro' answered HTTP 200 ok" in messages[0]
|
||||
|
||||
|
||||
def test_vision_trip_notifies_with_local_config_and_no_fallback(local_db, monkeypatch):
|
||||
import httpx
|
||||
|
||||
from devplacepy.services.openai_gateway.reliability import CircuitBreaker
|
||||
from devplacepy.services.openai_gateway.vision import VisionAugmenter, VisionCache
|
||||
|
||||
class FailVisionClient_openai_gateway:
|
||||
async def post(self, url, headers=None, json=None):
|
||||
raise httpx.ConnectError("vision down")
|
||||
|
||||
breaker = CircuitBreaker(1, 60)
|
||||
sink = []
|
||||
augmenter = VisionAugmenter(
|
||||
"https://vision.example/v1/chat/completions",
|
||||
"vis-model",
|
||||
"k",
|
||||
VisionCache(0),
|
||||
breaker=breaker,
|
||||
max_retries=0,
|
||||
trip_sink=sink,
|
||||
)
|
||||
image = {"type": "image_url", "image_url": {"url": "http://x/y.png"}}
|
||||
first = run_async(augmenter._describe_one(FailVisionClient_openai_gateway(), image))
|
||||
assert first.startswith("[vision call failed")
|
||||
assert breaker.is_open is True
|
||||
assert len(sink) == 1
|
||||
assert sink[0]["kind"] == "vision"
|
||||
assert sink[0]["detail"]["route"] == "vision:vis-model"
|
||||
second = run_async(augmenter._describe_one(FailVisionClient_openai_gateway(), image))
|
||||
assert second.startswith("[vision unavailable")
|
||||
assert len(sink) == 1
|
||||
rt, cfg = _target_runtime_openai_gateway()
|
||||
sent = []
|
||||
monkeypatch.setattr(
|
||||
gwmod.background, "submit", lambda fn, *a, **k: sent.append((fn, a))
|
||||
)
|
||||
rt._notify_trips(
|
||||
sink,
|
||||
"chat",
|
||||
"molodetz",
|
||||
"molodetz",
|
||||
[{"route": "molodetz"}],
|
||||
True,
|
||||
"fallback 'molodetz' answered HTTP 200 ok",
|
||||
"molodetz",
|
||||
)
|
||||
assert len(sent) == 1
|
||||
messages = _rendered_messages_openai_gateway(monkeypatch, [sent[0][1][1]])
|
||||
assert "Local config: 'vision:vis-model'" in messages[0]
|
||||
assert "Redirect: none, vision has no fallback model" in messages[0]
|
||||
assert "Outcome: fallback 'molodetz' answered HTTP 200 ok" in messages[0]
|
||||
|
||||
|
||||
def test_trip_notification_outcome_scrubs_secrets(local_db, monkeypatch):
|
||||
detail = gwmod._trip_detail(
|
||||
"s1-target",
|
||||
"https://s1.example/v1/chat/completions",
|
||||
30,
|
||||
"connection",
|
||||
"no response",
|
||||
"boom",
|
||||
0,
|
||||
route="s1",
|
||||
)
|
||||
detail["requested"] = "s1"
|
||||
detail["chain"] = "s1 (s1-target @ default)"
|
||||
detail["redirect"] = ""
|
||||
secret = "sk" + "-abc123"
|
||||
detail["outcome"] = "connection failed (Bearer " + secret + ")"
|
||||
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
|
||||
assert secret not in messages[0]
|
||||
assert "[redacted]" in messages[0]
|
||||
|
||||
|
||||
def test_passthrough_global_trip_carries_local_config(local_db, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
gwmod.httpx, "AsyncClient", FakeAlwaysFailClient_openai_gateway
|
||||
)
|
||||
svc = GatewayService()
|
||||
cfg = svc.effective_config()
|
||||
cfg["gateway_circuit_threshold"] = 1
|
||||
cfg["gateway_max_retries"] = 0
|
||||
rt = svc.runtime()
|
||||
sent = []
|
||||
monkeypatch.setattr(
|
||||
gwmod.background,
|
||||
"submit",
|
||||
lambda fn, *a, **k: sent.append((fn, a))
|
||||
if fn is gwmod._notify_gateway_status
|
||||
else None,
|
||||
)
|
||||
resp = run_async(
|
||||
rt.handle_passthrough(
|
||||
"POST",
|
||||
"responses",
|
||||
"application/json",
|
||||
b"{}",
|
||||
cfg,
|
||||
("guest", "ptrip_local"),
|
||||
"test",
|
||||
"default",
|
||||
)
|
||||
)
|
||||
assert resp.status_code == 500
|
||||
assert len(sent) == 1
|
||||
detail = sent[0][1][1]
|
||||
assert detail.get("global") is True
|
||||
assert detail["route"] == cfg["gateway_model"]
|
||||
assert "no fallback line" in detail["chain"]
|
||||
messages = _rendered_messages_openai_gateway(monkeypatch, [detail])
|
||||
assert f"Local config: '{cfg['gateway_model']}'" in messages[0]
|
||||
assert "passthrough POST responses (no fallback line)" in messages[0]
|
||||
assert "Outcome: all 1 route(s) failed" in messages[0]
|
||||
|
||||
@@ -804,3 +804,39 @@ def test_fetch_provider_models_tries_meta_listing_second(local_db, monkeypatch):
|
||||
]
|
||||
finally:
|
||||
r.provider_store.remove("probeprovmeta")
|
||||
|
||||
|
||||
def test_provider_key_rotation_follows_the_default_upstream_key(local_db):
|
||||
from devplacepy.database import get_setting, set_setting
|
||||
|
||||
original = get_setting(r.DEFAULT_KEY_SETTING, "")
|
||||
try:
|
||||
set_setting(r.DEFAULT_KEY_SETTING, "shared-old")
|
||||
r.provider_store.set(
|
||||
r.ProviderIn(name="utrot", base_url="https://up.example/v1/chat/completions", api_key="shared-old")
|
||||
)
|
||||
r.provider_store.set(
|
||||
r.ProviderIn(name="utrot", base_url="https://up.example/v1/chat/completions", api_key="shared-new")
|
||||
)
|
||||
assert get_setting(r.DEFAULT_KEY_SETTING, "") == "shared-new"
|
||||
finally:
|
||||
set_setting(r.DEFAULT_KEY_SETTING, original)
|
||||
_cleanup(["utrot"], [])
|
||||
|
||||
|
||||
def test_provider_key_rotation_leaves_an_unrelated_default_key_alone(local_db):
|
||||
from devplacepy.database import get_setting, set_setting
|
||||
|
||||
original = get_setting(r.DEFAULT_KEY_SETTING, "")
|
||||
try:
|
||||
set_setting(r.DEFAULT_KEY_SETTING, "default-key")
|
||||
r.provider_store.set(
|
||||
r.ProviderIn(name="utrot2", base_url="https://up.example/v1/chat/completions", api_key="a")
|
||||
)
|
||||
r.provider_store.set(
|
||||
r.ProviderIn(name="utrot2", base_url="https://up.example/v1/chat/completions", api_key="b")
|
||||
)
|
||||
assert get_setting(r.DEFAULT_KEY_SETTING, "") == "default-key"
|
||||
finally:
|
||||
set_setting(r.DEFAULT_KEY_SETTING, original)
|
||||
_cleanup(["utrot2"], [])
|
||||
|
||||
Reference in New Issue
Block a user