# retoor import http.server import json import socket import socketserver import threading import time import requests from devplacepy.database import get_table, refresh_snapshot from devplacepy.database.settings import internal_gateway_key from tests.api.admin.gateway.index import ( _unique_gateway, admin_session, ) from tests.conftest import BASE_URL CHAT_URL = f"{BASE_URL}/openai/v1/chat/completions" NOTICE_MARKER = "tripped its circuit breaker" SYNC_SECONDS = 1.5 NOTICE_WAIT_SECONDS = 10 OK_BODY = { "id": "x", "model": "m", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, } class _Upstream: def __init__(self, status, payload): self.calls = [] calls = self.calls body = json.dumps(payload).encode() code = status class Handler(http.server.BaseHTTPRequestHandler): def do_POST(self): length = int(self.headers.get("Content-Length") or 0) calls.append(json.loads(self.rfile.read(length) or b"{}")) self.send_response(code) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def log_message(self, *args): pass sock = socket.socket() sock.bind(("127.0.0.1", 0)) self.port = sock.getsockname()[1] sock.close() self.httpd = socketserver.ThreadingTCPServer(("127.0.0.1", self.port), Handler) threading.Thread(target=self.httpd.serve_forever, daemon=True).start() @property def host(self): return f"127.0.0.1:{self.port}" def close(self): self.httpd.shutdown() self.httpd.server_close() def _internal_headers(): refresh_snapshot() return {"Authorization": f"Bearer {internal_gateway_key()}", "Content-Type": "application/json"} def _route(admin, upstream, prefix, target=None): provider = _unique_gateway(f"{prefix}p").lower() source = _unique_gateway(f"{prefix}m").lower() target = target or f"vendor/{prefix}" admin.post( f"{BASE_URL}/admin/gateway/providers", json={"name": provider, "base_url": f"http://{upstream.host}/v1/chat/completions", "api_key": "sk-test"}, ) created = admin.post( f"{BASE_URL}/admin/gateway/models", json={"source_model": source, "provider": provider, "target_model": target, "kind": "chat"}, ) assert created.status_code == 200, created.text[:300] return provider, source def _cleanup(admin, provider, source): admin.delete(f"{BASE_URL}/admin/gateway/models/{source}") admin.delete(f"{BASE_URL}/admin/gateway/providers/{provider}") def _chat(source): return requests.post( CHAT_URL, headers=_internal_headers(), json={"model": source, "messages": [{"role": "user", "content": "hi"}]}, timeout=60, ) def _admin_notices(host): refresh_snapshot() rows = get_table("notifications").find(type="system") return [r for r in rows if NOTICE_MARKER in (r.get("message") or "") and host in r["message"]] def _wait_for_notices(host): deadline = time.time() + NOTICE_WAIT_SECONDS while time.time() < deadline: found = _admin_notices(host) if found: return found time.sleep(0.5) return [] def test_first_upstream_error_opens_the_target_and_the_second_call_never_leaves(seeded_db): admin = admin_session(seeded_db) upstream = _Upstream(500, {"error": {"message": "provider exploded"}}) provider, source = _route(admin, upstream, "trip") try: first = _chat(source) assert first.status_code >= 500 sent_after_first = len(upstream.calls) assert sent_after_first >= 1 second = _chat(source) assert second.status_code == 503 assert second.json()["error"]["type"] == "circuit_open" assert len(upstream.calls) == sent_after_first finally: _cleanup(admin, provider, source) upstream.close() def test_every_admin_gets_one_detailed_notification_per_incident(seeded_db): admin = admin_session(seeded_db) upstream = _Upstream(500, {"error": {"message": "provider exploded"}}) provider, source = _route(admin, upstream, "notify") try: _chat(source) _chat(source) _chat(source) notices = _wait_for_notices(upstream.host) assert notices, "no system notification reached the admins" admins = {u["uid"] for u in get_table("users").find(role="Admin")} per_admin = {} for row in notices: per_admin[row["user_uid"]] = per_admin.get(row["user_uid"], 0) + 1 assert set(per_admin) == admins assert set(per_admin.values()) == {1} message = notices[0]["message"] assert "vendor/notify" in message assert "Error class: upstream_error" in message assert "HTTP status: 500" in message assert "provider exploded" in message assert "sk-test" not in message assert notices[0]["target_url"] == "/admin/services/openai" finally: _cleanup(admin, provider, source) upstream.close() def test_a_failing_target_does_not_block_a_healthy_one(seeded_db): admin = admin_session(seeded_db) bad = _Upstream(500, {"error": {"message": "down"}}) good = _Upstream(200, OK_BODY) bad_provider, bad_source = _route(admin, bad, "isobad") good_provider, good_source = _route(admin, good, "isogood") try: _chat(bad_source) assert _chat(bad_source).status_code == 503 healthy = _chat(good_source) assert healthy.status_code == 200 assert healthy.json()["choices"][0]["message"]["content"] == "ok" assert len(good.calls) == 1 finally: _cleanup(admin, bad_provider, bad_source) _cleanup(admin, good_provider, good_source) bad.close() good.close() def test_a_malformed_request_trips_once_and_is_not_resent(seeded_db): admin = admin_session(seeded_db) upstream = _Upstream(400, {"error": {"message": "bad schema"}}) provider, source = _route(admin, upstream, "badreq") try: first = _chat(source) assert first.status_code == 400 sent = len(upstream.calls) assert sent == 1 assert _chat(source).status_code == 503 assert len(upstream.calls) == sent finally: _cleanup(admin, provider, source) upstream.close() def test_a_shape_fixable_400_does_not_open_the_target(seeded_db): admin = admin_session(seeded_db) upstream = _Upstream( 400, {"issues": [{"code": "unrecognized_keys", "keys": ["thinking"]}]} ) provider, source = _route(admin, upstream, "shape") try: _chat(source) sent = len(upstream.calls) assert sent >= 2 again = _chat(source) assert again.status_code == 400 assert len(upstream.calls) > sent finally: _cleanup(admin, provider, source) upstream.close() def test_the_breaker_is_per_model_not_per_upstream(seeded_db): admin = admin_session(seeded_db) first = _Upstream(500, {"error": {"message": "down"}}) second = _Upstream(200, OK_BODY) model = _unique_gateway("vendor/shared").lower() first_provider, first_source = _route(admin, first, "permodela", target=model) second_provider, second_source = _route(admin, second, "permodelb", target=model) try: _chat(first_source) blocked = _chat(second_source) assert blocked.status_code == 503 assert blocked.json()["error"]["type"] == "circuit_open" assert second.calls == [] finally: _cleanup(admin, first_provider, first_source) _cleanup(admin, second_provider, second_source) first.close() second.close() def test_a_healthy_target_stays_closed_and_raises_no_notification(seeded_db): admin = admin_session(seeded_db) upstream = _Upstream(200, OK_BODY) provider, source = _route(admin, upstream, "calm") try: for _ in range(3): assert _chat(source).status_code == 200 assert len(upstream.calls) == 3 time.sleep(SYNC_SECONDS) assert _admin_notices(upstream.host) == [] finally: _cleanup(admin, provider, source) upstream.close()