Files
devplacepy/tests/api/admin/gateway/circuit.py
T
retoorandClaude Sonnet 5.5 1a9201cb91 Admin pages: one shared specification, builders, fragments, caps and live refresh
Adds admin_sections registry, shared admin partials, WorkspaceList, admin CLAUDE.md spec,
capped node/flag listings, and api/unit tests enforcing the spec.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-15 01:50:50 +01:00

252 lines
8.4 KiB
Python

# retoor <retoor@molodetz.nl>
import http.server
import json
import socket
import socketserver
import threading
import time
import requests
from devplacepy.database import get_table, refresh_snapshot
from devplacepy.database.settings import internal_gateway_key
from tests.api.admin.gateway.index import (
_unique_gateway,
admin_session,
)
from tests.conftest import BASE_URL
CHAT_URL = f"{BASE_URL}/openai/v1/chat/completions"
NOTICE_MARKER = "tripped its circuit breaker"
SYNC_SECONDS = 1.5
NOTICE_WAIT_SECONDS = 10
OK_BODY = {
"id": "x",
"model": "m",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
class _Upstream:
def __init__(self, status, payload):
self.calls = []
calls = self.calls
body = json.dumps(payload).encode()
code = status
class Handler(http.server.BaseHTTPRequestHandler):
def do_POST(self):
length = int(self.headers.get("Content-Length") or 0)
calls.append(json.loads(self.rfile.read(length) or b"{}"))
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *args):
pass
sock = socket.socket()
sock.bind(("127.0.0.1", 0))
self.port = sock.getsockname()[1]
sock.close()
self.httpd = socketserver.ThreadingTCPServer(("127.0.0.1", self.port), Handler)
threading.Thread(target=self.httpd.serve_forever, daemon=True).start()
@property
def host(self):
return f"127.0.0.1:{self.port}"
def close(self):
self.httpd.shutdown()
self.httpd.server_close()
def _internal_headers():
refresh_snapshot()
return {"Authorization": f"Bearer {internal_gateway_key()}", "Content-Type": "application/json"}
def _route(admin, upstream, prefix, target=None):
provider = _unique_gateway(f"{prefix}p").lower()
source = _unique_gateway(f"{prefix}m").lower()
target = target or f"vendor/{prefix}"
admin.post(
f"{BASE_URL}/admin/gateway/providers",
json={"name": provider, "base_url": f"http://{upstream.host}/v1/chat/completions", "api_key": "sk-test"},
)
created = admin.post(
f"{BASE_URL}/admin/gateway/models",
json={"source_model": source, "provider": provider, "target_model": target, "kind": "chat"},
)
assert created.status_code == 200, created.text[:300]
return provider, source
def _cleanup(admin, provider, source):
admin.delete(f"{BASE_URL}/admin/gateway/models/{source}")
admin.delete(f"{BASE_URL}/admin/gateway/providers/{provider}")
def _chat(source):
return requests.post(
CHAT_URL,
headers=_internal_headers(),
json={"model": source, "messages": [{"role": "user", "content": "hi"}]},
timeout=60,
)
def _admin_notices(host):
refresh_snapshot()
rows = get_table("notifications").find(type="system")
return [r for r in rows if NOTICE_MARKER in (r.get("message") or "") and host in r["message"]]
def _wait_for_notices(host):
deadline = time.time() + NOTICE_WAIT_SECONDS
while time.time() < deadline:
found = _admin_notices(host)
if found:
return found
time.sleep(0.5)
return []
def test_first_upstream_error_opens_the_target_and_the_second_call_never_leaves(seeded_db):
admin = admin_session(seeded_db)
upstream = _Upstream(500, {"error": {"message": "provider exploded"}})
provider, source = _route(admin, upstream, "trip")
try:
first = _chat(source)
assert first.status_code >= 500
sent_after_first = len(upstream.calls)
assert sent_after_first >= 1
second = _chat(source)
assert second.status_code == 503
assert second.json()["error"]["type"] == "circuit_open"
assert len(upstream.calls) == sent_after_first
finally:
_cleanup(admin, provider, source)
upstream.close()
def test_every_admin_gets_one_detailed_notification_per_incident(seeded_db):
admin = admin_session(seeded_db)
upstream = _Upstream(500, {"error": {"message": "provider exploded"}})
provider, source = _route(admin, upstream, "notify")
try:
_chat(source)
_chat(source)
_chat(source)
notices = _wait_for_notices(upstream.host)
assert notices, "no system notification reached the admins"
admins = {u["uid"] for u in get_table("users").find(role="Admin")}
per_admin = {}
for row in notices:
per_admin[row["user_uid"]] = per_admin.get(row["user_uid"], 0) + 1
assert set(per_admin) == admins
assert set(per_admin.values()) == {1}
message = notices[0]["message"]
assert "vendor/notify" in message
assert "Error class: upstream_error" in message
assert "HTTP status: 500" in message
assert "provider exploded" in message
assert "sk-test" not in message
assert notices[0]["target_url"] == "/admin/services/openai"
finally:
_cleanup(admin, provider, source)
upstream.close()
def test_a_failing_target_does_not_block_a_healthy_one(seeded_db):
admin = admin_session(seeded_db)
bad = _Upstream(500, {"error": {"message": "down"}})
good = _Upstream(200, OK_BODY)
bad_provider, bad_source = _route(admin, bad, "isobad")
good_provider, good_source = _route(admin, good, "isogood")
try:
_chat(bad_source)
assert _chat(bad_source).status_code == 503
healthy = _chat(good_source)
assert healthy.status_code == 200
assert healthy.json()["choices"][0]["message"]["content"] == "ok"
assert len(good.calls) == 1
finally:
_cleanup(admin, bad_provider, bad_source)
_cleanup(admin, good_provider, good_source)
bad.close()
good.close()
def test_a_malformed_request_trips_once_and_is_not_resent(seeded_db):
admin = admin_session(seeded_db)
upstream = _Upstream(400, {"error": {"message": "bad schema"}})
provider, source = _route(admin, upstream, "badreq")
try:
first = _chat(source)
assert first.status_code == 400
sent = len(upstream.calls)
assert sent == 1
assert _chat(source).status_code == 503
assert len(upstream.calls) == sent
finally:
_cleanup(admin, provider, source)
upstream.close()
def test_a_shape_fixable_400_does_not_open_the_target(seeded_db):
admin = admin_session(seeded_db)
upstream = _Upstream(
400, {"issues": [{"code": "unrecognized_keys", "keys": ["thinking"]}]}
)
provider, source = _route(admin, upstream, "shape")
try:
_chat(source)
sent = len(upstream.calls)
assert sent >= 2
again = _chat(source)
assert again.status_code == 400
assert len(upstream.calls) > sent
finally:
_cleanup(admin, provider, source)
upstream.close()
def test_the_breaker_is_per_model_not_per_upstream(seeded_db):
admin = admin_session(seeded_db)
first = _Upstream(500, {"error": {"message": "down"}})
second = _Upstream(200, OK_BODY)
model = _unique_gateway("vendor/shared").lower()
first_provider, first_source = _route(admin, first, "permodela", target=model)
second_provider, second_source = _route(admin, second, "permodelb", target=model)
try:
_chat(first_source)
blocked = _chat(second_source)
assert blocked.status_code == 503
assert blocked.json()["error"]["type"] == "circuit_open"
assert second.calls == []
finally:
_cleanup(admin, first_provider, first_source)
_cleanup(admin, second_provider, second_source)
first.close()
second.close()
def test_a_healthy_target_stays_closed_and_raises_no_notification(seeded_db):
admin = admin_session(seeded_db)
upstream = _Upstream(200, OK_BODY)
provider, source = _route(admin, upstream, "calm")
try:
for _ in range(3):
assert _chat(source).status_code == 200
assert len(upstream.calls) == 3
time.sleep(SYNC_SECONDS)
assert _admin_notices(upstream.host) == []
finally:
_cleanup(admin, provider, source)
upstream.close()