forked from retoor/devplacepy
Adds admin_sections registry, shared admin partials, WorkspaceList, admin CLAUDE.md spec, capped node/flag listings, and api/unit tests enforcing the spec. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
252 lines
8.4 KiB
Python
252 lines
8.4 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import http.server
|
|
import json
|
|
import socket
|
|
import socketserver
|
|
import threading
|
|
import time
|
|
|
|
import requests
|
|
|
|
from devplacepy.database import get_table, refresh_snapshot
|
|
from devplacepy.database.settings import internal_gateway_key
|
|
from tests.api.admin.gateway.index import (
|
|
_unique_gateway,
|
|
admin_session,
|
|
)
|
|
from tests.conftest import BASE_URL
|
|
|
|
CHAT_URL = f"{BASE_URL}/openai/v1/chat/completions"
|
|
NOTICE_MARKER = "tripped its circuit breaker"
|
|
SYNC_SECONDS = 1.5
|
|
NOTICE_WAIT_SECONDS = 10
|
|
OK_BODY = {
|
|
"id": "x",
|
|
"model": "m",
|
|
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
|
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
|
}
|
|
|
|
|
|
class _Upstream:
|
|
def __init__(self, status, payload):
|
|
self.calls = []
|
|
calls = self.calls
|
|
body = json.dumps(payload).encode()
|
|
code = status
|
|
|
|
class Handler(http.server.BaseHTTPRequestHandler):
|
|
def do_POST(self):
|
|
length = int(self.headers.get("Content-Length") or 0)
|
|
calls.append(json.loads(self.rfile.read(length) or b"{}"))
|
|
self.send_response(code)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def log_message(self, *args):
|
|
pass
|
|
|
|
sock = socket.socket()
|
|
sock.bind(("127.0.0.1", 0))
|
|
self.port = sock.getsockname()[1]
|
|
sock.close()
|
|
self.httpd = socketserver.ThreadingTCPServer(("127.0.0.1", self.port), Handler)
|
|
threading.Thread(target=self.httpd.serve_forever, daemon=True).start()
|
|
|
|
@property
|
|
def host(self):
|
|
return f"127.0.0.1:{self.port}"
|
|
|
|
def close(self):
|
|
self.httpd.shutdown()
|
|
self.httpd.server_close()
|
|
|
|
|
|
def _internal_headers():
|
|
refresh_snapshot()
|
|
return {"Authorization": f"Bearer {internal_gateway_key()}", "Content-Type": "application/json"}
|
|
|
|
|
|
def _route(admin, upstream, prefix, target=None):
|
|
provider = _unique_gateway(f"{prefix}p").lower()
|
|
source = _unique_gateway(f"{prefix}m").lower()
|
|
target = target or f"vendor/{prefix}"
|
|
admin.post(
|
|
f"{BASE_URL}/admin/gateway/providers",
|
|
json={"name": provider, "base_url": f"http://{upstream.host}/v1/chat/completions", "api_key": "sk-test"},
|
|
)
|
|
created = admin.post(
|
|
f"{BASE_URL}/admin/gateway/models",
|
|
json={"source_model": source, "provider": provider, "target_model": target, "kind": "chat"},
|
|
)
|
|
assert created.status_code == 200, created.text[:300]
|
|
return provider, source
|
|
|
|
|
|
def _cleanup(admin, provider, source):
|
|
admin.delete(f"{BASE_URL}/admin/gateway/models/{source}")
|
|
admin.delete(f"{BASE_URL}/admin/gateway/providers/{provider}")
|
|
|
|
|
|
def _chat(source):
|
|
return requests.post(
|
|
CHAT_URL,
|
|
headers=_internal_headers(),
|
|
json={"model": source, "messages": [{"role": "user", "content": "hi"}]},
|
|
timeout=60,
|
|
)
|
|
|
|
|
|
def _admin_notices(host):
|
|
refresh_snapshot()
|
|
rows = get_table("notifications").find(type="system")
|
|
return [r for r in rows if NOTICE_MARKER in (r.get("message") or "") and host in r["message"]]
|
|
|
|
|
|
def _wait_for_notices(host):
|
|
deadline = time.time() + NOTICE_WAIT_SECONDS
|
|
while time.time() < deadline:
|
|
found = _admin_notices(host)
|
|
if found:
|
|
return found
|
|
time.sleep(0.5)
|
|
return []
|
|
|
|
|
|
def test_first_upstream_error_opens_the_target_and_the_second_call_never_leaves(seeded_db):
|
|
admin = admin_session(seeded_db)
|
|
upstream = _Upstream(500, {"error": {"message": "provider exploded"}})
|
|
provider, source = _route(admin, upstream, "trip")
|
|
try:
|
|
first = _chat(source)
|
|
assert first.status_code >= 500
|
|
sent_after_first = len(upstream.calls)
|
|
assert sent_after_first >= 1
|
|
second = _chat(source)
|
|
assert second.status_code == 503
|
|
assert second.json()["error"]["type"] == "circuit_open"
|
|
assert len(upstream.calls) == sent_after_first
|
|
finally:
|
|
_cleanup(admin, provider, source)
|
|
upstream.close()
|
|
|
|
|
|
def test_every_admin_gets_one_detailed_notification_per_incident(seeded_db):
|
|
admin = admin_session(seeded_db)
|
|
upstream = _Upstream(500, {"error": {"message": "provider exploded"}})
|
|
provider, source = _route(admin, upstream, "notify")
|
|
try:
|
|
_chat(source)
|
|
_chat(source)
|
|
_chat(source)
|
|
notices = _wait_for_notices(upstream.host)
|
|
assert notices, "no system notification reached the admins"
|
|
admins = {u["uid"] for u in get_table("users").find(role="Admin")}
|
|
per_admin = {}
|
|
for row in notices:
|
|
per_admin[row["user_uid"]] = per_admin.get(row["user_uid"], 0) + 1
|
|
assert set(per_admin) == admins
|
|
assert set(per_admin.values()) == {1}
|
|
message = notices[0]["message"]
|
|
assert "vendor/notify" in message
|
|
assert "Error class: upstream_error" in message
|
|
assert "HTTP status: 500" in message
|
|
assert "provider exploded" in message
|
|
assert "sk-test" not in message
|
|
assert notices[0]["target_url"] == "/admin/services/openai"
|
|
finally:
|
|
_cleanup(admin, provider, source)
|
|
upstream.close()
|
|
|
|
|
|
def test_a_failing_target_does_not_block_a_healthy_one(seeded_db):
|
|
admin = admin_session(seeded_db)
|
|
bad = _Upstream(500, {"error": {"message": "down"}})
|
|
good = _Upstream(200, OK_BODY)
|
|
bad_provider, bad_source = _route(admin, bad, "isobad")
|
|
good_provider, good_source = _route(admin, good, "isogood")
|
|
try:
|
|
_chat(bad_source)
|
|
assert _chat(bad_source).status_code == 503
|
|
healthy = _chat(good_source)
|
|
assert healthy.status_code == 200
|
|
assert healthy.json()["choices"][0]["message"]["content"] == "ok"
|
|
assert len(good.calls) == 1
|
|
finally:
|
|
_cleanup(admin, bad_provider, bad_source)
|
|
_cleanup(admin, good_provider, good_source)
|
|
bad.close()
|
|
good.close()
|
|
|
|
|
|
def test_a_malformed_request_trips_once_and_is_not_resent(seeded_db):
|
|
admin = admin_session(seeded_db)
|
|
upstream = _Upstream(400, {"error": {"message": "bad schema"}})
|
|
provider, source = _route(admin, upstream, "badreq")
|
|
try:
|
|
first = _chat(source)
|
|
assert first.status_code == 400
|
|
sent = len(upstream.calls)
|
|
assert sent == 1
|
|
assert _chat(source).status_code == 503
|
|
assert len(upstream.calls) == sent
|
|
finally:
|
|
_cleanup(admin, provider, source)
|
|
upstream.close()
|
|
|
|
|
|
def test_a_shape_fixable_400_does_not_open_the_target(seeded_db):
|
|
admin = admin_session(seeded_db)
|
|
upstream = _Upstream(
|
|
400, {"issues": [{"code": "unrecognized_keys", "keys": ["thinking"]}]}
|
|
)
|
|
provider, source = _route(admin, upstream, "shape")
|
|
try:
|
|
_chat(source)
|
|
sent = len(upstream.calls)
|
|
assert sent >= 2
|
|
again = _chat(source)
|
|
assert again.status_code == 400
|
|
assert len(upstream.calls) > sent
|
|
finally:
|
|
_cleanup(admin, provider, source)
|
|
upstream.close()
|
|
|
|
|
|
def test_the_breaker_is_per_model_not_per_upstream(seeded_db):
|
|
admin = admin_session(seeded_db)
|
|
first = _Upstream(500, {"error": {"message": "down"}})
|
|
second = _Upstream(200, OK_BODY)
|
|
model = _unique_gateway("vendor/shared").lower()
|
|
first_provider, first_source = _route(admin, first, "permodela", target=model)
|
|
second_provider, second_source = _route(admin, second, "permodelb", target=model)
|
|
try:
|
|
_chat(first_source)
|
|
blocked = _chat(second_source)
|
|
assert blocked.status_code == 503
|
|
assert blocked.json()["error"]["type"] == "circuit_open"
|
|
assert second.calls == []
|
|
finally:
|
|
_cleanup(admin, first_provider, first_source)
|
|
_cleanup(admin, second_provider, second_source)
|
|
first.close()
|
|
second.close()
|
|
|
|
|
|
def test_a_healthy_target_stays_closed_and_raises_no_notification(seeded_db):
|
|
admin = admin_session(seeded_db)
|
|
upstream = _Upstream(200, OK_BODY)
|
|
provider, source = _route(admin, upstream, "calm")
|
|
try:
|
|
for _ in range(3):
|
|
assert _chat(source).status_code == 200
|
|
assert len(upstream.calls) == 3
|
|
time.sleep(SYNC_SECONDS)
|
|
assert _admin_notices(upstream.host) == []
|
|
finally:
|
|
_cleanup(admin, provider, source)
|
|
upstream.close()
|