forked from retoor/devplacepy
Admin pages: one shared specification, builders, fragments, caps and live refresh
Adds admin_sections registry, shared admin partials, WorkspaceList, admin CLAUDE.md spec, capped node/flag listings, and api/unit tests enforcing the spec. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,251 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import http.server
|
||||
import json
|
||||
import socket
|
||||
import socketserver
|
||||
import threading
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
from devplacepy.database import get_table, refresh_snapshot
|
||||
from devplacepy.database.settings import internal_gateway_key
|
||||
from tests.api.admin.gateway.index import (
|
||||
_unique_gateway,
|
||||
admin_session,
|
||||
)
|
||||
from tests.conftest import BASE_URL
|
||||
|
||||
CHAT_URL = f"{BASE_URL}/openai/v1/chat/completions"
|
||||
NOTICE_MARKER = "tripped its circuit breaker"
|
||||
SYNC_SECONDS = 1.5
|
||||
NOTICE_WAIT_SECONDS = 10
|
||||
OK_BODY = {
|
||||
"id": "x",
|
||||
"model": "m",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
|
||||
|
||||
class _Upstream:
|
||||
def __init__(self, status, payload):
|
||||
self.calls = []
|
||||
calls = self.calls
|
||||
body = json.dumps(payload).encode()
|
||||
code = status
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
calls.append(json.loads(self.rfile.read(length) or b"{}"))
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
sock = socket.socket()
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
self.port = sock.getsockname()[1]
|
||||
sock.close()
|
||||
self.httpd = socketserver.ThreadingTCPServer(("127.0.0.1", self.port), Handler)
|
||||
threading.Thread(target=self.httpd.serve_forever, daemon=True).start()
|
||||
|
||||
@property
|
||||
def host(self):
|
||||
return f"127.0.0.1:{self.port}"
|
||||
|
||||
def close(self):
|
||||
self.httpd.shutdown()
|
||||
self.httpd.server_close()
|
||||
|
||||
|
||||
def _internal_headers():
|
||||
refresh_snapshot()
|
||||
return {"Authorization": f"Bearer {internal_gateway_key()}", "Content-Type": "application/json"}
|
||||
|
||||
|
||||
def _route(admin, upstream, prefix, target=None):
|
||||
provider = _unique_gateway(f"{prefix}p").lower()
|
||||
source = _unique_gateway(f"{prefix}m").lower()
|
||||
target = target or f"vendor/{prefix}"
|
||||
admin.post(
|
||||
f"{BASE_URL}/admin/gateway/providers",
|
||||
json={"name": provider, "base_url": f"http://{upstream.host}/v1/chat/completions", "api_key": "sk-test"},
|
||||
)
|
||||
created = admin.post(
|
||||
f"{BASE_URL}/admin/gateway/models",
|
||||
json={"source_model": source, "provider": provider, "target_model": target, "kind": "chat"},
|
||||
)
|
||||
assert created.status_code == 200, created.text[:300]
|
||||
return provider, source
|
||||
|
||||
|
||||
def _cleanup(admin, provider, source):
|
||||
admin.delete(f"{BASE_URL}/admin/gateway/models/{source}")
|
||||
admin.delete(f"{BASE_URL}/admin/gateway/providers/{provider}")
|
||||
|
||||
|
||||
def _chat(source):
|
||||
return requests.post(
|
||||
CHAT_URL,
|
||||
headers=_internal_headers(),
|
||||
json={"model": source, "messages": [{"role": "user", "content": "hi"}]},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
|
||||
def _admin_notices(host):
|
||||
refresh_snapshot()
|
||||
rows = get_table("notifications").find(type="system")
|
||||
return [r for r in rows if NOTICE_MARKER in (r.get("message") or "") and host in r["message"]]
|
||||
|
||||
|
||||
def _wait_for_notices(host):
|
||||
deadline = time.time() + NOTICE_WAIT_SECONDS
|
||||
while time.time() < deadline:
|
||||
found = _admin_notices(host)
|
||||
if found:
|
||||
return found
|
||||
time.sleep(0.5)
|
||||
return []
|
||||
|
||||
|
||||
def test_first_upstream_error_opens_the_target_and_the_second_call_never_leaves(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
upstream = _Upstream(500, {"error": {"message": "provider exploded"}})
|
||||
provider, source = _route(admin, upstream, "trip")
|
||||
try:
|
||||
first = _chat(source)
|
||||
assert first.status_code >= 500
|
||||
sent_after_first = len(upstream.calls)
|
||||
assert sent_after_first >= 1
|
||||
second = _chat(source)
|
||||
assert second.status_code == 503
|
||||
assert second.json()["error"]["type"] == "circuit_open"
|
||||
assert len(upstream.calls) == sent_after_first
|
||||
finally:
|
||||
_cleanup(admin, provider, source)
|
||||
upstream.close()
|
||||
|
||||
|
||||
def test_every_admin_gets_one_detailed_notification_per_incident(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
upstream = _Upstream(500, {"error": {"message": "provider exploded"}})
|
||||
provider, source = _route(admin, upstream, "notify")
|
||||
try:
|
||||
_chat(source)
|
||||
_chat(source)
|
||||
_chat(source)
|
||||
notices = _wait_for_notices(upstream.host)
|
||||
assert notices, "no system notification reached the admins"
|
||||
admins = {u["uid"] for u in get_table("users").find(role="Admin")}
|
||||
per_admin = {}
|
||||
for row in notices:
|
||||
per_admin[row["user_uid"]] = per_admin.get(row["user_uid"], 0) + 1
|
||||
assert set(per_admin) == admins
|
||||
assert set(per_admin.values()) == {1}
|
||||
message = notices[0]["message"]
|
||||
assert "vendor/notify" in message
|
||||
assert "Error class: upstream_error" in message
|
||||
assert "HTTP status: 500" in message
|
||||
assert "provider exploded" in message
|
||||
assert "sk-test" not in message
|
||||
assert notices[0]["target_url"] == "/admin/services/openai"
|
||||
finally:
|
||||
_cleanup(admin, provider, source)
|
||||
upstream.close()
|
||||
|
||||
|
||||
def test_a_failing_target_does_not_block_a_healthy_one(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
bad = _Upstream(500, {"error": {"message": "down"}})
|
||||
good = _Upstream(200, OK_BODY)
|
||||
bad_provider, bad_source = _route(admin, bad, "isobad")
|
||||
good_provider, good_source = _route(admin, good, "isogood")
|
||||
try:
|
||||
_chat(bad_source)
|
||||
assert _chat(bad_source).status_code == 503
|
||||
healthy = _chat(good_source)
|
||||
assert healthy.status_code == 200
|
||||
assert healthy.json()["choices"][0]["message"]["content"] == "ok"
|
||||
assert len(good.calls) == 1
|
||||
finally:
|
||||
_cleanup(admin, bad_provider, bad_source)
|
||||
_cleanup(admin, good_provider, good_source)
|
||||
bad.close()
|
||||
good.close()
|
||||
|
||||
|
||||
def test_a_malformed_request_trips_once_and_is_not_resent(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
upstream = _Upstream(400, {"error": {"message": "bad schema"}})
|
||||
provider, source = _route(admin, upstream, "badreq")
|
||||
try:
|
||||
first = _chat(source)
|
||||
assert first.status_code == 400
|
||||
sent = len(upstream.calls)
|
||||
assert sent == 1
|
||||
assert _chat(source).status_code == 503
|
||||
assert len(upstream.calls) == sent
|
||||
finally:
|
||||
_cleanup(admin, provider, source)
|
||||
upstream.close()
|
||||
|
||||
|
||||
def test_a_shape_fixable_400_does_not_open_the_target(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
upstream = _Upstream(
|
||||
400, {"issues": [{"code": "unrecognized_keys", "keys": ["thinking"]}]}
|
||||
)
|
||||
provider, source = _route(admin, upstream, "shape")
|
||||
try:
|
||||
_chat(source)
|
||||
sent = len(upstream.calls)
|
||||
assert sent >= 2
|
||||
again = _chat(source)
|
||||
assert again.status_code == 400
|
||||
assert len(upstream.calls) > sent
|
||||
finally:
|
||||
_cleanup(admin, provider, source)
|
||||
upstream.close()
|
||||
|
||||
|
||||
def test_the_breaker_is_per_model_not_per_upstream(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
first = _Upstream(500, {"error": {"message": "down"}})
|
||||
second = _Upstream(200, OK_BODY)
|
||||
model = _unique_gateway("vendor/shared").lower()
|
||||
first_provider, first_source = _route(admin, first, "permodela", target=model)
|
||||
second_provider, second_source = _route(admin, second, "permodelb", target=model)
|
||||
try:
|
||||
_chat(first_source)
|
||||
blocked = _chat(second_source)
|
||||
assert blocked.status_code == 503
|
||||
assert blocked.json()["error"]["type"] == "circuit_open"
|
||||
assert second.calls == []
|
||||
finally:
|
||||
_cleanup(admin, first_provider, first_source)
|
||||
_cleanup(admin, second_provider, second_source)
|
||||
first.close()
|
||||
second.close()
|
||||
|
||||
|
||||
def test_a_healthy_target_stays_closed_and_raises_no_notification(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
upstream = _Upstream(200, OK_BODY)
|
||||
provider, source = _route(admin, upstream, "calm")
|
||||
try:
|
||||
for _ in range(3):
|
||||
assert _chat(source).status_code == 200
|
||||
assert len(upstream.calls) == 3
|
||||
time.sleep(SYNC_SECONDS)
|
||||
assert _admin_notices(upstream.host) == []
|
||||
finally:
|
||||
_cleanup(admin, provider, source)
|
||||
upstream.close()
|
||||
@@ -80,3 +80,54 @@ def test_stats_model_detail_for_an_unknown_model(seeded_db):
|
||||
data = response.json()
|
||||
assert data["summary"]["total_requests"] == 0
|
||||
assert data["health"] is None
|
||||
|
||||
|
||||
def test_stats_tab_first_response_is_rendered(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.get(
|
||||
f"{BASE_URL}/admin/gateway?tab=stats", headers={"Accept": "text/html"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert "Loading..." not in response.text
|
||||
assert 'id="gw-stats-initial"' in response.text
|
||||
assert 'data-slot="tiles"' in response.text
|
||||
assert "/vendor/chartjs/chart.umd.min.js" in response.text
|
||||
assert "js/vendor/chart.js" not in response.text
|
||||
|
||||
|
||||
def test_stats_tab_honors_range_param(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.get(
|
||||
f"{BASE_URL}/admin/gateway?tab=stats&range=7d", headers={"Accept": "text/html"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert '<option value="7d" selected>' in response.text
|
||||
|
||||
|
||||
def test_stats_tab_falls_back_on_unknown_range(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.get(
|
||||
f"{BASE_URL}/admin/gateway?tab=stats&range=nonsense", headers={"Accept": "text/html"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert '<option value="24h" selected>' in response.text
|
||||
|
||||
|
||||
def test_stats_fragment_renders_slots(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.get(f"{BASE_URL}/admin/gateway/stats/fragment?range=24h")
|
||||
assert response.status_code == 200
|
||||
for slot in ("tiles", "models", "failures", "note"):
|
||||
assert f'data-slot="{slot}"' in response.text
|
||||
assert 'id="gw-stats-initial"' in response.text
|
||||
|
||||
|
||||
def test_stats_fragment_rejects_unknown_range(seeded_db):
|
||||
admin = admin_session(seeded_db)
|
||||
response = admin.get(f"{BASE_URL}/admin/gateway/stats/fragment?range=nonsense")
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
def test_stats_fragment_requires_admin(seeded_db):
|
||||
response = requests.get(f"{BASE_URL}/admin/gateway/stats/fragment", allow_redirects=False)
|
||||
assert response.status_code in (302, 303, 401, 403)
|
||||
|
||||
Reference in New Issue
Block a user