Add OpenCode Zen support, model health/stats dashboard, and gateway fallback fixes

AI gateway: - Add a generic, admin-selectable `client_profile` field on gateway_providers (e.g. "opencode") so a provider needing special request headers (OpenCode Zen's client-identity spoofing) is configured like any other provider, not hardcoded by name. - Track per-(provider, model) reliability/speed/latency health in memory, seeded from the existing gateway_usage_ledger at startup - purely observational, never influences routing. - New Stats tab on /admin/gateway: request volume, latency, per-model breakdowns, and reliability weight, charted with a vendored Chart.js and devplace's own theme tokens. - Record which model a failed request actually fell back to (fallback_used_route), surfaced in the Recent Failures table. - Stop excluding context_length errors from fallback, and skip a primary attempt outright when its known context window is already too small for the estimated request size, going straight to the fallback. - gateway_usage_ledger's provider/fallback_used_route columns and indexes are ensured centrally in database/schema.py's init_db(), the single point of truth for this table's schema. - Non-OpenAI upstream routing and client-model passthrough; trust only the upstream's own X-Gateway-Model header for served-model attribution. Devii agent: - Fix a real lockup: plan/verify tools could be individually disabled via the admin tool toggles while still being required by the protocol gate, permanently bricking any task that needed tools. They can no longer be disabled, and the gate now also checks the tool is actually offered. - Fix compaction being silently calibrated for a 1M-token model while running a much smaller one: context budget is now percentage-based and the summarizer's own request is sized to fit the real model. - Give a specific, actionable retry message when plan()'s own arguments get cut off by the output limit, and tighten its schema to discourage overlong plans. Other: - Backup service: offload completed backups to a remote Hetzner Storage Box. - Container manager: fix orphan blob leaks from sync races, add a two-phase plan/execute `system prune` CLI command. - Admin gateway UI: replace the JS-rendered model/provider tables with server-rendered forms and pages. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DhmEkvutuwtzFVcLbTrhdo
2026-09-09 07:38:24 +02:00
parent b475e7d6ed
commit 569f1dcc64
67 changed files with 6151 additions and 814 deletions
@@ -66,6 +66,32 @@ def test_gateway_page_requires_admin(seeded_db):

 def test_gateway_page_renders_for_admin(seeded_db):
    admin = admin_session(seeded_db)
-    response = admin.get(f"{BASE_URL}/admin/gateway", headers={})
+    response = admin.get(f"{BASE_URL}/admin/gateway", headers={"Accept": "text/html"})
    assert response.status_code == 200
    assert "Gateway routing" in response.text
+
+
+def test_gateway_page_tabs_default_and_select_content(seeded_db):
+    admin = admin_session(seeded_db)
+
+    default_page = admin.get(f"{BASE_URL}/admin/gateway", headers={"Accept": "text/html"})
+    assert "Model routes" in default_page.text
+    assert "GatewayAdmin" not in default_page.text
+
+    providers_page = admin.get(f"{BASE_URL}/admin/gateway?tab=providers", headers={"Accept": "text/html"})
+    assert "Providers" in providers_page.text
+    assert "Model routes" not in providers_page.text
+
+    quota_page = admin.get(f"{BASE_URL}/admin/gateway?tab=quota", headers={"Accept": "text/html"})
+    assert "Quota rules" in quota_page.text
+    assert "Model routes" not in quota_page.text
+
+    unknown_tab_page = admin.get(f"{BASE_URL}/admin/gateway?tab=bogus", headers={"Accept": "text/html"})
+    assert "Model routes" in unknown_tab_page.text
+
+
+def test_gateway_page_json_reports_active_tab(seeded_db):
+    admin = admin_session(seeded_db)
+    response = admin.get(f"{BASE_URL}/admin/gateway?tab=providers")
+    assert response.status_code == 200
+    assert response.json()["tab"] == "providers"
@@ -224,6 +224,113 @@ def test_model_route_fallback_rejects_self_reference(seeded_db):
    assert response.json()["ok"] is False


+def test_model_form_pages_require_admin(seeded_db):
+    assert (
+        requests.get(
+            f"{BASE_URL}/admin/gateway/models/new",
+            headers=JSON_gateway,
+            allow_redirects=False,
+        ).status_code
+        == 401
+    )
+    key = member_key()
+    assert (
+        requests.get(
+            f"{BASE_URL}/admin/gateway/models/new",
+            headers={**JSON_gateway, "X-API-KEY": key},
+            allow_redirects=False,
+        ).status_code
+        == 403
+    )
+
+
+def test_model_add_edit_delete_via_page(seeded_db):
+    admin = admin_session(seeded_db)
+    source = _unique_gateway("modelpage")
+
+    new_page = admin.get(f"{BASE_URL}/admin/gateway/models/new")
+    assert new_page.status_code == 200
+    assert new_page.json()["is_edit"] is False
+
+    created = admin.post(
+        f"{BASE_URL}/admin/gateway/models/new",
+        data={
+            "source_model": source,
+            "target_model": "vendor/page",
+            "kind": "chat",
+            "price_output_per_m": "1.5",
+            "off_peak_start": "22:30",
+            "off_peak_end": "06:00",
+            "off_peak_discount_pct": "10",
+            "is_active": "1",
+        },
+        allow_redirects=False,
+    )
+    assert created.status_code == 302
+    assert created.headers["location"] == "/admin/gateway?tab=models"
+
+    edit_page = admin.get(f"{BASE_URL}/admin/gateway/models/{source}/edit")
+    assert edit_page.status_code == 200
+    form = edit_page.json()["form"]
+    assert form["target_model"] == "vendor/page"
+    assert form["off_peak_start"] == "22:30"
+    assert form["off_peak_end"] == "06:00"
+
+    edit_html = admin.get(f"{BASE_URL}/admin/gateway/models/{source}/edit", headers={"Accept": "text/html"})
+    assert f'value="{source}"' in edit_html.text
+    assert "readonly" in edit_html.text
+
+    updated = admin.post(
+        f"{BASE_URL}/admin/gateway/models/{source}/edit",
+        data={"target_model": "vendor/page2", "kind": "chat", "is_active": "1"},
+        allow_redirects=False,
+    )
+    assert updated.status_code == 302
+
+    relisted = admin.get(f"{BASE_URL}/admin/gateway/models").json()
+    row = next(m for m in relisted["models"] if m["source_model"] == source)
+    assert row["target_model"] == "vendor/page2"
+
+    deleted = admin.post(
+        f"{BASE_URL}/admin/gateway/models/{source}/delete", allow_redirects=False
+    )
+    assert deleted.status_code == 302
+    assert admin.get(f"{BASE_URL}/admin/gateway/models/{source}/edit").status_code == 404
+
+
+def test_model_page_fallback_must_be_the_same_kind(seeded_db):
+    admin = admin_session(seeded_db)
+    embed_route = _unique_gateway("fbpage-embed")
+    chat_route = _unique_gateway("fbpage-chat")
+
+    admin.post(
+        f"{BASE_URL}/admin/gateway/models",
+        json={"source_model": embed_route, "target_model": "vendor/e", "kind": "embed"},
+    )
+    response = admin.post(
+        f"{BASE_URL}/admin/gateway/models/new",
+        data={
+            "source_model": chat_route,
+            "target_model": "vendor/c",
+            "kind": "chat",
+            "fallback_model": embed_route,
+        },
+    )
+    assert response.status_code == 400
+    assert "message" in response.json()["error"]
+
+    admin.delete(f"{BASE_URL}/admin/gateway/models/{embed_route}")
+
+
+def test_model_edit_page_404_for_missing_model(seeded_db):
+    admin = admin_session(seeded_db)
+    missing = _unique_gateway("ghostmodel")
+    assert admin.get(f"{BASE_URL}/admin/gateway/models/{missing}/edit").status_code == 404
+    assert (
+        admin.post(f"{BASE_URL}/admin/gateway/models/{missing}/delete").status_code == 404
+    )
+
+
 def test_model_route_validation(seeded_db):
    admin = admin_session(seeded_db)
    missing_target = admin.post(
@@ -0,0 +1,120 @@
+# retoor <retoor@molodetz.nl>
+
+import http.server
+import json
+import socket
+import socketserver
+import threading
+
+import requests
+
+from tests.conftest import BASE_URL
+from tests.api.admin.gateway.index import (
+    JSON_gateway,
+    admin_session,
+    member_key,
+    _unique_gateway,
+)
+
+
+def _fake_upstream(status_code, payload):
+    sock = socket.socket()
+    sock.bind(("127.0.0.1", 0))
+    port = sock.getsockname()[1]
+    sock.close()
+    body = json.dumps(payload).encode() if payload is not None else b""
+    seen_auth = {}
+
+    class Handler(http.server.BaseHTTPRequestHandler):
+        def do_GET(self):
+            seen_auth["path"] = self.path
+            seen_auth["authorization"] = self.headers.get("Authorization")
+            self.send_response(status_code)
+            self.send_header("Content-Type", "application/json")
+            self.end_headers()
+            self.wfile.write(body)
+
+        def log_message(self, *args):
+            pass
+
+    httpd = socketserver.TCPServer(("127.0.0.1", port), Handler)
+    thread = threading.Thread(target=httpd.serve_forever, daemon=True)
+    thread.start()
+    return httpd, port, seen_auth
+
+
+def test_provider_models_requires_admin(seeded_db):
+    assert (
+        requests.get(
+            f"{BASE_URL}/admin/gateway/provider-models",
+            headers=JSON_gateway,
+            allow_redirects=False,
+        ).status_code
+        == 401
+    )
+    key = member_key()
+    assert (
+        requests.get(
+            f"{BASE_URL}/admin/gateway/provider-models",
+            headers={**JSON_gateway, "X-API-KEY": key},
+            allow_redirects=False,
+        ).status_code
+        == 403
+    )
+
+
+def test_provider_models_returns_the_list_on_success(seeded_db):
+    admin = admin_session(seeded_db)
+    name = _unique_gateway("modelsupstream").lower()
+    httpd, port, seen = _fake_upstream(
+        200, {"data": [{"id": "vendor/a"}, {"id": "vendor/b"}]}
+    )
+    try:
+        admin.post(
+            f"{BASE_URL}/admin/gateway/providers",
+            json={
+                "name": name,
+                "base_url": f"http://127.0.0.1:{port}/v1/chat/completions",
+                "api_key": "sk-fake",
+            },
+        )
+        response = admin.get(
+            f"{BASE_URL}/admin/gateway/provider-models", params={"provider": name}
+        )
+        assert response.status_code == 200
+        assert response.json()["models"] == ["vendor/a", "vendor/b"]
+        assert seen["path"] == "/v1/models"
+        assert seen["authorization"] == "Bearer sk-fake"
+    finally:
+        httpd.shutdown()
+        admin.delete(f"{BASE_URL}/admin/gateway/providers/{name}")
+
+
+def test_provider_models_404_when_upstream_has_no_model_listing(seeded_db):
+    admin = admin_session(seeded_db)
+    name = _unique_gateway("modelsupstream404").lower()
+    httpd, port, _ = _fake_upstream(404, {})
+    try:
+        admin.post(
+            f"{BASE_URL}/admin/gateway/providers",
+            json={
+                "name": name,
+                "base_url": f"http://127.0.0.1:{port}/v1/chat/completions",
+            },
+        )
+        response = admin.get(
+            f"{BASE_URL}/admin/gateway/provider-models", params={"provider": name}
+        )
+        assert response.status_code == 404
+    finally:
+        httpd.shutdown()
+        admin.delete(f"{BASE_URL}/admin/gateway/providers/{name}")
+
+
+def test_provider_models_404_for_unknown_provider(seeded_db):
+    admin = admin_session(seeded_db)
+    response = admin.get(
+        f"{BASE_URL}/admin/gateway/provider-models",
+        params={"provider": _unique_gateway("ghostprovider")},
+    )
+    assert response.status_code == 404
@@ -82,3 +82,115 @@ def test_provider_name_validation(seeded_db):
    )
    assert bad.status_code == 400
    assert bad.json()["ok"] is False
+
+
+def test_provider_form_pages_require_admin(seeded_db):
+    name = _unique_gateway("provpage").lower()
+    assert (
+        requests.get(
+            f"{BASE_URL}/admin/gateway/providers/new",
+            headers=JSON_gateway,
+            allow_redirects=False,
+        ).status_code
+        == 401
+    )
+    key = member_key()
+    assert (
+        requests.get(
+            f"{BASE_URL}/admin/gateway/providers/new",
+            headers={**JSON_gateway, "X-API-KEY": key},
+            allow_redirects=False,
+        ).status_code
+        == 403
+    )
+    assert (
+        requests.post(
+            f"{BASE_URL}/admin/gateway/providers/new",
+            data={"name": name},
+            headers={**JSON_gateway, "X-API-KEY": key},
+            allow_redirects=False,
+        ).status_code
+        == 403
+    )
+
+
+def test_provider_add_edit_delete_via_page(seeded_db):
+    admin = admin_session(seeded_db)
+    name = _unique_gateway("provpage").lower()
+
+    new_page = admin.get(f"{BASE_URL}/admin/gateway/providers/new")
+    assert new_page.status_code == 200
+    assert new_page.json()["is_edit"] is False
+
+    created = admin.post(
+        f"{BASE_URL}/admin/gateway/providers/new",
+        data={
+            "name": name,
+            "base_url": "https://page.example/v1/chat/completions",
+            "api_key": "sk-page",
+            "is_active": "1",
+        },
+        allow_redirects=False,
+    )
+    assert created.status_code == 302
+    assert created.headers["location"] == "/admin/gateway?tab=providers"
+
+    edit_page = admin.get(f"{BASE_URL}/admin/gateway/providers/{name}/edit")
+    assert edit_page.status_code == 200
+    edit_body = edit_page.json()
+    assert edit_body["is_edit"] is True
+    assert edit_body["form"]["base_url"] == "https://page.example/v1/chat/completions"
+
+    edit_html = admin.get(f"{BASE_URL}/admin/gateway/providers/{name}/edit", headers={"Accept": "text/html"})
+    assert f'value="{name}"' in edit_html.text
+    assert "readonly" in edit_html.text
+
+    updated = admin.post(
+        f"{BASE_URL}/admin/gateway/providers/{name}/edit",
+        data={"base_url": "https://page2.example/v1/chat/completions", "is_active": "0"},
+        allow_redirects=False,
+    )
+    assert updated.status_code == 302
+
+    relisted = admin.get(f"{BASE_URL}/admin/gateway/providers").json()
+    match = next(p for p in relisted["providers"] if p["name"] == name)
+    assert match["base_url"] == "https://page2.example/v1/chat/completions"
+    assert match["is_active"] is False
+
+    deleted = admin.post(
+        f"{BASE_URL}/admin/gateway/providers/{name}/delete", allow_redirects=False
+    )
+    assert deleted.status_code == 302
+    assert admin.get(f"{BASE_URL}/admin/gateway/providers/{name}/edit").status_code == 404
+
+
+def test_provider_page_validation_error_rerenders_with_message(seeded_db):
+    admin = admin_session(seeded_db)
+    response = admin.post(
+        f"{BASE_URL}/admin/gateway/providers/new",
+        data={"name": "has spaces!"},
+    )
+    assert response.status_code == 400
+    assert "message" in response.json()["error"]
+
+
+def test_provider_page_validation_error_shows_banner_in_html(seeded_db):
+    admin = admin_session(seeded_db)
+    response = admin.post(
+        f"{BASE_URL}/admin/gateway/providers/new",
+        data={"name": "has spaces!"},
+        headers={"Accept": "text/html"},
+    )
+    assert response.status_code == 400
+    assert "gw-error" in response.text
+    assert "letters, numbers, hyphen, underscore" in response.text
+
+
+def test_provider_edit_page_404_for_missing_provider(seeded_db):
+    admin = admin_session(seeded_db)
+    missing = _unique_gateway("ghostprov").lower()
+    assert admin.get(f"{BASE_URL}/admin/gateway/providers/{missing}/edit").status_code == 404
+    assert (
+        admin.post(f"{BASE_URL}/admin/gateway/providers/{missing}/delete").status_code
+        == 404
+    )
@@ -11,6 +11,7 @@ from tests.api.admin.gateway.index import (
    JSON_gateway,
    admin_session,
    member_key,
+    _unique_gateway,
 )
 from tests.conftest import BASE_URL

@@ -160,3 +161,121 @@ def test_spend_recorded_after_a_reset_counts_again(seeded_db):
    )
    _burn(owner, "appa", 0.5)
    assert _spent(owner, "appa") == 0.5
+
+
+def test_quota_rule_form_pages_require_admin(seeded_db):
+    assert (
+        requests.get(
+            f"{BASE_URL}/admin/gateway/quota-rules/new",
+            headers=JSON_gateway,
+            allow_redirects=False,
+        ).status_code
+        == 401
+    )
+    key = member_key()
+    assert (
+        requests.get(
+            f"{BASE_URL}/admin/gateway/quota-rules/new",
+            headers={**JSON_gateway, "X-API-KEY": key},
+            allow_redirects=False,
+        ).status_code
+        == 403
+    )
+
+
+def test_quota_rule_add_edit_delete_reset_via_page(seeded_db):
+    admin = admin_session(seeded_db)
+    owner = _owner()
+    app_ref = _unique_gateway("qrapp").lower()
+
+    new_page = admin.get(f"{BASE_URL}/admin/gateway/quota-rules/new")
+    assert new_page.status_code == 200
+    assert new_page.json()["is_edit"] is False
+
+    created = admin.post(
+        f"{BASE_URL}/admin/gateway/quota-rules/new",
+        data={
+            "owner_kind": "user",
+            "owner_id": owner,
+            "app_reference": app_ref,
+            "limit_usd": "2.5",
+            "is_active": "1",
+            "label": "page test",
+        },
+        allow_redirects=False,
+    )
+    assert created.status_code == 302
+    assert created.headers["location"] == "/admin/gateway?tab=quota"
+
+    rules = admin.get(f"{BASE_URL}/admin/gateway/quota-rules").json()["rules"]
+    rule = next(r for r in rules if r["owner_id"] == owner and r["app_reference"] == app_ref)
+    uid = rule["uid"]
+    assert rule["limit_usd"] == 2.5
+
+    edit_page = admin.get(f"{BASE_URL}/admin/gateway/quota-rules/{uid}/edit")
+    assert edit_page.status_code == 200
+    assert edit_page.json()["form"]["limit_usd"] == "2.5"
+
+    updated = admin.post(
+        f"{BASE_URL}/admin/gateway/quota-rules/{uid}/edit",
+        data={
+            "owner_kind": "user",
+            "owner_id": owner,
+            "app_reference": app_ref,
+            "limit_usd": "5.0",
+            "is_active": "1",
+            "label": "page test updated",
+        },
+        allow_redirects=False,
+    )
+    assert updated.status_code == 302
+
+    rules = admin.get(f"{BASE_URL}/admin/gateway/quota-rules").json()["rules"]
+    rule = next(r for r in rules if r["uid"] == uid)
+    assert rule["limit_usd"] == 5.0
+    assert rule["label"] == "page test updated"
+
+    _burn(owner, app_ref, 1.0)
+    assert _spent(owner, app_ref) == 1.0
+    reset_response = admin.post(
+        f"{BASE_URL}/admin/gateway/quota-rules/{uid}/reset", allow_redirects=False
+    )
+    assert reset_response.status_code == 302
+    assert _spent(owner, app_ref) == 0.0
+
+    deleted = admin.post(
+        f"{BASE_URL}/admin/gateway/quota-rules/{uid}/delete", allow_redirects=False
+    )
+    assert deleted.status_code == 302
+    assert (
+        admin.get(f"{BASE_URL}/admin/gateway/quota-rules/{uid}/edit").status_code == 404
+    )
+
+
+def test_quota_rule_page_requires_a_dimension(seeded_db):
+    admin = admin_session(seeded_db)
+    response = admin.post(
+        f"{BASE_URL}/admin/gateway/quota-rules/new",
+        data={"limit_usd": "1.0"},
+    )
+    assert response.status_code == 400
+    assert "message" in response.json()["error"]
+
+
+def test_quota_rule_edit_page_404_for_missing_rule(seeded_db):
+    admin = admin_session(seeded_db)
+    from devplacepy.utils import generate_uid
+
+    missing = generate_uid()
+    assert (
+        admin.get(f"{BASE_URL}/admin/gateway/quota-rules/{missing}/edit").status_code
+        == 404
+    )
+    assert (
+        admin.post(f"{BASE_URL}/admin/gateway/quota-rules/{missing}/delete").status_code
+        == 404
+    )
+    assert (
+        admin.post(f"{BASE_URL}/admin/gateway/quota-rules/{missing}/reset").status_code
+        == 404
+    )
@@ -0,0 +1,82 @@
+# retoor <retoor@molodetz.nl>
+
+import requests
+
+from tests.api.admin.gateway.index import JSON_gateway, admin_session, member_key
+from tests.conftest import BASE_URL
+
+
+def test_stats_tab_renders_for_admin(seeded_db):
+    admin = admin_session(seeded_db)
+    response = admin.get(
+        f"{BASE_URL}/admin/gateway?tab=stats", headers={"Accept": "text/html"}
+    )
+    assert response.status_code == 200
+    assert "Model pool" in response.text
+    assert "Model routes" not in response.text
+
+
+def test_stats_data_requires_admin(seeded_db):
+    assert (
+        requests.get(
+            f"{BASE_URL}/admin/gateway/stats/data", headers=JSON_gateway, allow_redirects=False
+        ).status_code
+        == 401
+    )
+    key = member_key()
+    assert (
+        requests.get(
+            f"{BASE_URL}/admin/gateway/stats/data",
+            headers={**JSON_gateway, "X-API-KEY": key},
+            allow_redirects=False,
+        ).status_code
+        == 403
+    )
+
+
+def test_stats_data_shape_for_admin(seeded_db):
+    admin = admin_session(seeded_db)
+    response = admin.get(f"{BASE_URL}/admin/gateway/stats/data?range=24h", headers=JSON_gateway)
+    assert response.status_code == 200
+    data = response.json()
+    for key in (
+        "totals",
+        "per_model",
+        "per_endpoint",
+        "per_provider",
+        "timeseries",
+        "status_codes",
+        "streaming_split",
+        "failure_reasons",
+        "hourly_distribution",
+        "recent_failures",
+        "latency_histogram",
+        "tokens_per_second_histogram",
+        "bucket_seconds",
+    ):
+        assert key in data
+
+
+def test_stats_data_rejects_unknown_range(seeded_db):
+    admin = admin_session(seeded_db)
+    response = admin.get(f"{BASE_URL}/admin/gateway/stats/data?range=nonsense", headers=JSON_gateway)
+    assert response.status_code == 400
+
+
+def test_stats_models_returns_a_list(seeded_db):
+    admin = admin_session(seeded_db)
+    response = admin.get(f"{BASE_URL}/admin/gateway/stats/models", headers=JSON_gateway)
+    assert response.status_code == 200
+    assert isinstance(response.json()["models"], list)
+
+
+def test_stats_model_detail_for_an_unknown_model(seeded_db):
+    admin = admin_session(seeded_db)
+    response = admin.get(
+        f"{BASE_URL}/admin/gateway/stats/model/default/never-called-model?range=24h",
+        headers=JSON_gateway,
+    )
+    assert response.status_code == 200
+    data = response.json()
+    assert data["summary"]["total_requests"] == 0
+    assert data["health"] is None
@@ -0,0 +1,115 @@
+# retoor <retoor@molodetz.nl>
+
+import time
+import pytest
+import requests
+from tests.conftest import BASE_URL
+from devplacepy.database import clear_settings_cache, get_table, refresh_snapshot
+from devplacepy.services.devii import tool_prefs
+
+JSON = {"Accept": "application/json"}
+_counter = [0]
+
+
+@pytest.fixture(scope="module", autouse=True)
+def _settings(app_server):
+    from devplacepy.database import set_setting
+
+    set_setting("rate_limit_per_minute", "1000000")
+    set_setting("registration_open", "1")
+    yield
+
+
+def _admin(seeded_db):
+    refresh_snapshot()
+    key = get_table("users").find_one(username="alice_test")["api_key"]
+    s = requests.Session()
+    s.headers.update({"X-API-KEY": key, **JSON})
+    return s
+
+
+def _member():
+    _counter[0] += 1
+    name = f"dtmem{int(time.time() * 1000)}{_counter[0]}"
+    requests.post(
+        f"{BASE_URL}/auth/signup",
+        data={
+            "username": name,
+            "email": f"{name}@t.dev",
+            "password": "secret123",
+            "confirm_password": "secret123",
+            "birth_date": "1990-01-01",
+            "accept_terms": "1",
+        },
+        allow_redirects=True,
+    )
+    refresh_snapshot()
+    s = requests.Session()
+    s.headers.update(
+        {"X-API-KEY": get_table("users").find_one(username=name)["api_key"], **JSON}
+    )
+    return s
+
+
+def _all_tool_names():
+    return set(tool_prefs.GROUPS_BY_TOOL_NAME)
+
+
+def test_devii_service_page_has_tools_tab(seeded_db):
+    admin = _admin(seeded_db)
+    r = admin.get(f"{BASE_URL}/admin/services/devii")
+    assert r.status_code == 200
+    assert 'data-tab="tools"' in r.text
+    assert "devii-tools-form" in r.text
+
+
+def test_other_service_page_has_no_tools_tab(seeded_db):
+    admin = _admin(seeded_db)
+    r = admin.get(f"{BASE_URL}/admin/services/news")
+    assert r.status_code == 200
+    assert 'data-tab="tools"' not in r.text
+
+
+def test_admin_can_disable_and_reenable_a_tool(seeded_db):
+    admin = _admin(seeded_db)
+    all_names = _all_tool_names()
+    try:
+        enabled = sorted(all_names - {"create_post"})
+        r = admin.post(
+            f"{BASE_URL}/admin/services/devii/tools",
+            data=[("enabled", name) for name in enabled],
+        )
+        assert r.status_code == 200, r.text[:300]
+        assert r.json()["ok"] is True
+        clear_settings_cache()
+        assert tool_prefs.disabled_tool_names() == frozenset({"create_post"})
+
+        r2 = admin.post(
+            f"{BASE_URL}/admin/services/devii/tools",
+            data=[("enabled", name) for name in all_names],
+        )
+        assert r2.status_code == 200
+        clear_settings_cache()
+        assert tool_prefs.disabled_tool_names() == frozenset()
+    finally:
+        clear_settings_cache()
+        tool_prefs.set_disabled_tool_names(set())
+
+
+def test_member_cannot_save_tool_config(app_server):
+    member = _member()
+    r = member.post(
+        f"{BASE_URL}/admin/services/devii/tools",
+        data={"enabled": "create_post"},
+        allow_redirects=False,
+    )
+    assert r.status_code in (302, 303, 403)
+
+
+def test_guest_cannot_save_tool_config(app_server):
+    r = requests.post(
+        f"{BASE_URL}/admin/services/devii/tools",
+        data={"enabled": "create_post"},
+        allow_redirects=False,
+    )
+    assert r.status_code in (302, 303, 401)