Adds admin_sections registry, shared admin partials, WorkspaceList, admin CLAUDE.md spec, capped node/flag listings, and api/unit tests enforcing the spec. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
474 lines
18 KiB
Python
474 lines
18 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import json
|
|
import logging
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from typing import Annotated
|
|
|
|
from fastapi import APIRouter, Depends, Request
|
|
from fastapi.responses import HTMLResponse
|
|
|
|
from devplacepy.database import get_int_setting, get_setting, get_table, get_users_by_uids, set_setting
|
|
from devplacepy.database.nodes import (
|
|
ADMIN_LIST_LIMIT,
|
|
DEVICE_BREAKDOWN_COLUMNS,
|
|
LIMITS,
|
|
add_event,
|
|
count_all_devices,
|
|
count_flags,
|
|
count_open_requests,
|
|
device_breakdown,
|
|
cache_stats,
|
|
effective_status,
|
|
get_device,
|
|
lifetime_counters,
|
|
list_flags,
|
|
load_circuits,
|
|
recent_devices,
|
|
request_status_counts,
|
|
resolve_flag,
|
|
set_class_override,
|
|
suspend_device,
|
|
unsuspend_device,
|
|
usage_day_totals,
|
|
)
|
|
from devplacepy.dependencies import json_or_form
|
|
from devplacepy.models import (
|
|
AdminNodeClassForm,
|
|
AdminNodeKillswitchForm,
|
|
AdminNodeSettingsForm,
|
|
AdminNodeSuspendForm,
|
|
)
|
|
from devplacepy.responses import action_result, json_error, respond
|
|
from devplacepy.routers.admin._shared import deny_senior, is_senior_admin
|
|
from devplacepy.routers.nodes.social import SOCIAL_DAILY_DEFAULT
|
|
from devplacepy.schemas import (
|
|
AdminNodeCircuitOut,
|
|
AdminNodeDeviceOut,
|
|
AdminNodeFlagOut,
|
|
AdminNodeOutcomeOut,
|
|
AdminNodePoolOut,
|
|
AdminNodeQueueOut,
|
|
AdminNodeSettingOut,
|
|
AdminNodesOut,
|
|
)
|
|
from devplacepy.seo import base_seo_context, site_url, website_schema
|
|
from devplacepy.services.audit import record as audit
|
|
from devplacepy.services.base import BaseService
|
|
from devplacepy.services.nodes.circuit import CircuitBreaker
|
|
from devplacepy.services.nodes.geo import shared_geo
|
|
from devplacepy.services.nodes.registry import OUTCOME_PREFIX
|
|
from devplacepy.utils import not_found, require_admin
|
|
|
|
logger = logging.getLogger(__name__)
|
|
router = APIRouter()
|
|
|
|
LIST_URL = "/admin/nodes"
|
|
SERVICE_NAME = "nodes"
|
|
KILLSWITCH_KEY = "nodes_enabled"
|
|
QUOTA_USER_CEILING = 10_000
|
|
QUOTA_GUEST_CEILING = 1_000
|
|
QUOTA_SOCIAL_CEILING = 10_000
|
|
POOL_STATS = (
|
|
("connected", "connected"),
|
|
("capacity", "capacity"),
|
|
("accepted", "accepted"),
|
|
("kicked", "kicked"),
|
|
("jobs done", "jobs_done"),
|
|
("jobs failed", "jobs_failed"),
|
|
("open circuits", "open_circuits"),
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class NodeSettingField:
|
|
key: str
|
|
label: str
|
|
default: int
|
|
ceiling: int | None
|
|
kind: str = "int"
|
|
|
|
|
|
SETTING_FIELDS: tuple[NodeSettingField, ...] = (
|
|
NodeSettingField("nodes_max_total", "Maximum connected nodes", LIMITS.max_connected_nodes, LIMITS.max_connected_nodes),
|
|
NodeSettingField("nodes_daily_quota_user", "Daily node requests per member", LIMITS.default_daily_requests_user, QUOTA_USER_CEILING),
|
|
NodeSettingField("nodes_daily_quota_guest", "Daily node requests per guest", LIMITS.default_daily_requests_guest, QUOTA_GUEST_CEILING),
|
|
NodeSettingField("nodes_cache_ttl_seconds", "Result cache lifetime (seconds)", LIMITS.cache_ttl_seconds, LIMITS.cache_ttl_seconds),
|
|
NodeSettingField("nodes_daily_social_user", "Daily social searches per member", SOCIAL_DAILY_DEFAULT, QUOTA_SOCIAL_CEILING),
|
|
NodeSettingField("nodes_social_via_nodes", "Route social search through nodes", 0, None, "flag"),
|
|
NodeSettingField("nodes_allow_unknown_class", "Allow donors whose address class is unknown", 0, None, "flag"),
|
|
NodeSettingField("nodes_redundancy_percent", "Redundancy sample of fetch jobs (percent)", LIMITS.redundancy_percent, LIMITS.redundancy_max_percent),
|
|
)
|
|
SETTING_FIELDS_BY_KEY = {field.key: field for field in SETTING_FIELDS}
|
|
|
|
|
|
def kill_switch_on() -> bool:
|
|
return (get_setting(KILLSWITCH_KEY, "0") or "0") == "1"
|
|
|
|
|
|
def pool_view() -> AdminNodePoolOut:
|
|
row = get_table("service_state").find_one(name=SERVICE_NAME)
|
|
if not row or not row.get("heartbeat"):
|
|
return AdminNodePoolOut()
|
|
try:
|
|
beat = datetime.fromisoformat(row["heartbeat"])
|
|
except ValueError:
|
|
return AdminNodePoolOut()
|
|
if beat.tzinfo is None:
|
|
beat = beat.replace(tzinfo=timezone.utc)
|
|
if (datetime.now(timezone.utc) - beat).total_seconds() > BaseService.STALE_SECONDS * 2:
|
|
return AdminNodePoolOut(heartbeat=row["heartbeat"])
|
|
try:
|
|
stats = json.loads(row.get("metrics") or "{}").get("stats") or []
|
|
except ValueError:
|
|
stats = []
|
|
by_label = {item.get("label"): item.get("value") for item in stats}
|
|
values = {field: int(by_label.get(label) or 0) for label, field in POOL_STATS}
|
|
return AdminNodePoolOut(online=True, heartbeat=row["heartbeat"], **values)
|
|
|
|
|
|
def queue_view() -> AdminNodeQueueOut:
|
|
return AdminNodeQueueOut(
|
|
depth=count_open_requests(),
|
|
max_depth=LIMITS.max_queue_depth,
|
|
**request_status_counts(),
|
|
)
|
|
|
|
|
|
def setting_value(field: NodeSettingField) -> str:
|
|
if field.kind == "flag":
|
|
return "1" if (get_setting(field.key, str(field.default)) or "0") == "1" else "0"
|
|
return str(get_int_setting(field.key, field.default))
|
|
|
|
|
|
def settings_view() -> list[AdminNodeSettingOut]:
|
|
return [
|
|
AdminNodeSettingOut(
|
|
key=field.key,
|
|
label=field.label,
|
|
kind=field.kind,
|
|
value=setting_value(field),
|
|
default=str(field.default),
|
|
ceiling=field.ceiling,
|
|
)
|
|
for field in SETTING_FIELDS
|
|
]
|
|
|
|
|
|
def device_views(rows: list[dict], owners: dict) -> list[AdminNodeDeviceOut]:
|
|
return [
|
|
AdminNodeDeviceOut(
|
|
uid=row["uid"],
|
|
label=row.get("label") or "",
|
|
kind=row.get("kind") or "",
|
|
platform=row.get("platform") or "",
|
|
client_version=row.get("client_version") or "",
|
|
status=effective_status(row),
|
|
suspended_until=row.get("suspended_until") or None,
|
|
suspend_reason=row.get("suspend_reason") or "",
|
|
country=row.get("country") or "",
|
|
asn_class=row.get("asn_class") or "unknown",
|
|
asn=row.get("asn") or "",
|
|
class_override=row.get("class_override") or "",
|
|
last_seen_at=row.get("last_seen_at") or None,
|
|
created_at=row.get("created_at"),
|
|
owner_uid=row.get("user_uid") or "",
|
|
owner_username=(owners.get(row.get("user_uid")) or {}).get("username") or "",
|
|
)
|
|
for row in rows
|
|
]
|
|
|
|
|
|
def flag_views(rows: list[dict], owners: dict, devices: dict) -> list[AdminNodeFlagOut]:
|
|
return [
|
|
AdminNodeFlagOut(
|
|
uid=row["uid"],
|
|
kind=row.get("kind") or "",
|
|
detail=row.get("detail") or "",
|
|
status=row.get("status") or "open",
|
|
device_uid=row.get("device_uid") or "",
|
|
device_label=(devices.get(row.get("device_uid")) or {}).get("label") or "",
|
|
user_uid=row.get("user_uid") or "",
|
|
owner_username=(owners.get(row.get("user_uid")) or {}).get("username") or "",
|
|
created_at=row.get("created_at"),
|
|
)
|
|
for row in rows
|
|
]
|
|
|
|
|
|
def circuit_views() -> list[AdminNodeCircuitOut]:
|
|
now = datetime.now(timezone.utc).timestamp()
|
|
breaker = CircuitBreaker()
|
|
views = []
|
|
for row in load_circuits():
|
|
age = max(0.0, now - float(row.get("opened_at") or 0.0))
|
|
failures = int(row.get("failures") or 0)
|
|
if failures < breaker.failures:
|
|
state, left = "closed", 0
|
|
elif age < breaker.open_seconds:
|
|
state, left = "open", int(breaker.open_seconds - age)
|
|
else:
|
|
state, left = "half_open", 0
|
|
views.append(AdminNodeCircuitOut(key=row["key"], state=state, failures=failures, seconds_left=left))
|
|
return views
|
|
|
|
|
|
def outcome_views(counters: dict[str, int]) -> list[AdminNodeOutcomeOut]:
|
|
grouped: dict[tuple[str, str], dict[str, int]] = {}
|
|
for name, value in counters.items():
|
|
_, adapter, asn_class, kind = name.split(":")
|
|
grouped.setdefault((adapter, asn_class), {})[kind] = value
|
|
return [
|
|
AdminNodeOutcomeOut(
|
|
adapter=adapter,
|
|
asn_class=asn_class,
|
|
ok=kinds.get("ok", 0),
|
|
blocked=kinds.get("blocked", 0),
|
|
failed=kinds.get("failed", 0),
|
|
total=sum(kinds.values()),
|
|
)
|
|
for (adapter, asn_class), kinds in sorted(grouped.items())
|
|
]
|
|
|
|
|
|
def overview() -> dict:
|
|
lifetime = lifetime_counters()
|
|
outcomes = {name: value for name, value in lifetime.items() if name.startswith(OUTCOME_PREFIX)}
|
|
devices = recent_devices(ADMIN_LIST_LIMIT)
|
|
flags = list_flags(limit=ADMIN_LIST_LIMIT)
|
|
device_by_uid = {row["uid"]: row for row in devices}
|
|
missing = [row["device_uid"] for row in flags if row.get("device_uid") and row["device_uid"] not in device_by_uid]
|
|
for uid in missing:
|
|
found = get_device(uid)
|
|
if found:
|
|
device_by_uid[uid] = found
|
|
owner_uids = [row["user_uid"] for row in devices + flags if row.get("user_uid")]
|
|
owners = get_users_by_uids(list(dict.fromkeys(owner_uids)))
|
|
breakdowns = {f"devices_by_{column}": device_breakdown(column) for column in DEVICE_BREAKDOWN_COLUMNS}
|
|
return {
|
|
"enabled": kill_switch_on(),
|
|
"pool": pool_view(),
|
|
"queue": queue_view(),
|
|
**breakdowns,
|
|
"usage_today": usage_day_totals(),
|
|
"flags": flag_views(flags, owners, device_by_uid),
|
|
"flags_total": count_flags(),
|
|
"devices": device_views(devices, owners),
|
|
"devices_total": count_all_devices(),
|
|
"settings": settings_view(),
|
|
"limits": LIMITS.describe(),
|
|
"circuits": circuit_views(),
|
|
"outcomes": outcome_views(outcomes),
|
|
"lifetime": {name: value for name, value in lifetime.items() if name not in outcomes},
|
|
"geo": shared_geo().status(),
|
|
"cache": cache_stats(),
|
|
"admin_section": "nodes",
|
|
}
|
|
|
|
|
|
def seo(request: Request):
|
|
base = site_url(request)
|
|
return base_seo_context(
|
|
request,
|
|
title="Community nodes - Admin",
|
|
description="Community donor node network: pool, queue, devices and flags.",
|
|
breadcrumbs=[
|
|
{"name": "Home", "url": "/feed"},
|
|
{"name": "Admin", "url": "/admin"},
|
|
{"name": "Community nodes", "url": LIST_URL},
|
|
],
|
|
schemas=[website_schema(base)],
|
|
robots="noindex,nofollow",
|
|
)
|
|
|
|
|
|
@router.get("/nodes", response_class=HTMLResponse)
|
|
async def admin_nodes(request: Request):
|
|
admin = require_admin(request)
|
|
return respond(
|
|
request,
|
|
"admin_nodes.html",
|
|
{**seo(request), "request": request, "user": admin, **overview()},
|
|
model=AdminNodesOut,
|
|
)
|
|
|
|
|
|
@router.post("/nodes/killswitch")
|
|
async def admin_nodes_killswitch(
|
|
request: Request,
|
|
data: Annotated[AdminNodeKillswitchForm, Depends(json_or_form(AdminNodeKillswitchForm))],
|
|
):
|
|
admin = require_admin(request)
|
|
before = kill_switch_on()
|
|
after = (not before) if data.enabled is None else data.enabled
|
|
set_setting(KILLSWITCH_KEY, "1" if after else "0")
|
|
audit.record(
|
|
request,
|
|
"node.killswitch",
|
|
user=admin,
|
|
target_type="setting",
|
|
target_uid=KILLSWITCH_KEY,
|
|
target_label=KILLSWITCH_KEY,
|
|
old_value="1" if before else "0",
|
|
new_value="1" if after else "0",
|
|
summary=f"admin {admin['username']} switched community donation {'on' if after else 'off'}",
|
|
links=[audit.setting(KILLSWITCH_KEY)],
|
|
)
|
|
return action_result(request, LIST_URL, data={"enabled": after})
|
|
|
|
|
|
@router.post("/nodes/settings")
|
|
async def admin_nodes_settings(
|
|
request: Request,
|
|
data: Annotated[AdminNodeSettingsForm, Depends(json_or_form(AdminNodeSettingsForm))],
|
|
):
|
|
admin = require_admin(request)
|
|
changes: dict[str, dict] = {}
|
|
clamped: list[str] = []
|
|
for key, submitted in data.model_dump().items():
|
|
if submitted is None:
|
|
continue
|
|
field = SETTING_FIELDS_BY_KEY[key]
|
|
value = str(submitted)
|
|
if field.ceiling is not None and int(submitted) > field.ceiling:
|
|
value = str(field.ceiling)
|
|
clamped.append(key)
|
|
before = setting_value(field)
|
|
if before == value:
|
|
continue
|
|
set_setting(key, value)
|
|
changes[key] = {"old": before, "new": value}
|
|
audit.record(
|
|
request,
|
|
"node.admin.settings",
|
|
user=admin,
|
|
target_type="setting",
|
|
target_uid="nodes",
|
|
target_label="nodes",
|
|
metadata={"changes": changes, "clamped": clamped},
|
|
summary=f"admin {admin['username']} updated community node settings ({len(changes)} changed)",
|
|
links=[audit.setting(key) for key in changes],
|
|
)
|
|
return action_result(request, LIST_URL, data={"changed": list(changes), "clamped": clamped})
|
|
|
|
|
|
def require_device(device_uid: str) -> dict:
|
|
device = get_device(device_uid)
|
|
if device is None:
|
|
raise not_found("Unknown device")
|
|
return device
|
|
|
|
|
|
def owner_of(device: dict) -> dict | None:
|
|
return get_table("users").find_one(uid=device.get("user_uid"))
|
|
|
|
|
|
@router.post("/nodes/devices/{device_uid}/suspend")
|
|
async def admin_nodes_suspend(
|
|
request: Request,
|
|
device_uid: str,
|
|
data: Annotated[AdminNodeSuspendForm, Depends(json_or_form(AdminNodeSuspendForm))],
|
|
):
|
|
admin = require_admin(request)
|
|
device = require_device(device_uid)
|
|
owner = owner_of(device)
|
|
if is_senior_admin(admin, owner):
|
|
return deny_senior(request, admin, owner["uid"], owner, "node.suspend", LIST_URL)
|
|
if not suspend_device(device_uid, data.reason, data.minutes):
|
|
return json_error(409, "This device can no longer be suspended")
|
|
add_event("admin_suspend", device_uid=device_uid, user_uid=device.get("user_uid") or "", detail=data.reason)
|
|
audit.record(
|
|
request,
|
|
"node.suspend",
|
|
user=admin,
|
|
target_type="node_device",
|
|
target_uid=device_uid,
|
|
target_label=device.get("label") or device_uid,
|
|
metadata={"minutes": data.minutes, "reason": data.reason, "owner_uid": device.get("user_uid")},
|
|
summary=f"admin {admin['username']} suspended node device {device.get('label') or device_uid} for {data.minutes} minutes",
|
|
links=[audit.target("user", device.get("user_uid"), (owner or {}).get("username"))],
|
|
)
|
|
return action_result(request, LIST_URL, data={"uid": device_uid, "minutes": data.minutes})
|
|
|
|
|
|
@router.post("/nodes/devices/{device_uid}/class")
|
|
async def admin_nodes_class(
|
|
request: Request,
|
|
device_uid: str,
|
|
data: Annotated[AdminNodeClassForm, Depends(json_or_form(AdminNodeClassForm))],
|
|
):
|
|
admin = require_admin(request)
|
|
device = require_device(device_uid)
|
|
owner = owner_of(device)
|
|
if is_senior_admin(admin, owner):
|
|
return deny_senior(request, admin, owner["uid"], owner, "node.device.override", LIST_URL)
|
|
value = "allow" if data.value == "allow" else ""
|
|
before = device.get("class_override") or ""
|
|
set_class_override(device_uid, value)
|
|
audit.record(
|
|
request,
|
|
"node.device.override",
|
|
user=admin,
|
|
target_type="node_device",
|
|
target_uid=device_uid,
|
|
target_label=device.get("label") or device_uid,
|
|
old_value=before,
|
|
new_value=value,
|
|
metadata={"owner_uid": device.get("user_uid"), "asn_class": device.get("asn_class") or "unknown"},
|
|
summary=(
|
|
f"admin {admin['username']} {'allowed' if value else 'cleared the override for'} "
|
|
f"node device {device.get('label') or device_uid} regardless of its address class"
|
|
),
|
|
links=[audit.target("user", device.get("user_uid"), (owner or {}).get("username"))],
|
|
)
|
|
return action_result(request, LIST_URL, data={"uid": device_uid, "class_override": value})
|
|
|
|
|
|
@router.post("/nodes/devices/{device_uid}/unsuspend")
|
|
async def admin_nodes_unsuspend(request: Request, device_uid: str):
|
|
admin = require_admin(request)
|
|
device = require_device(device_uid)
|
|
owner = owner_of(device)
|
|
if is_senior_admin(admin, owner):
|
|
return deny_senior(request, admin, owner["uid"], owner, "node.unsuspend", LIST_URL)
|
|
if not unsuspend_device(device_uid):
|
|
return json_error(409, "This device is not suspended")
|
|
add_event("admin_unsuspend", device_uid=device_uid, user_uid=device.get("user_uid") or "")
|
|
audit.record(
|
|
request,
|
|
"node.unsuspend",
|
|
user=admin,
|
|
target_type="node_device",
|
|
target_uid=device_uid,
|
|
target_label=device.get("label") or device_uid,
|
|
metadata={"owner_uid": device.get("user_uid")},
|
|
summary=f"admin {admin['username']} lifted the suspension of node device {device.get('label') or device_uid}",
|
|
links=[audit.target("user", device.get("user_uid"), (owner or {}).get("username"))],
|
|
)
|
|
return action_result(request, LIST_URL, data={"uid": device_uid})
|
|
|
|
|
|
@router.post("/nodes/flags/{flag_uid}/resolve")
|
|
async def admin_nodes_flag_resolve(request: Request, flag_uid: str):
|
|
admin = require_admin(request)
|
|
flag = get_table("node_flags").find_one(uid=flag_uid, deleted_at=None)
|
|
if flag is None:
|
|
raise not_found("Unknown flag")
|
|
owner = get_table("users").find_one(uid=flag.get("user_uid")) if flag.get("user_uid") else None
|
|
if is_senior_admin(admin, owner):
|
|
return deny_senior(request, admin, owner["uid"], owner, "node.flag.resolve", LIST_URL)
|
|
if not resolve_flag(flag_uid, admin["uid"]):
|
|
return json_error(409, "This flag is already resolved")
|
|
audit.record(
|
|
request,
|
|
"node.flag.resolve",
|
|
user=admin,
|
|
target_type="node_flag",
|
|
target_uid=flag_uid,
|
|
target_label=flag.get("kind") or flag_uid,
|
|
metadata={"device_uid": flag.get("device_uid"), "owner_uid": flag.get("user_uid")},
|
|
summary=f"admin {admin['username']} resolved node flag {flag.get('kind') or flag_uid}",
|
|
)
|
|
return action_result(request, LIST_URL, data={"uid": flag_uid})
|