Files
devplacepy/devplacepy/routers/admin/nodes.py
T
retoorandClaude Sonnet 5.5 02ab747cd0 Code Farm: per-plot auto-replant pause and crop-lock check; community sync and node work
Continuous Delivery could never be turned off once a crop was planted. Add a
one-shot per-plot "stop after harvest" flag (POST /game/plot/stop, Devii tool
game_plot_stop, game_plots.stop_after_harvest), cleared by the next plant,
autoreplant claim or Refactor. Autoreplant now also respects crop locks through
the shared _crop_lock predicate. Plan and measurements in happyplanting.md.

Also includes the pending community sync, node network and related work in the
tree; the full suite passed (4999) with all of it.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-17 02:23:43 +02:00

470 lines
17 KiB
Python

# retoor <retoor@molodetz.nl>
import json
import logging
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Annotated
from fastapi import APIRouter, Depends, Request
from fastapi.responses import HTMLResponse
from devplacepy.database import get_int_setting, get_setting, get_table, get_users_by_uids, set_setting
from devplacepy.database.nodes import (
ADMIN_LIST_LIMIT,
DEVICE_BREAKDOWN_COLUMNS,
LIMITS,
add_event,
count_open_requests,
device_breakdown,
cache_stats,
effective_status,
get_device,
lifetime_counters,
list_flags,
load_circuits,
recent_devices,
request_status_counts,
resolve_flag,
set_class_override,
suspend_device,
unsuspend_device,
usage_day_totals,
)
from devplacepy.dependencies import json_or_form
from devplacepy.models import (
AdminNodeClassForm,
AdminNodeKillswitchForm,
AdminNodeSettingsForm,
AdminNodeSuspendForm,
)
from devplacepy.responses import action_result, json_error, respond
from devplacepy.routers.admin._shared import deny_senior, is_senior_admin
from devplacepy.routers.nodes.social import SOCIAL_DAILY_DEFAULT
from devplacepy.schemas import (
AdminNodeCircuitOut,
AdminNodeDeviceOut,
AdminNodeFlagOut,
AdminNodeOutcomeOut,
AdminNodePoolOut,
AdminNodeQueueOut,
AdminNodeSettingOut,
AdminNodesOut,
)
from devplacepy.seo import base_seo_context, site_url, website_schema
from devplacepy.services.audit import record as audit
from devplacepy.services.base import BaseService
from devplacepy.services.nodes.circuit import CircuitBreaker
from devplacepy.services.nodes.geo import shared_geo
from devplacepy.services.nodes.registry import OUTCOME_PREFIX
from devplacepy.utils import not_found, require_admin
logger = logging.getLogger(__name__)
router = APIRouter()
LIST_URL = "/admin/nodes"
SERVICE_NAME = "nodes"
KILLSWITCH_KEY = "nodes_enabled"
QUOTA_USER_CEILING = 10_000
QUOTA_GUEST_CEILING = 1_000
QUOTA_SOCIAL_CEILING = 10_000
POOL_STATS = (
("connected", "connected"),
("capacity", "capacity"),
("accepted", "accepted"),
("kicked", "kicked"),
("jobs done", "jobs_done"),
("jobs failed", "jobs_failed"),
("open circuits", "open_circuits"),
)
@dataclass(frozen=True)
class NodeSettingField:
key: str
label: str
default: int
ceiling: int | None
kind: str = "int"
SETTING_FIELDS: tuple[NodeSettingField, ...] = (
NodeSettingField("nodes_max_total", "Maximum connected nodes", LIMITS.max_connected_nodes, LIMITS.max_connected_nodes),
NodeSettingField("nodes_daily_quota_user", "Daily node requests per member", LIMITS.default_daily_requests_user, QUOTA_USER_CEILING),
NodeSettingField("nodes_daily_quota_guest", "Daily node requests per guest", LIMITS.default_daily_requests_guest, QUOTA_GUEST_CEILING),
NodeSettingField("nodes_cache_ttl_seconds", "Result cache lifetime (seconds)", LIMITS.cache_ttl_seconds, LIMITS.cache_ttl_seconds),
NodeSettingField("nodes_daily_social_user", "Daily social searches per member", SOCIAL_DAILY_DEFAULT, QUOTA_SOCIAL_CEILING),
NodeSettingField("nodes_social_via_nodes", "Route social search through nodes", 0, None, "flag"),
NodeSettingField("nodes_allow_unknown_class", "Allow donors whose address class is unknown", 0, None, "flag"),
NodeSettingField("nodes_redundancy_percent", "Redundancy sample of fetch jobs (percent)", LIMITS.redundancy_percent, LIMITS.redundancy_max_percent),
)
SETTING_FIELDS_BY_KEY = {field.key: field for field in SETTING_FIELDS}
def kill_switch_on() -> bool:
return (get_setting(KILLSWITCH_KEY, "0") or "0") == "1"
def pool_view() -> AdminNodePoolOut:
row = get_table("service_state").find_one(name=SERVICE_NAME)
if not row or not row.get("heartbeat"):
return AdminNodePoolOut()
try:
beat = datetime.fromisoformat(row["heartbeat"])
except ValueError:
return AdminNodePoolOut()
if beat.tzinfo is None:
beat = beat.replace(tzinfo=timezone.utc)
if (datetime.now(timezone.utc) - beat).total_seconds() > BaseService.STALE_SECONDS * 2:
return AdminNodePoolOut(heartbeat=row["heartbeat"])
try:
stats = json.loads(row.get("metrics") or "{}").get("stats") or []
except ValueError:
stats = []
by_label = {item.get("label"): item.get("value") for item in stats}
values = {field: int(by_label.get(label) or 0) for label, field in POOL_STATS}
return AdminNodePoolOut(online=True, heartbeat=row["heartbeat"], **values)
def queue_view() -> AdminNodeQueueOut:
return AdminNodeQueueOut(
depth=count_open_requests(),
max_depth=LIMITS.max_queue_depth,
**request_status_counts(),
)
def setting_value(field: NodeSettingField) -> str:
if field.kind == "flag":
return "1" if (get_setting(field.key, str(field.default)) or "0") == "1" else "0"
return str(get_int_setting(field.key, field.default))
def settings_view() -> list[AdminNodeSettingOut]:
return [
AdminNodeSettingOut(
key=field.key,
label=field.label,
kind=field.kind,
value=setting_value(field),
default=str(field.default),
ceiling=field.ceiling,
)
for field in SETTING_FIELDS
]
def device_views(rows: list[dict], owners: dict) -> list[AdminNodeDeviceOut]:
return [
AdminNodeDeviceOut(
uid=row["uid"],
label=row.get("label") or "",
kind=row.get("kind") or "",
platform=row.get("platform") or "",
client_version=row.get("client_version") or "",
status=effective_status(row),
suspended_until=row.get("suspended_until") or None,
suspend_reason=row.get("suspend_reason") or "",
country=row.get("country") or "",
asn_class=row.get("asn_class") or "unknown",
asn=row.get("asn") or "",
class_override=row.get("class_override") or "",
last_seen_at=row.get("last_seen_at") or None,
created_at=row.get("created_at"),
owner_uid=row.get("user_uid") or "",
owner_username=(owners.get(row.get("user_uid")) or {}).get("username") or "",
)
for row in rows
]
def flag_views(rows: list[dict], owners: dict, devices: dict) -> list[AdminNodeFlagOut]:
return [
AdminNodeFlagOut(
uid=row["uid"],
kind=row.get("kind") or "",
detail=row.get("detail") or "",
status=row.get("status") or "open",
device_uid=row.get("device_uid") or "",
device_label=(devices.get(row.get("device_uid")) or {}).get("label") or "",
user_uid=row.get("user_uid") or "",
owner_username=(owners.get(row.get("user_uid")) or {}).get("username") or "",
created_at=row.get("created_at"),
)
for row in rows
]
def circuit_views() -> list[AdminNodeCircuitOut]:
now = datetime.now(timezone.utc).timestamp()
breaker = CircuitBreaker()
views = []
for row in load_circuits():
age = max(0.0, now - float(row.get("opened_at") or 0.0))
failures = int(row.get("failures") or 0)
if failures < breaker.failures:
state, left = "closed", 0
elif age < breaker.open_seconds:
state, left = "open", int(breaker.open_seconds - age)
else:
state, left = "half_open", 0
views.append(AdminNodeCircuitOut(key=row["key"], state=state, failures=failures, seconds_left=left))
return views
def outcome_views(counters: dict[str, int]) -> list[AdminNodeOutcomeOut]:
grouped: dict[tuple[str, str], dict[str, int]] = {}
for name, value in counters.items():
_, adapter, asn_class, kind = name.split(":")
grouped.setdefault((adapter, asn_class), {})[kind] = value
return [
AdminNodeOutcomeOut(
adapter=adapter,
asn_class=asn_class,
ok=kinds.get("ok", 0),
blocked=kinds.get("blocked", 0),
failed=kinds.get("failed", 0),
total=sum(kinds.values()),
)
for (adapter, asn_class), kinds in sorted(grouped.items())
]
def overview() -> dict:
lifetime = lifetime_counters()
outcomes = {name: value for name, value in lifetime.items() if name.startswith(OUTCOME_PREFIX)}
devices = recent_devices(ADMIN_LIST_LIMIT)
flags = list_flags()[:ADMIN_LIST_LIMIT]
device_by_uid = {row["uid"]: row for row in devices}
missing = [row["device_uid"] for row in flags if row.get("device_uid") and row["device_uid"] not in device_by_uid]
for uid in missing:
found = get_device(uid)
if found:
device_by_uid[uid] = found
owner_uids = [row["user_uid"] for row in devices + flags if row.get("user_uid")]
owners = get_users_by_uids(list(dict.fromkeys(owner_uids)))
breakdowns = {f"devices_by_{column}": device_breakdown(column) for column in DEVICE_BREAKDOWN_COLUMNS}
return {
"enabled": kill_switch_on(),
"pool": pool_view(),
"queue": queue_view(),
**breakdowns,
"usage_today": usage_day_totals(),
"flags": flag_views(flags, owners, device_by_uid),
"devices": device_views(devices, owners),
"settings": settings_view(),
"limits": LIMITS.describe(),
"circuits": circuit_views(),
"outcomes": outcome_views(outcomes),
"lifetime": {name: value for name, value in lifetime.items() if name not in outcomes},
"geo": shared_geo().status(),
"cache": cache_stats(),
"admin_section": "nodes",
}
def seo(request: Request):
base = site_url(request)
return base_seo_context(
request,
title="Community nodes - Admin",
description="Community donor node network: pool, queue, devices and flags.",
breadcrumbs=[
{"name": "Home", "url": "/feed"},
{"name": "Admin", "url": "/admin"},
{"name": "Community nodes", "url": LIST_URL},
],
schemas=[website_schema(base)],
robots="noindex,nofollow",
)
@router.get("/nodes", response_class=HTMLResponse)
async def admin_nodes(request: Request):
admin = require_admin(request)
return respond(
request,
"admin_nodes.html",
{**seo(request), "request": request, "user": admin, **overview()},
model=AdminNodesOut,
)
@router.post("/nodes/killswitch")
async def admin_nodes_killswitch(
request: Request,
data: Annotated[AdminNodeKillswitchForm, Depends(json_or_form(AdminNodeKillswitchForm))],
):
admin = require_admin(request)
before = kill_switch_on()
after = (not before) if data.enabled is None else data.enabled
set_setting(KILLSWITCH_KEY, "1" if after else "0")
audit.record(
request,
"node.killswitch",
user=admin,
target_type="setting",
target_uid=KILLSWITCH_KEY,
target_label=KILLSWITCH_KEY,
old_value="1" if before else "0",
new_value="1" if after else "0",
summary=f"admin {admin['username']} switched community donation {'on' if after else 'off'}",
links=[audit.setting(KILLSWITCH_KEY)],
)
return action_result(request, LIST_URL, data={"enabled": after})
@router.post("/nodes/settings")
async def admin_nodes_settings(
request: Request,
data: Annotated[AdminNodeSettingsForm, Depends(json_or_form(AdminNodeSettingsForm))],
):
admin = require_admin(request)
changes: dict[str, dict] = {}
clamped: list[str] = []
for key, submitted in data.model_dump().items():
if submitted is None:
continue
field = SETTING_FIELDS_BY_KEY[key]
value = str(submitted)
if field.ceiling is not None and int(submitted) > field.ceiling:
value = str(field.ceiling)
clamped.append(key)
before = setting_value(field)
if before == value:
continue
set_setting(key, value)
changes[key] = {"old": before, "new": value}
audit.record(
request,
"node.admin.settings",
user=admin,
target_type="setting",
target_uid="nodes",
target_label="nodes",
metadata={"changes": changes, "clamped": clamped},
summary=f"admin {admin['username']} updated community node settings ({len(changes)} changed)",
links=[audit.setting(key) for key in changes],
)
return action_result(request, LIST_URL, data={"changed": list(changes), "clamped": clamped})
def require_device(device_uid: str) -> dict:
device = get_device(device_uid)
if device is None:
raise not_found("Unknown device")
return device
def owner_of(device: dict) -> dict | None:
return get_table("users").find_one(uid=device.get("user_uid"))
@router.post("/nodes/devices/{device_uid}/suspend")
async def admin_nodes_suspend(
request: Request,
device_uid: str,
data: Annotated[AdminNodeSuspendForm, Depends(json_or_form(AdminNodeSuspendForm))],
):
admin = require_admin(request)
device = require_device(device_uid)
owner = owner_of(device)
if is_senior_admin(admin, owner):
return deny_senior(request, admin, owner["uid"], owner, "node.suspend", LIST_URL)
if not suspend_device(device_uid, data.reason, data.minutes):
return json_error(409, "This device can no longer be suspended")
add_event("admin_suspend", device_uid=device_uid, user_uid=device.get("user_uid") or "", detail=data.reason)
audit.record(
request,
"node.suspend",
user=admin,
target_type="node_device",
target_uid=device_uid,
target_label=device.get("label") or device_uid,
metadata={"minutes": data.minutes, "reason": data.reason, "owner_uid": device.get("user_uid")},
summary=f"admin {admin['username']} suspended node device {device.get('label') or device_uid} for {data.minutes} minutes",
links=[audit.target("user", device.get("user_uid"), (owner or {}).get("username"))],
)
return action_result(request, LIST_URL, data={"uid": device_uid, "minutes": data.minutes})
@router.post("/nodes/devices/{device_uid}/class")
async def admin_nodes_class(
request: Request,
device_uid: str,
data: Annotated[AdminNodeClassForm, Depends(json_or_form(AdminNodeClassForm))],
):
admin = require_admin(request)
device = require_device(device_uid)
owner = owner_of(device)
if is_senior_admin(admin, owner):
return deny_senior(request, admin, owner["uid"], owner, "node.device.override", LIST_URL)
value = "allow" if data.value == "allow" else ""
before = device.get("class_override") or ""
set_class_override(device_uid, value)
audit.record(
request,
"node.device.override",
user=admin,
target_type="node_device",
target_uid=device_uid,
target_label=device.get("label") or device_uid,
old_value=before,
new_value=value,
metadata={"owner_uid": device.get("user_uid"), "asn_class": device.get("asn_class") or "unknown"},
summary=(
f"admin {admin['username']} {'allowed' if value else 'cleared the override for'} "
f"node device {device.get('label') or device_uid} regardless of its address class"
),
links=[audit.target("user", device.get("user_uid"), (owner or {}).get("username"))],
)
return action_result(request, LIST_URL, data={"uid": device_uid, "class_override": value})
@router.post("/nodes/devices/{device_uid}/unsuspend")
async def admin_nodes_unsuspend(request: Request, device_uid: str):
admin = require_admin(request)
device = require_device(device_uid)
owner = owner_of(device)
if is_senior_admin(admin, owner):
return deny_senior(request, admin, owner["uid"], owner, "node.unsuspend", LIST_URL)
if not unsuspend_device(device_uid):
return json_error(409, "This device is not suspended")
add_event("admin_unsuspend", device_uid=device_uid, user_uid=device.get("user_uid") or "")
audit.record(
request,
"node.unsuspend",
user=admin,
target_type="node_device",
target_uid=device_uid,
target_label=device.get("label") or device_uid,
metadata={"owner_uid": device.get("user_uid")},
summary=f"admin {admin['username']} lifted the suspension of node device {device.get('label') or device_uid}",
links=[audit.target("user", device.get("user_uid"), (owner or {}).get("username"))],
)
return action_result(request, LIST_URL, data={"uid": device_uid})
@router.post("/nodes/flags/{flag_uid}/resolve")
async def admin_nodes_flag_resolve(request: Request, flag_uid: str):
admin = require_admin(request)
flag = get_table("node_flags").find_one(uid=flag_uid, deleted_at=None)
if flag is None:
raise not_found("Unknown flag")
owner = get_table("users").find_one(uid=flag.get("user_uid")) if flag.get("user_uid") else None
if is_senior_admin(admin, owner):
return deny_senior(request, admin, owner["uid"], owner, "node.flag.resolve", LIST_URL)
if not resolve_flag(flag_uid, admin["uid"]):
return json_error(409, "This flag is already resolved")
audit.record(
request,
"node.flag.resolve",
user=admin,
target_type="node_flag",
target_uid=flag_uid,
target_label=flag.get("kind") or flag_uid,
metadata={"device_uid": flag.get("device_uid"), "owner_uid": flag.get("user_uid")},
summary=f"admin {admin['username']} resolved node flag {flag.get('kind') or flag_uid}",
)
return action_result(request, LIST_URL, data={"uid": flag_uid})