Files
devplacepy/devplacepy/routers/admin/communities.py
T
retoorandClaude Sonnet 5.5 1a9201cb91 Admin pages: one shared specification, builders, fragments, caps and live refresh
Adds admin_sections registry, shared admin partials, WorkspaceList, admin CLAUDE.md spec,
capped node/flag listings, and api/unit tests enforcing the spec.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-15 01:50:50 +01:00

498 lines
20 KiB
Python

# retoor <retoor@molodetz.nl>
import asyncio
import json
import logging
from datetime import datetime
from typing import Annotated
from urllib.parse import urlencode
from fastapi import APIRouter, Form, Request
from fastapi.responses import HTMLResponse
from devplacepy.database import build_pagination, get_setting, get_table, get_users_by_uids, resolve_object_url
from devplacepy.database.sync import count_links, identity_map
from devplacepy.responses import action_result, json_error, respond
from devplacepy.schemas import AdminCommunitiesOut, AdminCommunityLedgerOut, AdminCommunityOut, AdminCommunitySchemaOut
from devplacepy.models import CommunityAddForm, CommunityRunForm, CommunitySchemaForm, CommunityUpdateForm
from devplacepy.seo import base_seo_context, site_url, website_schema
from devplacepy.services.audit import record as audit
from devplacepy.services.community.discoverer import slug_for_query
from devplacepy.services.community.files import files
from devplacepy.services.community.ledger import (
LEDGER_KINDS,
LEVEL_TITLES,
OUTCOME_BY_KEY,
OUTCOMES,
Ledger,
explain,
tally_of,
verdict as run_verdict,
)
from devplacepy.services.community.ledger_store import ledger_store
from devplacepy.services.community.runner import TRIGGER_ADMIN, CommunityRunner
from devplacepy.services.community.service import DRY_RUN_KEY
from devplacepy.services.community.schemacheck import validate_schema
from devplacepy.services.community.spec import STATUS_LEARNING, STATUS_READY
from devplacepy.services.community.store import iso, now, store
from devplacepy.services.sync.gate import sync_enabled
from devplacepy.services.sync.options import SyncOptions
from devplacepy.utils import not_found, require_admin
logger = logging.getLogger(__name__)
router = APIRouter()
background_jobs: set[asyncio.Task] = set()
LIST_URL = "/admin/communities"
async def spawn(coroutine) -> None:
task = asyncio.create_task(coroutine)
background_jobs.add(task)
task.add_done_callback(background_jobs.discard)
await asyncio.sleep(0)
def service_flags() -> dict:
return {
"service_enabled": (get_setting("service_community_enabled", "0") or "0") == "1",
"service_dry_run": (get_setting(DRY_RUN_KEY, "1") or "1") == "1",
}
def public(row: dict) -> dict:
view = {key: value for key, value in row.items() if key not in ("login_password", "login_totp_secret")}
view["has_login"] = bool(row.get("login_password"))
view["has_totp"] = bool(row.get("login_totp_secret"))
return view
def require_community(slug: str) -> dict:
row = store.get(slug)
if row is None:
raise not_found("Unknown community")
return row
def credentials_of(data) -> dict:
return {"username": data.login_username.strip(), "email": data.login_email.strip(), "password": data.login_password, "totp": data.login_totp_secret.strip()}
def seo(request: Request, title: str, path: str, crumb: str):
base = site_url(request)
return base_seo_context(
request,
title=f"{title} - Admin",
description="External communities synchronised into this instance.",
breadcrumbs=[
{"name": "Home", "url": "/feed"},
{"name": "Admin", "url": "/admin"},
{"name": crumb, "url": path},
],
schemas=[website_schema(base)],
robots="noindex,nofollow",
)
FILTER_KEYS = ("kind", "outcome", "reason", "q")
def seconds_between(start: str | None, end: str | None) -> int | None:
try:
return max(0, round((datetime.fromisoformat(end) - datetime.fromisoformat(start)).total_seconds()))
except (TypeError, ValueError):
return None
def run_view(run: dict) -> dict:
view = dict(run)
view["dry_run"] = bool(run.get("dry_run"))
for column in ("items_stored", "items_dropped"):
view[column] = int(run.get(column) or 0)
for column in ("verdict", "headline"):
view[column] = run.get(column) or ""
if not view.get("tally") and view.get("pushed"):
view["tally"] = tally_of(view["pushed"], Ledger())
view["duration_seconds"] = seconds_between(run.get("started_at"), run.get("finished_at"))
if not view.get("verdict"):
outcome = run_verdict(run.get("status", ""), view["dry_run"], view.get("tally") or {})
view["verdict"], view["headline"] = outcome["level"], outcome["headline"]
return view
def local_links(rows: list[dict]) -> dict[str, str]:
wanted: dict[str, list[str]] = {"posts": [], "users": [], "comments": []}
for row in rows:
if row.get("local_uid") and row.get("kind") in wanted:
wanted[row["kind"]].append(row["local_uid"])
links: dict[str, str] = {}
if wanted["posts"]:
table = get_table("posts")
for found in table.find(table.table.columns.uid.in_(wanted["posts"]), deleted_at=None):
links[found["uid"]] = f"/posts/{found.get('slug') or found['uid']}"
for uid, found in get_users_by_uids(wanted["users"]).items():
links[uid] = f"/profile/{found['username']}"
if wanted["comments"]:
table = get_table("comments")
for found in table.find(table.table.columns.uid.in_(wanted["comments"]), deleted_at=None):
target = resolve_object_url(found["target_type"], found["target_uid"])
links[found["uid"]] = f"{target}#comment-{found['uid']}" if target else ""
return links
def item_view(row: dict, links: dict[str, str]) -> dict:
view = dict(row)
view["tone"] = OUTCOME_BY_KEY[row["outcome"]].tone if row.get("outcome") in OUTCOME_BY_KEY else "muted"
why = explain(row.get("reason", ""))
view.update(reason_title=why["title"], reason_text=why["text"], retried=why["retried"])
view["local_url"] = links.get(row.get("local_uid") or "", "")
return view
def ledger_options(scope: dict) -> dict:
facets = ledger_store.facets(**scope)
return {
"kind": [kind for kind in LEDGER_KINDS if kind in facets["kind"]],
"outcome": [{"value": outcome.key, "label": outcome.label} for outcome in OUTCOMES],
"reason": [{"value": reason, "title": explain(reason)["title"]} for reason in facets["reason"]],
}
def feed_summary(feed: dict) -> dict:
return {
"id": feed["id"],
"type": feed["type"],
"url": feed["url"],
"list_path": feed.get("list_path"),
"item_selector": feed.get("item_selector"),
"post_fields": feed["post"]["fields"],
"author_fields": (feed.get("author") or {}).get("fields", {}),
"comments": (feed.get("comments") or {}).get("mode", "none"),
"comments_reason": (feed.get("comments") or {}).get("reason", ""),
"pagination": (feed.get("pagination") or {}).get("style", "none"),
}
@router.get("/communities", response_class=HTMLResponse)
async def admin_communities(request: Request, page: int = 1):
admin = require_admin(request)
pagination = build_pagination(page, store.count())
offset = (pagination["page"] - 1) * pagination["per_page"]
return respond(
request,
"admin_communities.html",
{
**seo(request, "Communities", LIST_URL, "Communities"),
"request": request,
"user": admin,
"items": [public(row) for row in store.list(pagination["per_page"], offset)],
"pagination": pagination,
"sync_enabled": sync_enabled(),
**service_flags(),
"admin_section": "communities",
},
model=AdminCommunitiesOut,
)
@router.post("/communities")
async def admin_community_add(request: Request, data: Annotated[CommunityAddForm, Form()]):
admin = require_admin(request)
slug = slug_for_query(data.query)
if not slug:
return json_error(400, "Enter a community name or address")
existing = store.get(slug)
if existing is None:
secrets = credentials_of(data)
existing = store.create(
slug,
data.query,
admin["uid"],
name=data.query,
status=STATUS_LEARNING,
claimed_at=iso(now()),
login_username=secrets["username"],
login_email=secrets["email"],
login_password=secrets["password"],
login_totp_secret=secrets["totp"],
)
await spawn(CommunityRunner().discover(data.query, admin["uid"], relearn_slug=slug))
audit.record(
request,
"admin.community.add",
user=admin,
target_type="community",
target_uid=existing["uid"],
target_label=slug,
metadata={"query": data.query},
summary=f"{admin['username']} added community {data.query!r}",
)
return action_result(request, f"{LIST_URL}/{existing['slug']}")
@router.get("/communities/{slug}", response_class=HTMLResponse)
async def admin_community(request: Request, slug: str):
admin = require_admin(request)
row = require_community(slug)
store.reap_interrupted()
source = files.read(row["slug"])
return respond(
request,
"admin_community.html",
{
**seo(request, row["name"] or slug, f"{LIST_URL}/{slug}", "Communities"),
"request": request,
"user": admin,
"community": public(row),
"rules": row.get("rules") or "",
"favicon": row.get("favicon") or "",
"totals": row.get("totals") or {},
"feeds": [feed_summary(feed) for feed in (row.get("schema") or {}).get("feeds", [])],
"runs": [run_view(run) for run in store.run_summaries(row["uid"])],
"collectors_path": str(files.collectors_path(row["slug"])) if source else "",
"collectors_lines": len(source.splitlines()),
"busy": store.busy(row["uid"]),
"sync_enabled": sync_enabled(),
"admin_section": "communities",
},
model=AdminCommunityOut,
)
@router.get("/communities/{slug}/ledger", response_class=HTMLResponse)
async def admin_community_ledger(
request: Request,
slug: str,
run: str = "",
kind: str = "",
outcome: str = "",
reason: str = "",
q: str = "",
page: int = 1,
):
admin = require_admin(request)
row = require_community(slug)
run_row = None
if run:
found = store.run(run)
if found is None or found["community_uid"] != row["uid"]:
raise not_found("Unknown run")
run_row = run_view(found)
scope = {"run_uid": run_row["uid"]} if run_row else {"community_uid": row["uid"]}
filters = {key: value for key, value in zip(FILTER_KEYS, (kind, outcome, reason, q)) if value}
rows, pagination = ledger_store.page(filters, page, **scope)
links = local_links(rows)
listed = ledger_store.counts(**scope)
exact = (run_row or {}).get("tally") or listed
path = f"{LIST_URL}/{slug}/ledger"
query = urlencode({**({"run": run_row["uid"]} if run_row else {}), **filters})
return respond(
request,
"admin_community_ledger.html",
{
**seo(request, f"{row['name'] or slug} sync log", path, "Communities"),
"request": request,
"user": admin,
"community": public(row),
"scope": "run" if run_row else "community",
"run": run_row,
"run_errors": (run_row or {}).get("errors") or [],
"run_log": (run_row or {}).get("log") or "",
"busy": (run_row["status"] == "running") if run_row else store.busy(row["uid"]),
"verdict": {"level": run_row["verdict"], "title": LEVEL_TITLES.get(run_row["verdict"], ""), "headline": run_row["headline"]} if run_row else {},
"tiles": [
{"key": o.key, "label": o.label, "tone": o.tone, "help": o.help, "count": int(exact.get(o.key, 0)), "listed": int(listed.get(o.key, 0))}
for o in OUTCOMES
],
"items": [item_view(item, links) for item in rows],
"pagination": pagination,
"pagination_query": f"{query}&" if query else "",
"filters": filters,
"options": ledger_options(scope),
"admin_section": "communities",
},
model=AdminCommunityLedgerOut,
)
@router.post("/communities/{slug}/run")
async def admin_community_run(request: Request, slug: str, data: Annotated[CommunityRunForm, Form()]):
admin = require_admin(request)
row = require_community(slug)
if store.busy(row["uid"]):
return json_error(409, "A run is already in progress for this community")
await spawn(CommunityRunner().run(row, dry_run=data.dry_run, trigger=TRIGGER_ADMIN))
audit.record(
request,
"admin.community.run",
user=admin,
target_type="community",
target_uid=row["uid"],
target_label=slug,
metadata={"dry_run": data.dry_run},
summary=f"{admin['username']} started a {'dry run' if data.dry_run else 'run'} of community {slug}",
)
return action_result(request, f"{LIST_URL}/{slug}")
@router.post("/communities/{slug}/relearn")
async def admin_community_relearn(request: Request, slug: str):
admin = require_admin(request)
row = require_community(slug)
if store.busy(row["uid"]):
return json_error(409, "A run is already in progress for this community")
await spawn(CommunityRunner().discover(row["query"] or row["url"], admin["uid"], relearn_slug=slug))
audit.record(
request,
"admin.community.relearn",
user=admin,
target_type="community",
target_uid=row["uid"],
target_label=slug,
summary=f"{admin['username']} started relearning community {slug}",
)
return action_result(request, f"{LIST_URL}/{slug}")
@router.post("/communities/{slug}/update")
async def admin_community_update(request: Request, slug: str, data: Annotated[CommunityUpdateForm, Form()]):
admin = require_admin(request)
row = require_community(slug)
changes = {"interval_seconds": data.interval_seconds, "enabled": int(data.enabled)}
if data.clear_login:
changes.update(login_username="", login_email="", login_password="", login_totp_secret="")
else:
for column, value in (("login_username", data.login_username.strip()), ("login_email", data.login_email.strip()), ("login_password", data.login_password), ("login_totp_secret", data.login_totp_secret.strip())):
if value:
changes[column] = value
submitted = {name: getattr(data, name) for name in SyncOptions.__dataclass_fields__ if getattr(data, name) is not None}
if submitted:
try:
options = SyncOptions.from_row(row).merged(**submitted)
except ValueError as exc:
return json_error(400, str(exc))
if options.alias and not SyncOptions.from_row(row).alias and not identity_map(row["origin"]) and count_links(row["origin"]):
return json_error(400, "This community already has records stored under their original names; aliasing cannot be switched on afterwards")
changes["sync_options"] = options.to_dict()
store.update(row["uid"], **changes)
audit.record(
request,
"admin.community.update",
user=admin,
target_type="community",
target_uid=row["uid"],
target_label=slug,
metadata={"interval_seconds": data.interval_seconds, "enabled": data.enabled, "sync_options": changes.get("sync_options"), "login_changed": data.clear_login or bool(data.login_password or data.login_username or data.login_email or data.login_totp_secret)},
summary=f"{admin['username']} updated community {slug}",
)
return action_result(request, f"{LIST_URL}/{slug}")
def schema_text(schema) -> str:
return json.dumps(schema, indent=2, ensure_ascii=False)
def schema_page(request: Request, admin: dict, row: dict, text: str, errors: list[str], notice: str, status_code: int):
context = {
**seo(request, f"{row['name'] or row['slug']} schema", f"{LIST_URL}/{row['slug']}/schema", "Communities"),
"request": request,
"user": admin,
"community": public(row),
"schema_text": text,
"has_previous": bool(row.get("schema_previous")),
"errors": errors,
"notice": notice,
"admin_section": "communities",
}
response = respond(request, "admin_community_schema.html", context, model=AdminCommunitySchemaOut)
response.status_code = status_code
return response
@router.get("/communities/{slug}/schema", response_class=HTMLResponse)
async def admin_community_schema(request: Request, slug: str):
admin = require_admin(request)
row = require_community(slug)
return schema_page(request, admin, row, schema_text(row.get("schema") or {}), [], "", 200)
@router.post("/communities/{slug}/schema")
async def admin_community_schema_save(request: Request, slug: str, data: Annotated[CommunitySchemaForm, Form()]):
admin = require_admin(request)
row = require_community(slug)
if store.busy(row["uid"]):
return json_error(409, "A run is already in progress for this community")
try:
schema = json.loads(data.schema_text)
except ValueError as exc:
return schema_page(request, admin, row, data.schema_text, [f"The text is not valid JSON: {exc}"], "", 400)
errors = validate_schema(schema)
if errors:
return schema_page(request, admin, row, data.schema_text, errors, "", 400)
verdict = await CommunityRunner().verify_schema(row, schema)
if not verdict.ok:
failed = ["The schema was checked against the live source and was not saved."] + verdict.problems[:10]
return schema_page(request, admin, row, data.schema_text, failed, "", 422)
store.update(
row["uid"],
schema=schema,
schema_previous=row.get("schema") or {},
status=STATUS_READY,
last_error="",
failure_count=0,
enabled=1,
)
files.write(row["slug"], schema)
audit.record(
request,
"admin.community.schema",
user=admin,
target_type="community",
target_uid=row["uid"],
target_label=slug,
metadata={"feeds": len(schema["feeds"]), "sample_posts": verdict.posts, "sample_comments": verdict.comments},
summary=f"{admin['username']} replaced the learned schema of community {slug}",
)
return action_result(request, f"{LIST_URL}/{slug}/schema", data={"posts": verdict.posts, "comments": verdict.comments})
@router.post("/communities/{slug}/schema/restore")
async def admin_community_schema_restore(request: Request, slug: str):
admin = require_admin(request)
row = require_community(slug)
previous = row.get("schema_previous") or {}
if not previous:
return json_error(409, "There is no previous schema to restore")
errors = validate_schema(previous)
if errors:
return schema_page(request, admin, row, schema_text(previous), errors, "", 400)
store.update(row["uid"], schema=previous, schema_previous=row.get("schema") or {})
files.write(row["slug"], previous)
audit.record(
request,
"admin.community.schema_restore",
user=admin,
target_type="community",
target_uid=row["uid"],
target_label=slug,
summary=f"{admin['username']} restored the previous schema of community {slug}",
)
return action_result(request, f"{LIST_URL}/{slug}/schema")
@router.post("/communities/{slug}/delete")
async def admin_community_delete(request: Request, slug: str):
admin = require_admin(request)
row = require_community(slug)
store.delete(row["uid"], admin["uid"])
audit.record(
request,
"admin.community.delete",
user=admin,
target_type="community",
target_uid=row["uid"],
target_label=slug,
summary=f"{admin['username']} deleted community {slug}",
)
return action_result(request, LIST_URL)