Adds admin_sections registry, shared admin partials, WorkspaceList, admin CLAUDE.md spec, capped node/flag listings, and api/unit tests enforcing the spec. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
498 lines
20 KiB
Python
498 lines
20 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
from datetime import datetime
|
|
from typing import Annotated
|
|
from urllib.parse import urlencode
|
|
|
|
from fastapi import APIRouter, Form, Request
|
|
from fastapi.responses import HTMLResponse
|
|
|
|
from devplacepy.database import build_pagination, get_setting, get_table, get_users_by_uids, resolve_object_url
|
|
from devplacepy.database.sync import count_links, identity_map
|
|
from devplacepy.responses import action_result, json_error, respond
|
|
from devplacepy.schemas import AdminCommunitiesOut, AdminCommunityLedgerOut, AdminCommunityOut, AdminCommunitySchemaOut
|
|
from devplacepy.models import CommunityAddForm, CommunityRunForm, CommunitySchemaForm, CommunityUpdateForm
|
|
from devplacepy.seo import base_seo_context, site_url, website_schema
|
|
from devplacepy.services.audit import record as audit
|
|
from devplacepy.services.community.discoverer import slug_for_query
|
|
from devplacepy.services.community.files import files
|
|
from devplacepy.services.community.ledger import (
|
|
LEDGER_KINDS,
|
|
LEVEL_TITLES,
|
|
OUTCOME_BY_KEY,
|
|
OUTCOMES,
|
|
Ledger,
|
|
explain,
|
|
tally_of,
|
|
verdict as run_verdict,
|
|
)
|
|
from devplacepy.services.community.ledger_store import ledger_store
|
|
from devplacepy.services.community.runner import TRIGGER_ADMIN, CommunityRunner
|
|
from devplacepy.services.community.service import DRY_RUN_KEY
|
|
from devplacepy.services.community.schemacheck import validate_schema
|
|
from devplacepy.services.community.spec import STATUS_LEARNING, STATUS_READY
|
|
from devplacepy.services.community.store import iso, now, store
|
|
from devplacepy.services.sync.gate import sync_enabled
|
|
from devplacepy.services.sync.options import SyncOptions
|
|
from devplacepy.utils import not_found, require_admin
|
|
|
|
logger = logging.getLogger(__name__)
|
|
router = APIRouter()
|
|
|
|
background_jobs: set[asyncio.Task] = set()
|
|
LIST_URL = "/admin/communities"
|
|
|
|
|
|
async def spawn(coroutine) -> None:
|
|
task = asyncio.create_task(coroutine)
|
|
background_jobs.add(task)
|
|
task.add_done_callback(background_jobs.discard)
|
|
await asyncio.sleep(0)
|
|
|
|
|
|
def service_flags() -> dict:
|
|
return {
|
|
"service_enabled": (get_setting("service_community_enabled", "0") or "0") == "1",
|
|
"service_dry_run": (get_setting(DRY_RUN_KEY, "1") or "1") == "1",
|
|
}
|
|
|
|
|
|
def public(row: dict) -> dict:
|
|
view = {key: value for key, value in row.items() if key not in ("login_password", "login_totp_secret")}
|
|
view["has_login"] = bool(row.get("login_password"))
|
|
view["has_totp"] = bool(row.get("login_totp_secret"))
|
|
return view
|
|
|
|
|
|
def require_community(slug: str) -> dict:
|
|
row = store.get(slug)
|
|
if row is None:
|
|
raise not_found("Unknown community")
|
|
return row
|
|
|
|
|
|
def credentials_of(data) -> dict:
|
|
return {"username": data.login_username.strip(), "email": data.login_email.strip(), "password": data.login_password, "totp": data.login_totp_secret.strip()}
|
|
|
|
|
|
def seo(request: Request, title: str, path: str, crumb: str):
|
|
base = site_url(request)
|
|
return base_seo_context(
|
|
request,
|
|
title=f"{title} - Admin",
|
|
description="External communities synchronised into this instance.",
|
|
breadcrumbs=[
|
|
{"name": "Home", "url": "/feed"},
|
|
{"name": "Admin", "url": "/admin"},
|
|
{"name": crumb, "url": path},
|
|
],
|
|
schemas=[website_schema(base)],
|
|
robots="noindex,nofollow",
|
|
)
|
|
|
|
|
|
FILTER_KEYS = ("kind", "outcome", "reason", "q")
|
|
|
|
|
|
def seconds_between(start: str | None, end: str | None) -> int | None:
|
|
try:
|
|
return max(0, round((datetime.fromisoformat(end) - datetime.fromisoformat(start)).total_seconds()))
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def run_view(run: dict) -> dict:
|
|
view = dict(run)
|
|
view["dry_run"] = bool(run.get("dry_run"))
|
|
for column in ("items_stored", "items_dropped"):
|
|
view[column] = int(run.get(column) or 0)
|
|
for column in ("verdict", "headline"):
|
|
view[column] = run.get(column) or ""
|
|
if not view.get("tally") and view.get("pushed"):
|
|
view["tally"] = tally_of(view["pushed"], Ledger())
|
|
view["duration_seconds"] = seconds_between(run.get("started_at"), run.get("finished_at"))
|
|
if not view.get("verdict"):
|
|
outcome = run_verdict(run.get("status", ""), view["dry_run"], view.get("tally") or {})
|
|
view["verdict"], view["headline"] = outcome["level"], outcome["headline"]
|
|
return view
|
|
|
|
|
|
def local_links(rows: list[dict]) -> dict[str, str]:
|
|
wanted: dict[str, list[str]] = {"posts": [], "users": [], "comments": []}
|
|
for row in rows:
|
|
if row.get("local_uid") and row.get("kind") in wanted:
|
|
wanted[row["kind"]].append(row["local_uid"])
|
|
links: dict[str, str] = {}
|
|
if wanted["posts"]:
|
|
table = get_table("posts")
|
|
for found in table.find(table.table.columns.uid.in_(wanted["posts"]), deleted_at=None):
|
|
links[found["uid"]] = f"/posts/{found.get('slug') or found['uid']}"
|
|
for uid, found in get_users_by_uids(wanted["users"]).items():
|
|
links[uid] = f"/profile/{found['username']}"
|
|
if wanted["comments"]:
|
|
table = get_table("comments")
|
|
for found in table.find(table.table.columns.uid.in_(wanted["comments"]), deleted_at=None):
|
|
target = resolve_object_url(found["target_type"], found["target_uid"])
|
|
links[found["uid"]] = f"{target}#comment-{found['uid']}" if target else ""
|
|
return links
|
|
|
|
|
|
def item_view(row: dict, links: dict[str, str]) -> dict:
|
|
view = dict(row)
|
|
view["tone"] = OUTCOME_BY_KEY[row["outcome"]].tone if row.get("outcome") in OUTCOME_BY_KEY else "muted"
|
|
why = explain(row.get("reason", ""))
|
|
view.update(reason_title=why["title"], reason_text=why["text"], retried=why["retried"])
|
|
view["local_url"] = links.get(row.get("local_uid") or "", "")
|
|
return view
|
|
|
|
|
|
def ledger_options(scope: dict) -> dict:
|
|
facets = ledger_store.facets(**scope)
|
|
return {
|
|
"kind": [kind for kind in LEDGER_KINDS if kind in facets["kind"]],
|
|
"outcome": [{"value": outcome.key, "label": outcome.label} for outcome in OUTCOMES],
|
|
"reason": [{"value": reason, "title": explain(reason)["title"]} for reason in facets["reason"]],
|
|
}
|
|
|
|
|
|
def feed_summary(feed: dict) -> dict:
|
|
return {
|
|
"id": feed["id"],
|
|
"type": feed["type"],
|
|
"url": feed["url"],
|
|
"list_path": feed.get("list_path"),
|
|
"item_selector": feed.get("item_selector"),
|
|
"post_fields": feed["post"]["fields"],
|
|
"author_fields": (feed.get("author") or {}).get("fields", {}),
|
|
"comments": (feed.get("comments") or {}).get("mode", "none"),
|
|
"comments_reason": (feed.get("comments") or {}).get("reason", ""),
|
|
"pagination": (feed.get("pagination") or {}).get("style", "none"),
|
|
}
|
|
|
|
|
|
@router.get("/communities", response_class=HTMLResponse)
|
|
async def admin_communities(request: Request, page: int = 1):
|
|
admin = require_admin(request)
|
|
pagination = build_pagination(page, store.count())
|
|
offset = (pagination["page"] - 1) * pagination["per_page"]
|
|
return respond(
|
|
request,
|
|
"admin_communities.html",
|
|
{
|
|
**seo(request, "Communities", LIST_URL, "Communities"),
|
|
"request": request,
|
|
"user": admin,
|
|
"items": [public(row) for row in store.list(pagination["per_page"], offset)],
|
|
"pagination": pagination,
|
|
"sync_enabled": sync_enabled(),
|
|
**service_flags(),
|
|
"admin_section": "communities",
|
|
},
|
|
model=AdminCommunitiesOut,
|
|
)
|
|
|
|
|
|
@router.post("/communities")
|
|
async def admin_community_add(request: Request, data: Annotated[CommunityAddForm, Form()]):
|
|
admin = require_admin(request)
|
|
slug = slug_for_query(data.query)
|
|
if not slug:
|
|
return json_error(400, "Enter a community name or address")
|
|
existing = store.get(slug)
|
|
if existing is None:
|
|
secrets = credentials_of(data)
|
|
existing = store.create(
|
|
slug,
|
|
data.query,
|
|
admin["uid"],
|
|
name=data.query,
|
|
status=STATUS_LEARNING,
|
|
claimed_at=iso(now()),
|
|
login_username=secrets["username"],
|
|
login_email=secrets["email"],
|
|
login_password=secrets["password"],
|
|
login_totp_secret=secrets["totp"],
|
|
)
|
|
await spawn(CommunityRunner().discover(data.query, admin["uid"], relearn_slug=slug))
|
|
audit.record(
|
|
request,
|
|
"admin.community.add",
|
|
user=admin,
|
|
target_type="community",
|
|
target_uid=existing["uid"],
|
|
target_label=slug,
|
|
metadata={"query": data.query},
|
|
summary=f"{admin['username']} added community {data.query!r}",
|
|
)
|
|
return action_result(request, f"{LIST_URL}/{existing['slug']}")
|
|
|
|
|
|
@router.get("/communities/{slug}", response_class=HTMLResponse)
|
|
async def admin_community(request: Request, slug: str):
|
|
admin = require_admin(request)
|
|
row = require_community(slug)
|
|
store.reap_interrupted()
|
|
source = files.read(row["slug"])
|
|
return respond(
|
|
request,
|
|
"admin_community.html",
|
|
{
|
|
**seo(request, row["name"] or slug, f"{LIST_URL}/{slug}", "Communities"),
|
|
"request": request,
|
|
"user": admin,
|
|
"community": public(row),
|
|
"rules": row.get("rules") or "",
|
|
"favicon": row.get("favicon") or "",
|
|
"totals": row.get("totals") or {},
|
|
"feeds": [feed_summary(feed) for feed in (row.get("schema") or {}).get("feeds", [])],
|
|
"runs": [run_view(run) for run in store.run_summaries(row["uid"])],
|
|
"collectors_path": str(files.collectors_path(row["slug"])) if source else "",
|
|
"collectors_lines": len(source.splitlines()),
|
|
"busy": store.busy(row["uid"]),
|
|
"sync_enabled": sync_enabled(),
|
|
"admin_section": "communities",
|
|
},
|
|
model=AdminCommunityOut,
|
|
)
|
|
|
|
|
|
@router.get("/communities/{slug}/ledger", response_class=HTMLResponse)
|
|
async def admin_community_ledger(
|
|
request: Request,
|
|
slug: str,
|
|
run: str = "",
|
|
kind: str = "",
|
|
outcome: str = "",
|
|
reason: str = "",
|
|
q: str = "",
|
|
page: int = 1,
|
|
):
|
|
admin = require_admin(request)
|
|
row = require_community(slug)
|
|
run_row = None
|
|
if run:
|
|
found = store.run(run)
|
|
if found is None or found["community_uid"] != row["uid"]:
|
|
raise not_found("Unknown run")
|
|
run_row = run_view(found)
|
|
scope = {"run_uid": run_row["uid"]} if run_row else {"community_uid": row["uid"]}
|
|
filters = {key: value for key, value in zip(FILTER_KEYS, (kind, outcome, reason, q)) if value}
|
|
rows, pagination = ledger_store.page(filters, page, **scope)
|
|
links = local_links(rows)
|
|
listed = ledger_store.counts(**scope)
|
|
exact = (run_row or {}).get("tally") or listed
|
|
path = f"{LIST_URL}/{slug}/ledger"
|
|
query = urlencode({**({"run": run_row["uid"]} if run_row else {}), **filters})
|
|
return respond(
|
|
request,
|
|
"admin_community_ledger.html",
|
|
{
|
|
**seo(request, f"{row['name'] or slug} sync log", path, "Communities"),
|
|
"request": request,
|
|
"user": admin,
|
|
"community": public(row),
|
|
"scope": "run" if run_row else "community",
|
|
"run": run_row,
|
|
"run_errors": (run_row or {}).get("errors") or [],
|
|
"run_log": (run_row or {}).get("log") or "",
|
|
"busy": (run_row["status"] == "running") if run_row else store.busy(row["uid"]),
|
|
"verdict": {"level": run_row["verdict"], "title": LEVEL_TITLES.get(run_row["verdict"], ""), "headline": run_row["headline"]} if run_row else {},
|
|
"tiles": [
|
|
{"key": o.key, "label": o.label, "tone": o.tone, "help": o.help, "count": int(exact.get(o.key, 0)), "listed": int(listed.get(o.key, 0))}
|
|
for o in OUTCOMES
|
|
],
|
|
"items": [item_view(item, links) for item in rows],
|
|
"pagination": pagination,
|
|
"pagination_query": f"{query}&" if query else "",
|
|
"filters": filters,
|
|
"options": ledger_options(scope),
|
|
"admin_section": "communities",
|
|
},
|
|
model=AdminCommunityLedgerOut,
|
|
)
|
|
|
|
|
|
@router.post("/communities/{slug}/run")
|
|
async def admin_community_run(request: Request, slug: str, data: Annotated[CommunityRunForm, Form()]):
|
|
admin = require_admin(request)
|
|
row = require_community(slug)
|
|
if store.busy(row["uid"]):
|
|
return json_error(409, "A run is already in progress for this community")
|
|
await spawn(CommunityRunner().run(row, dry_run=data.dry_run, trigger=TRIGGER_ADMIN))
|
|
audit.record(
|
|
request,
|
|
"admin.community.run",
|
|
user=admin,
|
|
target_type="community",
|
|
target_uid=row["uid"],
|
|
target_label=slug,
|
|
metadata={"dry_run": data.dry_run},
|
|
summary=f"{admin['username']} started a {'dry run' if data.dry_run else 'run'} of community {slug}",
|
|
)
|
|
return action_result(request, f"{LIST_URL}/{slug}")
|
|
|
|
|
|
@router.post("/communities/{slug}/relearn")
|
|
async def admin_community_relearn(request: Request, slug: str):
|
|
admin = require_admin(request)
|
|
row = require_community(slug)
|
|
if store.busy(row["uid"]):
|
|
return json_error(409, "A run is already in progress for this community")
|
|
await spawn(CommunityRunner().discover(row["query"] or row["url"], admin["uid"], relearn_slug=slug))
|
|
audit.record(
|
|
request,
|
|
"admin.community.relearn",
|
|
user=admin,
|
|
target_type="community",
|
|
target_uid=row["uid"],
|
|
target_label=slug,
|
|
summary=f"{admin['username']} started relearning community {slug}",
|
|
)
|
|
return action_result(request, f"{LIST_URL}/{slug}")
|
|
|
|
|
|
@router.post("/communities/{slug}/update")
|
|
async def admin_community_update(request: Request, slug: str, data: Annotated[CommunityUpdateForm, Form()]):
|
|
admin = require_admin(request)
|
|
row = require_community(slug)
|
|
changes = {"interval_seconds": data.interval_seconds, "enabled": int(data.enabled)}
|
|
if data.clear_login:
|
|
changes.update(login_username="", login_email="", login_password="", login_totp_secret="")
|
|
else:
|
|
for column, value in (("login_username", data.login_username.strip()), ("login_email", data.login_email.strip()), ("login_password", data.login_password), ("login_totp_secret", data.login_totp_secret.strip())):
|
|
if value:
|
|
changes[column] = value
|
|
submitted = {name: getattr(data, name) for name in SyncOptions.__dataclass_fields__ if getattr(data, name) is not None}
|
|
if submitted:
|
|
try:
|
|
options = SyncOptions.from_row(row).merged(**submitted)
|
|
except ValueError as exc:
|
|
return json_error(400, str(exc))
|
|
if options.alias and not SyncOptions.from_row(row).alias and not identity_map(row["origin"]) and count_links(row["origin"]):
|
|
return json_error(400, "This community already has records stored under their original names; aliasing cannot be switched on afterwards")
|
|
changes["sync_options"] = options.to_dict()
|
|
store.update(row["uid"], **changes)
|
|
audit.record(
|
|
request,
|
|
"admin.community.update",
|
|
user=admin,
|
|
target_type="community",
|
|
target_uid=row["uid"],
|
|
target_label=slug,
|
|
metadata={"interval_seconds": data.interval_seconds, "enabled": data.enabled, "sync_options": changes.get("sync_options"), "login_changed": data.clear_login or bool(data.login_password or data.login_username or data.login_email or data.login_totp_secret)},
|
|
summary=f"{admin['username']} updated community {slug}",
|
|
)
|
|
return action_result(request, f"{LIST_URL}/{slug}")
|
|
|
|
|
|
def schema_text(schema) -> str:
|
|
return json.dumps(schema, indent=2, ensure_ascii=False)
|
|
|
|
|
|
def schema_page(request: Request, admin: dict, row: dict, text: str, errors: list[str], notice: str, status_code: int):
|
|
context = {
|
|
**seo(request, f"{row['name'] or row['slug']} schema", f"{LIST_URL}/{row['slug']}/schema", "Communities"),
|
|
"request": request,
|
|
"user": admin,
|
|
"community": public(row),
|
|
"schema_text": text,
|
|
"has_previous": bool(row.get("schema_previous")),
|
|
"errors": errors,
|
|
"notice": notice,
|
|
"admin_section": "communities",
|
|
}
|
|
response = respond(request, "admin_community_schema.html", context, model=AdminCommunitySchemaOut)
|
|
response.status_code = status_code
|
|
return response
|
|
|
|
|
|
@router.get("/communities/{slug}/schema", response_class=HTMLResponse)
|
|
async def admin_community_schema(request: Request, slug: str):
|
|
admin = require_admin(request)
|
|
row = require_community(slug)
|
|
return schema_page(request, admin, row, schema_text(row.get("schema") or {}), [], "", 200)
|
|
|
|
|
|
@router.post("/communities/{slug}/schema")
|
|
async def admin_community_schema_save(request: Request, slug: str, data: Annotated[CommunitySchemaForm, Form()]):
|
|
admin = require_admin(request)
|
|
row = require_community(slug)
|
|
if store.busy(row["uid"]):
|
|
return json_error(409, "A run is already in progress for this community")
|
|
try:
|
|
schema = json.loads(data.schema_text)
|
|
except ValueError as exc:
|
|
return schema_page(request, admin, row, data.schema_text, [f"The text is not valid JSON: {exc}"], "", 400)
|
|
errors = validate_schema(schema)
|
|
if errors:
|
|
return schema_page(request, admin, row, data.schema_text, errors, "", 400)
|
|
verdict = await CommunityRunner().verify_schema(row, schema)
|
|
if not verdict.ok:
|
|
failed = ["The schema was checked against the live source and was not saved."] + verdict.problems[:10]
|
|
return schema_page(request, admin, row, data.schema_text, failed, "", 422)
|
|
store.update(
|
|
row["uid"],
|
|
schema=schema,
|
|
schema_previous=row.get("schema") or {},
|
|
status=STATUS_READY,
|
|
last_error="",
|
|
failure_count=0,
|
|
enabled=1,
|
|
)
|
|
files.write(row["slug"], schema)
|
|
audit.record(
|
|
request,
|
|
"admin.community.schema",
|
|
user=admin,
|
|
target_type="community",
|
|
target_uid=row["uid"],
|
|
target_label=slug,
|
|
metadata={"feeds": len(schema["feeds"]), "sample_posts": verdict.posts, "sample_comments": verdict.comments},
|
|
summary=f"{admin['username']} replaced the learned schema of community {slug}",
|
|
)
|
|
return action_result(request, f"{LIST_URL}/{slug}/schema", data={"posts": verdict.posts, "comments": verdict.comments})
|
|
|
|
|
|
@router.post("/communities/{slug}/schema/restore")
|
|
async def admin_community_schema_restore(request: Request, slug: str):
|
|
admin = require_admin(request)
|
|
row = require_community(slug)
|
|
previous = row.get("schema_previous") or {}
|
|
if not previous:
|
|
return json_error(409, "There is no previous schema to restore")
|
|
errors = validate_schema(previous)
|
|
if errors:
|
|
return schema_page(request, admin, row, schema_text(previous), errors, "", 400)
|
|
store.update(row["uid"], schema=previous, schema_previous=row.get("schema") or {})
|
|
files.write(row["slug"], previous)
|
|
audit.record(
|
|
request,
|
|
"admin.community.schema_restore",
|
|
user=admin,
|
|
target_type="community",
|
|
target_uid=row["uid"],
|
|
target_label=slug,
|
|
summary=f"{admin['username']} restored the previous schema of community {slug}",
|
|
)
|
|
return action_result(request, f"{LIST_URL}/{slug}/schema")
|
|
|
|
|
|
@router.post("/communities/{slug}/delete")
|
|
async def admin_community_delete(request: Request, slug: str):
|
|
admin = require_admin(request)
|
|
row = require_community(slug)
|
|
store.delete(row["uid"], admin["uid"])
|
|
audit.record(
|
|
request,
|
|
"admin.community.delete",
|
|
user=admin,
|
|
target_type="community",
|
|
target_uid=row["uid"],
|
|
target_label=slug,
|
|
summary=f"{admin['username']} deleted community {slug}",
|
|
)
|
|
return action_result(request, LIST_URL)
|