Verify live game panels and leaderboard API contents

Document auto-replant and POST /game/plot/stop in detail for automation authors
(exact conditions, quest/audit differences, race behavior, rotation recipe),
add an Exact formulas section (build time, rewards, raid payout, quests,
contracts, market supply), document the remaining crop/perk/legacy state fields.
Fix: Golden Parachute carry-over is up to 60%, crops reward_xp is base XP, and
stale game CLAUDE.md notes (Mastery step 5, raid notification names the raider).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

.coveragerc

@@ -1,7 +1,7 @@
 # retoor <retoor@molodetz.nl>
 [run]
 source = devplacepy
-parallel = true
+parallel = false
 sigterm = true
 omit =
     tests/*

.env.example

@@ -29,9 +29,16 @@ SECRET_KEY=change-me
 # from the request.
 DEVPLACE_SITE_URL=
 
-# Host port the nginx front door binds.
+# Host port the nginx front door binds (Docker only - the app container's own
+# internal port stays fixed).
 PORT=10500
 
+# Port the uvicorn process itself binds to for `make dev`/`make prod` (bare
+# metal, no Docker/nginx in front). Also what the app calls itself on
+# internally (DEVII_BASE_URL default, INTERNAL_GATEWAY_URL). Unrelated to
+# PORT above - leave unset unless running bare metal on a non-default port.
+# DEVPLACE_PORT=10500
+
 # nginx upload ceiling. Must be >= the admin-configurable max_upload_size_mb.
 NGINX_MAX_BODY_SIZE=50m
 
@@ -42,3 +49,13 @@ NGINX_CACHE_MAX_SIZE=1g
 # Run the app container as this host user so shared files keep dev ownership.
 DEVPLACE_UID=1000
 DEVPLACE_GID=1000
+
+# OpenCode Zen client identity (devplacepy/services/openai_gateway/opencode_zen.py).
+# Optional: both already default to these exact values, which match the real
+# opencode CLI's own headers - only override if opencode ships a new version
+# and Zen starts rejecting the old one. The provider's base URL, API key, and
+# which model(s) route to it are NOT set here - those live in the
+# gateway_providers/gateway_models tables, configured at /admin/gateway
+# (provider client profile "OpenCode Zen").
+# OPENCODE_CLIENT_NAME=cli
+# OPENCODE_CLIENT_USER_AGENT=opencode/1.18.29 ai-sdk/provider-utils/4.0.23 runtime/bun/1.3.15

.githooks/pre-commit

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+# retoor <retoor@molodetz.nl>
+
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(
+    subprocess.run(
+        ["git", "rev-parse", "--show-toplevel"],
+        capture_output=True,
+        text=True,
+        check=True,
+    ).stdout.strip()
+)
+PYPROJECT = REPO_ROOT / "pyproject.toml"
+VERSION_LINE = re.compile(r'^version = "(\d+)\.(\d+)\.(\d+)"$', re.MULTILINE)
+STAGED_VERSION_CHANGE = re.compile(r'^[+-]version = "\d+\.\d+\.\d+"$', re.MULTILINE)
+
+
+def staged_diff(path: Path) -> str:
+    result = subprocess.run(
+        ["git", "diff", "--cached", "--unified=0", "--", str(path)],
+        capture_output=True,
+        text=True,
+        cwd=REPO_ROOT,
+    )
+    return result.stdout
+
+
+def main() -> int:
+    if (REPO_ROOT / ".git" / "MERGE_HEAD").exists():
+        return 0
+    if not PYPROJECT.is_file():
+        return 0
+    if STAGED_VERSION_CHANGE.search(staged_diff(PYPROJECT)):
+        return 0
+    text = PYPROJECT.read_text()
+    match = VERSION_LINE.search(text)
+    if not match:
+        return 0
+    major, minor, patch = (int(part) for part in match.groups())
+    bumped = f'version = "{major}.{minor}.{patch + 1}"'
+    PYPROJECT.write_text(VERSION_LINE.sub(bumped, text, count=1))
+    subprocess.run(["git", "add", str(PYPROJECT)], cwd=REPO_ROOT, check=True)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.gitignore

@@ -1,3 +1,7 @@
+.docs
+.dpc
+.devplace
+dpc.log
 .cache
 .local
 .devplace_bots/
@@ -13,6 +17,10 @@ devplace-init.lock
 notification-private.pem
 notification-private.pkcs8.pem
 notification-public.pem
+# HOME=/app in the Docker app container, so an rclone.conf created via
+# `rclone config` or DEVPLACE_RCLONE_CONFIG's default lands here on the
+# bind-mounted host tree - it holds live remote-storage credentials.
+/.config/
 .pytest_cache/
 .ruff_cache/
 .opencode

ARCHITECTURE.md

@@ -1,636 +0,0 @@
-# DevPlace architecture atlas
-
-Author: retoor <retoor@molodetz.nl>
-
-Every diagram below was derived from the source tree, the live SQLite schema and the imported FastAPI app object, not from the documentation. Where the repository documentation and the running code disagree, the disagreement is recorded in the last section.
-
-| Measure | Value |
-|---|---|
-| python files | 644 |
-| routers | 38 |
-| http paths | 348 |
-| websockets | 12 |
-| services | 27 |
-| db tables | 102 |
-| agent tools | 318 |
-| test functions | 3073 |
-
-## 01 Deployment topology
-
-Source: `docker-compose.yml / Dockerfile / nginx/nginx.conf.template`
-
-Two containers on one bridge network, on a single host. nginx is the only published listener and binds to loopback only. The app container bind-mounts the repository root, so the production process reads the same SQLite file and the same `data/` tree as `make dev`.
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"13px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart TB
-  client["Browser / API client / Telegram / devRant client"]
-  subgraph host["Single host"]
-    subgraph appnet["docker network: appnet"]
-      nginx["nginx container<br/>127.0.0.1:PORT to :80<br/>8 websocket upgrade locations"]
-      app["app container<br/>uvicorn devplacepy.main:app<br/>--workers 2 --backlog 8192"]
-    end
-    repo["repository root<br/>bind mount .:/app"]
-    static["devplacepy/static<br/>read only mount"]
-    data["DEVPLACE_DATA_DIR = data/<br/>23 registered paths"]
-    sqlite[("data/devplace.db<br/>SQLite, WAL, 256MB mmap")]
-    lock["data/locks/devplace-services.lock<br/>flock, elects one service owner"]
-    docker["host Docker daemon<br/>ppy:latest instances"]
-  end
-  client --> nginx
-  nginx -->|"proxy_pass, X-Real-IP"| app
-  nginx -->|"/static/, /static/uploads/"| static
-  app --> repo
-  app --> data
-  data --> sqlite
-  app --> lock
-  app -->|"docker CLI backend"| docker
-```
-
-nginx depends_on app with condition service_healthy. App healthcheck: interval 30s, timeout 10s, retries 3, start_period 120s, start_interval 2s.
-
-## 02 Worker boot and service election
-
-Source: `devplacepy/main.py lifespan`
-
-Every uvicorn worker runs the full lifespan. Schema work is serialized behind an exclusive lock so workers pay it one after another; background services are elected, so exactly one worker in the host owns them.
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"13px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart TB
-  s1["ensure_data_dirs<br/>creates all 23 DATA_PATHS"]
-  s2["init_lock<br/>exclusive flock on INIT_LOCK_FILE"]
-  s3["init_db<br/>idempotent columns and indexes"]
-  s4["ensure_certificates<br/>VAPID keys in data/keys"]
-  s5["service_manager.register x27"]
-  gate{"DEVPLACE_DISABLE_SERVICES set?"}
-  skip["services registered but never started<br/>test mode"]
-  bg["background.start<br/>per worker asyncio queue"]
-  elect{"acquire_service_lock"}
-  owner["set_lock_owner True<br/>supervise all 27 services"]
-  decline["declined, another worker owns them"]
-  vis["start_visit_flusher"]
-  serve(["serving on port 10500"])
-  s1 --> s2 --> s3 --> s4 --> s5 --> gate
-  gate -->|yes| skip --> vis
-  gate -->|no| bg --> elect
-  elect -->|"lock acquired"| owner --> vis
-  elect -->|"lock held elsewhere"| decline --> vis
-  vis --> serve
-```
-
-Shutdown reverses it: flush_visits, service_manager.shutdown_all, background.stop.
-
-## 03 Request pipeline
-
-Source: `devplacepy/main.py, verified against app.user_middleware`
-
-Ten middlewares wrap every request. The order below is the real execution order read off the built middleware stack, outermost first. Two of them are plain ASGI classes; the other eight are `BaseHTTPMiddleware` dispatch functions.
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"13px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart TB
-  req(["incoming ASGI scope"])
-  m0["0 GZipMiddleware<br/>minimum_size 512, level 5"]
-  m1["1 TunnelDispatchMiddleware<br/>host matches a workspace tunnel?"]
-  tun["routers/tunnel.py<br/>handle_http / handle_ws"]
-  m2["2 response_timing<br/>request.state.request_start, X-Response-Time"]
-  m3["3 visit_statistics"]
-  m4["4 track_presence<br/>throttled last_seen write"]
-  m5["5 maintenance_middleware<br/>503 unless static, avatar, auth, admin"]
-  m6["6 rate_limit_middleware<br/>mutating methods only, per IP bucket"]
-  m7["7 add_security_headers"]
-  m8["8 await_pending_corrections<br/>drains sync AI correction futures"]
-  m9["9 refresh_db_snapshot"]
-  mounts["mounts: /static/uploads, /static/v{ts}, /static"]
-  routes["38 included routers"]
-  handlers["exception handlers<br/>404, 500, RequestValidationError"]
-  req --> m0 --> m1
-  m1 -->|"tunnel host"| tun
-  m1 -->|"normal host"| m2 --> m3 --> m4 --> m5 --> m6 --> m7 --> m8 --> m9
-  m9 --> mounts
-  m9 --> routes
-  routes --> handlers
-```
-
-Rate limiting reads rate_limit_per_minute and rate_limit_window_seconds from site_settings and exempts reads and the /openai gateway. Bucket key is X-Real-IP falling back to request.client.host.
-
-## 04 URL surface
-
-Source: `devplacepy/main.py include_router calls`
-
-38 routers, each mounted under one prefix, grouped here by the concern they serve. Directory shape mirrors the URL: a domain with one resource is a flat module, a domain with several sub-resources is a package that aggregates its leaves in `__init__.py`.
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"12px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart LR
-  app(["FastAPI app"])
-  subgraph identity["Identity and profile"]
-    r1["/auth - auth/"]
-    r2["/profile - profile/"]
-    r3["/avatar - avatar.py"]
-    r4["/follow - follow.py"]
-    r5["(none) - relations.py block, mute"]
-  end
-  subgraph social["Content and engagement"]
-    r6["/feed - feed.py"]
-    r7["/posts - posts.py"]
-    r8["/comments - comments.py"]
-    r9["/gists - gists.py"]
-    r10["/news - news.py"]
-    r11["/votes - votes.py"]
-    r12["/reactions - reactions.py"]
-    r13["/bookmarks - bookmarks.py"]
-    r14["/polls - polls.py"]
-    r15["/awards - awards.py"]
-    r16["/leaderboard - leaderboard.py"]
-    r17["/notifications - notifications.py"]
-    r18["/messages - messages.py"]
-    r19["/uploads - uploads.py"]
-    r20["/media - media.py"]
-  end
-  subgraph work["Projects and compute"]
-    r21["/projects - projects/ incl files, containers"]
-    r22["/p - proxy.py container ingress"]
-    r23["/zips - zips.py"]
-    r24["/forks - forks.py"]
-    r25["/issues - issues/"]
-  end
-  subgraph aiml["AI and tools"]
-    r26["/openai - openai_gateway.py"]
-    r27["/devii - devii.py"]
-    r28["/tools - tools/ seo, deepsearch, isslop"]
-  end
-  subgraph play["Play"]
-    r29["/game - game/"]
-    r30["/quizzes - quizzes/"]
-  end
-  subgraph platform["Platform"]
-    r31["/admin - admin/ 23 modules"]
-    r32["/docs - docs/"]
-    r33["(none) - push.py, seo.py"]
-  end
-  subgraph machine["Machine interfaces"]
-    r34["/api - devrant/"]
-    r35["/dbapi - dbapi/ read only"]
-    r36["/xmlrpc - xmlrpc.py"]
-    r37["/pubsub - pubsub.py"]
-  end
-  app --> identity
-  app --> social
-  app --> work
-  app --> aiml
-  app --> play
-  app --> platform
-  app --> machine
-```
-
-| Prefix | Paths | Prefix | Paths | Prefix | Paths |
-|---|---|---|---|---|---|
-| /admin | 92 | /profile | 17 | /gists | 5 |
-| /projects | 46 | /issues | 13 | /messages | 5 |
-| /tools | 27 | /dbapi | 9 | /notifications | 5 |
-| /game | 23 | /auth | 6 | /uploads | 5 |
-| /quizzes | 21 | /devii | 5 | /docs | 4 |
-| /api | 20 | /posts | 4 | everything else | 36 |
-
-## 05 Route fan-out
-
-Source: `devplacepy/responses.py, schemas/, docs_api/, services/devii/actions/`
-
-One handler serves four consumers. `respond()` feeds the same context dict to a Pydantic model and to a Jinja template, so a key missing from the `*Out` schema is silently absent from JSON while still rendering in HTML. The agent catalog and the docs registry are separate declarations that must be kept in step by hand.
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"13px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart TB
-  form["models.py<br/>Pydantic form model via Annotated Form"]
-  guard["guard: get_current_user<br/>require_user / require_admin"]
-  handler["router handler"]
-  helpers["database/ batch helpers<br/>get_users_by_uids, build_pagination"]
-  ctx["context dict"]
-  neg{"wants_json(request)"}
-  json["model.model_validate<br/>JSONResponse"]
-  html["templates.TemplateResponse<br/>extends base.html"]
-  action["services/devii/actions catalog<br/>Action, requires_auth mirrors guard"]
-  docs["docs_api endpoint()<br/>params and sample_response"]
-  seo["seo.py base_seo_context<br/>JSON-LD, sitemap entry"]
-  form --> handler
-  guard --> handler
-  handler --> helpers --> ctx --> neg
-  neg -->|"json accepted"| json
-  neg -->|"otherwise"| html
-  handler -.->|"same capability"| action
-  handler -.->|"same contract"| docs
-  html -.-> seo
-```
-
-wants_json is true when the request content-type starts with application/json, or Accept contains application/json and does not contain text/html. Registry sizes: 318 agent actions, 259 documented endpoint entries across 20 API groups, 122 prose docs pages.
-
-## 06 Identity resolution
-
-Source: `devplacepy/utils/auth.py`
-
-A single resolver serves browsers, API clients, the devRant compatibility layer and issued access tokens. The result is memoised on `request.state` for the request and in a process TTL cache keyed by credential.
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"13px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart TB
-  start(["get_current_user"])
-  cached{"request.state._auth_user set?"}
-  sess{"session cookie<br/>64 hex chars"}
-  xapi{"X-API-KEY header"}
-  bearer{"Authorization: Bearer"}
-  basic{"Authorization: Basic"}
-  triple["try in order:<br/>users.api_key,<br/>devrant token 40 hex,<br/>access token 64 chars"]
-  user(["user dict"])
-  guest(["None, guest"])
-  gu["require_user: 303 to /"]
-  ga["require_admin: redirect to /feed"]
-  start --> cached
-  cached -->|yes| user
-  cached -->|no| sess
-  sess -->|match| user
-  sess -->|no| xapi
-  xapi -->|present| triple
-  triple -->|match| user
-  triple -->|no match| bearer
-  xapi -->|absent| bearer
-  bearer -->|present| triple
-  bearer -->|absent| basic
-  basic -->|"username-or-email:password, pbkdf2_sha256"| user
-  basic -->|no| guest
-  guest --> gu
-  user --> ga
-```
-
-Roles are stored capitalized as Admin or Member and tested through the is_admin global. Admin seniority is enforced per user mutation in routers/admin/users.py.
-
-## 07 Background service fleet
-
-Source: `devplacepy/services/manager.py, base.py, jobs/base.py`
-
-27 singletons registered at boot, supervised only by the worker that won the service lock. Sixteen are long-lived loops on `BaseService`; eleven are queue consumers on `JobService`, which adds retention, concurrency and timeout settings on top and drains rows from the shared `jobs` table.
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"12px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart LR
-  mgr["ServiceManager<br/>supervise, shutdown_all"]
-  settings[("site_settings<br/>enabled flag and interval per service")]
-  jobs[("jobs table")]
-  subgraph base["BaseService loops - 16"]
-    b1["NewsService"]
-    b2["BotsService"]
-    b3["GatewayService"]
-    b4["DeviiService"]
-    b5["PubSubService"]
-    b6["NotificationRelayService"]
-    b7["LiveViewRelayService"]
-    b8["PresenceRelayService"]
-    b9["IssueTrackerService"]
-    b10["ContainerService"]
-    b11["WorkspaceService"]
-    b12["XmlrpcService"]
-    b13["AuditService"]
-    b14["PushService"]
-    b15["TelegramService"]
-    b16["TelegramOutboxService"]
-  end
-  subgraph job["JobService consumers - 11"]
-    j1["ZipService"]
-    j2["ForkService"]
-    j3["SeoService"]
-    j4["SeoMetaService"]
-    j5["AwardService"]
-    j6["BackupService"]
-    j7["DbApiJobService"]
-    j8["DeepsearchService"]
-    j9["IsslopService"]
-    j10["IssueCreateService"]
-    j11["PlanningReportService"]
-  end
-  mgr --> base
-  mgr --> job
-  settings --> mgr
-  job --> jobs
-```
-
-Separate from the fleet, services/background.py offers a fire and forget queue per worker for audit writes, XP awards and notifications; when no consumer runs, as in tests, the callable executes inline so ordering stays deterministic.
-
-## 08 Data layer
-
-Source: `devplacepy/database/, devplacepy/config.py`
-
-One SQLite file reached through `dataset`, called synchronously from async handlers by design. 102 tables exist in the live schema; 52 of them carry the soft delete pair and are restorable as one event from the admin trash.
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"12px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart TB
-  db[("data/devplace.db")]
-  subgraph g1["Identity and access - 14"]
-    t1["users, sessions, password_resets<br/>access_tokens, devrant_tokens<br/>user_relations, follows<br/>notification_preferences, user_customizations<br/>push_registration, email_accounts<br/>telegram_links, telegram_pairings, telegram_outbox"]
-  end
-  subgraph g2["Content and engagement - 16"]
-    t2["posts, comments, gists<br/>projects, project_files, project_forks<br/>attachments, news, news_images, news_sync<br/>polls, poll_options, poll_votes<br/>votes, reactions, bookmarks"]
-  end
-  subgraph g3["Reputation - 5"]
-    t3["awards, badges, award_usage<br/>user_activity, user_activity_seen"]
-  end
-  subgraph g4["Code Farm - 9"]
-    t4["game_farms, game_plots, game_quests<br/>game_cosmetics, game_market_ticks<br/>game_steals, game_treasury<br/>game_eras, game_era_results"]
-  end
-  subgraph g5["Quizzes - 5"]
-    t5["quizzes, quiz_questions, quiz_options<br/>quiz_attempts, quiz_answers"]
-  end
-  subgraph g6["Live delivery - 3"]
-    t6["messages, notifications, ws_tickets"]
-  end
-  subgraph g7["Devii - 8"]
-    t7["devii_conversations, devii_turns<br/>devii_tasks, devii_task_runs<br/>devii_lessons, devii_virtual_tools<br/>devii_behavior, devii_usage_ledger"]
-  end
-  subgraph g8["AI gateway - 6"]
-    t8["gateway_providers, gateway_models<br/>gateway_usage_ledger<br/>gateway_quota_rules, gateway_quota_resets<br/>gateway_concurrency_samples"]
-  end
-  subgraph g9["Jobs, tools and usage - 20"]
-    t9["jobs, backups, backup_schedules<br/>deepsearch_sessions, deepsearch_messages, deepsearch_url_cache<br/>isslop_analyses, isslop_reports, isslop_events<br/>isslop_dom_results, isslop_file_results, isslop_image_results<br/>seo_metadata, seo_usage<br/>issue_tickets, issue_comment_authors, issue_usage<br/>correction_usage, modifier_usage, news_usage"]
-  end
-  subgraph g10["Containers - 9"]
-    t10["instances, instance_events, instance_metrics<br/>tunnels, workspace_flags, workspace_quota_rules<br/>builds, dockerfiles, dockerfile_versions"]
-  end
-  subgraph g11["Platform state - 7"]
-    t11["site_settings, cache_state, service_state<br/>audit_log, audit_log_links<br/>visit_stats_hourly, visit_unique_slots"]
-  end
-  db --> g1
-  db --> g2
-  db --> g3
-  db --> g4
-  db --> g5
-  db --> g6
-  db --> g7
-  db --> g8
-  db --> g9
-  db --> g10
-  db --> g11
-```
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"12px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart LR
-  root["DEVPLACE_DATA_DIR<br/>default repo/data"]
-  blobs["uploads, attachments, project_files<br/>sharded xx/yy on the uuid7 random tail"]
-  jobsdir["zips, zip_staging, fork_staging<br/>backups, backup_staging"]
-  reports["seo_reports, planning_reports<br/>dbapi, deepsearch, deepsearch_chroma"]
-  isslop["isslop, isslop_workspaces<br/>isslop_runs, isslop_media"]
-  ws["container_workspaces, workspace_state"]
-  keys["keys VAPID, bot, locks"]
-  dbs["devplace.db, devii_tasks.db, devii_lessons.db"]
-  root --> blobs
-  root --> jobsdir
-  root --> reports
-  root --> isslop
-  root --> ws
-  root --> keys
-  root --> dbs
-```
-
-config.DATA_PATHS registers 23 directories and ensure_data_dirs creates the whole tree before any write. Uploads live under data/uploads but are served at the unchanged /static/uploads URL.
-
-## 09 Content rendering
-
-Source: `devplacepy/rendering.py, static/js/ContentRenderer.js`
-
-Two pipelines with a deliberate split: anything that exists at request time is rendered on the server for SEO, and the client pipeline is reserved for content that does not exist yet. Each has its own XSS control at a different point.
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"13px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart TB
-  subgraph server["Server, rendering.py, lru_cache"]
-    a1["raw text"]
-    a2["normalize dashes to hyphen"]
-    a3["emoji shortcodes, 4869 names"]
-    a4["mistune GFM, escape=True"]
-    a5["media pass: bare URLs to embeds, mentions to links"]
-    a6["mask emails on rendered text nodes"]
-    a7["render_content / render_title in template"]
-    a8["ContentEnhancer adds highlighting and copy buttons"]
-    a1 --> a2 --> a3 --> a4 --> a5 --> a6 --> a7 --> a8
-  end
-  subgraph client["Client, ContentRenderer.js"]
-    b1["live text: comments, DM bubbles, Devii, DeepSearch"]
-    b2["marked"]
-    b3["DOMPurify.sanitize, fail closed"]
-    b4["highlight.js"]
-    b5["media and autolink pass"]
-    b1 --> b2 --> b3 --> b4 --> b5
-  end
-```
-
-The server escapes at the markdown step, the client sanitizes after parsing. Server rendered content must not carry data-render, or both pipelines run over it.
-
-## 10 AI plane
-
-Source: `devplacepy/routers/openai_gateway.py, services/openai_gateway/, services/devii/`
-
-Every model call in the product, internal or external, leaves through one gateway, which is where routing, quota and cost attribution live. Internal consumers call it over loopback HTTP rather than importing a client, so their spend lands in the same ledger.
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"12px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart TB
-  ext["external client<br/>/openai/v1/*"]
-  subgraph consumers["Internal consumers via INTERNAL_GATEWAY_URL"]
-    c1["services/devii"]
-    c2["services/correction<br/>AI correction and modifier"]
-    c3["services/news"]
-    c4["services/bot"]
-    c5["services/deepsearch + jobs/deepsearch"]
-    c6["services/dbapi nl2sql"]
-    c7["services/gitea enhance + planning"]
-    c8["jobs/isslop"]
-  end
-  gw["gateway.py<br/>routing.py, quota.py, usage.py<br/>reliability.py, vision.py"]
-  ledger[("gateway_usage_ledger<br/>gateway_quota_rules")]
-  stealth["stealth_async_client<br/>curl_cffi Chrome 146 fingerprint"]
-  up["upstream provider"]
-  subgraph devii["Devii assistant"]
-    d1["registry.py CATALOG<br/>318 tools"]
-    d2["role gating<br/>guest 108, member 246<br/>admin 312, primary 318"]
-    d3["45 tools behind CONFIRM_REQUIRED"]
-    d4["session, tasks, lessons<br/>virtual tools, behavior"]
-    d5["surfaces: /devii ws, Telegram, CLI"]
-  end
-  ext --> gw
-  consumers --> gw
-  gw --> ledger
-  gw --> stealth --> up
-  d1 --> d2 --> d3
-  devii --> c1
-  d4 --> d1
-  d5 --> d4
-```
-
-Cleartext loopback calls are forced to HTTP/1.1 in curl_transport.http_version_for, because the Chrome impersonation profile would otherwise negotiate HTTP/2 against an HTTP/1.1 only uvicorn.
-
-## 11 Containers, workspaces and ingress
-
-Source: `devplacepy/services/containers/, routers/projects/containers/, routers/proxy.py, routers/tunnel.py`
-
-One shared image serves every instance. Reconciliation is a loop that compares desired state in the database against what the Docker daemon actually reports, and there are two independent ways in from the outside: a path prefix and a hostname.
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"12px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart TB
-  ui["/projects/{slug}/containers<br/>instances, schedules, workspace"]
-  api["services/containers/api.py"]
-  store[("instances, instance_events<br/>instance_metrics, tunnels<br/>workspace_flags, workspace_quota_rules")]
-  svc["ContainerService<br/>reconcile desired vs docker ps"]
-  wsvc["WorkspaceService<br/>provision, certs, quota, tunnels"]
-  backend["backend/docker_cli.py<br/>backend/fake.py for tests"]
-  image["single shared image ppy:latest"]
-  inst["running instance<br/>rootless workflow, aptroot"]
-  wsdir["data/container_workspaces/{uid}"]
-  proxy["/p/{slug}<br/>routers/proxy.py, relays headers verbatim"]
-  tunnel["host based tunnel<br/>TunnelDispatchMiddleware to routers/tunnel.py"]
-  code["workspace editor over websocket<br/>/projects/{slug}/containers/instances/{uid}/code"]
-  exec["terminal over websocket<br/>.../exec/ws"]
-  ui --> api --> store
-  svc --> store
-  wsvc --> store
-  api --> backend
-  svc --> backend
-  wsvc --> wsdir
-  backend --> image --> inst
-  proxy --> inst
-  tunnel --> inst
-  code --> inst
-  exec --> inst
-```
-
-Access uses stricter predicates than the rest of the product: owns_instance, can_view_project_containers, can_view_instance, can_manage_instance. The primary administrator sees and manages every container; any other admin can only view others on public projects and manage instances they own.
-
-## 12 Real time plane
-
-Source: `routers/*.py websocket handlers, services relays, nginx.conf.template`
-
-Twelve WebSocket endpoints, each of which needs its own nginx upgrade location because the catch-all location strips upgrade headers. Fan-out to connected sockets goes through in-process hubs driven by relay services on the lock-owning worker.
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"12px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart LR
-  subgraph sockets["WebSocket endpoints"]
-    s1["/devii/ws"]
-    s2["/messages/ws"]
-    s3["/pubsub/ws"]
-    s4["/tools/seo/{uid}/ws"]
-    s5["/tools/deepsearch/{uid}/ws"]
-    s6["/tools/deepsearch/{uid}/chat"]
-    s7["/dbapi/query/{uid}/ws"]
-    s8[".../containers/instances/{uid}/exec/ws"]
-    s9[".../containers/instances/{uid}/code"]
-    s10[".../code/{path}"]
-    s11["/p/{slug}"]
-    s12["/p/{slug}/{path}"]
-  end
-  subgraph relays["Relay services"]
-    r1["NotificationRelayService"]
-    r2["LiveViewRelayService"]
-    r3["PresenceRelayService<br/>track limit 500, online limit 30"]
-    r4["PubSubService"]
-  end
-  hubs["messaging/hub.py, pubsub/hub.py<br/>devii/hub.py"]
-  tick[("ws_tickets<br/>expiring auth tickets")]
-  s2 --> hubs
-  s3 --> hubs
-  s1 --> hubs
-  relays --> hubs
-  tick --> s2
-```
-
-Presence is authoritative from the relay: PRESENCE_TRACK_LIMIT sets the tracked online set, PRESENCE_ONLINE_LIMIT only caps how many the feed panel displays, and PRESENCE_ONLINE_MARGIN_SECONDS provides hysteresis at the boundary.
-
-## 13 Frontend
-
-Source: `devplacepy/static/, devplacepy/templates/`
-
-No framework and no package manager. 131 ES6 modules, one class per file, hung off a single global `app`; 33 of them are custom elements. 217 Jinja templates, of which 107 are the docs site.
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"12px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart TB
-  base["templates/base.html<br/>extra_head for CSS, extra_js for JS"]
-  partials["shared partials<br/>_avatar_link, _user_link, _comment_section<br/>_post_card, _pagination, _sidebar_search"]
-  appjs["static/js/Application.js<br/>instantiated once as app"]
-  subgraph modules["131 ES6 modules"]
-    u1["utilities<br/>Http, Poller, JobPoller<br/>OptimisticAction, FloatingWindow, ScrollMemory"]
-    u2["33 custom elements<br/>components/ with dp- prefix"]
-    u3["feature modules<br/>chat/, devii/, autoload/"]
-  end
-  css["49 stylesheets<br/>variables.css tokens, per page files"]
-  vers["static_url in Jinja, assetUrl in JS<br/>/static/v{boot ts}/, immutable for a year"]
-  cust["per user CSS and JS injection<br/>custom_css_tag, custom_js_tag"]
-  base --> partials
-  base --> appjs --> modules
-  base --> css
-  base --> cust
-  vers --> css
-  vers --> modules
-```
-
-Per user customizations are configured only through Devii, stored in user_customizations, scoped globally or per matched route template, and run solely in that owner's own browser sessions.
-
-## 14 Test and delivery topology
-
-Source: `tests/, pyproject.toml, .gitea/workflows/test.yaml`
-
-Three tiers separated by what they exercise, decided by fixtures, run serially in a single process against one uvicorn subprocess on port 10501 with a temporary database.
-
-```mermaid
-%%{init:{"theme":"base","themeVariables":{"background":"#F4F8F7","primaryColor":"#E4EEED","primaryTextColor":"#0F1A1B","primaryBorderColor":"#0A6D6A","secondaryColor":"#EDE6D9","tertiaryColor":"#F4F8F7","lineColor":"#6B7C7C","textColor":"#0F1A1B","fontFamily":"ui-monospace,SFMono-Regular,Menlo,monospace","fontSize":"12px","clusterBkg":"#FAFCFC","clusterBorder":"#C6D4D3"}}}%%
-flowchart LR
-  unit["tests/unit - 140 modules<br/>mirrors source module path<br/>local_db or no fixture"]
-  api["tests/api - 221 modules<br/>mirrors URL path<br/>app_server, seeded_db"]
-  e2e["tests/e2e - 131 modules<br/>mirrors URL path<br/>page, alice, bob"]
-  srv["uvicorn subprocess :10501<br/>temp SQLite, own DATA_DIR<br/>DEVPLACE_DISABLE_SERVICES=1"]
-  pw["Playwright Chromium<br/>session scoped context"]
-  ci["Gitea Actions on push and PR to master<br/>full suite under coverage"]
-  prod["promotion to production<br/>docker compose"]
-  unit --> ci
-  api --> srv --> ci
-  e2e --> srv
-  e2e --> pw --> ci
-  ci --> prod
-```
-
-pytest-xdist is not a dependency and -n is rejected centrally, so no tier can be parallelised by accident.
-
-## 15 Findings
-
-Source: `measured against the repository documentation`
-
-The structure holds up: the router tree mirrors the URL tree, the test tree mirrors both, every runtime path is registered in one place, and every model call has a single exit. The items below are the places where the running code and the written record have drifted apart, or where a structure exists that nothing currently uses. None of them is a functional defect.
-
-### Middleware order in the documentation is stale
-
-The root CLAUDE.md states that response_timing is the outermost middleware. Measured from app.user_middleware it is third, behind GZipMiddleware and TunnelDispatchMiddleware, both added after it. The consequence is minor but real: X-Response-Time excludes compression time and excludes tunnel-host dispatch entirely.
-
-### Two catalogue counts have drifted
-
-CLAUDE.md cites around 2882 tests and an events.md catalogue of 288 keys. Measured now: 3073 functions named test_ across 492 modules, and 312 distinct dotted event keys in events.md. Both are undercounts in the docs, not missing implementation.
-
-### instance_schedules is declared soft-deletable but has no table
-
-It appears in database.SOFT_DELETE_TABLES and in the routers under /projects/{slug}/containers/schedules, but no such table exists in the live schema. dataset creates it lazily on first insert, so this is correct only for as long as every read path tolerates the table being absent.
-
-### Three legacy container tables still exist
-
-builds, dockerfiles and dockerfile_versions are present in the live database. The project replaced per-project images with the single shared ppy:latest image and ships devplace containers prune-builds as the one-time cleanup for exactly these rows. The tables are still carried.
-
-### Five duplicate OpenAPI operation IDs
-
-Generating the schema warns on editor_proxy twice in routers/projects/containers/workspace.py, passthrough in routers/openai_gateway.py, and proxy_http twice in routers/proxy.py. These are catch-all routes registered for several methods, so a generated client would collide on those names.
-
-### The one deliberate asymmetry is documented and intentional
-
-routers/__init__.py holds nothing but the attribution line; unlike every nested router package, the top level does not aggregate. main.py performs all 38 include_router calls directly, which keeps prefix ownership in a single readable block.
-
----
-
-Sources: devplacepy/main.py, the imported FastAPI app object, data/devplace.db sqlite_master, and the working tree at HEAD 192df12b with uncommitted local modifications present. Counts exclude __pycache__ and the virtual environment.

CLAUDE.md

@@ -19,7 +19,7 @@ DevPlace is a server-rendered social network for developers. FastAPI backend ser
 ## Commands
 
 ```bash
-make install        # pip install -e . + playwright install chromium
+make install        # create .venv if needed, pip install -e ".[dev]", playwright chromium
 make ppy            # build the single shared container image (ppy:latest); run once before launching instances
 make dev            # uvicorn --reload on port 10500, backlog 4096
 make prod           # uvicorn --workers $(WEB_WORKERS) (defaults to nproc), port 10500 (backlog 8192)
@@ -33,6 +33,8 @@ make locust         # Locust load test, interactive web UI
 make locust-headless  # Locust CLI mode for CI
 ```
 
+Every Python make target (`install`, `dev`, `prod`, `test*`, `coverage*`, `locust*`) uses `.venv/bin/python`. If `.venv` is missing, make creates it from `python3`, installs `-e ".[dev]"`, and installs Playwright Chromium before running the target. `make install` refreshes that environment. `make clean` removes `.venv` as well as stray bytecode.
+
 The Makefile exports `PYTHONDONTWRITEBYTECODE=1` for every recipe, so no `.pyc` files or `__pycache__` directories are written by any `make` target. Keep it that way - do not add a target that re-enables bytecode writing. `make clean` removes any stray bytecode left from running Python outside make.
 
 Preliminary validation: confirm `python -c "from devplacepy.main import app"` imports clean and check each touched language manually (Python compiles/imports, JS parses, CSS braces and HTML tags balance). These checks are gates on the way to the real validation, never a substitute for it: **every change ends with the full test suite (`make test` - all three tiers, every test) and it must pass.**
@@ -56,6 +58,7 @@ devplace token prune                # soft-delete all expired access tokens
 devplace news clear         # delete all news rows
 devplace news sanitize      # strip HTML from news descriptions/content
 devplace attachments prune  # remove orphan attachment records/files
+devplace system prune [--dry-run]  # safe but aggressive: purges soft-deleted attachment/project-file blobs, sweeps blob files with zero DB reference at all (e.g. left by an interrupted/racing sync), and GCs orphaned container workspace dirs
 devplace devii reset-quota <username>   # reset one user's rolling 24h AI quota
 devplace devii reset-quota --guests     # reset every guest quota
 devplace devii reset-quota --all        # reset every quota (users and guests)
@@ -70,6 +73,10 @@ devplace zips prune         # delete expired zip archives + job rows
 devplace zips clear         # delete every zip archive + job row
 devplace forks prune        # delete expired completed fork job rows (forked projects persist)
 devplace forks clear        # delete every fork job row (forked projects persist)
+devplace community discover <query> [--run] [--relearn] [--dry-run]  # discover, learn and generate collectors for an external community
+devplace community run <slug> [--dry-run]  # run a community's collectors through the sync API
+devplace community login <slug> [--login-user U] [--login-email E] [--login-password P | --clear]  # store or remove the account used to log in
+devplace community list     # list communities
 devplace messaging prune-tickets  # delete expired WebSocket auth tickets (ws_tickets)
 devplace seo prune          # delete expired SEO audit reports + job rows
 devplace seo clear          # delete every SEO audit report + job row
@@ -114,6 +121,7 @@ Tests run on port 10501 with a tempfile SQLite DB and a dedicated `DEVPLACE_DATA
 | Var | Default | Purpose |
 |-----|---------|---------|
 | `DEVPLACE_DATABASE_URL` | `sqlite:///<repo>/data/devplace.db` | Override DB path (tests use this) |
+| `DEVPLACE_PORT` | `10500` | Port the app itself binds to bare-metal (`make dev`/`make prod`, wired to uvicorn's `--port`; tests use their own `DEVPLACE_TEST_PORT`, default `10501`). Also `config.PORT`'s source, so `DEVPLACE_INTERNAL_BASE_URL`'s default and Devii's self-dial URL (`DEFAULT_BASE_URL`/`INSTANCE_ORIGIN_DEFAULT`) follow it automatically. The Docker app container pins it to `10500` in `docker-compose.yml` regardless of `.env` - Docker's externally reachable port is the unrelated `PORT` var (nginx's host mapping), never this one. |
 | `SECRET_KEY` | hardcoded fallback | Session signing |
 | `DEVPLACE_DISABLE_SERVICES` | unset | When `1`, NewsService and other background services skip start (set by test conftest) |
 | `PLAYWRIGHT_HEADLESS` | `1` in tests | Toggle headed mode |
@@ -143,7 +151,7 @@ Nested `CLAUDE.md` files (loaded automatically by Claude Code only when a file i
 | `devplacepy/services/moderation/CLAUDE.md` | Trust and safety: the reportable-target registry, the content filter and its five choke points, the report queue and its atomic resolution, enforcement, consent, maturity, account deletion |
 | `devplacepy/services/acceptance/CLAUDE.md` | Acceptance convergence: the opt-in service that grants every policy agreement to every account that has not declined it, its invisibility contract and the ledger-as-decline-register rule |
 | `devplacepy/services/audit/CLAUDE.md` | Audit log: recorders, categories, retention |
-| `devplacepy/services/backup/CLAUDE.md` | Backup service: targets, worker, schedules, primary-admin-only download |
+| `devplacepy/services/backup/CLAUDE.md` | Backup service: targets, worker, schedules, primary-admin-only download, remote offload to Hetzner Storage Box |
 | `devplacepy/services/telegram/CLAUDE.md` | Telegram bot bridge |
 | `devplacepy/services/email/CLAUDE.md` | Devii IMAP/SMTP email tools |
 | `devplacepy/services/gitea/CLAUDE.md` | Issue tracker (Gitea-backed, no local issue store) |
@@ -155,7 +163,12 @@ Nested `CLAUDE.md` files (loaded automatically by Claude Code only when a file i
 | `devplacepy/services/pubsub/CLAUDE.md` | Database-free pub/sub bus |
 | `devplacepy/push/CLAUDE.md` | Push notification providers: the `PushProvider` protocol, the registry, Web Push and APNs, registration storage |
 | `devplacepy/services/game/CLAUDE.md` | Code Farm idle game (economy invariants, raids, the one-pure-function rule) |
+| `devplacepy/services/opinionwar/CLAUDE.md` | Opinion Wars (week-long faction battles on posts: atomic fight/resolve transitions, cooldown-before-coins compensation, event seq allocation, relay-on-lock-owner) |
 | `devplacepy/services/quiz/CLAUDE.md` | Quizzes (the terminal publish lock, attempt atomicity, answer-key withholding, AI free-text grading, the best-attempt scoreboard) |
+| `devplacepy/services/skills/CLAUDE.md` | Agent Skills: strict `SKILL.md` parser, SSRF-guarded fetcher, the bounded prompt index, per-user privacy, soft delete and account-deletion cascade, why a skill is data and never code |
+| `devplacepy/services/nodes/CLAUDE.md` | Community node network: the protocol specification (single source of truth), the no-open-proxy rule, the two transports and their trust levels, the decisions taken, and the gaps per phase |
+| `devplacepy/services/community/CLAUDE.md` | Community discovery: rsearch + feed detection, one-time structure learning, generated collectors, dry run, the failure floor, admin-only visibility |
+| `devplacepy/services/sync/CLAUDE.md` | Community sync: the origin link table, idempotent batch upserts, normalization and rejection rules, what is deliberately not imported, the race that was measured |
 | `devplacepy/services/CLAUDE.md` | Background task queue, AI correction/modifier, presence, live view relay, `BaseService`/`ServiceManager` |
 | `devplacepy/database/CLAUDE.md` | Dataset rules, indexing conventions, soft delete, tables, site settings |
 | `devplacepy/utils/CLAUDE.md` | Notifications and gamification (XP/levels/badges/leaderboard) |
@@ -170,9 +183,9 @@ The AI usage analyzer (`devplace isslop analyze`, the `/tools/isslop` job servic
 
 ### Request pipeline
 
-`main.py` mounts `/static`, registers every router with its prefix, installs middlewares (security headers, a per-IP rate limit in an in-process `defaultdict`, a maintenance gate), and registers global 404/500 handlers. Rate limiting reads `rate_limit_per_minute`/`rate_limit_window_seconds` from `site_settings` (default 60/60s) and **applies only to mutating methods** (`POST`/`PUT`/`DELETE`/`PATCH`); reads and the whole `/openai` gateway are exempt. Bucket key is `X-Real-IP` falling back to `request.client.host`; an over-limit request gets `429` with `Retry-After`. The maintenance middleware short-circuits non-admin requests with a 503 when `maintenance_mode="1"`, but always allows `/static`, `/avatar`, `/auth`, `/admin`, and admin users. The outermost middleware is `response_timing`: stamps `request.state.request_start` and sets `X-Response-Time` on every response; the `response_time_ms(request)` Jinja global renders it as a fixed bottom-left badge in `base.html`. This is the single generic timing mechanism - never re-time per route.
+`main.py` mounts `/static`, registers every router with its prefix, installs middlewares from `middleware/` (`register()` in addition order: snapshot, corrections, security headers, rate limit, maintenance, terms, presence, visits, timing, tunnel dispatch, gzip), and registers the global handlers from `errors.py` (404/500/`ContentRefused`/`RequestValidationError`). The per-IP rate limit state lives in `middleware/state.py`; app lifespan (locks, `SERVICE_CLASSES`, startup/shutdown) lives in `lifespan.py`; the home route lives in `routers/landing.py`; the static-file classes live in `static_files.py`. Rate limiting reads `rate_limit_per_minute`/`rate_limit_window_seconds` from `site_settings` (default 60/60s) and **applies only to mutating methods** (`POST`/`PUT`/`DELETE`/`PATCH`); reads and the whole `/openai` gateway are exempt. Bucket key is `X-Real-IP` falling back to `request.client.host`; an over-limit request gets `429` with `Retry-After`. The maintenance middleware short-circuits non-admin requests with a 503 when `maintenance_mode="1"`, but always allows `/static`, `/avatar`, `/auth`, `/admin`, and admin users. The outermost middleware is `response_timing`: stamps `request.state.request_start` and sets `X-Response-Time` on every response; the `response_time_ms(request)` Jinja global renders it as a fixed bottom-left badge in `base.html`. This is the single generic timing mechanism - never re-time per route.
 
-`@app.on_event("startup")` calls `init_db()` and, unless `DEVPLACE_DISABLE_SERVICES` is set, registers `NewsService` and kicks off `start_all()`. `GET /` never redirects: guests get the marketing splash, authenticated users get a personalized home, both sharing Latest Posts + Developer News. The feed and Latest Posts enforce **author diversity** by interleaving authors (never dropping posts) via `interleave_by_author`/`paginate_diverse` in `database/`.
+`lifespan.py` `lifespan()` calls `init_db()` and, unless `DEVPLACE_DISABLE_SERVICES` is set, registers every service in `SERVICE_CLASSES` and supervises them in the lock owner. `GET /` never redirects: guests get the marketing splash, authenticated users get a personalized home, both sharing Latest Posts + Developer News. The feed and Latest Posts enforce **author diversity** by interleaving authors (never dropping posts) via `interleave_by_author`/`paginate_diverse` in `database/`.
 
 ### Routing layout
 
@@ -182,11 +195,13 @@ Routers in `devplacepy/routers/` are organised as a **directory tree that mirror
 |--------|--------|
 | `/auth` | auth/ package |
 | `/feed`, `/posts`, `/comments` | flat files |
+| `/topics` | topics.py - crawlable per-topic category index pages (`/topics` hub, `/topics/{topic}` listing) |
 | `/projects`, `/projects/{slug}/files`, `/projects/{slug}/containers` | projects/ package - see `routers/projects/CLAUDE.md` |
 | `/profile` | profile/ package (customization, notifications, ai-correction, ai-modifier, interactions, telegram, usage) |
 | `/messages` | messages.py - see `services/messaging/CLAUDE.md` |
 | `/notifications`, `/votes`, `/reactions`, `/bookmarks`, `/polls`, `/avatar`, `/follow`, `/leaderboard` | flat files |
 | (none) | relations.py - block/mute |
+| `/` | landing.py - home route (guests get the splash, users the dashboard) |
 | `/admin`, `/admin/services`, `/admin/containers` | admin/ package |
 | `/issues` | issues/ package - see `services/gitea/CLAUDE.md` |
 | `/gists`, `/news`, `/uploads`, `/media` | flat files |
@@ -198,9 +213,11 @@ Routers in `devplacepy/routers/` are organised as a **directory tree that mirror
 | `/xmlrpc` | xmlrpc.py - see `services/xmlrpc/CLAUDE.md` |
 | `/api` | devrant/ package - see `routers/devrant/CLAUDE.md` |
 | `/dbapi` | dbapi/ package, **primary-administrator-only, strictly READ-ONLY** - see `services/dbapi/CLAUDE.md` |
+| `/sync` | sync/ package, **administrator only**, off by default - see `services/sync/CLAUDE.md` |
 | `/game` | game/ package - see `services/game/CLAUDE.md` |
 | `/reports`, `/admin/moderation`, `/workspaces` | reports.py, admin/moderation.py, workspaces.py - see `services/moderation/CLAUDE.md` |
 | `/quizzes` | quizzes/ package - see `services/quiz/CLAUDE.md` |
+| `/battles` | battles.py - see `services/opinionwar/CLAUDE.md` |
 | (none) | push.py (web push/PWA), docs.py (see `routers/docs/CLAUDE.md`) |
 | `/pubsub` | pubsub.py - see `services/pubsub/CLAUDE.md` |
 | (none) | seo.py - `/robots.txt`, `/sitemap.xml` |
@@ -256,7 +273,7 @@ Users and guests inject their own CSS and JS, scoped to a page type or globally,
 
 ### Container manager, Devii assistant, AI gateway, async jobs, audit log
 
-DevPlace ships a Docker-backed **Container manager** (admin-only, one shared `ppy:latest` image, security-hardened rootless workflow via a `sudo` superclone and `aptroot`), the **Devii** in-platform agentic assistant (WebSocket terminal, per-owner-channel sessions, scheduler/reminders, self-learning lessons, user-defined virtual tools, self-configured behavior, browser-automation client tools), the **AI gateway** (`/openai/v1/*`, single point of truth for every AI call, per-user cost attribution, provider/model routing overlay), **async job services** (`JobService` base pattern: zip, fork, SEO diagnostics, SEO metadata generation, DeepSearch, AI Usage Analyzer), and an admin-only append-only **audit log** (`record`/`record_system` entrypoints, `events.md` catalogue of 288 keys, never raises into the caller). These are among the largest subsystems in the codebase - read their dedicated nested `CLAUDE.md` files (see Subsystem map) before working in any of them; do not assume prior knowledge from this summary.
+DevPlace ships a Docker-backed **Container manager** (admin-only, one shared `ppy:latest` image, security-hardened rootless workflow via a `sudo` superclone and `aptroot`), the **Devii** in-platform agentic assistant (WebSocket terminal, per-owner-channel sessions, scheduler/reminders, self-learning lessons, user-defined virtual tools, self-configured behavior, browser-automation client tools), the **AI gateway** (`/openai/v1/*`, single point of truth for every AI call, per-user cost attribution, provider/model routing overlay), **async job services** (`JobService` base pattern: zip, fork, SEO diagnostics, SEO metadata generation, DeepSearch, AI Usage Analyzer), and an admin-only append-only **audit log** (`record`/`record_system` entrypoints, `events.md` catalogue of every recorded event key, never raises into the caller). These are among the largest subsystems in the codebase - read their dedicated nested `CLAUDE.md` files (see Subsystem map) before working in any of them; do not assume prior knowledge from this summary.
 
 ### Telegram bot, email, devRant compatibility API, issue tracker
 
@@ -280,8 +297,8 @@ The escape hatch is deliberately two-factor and must never be self-served: after
 
 - **No comments, no docstrings in source.** Code is self-documenting.
 - **Forbidden variable name patterns:** `_new`, `_old`, `_temp`, `_v2`, `better_`, `my_`, `the_`.
-- **Outbound HTTP goes through the stealth client only.** Never instantiate a bare `httpx.AsyncClient(...)`/`httpx.Client(...)` for outbound traffic - build it via `devplacepy.stealth.stealth_async_client(...)`/`stealth_sync_client(...)`. The async factory routes bytes through **`curl_cffi` (curl-impersonate, BoringSSL)** behind an `httpx` transport adapter, so TLS JA3/JA4 and the HTTP/2 fingerprint match Chrome 146 byte-for-byte; falls back to a pure-httpx Chrome-aligned transport if `curl_cffi` is missing. Applies to news, gateway upstream, Devii/DeepSearch/SEO fetch, gitea, push, attachments. SSRF-guarded fetches via `net_guard.guarded_async_client(...)` get the fingerprint automatically. Sole exception: the reverse-proxy forwarding core `services/containers/forward.py` (and the three routers that call it - `routers/proxy.py`, `routers/tunnel.py`, the workspace editor route), which relays the user's own headers verbatim and therefore keeps a plain `httpx.AsyncClient`; bolting the Chrome identity onto it would overwrite the very headers it exists to forward. **Load-bearing gotcha:** cleartext `http://` is forced to HTTP/1.1 (`curl_transport.http_version_for`) - the Chrome-impersonation profile defaults to HTTP/2, but a plaintext origin has no TLS/ALPN to negotiate h2 and uvicorn (HTTP/1.1 only) returns `400 "Invalid HTTP request received."` once a cleartext HTTP/2 body crosses ~128KB. Every internal AI consumer calls `INTERNAL_GATEWAY_URL` = `http://localhost:<port>/openai/v1/...`, so without this override a large Devii turn (big system prompt + tool list) fails on every request. `https://` keeps the HTTP/2 default. `stealth_sync_client` stays pure-httpx Chrome-aligned (no curl_cffi sync adapter). **Never bolt a hand-written browser-header dict (UA/`sec-ch-ua`/`Sec-Fetch-*`) onto a `stealth_async_client`/`stealth_sync_client` call** - curl_cffi's impersonation already emits an internally-consistent Chrome 146 identity across TLS, UA, and Client Hints; a caller-supplied header set claiming a different Chrome version or platform creates a UA-vs-TLS-vs-Client-Hints mismatch that is a textbook anti-bot tripwire (this was a real bug in Devii's `fetch_url`/`http_request` tool, fixed by deleting its hardcoded Chrome-131 header block). `stealth.detect_consent_gate(html)` recognizes the common JS-redirect cookie-consent wall pattern (DPG Media's `nu.nl` and sibling sites, and generically any page gated behind a `decodeURIComponent(...)` callback referencing a "privacy gate"/`consent.js`) - reuse it wherever a fetched page's real content sits behind that gate rather than special-casing a domain. `stealth_transport`/`stealth_async_client`/`stealth_sync_client` all accept a `proxy=` override, defaulting to `stealth.configured_proxy_url()` (the admin-editable `outbound_proxy_url` site setting, falling back to `DEVPLACE_OUTBOUND_PROXY_URL` when the setting is unset) - this is the only real fix for a destination that blocks by IP/ASN reputation (e.g. reddit.com returns an identical "blocked by network security" page across every TLS/header fingerprint tried, confirmed by testing curl_cffi's full impersonation list - it is blocking the datacenter ASN outright, not fingerprinting the client).
-- **Form validation is Pydantic-native.** Routers declare `data: Annotated[SomeForm, Form()]` (models in `models.py`); the global `RequestValidationError` handler re-renders auth pages with messages (400) or redirects other routes. Read raw `await request.form()` only when also reading an uploaded file alongside a model.
+- **Outbound HTTP goes through the stealth client only.** Never instantiate a bare `httpx.AsyncClient(...)`/`httpx.Client(...)` for outbound traffic - build it via `devplacepy.stealth.stealth_async_client(...)`/`stealth_sync_client(...)`. The async factory routes bytes through **`curl_cffi` (curl-impersonate, BoringSSL)** behind an `httpx` transport adapter, so TLS JA3/JA4 and the HTTP/2 fingerprint match Chrome 146 byte-for-byte; falls back to a pure-httpx Chrome-aligned transport if `curl_cffi` is missing. Applies to news, gateway upstream, Devii/DeepSearch/SEO fetch, gitea, web push, attachments. SSRF-guarded fetches via `net_guard.guarded_async_client(...)` get the fingerprint automatically. Two exceptions keep a plain `httpx.AsyncClient`: (1) the reverse-proxy forwarding core `services/containers/forward.py` (and the three routers that call it - `routers/proxy.py`, `routers/tunnel.py`, the workspace editor route), which relays the user's own headers verbatim - bolting the Chrome identity onto it would overwrite the very headers it exists to forward; (2) the APNs provider (`push/providers/apns.py` `gateway_client` / `delivery_client`) which talks to Apple's HTTP/2 provider API with `httpx.AsyncClient(http2=True, trust_env=False)` - Chrome impersonation, HTTP/2 PRIORITY frames, HPACK indexing of `:path`/`authorization`, extra browser headers, and the outbound proxy all break or starve that API. Web Push stays on stealth. **Load-bearing gotcha:** cleartext `http://` is forced to HTTP/1.1 (`curl_transport.http_version_for`) - the Chrome-impersonation profile defaults to HTTP/2, but a plaintext origin has no TLS/ALPN to negotiate h2 and uvicorn (HTTP/1.1 only) returns `400 "Invalid HTTP request received."` once a cleartext HTTP/2 body crosses ~128KB. Every internal AI consumer calls `INTERNAL_GATEWAY_URL` = `http://localhost:<port>/openai/v1/...`, so without this override a large Devii turn (big system prompt + tool list) fails on every request. `https://` keeps the HTTP/2 default. `stealth_sync_client` stays pure-httpx Chrome-aligned (no curl_cffi sync adapter). **Never bolt a hand-written browser-header dict (UA/`sec-ch-ua`/`Sec-Fetch-*`) onto a `stealth_async_client`/`stealth_sync_client` call** - curl_cffi's impersonation already emits an internally-consistent Chrome 146 identity across TLS, UA, and Client Hints; a caller-supplied header set claiming a different Chrome version or platform creates a UA-vs-TLS-vs-Client-Hints mismatch that is a textbook anti-bot tripwire (this was a real bug in Devii's `fetch_url`/`http_request` tool, fixed by deleting its hardcoded Chrome-131 header block). `stealth.detect_consent_gate(html)` recognizes the common JS-redirect cookie-consent wall pattern (DPG Media's `nu.nl` and sibling sites, and generically any page gated behind a `decodeURIComponent(...)` callback referencing a "privacy gate"/`consent.js`) - reuse it wherever a fetched page's real content sits behind that gate rather than special-casing a domain. `stealth_transport`/`stealth_async_client`/`stealth_sync_client` all accept a `proxy=` override, defaulting to `stealth.configured_proxy_url()` (the admin-editable `outbound_proxy_url` site setting, falling back to `DEVPLACE_OUTBOUND_PROXY_URL` when the setting is unset) - this is the only real fix for a destination that blocks by IP/ASN reputation (e.g. reddit.com returns an identical "blocked by network security" page across every TLS/header fingerprint tried, confirmed by testing curl_cffi's full impersonation list - it is blocking the datacenter ASN outright, not fingerprinting the client).
+- **Form validation is Pydantic-native.** Routers declare `data: Annotated[SomeForm, Form()]` (models in `models/`, one module per domain, re-exported from `models/__init__.py`); the global `RequestValidationError` handler re-renders auth pages with messages (400) or redirects other routes. Read raw `await request.form()` only when also reading an uploaded file alongside a model.
 - **`RedirectResponse(url=..., status_code=302)`** - always pass `status_code` as an integer literal, never `status.HTTP_302_FOUND`.
 - **Ownership checks:** compare `resource["user_uid"] == user["uid"]` before edit/delete (use `content.is_owner`). **Deletes additionally allow any admin:** `is_owner(...) or is_admin(user)` on every content delete endpoint, so an administrator may delete any member's content while members are limited to their own. Edits stay owner-only. Deletes are **soft** and cascade under one shared stamp - see `devplacepy/database/CLAUDE.md`.
 - **All dates shown to users are DD/MM/YYYY** (European), rendered in the viewer's own timezone client-side. Timestamps are stored/emitted as UTC ISO. Use the `local_dt(iso, mode)`/`dt_ago(iso)` Jinja globals for any user-facing instant - they emit `<time data-dt>` and `static/js/LocalTime.js` reformats to local timezone with a `MutationObserver` for dynamic content. `format_date()`/`time_ago()` stay as plain-text helpers for JSON responses, no-JS fallbacks, and non-timestamp date fields (e.g. project `release_date`) - do NOT wrap those in `local_dt`.
@@ -346,17 +363,17 @@ Every feature in DevPlace is **one data source fanning out into several consumer
 
 1. **HTML** - `respond()` returns a rendered template for browsers.
 2. **JSON** - the SAME `respond(..., model=XOut)` returns JSON when `Accept: application/json`. The `*Out` schema is the gate: a context key not declared on `*Out` is silently dropped from JSON even though the template still sees it.
-3. **Agent tool** - `services/devii/actions/catalog.py` exposes the route as an `Action` so Devii can call it on the user's behalf. `requires_auth` mirrors the route guard.
+3. **Agent tool** - `services/devii/actions/catalog/` (the relevant module inside the package, e.g. `posts.py`, `admin.py`) exposes the route as an `Action` so Devii can call it on the user's behalf. `requires_auth` mirrors the route guard.
 4. **Documented endpoint** - `docs_api.py` `endpoint()` describes it (params, auth, `sample_response`) in the right group, rendered at `/docs/{group}.html`.
 
 A new public read almost always needs all four. The cardinal failure mode is changing one face and forgetting a connected one - a real worked example (followers/following listing) touched nine files across data helpers, output schema, route, view, agent tool, API docs, and the docs trio below for one conceptual feature; that count is the norm, not the exception. If a change touches only one file, confirm none of the four faces were missed. Checklist, ordered by data flow:
 
 1. **Understand.** Read the router, template, matching tests (per the tier naming rule above), and the matching nested `CLAUDE.md`. Trace input model -> router -> data helper -> response (HTML and JSON).
-2. **Data layer.** `database/` for query/batch helpers (never inline N+1 loops - use `get_users_by_uids`, `build_pagination`, `_in_clause`; guard raw SQL with `if "table" in db.tables`). `models.py` for the Pydantic `Form` input model. `schemas/` for the `*Out` JSON response model - every context key a JSON route exposes via `respond(..., model=XOut)` MUST exist on `XOut` or it is silently dropped. Schema auto-syncs via `dataset`; add indexes in `init_db()` with `CREATE INDEX IF NOT EXISTS`.
+2. **Data layer.** `database/` for query/batch helpers (never inline N+1 loops - use `get_users_by_uids`, `build_pagination`, `_in_clause`; guard raw SQL with `if "table" in db.tables`). `models/` for the Pydantic `Form` input model (add the class to the matching domain module and to the re-export list plus `__all__` in `models/__init__.py`). `schemas/` for the `*Out` JSON response model - every context key a JSON route exposes via `respond(..., model=XOut)` MUST exist on `XOut` or it is silently dropped. Schema auto-syncs via `dataset`; add indexes in `init_db()` with `CREATE INDEX IF NOT EXISTS`.
 3. **Server layer.** Handler with the right auth guard (`get_current_user` public read, `require_user` member POST, `require_admin` admin). Specific paths (`/{username}/followers`) declared before catch-alls (`/{username}`). Return HTML+JSON via `respond(request, template, ctx, model=XOut)` or pure JSON via `JSONResponse`. Register any NEW router in `main.py`.
 4. **View layer.** Extend `base.html`; page CSS in `{% block extra_head %}`, page JS in `{% block extra_js %}`. Reuse partials and template globals. JS is ES6 modules, one class per file, instantiated on `app`.
 5. **Safety layer (skip only when the feature stores nothing and shows nothing).** If the feature adds a surface that carries user-generated content, register it in `database/moderation.py` `REPORTABLE_TARGETS` (or in `UNREPORTABLE_TABLES` with a reason), resolve it in `resolve_object_url`, and include `_report_button.html` in its action bar - the registry test and the e2e coverage test both fail otherwise. If it collects or transmits a new *category* of personal data, or sends anything new to a third party, update `/docs/privacy.html` and the app-store privacy declarations in the same change; that disclosure is part of the feature, not a follow-up.
-6. **Agent + docs layer (the most-forgotten step - do not skip).** `services/devii/actions/catalog.py` - add an `Action` tool if a user could ask Devii to do this (`requires_auth=False` for public reads). `docs_api.py` - every public/auth endpoint gets an `endpoint()` entry with params and a `sample_response`. `routers/docs/pages.py` `DOCS_PAGES` for prose pages. SEO (`seo.py`) - `base_seo_context` and any JSON-LD for a new public page; add to `routers/seo.py` sitemap if indexable.
+6. **Agent + docs layer (the most-forgotten step - do not skip).** `services/devii/actions/catalog/` (the relevant module inside the package) - add an `Action` tool if a user could ask Devii to do this (`requires_auth=False` for public reads). `docs_api.py` - every public/auth endpoint gets an `endpoint()` entry with params and a `sample_response`. `routers/docs/pages.py` `DOCS_PAGES` for prose pages. SEO (`seo.py`) - `base_seo_context` and any JSON-LD for a new public page; add to `routers/seo.py` sitemap if indexable.
 7. **Document.** `README.md` (product-facing, routes/config/dependencies/user-visible features), the relevant nested `CLAUDE.md` (any new mechanic, helper, pitfall, or cross-layer wiring), this root `CLAUDE.md` only when a NEW architectural rule or workflow step is introduced (not per feature).
 8. **Validate.** Confirm a clean import (`python -c "from devplacepy.main import app"`), check each touched language manually, grep for em-dash in touched files (neither the character nor its HTML entity - use a hyphen), and run `pyflakes`/`ruff check` on every touched file. **If the feature touches a spendable resource, a bounded state machine, or a read-then-write mutation reachable from more than one request path, also run the four-layer procedure in "Rigorous correctness verification" above** - it is the default for that shape of feature, not an optional extra. **Finish by running the full persisted test suite (`make test` - every tier, every test) and fixing any failure before the change is done.** Write new tests in the matching tier/path, following the required patterns above.
 
@@ -366,15 +383,21 @@ Failures at any implementation step block the workflow - never skip a failed ste
 
 Gitea Actions workflow at `.gitea/workflows/test.yaml` runs on every push/PR to `master`: installs dependencies + Playwright Chromium, runs the full suite serially under coverage, publishes coverage HTML as an artifact, uploads failure screenshots. CI must be green before merging. Changes move through DTAP: Development (`make dev`) -> Test (CI suite + coverage on `master`) -> Acceptance (`master` to `production` promotion via `make deploy`) -> Production (Docker Compose stack). Only CI-green `master` commits are promoted to `production`.
 
+## Version bumping
+
+`pyproject.toml` `version` is bumped automatically, by a real git `pre-commit` hook, not by an agent remembering to edit it. The hook lives at `.githooks/pre-commit` (tracked in the repo, plain stdlib Python) and `make install` points git at it with `git config core.hooksPath .githooks` - run `make install` once per clone (or that one `git config` line by hand) to activate it; a clone that has never run `make install` simply gets no auto-bump, which is a safe, backwards-compatible no-op, never a broken commit.
+
+On every commit the hook increments the patch component (`1.0.0` -> `1.0.1`) and stages the change, so the bump rides in the same commit with no extra step. It defers to a deliberate version edit already staged in that same commit (a hand-set major/minor bump in `pyproject.toml` is left exactly as written, never incremented further) and does nothing during a merge (`.git/MERGE_HEAD` present) or when `pyproject.toml` does not exist. It never blocks a commit - a missing or unparsable version line is a silent no-op, not a failure.
+
 ## Diagnosing a production failure (the order that finds it fastest)
 
 This procedure exists because a single "the editor is down" report turned out to be **three unrelated faults stacked on each other** (a stale URL, a firewalled network leg, and a corrupt database), and the investigation wasted hours by guessing before measuring. Work the layers outward from the browser; each step is cheap and each one eliminates a whole class of cause. **Never skip to a hypothesis, and never repair anything before the layer above it is proven healthy.**
 
-**Layer 0 - is the request even arriving here?** Fetch the hostname over the public internet exactly as it resolves (`curl -sS -o /dev/null -w '%{http_code} %{remote_ip}' https://host/`). Compare the answering IP against this machine's own addresses (`ip -6 addr`, `curl https://api.ipify.org`). Two hostnames serve this platform by different routes - see the topology section above. **Never use `curl --resolve` to force a hostname onto an IP it does not resolve to**; that fabricates a path that no real traffic takes and produces confident, wrong conclusions.
+**Layer 0 - is the request even arriving here?** Fetch the hostname over the public internet exactly as it resolves (`curl -sS -o /dev/null -w '%{http_code} %{remote_ip}' https://host/`). Compare the answering IP against this machine's own addresses (`ip -6 addr`, `curl https://api.ipify.org`). DevPlace is served publicly on `devplace.net` only - see the topology section below. **Never use `curl --resolve` to force a hostname onto an IP it does not resolve to**; that fabricates a path that no real traffic takes and produces confident, wrong conclusions.
 
-**Layer 1 - which edge answered?** The error body identifies it. `application/problem+json` with `No site configured for host` is molohttp. A DevPlace HTML error page is the application. An nginx error page is the nginx container. A browser `ERR_*` with no body means nothing well-formed was returned at all.
+**Layer 1 - which edge answered?** The error body identifies it. A DevPlace HTML error page is the application. An nginx error page is the nginx container. A browser `ERR_*` with no body means nothing well-formed was returned at all.
 
-**Layer 2 - same failure on both hostnames?** Run the identical authenticated request against `pravda.education` and `devplace.net`. Failing on **both** means the application or the database; failing on **one** means that host's edge. This single comparison is the highest-value measurement available and costs one command.
+**Layer 2 - same failure on the public hostname and on the host itself?** Run the identical authenticated request against `https://devplace.net` and `http://127.0.0.1:10500`. Failing on **both** means the application or the database; failing only on `devplace.net` means the front host's proxy or the SSH tunnel. This single comparison is the highest-value measurement available and costs one command.
 
 **Layer 3 - the application log, before any theory.** `docker logs --since 5m devplace-app-1`. Count error classes rather than reading prose (`grep -c malformed`). A recurring service-loop error is a systemic fault even when it looks unrelated to the symptom.
 
@@ -395,23 +418,20 @@ This procedure exists because a single "the editor is down" report turned out to
 
 ## Production hostnames and the devplace.net SSH tunnel (verified topology, do not re-derive)
 
-**The platform answers on two public hostnames, and they reach the same application by two completely different paths.** This has already cost one debugging session; the failure mode is that a `curl --resolve devplace.net:443:<production ip>` "test" reports `No site configured for host: devplace.net` and looks like a total outage, when in fact devplace.net never touches the production edge at all.
+**DevPlace answers publicly on `devplace.net` only.** `pravda.education` used to be a second hostname for it; it is now a different site. Measured 2026-10-14: it still resolves to this machine (`95.216.15.238`) but answers with `Server: grr` and a different application (`/robots.txt` and `/openapi.json` are 404, `/docs` is 200 where DevPlace returns 302), and the owner confirmed it is no longer DevPlace. Never use it to verify a DevPlace change, and never read a 404 there as a DevPlace fault.
 
-| | `pravda.education` | `devplace.net` |
-|---|---|---|
-| DNS | `95.216.15.238`, `2a01:4f9:2a:100e::2` | `88.198.21.243`, `2a01:4f8:222:2c45::2` |
-| Machine | the production host itself | a separate front host (Hetzner, PTR `static.88-198-21-243.clients.your-server.de`) |
-| Path in | molohttp on `:443` -> `127.0.0.1:10500` | its own proxy -> **SSH tunnel** -> `127.0.0.1:10500` on production |
-| Reaches molohttp | yes | **no, never** |
+| | `devplace.net` |
+|---|---|
+| DNS | `88.198.21.243`, `2a01:4f8:222:2c45::2` |
+| Machine | a separate front host (Hetzner, PTR `static.88-198-21-243.clients.your-server.de`) |
+| Path in | its own proxy -> **SSH tunnel** -> `127.0.0.1:10500` on this host |
 
-**`devplace.net` is a front host that forwards over SSH.** It holds a persistent SSH session into the production host (visible there as an established inbound connection from `88.198.21.243` to port 22) and forwards through it to `127.0.0.1:10500`, which is the `docker-proxy` for the `devplace-nginx` container. The listening socket lives on the **front** host (an `ssh -L` style local forward), so the production host shows **no** sshd-owned listener - that absence is expected and is not evidence against the tunnel.
+**`devplace.net` is a front host that forwards over SSH.** It holds a persistent SSH session into this host (visible here as an established inbound connection from `88.198.21.243` to port 22) and forwards through it to `127.0.0.1:10500`. In the Docker Compose layout that port is the `docker-proxy` for the `devplace-nginx` container; on a host running `make dev` it is uvicorn itself. The listening socket lives on the **front** host (an `ssh -L` style local forward), so this host shows **no** sshd-owned listener - that absence is expected and is not evidence against the tunnel.
 
-Two consequences that must not be forgotten:
+- **The front host's proxy is the edge to suspect when only `devplace.net` fails** while `127.0.0.1:10500` is healthy (WebSocket `Upgrade` headers are the usual culprit, exactly as for the production nginx locations below).
+- **The machine the repository is checked out on may run `make dev`, not the Compose stack** (observed 2026-10-14: `uvicorn --reload` on port 10500, no `docker` binary, `podman` present). `make docker-reload` fails there with `docker: No such file or directory`, and the `--reload` server already serves committed code. Check `ss -ltnp | grep 10500` and `which docker` before assuming a deploy path.
 
-- **molohttp has no `devplace.net` site, and that is correct.** Its site list is `mail`/`smtp`/`imap.molodetz.nl`, `pravda.education` and `*.tunnel.pravda.education`. devplace.net traffic enters below molohttp, straight into `127.0.0.1:10500`, so it needs no site. **Never "fix" this by adding a devplace.net site to molohttp** - devplace.net does not resolve to the production host, so such a site could never match, and its absence is not a bug.
-- **Both hostnames land on the same nginx and the same app**, so a request that fails on both is failing in the application, not in either edge. That comparison is the fastest triage available here: run the same authenticated request against both hostnames. Same failure on both means look at the app or the database; a failure only on devplace.net means look at the front host's proxy (WebSocket `Upgrade` headers are the usual culprit, exactly as for the production nginx locations below).
-
-**Testing rule.** Never point a hostname at an IP it does not resolve to in order to "test" it. Fetch each hostname over the public internet as it really resolves (`curl https://devplace.net/...` and `curl https://pravda.education/...`), because forcing devplace.net onto the production IP tests molohttp with a `Host` it deliberately does not serve and proves nothing about the real path.
+**Testing rule.** Never point a hostname at an IP it does not resolve to in order to "test" it. Fetch `https://devplace.net/...` over the public internet as it really resolves, and compare it with `http://127.0.0.1:10500/...` on the host itself.
 
 ## Production deployment
 

Dockerfile

@@ -1,9 +1,9 @@
-FROM python:3.13-slim
+FROM docker.io/library/python:3.13-slim
 
 WORKDIR /app
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    curl ca-certificates \
+    curl ca-certificates rclone git \
     libglib2.0-0 libpango-1.0-0 libpangoft2-1.0-0 libharfbuzz0b libfontconfig1 \
     fonts-dejavu-core \
     && rm -rf /var/lib/apt/lists/*

Makefile

@@ -8,21 +8,42 @@ LOCUST_RUN_TIME ?= 120s
 LOCUST_WEB_WORKERS ?= 4
 WEB_WORKERS ?= $(shell nproc 2>/dev/null || echo 2)
 DEVPLACE_RATE_LIMIT ?= 1000000
+DEVPLACE_PORT ?= 10500
+
+VENV ?= $(CURDIR)/.venv
+PYTHON := $(VENV)/bin/python
+VENV_STAMP := $(VENV)/.installed
+BOOTSTRAP_PYTHON := $(shell command -v python3 2>/dev/null || command -v python 2>/dev/null)
 
 PYTHONDONTWRITEBYTECODE := 1
 export PYTHONDONTWRITEBYTECODE
 
-.PHONY: install dev clean tree tree-loc zip test test-headed test-unit test-api test-e2e test-fast test-failed test-first-failure test-slowest test-cache-clean coverage coverage-headed coverage-html locust locust-headless
+.PHONY: venv install dev prod clean tree tree-loc zip test test-headed test-unit test-api test-e2e test-fast test-failed test-first-failure test-slowest test-cache-clean coverage coverage-headed coverage-html locust locust-headless prune prune-dry-run
 
-install:
-	pip install -e .
-	python -m playwright install chromium
+$(PYTHON):
+	@test -n "$(BOOTSTRAP_PYTHON)" || { echo "python3 is required to create $(VENV)"; exit 1; }
+	$(BOOTSTRAP_PYTHON) -m venv $(VENV)
 
-dev:
-	uvicorn devplacepy.main:app --reload --reload-dir devplacepy --host 0.0.0.0 --port 10500 --backlog 4096
+$(VENV_STAMP): $(PYTHON) pyproject.toml
+	$(PYTHON) -m pip install -U pip
+	$(PYTHON) -m pip install -e ".[dev]"
+	$(PYTHON) -m playwright install chromium
+	touch $(VENV_STAMP)
 
-prod:
-	DEVPLACE_STATIC_VERSION=$$(date +%s) DEVPLACE_TEMPLATE_AUTO_RELOAD=0 DEVPLACE_WEB_WORKERS=$(WEB_WORKERS) uvicorn devplacepy.main:app --host 0.0.0.0 --port 10500 --workers $(WEB_WORKERS) --backlog 8192 --proxy-headers --forwarded-allow-ips '*'
+venv: $(VENV_STAMP)
+
+install: $(PYTHON)
+	$(PYTHON) -m pip install -U pip
+	$(PYTHON) -m pip install -e ".[dev]"
+	$(PYTHON) -m playwright install chromium
+	git config core.hooksPath .githooks
+	touch $(VENV_STAMP)
+
+dev: $(VENV_STAMP)
+	DEVPLACE_PORT=$(DEVPLACE_PORT) $(PYTHON) -m uvicorn devplacepy.main:app --reload --reload-dir devplacepy --host 0.0.0.0 --port $(DEVPLACE_PORT) --backlog 4096
+
+prod: $(VENV_STAMP)
+	DEVPLACE_STATIC_VERSION=$$(date +%s) DEVPLACE_TEMPLATE_AUTO_RELOAD=0 DEVPLACE_WEB_WORKERS=$(WEB_WORKERS) DEVPLACE_PORT=$(DEVPLACE_PORT) $(PYTHON) -m uvicorn devplacepy.main:app --host 0.0.0.0 --port $(DEVPLACE_PORT) --workers $(WEB_WORKERS) --backlog 8192 --proxy-headers --forwarded-allow-ips '*'
 
 delete-pyc:
 	find . -name "__pycache__" -type d -prune -exec rm -rf {} + 2>/dev/null || true
@@ -42,78 +63,78 @@ zip:
 	@git ls-files -z | xargs -0 zip -q $(notdir $(CURDIR)).zip
 	@printf 'Wrote %s (%s files)\n' "$(notdir $(CURDIR)).zip" "$$(git ls-files | wc -l)"
 
-test:
-	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/
+test: $(VENV_STAMP)
+	PLAYWRIGHT_HEADLESS=1 $(PYTHON) -m pytest tests/
 
-test-headed:
-	PLAYWRIGHT_HEADLESS=0 python -m pytest tests/
+test-headed: $(VENV_STAMP)
+	PLAYWRIGHT_HEADLESS=0 $(PYTHON) -m pytest tests/
 
-test-unit:
-	python -m pytest tests/unit
+test-unit: $(VENV_STAMP)
+	$(PYTHON) -m pytest tests/unit
 
-test-api:
-	python -m pytest tests/api
+test-api: $(VENV_STAMP)
+	$(PYTHON) -m pytest tests/api
 
-test-e2e:
-	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/e2e
+test-e2e: $(VENV_STAMP)
+	PLAYWRIGHT_HEADLESS=1 $(PYTHON) -m pytest tests/e2e
 
-test-fast:
-	python -m pytest tests/unit tests/api
+test-fast: $(VENV_STAMP)
+	$(PYTHON) -m pytest tests/unit tests/api
 
-test-failed:
-	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/ --last-failed --last-failed-no-failures none
+test-failed: $(VENV_STAMP)
+	PLAYWRIGHT_HEADLESS=1 $(PYTHON) -m pytest tests/ --last-failed --last-failed-no-failures none
 
-test-first-failure:
-	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/ -x
+test-first-failure: $(VENV_STAMP)
+	PLAYWRIGHT_HEADLESS=1 $(PYTHON) -m pytest tests/ -x
 
-test-slowest:
-	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/ --durations=40
+test-slowest: $(VENV_STAMP)
+	PLAYWRIGHT_HEADLESS=1 $(PYTHON) -m pytest tests/ --durations=40
 
-coverage:
+coverage: $(VENV_STAMP)
 	rm -f .coverage .coverage.*
 	COVERAGE_PROCESS_START=$(CURDIR)/.coveragerc PLAYWRIGHT_HEADLESS=1 \
-		python -m coverage run -m pytest tests/
-	python -m coverage combine
-	python -m coverage report
+		$(PYTHON) -m coverage run -m pytest tests/
+	$(PYTHON) -m coverage combine
+	$(PYTHON) -m coverage report
 
-coverage-headed:
+coverage-headed: $(VENV_STAMP)
 	rm -f .coverage .coverage.*
 	COVERAGE_PROCESS_START=$(CURDIR)/.coveragerc PLAYWRIGHT_HEADLESS=0 \
-		python -m coverage run -m pytest tests/
-	python -m coverage combine
-	python -m coverage report
+		$(PYTHON) -m coverage run -m pytest tests/
+	$(PYTHON) -m coverage combine
+	$(PYTHON) -m coverage report
 
 coverage-html: coverage
-	python -m coverage html
+	$(PYTHON) -m coverage html
 	@echo "Report written to htmlcov/index.html"
 
-locust:
+locust: $(VENV_STAMP)
 	export DEVPLACE_DATABASE_URL="sqlite:///$(LOCUST_DB)"; \
 	export DEVPLACE_RATE_LIMIT=$(DEVPLACE_RATE_LIMIT); \
 	fuser -k $(LOCUST_PORT)/tcp 2>/dev/null || true; \
 	sleep 1; \
 	mkdir -p $(LOCUST_DB_DIR); \
 	rm -f $(LOCUST_DB); \
-	DEVPLACE_TEMPLATE_AUTO_RELOAD=0 DEVPLACE_WEB_WORKERS=$(LOCUST_WEB_WORKERS) uvicorn devplacepy.main:app --host 127.0.0.1 --port $(LOCUST_PORT) --workers $(LOCUST_WEB_WORKERS) --backlog 8192 > /tmp/devplace_locust_server.log 2>&1 & \
+	DEVPLACE_TEMPLATE_AUTO_RELOAD=0 DEVPLACE_WEB_WORKERS=$(LOCUST_WEB_WORKERS) $(PYTHON) -m uvicorn devplacepy.main:app --host 127.0.0.1 --port $(LOCUST_PORT) --workers $(LOCUST_WEB_WORKERS) --backlog 8192 > /tmp/devplace_locust_server.log 2>&1 & \
 	PID=$$!; \
 	while kill -0 $$PID 2>/dev/null && ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
 	if ! kill -0 $$PID 2>/dev/null; then echo "Server failed to start (port $(LOCUST_PORT) busy?). See /tmp/devplace_locust_server.log"; exit 1; fi; \
-	locust --host http://127.0.0.1:$(LOCUST_PORT) --web-port $(LOCUST_WEB_PORT); \
+	$(PYTHON) -m locust --host http://127.0.0.1:$(LOCUST_PORT) --web-port $(LOCUST_WEB_PORT); \
 	kill $$PID 2>/dev/null || true; \
 	rm -rf $(LOCUST_DB_DIR)
 
-locust-headless:
+locust-headless: $(VENV_STAMP)
 	export DEVPLACE_DATABASE_URL="sqlite:///$(LOCUST_DB)"; \
 	export DEVPLACE_RATE_LIMIT=$(DEVPLACE_RATE_LIMIT); \
 	fuser -k $(LOCUST_PORT)/tcp 2>/dev/null || true; \
 	sleep 1; \
 	mkdir -p $(LOCUST_DB_DIR); \
 	rm -f $(LOCUST_DB); \
-	DEVPLACE_TEMPLATE_AUTO_RELOAD=0 DEVPLACE_WEB_WORKERS=$(LOCUST_WEB_WORKERS) uvicorn devplacepy.main:app --host 127.0.0.1 --port $(LOCUST_PORT) --workers $(LOCUST_WEB_WORKERS) --backlog 8192 > /tmp/devplace_locust_server.log 2>&1 & \
+	DEVPLACE_TEMPLATE_AUTO_RELOAD=0 DEVPLACE_WEB_WORKERS=$(LOCUST_WEB_WORKERS) $(PYTHON) -m uvicorn devplacepy.main:app --host 127.0.0.1 --port $(LOCUST_PORT) --workers $(LOCUST_WEB_WORKERS) --backlog 8192 > /tmp/devplace_locust_server.log 2>&1 & \
 	PID=$$!; \
 	while kill -0 $$PID 2>/dev/null && ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
 	if ! kill -0 $$PID 2>/dev/null; then echo "Server failed to start (port $(LOCUST_PORT) busy?). See /tmp/devplace_locust_server.log"; exit 1; fi; \
-	locust --host http://127.0.0.1:$(LOCUST_PORT) --headless -u $(LOCUST_USERS) -r $(LOCUST_SPAWN_RATE) --run-time $(LOCUST_RUN_TIME) --html=$(LOCUST_DB_DIR)/report.html; \
+	$(PYTHON) -m locust --host http://127.0.0.1:$(LOCUST_PORT) --headless -u $(LOCUST_USERS) -r $(LOCUST_SPAWN_RATE) --run-time $(LOCUST_RUN_TIME) --html=$(LOCUST_DB_DIR)/report.html; \
 	kill $$PID 2>/dev/null || true; \
 	rm -rf $(LOCUST_DB_DIR)
 
@@ -127,6 +148,23 @@ clean:
 test-cache-clean:
 	rm -rf .pytest_cache
 
+# Builds the donor clients (Go binaries for five platforms and the browser
+# extension zip) into the directory the /nodes/download route serves.
+NODE_CLIENT_OUT ?= data/node_clients
+.PHONY: node-client
+node-client:
+	sh nodes/client/build.sh $(NODE_CLIENT_OUT)
+	sh nodes/extension/build.sh $(NODE_CLIENT_OUT)
+
+# Safe but aggressive disk-space cleanup, acting on the REAL data/devplace.db
+# and data/ tree (never the test database) - see CLAUDE.md "devplace system
+# prune". prune-dry-run reports what would be removed without deleting.
+prune: $(VENV_STAMP)
+	$(PYTHON) -m devplacepy.cli system prune
+
+prune-dry-run: $(VENV_STAMP)
+	$(PYTHON) -m devplacepy.cli system prune --dry-run
+
 # Container Manager works out of the box: the overlay installs the docker CLI in
 # the image and mounts the host socket. DOCKER_GID is read straight from the
 # socket so the UID-1000 app can use it; the data dir is the project's own data/
@@ -183,6 +221,36 @@ docker-clean:
 
 docker-bup: docker-build docker-up
 
+# Rootless-podman hosts have no docker daemon: the podman overlay runs the app
+# as container root (== invoking host user under rootless, see
+# docker-compose.podman.yml) so the bind-mounted checkout stays writable.
+# Always update through these targets - a bare `up` without the overlay file
+# recreates the app with the base `user: 1000:1000` and it crashes on startup
+# with PermissionError (container uid 1000 maps to a subordinate host uid).
+PCOMPOSE := podman-compose -f docker-compose.yml -f docker-compose.podman.yml
+
+.PHONY: podman-build podman-up podman-reload podman-down podman-logs podman-ps
+
+podman-build: docker-prep
+	$(PCOMPOSE) build
+
+podman-up: docker-prep
+	$(PCOMPOSE) up -d
+	@loginctl show-user "$$(id -un)" 2>/dev/null | grep -q 'Linger=yes' || echo "WARNING: user linger is off - containers will NOT restart after a reboot until someone logs in (fix: sudo loginctl enable-linger $$(id -un))"
+
+podman-reload:
+	$(PCOMPOSE) restart app
+	$(PCOMPOSE) up -d
+
+podman-down:
+	$(PCOMPOSE) down
+
+podman-logs:
+	$(PCOMPOSE) logs -f
+
+podman-ps:
+	$(PCOMPOSE) ps
+
 deploy:
 	git checkout production
 	git merge master

README.md

@@ -7,7 +7,7 @@ Server-rendered social network for developers. FastAPI backend serving Jinja2 te
 ## Quick start
 
 ```bash
-make install        # pip install -e .
+make install        # create .venv if needed, pip install -e ".[dev]", playwright chromium
 make dev            # uvicorn --reload on port 10500
 make test           # Playwright integration + unit tests, headless, fail-fast
 make test-headed    # same tests in visible browser
@@ -42,12 +42,15 @@ sudo apt-get install -y libglib2.0-0 libpango-1.0-0 libpangoft2-1.0-0 libharfbuz
 ```
 devplacepy/
   main.py             # FastAPI app, router registration
+  middleware/         # HTTP middlewares + shared request state (package)
+  errors.py           # Global exception handlers
+  lifespan.py         # Startup/shutdown, service registry
   config.py           # Settings from env vars + .env
   database/           # dataset connection, index creation (package)
   templating.py       # Shared Jinja2 environment + globals
   avatar.py           # Multiavatar generation, URL builder
   utils/              # Password hashing, session mgmt, time_ago, notification hook (package)
-  models.py           # Pydantic schemas
+  models/             # Pydantic Form models, one module per domain (package)
   push/               # Push delivery: provider protocol, Web Push, APNs, registrations
   routers/            # One file per domain (auth, feed, posts, push, ...)
   templates/          # Jinja2 HTML templates
@@ -63,17 +66,21 @@ devplacepy/
 | `/` | Home page: marketing splash for guests, personalized home (welcome, feed shortcut, latest posts, news) for signed-in users. Does not redirect. Latest-posts section interleaves authors so no two consecutive posts share an author. |
 | `/auth` | Signup, login, logout, forgot/reset password |
 | `/feed` | Post feed with topic/tab filtering and free-text `search` (title, content, and author username) in the left panel (public). Each page interleaves authors so no two consecutive posts share an author. |
+| `/topics` | Crawlable per-topic category pages (public): `/topics` hub links every topic with a live post count, `/topics/{topic}` lists that topic's posts with its own canonical URL, title, and breadcrumbs |
+| `/skills` | Per-user Agent Skills: list, detail, install from a URL, enable/disable, delete. Private to the owner, `noindex` |
+| `/nodes` | Community node network: `GET /nodes/spec.json`, `GET/POST /nodes/devices` (+ `/rotate`, `/delete`), `POST /nodes/ticket`, `WS /nodes/ws`. Off by default; also `GET /nodes/me`, `POST /nodes/settings`, `GET /nodes/social/search`, `POST /nodes/internal/fetch`, `GET /nodes/stats`, `GET /nodes/leaderboard`, `GET /nodes/client-version`, `GET /nodes/download/{artifact}` (see `plan.md`) |
+| `/sync` | Community sync: `GET /sync/spec.json`, `POST /sync/{users,projects,posts,comments,polls,votes,reactions,follows,poll-votes,attachments}`, `GET /sync/export/{kind}`, `GET /sync/deletions`, `POST /sync/deletions/ack`. Administrator only, off by default (`sync_enabled`), idempotent batch upserts keyed on the origin community's ids |
 | `/news` | Developer news listing, detail page with comments |
 | `/posts` | Post detail, creation |
-| `/gists` | Code gist listing, detail, creation, and editing; left panel offers language filtering and free-text `search` (title, description, and author username), public read |
-| `/comments` | Comment creation, owner editing (`POST /comments/edit/{comment_uid}`), deletion |
+| `/gists` | Code gist listing, detail, creation, and editing; left panel offers language filtering and free-text `search` (title, description, and author username), public read. A plain Markdown gist has a **View rendered / View raw** toggle beside its Copy button, switching between the raw source and the same rendered view a `Markdown Rendered` gist always shows |
+| `/comments` | Comment creation, owner editing (`POST /comments/edit/{comment_uid}`), deletion. Every comment has a **Copy link** button that copies its permalink (the parent post/gist/project/news URL plus `#comment-{uid}`) to the clipboard |
 | `/projects` | Project listing (left panel offers type filtering and free-text `search` over title, description, and author username), creation, owner editing (`POST /projects/edit/{slug}`), and per-project visibility toggles: `POST /projects/{slug}/private` (owner-only visibility) and `POST /projects/{slug}/readonly` (immutable files). A project hidden by a member stays visible to administrators, but a project hidden by an administrator is visible only to that owner administrator - other administrators cannot see it, its files, or its containers (web UI and REST API alike). The primary administrator (the first Admin account) is the single exception and retains full visibility |
 | `/projects/{slug}` | Dedicated project page: one encompassing card with a cover banner and project logo (owner-uploaded through the standard attachment uploader), the title overlaid on the banner, status/type/platform chips, owner-set Website and Repository links, section tabs (Overview, Devlog, Screenshots, Comments, Files), an About section, the Devlog timeline of every post linked to the project (owners post updates straight from the page via the shared composer preset to the `devlog` topic), a Screenshots gallery built from image attachments (owners add more from the More menu), and a sidebar with links, stats and the author card |
 | `/projects/{slug}/files` | Per-project filesystem: directory and file CRUD, upload, inline editing, and line-range operations (`lines` read, `replace-lines`, `insert-lines`, `delete-lines`, `append`) for surgical edits to large text files (public read, owner write; all writes refused while the project is read-only) |
 | `/zips` | Zip job status (`/zips/{uid}`) and archive download (`/zips/{uid}/download`); archives are queued via `/projects/{slug}/zip` and `/projects/{slug}/files/zip` |
 | `/forks` | Fork job status (`/forks/{uid}`); forks are queued via `/projects/{slug}/fork`. Any signed-in user can fork a project they can view into a new project they own; once the job finishes the response carries the new project URL |
-| `/tools` | Public developer tools. `/tools/seo` is **SEO Diagnostics**: audit any URL or sitemap and stream live progress over a websocket. Queue with `POST /tools/seo/run`, poll `GET /tools/seo/{uid}`, read the full report at `GET /tools/seo/{uid}/report`. `/tools/deepsearch` is **DeepSearch**: a multi-agent deep web researcher that crawls and indexes sources, synthesises a cited report with confidence scoring, and lets you chat over the gathered evidence. Queue with `POST /tools/deepsearch/run`, poll `GET /tools/deepsearch/{uid}`, read the report at `GET /tools/deepsearch/{uid}/session`, export at `/export.{md,json,pdf}`. `/tools/isslop` is the **AI Usage Analyzer**: classify a git repository or website as AI slop, sophisticated AI-assisted work or genuine human work. Queue with `POST /tools/isslop/run`, poll `GET /tools/isslop/{uid}` or the event trail at `GET /tools/isslop/{uid}/events`, read the report at `GET /tools/isslop/{uid}/report` (`.md` to download) and embed the SVG authenticity badge from `GET /tools/isslop/{uid}/badge.svg` |
-| `/projects/{slug}/containers` | Admin per-project container manager: create and control container instances, all running the shared prebuilt `ppy` image (there is no in-app image building). Reachable from the project page via the admin-only **Containers** button |
+| `/tools` | Public developer tools. `/tools/seo` is **SEO Diagnostics**: audit any URL or sitemap and stream live progress over a websocket. Queue with `POST /tools/seo/run`, poll `GET /tools/seo/{uid}`, read the full report at `GET /tools/seo/{uid}/report`. `/tools/deepsearch` is **DeepSearch**: a multi-agent deep web researcher that crawls and indexes sources, synthesises a cited report with confidence scoring, and lets you chat over the gathered evidence. Queue with `POST /tools/deepsearch/run`, poll `GET /tools/deepsearch/{uid}`, read the report at `GET /tools/deepsearch/{uid}/session`, export at `/export.{md,json,pdf}`. Every past run is kept at `GET /tools/deepsearch/history`, which lists your own research sessions with a link back to reopen the report and continue its grounded chat. `/tools/isslop` is the **AI Usage Analyzer**: classify a git repository or website as AI slop, sophisticated AI-assisted work or genuine human work. Queue with `POST /tools/isslop/run`, poll `GET /tools/isslop/{uid}` or the event trail at `GET /tools/isslop/{uid}/events`, read the report at `GET /tools/isslop/{uid}/report` (`.md` to download) and embed the SVG authenticity badge from `GET /tools/isslop/{uid}/badge.svg` |
+| `/projects/{slug}/containers` | Admin per-project container manager: create and control container instances, all running the shared prebuilt `ppy` image (there is no in-app image building). Reachable by direct URL and from the admin index |
 | `/admin/containers` | Admin **Containers** manager: list, create, edit, and control container instances across projects, under strict per-user isolation: the primary administrator sees and manages every instance; every other administrator sees instances on public projects plus their own (instances attached to another user's private project are excluded entirely) and manages only the instances they own (created by them or attached to their own project) - all other rows are view-only. The list (`/admin/containers`) has inline start/stop/restart/terminal/edit/delete on each row and a create form (pick a project, optionally a run-as user, a boot language with a source editor, restart policy, start-on-boot, plus env/ports/limits/ingress). Each instance has a detail page (`/admin/containers/{uid}`) with lifecycle controls, live logs and metrics, an interactive terminal, schedules, ingress, workspace sync, and a status history, and an edit page (`/admin/containers/{uid}/edit`) |
 | `/p/{slug}` | Public ingress proxy (HTTP + WebSocket) to a running container instance's published port, opt-in per instance via `ingress_slug` |
 | `/profile` | Profile view, editing, a public **Media** tab (`?tab=media`) showing every attachment a user uploaded newest first, and a live **online / last-seen** presence indicator |
@@ -82,10 +89,11 @@ devplacepy/
 | `/admin/trash` | Admin **Trash**: review, restore, and permanently purge soft-deleted content (posts, comments, gists, projects, news, project files, attachments) across the platform |
 | `/admin/devii-tasks` | Admin **Devii tasks**: every scheduled task across all owners with its schedule, run count, expiry and failure streak, plus per-task disable and delete |
 | `/notifications` | Notification list, mark read, live unread counts (`/notifications/counts`) |
-| `/messages` | Real-time direct messaging over WebSocket (`/messages/ws`): live bidirectional delivery, optimistic send, typing indicators, read receipts, and online/last-seen presence. Messages render through the shared content pipeline (emoji shortcodes, image, video and audio embeds, autolink, sanitization). AI content correction and the AI modifier apply to direct messages, so typing an inline `@ai <instruction>` in a message executes it and the resolved result appears live in the chat for both participants (the `message` WS frame carries an additive `ai_processed` flag when a pending correction/modification was applied). An opened conversation loads its 500 most recent messages; older history is retained in the database. `GET /messages/conversations` returns the conversation list as JSON for a live client refresh; the `POST /messages/send` form remains as a no-JavaScript fallback and now accepts an attachment-only, empty-content message. `POST /messages/ws-ticket` exchanges a session/API-key auth to a short-lived (30s), single-use ticket a browser WebSocket can carry as `?ticket=...` on the handshake, since a native `WebSocket` cannot set an `Authorization`/`X-API-KEY` header - this is what makes off-session embedding of the chat possible. CLI: `devplace messaging prune-tickets` removes expired tickets |
+| `/messages` | Real-time direct messaging over WebSocket (`/messages/ws`): live bidirectional delivery, optimistic send, typing indicators, read receipts, reconnect catch-up, and online/last-seen presence. Messages render through the shared content pipeline (emoji shortcodes, image, video and audio embeds, autolink, sanitization). AI content correction and the AI modifier apply to direct messages: the original is delivered immediately, then a second `message` frame with `ai_processed` replaces the bubble for both participants when the rewrite lands (including across workers). An opened conversation loads its 500 most recent messages and can page older rows with `?before=`; switching threads does not drop the socket. `GET /messages/conversations` returns the conversation list as JSON for a live client refresh; the `POST /messages/send` form remains as a no-JavaScript fallback and now accepts an attachment-only, empty-content message. `POST /messages/ws-ticket` exchanges a session/API-key auth to a short-lived (30s), single-use ticket a browser WebSocket can carry as `?ticket=...` on the handshake, since a native `WebSocket` cannot set an `Authorization`/`X-API-KEY` header - this is what makes off-session embedding of the chat possible. CLI: `devplace messaging prune-tickets` removes expired tickets |
 | `/votes` | Upvote/downvote on posts, comments, projects |
 | `/reactions` | Emoji reactions on posts, comments, gists, projects |
 | `/bookmarks` | Save/unsave content; `/bookmarks/saved` personal list |
+| `/notes` | Private per-user notes attached to a post, gist, project, or news article; `POST /notes/{target_type}/{target_uid}` adds or replaces the note, `POST /notes/{target_type}/{target_uid}/delete` removes it, `/notes/saved` is your personal notes list |
 | `/polls` | Vote on post-attached polls |
 | `/follow` | Follow/unfollow users |
 | `/reports` | Content reporting: `POST /reports/{target_type}/{target_uid}` files a report against any user-generated surface, `GET /reports/mine` lists the reports you filed and their outcome, `GET /reports/reasons` serves the reason registry so every client renders the same dialog |
@@ -97,6 +105,7 @@ devplacepy/
 | `/leaderboard` | Contributor ranking by total stars earned |
 | `/game` | **Code Farm** cooperative idle game (member-only): plant projects that build over real time, harvest coins and XP, upgrade CI, buy plots, and water friends' builds at `/game/farm/{username}`. Live over pub/sub; every endpoint negotiates JSON |
 | `/quizzes` | **Quizzes**: author quizzes, play them, and climb the cross-quiz scoreboard. Three-column hub with filters (`all`/`todo`/`done`/`mine`/`drafts`), search, per-viewer state badges, and the scoreboard rail; `/quizzes/{slug}` detail, `/quizzes/{slug}/edit` builder, `/quizzes/{slug}/attempts/{uid}` player, `/quizzes/scoreboard` JSON. Publishing is permanent. Every endpoint negotiates JSON |
+| `/battles` | **Opinion Wars**: week-long two-faction battles attached to posts, started from the composer's *Start Opinion War* builder. Members join a side and fight once a day (25 Code Farm coins, level-weighted damage); the pixel-art battle card shows live HP bars, a countdown, top contributors and an event ticker. `/battles` lists battles (`active`/`ended`/`mine` + search); `/battles/{uid}` state, `/battles/{uid}/events` replay, `/battles/{uid}/join` and `/battles/{uid}/fight` actions |
 | `/avatar` | Multiavatar proxy with in-memory cache |
 | `/issues` | Issue tracker backed by Gitea: list/detail/comment/status, AI-enhanced filing, an admin planning report over a selectable set of open tickets (each ticket's full text reproduced verbatim so the document hands straight to a coding agent), and file attachments on open issues and comments (mirrored to the Gitea tracker) |
 | `/admin/services` | Background service management (start/stop, config, status, logs) |
@@ -106,7 +115,7 @@ devplacepy/
 | `/openai` | OpenAI-compatible LLM gateway service (`/openai/v1/chat/completions`, `/openai/v1/*`) |
 | `/devii` | Devii agentic assistant: WebSocket terminal (`/devii/ws`), standalone page, usage (`/devii/usage`), session bootstrap |
 | `(none)` | `/robots.txt`, `/sitemap.xml` (SEO) |
-| `(none)` | `/push.json` (VAPID key + subscribe), `/service-worker.js`, `/manifest.json` (push + PWA) |
+| `(none)` | `/push.json` (VAPID key + subscribe/unsubscribe), `/service-worker.js`, `/manifest.json` (push + PWA) |
 
 ## Gamification
 
@@ -169,10 +178,33 @@ scoreboard on the right. Guests read published quizzes and see the board; they c
 Retention: completed attempts are permanent; abandoned and expired ones are garbage-collected by
 `devplace quiz prune`.
 
+## Opinion Wars
+
+**Opinion Wars** (`/battles`) are week-long two-faction battles attached to posts, in the spirit of
+old eRepublik battles: settle tabs-versus-spaces by showing up daily and fighting for your side.
+
+- **Start one from the composer.** The *Start Opinion War* button next to *Add poll* names the two
+  factions; the battle runs for exactly 7 days from the moment the post is published.
+- **Join and fight.** Any signed-in member picks a side and may fight once every 24 hours per
+  battle. A fight costs 25 Code Farm coins and deals deterministic, level-weighted damage
+  (100 HP + 10 per site level, capped at level 20) - no randomness, dedication wins wars.
+- **Defection is allowed.** Switch factions any time; damage already dealt stays where it landed.
+- **Live pixel-art card.** The battle renders on the post as a CSS pixel-art battlefield with HP
+  bars, a countdown, your rank, top contributors and a live event ticker (joins, defections,
+  fights, lead changes) over pub/sub with an incremental replay fallback.
+- **Rewards.** When the week ends the bigger total wins: every fighter earns XP, the winning side
+  and the top damage dealer earn bonuses, and battle badges (*Instigator*, *First Blood*,
+  *War Veteran*, *Champion*) mark the milestones. Notifications cover lead changes, the result and
+  your next fight being ready.
+
 ## Code Farm
 
 The **Code Farm** (`/game`) is a cooperative idle game in the spirit of Farmville, themed for developers. Each member owns a farm of plots and plays asynchronously - nothing has to happen in real time.
 
+The game presents plots as individual servers in one front-view rack. Select a server to spin up a workload from the right-hand card picker, watch its build progress, and harvest the result. Buying a plot installs the next server, up to the existing twelve-server limit. Cooling reflects the Build Cache perk, networking reflects the CI tier, and power reflects the Optimizer perk; installed infrastructure also lights up the rack. These are visual representations of the existing upgrades and rules. Keyboard navigation, mobile controls, and the server-rendered no-JavaScript forms are supported.
+
+The rack's **control terminal** supports every current game API route. Try `help`, `servers`, `plant 3 python`, `harvest all`, or `ci upgrade`; server numbers match the rack (1–12). Up/Down recalls commands and Tab completes commands and upgrade keys. `api` lists the route coverage, while `state` exposes the full JSON snapshot. Terminal commands and page controls share the same action handler and backend permissions. Refactoring, raids, infrastructure/cosmetic purchases, and defense downgrades retain confirmation dialogs. The terminal keeps drafts and history through live refreshes and reports actual control actions and completed builds.
+
 - **Plant.** Plant a software project (a shell script, Python script, web app, Go service, Rust engine, compiler, or kernel) in an empty plot for a coin cost. Higher-tier crops unlock as your farm level rises.
 - **Build and harvest.** A planted crop builds over real time; when the build finishes, harvest it for coins and XP. Harvesting also awards site XP and the **Green Thumb** / **Master Farmer** badges.
 - **Upgrade CI.** Spend coins to raise your CI tier (Local Build through Distributed Cache); each tier makes every build faster.
@@ -195,7 +227,7 @@ The **Code Farm** (`/game`) is a cooperative idle game in the spirit of Farmvill
 - **Leaderboards.** Several boards, selectable from the game page: **Overall score** (a composite weighing refactor/prestige count, XP, lifetime harvests, a capped coin contribution, CI tier, plots, perks, and streak - the cap keeps it a measure of what you built rather than what you hoard), **Prestige**, **Harvests this week**, **Raid efficiency** (average coins per successful raid), **Fastest to Kernel** (time since your last refactor), **Fair play** (rewards recent activity over hoarding), and (when running) the current **Era** board.
 - **Eras (admin-managed seasons).** Administrators can start an Era at `/admin/game`: every farm's *visible* Era coins/harvests counters reset to zero, but real coin balances, prestige, Stars, Legacy, and Mastery are never touched. Ending an Era ranks farms by Era score (which gives prestige only partial weight, so veterans keep an edge without it being insurmountable), awards Stars to the top 10, and permanently records the results.
 
-The farm refreshes live over the pub/sub bus (a watered build appears on the owner's screen at once) and every plot countdown ticks client-side. Every endpoint also answers JSON, and Devii can play the game on the member's behalf via the `game_*` tools (`game_state`, `game_plant`, `game_harvest`, `game_buy_plot`, `game_upgrade_ci`, `game_fertilize`, `game_daily`, `game_claim_quest`, `game_water`, `game_steal`, `game_view_farm`, `game_leaderboard`, `game_upgrade_perk`, `game_upgrade_legacy`, `game_prestige`, `game_claim_grant`, `game_upgrade_mastery`, `game_buy_infrastructure`, `game_upgrade_defense`, `game_buy_cosmetic`, `game_equip_cosmetic`, `game_downgrade_defense`). See the API reference group **Code Farm** and the full player guide at `/docs/code-farm.html`.
+The farm refreshes live over the pub/sub bus (a watered build appears on the owner's screen at once) and every plot countdown ticks client-side. Every endpoint also answers JSON, and Devii can play the game on the member's behalf via the `game_*` tools (`game_state`, `game_plant`, `game_harvest`, `game_plot_stop`, `game_buy_plot`, `game_upgrade_ci`, `game_fertilize`, `game_daily`, `game_claim_quest`, `game_water`, `game_steal`, `game_view_farm`, `game_leaderboard`, `game_upgrade_perk`, `game_upgrade_legacy`, `game_prestige`, `game_claim_grant`, `game_upgrade_mastery`, `game_buy_infrastructure`, `game_upgrade_defense`, `game_buy_cosmetic`, `game_equip_cosmetic`, `game_downgrade_defense`). See the API reference group **Code Farm** and the full player guide at `/docs/code-farm.html`.
 
 ## Engagement
 
@@ -204,6 +236,7 @@ The farm refreshes live over the pub/sub bus (a watered build appears on the own
 - **Polls** - a post can carry a poll (question plus up to six options); results appear as live bars once the viewer votes, one vote per member. A poll can be attached when the post is created or added later by editing a post that has none.
 - **Paste an image to attach it** - pressing Ctrl+V (Cmd+V) with a screenshot or copied image on the clipboard while writing a post, a comment, a direct message, an issue, a gist, or a project attaches it immediately, with no trip through the file picker. The upload, its limits, and the resulting attachment are identical to picking the file by hand.
 - **Bookmarks** - save posts, gists, projects, and news to a personal list at `/bookmarks/saved`.
+- **Personal notes** - attach a private note to a post, gist, project, or news article; nobody else, not even the content's author, can ever see it. Managed from an "Add note"/"Edit note" button on the item and listed on a personal page at `/notes/saved`.
 - **Private projects** - an owner can mark a project private so it is visible only to them (and administrators) and excluded from listings, profiles, search, the sitemap, and zip access. Set at creation or toggled later from the project page.
 - **Read-only projects** - an owner can mark a project read-only, making its entire virtual filesystem immutable: every write, edit, line-edit, move, delete, and upload is refused from all paths (the web UI, the HTTP API, the Devii agent, and container workspace sync) until read-only is turned off. Devii may toggle read-only only after the user explicitly confirms.
 
@@ -223,6 +256,26 @@ DevPlace is open in the sense that it does not editorialise technical opinion. I
 - **Account deletion.** Self-service at `/profile/{username}/delete`, reauthenticated with the account password. Sessions and tokens are revoked, the profile is anonymised immediately, and all content is removed under one deletion event that stays restorable for `account_deletion_grace_hours` (default 24) before `devplace accounts prune` (and the Moderation housekeeping service) purges it permanently. The devRant `DELETE /api/users/me` routes into the same cascade.
 - **Legal pages**: [Terms of Service](/docs/terms.html), [Community Guidelines](/docs/community-guidelines.html), [Privacy Policy](/docs/privacy.html), [How moderation works](/docs/content-moderation.html), [Notice and takedown](/docs/intellectual-property.html) and [Contact](/docs/contact.html). All six are indexed in the sitemap and reachable from the docs index; the footer of every page links Terms, Privacy, Community Guidelines and Contact. Contact details come from the `contact_email`, `contact_phone` and `contact_address` settings, so the in-product page and any app-store trader declaration have one source of truth.
 
+## Skills
+
+Members can install [Agent Skills](https://agentskills.io) from a URL at `/skills` so Devii can use them. A skill is a `SKILL.md` (YAML frontmatter plus markdown instructions) with optional text files under `scripts/`, `references/` and `assets/`. Skills are private to their owner, count against a per-user cap, and only the name and a one-line description of enabled skills are added to Devii's system prompt; Devii loads the full text on demand. A skill is untrusted third-party text: it can never grant Devii a tool, bypass a confirmation or run a script. Deleting a skill is a soft delete and needs confirmation when Devii does it. Documented in the API docs under *Skills* and in `devplacepy/services/skills/CLAUDE.md`.
+
+## Community sync
+
+Users, posts and comments can be copied in from another community. The sending side posts batches of up to 50 records to `POST /sync/users`, `/sync/posts` and `/sync/comments` (administrator API key, `sync_enabled` set to `1`). Each record carries the id it has in the origin community (`origin_uid`); the server keeps a `sync_links` row from `(origin, kind, origin_uid)` to the local row, so a repeated record is an update or a no-op and never a duplicate. Records are validated and normalized with Pydantic (`devplacepy/services/sync/records.py`) and every record gets its own result, so one bad record never fails the batch. Stars, XP and levels cannot be verified across communities and are not imported; they are listed under `ignored` in the result. A record whose post, parent, poll or project has not arrived yet is parked in a bounded pending queue and released when it does. Synced content sends no notifications, awards no XP, runs no AI correction and passes the content filter. `sync_origins` optionally restricts which origins may send. Limits, statuses and the record schemas are published at `GET /sync/spec.json`. The opt-in Community sync service (`CommunityService`) is the scheduled sender: it learns an external community, runs its generated collectors and feeds this API. A run refused by the kill switch, the allowlist or the aliasing switch is recorded as `skipped` and is never counted as a failure. Every run also writes a per-record sync log (`/admin/communities/{slug}/ledger?run=<uid>`): a verdict, exact counts per outcome (created, updated, unchanged, stale, skipped, warning, rejected, failed, error), and a filterable, paginated list with the reason each record was not synced in plain words. It is index-bound and capped, so it stays fast however large a run is. The origin can read what we hold through `GET /sync/export/{kind}` (public data only, rows that arrived by sync are never echoed back) and which of its records we removed through `GET /sync/deletions`. Usage is written up in `prompt.md`.
+
+### Community discovery
+
+Administrators can add an external community by name or address at `/admin/communities` (or `devplace community discover`). The system searches (rsearch), detects RSS, Atom, JSON and HTML feeds, learns the structure once, records the community's description, rules and size for the admin detail page, generates standalone collector code under `data/communities/<slug>/`, and then runs it on a schedule through the sync API above. Members never see these communities. Login-protected communities take a stored account (form, JSON-API and TOTP logins; a captcha is detected and reported, never solved); a run keeps checkpoints so a killed run stays visible; administrators can edit and restore a community's learned schema at `/admin/communities/{slug}/schema`; new accounts created by a community run get a realistic handle derived from the author (the person behind a `by`/`via` is extracted from feed strings, then a seeded, weighted pattern adds a year, number or light humor; existing accounts are never renamed; `devplacepy/services/sync/CLAUDE.md`, "Realistic handles"); JavaScript pages are rendered through stealthii; the internal AI model helps with HTML structure it cannot read otherwise. Administrators can manage everything through the admin pages, `devplace community ...` and Devii. Dry run (`--dry-run`, the admin button, and the default of the `Community sync` service) validates and logs everything without writing. Design and limits: `add_community.md` and `devplacepy/services/community/CLAUDE.md`. Per community, administrators can optionally (all off by default) sample which posts are accepted, filter by topic and minimum comments, jitter the schedule, translate content, and store authors under permanent pseudonyms with every mention rewritten (`community_sync_additional.md`); this is administrator-only and invisible to members.
+
+## Community nodes (built, off by default)
+
+Members will be able to opt in to donate a little of their connection to the DevPlace community, so search can reach sites that block datacenter addresses. **Today the server side exists:** the specification (`GET /nodes/spec.json` and the [Node network protocol](/docs/nodes-protocol.html) docs page, both generated from `devplacepy/services/nodes/spec.py`), device registration (`/nodes/devices`), the socket ticket (`POST /nodes/ticket`), the node socket (`WS /nodes/ws`), the consent gate and the administrator kill switch (`nodes_enabled`, off by default). The server queues, schedules and meters jobs and relays them either as a `tunnel` (raw TCP, TLS stays end to end between DevPlace and the site, so the donor cannot read or forge the page) or as a `fetch` (extension donors, results verified and labelled). Donor clients are a Go binary (`nodes/client/`) and a browser extension (`nodes/extension/`); `make node-client` builds them into `data/node_clients/`, served at `/nodes/client-version` and `/nodes/download/{artifact}`. Social search (`GET /nodes/social/search`, Devii `social_search`, optional DeepSearch source) reads Hacker News, Stack Overflow, GitHub, dev.to and Mastodon directly and Reddit through donors when `nodes_social_via_nodes` is on. Whether Reddit works from residential donors is not measured yet. Administrators manage it at `/admin/nodes` (pool and queue summary, devices by status, kind, address class and country, open flags, timed device suspension, the kill switch and the caps), with the same operations available to Devii for administrators.
+
+Members manage their donation on the owner-only **Donate** tab of their profile (`/profile/{username}?tab=donate`, noindex): the explanation, the `bandwidth_sharing` consent, limits and active hours, devices with a one-time secret and install lines, and a live "Donated by you" card. The user guide is [Donate your connection](/docs/nodes.html).
+
+What is fixed already, because the specification enforces it: nobody is enrolled by default; a node is not an open proxy and only ever contacts a short published allowlist of websites, on ports 80 and 443, never a private address; the refusal is enforced on the donor's machine as well as on the server; every limit is a published hard cap. The implementation plan, the measured and unmeasured claims, and the legal gate are in `plan.md`.
+
 ## Vibe coding (Alpha, admin only)
 
 Build software by talking to an AI agent instead of typing every line. Create a project for storage, attach a container to it (the shared `ppy` image, your files mounted at `/app`), start it, and open a terminal. The whole flow is drivable conversationally through Devii. Inside every container three agents ship preinstalled and run on **your own API key**, so all AI usage is metered to your account: **DevPlace Code (`dpc`)**, a coding agent in the same class as Claude Code; **`botje.py`**, a plug-and-play DevPlace bot you can copy and customise; and **`pagent`**, a minimal zero-dependency agent. Each container is launched with `DEVPLACE_BASE_URL`, `DEVPLACE_OPENAI_URL`, `DEVPLACE_API_KEY`, `DEVPLACE_USER_UID`, `DEVPLACE_CONTAINER_NAME`, `DEVPLACE_CONTAINER_UID`, and `DEVPLACE_INGRESS_URL` already set. Publish a container port to a public URL at `/p/<slug>` by setting an `ingress_slug` and `ingress_port` (ask Devii to do it at create time). The feature is in **Alpha** and currently limited to administrators; the full walkthrough, including a tutorial that vibes a web app and puts it online, is at `/docs/getting-started-vibing.html`.
@@ -239,12 +292,14 @@ The log is **administrator-only**. `/admin/audit-log` is a paginated, filterable
 |---------|---------|---------|
 | `DEVPLACE_DATABASE_URL` | `sqlite:///<repo>/data/devplace.db` | Database connection string |
 | `DEVPLACE_DATA_DIR` | `<repo>/data` | Single root for every runtime/user-generated artifact (DB, uploads, VAPID keys, locks, bot state, job staging, container workspaces), outside the package and not served via `/static`. Point at a volume in production. Defined once in `config.py` (`DATA_PATHS` registry, created by `ensure_data_dirs()`) |
+| `DEVPLACE_PORT` | `10500` | Port the app itself binds to (`make dev`/`make prod`, and what `Makefile`'s `dev`/`prod` targets pass to uvicorn's `--port`). Also feeds `config.PORT`, so `DEVPLACE_INTERNAL_BASE_URL`'s default and Devii's own self-dial URL follow it automatically. Bare metal only - the Docker app container always binds its fixed internal port regardless of this var; for Docker, use `PORT` (below) to change the externally reachable port |
+| `PORT` | `10500` | Docker only: the host port `docker-compose.yml` publishes nginx on (`127.0.0.1:${PORT}:80`). Unrelated to `DEVPLACE_PORT` above - it never reaches the app container |
 | `SECRET_KEY` | hardcoded fallback | Session signing key |
 | `DEVPLACE_VAPID_SUB` | `mailto:retoor@molodetz.nl` | Contact address in the VAPID JWT `sub` claim |
 | `DEVPLACE_INTERNAL_BASE_URL` | `http://localhost:10500` | Base URL the platform's own services dial for the AI gateway |
 | `DEVPLACE_XMLRPC_PORT` | `10550` | Loopback port the forking XML-RPC bridge binds; the app and nginx reverse-proxy `/xmlrpc` to it |
 | `DEVPLACE_XMLRPC_BIND` | `127.0.0.1` | Bind address for the XML-RPC bridge (loopback; the app and nginx are the intended front doors) |
-| `DEVPLACE_STATIC_VERSION` | server boot unix timestamp | Cache-busting version stamped into every static asset URL (`/static/v<version>/...`). Set it at launch so multiple workers agree (the `prod` target and Docker image do this); leave unset in dev to refresh on each reload. See [Static asset caching](#static-asset-caching) |
+| `DEVPLACE_STATIC_VERSION` | server boot unix timestamp | The boot-id half of the cache-busting version stamped into every static asset URL (`/static/v<app-version>-<boot-id>/...`, e.g. `/static/v1.0.1-1718040000/...`). Set it at launch so multiple workers agree (the `prod` target and Docker image do this); leave unset in dev to refresh on each reload. See [Static asset caching](#static-asset-caching) |
 | `DEEPSEEK_API_KEY` / `OPENROUTER_API_KEY` | unset | Upstream provider keys; migrated into the gateway settings on first boot |
 | `DEVPLACE_PRESENCE_TIMEOUT_SECONDS` | `60` | Online-presence window: a user counts as online for this many seconds after their last activity. `last_seen` is refreshed by a throttled in-place update at most once per half this interval per worker (no per-load inserts, no data growth) |
 | `DEVPLACE_PRESENCE_ONLINE_LIMIT` | `30` | Maximum avatars shown in the feed's live "Online now" panel (ordered alphabetically by username) |
@@ -265,6 +320,7 @@ Operational behavior is tunable live from `/admin/settings` (stored in `site_set
 | `registration_open` | `1` | When `0`, new sign-ups are rejected |
 | `maintenance_mode` | `0` | When `1`, non-admins see the maintenance page; admins retain access |
 | `maintenance_message` | scheduled-maintenance text | Message shown during maintenance |
+| `happy_404_enabled` | `1` | When `1`, an HTML page that would otherwise 404 renders a random existing post instead ("Happy 404"); JSON/API requests always get a real 404 regardless. See [Happy 404](#happy-404) |
 | `customization_enabled` | `1` | When `0`, no user CSS/JS customization is injected on any page |
 | `customization_js_enabled` | `1` | When `0`, user custom CSS is still served but custom JavaScript is suppressed |
 | `audit_log_retention_days` | `90` | Audit rows older than this are pruned daily by the Audit retention service; `0` disables pruning |
@@ -285,9 +341,13 @@ accepts three header-based methods, resolved centrally in `get_current_user`
 Every user has an `api_key` (a uuid7), set at signup and backfilled for existing
 users on startup (and via `devplace apikey backfill`). The key is shown on the
 profile page to its owner (and to admins for any user) with a live **Regenerate**
-button; regenerating invalidates the old key immediately. Invalid credentials
-return `401`; requests with no credentials behave as anonymous (browser redirect to
-login). Full details and copy-paste `curl` examples live at `/docs/authentication.html`.
+button; regenerating invalidates the old key immediately. A successful JSON
+`POST /auth/login` or `POST /auth/signup` also returns the account's `email`,
+`api_key`, and `is_admin` flag, and `GET /profile/{username}` as JSON reports
+`profile_is_admin` plus the key (null unless the viewer owns the profile or is
+an admin). Invalid credentials return `401`; requests with no credentials behave
+as anonymous (browser redirect to login). Full details and copy-paste `curl`
+examples live at `/docs/authentication.html`.
 
 ```bash
 curl -H "X-API-KEY: <your-key>" https://your-host/notifications
@@ -303,8 +363,9 @@ website does is automatable from the same URLs. A request gets JSON when it send
 `Accept: application/json` or `Content-Type: application/json`; a normal browser navigation
 (`Accept: text/html`) always gets HTML, so existing behaviour is unchanged (the legacy
 `X-Requested-With: fetch` AJAX header still drives the four engagement endpoints only). JSON responses are defined by Pydantic models in `devplacepy/schemas/` and built
-from the same context the templates use (sensitive user fields like `email`/`api_key`/
-`password_hash` are never exposed). Page GETs return the page payload; form actions return a
+from the same context the templates use (`password_hash` is never exposed; `email` and
+`api_key` are exposed only to the account owner or an admin, never to third parties).
+Page GETs return the page payload; form actions return a
 uniform envelope `{ "ok": true, "redirect": "…", "data": {…} }`; errors return
 `{ "error": { "status", "message" } }` (validation → `422` with a `fields` map, unauthenticated
 JSON → `401`, non-admin → `403`). The core lives in `devplacepy/responses.py`
@@ -319,6 +380,27 @@ curl -H "Accept: application/json" https://your-host/feed
 curl -H "Accept: application/json" -X POST -d "content=hi&title=T&topic=devlog" https://your-host/posts/create
 ```
 
+## Happy 404
+
+Instead of a bare error page, an HTML request that would 404 (an unmatched route, or an app route
+raising `not_found()` for a missing resource) instead renders a random existing post at that URL,
+using the exact same template and context as the real `/posts/{slug}` page. JSON/API requests are
+unaffected and still get a normal `404` - the substitution only ever applies to a browser HTML
+navigation.
+
+- **Toggle:** `happy_404_enabled` site setting (`/admin/settings`, on by default).
+- **Scope:** any HTML `GET` 404, app-wide - not just under `/posts`.
+- **Performance:** a small pool of random post slugs is cached in-process for a few minutes and
+  refreshed with a fresh random sample on expiry, so every request only does an in-memory pick plus
+  one indexed lookup - no per-request full-table scan - while the pool composition still cycles
+  through the whole `posts` table over time.
+- **SEO safety:** the substituted page is always marked `noindex,nofollow` so the decoy URL is never
+  indexed under the wrong address.
+- **Fail-closed:** any error while building the substitute page falls straight through to the normal
+  404 page - this feature can never turn a real error into a worse one.
+
+Implementation: `devplacepy/happy404.py`, wired into the `404` exception handler in `main.py`.
+
 ## XML-RPC bridge
 
 The full REST API is also reachable over XML-RPC at `/xmlrpc`. A standalone forking XML-RPC
@@ -432,16 +514,16 @@ and its full configuration are documented automatically - including future servi
 - **`ConfigField`** - declarative parameter spec (type, default, validation, secret) a service uses to declare its editable settings
 - **`BaseService`** - abstract class with a reconciling run loop that honors the persisted `enabled`/command/interval state, plus `config_fields`, `get_config()`, `describe()`, and a log buffer
 - **`ServiceManager`** - singleton: `register`, `describe_all`, `set_enabled`, `send_command`, `save_config`, `supervise`, `shutdown_all`
-- **`NewsService`** - a fully automatic, zero-maintenance news pipeline: fetches news from `news.app.molodetz.nl/api`, cleans each article, fetches and perceptually compares the images to reject placeholders and detect uniqueness, grades each one deterministically, reformats every valid article into clean Markdown (paragraphs, headings, lists) with the AI so the source wall of text reads as a proper article, and auto-rotates the best articles to Featured and the landing page. Its AI spend is metered from the gateway response headers and reported on the admin Services page (calls, tokens, total cost, and per-call averages)
+- **`NewsService`** - a fully automatic, zero-maintenance news pipeline: fetches news from `news.app.molodetz.nl/api`, cleans each article, fetches and perceptually compares the images to reject placeholders and detect uniqueness, grades each one against a free, local, deterministic quality model (aquality - no billed LLM call, no API key), optionally reformats articles that clear the publish grade threshold into clean Markdown with a generative AI model (off by default, since the default grading model cannot generate text), and auto-rotates the best articles to Featured and the landing page. Any AI spend it does incur is metered from the gateway response headers and reported on the admin Services page (calls, tokens, total cost, and per-call averages)
 - **`BotsService`** - Playwright fleet of AI personas that browse and interact with a DevPlace instance, with live cost/usage metrics and a live screenshot monitor at `/admin/bots` (opt-in; install the `bots` extra)
-- **`GatewayService`** - OpenAI-compatible LLM gateway at `/openai/v1/*`, forwarding to DeepSeek (default) with optional vision augmentation; the single point of truth for AI that every other service routes through (enabled by default). Every chat request is made date-aware by injecting the current date in EU `DD/MM/YYYY` format into the system message when it contains no date already (date only, never time, to keep upstream prompt caching effective), and an admin-configurable system preamble (`gateway_system_preamble`) can be prepended ahead of the client's system message on every call. **Provider and model routing** (admin **Gateway** page, `/admin/gateway`) maps any number of requested model names onto named upstream providers and target models, each with its own pricing economy (input, output and cache-hit/cache-miss rates per million tokens) and an optional vision model that describes image content before forwarding, so text and vision models are merged transparently. Unmapped requests fall through to the default upstream unchanged, so existing clients are unaffected
+- **`GatewayService`** - OpenAI-compatible LLM gateway at `/openai/v1/*`, forwarding to DeepSeek (default) with optional vision augmentation; the single point of truth for AI that every other service routes through (enabled by default). Every chat request is made date-aware by injecting the current date in EU `DD/MM/YYYY` format into the system message when it contains no date already (date only, never time, to keep upstream prompt caching effective), and an admin-configurable system preamble (`gateway_system_preamble`) can be prepended ahead of the client's system message on every call. **Thinking is disabled by default** on every chat and vision call (`thinking.type=disabled` on DeepSeek, `reasoning.effort=none` on OpenRouter, `think=false` on Ollama) so the fast path is the default; a client may re-enable it per request, and an administrator may flip `gateway_thinking` on. **Provider and model routing** (admin **Gateway** page, `/admin/gateway`) maps any number of requested model names onto named upstream providers and target models, each with its own pricing economy (input, output and cache-hit/cache-miss rates per million tokens) and an optional vision model that describes image content before forwarding, so text and vision models are merged transparently. Unmapped requests fall through to the default upstream unchanged, so existing clients are unaffected. Each route may also name a **fallback model** - another already-configured model of the same kind, picked from a select box of the public model names (never a free-text provider model id) - that the gateway retries once, automatically, whenever the primary model fails after its own retries are exhausted, so a struggling model degrades to a working one instead of failing the caller
 - **`JobService` / `ZipService` / `ForkService`** - generic async job framework (`services/jobs/`) for heavy, blocking work run off the request path; `ZipService` builds project zip archives in a subprocess, `ForkService` copies a project into a new project owned by the forking user
 - **`ContainerService`** - the admin container manager (`services/containers/`): a reconciling supervisor for container instances, all running one shared prebuilt image
 - **`AcceptanceService`** - grants every policy agreement (Terms of Service, Privacy Policy, third-party AI processing, activity recording, container credentials) to every account that has not declined it, so an instance kept production-identical for extended manual testing never interrupts with an acceptance dialog. Administrator-only, **off by default**, with a separate switch per agreement type and a dry run that reports what it would do without writing. An account that withdrew a consent is never granted it again, with no further action: the consent ledger itself is the decline register. It is not appropriate on a real production host
 
 ### Container manager (admin only)
 
-`services/containers/` runs supervised container instances from the web UI, the HTTP API, and Devii. It drives the `docker` CLI via async subprocesses behind a pluggable `Backend` interface (a `DockerCliBackend` plus a `FakeBackend` for tests). **There is no in-app image building.** Every instance runs ONE shared prebuilt image, `ppy:latest` (override `DEVPLACE_CONTAINER_IMAGE`), built once with **`make ppy`** from `ppy.Dockerfile`: a Python + Playwright base with a broad set of common libraries preinstalled, the `pravda` (uid 1000) user, the sudo superclone, `pagent` at `/usr/bin/pagent.py`, and the `code-server` browser IDE that backs dev workspaces all baked in. Creating an instance is then an instant `docker run` (it fails fast with a clear error if the `ppy` image has not been built yet). `ContainerService` reconciles desired instance state against `docker ps` each tick (containers are labeled `devplace.instance=<uid>`, so orphans are reaped and no state is lost), applies restart policies, fires cron/interval/one-time schedules, and samples metrics. The container's `/app` is bind-mounted to a persistent project workspace (materialized from the project files) and **stays in sync automatically**: the manager runs a bidirectional, newer-wins sync between the project files and the workspace on every start and roughly once a minute while running, so edits made inside the container and edits made in the project file editor converge without manual intervention (the sync only ever creates or overwrites the older copy of a file, never deletes one; a read-only project is export-only). Projects that need extra packages use runtime `pip install` (pravda owns the site-packages, no sudo needed) or `apt install` directly (the `pravda` user runs `apt`/`dpkg` through a fakeroot wrapper, so system packages install without root) or add the library to `ppy.Dockerfile` and rerun `make ppy`. Each instance can run a **boot script** in Python or Bash (written into the workspace and run on launch) or a plain boot command, can be set to **start automatically** whenever the container service starts, and can be configured to **run as** a chosen DevPlace user - which only selects whose identity and API key are injected into the container (the container always runs as the unprivileged `pravda` user). Every status change is recorded and shown as a status history on the instance page. A running instance can be **published** with an `ingress_slug`, making its service reachable (HTTP and WebSocket) at `/p/<slug>` through DevPlace. The manager is reached two ways: the admin **Containers** sidebar entry (`/admin/containers`) lists, creates, edits, and controls every instance across all projects and opens a dedicated detail page per instance, and each project page carries an admin-only **Containers** button to its own instance manager.
+`services/containers/` runs supervised container instances from the web UI, the HTTP API, and Devii. It drives the `docker` CLI via async subprocesses behind a pluggable `Backend` interface (a `DockerCliBackend` plus a `FakeBackend` for tests). **There is no in-app image building.** Every instance runs ONE shared prebuilt image, `ppy:latest` (override `DEVPLACE_CONTAINER_IMAGE`), built once with **`make ppy`** from `ppy.Dockerfile`: a Python + Playwright base with a broad set of common libraries preinstalled, the `pravda` (uid 1000) user, the sudo superclone, `pagent` at `/usr/bin/pagent.py`, and the `code-server` browser IDE that backs dev workspaces all baked in. Creating an instance is then an instant `docker run` (it fails fast with a clear error if the `ppy` image has not been built yet). `ContainerService` reconciles desired instance state against `docker ps` each tick (containers are labeled `devplace.instance=<uid>`, so orphans are reaped and no state is lost), applies restart policies, fires cron/interval/one-time schedules, and samples metrics. The container's `/app` is bind-mounted to a persistent project workspace (materialized from the project files) and **stays in sync automatically**: the manager runs a bidirectional sync between the project files and the workspace on every start and roughly once a minute while running, so edits made inside the container and edits made in the project file editor converge without manual intervention. Deletions propagate too, in both directions: deleting a file inside the container removes it from the project, and deleting it in the project's file editor removes it from the container, tracked against a per-file sync baseline so a genuine deletion is never confused with a file that simply has not been materialized to that workspace yet; an edit made after a conflicting deletion always wins and restores the file (a read-only project is export-only and always mirrors the project verbatim, including removing files the project no longer has). Projects that need extra packages use runtime `pip install` (pravda owns the site-packages, no sudo needed) or `apt install` directly (the `pravda` user runs `apt`/`dpkg` through a fakeroot wrapper, so system packages install without root) or add the library to `ppy.Dockerfile` and rerun `make ppy`. Each instance can run a **boot script** in Python or Bash (written into the workspace and run on launch) or a plain boot command, can be set to **start automatically** whenever the container service starts, and can be configured to **run as** a chosen DevPlace user - which only selects whose identity and API key are injected into the container (the container always runs as the unprivileged `pravda` user). Every status change is recorded and shown as a status history on the instance page. A running instance can be **published** with an `ingress_slug`, making its service reachable (HTTP and WebSocket) at `/p/<slug>` through DevPlace. The manager is reached two ways: the admin **Containers** sidebar entry (`/admin/containers`) lists, creates, edits, and controls every instance across all projects and opens a dedicated detail page per instance, and each project page carries an admin-only **Containers** button to its own instance manager.
 
 **Security:** this requires mounting the Docker socket, which grants host root. Every run, exec, lifecycle, and schedule operation is administrator-only; `--privileged` is never used and all docker calls are argument-list subprocesses. Containers are additionally isolated per user: an instance is managed only by its owner (the administrator who created it, or the owner of its project) and by the primary administrator, who alone sees and manages every instance including those on private projects; other administrators get a read-only view of instances on public projects and none of another user's private-project instances (exec, terminals, schedules, edits, and lifecycle actions are all refused and audited as denied). The service is disabled by default; an admin enables **Containers** on `/admin/services`. CLI: `devplace containers list | reconcile | prune | prune-builds | gc-workspaces` (`prune-builds` is a one-time cleanup that removes legacy per-project images and the old dockerfiles/builds tables).
 
@@ -450,10 +532,19 @@ and its full configuration are documented automatically - including future servi
 ### Dev Workspaces and the browser editor
 
 A **workspace** is a member-facing container running the DevPlace browser editor, layered on the
-container runtime above. It is opened from a project's **Workspace** page and reached at
-`/projects/{slug}/workspace`; the editor itself is proxied at
-`/projects/{slug}/containers/instances/{uid}/code/`, and an **Editor** button appears on the project
-page whenever the workspace is running.
+container runtime above. It is reached at `/projects/{slug}/workspace`; the editor itself is proxied
+at `/projects/{slug}/containers/instances/{uid}/code/`, and the workspace page shows an **Open
+editor** link whenever the editor is actually reachable.
+
+**The workspace page reports the editor's real state, live.** A workspace has a *phase* derived on
+the server from its desired state, its container status and a TCP probe of the editor port:
+`stopped`, `starting`, `ready`, `stopping`, `crashed` or `suspended`. The page renders the control
+that matches the phase, so pressing **Start** turns the button into a spinner reading *Starting the
+editor* and the **Open editor** link appears only once code-server answers, never while the
+container is still booting. While a workspace is in transition the page polls every two seconds
+(twenty seconds otherwise) and also receives pushed updates on the owner's private pub/sub topic, so
+the label changes on its own without a reload; the project page's **Editor** button follows the same
+readiness rule. The phase, its label and `editor_ready` are part of the workspace JSON.
 
 The editor is `code-server`, rebranded as DevPlace end to end: the application name, the browser tab
 icon and PWA icons, the login page styling, and `product.json` all carry DevPlace, and a bundled
@@ -466,10 +557,21 @@ coding agent baked into the image, and a plain login shell beside it with the Py
 Swift toolchains on `PATH`. Both are configurable, and `bash` stays the default profile for
 terminals the member opens later.
 
-The workspace opens straight onto the member's files rather than a welcome page, and the editor's
-own built-in chat assistant is suppressed so `dpc` is the only agent on offer and every token it
-spends is ledgered against the member's DevPlace account. `dpc`'s own working files (`.dpc/`,
-`dpc.log`) are in `SYNC_SKIP_NAMES`, so running an agent on every boot never pollutes the project.
+**The first boot of a workspace opens the DevPlace welcome page** beside the terminals: a webview
+introducing the workspace and DevPlace Code (its 900k token context, vision, parallel sub-agents,
+deep research, safety gates and the daily credits it runs on), with example prompts and buttons that
+focus the agent terminal, start the walkthrough, show the public tunnels and open the workspace
+settings. It is shown once per workspace and can be reopened with **DevPlace: Show the welcome
+page**. The editor's own built-in chat assistant and VS Code's own welcome page stay suppressed so
+`dpc` is the only agent on offer and every token it spends is ledgered against the member's DevPlace
+account. `dpc`'s own working files (`.dpc/`, `dpc.log`) are in `SYNC_SKIP_NAMES`, so running an
+agent on every boot never pollutes the project.
+
+**The terminal panel gets about a third of the window by default.** Every preset is a fixed number
+of steps up from the panel's minimum height (`normal`, the default, lands at roughly a third of a
+typical window; `short` at a fifth, `tall` at about half) and `maximized` fills the editor area.
+A preset is applied on the first boot of a workspace and again whenever it changes; a height the
+member drags themselves is kept across restarts.
 
 **Every workspace is trusted.** VS Code Restricted Mode is disabled at the command line and in the
 seeded settings, so nothing prompts and automatic tasks run. This is a deliberate default with a
@@ -500,15 +602,15 @@ restart.
 
 `ForkService` copies a project into a new project owned by the forking user. The **Fork** button on the project page (any signed-in user) prompts for a name; the job creates the destination project, duplicates the entire virtual filesystem, and records a directional `project_forks` relation so each fork shows a "Forked from X" link. The frontend `app.projectForker` enqueues, polls `/forks/{uid}`, and redirects to the new project once it is done; on failure the partially created project is rolled back. The forked project is permanent, so retention removes only the job tracking row. CLI: `devplace forks prune` / `devplace forks clear` (job rows only).
 
-`SeoService` powers the public **Tools -> SEO Diagnostics** auditor. It runs a headless-browser (Playwright) crawl of a single URL or a sitemap (capped pages) in a subprocess and runs a broad battery of checks across eleven categories: crawlability and indexing (status, redirects, HTTPS/HSTS, canonical, robots/meta-robots, sitemap, URL hygiene, mixed content), on-page meta and content (title, description, headings, language, charset, viewport, favicon, content depth), links, structured data and rich results (JSON-LD validity and required properties, microdata/RDFa), social cards (Open Graph, Twitter), Core Web Vitals and performance (LCP, CLS, FCP, TTFB, page weight, requests, DOM size, compression, caching, image optimisation, console errors), mobile and accessibility (responsive layout, tap targets, image alt, form labels), security headers, and AI/LLM-search readiness (server-rendered-vs-JS content parity, `llms.txt`, semantic HTML). It produces a weighted score and grade with per-category subscores and a recommendation for every finding. Progress streams live over `WS /tools/seo/{uid}/ws`; the full report is available at `/tools/seo/{uid}/report` (HTML or JSON). CLI: `devplace seo prune` / `devplace seo clear`. Playwright is a core dependency; `make install` fetches the Chromium browser.
+`SeoService` powers the public **SEO Diagnostics** auditor at `/tools/seo`. It runs a headless-browser (Playwright) crawl of a single URL or a sitemap (capped pages) in a subprocess and runs a broad battery of checks across eleven categories: crawlability and indexing (status, redirects, HTTPS/HSTS, canonical, robots/meta-robots, sitemap, URL hygiene, mixed content), on-page meta and content (title, description, headings, language, charset, viewport, favicon, content depth), links, structured data and rich results (JSON-LD validity and required properties, microdata/RDFa), social cards (Open Graph, Twitter), Core Web Vitals and performance (LCP, CLS, FCP, TTFB, page weight, requests, DOM size, compression, caching, image optimisation, console errors), mobile and accessibility (responsive layout, tap targets, image alt, form labels), security headers, and AI/LLM-search readiness (server-rendered-vs-JS content parity, `llms.txt`, semantic HTML). It produces a weighted score and grade with per-category subscores and a recommendation for every finding. Progress streams live over `WS /tools/seo/{uid}/ws`; the full report is available at `/tools/seo/{uid}/report` (HTML or JSON). CLI: `devplace seo prune` / `devplace seo clear`. Playwright is a core dependency; `make install` fetches the Chromium browser.
 
 `SeoMetaService` is a separate AI subservice that generates a clean, search-optimised title, description and short keyword list for every published post, project, gist, news article and issue, entirely off the request path so it never slows the web server. The work is queued whenever content is created, edited or published; until the AI value is ready a plain-content default (built from the markdown-stripped text and clamped to safe lengths) fills the fields, so a page's metadata is **always populated, never empty**. The service uses the built-in internal AI gateway and meters its own AI cost and statistics in a dedicated usage table, surfaced together with its live task pipeline on the **Admin -> Services** page. This release also fixes the on-page metadata: the `<meta name="description">` is now stripped of markdown markup (it previously leaked `#`, `**` and `[](...)` from the raw body), a `<meta name="keywords">` tag is emitted (a short honest list, not stuffed), and social-card image dimensions and alt text are added. CLI: `devplace seo-meta prune` / `devplace seo-meta clear` (job rows only; the generated metadata persists).
 
-`DeepsearchService` powers the public **Tools -> DeepSearch** researcher. Given a single research question it plans a set of diverse web search queries, interleaves their results so every angle contributes sources, and crawls the most relevant pages in a subprocess (concurrent fetches, plain HTTP first with a headless-browser fallback for JavaScript-heavy pages, PDF documents streamed and text-extracted, every URL SSRF-guarded). Social sites that block bots (X, YouTube, Reddit and similar) contribute the readable text supplied by the search engine, so their content is not lost to a login wall. A readability-grade extractor isolates the main article content of each page (navigation, cookie banners and footers are discarded), and the configured depth follows the most relevant in-page links for deeper coverage. Content is de-duplicated and indexed into a per-session ChromaDB vector collection (embeddings via the AI gateway with a local fallback). A chain of agents (report writer, findings extractor, linker) then synthesises a thorough cited markdown report grounded on the passages retrieved from that index, with key findings, a confidence score and source diversity; if synthesis fails the report is clearly marked as degraded instead of silently shipping raw source material. Progress streams live over `WS /tools/deepsearch/{uid}/ws`; the report is at `/tools/deepsearch/{uid}/session` (HTML or JSON) and can be exported as Markdown, JSON or PDF. A grounded chat over the session's evidence runs at `WS /tools/deepsearch/{uid}/chat` using hybrid retrieval (vector + keyword/BM25). Runs can be paused, resumed or cancelled. A cross-session URL cache avoids re-fetching pages seen by earlier runs. CLI: `devplace deepsearch prune` / `devplace deepsearch clear`. ChromaDB, weasyprint, pypdf and Playwright are core dependencies.
+`DeepsearchService` powers the public **DeepSearch** researcher at `/tools/deepsearch`, an iterative agent modelled on the way Perplexity and OpenAI's own deep-research agents work: search, read, decide what is still missing, search again. Given a single research question it plans a set of diverse web search queries, interleaves their results so every angle contributes sources, and crawls the most relevant pages in a subprocess (concurrent fetches, plain HTTP first with a headless-browser fallback for JavaScript-heavy pages, PDF documents streamed and text-extracted, every URL SSRF-guarded). After the first pass it runs up to two automatic gap-filling refinement rounds - a lightweight planner looks at what has been gathered so far and proposes a few more targeted queries only if real coverage gaps remain, bounded by the page budget, and stops the moment nothing more is needed. Social sites that block bots (X, YouTube, Reddit and similar) contribute the readable text supplied by the search engine, so their content is not lost to a login wall. A readability-grade extractor isolates the main article content of each page (navigation, cookie banners and footers are discarded), and the configured depth follows the most relevant in-page links for deeper coverage. Content is de-duplicated and indexed into a per-session ChromaDB vector collection (embeddings via the AI gateway with a local fallback); retrieval reranks candidate passages with Maximal Marginal Relevance so the report is grounded on genuinely diverse evidence rather than several near-duplicate hits on the same page. A chain of agents (report writer, findings extractor, linker) then synthesises a thorough cited markdown report grounded on the passages retrieved from that index, with key findings, a confidence score, source diversity and a short list of suggested follow-up questions you can ask straight into the chat; if synthesis fails the report is clearly marked as degraded instead of silently shipping raw source material. Progress streams live over `WS /tools/deepsearch/{uid}/ws`; the report is at `/tools/deepsearch/{uid}/session` (HTML or JSON) and can be exported as Markdown, JSON or PDF. A grounded chat over the session's evidence runs at `WS /tools/deepsearch/{uid}/chat` using hybrid retrieval (vector + keyword/BM25). Runs can be paused, resumed or cancelled. Every run you have started is listed at `GET /tools/deepsearch/history`, newest first with its query, status and score, so you can reopen a finished report and pick the chat back up later. A cross-session URL cache avoids re-fetching pages seen by earlier runs. CLI: `devplace deepsearch prune` / `devplace deepsearch clear`. ChromaDB, weasyprint, pypdf and Playwright are core dependencies.
 
-`IsslopService` powers the public **Tools -> AI Usage Analyzer**, which classifies a git repository or website as AI slop, sophisticated AI-assisted work or genuine human work. Sources are acquired in an isolated subprocess (git URLs are probed with `git ls-remote` and shallow-cloned with a 3 GB guard; websites render in a stealth headless browser with an HTTP fallback, bounded by depth, file and byte caps), inventoried with exclusion rules, and scored by a multi-signal static engine (twenty-one detector families, 126 signal types across an origin axis and a quality-deficit axis). For a live website, the analyzer also opens the home page in a headless browser and inspects what actually renders: it fingerprints AI website-builder platforms directly (Lovable, Bolt.new, Framer and others) and checks the page's real computed styles, layout and build artifacts, not just its file contents. Representative files receive an AI review pass and images an AI-generation review through the internal gateway (model `molodetz`, internal key); the static engine stays authoritative when the gateway is unreachable. Every pipeline step is persisted as an ordered event trail and streamed live over the pub/sub topic `public.isslop.{uid}`. The verdict is an A-F authenticity grade with a human/AI split and one of five categories (`ai-slop`, `sophisticated-ai`, `human-clean`, `human-messy`, `uncertain`), published as a persistent report with an embeddable SVG badge. Members keep their analysis history on their account; guest history is session-bound and claimed by the account on the first signed-in visit. Admin settings (private-host allowance, AI/image review toggles, image cap, retention, concurrency) live on `/admin/services`. CLI: `devplace isslop analyze <url>` / `devplace isslop prune` / `devplace isslop clear`. Playwright plus playwright-stealth back the website crawler.
+`IsslopService` powers the public **AI Usage Analyzer** at `/tools/isslop`, which classifies a git repository or website as AI slop, sophisticated AI-assisted work or genuine human work. Sources are acquired in an isolated subprocess (git URLs are probed with `git ls-remote` and shallow-cloned with a 3 GB guard; websites render in a stealth headless browser with an HTTP fallback, bounded by depth, file and byte caps), inventoried with exclusion rules, and scored by a multi-signal static engine (twenty-one detector families, 126 signal types across an origin axis and a quality-deficit axis). For a live website, the analyzer also opens the home page in a headless browser and inspects what actually renders: it fingerprints AI website-builder platforms directly (Lovable, Bolt.new, Framer and others) and checks the page's real computed styles, layout and build artifacts, not just its file contents. Representative files receive an AI review pass and images an AI-generation review through the internal gateway (model `molodetz`, internal key); the static engine stays authoritative when the gateway is unreachable. Every pipeline step is persisted as an ordered event trail and streamed live over the pub/sub topic `public.isslop.{uid}`. The verdict is an A-F authenticity grade with a human/AI split and one of five categories (`ai-slop`, `sophisticated-ai`, `human-clean`, `human-messy`, `uncertain`), published as a persistent report with an embeddable SVG badge. Members keep their analysis history on their account; guest history is session-bound and claimed by the account on the first signed-in visit. Admin settings (private-host allowance, AI/image review toggles, image cap, retention, concurrency) live on `/admin/services`. CLI: `devplace isslop analyze <url>` / `devplace isslop prune` / `devplace isslop clear`. Playwright plus playwright-stealth back the website crawler.
 
-`BackupService` powers the admin **Admin -> Backups** dashboard, an enterprise-grade backup system that runs entirely as asynchronous jobs so it never impacts the running server. An administrator can back up one of four targets: the **database** (a consistent SQLite snapshot of the main database and the Devii task/lesson databases, taken with SQLite's online backup API so it is consistent under WAL), **uploads** (every attachment and project file), **keys and config** (VAPID keys), or the **full data directory** (database snapshot, uploads, and keys in one archive, excluding regenerable staging, locks, caches, and container workspaces). Each backup is compressed to a `tar.gz` in a stdlib subprocess off the request path and recorded with its size, file count, and a SHA-256 checksum. Archives live under `data/backups/` (sharded on the random uuid tail) and are served only through `/admin/backups/{uid}/download`, which is restricted to the **primary administrator** - the first user created with the Admin role. Every other administrator receives a 403 from the endpoint and sees the Download button disabled with the tooltip `Not available`; creating, running, deleting, and scheduling backups remain available to all administrators. The dashboard reports detailed storage usage - the size and file count of every major data area, the total data-directory footprint, the total size and count of stored backups, and disk usage (total, used, free, percent), computed in a worker thread and cached briefly so the page never blocks. Backups can be **scheduled** (CRUD) on an interval or 5-field cron expression with a `keep_last` rotation count that prunes older backups of the same schedule; the service evaluates schedules only on the lock-owning worker so each fires exactly once. Backup archives are permanent operational artifacts: job retention only removes the tracking row, never the archive, which is deleted only by an administrator, by schedule rotation, or via the CLI. CLI: `devplace backups list` / `devplace backups run <target>` / `devplace backups prune` / `devplace backups clear`. Devii tools: `backups_overview`, `backup_run`, `backup_status`, `backup_delete`, `backup_schedule_create`, `backup_schedule_delete` (all admin-only). The service creates and stores backups but does not restore them into a live server; restore is a documented manual procedure (stop the server, unpack the archive over the data directory, verify the checksum, restart).
+`BackupService` powers the admin **Admin -> Backups** dashboard, an enterprise-grade backup system that runs entirely as asynchronous jobs so it never impacts the running server. An administrator can back up one of four targets: the **database** (a consistent SQLite snapshot of the main database and the Devii task/lesson databases, taken with SQLite's online backup API so it is consistent under WAL), **uploads** (every attachment and project file), **keys and config** (VAPID keys), or the **full data directory** (database snapshot, uploads, and keys in one archive, excluding regenerable staging, locks, caches, and container workspaces). Each backup is compressed to a `tar.gz` in a stdlib subprocess off the request path and recorded with its size, file count, and a SHA-256 checksum. Archives live under `data/backups/` (sharded on the random uuid tail) and are served only through `/admin/backups/{uid}/download`, which is restricted to the **primary administrator** - the first user created with the Admin role. Every other administrator receives a 403 from the endpoint and sees the Download button disabled with the tooltip `Not available`; creating, running, deleting, and scheduling backups remain available to all administrators. The dashboard reports detailed storage usage - the size and file count of every major data area, the total data-directory footprint, the total size and count of stored backups, and disk usage (total, used, free, percent). Disk percent is an O(1) volume stat used by the backup service tick; per-directory file counts are a single walk of the data directory, run in a worker thread and cached for minutes, so neither the request path nor the event loop ever walks the tree. Backups can be **scheduled** (CRUD) on an interval or 5-field cron expression with a `keep_last` rotation count that prunes older backups of the same schedule; the service evaluates schedules only on the lock-owning worker so each fires exactly once. Backup archives are permanent operational artifacts: job retention only removes the tracking row, never the archive, which is deleted only by an administrator, by schedule rotation, or via the CLI. CLI: `devplace backups list` / `devplace backups run <target>` / `devplace backups prune` / `devplace backups clear`. Devii tools: `backups_overview`, `backup_run`, `backup_status`, `backup_delete`, `backup_schedule_create`, `backup_schedule_delete` (all admin-only). The service creates and stores backups but does not restore them into a live server; restore is a documented manual procedure (stop the server, unpack the archive over the data directory, verify the checksum, restart).
 
 ### Adding a service
 
@@ -537,9 +639,10 @@ Configuration on the Services tab (`/admin/services`):
 |-----------|---------|---------|
 | `news_grade_threshold` | `7` | Minimum AI grade for auto-publish |
 | `news_api_url` | `https://news.app.molodetz.nl/api` | News source |
-| `news_ai_url` | `http://localhost:10500/openai/v1/chat/completions` | AI grading endpoint (the internal gateway) |
-| `news_ai_model` | `molodetz` | Generic model name; the gateway maps it to the real model |
-| `news_ai_key` | internal key | AI API key (`NEWS_AI_KEY` env, then the auto-generated gateway internal key) |
+| `news_ai_url` | `https://aquality.cloud.pravda.education/v1/chat/completions` | AI grading endpoint - a free, local, deterministic quality model by default |
+| `news_ai_model` | `aquality` | Generic model name; echoed back, not used for routing |
+| `news_ai_key` | internal key | AI API key (`NEWS_AI_KEY` env, then the auto-generated gateway internal key); unused by the default aquality endpoint |
+| `news_format_enabled` | `false` | Reformat article bodies into Markdown with AI after grading; off by default since aquality only scores and cannot generate text - enable when `news_ai_url` points at a generative chat model |
 | `news_service_interval` | `3600` | Seconds between fetch cycles (min 60) |
 
 News articles have detail pages at `/news/{slug}` with full comment support (same component as posts/projects). The landing page can display curated articles toggled from admin.
@@ -636,6 +739,7 @@ Configuration on the Services tab:
 | `gateway_upstream_url` | `https://api.deepseek.com/chat/completions` | Where requests are forwarded |
 | `gateway_model` | `deepseek-v4-flash` | Real model sent upstream (what `molodetz` maps to); 1M-token context, 384K max output |
 | `gateway_force_model` | on | Override the client-requested model (and the `molodetz` alias) |
+| `gateway_thinking` | off | When off, chat completions disable model thinking unless the client explicitly enables it (`think` / `thinking` / `reasoning`). Fastest default. |
 | `gateway_api_key` | migrated from env | Upstream key, shown and editable (auto-migrated from `DEEPSEEK_API_KEY`/`OPENROUTER_API_KEY`) |
 | `gateway_instances` | `4` | Max concurrent upstream forwards per worker (pool + semaphore) |
 | `gateway_timeout` | `300` | Upstream timeout (seconds); minimum five minutes; also bounds the vision describe-image call |
@@ -645,6 +749,8 @@ Configuration on the Services tab:
 | `gateway_allow_admins` / `gateway_allow_users` | on / off | Which DevPlace users may call it (any auth scheme) |
 | `gateway_access_key` | empty | A standalone key (sent as `X-API-KEY`/Bearer) that always grants access |
 | `gateway_internal_key` | auto (uuid4) | Auto-generated on boot; DevPlace's own services authenticate with this. Clear and restart to rotate |
+| `gateway_auth_throttle_enabled` | on | Track failed authentication attempts per IP and block further unauthenticated attempts once an IP crosses the failure threshold. Never blocks a request presenting valid credentials |
+| `gateway_auth_throttle_max_failures` / `_window_seconds` | 10 / 60 | Failed-auth attempts allowed per IP within the sliding window before further unauthenticated attempts get `429` |
 | `gateway_price_cache_hit_per_m` / `_cache_miss_per_m` / `_output_per_m` | 0.0028 / 0.14 / 0.28 | Chat cost per 1M tokens, used when the upstream returns no native cost (DeepSeek) |
 | `gateway_vision_price_input_per_m` / `_output_per_m` | 0 / 0 | Vision cost per 1M tokens, used only when the vision upstream returns no native cost |
 | `gateway_embed_price_input_per_m` | 0.01 | Embeddings cost per 1M input tokens, used only when the embeddings upstream returns no native cost |
@@ -899,13 +1005,25 @@ receives a notification through every provider they hold a live subscription for
 | Provider | Registration | Transport |
 |----------|--------------|-----------|
 | `webpush` | `PushSubscription` from the browser `PushManager` (endpoint + `p256dh`/`auth` keys) | Web Push Protocol, VAPID signed, `aesgcm` encrypted payload |
-| `apns` | Hexadecimal device token | `POST https://api.push.apple.com/3/device/{token}` over HTTP/2, ES256 provider token |
+| `apns` | Hexadecimal device token, optional stable `client_id` | Dedicated HTTP/2 client to `api.push.apple.com` or `api.sandbox.push.apple.com` (`POST /3/device/{token}`), ES256 provider token. Not the stealth/Chrome client. |
 
 `POST /push.json` accepts a registration for any active provider; a body without a
-`provider` field is a `webpush` body, so browsers need no change. `GET /push.json` returns
-the VAPID public key plus the providers currently accepting registrations. A provider that
-is disabled or not fully configured accepts no registrations and is skipped during
-delivery, so an unconfigured provider is inert rather than an error.
+`provider` field is a `webpush` body, so browsers need no change. An APNs body is
+`{"provider": "apns", "token": "...", "client_id": "..."}` - `client_id` is optional and
+identifies the device across Apple token rotations, so a new token updates that row
+instead of inserting another. A token-only body still works and revives a previously
+dead token. `GET /push.json` returns the VAPID public key plus the providers currently
+accepting registrations; when `apns` is active it includes `environment` (`production` or
+`sandbox`). A provider that is disabled or not fully configured accepts no registrations
+and is skipped during delivery, so an unconfigured provider is inert rather than an error.
+
+`DELETE /push.json` unregisters exactly one registration, identified the same way it was
+created (`endpoint` for webpush, `token` or `client_id` for apns). It is idempotent - an
+unknown or already-removed identity still returns `200 {"unregistered": false}`. The web
+frontend calls it automatically before navigating to `/auth/logout` (`PushManager.js`), so a
+browser subscription stops receiving notifications the moment the user signs out; a native
+app integrating `apns` must call it itself at logout, since the server has no way to detect
+a native client closing on its own.
 
 Every provider setting is edited at **`/admin/services/push`**: per provider an `Enabled`
 toggle, the VAPID subject for `webpush`, and team id, key id, `.p8` auth key (stored as a
@@ -931,6 +1049,7 @@ in real time, bridged onto the in-process pub/sub bus by a lock-owner relay:
 | Direct message received | receiver |
 | Comment on your post | post author |
 | Reply to your comment | comment author |
+| Any comment on a post you've also commented on | every other commenter on that post |
 | `@mention` in any content | mentioned user |
 | Upvote on your content | content owner |
 | New follower | followed user |
@@ -940,13 +1059,15 @@ in real time, bridged onto the in-process pub/sub bus by a lock-owner relay:
 `create_notification` schedules delivery as a fire-and-forget async task, so a dead
 subscription or push-service error never blocks the triggering request. Delivery
 (`push.notify_user`) reads a user's subscriptions once, groups them by provider, builds
-each provider's payload once, and sends over a single shared HTTP client. A subscription
-the push service reports as gone (`404`/`410` for Web Push, `410` or an `Unregistered`
-class reason for APNs) is soft-deleted; any other failure is logged and the subscription is
-kept.
+each provider's payload once, and sends over that provider's own HTTP client (stealth for
+Web Push, a dedicated HTTP/2 client for APNs). A subscription the push service reports as
+gone (`404`/`410` for Web Push, `410` or an `Unregistered` class reason for APNs) is
+soft-deleted and the provider reason is logged; any other failure is logged and the
+subscription is kept. APNs environment is stored per registration so a sandbox debug
+token and a production token can coexist.
 
 A notification is also **marked read automatically when you open the page that shows its
-content** - viewing a post clears its comment, reply, upvote and mention notifications;
+content** - viewing a post clears its comment, reply, thread, upvote and mention notifications;
 opening a conversation clears its direct-message notifications; visiting a profile clears
 the matching follow, badge and level notifications; and the issue, reminder and farm-raid
 notifications clear on their respective pages. You no longer have to dismiss each one by
@@ -1002,7 +1123,7 @@ offline. Installation requires a secure origin (HTTPS, or `localhost` for develo
 |------|------|
 | `devplacepy/push/providers/` | Provider protocol, Web Push (VAPID keys, payload encryption), APNs |
 | `devplacepy/push/store.py` | `push_registration` reads and writes |
-| `devplacepy/push/delivery.py` | `notify_user` - group by provider, deliver, reap dead subscriptions |
+| `devplacepy/push/delivery.py` | `notify_user` / `notify_registration` - group by provider, per-provider client, reap dead subscriptions |
 | `devplacepy/services/push/service.py` | Provider configuration at `/admin/services/push`, retention sweep, metrics |
 | `devplacepy/routers/push.py` | `/push.json`, `/service-worker.js`, `/manifest.json` |
 | `static/js/PushManager.js` | Service-worker registration + subscribe + opt-in UI |
@@ -1099,6 +1220,17 @@ Open `http://<host>:${PORT}` (default 10500). `make docker-logs` tails output; `
 
 `make docker-build` and `make docker-up` always apply the `docker-compose.containers.yml` overlay, so the admin-only Container Manager works with no extra setup. Mounting the docker socket grants the app **root on the host**; the feature itself stays disabled until an admin enables **Containers** on `/admin/services` (and the `ppy` image is built once with `make ppy`), but treat the whole deployment as trusted-admins-only. Always update through the make targets - a plain `docker compose up -d` drops the overlay and silently removes the socket and CLI.
 
+### Podman (rootless) hosts
+
+Hosts without a docker daemon run the same stack via `podman-compose` plus the `docker-compose.podman.yml` overlay - see [PODMAN.md](PODMAN.md) for the full notes. Short version:
+
+```bash
+make podman-build           # build the images
+make podman-up              # start (warns if user linger is off)
+```
+
+The base compose file runs the app as `user: 1000:1000`, which is wrong under rootless podman (container uid 1000 maps to a subordinate host uid that cannot write the bind-mounted checkout, so the app crashes with `PermissionError` and nginx never starts). The overlay runs the app as container root instead, which maps back to the invoking host user and stays unprivileged on the host. Always use the `make podman-*` targets - a bare `podman-compose up -d` drops the overlay and recreates the crashing container. Never mix `podman-compose` with native `podman compose` in this directory (different container names, they fight over port 10500). Reboot survival needs both `restart: unless-stopped` (already set) and systemd user linger (`sudo loginctl enable-linger <user>`); `make podman-up` warns when linger is off.
+
 ### Updating
 
 ```bash
@@ -1149,13 +1281,13 @@ reverse_proxy localhost:10500 {
 
 ### Static asset caching
 
-Static assets (CSS, JS, vendored libraries) are served with a **one-year immutable cache** for the best Lighthouse "efficient cache policy" score, while deploys still take effect immediately. Every app-owned static URL carries a boot-time version path segment, `/static/v<timestamp>/...`, where `<timestamp>` is the unix time the server process started (`config.STATIC_VERSION`). A restart changes the segment, so every asset URL changes and returning browsers refetch on their next page load - no cache purge, no hashing build step.
+Static assets (CSS, JS, vendored libraries) are served with a **one-year immutable cache** for the best Lighthouse "efficient cache policy" score, while deploys still take effect immediately. Every app-owned static URL carries a version path segment, `/static/v<app-version>-<boot-id>/...` (`config.STATIC_VERSION`) - `<app-version>` is `pyproject.toml`'s `version` (auto-bumped on every commit, see "Version bumping" in `CLAUDE.md`) and `<boot-id>` is the unix time the server process started. A restart changes the segment, so every asset URL changes and returning browsers refetch on their next page load - no cache purge, no hashing build step.
 
 The version sits in the **path**, not a query string, because the frontend is unbundled ES6 modules wired with relative imports: a path segment is inherited automatically by every transitively imported module and relative CSS `url()`, so the whole graph busts on deploy. Templates emit URLs through the `static_url` Jinja global and runtime JavaScript through the `assetUrl` helper (`static/js/assetVersion.js`, reading `<meta name="asset-version">`). User uploads under `/static/uploads/` and the `service-worker.js` route are excluded. Set `DEVPLACE_STATIC_VERSION` at launch so multiple workers share one value (the `prod` target and Docker image do this). Full detail: `/docs/static-caching.html`.
 
 ### Bare-metal alternative
 
-`make prod` runs the same app without containers (`uvicorn ... --workers $(WEB_WORKERS) --proxy-headers`, where `WEB_WORKERS` defaults to `nproc`) from the project root, sharing the identical database and files. Note it binds port 10500, so it conflicts with the Docker front door on the same port - run one, or set a different `PORT`.
+`make prod` runs the same app without containers (`uvicorn ... --workers $(WEB_WORKERS) --proxy-headers`, where `WEB_WORKERS` defaults to `nproc`) from the project root, sharing the identical database and files. It binds port 10500 by default, so it conflicts with the Docker front door on the same port - run one, or set `DEVPLACE_PORT=<other-port> make prod` (also honoured by `make dev`).
 
 ### Multi-worker safety
 

devplacepy/attachments.py

@@ -3,6 +3,7 @@
 import asyncio
 import ipaddress
 import logging
+import os
 import socket
 from datetime import datetime, timezone
 from pathlib import Path
@@ -10,7 +11,7 @@ from urllib.parse import urlparse
 from PIL import Image
 from io import BytesIO
 import httpx
-from devplacepy import stealth
+from devplacepy.net_guard import BlockedAddressError, guarded_async_client
 from devplacepy.database import get_table, db, get_setting
 from devplacepy.config import UPLOADS_DIR, ATTACHMENTS_DIR
 from devplacepy.utils import generate_uid
@@ -273,14 +274,14 @@ def delete_inline_image(filename):
         logger.warning(f"Failed to delete inline image {filename}: {e}")
 
 
-def store_attachment(file_bytes, original_filename, user_uid):
+def store_attachment(file_bytes, original_filename, user_uid, uid=None):
     if len(file_bytes) > _get_max_upload_bytes():
         return None
     ext = Path(original_filename).suffix.lower()
     if not is_extension_allowed(ext):
         return None
 
-    uid = generate_uid()
+    uid = uid or generate_uid()
     stored_name = f"{uid}{ext}"
     directory = _directory_for(uid)
     file_dir = ATTACHMENTS_DIR / directory
@@ -389,7 +390,7 @@ async def fetch_remote_file(url, filename=None):
     await _guard_public_url(url)
     max_bytes = _get_max_upload_bytes()
     try:
-        async with stealth.stealth_async_client(
+        async with guarded_async_client(
             follow_redirects=True,
             timeout=REMOTE_FETCH_TIMEOUT,
             headers={"User-Agent": REMOTE_FETCH_USER_AGENT},
@@ -412,6 +413,8 @@ async def fetch_remote_file(url, filename=None):
                             413,
                         )
                 data = b"".join(chunks)
+    except BlockedAddressError as exc:
+        raise RemoteFetchError(str(exc), 400) from exc
     except httpx.HTTPError as exc:
         raise RemoteFetchError(f"Could not fetch {url}: {exc}", 400) from exc
 
@@ -453,6 +456,33 @@ def link_attachments(uids, target_type, target_uid):
         )
 
 
+def split_attachment_uids(raw):
+    return [
+        uid.strip() for item in raw or [] for uid in str(item).split(",") if uid.strip()
+    ]
+
+
+def get_orphan_attachments_batch(uids, user, admin=False):
+    if not uids:
+        return []
+    placeholders = ",".join(f":p{i}" for i in range(len(uids)))
+    params = {f"p{i}": uid for i, uid in enumerate(uids)}
+    rows = db.query(
+        f"SELECT * FROM attachments WHERE uid IN ({placeholders}) AND deleted_at IS NULL",
+        **params,
+    )
+    by_uid = {row["uid"]: row for row in rows}
+    owned = []
+    for uid in uids:
+        row = by_uid.get(uid)
+        if not row or row.get("target_uid"):
+            continue
+        if row.get("user_uid") and row["user_uid"] != user["uid"] and not admin:
+            continue
+        owned.append(uid)
+    return owned
+
+
 def set_gitea_asset_id(uid, asset_id):
     get_table("attachments").update(
         {"uid": uid, "gitea_asset_id": int(asset_id)}, ["uid"]
@@ -472,7 +502,7 @@ async def mirror_attachment_to_gitea(uid):
         return None
     path = ATTACHMENTS_DIR / row.get("directory", "") / row.get("stored_name", "")
     try:
-        data = path.read_bytes()
+        data = await asyncio.to_thread(path.read_bytes)
     except OSError as exc:
         logger.warning("Cannot read attachment %s for Gitea mirror: %s", uid, exc)
         return None
@@ -516,6 +546,28 @@ async def remove_gitea_asset(row):
         logger.warning("Gitea asset delete failed for %s: %s", row.get("uid"), exc)
 
 
+_pending_gitea_tasks: set[asyncio.Task] = set()
+
+
+def _fire_and_forget(coro) -> None:
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        coro.close()
+        return
+    task = loop.create_task(coro)
+    _pending_gitea_tasks.add(task)
+    task.add_done_callback(_pending_gitea_tasks.discard)
+
+
+def schedule_gitea_mirror(uid: str) -> None:
+    _fire_and_forget(mirror_attachment_to_gitea(uid))
+
+
+def schedule_gitea_removal(row: dict) -> None:
+    _fire_and_forget(remove_gitea_asset(row))
+
+
 def _unlink_attachment_files(row):
     stored_name = row.get("stored_name", "")
     directory = row.get("directory", "")
@@ -586,6 +638,73 @@ def restore_attachment(uid):
     return True
 
 
+def purge_soft_deleted_attachments(*, dry_run: bool = False) -> tuple[int, int]:
+    if "attachments" not in db.tables:
+        return 0, 0
+    table = get_table("attachments")
+    rows = list(table.find(table.table.columns.deleted_at.isnot(None)))
+    removed = 0
+    freed = 0
+    for row in rows:
+        directory = row.get("directory")
+        stored_name = row.get("stored_name")
+        if directory and stored_name:
+            path = ATTACHMENTS_DIR / directory / stored_name
+            try:
+                freed += path.stat().st_size
+            except OSError:
+                pass
+            for thumb in (ATTACHMENTS_DIR / directory).glob(
+                f"{Path(stored_name).stem}_thumb.*"
+            ):
+                try:
+                    freed += thumb.stat().st_size
+                except OSError:
+                    pass
+            if not dry_run:
+                _unlink_attachment_files(row)
+        if not dry_run:
+            table.delete(id=row["id"])
+        removed += 1
+    return removed, freed
+
+
+def sweep_orphan_attachment_blobs(*, dry_run: bool = False) -> tuple[int, int]:
+    if not ATTACHMENTS_DIR.exists():
+        return 0, 0
+    referenced_stems = set()
+    if "attachments" in db.tables:
+        for row in get_table("attachments").find():
+            directory = row.get("directory")
+            stored_name = row.get("stored_name")
+            if directory and stored_name:
+                referenced_stems.add((directory, Path(stored_name).stem))
+    removed = 0
+    freed = 0
+    base = str(ATTACHMENTS_DIR)
+    for root, _dirs, files in os.walk(base):
+        directory = os.path.relpath(root, base)
+        for name in files:
+            stem = Path(name).stem
+            if stem.endswith("_thumb"):
+                stem = stem[: -len("_thumb")]
+            if (directory, stem) in referenced_stems:
+                continue
+            file_path = os.path.join(root, name)
+            try:
+                size = os.path.getsize(file_path)
+            except OSError:
+                continue
+            if not dry_run:
+                try:
+                    os.unlink(file_path)
+                except OSError:
+                    continue
+            removed += 1
+            freed += size
+    return removed, freed
+
+
 def soft_delete_target_attachments(target_type, target_uid, deleted_by):
     stamp = datetime.now(timezone.utc).isoformat()
     for row in get_table("attachments").find(

devplacepy/cli/__init__.py

@@ -45,6 +45,7 @@ from devplacepy.cli.containers import (
 )
 from devplacepy.cli.quiz import cmd_quiz_prune
 from devplacepy.cli.migrate import cmd_emoji_sync, cmd_migrate_data
+from devplacepy.cli.system import cmd_system_prune
 
 __all__ = [
     "main",
@@ -91,4 +92,5 @@ __all__ = [
     "cmd_quiz_prune",
     "cmd_emoji_sync",
     "cmd_migrate_data",
+    "cmd_system_prune",
 ]

devplacepy/cli/community.py

@@ -0,0 +1,160 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+import os
+import sys
+
+from devplacepy.cli._shared import _audit_cli
+
+
+def _prepare():
+    from devplacepy.database import init_db
+
+    init_db()
+
+
+def _credentials(args) -> dict:
+    return {
+        "username": args.login_user or "",
+        "email": args.login_email or "",
+        "password": args.login_password or os.environ.get("COMMUNITY_PASSWORD", ""),
+        "totp": args.login_totp or os.environ.get("COMMUNITY_TOTP_SECRET", ""),
+    }
+
+
+def _print_report(report):
+    mode = "DRY RUN" if report.dry_run else "LIVE"
+    print(f"\n[{mode}] status: {report.status}")
+    print(f"collected: {report.collected}")
+    for kind, counts in report.pushed.items():
+        print(f"  {kind}: {counts or 'nothing'}")
+    for error in report.errors[:10]:
+        print(f"  problem: {error}")
+
+
+def cmd_community_discover(args):
+    from devplacepy.services.community.logbook import LogBook
+    from devplacepy.services.community.runner import TRIGGER_CLI, CommunityRunner
+    from devplacepy.services.community.spec import STATUS_READY
+    from devplacepy.services.community.store import store
+
+    from devplacepy.services.community.discoverer import slug_for_query
+
+    _prepare()
+    runner = CommunityRunner()
+    existing = store.get(slug_for_query(args.query))
+    if existing and not args.relearn and not args.dry_run:
+        print(f"Community '{existing['slug']}' already exists ({existing['status']}); pass --relearn to learn it again")
+        if args.run and existing["status"] == STATUS_READY:
+            with LogBook(echo=True):
+                _print_report(asyncio.run(runner.run(existing, dry_run=False, trigger=TRIGGER_CLI)))
+        return
+    with LogBook(echo=True):
+        result, row, _ = asyncio.run(
+            runner.discover(
+                args.query,
+                "cli",
+                dry_run=args.dry_run,
+                relearn_slug=existing["slug"] if existing else None,
+                credentials=_credentials(args),
+            )
+        )
+    print(f"\n{result.slug or '-'}: {result.status}" + (f" ({result.error})" if result.error else ""))
+    if result.status == STATUS_READY:
+        for feed in result.schema["feeds"]:
+            print(f"  feed {feed['id']}: {feed['type']} {feed['url']} comments={feed['comments']['mode']} pagination={feed['pagination']['style']}")
+        print(f"  sample: { {kind: len(rows) for kind, rows in result.sample['records'].items()} }")
+    if args.dry_run:
+        print("dry run: nothing was saved")
+    else:
+        _audit_cli("cli.community.discover", f"CLI discovered community {result.slug}: {result.status}", target_label=result.slug, metadata={"status": result.status})
+    if result.status != STATUS_READY and not args.dry_run and result.status != "login_required":
+        sys.exit(1)
+    if args.run and row and result.status == STATUS_READY and not args.dry_run:
+        with LogBook(echo=True):
+            _print_report(asyncio.run(runner.run(store.get(row["uid"]), dry_run=False, trigger=TRIGGER_CLI)))
+
+
+def cmd_community_run(args):
+    from devplacepy.services.community.logbook import LogBook
+    from devplacepy.services.community.runner import TRIGGER_CLI, CommunityRunner
+    from devplacepy.services.community.store import store
+
+    _prepare()
+    row = store.get(args.slug)
+    if row is None:
+        print(f"Unknown community '{args.slug}'")
+        sys.exit(1)
+    with LogBook(echo=True):
+        report = asyncio.run(CommunityRunner().run(row, dry_run=args.dry_run, trigger=TRIGGER_CLI))
+    _print_report(report)
+    if not args.dry_run:
+        _audit_cli("cli.community.run", f"CLI ran community {row['slug']}: {report.status}", target_label=row["slug"], metadata={"status": report.status, "collected": report.collected})
+    if report.status == "failed":
+        sys.exit(1)
+
+
+def cmd_community_login(args):
+    from devplacepy.services.community.store import store
+
+    _prepare()
+    row = store.get(args.slug)
+    if row is None:
+        print(f"Unknown community '{args.slug}'")
+        sys.exit(1)
+    if args.clear:
+        store.update(row["uid"], login_username="", login_email="", login_password="", login_totp_secret="")
+        print(f"Removed the stored account of {row['slug']}")
+    else:
+        secrets = _credentials(args)
+        if not secrets["password"] or not (secrets["username"] or secrets["email"]):
+            print("Give --login-user or --login-email, and --login-password (or COMMUNITY_PASSWORD)")
+            sys.exit(1)
+        store.update(row["uid"], login_username=secrets["username"], login_email=secrets["email"], login_password=secrets["password"], login_totp_secret=secrets["totp"])
+        print(f"Stored the account for {row['slug']}; run 'community discover \"{row['query']}\" --relearn' to learn behind the login")
+    _audit_cli("cli.community.login", f"CLI changed the stored account of community {row['slug']}", target_label=row["slug"])
+
+
+def cmd_community_list(args):
+    from devplacepy.services.community.store import store
+
+    _prepare()
+    rows = store.list()
+    if not rows:
+        print("No communities")
+        return
+    for row in rows:
+        print(
+            f"{row['slug']:<28} {row['status']:<15} runs={row.get('run_count') or 0:<4} "
+            f"last={row.get('last_run_at') or '-'} next={row.get('next_run_at') or '-'}"
+        )
+
+
+def add_login_arguments(parser):
+    parser.add_argument("--login-user", help="username of an account on the community")
+    parser.add_argument("--login-email", help="email of an account on the community")
+    parser.add_argument("--login-password", help="password (or set COMMUNITY_PASSWORD)")
+    parser.add_argument("--login-totp", help="authenticator secret of the account, for a login with two-factor codes (or set COMMUNITY_TOTP_SECRET)")
+
+
+def register_community(subparsers):
+    community = subparsers.add_parser("community", help="External community discovery and sync")
+    sub = community.add_subparsers(title="action", dest="action")
+    discover = sub.add_parser("discover", help="Discover, learn and generate collectors for a community")
+    discover.add_argument("query", help="community name, address or search phrase")
+    discover.add_argument("--run", action="store_true", help="also run the collectors once, live")
+    discover.add_argument("--relearn", action="store_true", help="learn again even if the community exists")
+    discover.add_argument("--dry-run", action="store_true", help="learn and report, save nothing")
+    add_login_arguments(discover)
+    discover.set_defaults(func=cmd_community_discover)
+    run = sub.add_parser("run", help="Run the generated collectors of a community")
+    run.add_argument("slug")
+    run.add_argument("--dry-run", action="store_true", help="collect and validate, write nothing, keep the watermark")
+    run.set_defaults(func=cmd_community_run)
+    login = sub.add_parser("login", help="Store or remove the account used to log in to a community")
+    login.add_argument("slug")
+    login.add_argument("--clear", action="store_true", help="remove the stored account")
+    add_login_arguments(login)
+    login.set_defaults(func=cmd_community_login)
+    listing = sub.add_parser("list", help="List communities")
+    listing.set_defaults(func=cmd_community_list)

devplacepy/cli/containers.py

@@ -66,23 +66,27 @@ def cmd_containers_prune_builds(args):
     )
 
 
+def _human_bytes(n):
+    n = float(n)
+    for unit in ("B", "KB", "MB", "GB", "TB"):
+        if n < 1024:
+            return f"{n:.1f}{unit}"
+        n /= 1024
+    return f"{n:.1f}PB"
+
+
 def cmd_containers_gc_workspaces(args):
-    import shutil
-    from pathlib import Path
-    from devplacepy import config
     from devplacepy.services.containers import store
 
-    active = {inst["project_uid"] for inst in store.all_instances()}
-    base = Path(config.CONTAINER_WORKSPACES_DIR)
-    removed = 0
-    if base.is_dir():
-        for child in base.iterdir():
-            if child.is_dir() and child.name not in active:
-                shutil.rmtree(child, ignore_errors=True)
-                removed += 1
-    _audit_cli("cli.containers.gc_workspaces", f"CLI removed {removed} unused workspace dirs", metadata={"count": removed})
+    removed, freed = store.gc_workspaces()
+    _audit_cli(
+        "cli.containers.gc_workspaces",
+        f"CLI removed {removed} unused workspace dirs, freed {_human_bytes(freed)}",
+        metadata={"count": removed, "bytes_freed": freed},
+    )
     print(
         f"Removed {removed} unused workspace director{'y' if removed == 1 else 'ies'}"
+        f" ({_human_bytes(freed)})"
     )
 
 

devplacepy/cli/main.py

@@ -17,6 +17,9 @@ from devplacepy.cli.game import register_game
 from devplacepy.cli.quiz import register_quiz
 from devplacepy.cli.gateway import register_gateway
 from devplacepy.cli.messaging import register_messaging
+from devplacepy.cli.system import register_system
+from devplacepy.cli.community import register_community
+from devplacepy.cli.nodes import register_nodes
 
 
 def build_parser():
@@ -38,6 +41,9 @@ def build_parser():
     register_gateway(sub)
     register_messaging(sub)
     register_accounts(sub)
+    register_system(sub)
+    register_community(sub)
+    register_nodes(sub)
 
     return parser
 

devplacepy/cli/nodes.py

@@ -0,0 +1,164 @@
+# retoor <retoor@molodetz.nl>
+
+import sys
+
+from devplacepy.cli._shared import _audit_cli
+
+LIST_LIMIT = 200
+SUSPEND_DEFAULT_MINUTES = 60
+SUSPEND_MAX_MINUTES = 7 * 24 * 60
+
+
+def _prepare():
+    from devplacepy.database import init_db
+
+    init_db()
+
+
+def cmd_nodes_list(args):
+    from devplacepy.database import get_table
+    from devplacepy.database.nodes import effective_status
+
+    _prepare()
+    rows = list(get_table("node_devices").find(deleted_at=None, order_by=["-created_at"], _limit=LIST_LIMIT))
+    if not rows:
+        print("No devices")
+        return
+    for row in rows:
+        print(
+            f"{row['uid']}  {effective_status(row):<10} {row.get('kind') or '-':<9} "
+            f"{row.get('platform') or '-':<14} user={row['user_uid']} last={row.get('last_seen_at') or '-'}"
+        )
+
+
+def cmd_nodes_stats(args):
+    from devplacepy.database.nodes import list_flags, public_stats
+
+    _prepare()
+    stats = public_stats()
+    print(f"donors: {stats['donors']}  requests: {stats['requests']}  bytes: {stats['bytes']}")
+    print(f"open flags: {len(list_flags())}")
+
+
+def cmd_nodes_flags(args):
+    from devplacepy.database.nodes import list_flags
+
+    _prepare()
+    flags = list_flags()
+    if not flags:
+        print("No open flags")
+        return
+    for flag in flags:
+        print(f"{flag['uid']}  {flag.get('kind') or '-':<22} device={flag.get('device_uid') or '-'} {flag.get('detail') or ''}")
+
+
+def cmd_nodes_suspend(args):
+    from devplacepy.database.nodes import suspend_device
+
+    _prepare()
+    minutes = max(1, min(args.minutes, SUSPEND_MAX_MINUTES))
+    if suspend_device(args.device_uid, args.reason, minutes):
+        print(f"Suspended {args.device_uid} for {minutes} minutes")
+        _audit_cli(
+            "cli.nodes.suspend",
+            f"CLI suspended device {args.device_uid}",
+            target_type="node_device",
+            target_uid=args.device_uid,
+            metadata={"minutes": minutes, "reason": args.reason},
+        )
+        return
+    print(f"No active device {args.device_uid}")
+    sys.exit(1)
+
+
+def cmd_nodes_unsuspend(args):
+    from devplacepy.database.nodes import unsuspend_device
+
+    _prepare()
+    if unsuspend_device(args.device_uid):
+        print(f"Lifted the suspension of {args.device_uid}")
+        _audit_cli(
+            "cli.nodes.unsuspend",
+            f"CLI lifted the suspension of device {args.device_uid}",
+            target_type="node_device",
+            target_uid=args.device_uid,
+        )
+        return
+    print(f"{args.device_uid} is not suspended")
+    sys.exit(1)
+
+
+def cmd_nodes_prune(args):
+    from devplacepy.services.nodes.ledger import UsageLedger
+
+    _prepare()
+    counts = UsageLedger().prune(dry_run=args.dry_run)
+    label = "would remove" if args.dry_run else "removed"
+    for name, count in counts.items():
+        print(f"{label} {count} {name}")
+
+
+def cmd_nodes_clear(args):
+    from devplacepy.database.nodes import clear_ephemeral
+
+    _prepare()
+    counts = clear_ephemeral(dry_run=not args.force)
+    label = "removed" if args.force else "would remove (pass --force)"
+    for name, count in counts.items():
+        print(f"{label} {count} {name}")
+    if args.force:
+        _audit_cli("cli.nodes.clear","CLI cleared the node job queue and event log", metadata=counts)
+
+
+def cmd_nodes_geo_sync(args):
+    import asyncio
+
+    from devplacepy.services.nodes.geo import GeoError, geo_sync
+
+    try:
+        summary = asyncio.run(geo_sync(dry_run=args.dry_run, month=args.month))
+    except GeoError as exc:
+        print(f"geo-sync refused: {exc}")
+        sys.exit(1)
+    verb = "installed" if summary["installed"] else "verified, not installed (dry run)"
+    print(f"{verb}: month {summary['month']}, {summary['asn_rows']} ASN ranges, {summary['country_rows']} country ranges")
+    if summary["installed"]:
+        _audit_cli("cli.nodes.geo_sync", f"CLI installed the node address database for {summary['month']}", metadata={"month": summary["month"]})
+
+
+def cmd_nodes_geo_status(args):
+    from devplacepy.services.nodes.geo import shared_geo
+
+    status = shared_geo().status()
+    if not status["available"]:
+        print(f"not installed at {status['path']}; every donor is classed unknown")
+        return
+    print(f"month {status.get('month')}, {status.get('asn_rows')} ASN ranges, {status.get('country_rows')} country ranges, loaded {status.get('loaded_at')}")
+    print(status["attribution"])
+
+
+def register_nodes(subparsers):
+    nodes = subparsers.add_parser("nodes", help="Community donor node network")
+    sub = nodes.add_subparsers(title="action", dest="action")
+    sub.add_parser("list", help="List devices").set_defaults(func=cmd_nodes_list)
+    sub.add_parser("stats", help="Aggregate donation totals and open flags").set_defaults(func=cmd_nodes_stats)
+    sub.add_parser("flags", help="List open flags that need a human").set_defaults(func=cmd_nodes_flags)
+    suspend = sub.add_parser("suspend", help="Suspend one device for a while")
+    suspend.add_argument("device_uid")
+    suspend.add_argument("--minutes", type=int, default=SUSPEND_DEFAULT_MINUTES)
+    suspend.add_argument("--reason", default="suspended from the command line")
+    suspend.set_defaults(func=cmd_nodes_suspend)
+    unsuspend = sub.add_parser("unsuspend", help="Lift the suspension of a device")
+    unsuspend.add_argument("device_uid")
+    unsuspend.set_defaults(func=cmd_nodes_unsuspend)
+    prune = sub.add_parser("prune", help="Prune usage buckets, events and finished requests past their retention")
+    prune.add_argument("--dry-run", action="store_true", help="report what would be removed")
+    prune.set_defaults(func=cmd_nodes_prune)
+    clear = sub.add_parser("clear", help="Empty the job queue and the event log (never usage or donor totals)")
+    clear.add_argument("--force", action="store_true", help="actually delete; without it only counts are shown")
+    clear.set_defaults(func=cmd_nodes_clear)
+    geo_sync = sub.add_parser("geo-sync", help="Download the DB-IP address database used to class donor addresses")
+    geo_sync.add_argument("--dry-run", action="store_true", help="download and verify, do not install")
+    geo_sync.add_argument("--month", default="", help="YYYY-MM, default the newest published")
+    geo_sync.set_defaults(func=cmd_nodes_geo_sync)
+    sub.add_parser("geo-status", help="Show the installed address database").set_defaults(func=cmd_nodes_geo_status)

devplacepy/cli/system.py

@@ -0,0 +1,202 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.cli._shared import _audit_cli
+
+RULE_WIDTH = 72
+
+
+def _rule(char: str = "=") -> str:
+    return char * RULE_WIDTH
+
+
+def _human(n) -> str:
+    n = float(n)
+    for unit in ("B", "KB", "MB", "GB", "TB"):
+        if n < 1024:
+            return f"{n:.1f}{unit}"
+        n /= 1024
+    return f"{n:.1f}PB"
+
+
+def _dir_stats(path) -> tuple[int, int]:
+    import os
+
+    count = 0
+    total = 0
+    for root, _dirs, files in os.walk(str(path)):
+        for name in files:
+            count += 1
+            try:
+                total += os.path.getsize(os.path.join(root, name))
+            except OSError:
+                continue
+    return count, total
+
+
+def _section(title: str) -> None:
+    print()
+    print(_rule())
+    print(title)
+    print(_rule())
+
+
+def _build_areas():
+    from pathlib import Path
+
+    from devplacepy import config
+    from devplacepy.attachments import (
+        ATTACHMENTS_DIR,
+        purge_soft_deleted_attachments,
+        sweep_orphan_attachment_blobs,
+    )
+    from devplacepy.project_files import (
+        PROJECT_FILES_DIR,
+        purge_soft_deleted_project_files,
+        sweep_orphan_project_file_blobs,
+    )
+    from devplacepy.services.containers import store as container_store
+
+    return [
+        {
+            "label": "Attachments",
+            "path": ATTACHMENTS_DIR,
+            "checks": [
+                ("soft-deleted attachments", purge_soft_deleted_attachments),
+                (
+                    "orphan attachment blobs (zero DB reference)",
+                    sweep_orphan_attachment_blobs,
+                ),
+            ],
+        },
+        {
+            "label": "Project files",
+            "path": PROJECT_FILES_DIR,
+            "checks": [
+                ("soft-deleted project files", purge_soft_deleted_project_files),
+                (
+                    "orphan project-file blobs (zero DB reference)",
+                    sweep_orphan_project_file_blobs,
+                ),
+            ],
+        },
+        {
+            "label": "Container workspaces",
+            "path": Path(config.CONTAINER_WORKSPACES_DIR),
+            "checks": [
+                (
+                    "orphaned workspace directories (no live instance)",
+                    container_store.gc_workspaces,
+                ),
+            ],
+        },
+    ]
+
+
+def cmd_system_prune(args):
+    import time
+
+    dry_run = bool(args.dry_run)
+    areas = _build_areas()
+
+    print("System prune - safe but aggressive disk-space cleanup")
+    print(
+        "Removes only content with zero live reference: soft-deleted attachment/"
+        "\nproject-file blobs, blob files with no matching database row at all"
+        "\n(orphans, e.g. left by an interrupted or racing sync), and orphaned"
+        "\ncontainer workspace directories. Never touches live content."
+    )
+    if dry_run:
+        print("DRY RUN: this pass only reports; nothing will be deleted.")
+
+    started = time.monotonic()
+
+    _section("Where the data is, and what can be reclaimed (estimate)")
+    estimate_items = 0
+    estimate_bytes = 0
+    for area in areas:
+        count, size = _dir_stats(area["path"])
+        print(f"\n{area['label']}")
+        print(f"  location: {area['path']}")
+        print(f"  currently on disk: {count} file(s), {_human(size)}")
+        for check_label, fn in area["checks"]:
+            c, f = fn(dry_run=True)
+            estimate_items += c
+            estimate_bytes += f
+            print(f"  expected to reclaim - {check_label}: {c} item(s), {_human(f)}")
+
+    print()
+    print(_rule("-"))
+    print(
+        f"Estimated total: {estimate_items} item(s), {_human(estimate_bytes)} reclaimable"
+    )
+    print(_rule("-"))
+
+    if dry_run:
+        print()
+        print("DRY RUN complete: nothing was deleted. Re-run without --dry-run to apply.")
+        return
+
+    _section("Executing")
+    actual_items = 0
+    actual_bytes = 0
+    for area in areas:
+        print(f"\n{area['label']} - {area['path']}")
+        for check_label, fn in area["checks"]:
+            c, f = fn(dry_run=False)
+            actual_items += c
+            actual_bytes += f
+            print(f"  removed - {check_label}: {c} item(s), {_human(f)}")
+
+    _section("After")
+    for area in areas:
+        count, size = _dir_stats(area["path"])
+        print(f"  {area['label']} ({area['path']}): {count} file(s), {_human(size)}")
+
+    elapsed = time.monotonic() - started
+
+    print()
+    print(_rule())
+    print(
+        f"Removed {actual_items} item(s) total, freed {_human(actual_bytes)} "
+        f"in {elapsed:.1f}s"
+    )
+    if estimate_bytes != actual_bytes or estimate_items != actual_items:
+        print(
+            f"(Estimate was {estimate_items} item(s), {_human(estimate_bytes)} - "
+            "state changed between the estimate and execution passes, e.g. the "
+            "live server wrote or soft-deleted something in between.)"
+        )
+    print(_rule())
+
+    _audit_cli(
+        "cli.system.prune",
+        f"CLI system prune removed {actual_items} item(s), freed {_human(actual_bytes)}",
+        metadata={
+            "items": actual_items,
+            "bytes_freed": actual_bytes,
+            "estimated_items": estimate_items,
+            "estimated_bytes": estimate_bytes,
+            "elapsed_seconds": round(elapsed, 3),
+        },
+    )
+
+
+def register_system(subparsers):
+    system = subparsers.add_parser("system", help="Cross-cutting system maintenance")
+    system_sub = system.add_subparsers(title="action", dest="action")
+    prune = system_sub.add_parser(
+        "prune",
+        help=(
+            "Safe but aggressive disk-space cleanup: purges soft-deleted "
+            "attachment/project-file blobs and any blob with zero database "
+            "reference at all, plus orphaned container workspace directories. "
+            "Reports location, current size, and estimated reclaim per area "
+            "before acting, then confirms what was actually freed."
+        ),
+    )
+    prune.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Report location, size, and estimated reclaim without deleting anything",
+    )
+    prune.set_defaults(func=cmd_system_prune)

devplacepy/config.py

@@ -1,6 +1,7 @@
 # retoor <retoor@molodetz.nl>
 
 import time
+import tomllib
 from pathlib import Path
 from dotenv import load_dotenv
 from os import environ
@@ -11,6 +12,8 @@ BASE_DIR = Path(__file__).resolve().parent.parent
 STATIC_DIR = BASE_DIR / "devplacepy" / "static"
 TEMPLATES_DIR = BASE_DIR / "devplacepy" / "templates"
 
+LOG_LEVEL = environ.get("DEVPLACE_LOG_LEVEL", "INFO").upper()
+
 DATA_DIR = Path(environ.get("DEVPLACE_DATA_DIR", str(BASE_DIR / "data")))
 UPLOADS_DIR = DATA_DIR / "uploads"
 ATTACHMENTS_DIR = UPLOADS_DIR / "attachments"
@@ -22,6 +25,18 @@ ZIP_STAGING_DIR = DATA_DIR / "zip_staging"
 FORK_STAGING_DIR = DATA_DIR / "fork_staging"
 BACKUPS_DIR = DATA_DIR / "backups"
 BACKUP_STAGING_DIR = DATA_DIR / "backup_staging"
+NODE_CLIENTS_DIR = DATA_DIR / "node_clients"
+NODE_GEO_DIR = DATA_DIR / "node_geo"
+NODE_GEO_DB = NODE_GEO_DIR / "geo.db"
+
+RCLONE_BIN = environ.get("DEVPLACE_RCLONE_BIN", "rclone")
+RCLONE_CONFIG_FILE = environ.get(
+    "DEVPLACE_RCLONE_CONFIG",
+    str(Path(environ.get("HOME", "/root")) / ".config" / "rclone" / "rclone.conf"),
+)
+BACKUP_OFFLOAD_REMOTE = environ.get(
+    "DEVPLACE_BACKUP_OFFLOAD_REMOTE", "storagebox:devplacepy-backups"
+)
 SEO_REPORTS_DIR = DATA_DIR / "seo_reports"
 PLANNING_REPORTS_DIR = DATA_DIR / "planning_reports"
 DBAPI_DIR = DATA_DIR / "dbapi"
@@ -33,6 +48,7 @@ ISSLOP_RUNS_DIR = ISSLOP_DIR / "runs"
 ISSLOP_MEDIA_DIR = ISSLOP_DIR / "media"
 KEYS_DIR = DATA_DIR / "keys"
 BOT_DIR = DATA_DIR / "bot"
+COMMUNITIES_DIR = DATA_DIR / "communities"
 LOCKS_DIR = DATA_DIR / "locks"
 
 DEVII_TASKS_DB = DATA_DIR / "devii_tasks.db"
@@ -45,7 +61,7 @@ SECRET_KEY = environ.get("SECRET_KEY", "devplace-secret-key-change-in-production
 SECONDS_PER_DAY = 86400
 SESSION_MAX_AGE = SECONDS_PER_DAY * 7
 SESSION_MAX_AGE_REMEMBER = SECONDS_PER_DAY * 30
-PORT = 10500
+PORT = int(environ.get("DEVPLACE_PORT", "10500"))
 SITE_URL = environ.get("DEVPLACE_SITE_URL", "").rstrip("/")
 
 PRESENCE_TIMEOUT_SECONDS = int(environ.get("DEVPLACE_PRESENCE_TIMEOUT_SECONDS", "60"))
@@ -59,7 +75,9 @@ PRESENCE_ONLINE_MARGIN_SECONDS = int(
 XMLRPC_BIND = environ.get("DEVPLACE_XMLRPC_BIND", "127.0.0.1")
 XMLRPC_PORT = int(environ.get("DEVPLACE_XMLRPC_PORT", "10550"))
 
-STATIC_VERSION = environ.get("DEVPLACE_STATIC_VERSION") or str(int(time.time()))
+APP_VERSION = tomllib.loads((BASE_DIR / "pyproject.toml").read_text())["project"]["version"]
+BOOT_ID = environ.get("DEVPLACE_STATIC_VERSION") or str(int(time.time()))
+STATIC_VERSION = f"{APP_VERSION}-{BOOT_ID}"
 
 TEMPLATE_AUTO_RELOAD = environ.get("DEVPLACE_TEMPLATE_AUTO_RELOAD", "1") != "0"
 
@@ -72,11 +90,16 @@ INTERNAL_MODEL = "molodetz"
 INTERNAL_EMBED_MODEL = "molodetz~embed"
 INTERNAL_IMAGE_MODEL = "molodetz-img-small"
 
+AQUALITY_NEWS_GRADING_URL = "https://aquality.cloud.pravda.education/v1/chat/completions"
+AQUALITY_NEWS_GRADING_MODEL = "aquality"
+
 AWARD_GIVE_COOLDOWN_HOURS_DEFAULT = 24
 AWARD_RECEIVE_COOLDOWN_HOURS_DEFAULT = 24
 AWARD_DISPLAY_HOURS_DEFAULT = 24
 AWARD_DESCRIPTION_MAX = 125
 AWARD_IMAGE_MODEL_DEFAULT = "molodetz-img-small"
+USER_IMAGE_SIZE_DEFAULT = "1024x1024"
+USER_IMAGE_TIMEOUT_SECONDS = 120.0
 AWARD_IMAGE_SIZE_DEFAULT = "512x512"
 AWARD_GENERATION_TIMEOUT_SECONDS = 120.0
 AWARD_IMAGE_PROMPT_DEFAULT = (
@@ -97,6 +120,8 @@ QUIZ_SCOREBOARD_LIMIT = 20
 QUIZ_SCOREBOARD_CACHE_SECONDS = 15
 QUIZ_LIST_PER_PAGE = 20
 
+BATTLES_LIST_PER_PAGE = 10
+
 DEFAULT_CORRECTION_PROMPT = "Leave literary as is, only do punctuation and casing"
 DEFAULT_MODIFIER_PROMPT = (
     "Execute what is behind `@ai` (the prompt) and replace that part including `@ai`"
@@ -131,6 +156,8 @@ DATA_PATHS: dict[str, Path] = {
     "fork_staging": FORK_STAGING_DIR,
     "backups": BACKUPS_DIR,
     "backup_staging": BACKUP_STAGING_DIR,
+    "node_clients": NODE_CLIENTS_DIR,
+    "node_geo": NODE_GEO_DIR,
     "seo_reports": SEO_REPORTS_DIR,
     "planning_reports": PLANNING_REPORTS_DIR,
     "dbapi": DBAPI_DIR,
@@ -142,6 +169,7 @@ DATA_PATHS: dict[str, Path] = {
     "isslop_media": ISSLOP_MEDIA_DIR,
     "keys": KEYS_DIR,
     "bot": BOT_DIR,
+    "communities": COMMUNITIES_DIR,
     "locks": LOCKS_DIR,
 }
 

devplacepy/constants.py

@@ -2,6 +2,16 @@
 
 TOPICS = ["devlog", "showcase", "question", "rant", "fun", "random", "politics"]
 
+TOPIC_LABELS = {
+    "devlog": "Devlog",
+    "showcase": "Showcase",
+    "question": "Question",
+    "rant": "Rant",
+    "fun": "Fun",
+    "random": "Random",
+    "politics": "Politics",
+}
+
 REACTION_EMOJI = [
     "\U0001f44d",
     "❤️",

devplacepy/content.py

@@ -19,6 +19,7 @@ from devplacepy.database import (
     get_user_votes,
     get_reactions_by_targets,
     get_user_bookmarks,
+    get_user_notes,
     get_blocked_uids,
     get_poll_for_post,
     update_target_stars,
@@ -38,6 +39,8 @@ from devplacepy.database import (
     get_int_setting,
     _now_iso,
     db,
+    get_user_recent_items,
+    invalidate_user_recent_cache,
 )
 from devplacepy.utils import (
     time_ago,
@@ -46,6 +49,7 @@ from devplacepy.utils import (
     award_rewards,
     track_action,
     create_notification,
+    create_thread_notifications,
     create_mention_notifications,
     is_admin,
     is_primary_admin,
@@ -53,6 +57,7 @@ from devplacepy.utils import (
     XP_UPVOTE,
 )
 from devplacepy.services.audit import record as audit
+from devplacepy.services.opinionwar import store as war_store
 from devplacepy.services.correction import schedule_correction
 from devplacepy.services.ai_modifier import schedule_modification
 from devplacepy.services.seo_meta import schedule_seo_meta_for_table
@@ -66,6 +71,7 @@ CREATE_METADATA_KEYS = ("project_type", "is_private", "language", "topic", "stat
 
 BOOKMARKABLE_TYPES = {"post", "gist", "project", "news", "quiz"}
 REACTABLE_TYPES = {"post", "comment", "gist", "project", "quiz"}
+NOTABLE_TYPES = {"post", "gist", "project", "news"}
 
 logger = logging.getLogger(__name__)
 
@@ -131,6 +137,18 @@ def can_view_project(project: dict | None, user: dict | None) -> bool:
     return not _owner_is_admin(project)
 
 
+def get_user_sidebar_gists(user_uid: str, limit: int = 5) -> list[dict]:
+    return get_user_recent_items("gists", user_uid)[:limit]
+
+
+def get_user_sidebar_projects(
+    user_uid: str, viewer: dict | None, limit: int = 5
+) -> list[dict]:
+    rows = get_user_recent_items("projects", user_uid)
+    visible = [p for p in rows if can_view_project(p, viewer)]
+    return visible[:limit]
+
+
 def owns_instance(
     instance: dict | None, project: dict | None, user: dict | None
 ) -> bool:
@@ -262,6 +280,7 @@ def create_content_item(
         from devplacepy.templating import clear_user_projects_cache
 
         clear_user_projects_cache(user["uid"])
+    invalidate_user_recent_cache(table_name, user["uid"])
     award_rewards(user["uid"], xp, badge)
     if attachment_uids:
         link_attachments(attachment_uids, target_type, uid)
@@ -308,10 +327,16 @@ def apply_vote(request, user: dict, target_type: str, target_uid: str, value: in
     existing = votes.find_one(
         user_uid=user["uid"], target_uid=target_uid, target_type=target_type
     )
-    old_value = int(existing["value"]) if existing else 0
+    old_value = int(existing["value"]) if existing and not existing.get("deleted_at") else 0
     did_upvote = False
     new_value = value
-    if existing:
+    if value == 0:
+        if existing and not existing.get("deleted_at"):
+            votes.update(
+                {"id": existing["id"], "deleted_at": _now_iso(), "deleted_by": user["uid"]},
+                ["id"],
+            )
+    elif existing:
         if existing.get("deleted_at"):
             votes.update(
                 {
@@ -437,6 +462,7 @@ def create_comment_record(
     comment_url = f"{redirect_url}#comment-{comment_uid}"
 
     if target_type == "post":
+        already_notified = {user["uid"]}
         if parent_uid:
             parent = get_table("comments").find_one(uid=parent_uid, deleted_at=None)
             if parent and parent["user_uid"] != user["uid"]:
@@ -447,6 +473,7 @@ def create_comment_record(
                     user["uid"],
                     comment_url,
                 )
+                already_notified.add(parent["user_uid"])
         else:
             posts = get_table("posts")
             post = posts.find_one(uid=target_uid)
@@ -460,6 +487,9 @@ def create_comment_record(
                     user["uid"],
                     comment_url,
                 )
+                already_notified.add(post["user_uid"])
+
+        create_thread_notifications(target_uid, user["uid"], comment_url, already_notified)
 
     create_mention_notifications(content, user["uid"], comment_url)
     record_screening(
@@ -626,7 +656,9 @@ def detail_context(
         "attachments": detail["attachments"],
         "reactions": detail.get("reactions", {"counts": {}, "mine": []}),
         "bookmarked": detail.get("bookmarked", False),
+        "note_content": detail.get("note_content"),
         "poll": detail.get("poll"),
+        "war": detail.get("war"),
         "project_link": detail.get("project_link"),
         "maturity": detail.get("maturity", "general"),
     }
@@ -670,6 +702,7 @@ def edit_content_item(
         "updated_at": datetime.now(timezone.utc).isoformat(),
     }
     table.update({"uid": item["uid"], **update_fields}, ["uid"])
+    invalidate_user_recent_cache(table_name, item["user_uid"])
     record_screening(
         screening,
         target_type=kind,
@@ -761,6 +794,7 @@ def delete_content_item(
         soft_delete_all_project_files(item["uid"], actor)
         soft_delete_fork_relations(item["uid"], actor)
         clear_user_projects_cache(item["user_uid"])
+    invalidate_user_recent_cache(table_name, item["user_uid"])
     soft_delete(table_name, actor, stamp=stamp, uid=item["uid"])
     logger.info(f"{table_name} {item['uid']} soft-deleted by {user['username']}")
     audit.record(
@@ -803,6 +837,11 @@ def load_detail(
         and target_type in BOOKMARKABLE_TYPES
         and item["uid"] in get_user_bookmarks(user["uid"], target_type, [item["uid"]])
     )
+    note_content = (
+        get_user_notes(user["uid"], target_type, [item["uid"]]).get(item["uid"])
+        if user and target_type in NOTABLE_TYPES
+        else None
+    )
     return {
         "item": item,
         "author": author,
@@ -816,7 +855,11 @@ def load_detail(
         "time_ago": time_ago(item["created_at"]),
         "reactions": reactions,
         "bookmarked": bookmarked,
+        "note_content": note_content,
         "poll": get_poll_for_post(item["uid"], user) if target_type == "post" else None,
+        "war": war_store.get_war_serialized_for_post(item["uid"], user)
+        if target_type == "post"
+        else None,
         "project_link": get_project_by_uid(item.get("project_uid")) if target_type == "post" else None,
         "maturity": get_maturity(target_type, item["uid"])["level"],
     }

devplacepy/database/CLAUDE.md

@@ -124,7 +124,7 @@ Profile with the real database before and after any change here (`cProfile` arou
 
 Attachments (see "Profile media gallery and soft-deleted attachments" below) are one instance of a platform-wide model: **every removal is a soft delete; only garbage collection is a hard delete.** Every removable row carries two columns, `deleted_at` (ISO timestamp) and `deleted_by` (actor uid, or `system`). A live row has `deleted_at = NULL`, and every list/count read filters `deleted_at IS NULL`.
 
-- **Tables (`database.SOFT_DELETE_TABLES`):** posts, comments, gists, projects, news, news_images, project_files, attachments, votes, reactions, bookmarks, follows, poll_votes, polls, poll_options, sessions, instances, instance_schedules, devii_conversations, devii_tasks, devii_lessons, devii_virtual_tools, user_customizations, project_forks. `init_db` loops `ensure_soft_delete_columns(t)` for each (adds both columns + `idx_<t>_deleted` / the partial trash index described above). The Devii task/lesson/virtual-tool stores run on their own DB handles, so their `_ensure_indexes` adds the columns there too.
+- **Tables (`database.SOFT_DELETE_TABLES`):** posts, comments, gists, projects, news, news_images, project_files, attachments, votes, reactions, bookmarks, notes, follows, poll_votes, polls, poll_options, sessions, instances, instance_schedules, devii_conversations, devii_tasks, devii_lessons, devii_virtual_tools, user_customizations, project_forks. `init_db` loops `ensure_soft_delete_columns(t)` for each (adds both columns + `idx_<t>_deleted` / the partial trash index described above). The Devii task/lesson/virtual-tool stores run on their own DB handles, so their `_ensure_indexes` adds the columns there too.
 - **Every INSERT into a soft-deletable table MUST write `deleted_at: None, deleted_by: None`.** `dataset.find(deleted_at=None)` on a table that lacks the column matches NOTHING (a false predicate), silently hiding all rows - the born-live insert is what creates the column. Add the pair to any new insert.
 - **Central helpers (`database/`):** `soft_delete(table, deleted_by, *, stamp=None, **criteria)` (equality), `soft_delete_in(table, column, uids, deleted_by, *, stamp=None, **extra)` (IN-clause cascade), `restore(table, **criteria)`, `purge(table, **criteria)` (real delete), `list_deleted(table, page)` / `count_deleted(table)` (trash listings), and the event helpers `restore_event(stamp)` / `purge_event(stamp)` that act across ALL tables sharing one `deleted_at` stamp.
 - **Two generic chokepoints are conditionally filtered:** `resolve_by_slug(table, slug, include_deleted=False)` (detail-page lookups; restore passes `include_deleted=True`) and `paginate(table, ...)` (auto-appends `deleted_at IS NULL` when the table has the column and the caller did not pass `deleted_at`). `seo._collect` does the same for the sitemap. Read filters were threaded through every batch helper, analytics/activity/leaderboard UNION, feed/profile/listing route, and store; never re-introduce an unfiltered read of a soft-deletable table.
@@ -198,8 +198,8 @@ Site settings are seeded on startup (`site_settings` table):
 | `site_name` / `site_description` / `site_tagline` | DevPlace branding | General site metadata |
 | `news_grade_threshold` | `"7"` | Minimum AI grade for auto-publish |
 | `news_api_url` | `"https://news.app.molodetz.nl/api"` | News source API |
-| `news_ai_url` | `"https://openai.app.molodetz.nl/v1/chat/completions"` | AI grading endpoint |
-| `news_ai_model` | `"molodetz"` | AI model identifier |
+| `news_ai_url` | `"https://aquality.cloud.pravda.education/v1/chat/completions"` | AI grading endpoint - the free, local aquality quality model by default (see `devplacepy/services/news/CLAUDE.md`) |
+| `news_ai_model` | `"aquality"` | AI model identifier |
 | `max_upload_size_mb` / `allowed_file_types` / `max_attachments_per_resource` | `"10"` / `""` / `"10"` | Upload limits |
 | `rate_limit_per_minute` | `"60"` | Mutating requests per IP per window (`main.py` middleware); a `429` carries a `Retry-After: <window>` header |
 | `rate_limit_window_seconds` | `"60"` | Rolling window for the rate limit (also the `Retry-After` value) |
@@ -211,6 +211,7 @@ Site settings are seeded on startup (`site_settings` table):
 | `maintenance_message` | scheduled-maintenance text | Body shown on the maintenance 503 page |
 | `customization_enabled` | `"1"` | When `"0"`, `custom_css_tag`/`custom_js_tag` inject nothing (feature off) |
 | `customization_js_enabled` | `"1"` | When `"0"`, custom CSS still serves but custom JS is suppressed |
+| `happy_404_enabled` | `"1"` | When `"0"`, `happy404.render()` is a no-op and a real 404 page is shown; when on, an HTML 404 renders a random existing post instead (see below) |
 | `moderation_sla_hours` | `"24"` | The published moderation response window; the admin queue badge turns red past it |
 | `moderation_filter_mode` | `"review"` | `off`/`label`/`review`/`block` - how the content filter acts on a match |
 | `moderation_filter_review_score` | `"2"` | Rule score at which a match becomes a report rather than a label |
@@ -222,6 +223,7 @@ Site settings are seeded on startup (`site_settings` table):
 | `terms_version` / `privacy_version` / `guidelines_version` | `"1"` | Bumping `terms_version` forces re-acceptance before the next write. **Every reader uses `get_setting(key, "1") or "1"`** - an empty stored value must read as the default or the gate 403s every write |
 | `ai_third_party_provider` | `""` | Named in the consent copy and the privacy policy |
 | `extra_head` | `""` | Raw HTML emitted verbatim into every page `<head>` by `templating.extra_head_tag()`; site-wide trusted-admin input, not sanitized |
+| `sync_enabled` / `sync_origins` | `"0"` / `""` | Community sync kill switch (`routers/sync/_shared.py`) and an optional comma separated allowlist of sending origins; empty allows any origin |
 
 Besides the site/news/upload keys above, the **Operational** group is admin-editable at `/admin/settings`: `site_url` (public origin for absolute links incl. container ingress; resolved by `seo.public_base_url()` = setting -> `DEVPLACE_SITE_URL` env -> request origin), `rate_limit_per_minute`, `rate_limit_window_seconds`, `news_service_interval`, `session_max_age_days`, `session_remember_days`, `registration_open`, `maintenance_mode`, `maintenance_message`, `docs_search_mode` (`agent`|`bm25`, default `agent` - picks the `/docs/search.html` surface: the in-page Devii chat or the classic BM25 list; a viewer over their daily AI limit, or a guest when Devii is disabled, auto-falls-back to BM25 via `routers/docs/views.py` `_agent_search_state`), `outbound_proxy_url` (empty by default - when set, every `stealth.stealth_async_client`/`stealth_sync_client` call across the whole app routes through it via `stealth.configured_proxy_url()`; validated as `http(s)://`/`socks5(h)://` with a host in `AdminSettingsForm`; falls back to `DEVPLACE_OUTBOUND_PROXY_URL` when unset - see the "Outbound HTTP" note in the root `CLAUDE.md`). The **Custom Code** key `extra_head` and the `ios_app_url` badge link are the keys in the settings handler's `CLEARABLE_SETTINGS` set, so saving an empty textarea/field removes them (the default loop skips empty values).
 
@@ -239,6 +241,7 @@ Operational settings - read sites and rules:
 | `service_<name>_enabled` / `service_<name>_command` / `service_<name>_log_size` | `BaseService` reconciling loop | Generic per-service controls written by the Services tab; the loop reconciles within ~1s |
 | `session_max_age_days` / `session_remember_days` | `auth.py` signup + login | Multiplied by `SECONDS_PER_DAY`; passed to `create_session(uid, max_age)` so the cookie and the DB session row expire together |
 | `registration_open` | `auth.py` `signup_page` (GET) and `signup` (POST) | POST returns before any DB write when closed |
+| `happy_404_enabled` | `happy404.render()`, called from the `404` handler in `main.py` | Gates the whole feature; checked per-request via the normal 60s `get_setting` TTL cache, so a toggle takes effect within a minute across all workers with no restart |
 
 **Booleans are `<select>`, never checkboxes.** The settings save handler (`admin.py`) skips empty form values so empty fields don't clobber existing rows. An unchecked checkbox submits nothing, so it could never be turned off - `registration_open` and `maintenance_mode` use `<option value="1">`/`<option value="0">` so a value is always submitted.
 

devplacepy/database/__init__.py

@@ -8,7 +8,7 @@ from .users import get_users_by_uids, _admins_cache, invalidate_admins_cache, ge
 from .relations import _relations_cache, get_user_relations, get_blocked_uids, get_muted_uids, get_silenced_uids, invalidate_user_relations
 from .pagination import PAGE_SIZE, paginate, interleave_by_author, paginate_diverse, get_user_post_count, clear_user_post_count, build_pagination
 from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns, soft_delete, soft_delete_in, restore, purge, list_deleted, count_deleted, restore_event, purge_event
-from .engagement import _comment_count_cache, get_comment_counts_by_post_uids, get_post_counts_by_user_uids, get_vote_counts, get_user_votes, get_reactions_by_targets, get_user_bookmarks, get_polls_by_post_uids, get_poll_for_post
+from .engagement import _comment_count_cache, get_comment_counts_by_post_uids, get_post_counts_by_user_uids, get_vote_counts, get_user_votes, get_reactions_by_targets, get_user_bookmarks, get_user_notes, get_polls_by_post_uids, get_poll_for_post
 from .usage import _add_usage, _get_usage, add_correction_usage, get_correction_usage, add_modifier_usage, get_modifier_usage, NEWS_USAGE_KEY, add_news_usage, get_news_usage, ISSUE_USAGE_KEY, add_issue_usage, get_issue_usage, SEO_USAGE_KEY, add_seo_usage, get_seo_usage, AWARD_USAGE_KEY, add_award_usage, get_award_usage
 from .awards import (
     AWARDS_PER_PAGE,
@@ -34,7 +34,7 @@ from .email import EMAIL_ACCOUNT_DEFAULTS, list_email_accounts, get_email_accoun
 from .notifications import NOTIFICATION_TYPES, NOTIFICATION_CHANNELS, _NOTIFICATION_CHANNEL_COLUMNS, _NOTIFICATION_CHANNEL_DEFAULTS, _NOTIFICATION_TYPE_KEYS, _notification_prefs_cache, _notification_default, get_notification_default, set_notification_default, _notification_overrides, notification_enabled, get_notification_prefs, set_notification_pref, reset_notification_prefs, mark_notifications_read_by_target
 from .forks import record_fork, get_fork_parent, count_forks, soft_delete_fork_relations, delete_fork_relations
 from .follows import get_follow_counts, get_follow_list, get_following_among
-from .deepsearch import _ds_now, create_deepsearch_session, update_deepsearch_session, get_deepsearch_session, add_deepsearch_message, get_deepsearch_messages, get_cached_deepsearch_url, upsert_deepsearch_url_cache
+from .deepsearch import _ds_now, create_deepsearch_session, update_deepsearch_session, get_deepsearch_session, list_deepsearch_sessions, add_deepsearch_message, get_deepsearch_messages, get_cached_deepsearch_url, upsert_deepsearch_url_cache
 from .ranking import VOTABLE_TARGETS, STAR_TARGETS, _authors_cache, _ranked_authors, _rank_map, get_top_authors, get_leaderboard, get_user_rank, get_user_stars, clear_user_stars, update_target_stars, soft_delete_engagement, delete_engagement, get_target_owner_uid
 from .moderation import (
     ACTIONS_TABLE,
@@ -43,6 +43,7 @@ from .moderation import (
     CONSENTS_TABLE,
     CONSENT_KINDS,
     CONSENT_STATES,
+    MEMBER_ONLY_CONSENTS,
     MATURITY_LEVELS,
     MATURITY_SOURCES,
     MATURITY_TABLE,
@@ -62,6 +63,7 @@ from .moderation import (
     band_allows_mature,
     band_allows_restricted,
     consent_granted,
+    consent_granted_many,
     consent_state,
     get_maturity,
     get_maturity_by_targets,
@@ -74,7 +76,7 @@ from .moderation import (
     years_between,
 )
 from .comments import _drop_blocked, _build_comment_items, load_comments, get_recent_comments_by_target_uids, get_recent_comments_by_post_uids, load_comments_by_target_uids
-from .content import resolve_by_slug, resolve_object_url, get_uids_by_username_match, text_search_clause, get_daily_topic, get_featured_news, get_trending_topics
+from .content import resolve_by_slug, resolve_object_url, get_projects_by_uids, get_uids_by_username_match, text_search_clause, get_daily_topic, get_featured_news, get_featured_topics, get_trending_topics, get_user_recent_items, invalidate_user_recent_cache
 from .attachments_data import get_attachments, get_attachments_by_type, get_news_images_by_uids, delete_attachment_record, delete_attachments, _delete_attachment_file, get_user_media, get_user_attachments, get_user_attachment, get_deleted_media
 from .stats import _stats_cache, get_site_stats, _analytics_cache, get_platform_analytics, _gist_languages_cache, get_gist_languages
 from .schema import BUG_TABLE_RENAMES, migrate_bug_tables_to_issue_tables, init_db, _refresh_query_planner_stats, OLD_GATEWAY_URL, migrate_ai_gateway_settings, backfill_api_keys, _backfill_gamification
@@ -157,6 +159,7 @@ __all__ = [
     "get_user_votes",
     "get_reactions_by_targets",
     "get_user_bookmarks",
+    "get_user_notes",
     "get_polls_by_post_uids",
     "get_poll_for_post",
     "_add_usage",
@@ -236,6 +239,7 @@ __all__ = [
     "create_deepsearch_session",
     "update_deepsearch_session",
     "get_deepsearch_session",
+    "list_deepsearch_sessions",
     "add_deepsearch_message",
     "get_deepsearch_messages",
     "get_cached_deepsearch_url",
@@ -259,6 +263,7 @@ __all__ = [
     "AGE_BANDS",
     "CONSENTS_TABLE",
     "CONSENT_KINDS",
+    "MEMBER_ONLY_CONSENTS",
     "CONSENT_STATES",
     "MATURITY_LEVELS",
     "MATURITY_SOURCES",
@@ -279,6 +284,7 @@ __all__ = [
     "band_allows_mature",
     "band_allows_restricted",
     "consent_granted",
+    "consent_granted_many",
     "consent_state",
     "get_maturity",
     "get_maturity_by_targets",
@@ -296,10 +302,14 @@ __all__ = [
     "load_comments_by_target_uids",
     "resolve_by_slug",
     "resolve_object_url",
+    "get_projects_by_uids",
     "get_uids_by_username_match",
     "text_search_clause",
     "get_daily_topic",
     "get_featured_news",
+    "get_featured_topics",
+    "get_user_recent_items",
+    "invalidate_user_recent_cache",
     "get_trending_topics",
     "get_attachments",
     "get_attachments_by_type",

devplacepy/database/content.py

@@ -1,5 +1,7 @@
 # retoor <retoor@molodetz.nl>
 
+import os
+import random
 from collections import Counter
 
 from devplacepy.cache import TTLCache
@@ -8,6 +10,34 @@ from .core import db, get_table, or_
 
 _daily_topic_cache = TTLCache(ttl=60, max_size=1)
 _trending_cache = TTLCache(ttl=15, max_size=1)
+FEATURED_TOPICS_POOL_TTL = int(os.environ.get("DEVPLACE_FEATURED_TOPICS_POOL_TTL", "300"))
+FEATURED_TOPICS_POOL_SIZE = 20
+_featured_topics_cache = TTLCache(ttl=FEATURED_TOPICS_POOL_TTL, max_size=1)
+
+USER_RECENT_ITEMS_TTL = int(os.environ.get("DEVPLACE_USER_RECENT_ITEMS_TTL", "15"))
+_user_recent_cache = TTLCache(ttl=USER_RECENT_ITEMS_TTL, max_size=2000)
+
+
+def _user_recent_key(table_name: str, user_uid: str) -> str:
+    return f"{table_name}:{user_uid}"
+
+
+def invalidate_user_recent_cache(table_name: str, user_uid: str) -> None:
+    _user_recent_cache.pop(_user_recent_key(table_name, user_uid))
+
+
+def get_user_recent_items(table_name: str, user_uid: str) -> list[dict]:
+    key = _user_recent_key(table_name, user_uid)
+    cached = _user_recent_cache.get(key)
+    if cached is not None:
+        return cached
+    items = []
+    if table_name in db.tables:
+        rows = list(get_table(table_name).find(user_uid=user_uid, deleted_at=None))
+        rows.sort(key=lambda row: row.get("updated_at") or row.get("created_at") or "", reverse=True)
+        items = rows
+    _user_recent_cache.set(key, items)
+    return items
 
 
 def resolve_by_slug(table, slug, include_deleted=False):
@@ -89,6 +119,11 @@ def resolve_object_url(target_type: str, target_uid: str) -> str:
         if not poll:
             return "/feed"
         return resolve_object_url("post", poll.get("post_uid", ""))
+    if target_type == "battle":
+        war = get_table("opinion_wars").find_one(uid=target_uid)
+        if not war:
+            return "/battles"
+        return resolve_object_url("post", war.get("post_uid", ""))
     if target_type == "workspace":
         instance = get_table("instances").find_one(uid=target_uid)
         return f"/admin/containers/{instance['uid']}" if instance else "/admin/containers"
@@ -97,6 +132,17 @@ def resolve_object_url(target_type: str, target_uid: str) -> str:
     return "/feed"
 
 
+def get_projects_by_uids(uids):
+    if not uids or "projects" not in db.tables:
+        return {}
+    projects = get_table("projects")
+    if "uid" not in projects.columns:
+        return {}
+    seen = set()
+    unique = [u for u in uids if u not in seen and not seen.add(u)]
+    return {p["uid"]: p for p in projects.find(projects.table.columns.uid.in_(unique))}
+
+
 def get_uids_by_username_match(search, limit=200):
     term = (search or "").strip()
     if not term or "users" not in db.tables:
@@ -155,6 +201,49 @@ def _load_daily_topic():
     }
 
 
+def get_featured_topics(count: int = 3) -> list[dict]:
+    pool = _featured_topics_pool()
+    if not pool:
+        return []
+    return random.sample(pool, min(count, len(pool)))
+
+
+def _featured_topics_pool() -> list[dict]:
+    cached = _featured_topics_cache.get("pool")
+    if cached is not None:
+        return cached
+    pool = _load_featured_topics_pool()
+    _featured_topics_cache.set("pool", pool)
+    return pool
+
+
+def _load_featured_topics_pool(limit: int = FEATURED_TOPICS_POOL_SIZE) -> list[dict]:
+    if "news" not in db.tables:
+        return []
+    rows = db["news"].find(
+        featured=1,
+        status="published",
+        deleted_at=None,
+        order_by=["-synced_at"],
+        _limit=limit,
+    )
+    topics = []
+    for article in rows:
+        desc = (article.get("description") or "")[:160] or (
+            article.get("content") or ""
+        )[:160]
+        topics.append(
+            {
+                "title": article.get("title", ""),
+                "summary": desc,
+                "slug": article.get("slug", ""),
+                "url": article.get("url", ""),
+                "image_url": article.get("image_url", "") or "",
+            }
+        )
+    return topics
+
+
 def get_featured_news(limit=5):
     if "news" not in db.tables:
         return []

devplacepy/database/core.py

@@ -2,6 +2,7 @@
 
 import dataset
 import logging
+import warnings
 from pathlib import Path
 from sqlalchemy import or_
 from sqlalchemy.pool import NullPool
@@ -9,6 +10,8 @@ from collections import defaultdict
 from datetime import datetime, timedelta, timezone
 from devplacepy.cache import TTLCache
 from devplacepy.config import (
+    AQUALITY_NEWS_GRADING_MODEL,
+    AQUALITY_NEWS_GRADING_URL,
     DATABASE_URL,
     DEFAULT_CORRECTION_PROMPT,
     DEFAULT_MODIFIER_PROMPT,
@@ -18,6 +21,8 @@ from devplacepy.config import (
 
 logger = logging.getLogger(__name__)
 
+warnings.filterwarnings("ignore", message="Skipped unsupported reflection of expression-based index")
+
 ensure_data_dirs()
 if DATABASE_URL.startswith("sqlite:///"):
     _db_file = DATABASE_URL[len("sqlite:///") :]

devplacepy/database/deepsearch.py

@@ -54,6 +54,20 @@ def get_deepsearch_session(uid: str) -> dict | None:
     return get_table("deepsearch_sessions").find_one(uid=uid, deleted_at=None)
 
 
+def list_deepsearch_sessions(owner_kind: str, owner_id: str, limit: int = 50) -> list[dict]:
+    if "deepsearch_sessions" not in db.tables:
+        return []
+    return list(
+        get_table("deepsearch_sessions").find(
+            owner_kind=owner_kind,
+            owner_id=owner_id,
+            deleted_at=None,
+            order_by=["-created_at"],
+            _limit=limit,
+        )
+    )
+
+
 def add_deepsearch_message(
     uid: str, session_uid: str, role: str, content: str, citations: str = ""
 ) -> None:

devplacepy/database/engagement.py

@@ -116,6 +116,19 @@ def get_user_bookmarks(user_uid, target_type, target_uids):
     return {row["target_uid"] for row in rows}
 
 
+def get_user_notes(user_uid, target_type, target_uids):
+    if not user_uid or not target_uids or "notes" not in db.tables:
+        return {}
+    placeholders, params = _in_clause(target_uids)
+    params["u"] = user_uid
+    params["tt"] = target_type
+    rows = db.query(
+        f"SELECT target_uid, content FROM notes WHERE user_uid=:u AND target_type=:tt AND target_uid IN ({placeholders}) AND deleted_at IS NULL",
+        **params,
+    )
+    return {row["target_uid"]: row["content"] for row in rows}
+
+
 def get_polls_by_post_uids(post_uids, user=None):
     if not post_uids or "polls" not in db.tables:
         return {}

devplacepy/database/moderation.py

@@ -17,6 +17,7 @@ REPORTABLE_TARGETS: dict[str, str] = {
     "message": "messages",
     "quiz": "quizzes",
     "poll": "polls",
+    "battle": "opinion_wars",
     "award": "awards",
     "user": "users",
     "issue": "issue_tickets",
@@ -46,8 +47,19 @@ UNREPORTABLE_TABLES: dict[str, str] = {
     "votes": "engagement counters, carry no authored content",
     "reactions": "engagement counters, carry no authored content",
     "bookmarks": "private to the owner",
+    "notes": "private to the owner",
+    "skills": "private to the owner, never shown to others",
+    "communities": "administrator-only configuration, never shown to members",
+    "node_devices": "private to the owner, the label is never shown to others",
+    "node_settings": "private to the owner",
+    "node_usage_daily": "derived donation counters, carry no authored content",
+    "node_donors": "derived donation totals, carry no authored content",
+    "node_caller_daily": "derived daily request counters, carry no authored content",
+    "node_flags": "operational flags for administrators, carry no authored content",
     "follows": "relationship rows, carry no authored content",
     "poll_votes": "private ballots",
+    "opinion_war_fighters": "membership and damage counters, carry no authored content",
+    "opinion_war_events": "server-composed battle log rows, not authored content",
     "quiz_attempts": "private to the participant",
     "quiz_answers": "private to the participant",
     "sessions": "authentication state",
@@ -128,8 +140,13 @@ CONSENT_KINDS: dict[str, str] = {
         "Sharing your DevPlace credentials with software another member runs "
         "in a container"
     ),
+    "bandwidth_sharing": (
+        "Sharing part of your internet connection with the DevPlace community "
+        "so search can reach sites that block datacenters"
+    ),
 }
 CONSENT_STATES: tuple[str, ...] = ("granted", "withdrawn")
+MEMBER_ONLY_CONSENTS: frozenset[str] = frozenset({"bandwidth_sharing"})
 
 AGE_BANDS: tuple[str, ...] = ("under_min", "13_15", "16_17", "adult")
 
@@ -286,6 +303,23 @@ def consent_granted(owner_kind: str, owner_id: str, kind: str) -> bool:
     return bool(row and row.get("state") == "granted")
 
 
+def consent_granted_many(owner_kind: str, owner_ids, kind: str) -> set[str]:
+    unique = list(dict.fromkeys(owner_id for owner_id in owner_ids if owner_id))
+    latest: dict[str, str] = {}
+    for start in range(0, len(unique), 500):
+        placeholders, params = _in_clause(unique[start : start + 500])
+        for row in db.query(
+            f"SELECT owner_id, state FROM {CONSENTS_TABLE} WHERE owner_kind = :owner_kind "
+            f"AND kind = :kind AND deleted_at IS NULL AND owner_id IN ({placeholders}) "
+            "ORDER BY created_at, id",
+            owner_kind=owner_kind,
+            kind=kind,
+            **params,
+        ):
+            latest[row["owner_id"]] = row["state"]
+    return {owner_id for owner_id, state in latest.items() if state == "granted"}
+
+
 def set_consent(
     owner_kind: str, owner_id: str, kind: str, granted: bool, version: str = "1"
 ) -> dict | None:

devplacepy/database/notifications.py

@@ -8,6 +8,7 @@ from .soft_delete import soft_delete
 NOTIFICATION_TYPES = [
     {"key": "comment", "label": "Comments", "description": "Someone comments on your post"},
     {"key": "reply", "label": "Replies", "description": "Someone replies to your comment"},
+    {"key": "thread", "label": "Thread activity", "description": "Someone else comments on a post you've commented on"},
     {"key": "mention", "label": "Mentions", "description": "Someone mentions you with @username"},
     {"key": "vote", "label": "Upvotes", "description": "Someone ++'d your content"},
     {"key": "follow", "label": "Followers", "description": "Someone starts following you"},
@@ -19,9 +20,11 @@ NOTIFICATION_TYPES = [
     {"key": "harvest_stolen", "label": "Farm raids", "description": "Someone steals a ready build from your Code Farm"},
     {"key": "award", "label": "Awards", "description": "Someone gives you an award on your profile"},
     {"key": "quiz_attempt", "label": "Quiz attempts", "description": "Someone completes one of your quizzes"},
+    {"key": "battle", "label": "Opinion Wars", "description": "Lead changes, results and fight-ready alerts for battles you joined"},
     {"key": "workspace", "label": "Dev workspaces", "description": "Idle, quota, retention and moderation notices for your dev workspaces"},
     {"key": "moderation", "label": "Moderation", "description": "Acknowledgements for reports you file and decisions taken on your content"},
     {"key": "system", "label": "System alerts", "description": "Platform infrastructure alerts (e.g. the AI gateway going down)"},
+    {"key": "ai_quota_warning", "label": "AI quota warnings", "description": "You're approaching your daily Devii AI usage limit"},
 ]
 
 

devplacepy/database/pagination.py

@@ -18,6 +18,7 @@ def paginate(
     order=None,
     cursor_field="created_at",
     viewer_uid=None,
+    limit=PAGE_SIZE,
     **filters,
 ):
     order = order or ["-" + cursor_field]
@@ -30,9 +31,9 @@ def paginate(
             clauses.append(table.table.columns.user_uid.notin_(blocked))
     if before:
         clauses.append(table.table.columns[cursor_field] < before)
-    rows = list(table.find(*clauses, **filters, order_by=order, _limit=PAGE_SIZE + 1))
-    has_more = len(rows) > PAGE_SIZE
-    rows = rows[:PAGE_SIZE]
+    rows = list(table.find(*clauses, **filters, order_by=order, _limit=limit + 1))
+    has_more = len(rows) > limit
+    rows = rows[:limit]
     next_cursor = rows[-1][cursor_field] if has_more and rows else None
     return rows, next_cursor
 
@@ -64,6 +65,7 @@ def paginate_diverse(
     cursor_field="created_at",
     uid_key="user_uid",
     viewer_uid=None,
+    limit=PAGE_SIZE,
     **filters,
 ):
     rows, next_cursor = paginate(
@@ -73,6 +75,7 @@ def paginate_diverse(
         order=order,
         cursor_field=cursor_field,
         viewer_uid=viewer_uid,
+        limit=limit,
         **filters,
     )
     return interleave_by_author(rows, uid_key=uid_key), next_cursor

devplacepy/database/ranking.py

@@ -4,7 +4,7 @@ import os
 
 from .core import TTLCache, _in_clause, _now_iso, db, get_table
 from .users import get_users_by_uids
-from .soft_delete import soft_delete, soft_delete_in
+from .soft_delete import soft_delete_in
 
 
 VOTABLE_TARGETS: dict[str, str] = {
@@ -133,6 +133,26 @@ def update_target_stars(target_type: str, target_uid: str, net_stars: int) -> No
         get_table(table_name).update({"uid": target_uid, "stars": net_stars}, ["uid"])
 
 
+def _child_uids(table_name, parent_column, parent_uids, live_only=False):
+    if table_name not in db.tables:
+        return []
+    table = db[table_name]
+    if parent_column not in table.columns:
+        return []
+    clause = table.table.columns[parent_column].in_(parent_uids)
+    if live_only:
+        return [row["uid"] for row in table.find(clause, deleted_at=None)]
+    return [row["uid"] for row in table.find(clause)]
+
+
+def _delete_in(table_name, column, uids):
+    placeholders, params = _in_clause(uids)
+    with db:
+        db.query(
+            f"DELETE FROM {table_name} WHERE {column} IN ({placeholders})", **params
+        )
+
+
 def soft_delete_engagement(target_type: str, target_uids: list, deleted_by: str) -> None:
     uids = [uid for uid in (target_uids or []) if uid]
     if not uids:
@@ -145,11 +165,15 @@ def soft_delete_engagement(target_type: str, target_uids: list, deleted_by: str)
         "bookmarks", "target_uid", uids, deleted_by, stamp=stamp, target_type=target_type
     )
     if target_type == "post" and "polls" in db.tables:
-        for uid in uids:
-            for poll in db["polls"].find(post_uid=uid, deleted_at=None):
-                soft_delete("poll_votes", deleted_by, stamp=stamp, poll_uid=poll["uid"])
-                soft_delete("poll_options", deleted_by, stamp=stamp, poll_uid=poll["uid"])
-            soft_delete("polls", deleted_by, stamp=stamp, post_uid=uid)
+        poll_uids = _child_uids("polls", "post_uid", uids, live_only=True)
+        soft_delete_in("poll_votes", "poll_uid", poll_uids, deleted_by, stamp=stamp)
+        soft_delete_in("poll_options", "poll_uid", poll_uids, deleted_by, stamp=stamp)
+        soft_delete_in("polls", "post_uid", uids, deleted_by, stamp=stamp)
+    if target_type == "post" and "opinion_wars" in db.tables:
+        war_uids = _child_uids("opinion_wars", "post_uid", uids, live_only=True)
+        soft_delete_in("opinion_war_fighters", "war_uid", war_uids, deleted_by, stamp=stamp)
+        soft_delete_in("opinion_war_events", "war_uid", war_uids, deleted_by, stamp=stamp)
+        soft_delete_in("opinion_wars", "post_uid", uids, deleted_by, stamp=stamp)
 
 
 def delete_engagement(target_type: str, target_uids: list) -> None:
@@ -174,13 +198,21 @@ def delete_engagement(target_type: str, target_uids: list) -> None:
                 **params,
             )
     if target_type == "post" and "polls" in tables:
-        for uid in uids:
-            for poll in db["polls"].find(post_uid=uid):
-                if "poll_votes" in tables:
-                    db["poll_votes"].delete(poll_uid=poll["uid"])
-                if "poll_options" in tables:
-                    db["poll_options"].delete(poll_uid=poll["uid"])
-            db["polls"].delete(post_uid=uid)
+        poll_uids = _child_uids("polls", "post_uid", uids)
+        if poll_uids:
+            if "poll_votes" in tables:
+                _delete_in("poll_votes", "poll_uid", poll_uids)
+            if "poll_options" in tables:
+                _delete_in("poll_options", "poll_uid", poll_uids)
+        _delete_in("polls", "post_uid", uids)
+    if target_type == "post" and "opinion_wars" in tables:
+        war_uids = _child_uids("opinion_wars", "post_uid", uids)
+        if war_uids:
+            if "opinion_war_fighters" in tables:
+                _delete_in("opinion_war_fighters", "war_uid", war_uids)
+            if "opinion_war_events" in tables:
+                _delete_in("opinion_war_events", "war_uid", war_uids)
+        _delete_in("opinion_wars", "post_uid", uids)
 
 
 def get_target_owner_uid(target_type: str, target_uid: str) -> str | None:

devplacepy/database/schema.py

@@ -1,6 +1,7 @@
 # retoor <retoor@molodetz.nl>
 
-from .core import DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _drop_index, _index, _uid_index, db, defaultdict, get_table, logger
+from devplacepy.services.nodes.spec import SPEC as NODES_SPEC
+from .core import AQUALITY_NEWS_GRADING_MODEL, AQUALITY_NEWS_GRADING_URL, DEFAULT_CORRECTION_PROMPT, DEFAULT_MODIFIER_PROMPT, INTERNAL_GATEWAY_URL, _drop_index, _index, _uid_index, db, defaultdict, get_table, logger
 from .settings import get_setting, set_setting
 from .soft_delete import SOFT_DELETE_TABLES, ensure_soft_delete_columns
 from .ranking import _authors_cache
@@ -30,6 +31,54 @@ def migrate_bug_tables_to_issue_tables() -> None:
         logger.info("Dropped table %s after migration", source_name)
 
 
+_TUNNEL_STATUS_RANK = {
+    "active": 4,
+    "provisioning": 3,
+    "pending": 2,
+    "failed": 1,
+    "suspended": 0,
+}
+
+
+def _dedupe_tunnel_hostnames() -> None:
+    if "tunnels" not in db.tables:
+        return
+    table = get_table("tunnels")
+    if not table.has_column("hostname") or not table.has_column("uid"):
+        return
+    groups = list(
+        db.query(
+            "SELECT hostname FROM tunnels "
+            "WHERE hostname IS NOT NULL AND hostname != '' "
+            "GROUP BY hostname HAVING COUNT(*) > 1"
+        )
+    )
+    for group in groups:
+        hostname = group["hostname"]
+        dupes = list(table.find(hostname=hostname))
+        dupes.sort(
+            key=lambda r: (
+                _TUNNEL_STATUS_RANK.get(r.get("status") or "", -1),
+                r.get("deleted_at") is None,
+                r.get("created_at") or "",
+                r.get("id") or 0,
+            ),
+            reverse=True,
+        )
+        losers = [r["uid"] for r in dupes[1:]]
+        if not losers:
+            continue
+        with db:
+            for uid in losers:
+                db.query("DELETE FROM tunnels WHERE uid = :uid", uid=uid)
+        logger.warning(
+            "Removed %d duplicate tunnel row(s) for hostname %s, kept %s",
+            len(losers),
+            hostname,
+            dupes[0]["uid"],
+        )
+
+
 def init_db():
     tables = db.tables
     _index(db, "users", "idx_users_username", ["username"])
@@ -38,6 +87,7 @@ def init_db():
     _index(db, "users", "idx_users_role", ["role", "created_at"])
     _index(db, "users", "idx_users_last_seen", ["last_seen"])
     _index(db, "users", "idx_users_created_at", ["created_at"])
+    _ensure_users_uid()
     _index(db, "posts", "idx_posts_user_uid", ["user_uid"])
     _index(db, "posts", "idx_posts_created_at", ["created_at"])
     _index(db, "posts", "idx_posts_topic", ["topic"])
@@ -129,6 +179,8 @@ def init_db():
         ("content", ""),
         ("read", False),
         ("created_at", ""),
+        ("updated_at", ""),
+        ("client_id", ""),
     ):
         if not messages.has_column(column):
             messages.create_column_by_example(column, example)
@@ -143,6 +195,8 @@ def init_db():
         "idx_messages_conversation_rev",
         ["receiver_uid", "sender_uid"],
     )
+    _index(db, "messages", "idx_messages_updated_at", ["updated_at"])
+    _index(db, "messages", "idx_messages_dedupe", ["sender_uid", "client_id"])
     _index(db, "notifications", "idx_notifications_user", ["user_uid"])
     _index(db, "notifications", "idx_notifications_user_read", ["user_uid", "read"])
     push_registration = get_table("push_registration")
@@ -154,19 +208,38 @@ def init_db():
         ("key_auth", ""),
         ("key_p256dh", ""),
         ("token", ""),
+        ("client_id", ""),
+        ("environment", ""),
         ("created_at", ""),
+        ("registered_at", ""),
         ("deleted_at", ""),
     ):
         if not push_registration.has_column(column):
             push_registration.create_column_by_example(column, example)
     _index(db, "push_registration", "idx_push_registration_user", ["user_uid"])
     _index(db, "push_registration", "idx_push_registration_provider", ["provider"])
+    _index(
+        db,
+        "push_registration",
+        "idx_push_registration_client",
+        ["user_uid", "provider", "client_id"],
+    )
+    _index(
+        db,
+        "push_registration",
+        "idx_push_registration_token",
+        ["user_uid", "provider", "token"],
+    )
     if "push_registration" in db.tables:
         with db:
             db.query(
                 "UPDATE push_registration SET provider = 'webpush' "
                 "WHERE provider IS NULL OR provider = ''"
             )
+            db.query(
+                "UPDATE push_registration SET registered_at = created_at "
+                "WHERE registered_at IS NULL OR registered_at = ''"
+            )
     _index(db, "sessions", "idx_sessions_token", ["session_token"])
     projects = get_table("projects")
     for column, example in (
@@ -224,6 +297,22 @@ def init_db():
     _index(
         db, "project_files", "idx_project_files_parent", ["project_uid", "parent_path"]
     )
+    project_file_sync_state = get_table("project_file_sync_state")
+    for column, example in (
+        ("project_uid", ""),
+        ("path", ""),
+        ("db_epoch", 0.0),
+        ("fs_epoch", 0.0),
+    ):
+        if not project_file_sync_state.has_column(column):
+            project_file_sync_state.create_column_by_example(column, example)
+    _index(
+        db,
+        "project_file_sync_state",
+        "idx_project_file_sync_state_path",
+        ["project_uid", "path"],
+        unique=True,
+    )
     _index(db, "badges", "idx_badges_user", ["user_uid"])
     _index(db, "badges", "idx_badges_user_name", ["user_uid", "badge_name"])
     _drop_index(db, "idx_follows_follower")
@@ -286,6 +375,8 @@ def init_db():
     )
     _index(db, "bookmarks", "idx_bookmarks_user", ["user_uid"])
     _index(db, "bookmarks", "idx_bookmarks_target", ["target_type", "target_uid"])
+    _index(db, "notes", "idx_notes_user", ["user_uid"])
+    _index(db, "notes", "idx_notes_target", ["target_type", "target_uid"])
     _index(db, "polls", "idx_polls_post", ["post_uid"])
     _index(db, "poll_options", "idx_poll_options_poll", ["poll_uid"])
     _index(db, "poll_votes", "idx_poll_votes_poll", ["poll_uid"])
@@ -407,6 +498,8 @@ def init_db():
         ("uid", ""),
         ("token", ""),
         ("user_uid", ""),
+        ("scope", ""),
+        ("device_uid", ""),
         ("created_at", ""),
         ("expires_at", ""),
         ("used_at", ""),
@@ -510,7 +603,60 @@ def init_db():
         "idx_user_cust_scope",
         ["owner_kind", "owner_id", "scope", "lang"],
     )
+    gateway_usage_ledger = get_table("gateway_usage_ledger")
+    for column, example in (
+        ("created_at", ""),
+        ("owner_kind", ""),
+        ("owner_id", ""),
+        ("backend", ""),
+        ("endpoint", ""),
+        ("requested_model", ""),
+        ("model", ""),
+        ("status_code", 0),
+        ("success", 0),
+        ("error_category", ""),
+        ("upstream_latency_ms", 0.0),
+        ("gateway_overhead_ms", 0.0),
+        ("queue_wait_ms", 0.0),
+        ("connect_ms", 0.0),
+        ("total_latency_ms", 0.0),
+        ("prompt_tokens", 0),
+        ("completion_tokens", 0),
+        ("cache_hit_tokens", 0),
+        ("cache_miss_tokens", 0),
+        ("reasoning_tokens", 0),
+        ("total_tokens", 0),
+        ("tokens_per_second", 0.0),
+        ("context_window", 0),
+        ("context_utilization", 0.0),
+        ("cost_usd", 0.0),
+        ("input_cost_usd", 0.0),
+        ("output_cost_usd", 0.0),
+        ("native_cost", 0),
+        ("stream_requested", 0),
+        ("temperature", 0.0),
+        ("top_p", 0.0),
+        ("max_tokens", 0),
+        ("has_tools", 0),
+        ("retries_attempted", 0),
+        ("retry_succeeded", 0),
+        ("circuit_open", 0),
+        ("user_agent", ""),
+        ("app_reference", ""),
+        ("ttft_ms", 0.0),
+        ("inter_token_ms", 0.0),
+        ("provider", ""),
+        ("fallback_used_route", ""),
+    ):
+        if not gateway_usage_ledger.has_column(column):
+            gateway_usage_ledger.create_column_by_example(column, example)
     _index(db, "gateway_usage_ledger", "idx_gw_usage_time", ["created_at"])
+    _index(
+        db,
+        "gateway_usage_ledger",
+        "idx_gw_usage_provider_time",
+        ["provider", "created_at"],
+    )
     _index(
         db,
         "gateway_usage_ledger",
@@ -702,7 +848,9 @@ def init_db():
     _index(db, "instances", "idx_instances_workspace", ["is_workspace", "status"])
     _index(db, "instances", "idx_instances_workspace_owner", ["workspace_owner_uid"])
     _index(db, "instances", "idx_instances_tunnel_name", ["tunnel_name"])
-    _index(db, "tunnels", "idx_tunnels_hostname", ["hostname"])
+    _dedupe_tunnel_hostnames()
+    _drop_index(db, "idx_tunnels_hostname")
+    _index(db, "tunnels", "idx_tunnels_hostname", ["hostname"], unique=True)
     _index(db, "tunnels", "idx_tunnels_instance", ["instance_uid"])
     _index(db, "tunnels", "idx_tunnels_user", ["user_uid"])
     _index(db, "tunnels", "idx_tunnels_state", ["desired_state", "status"])
@@ -1107,6 +1255,12 @@ def init_db():
     _index(db, "deepsearch_sessions", "idx_deepsearch_sessions_owner", ["owner_kind", "owner_id"])
     _index(db, "deepsearch_sessions", "idx_deepsearch_sessions_status", ["status"])
     _index(db, "deepsearch_sessions", "idx_deepsearch_sessions_created", ["created_at"])
+    _index(
+        db,
+        "deepsearch_sessions",
+        "idx_deepsearch_sessions_owner_created",
+        ["owner_kind", "owner_id", "created_at"],
+    )
 
     deepsearch_messages = get_table("deepsearch_messages")
     for column, example in (
@@ -1352,6 +1506,7 @@ def init_db():
         ("ready_at", ""),
         ("watered_by", "[]"),
         ("raided_fraction", 0.0),
+        ("stop_after_harvest", 0),
         ("created_at", ""),
         ("updated_at", ""),
     ):
@@ -1580,6 +1735,98 @@ def init_db():
     )
     _index(db, "quiz_answers", "idx_quiz_answers_quiz", ["quiz_uid"])
 
+    opinion_wars = get_table("opinion_wars")
+    for column, example in (
+        ("uid", ""),
+        ("post_uid", ""),
+        ("user_uid", ""),
+        ("faction_a", ""),
+        ("faction_b", ""),
+        ("hp_a", 0),
+        ("hp_b", 0),
+        ("leader", ""),
+        ("status", "active"),
+        ("winner", ""),
+        ("created_at", ""),
+        ("ends_at", ""),
+        ("resolved_at", ""),
+        ("updated_at", ""),
+        ("deleted_at", ""),
+        ("deleted_by", ""),
+    ):
+        if not opinion_wars.has_column(column):
+            opinion_wars.create_column_by_example(column, example)
+    _index(db, "opinion_wars", "idx_opinion_wars_post", ["post_uid"], unique=True)
+    _index(db, "opinion_wars", "idx_opinion_wars_status_ends", ["status", "ends_at"])
+    _index(
+        db,
+        "opinion_wars",
+        "idx_opinion_wars_live_created",
+        ["created_at"],
+        where="deleted_at IS NULL",
+    )
+
+    opinion_war_fighters = get_table("opinion_war_fighters")
+    for column, example in (
+        ("uid", ""),
+        ("war_uid", ""),
+        ("user_uid", ""),
+        ("faction", ""),
+        ("hp_a", 0),
+        ("hp_b", 0),
+        ("fight_count", 0),
+        ("last_fight_at", ""),
+        ("cooldown_notified_at", ""),
+        ("created_at", ""),
+        ("updated_at", ""),
+        ("deleted_at", ""),
+        ("deleted_by", ""),
+    ):
+        if not opinion_war_fighters.has_column(column):
+            opinion_war_fighters.create_column_by_example(column, example)
+    _index(
+        db,
+        "opinion_war_fighters",
+        "idx_opinion_war_fighters_war_user",
+        ["war_uid", "user_uid"],
+        unique=True,
+    )
+    _index(
+        db,
+        "opinion_war_fighters",
+        "idx_opinion_war_fighters_war_faction",
+        ["war_uid", "faction"],
+    )
+    _index(
+        db,
+        "opinion_war_fighters",
+        "idx_opinion_war_fighters_last_fight",
+        ["last_fight_at"],
+    )
+
+    opinion_war_events = get_table("opinion_war_events")
+    for column, example in (
+        ("uid", ""),
+        ("war_uid", ""),
+        ("seq", 0),
+        ("kind", ""),
+        ("message", ""),
+        ("payload", ""),
+        ("actor_uid", ""),
+        ("created_at", ""),
+        ("deleted_at", ""),
+        ("deleted_by", ""),
+    ):
+        if not opinion_war_events.has_column(column):
+            opinion_war_events.create_column_by_example(column, example)
+    _index(
+        db,
+        "opinion_war_events",
+        "idx_opinion_war_events_war_seq",
+        ["war_uid", "seq"],
+        unique=True,
+    )
+
     _index(db, "posts", "idx_posts_user_created", ["user_uid", "created_at"])
     _index(
         db,
@@ -1606,6 +1853,10 @@ def init_db():
     )
 
     _ensure_moderation_tables()
+    _ensure_skills_table()
+    _ensure_nodes_tables()
+    _ensure_sync_tables()
+    _ensure_community_tables()
 
     for table in db.tables:
         _uid_index(db, table)
@@ -1613,6 +1864,10 @@ def init_db():
     for table in SOFT_DELETE_TABLES:
         ensure_soft_delete_columns(table)
 
+    from .sync import ensure_export_indexes
+
+    ensure_export_indexes()
+
     _index(
         db,
         "comments",
@@ -1706,8 +1961,8 @@ def init_db():
         news_defaults = {
             "news_grade_threshold": "7",
             "news_api_url": "https://news.app.molodetz.nl/api",
-            "news_ai_url": INTERNAL_GATEWAY_URL,
-            "news_ai_model": "molodetz",
+            "news_ai_url": AQUALITY_NEWS_GRADING_URL,
+            "news_ai_model": AQUALITY_NEWS_GRADING_MODEL,
         }
         for key, value in news_defaults.items():
             existing = db["site_settings"].find_one(key=key)
@@ -1739,6 +1994,7 @@ def init_db():
             "maintenance_message": "DevPlace is undergoing scheduled maintenance. Please check back shortly.",
             "customization_enabled": "1",
             "customization_js_enabled": "1",
+            "happy_404_enabled": "1",
             "audit_log_retention_days": "90",
             "statistics_tracking_enabled": "1",
             "docs_search_mode": "agent",
@@ -1759,6 +2015,14 @@ def init_db():
             "guidelines_version": "1",
             "ai_third_party_provider": "",
             "account_deletion_grace_hours": "24",
+            "nodes_enabled": "0",
+            "nodes_max_total": str(NODES_SPEC.limits.max_connected_nodes),
+            "nodes_daily_quota_user": str(NODES_SPEC.limits.default_daily_requests_user),
+            "nodes_daily_quota_guest": str(NODES_SPEC.limits.default_daily_requests_guest),
+            "nodes_cache_ttl_seconds": str(NODES_SPEC.limits.cache_ttl_seconds),
+            "nodes_social_via_nodes": "0",
+            "sync_enabled": "0",
+            "sync_origins": "",
         }
         for key, value in operational_defaults.items():
             existing = db["site_settings"].find_one(key=key)
@@ -1828,6 +2092,7 @@ def init_db():
     _index(db, "reactions", "idx_reactions_created_at", ["created_at"])
     _index(db, "votes", "idx_votes_created_at", ["created_at"])
     _index(db, "bookmarks", "idx_bookmarks_created_at", ["created_at"])
+    _index(db, "notes", "idx_notes_created_at", ["created_at"])
     _index(db, "badges", "idx_badges_created_at", ["created_at"])
     _index(db, "audit_log", "idx_audit_result_created", ["result", "created_at"])
     _index(db, "jobs", "idx_jobs_created_at", ["created_at"])
@@ -1906,6 +2171,450 @@ MODERATION_INDEXES: tuple[tuple[str, str, list[str]], ...] = (
 )
 
 
+SKILLS_COLUMNS: tuple[tuple[str, object], ...] = (
+    ("uid", ""),
+    ("user_uid", ""),
+    ("name", ""),
+    ("description", ""),
+    ("body", ""),
+    ("files", ""),
+    ("source_url", ""),
+    ("license", ""),
+    ("compatibility", ""),
+    ("metadata", ""),
+    ("allowed_tools", ""),
+    ("enabled", 1),
+    ("sha256", ""),
+    ("skipped", ""),
+    ("warnings", ""),
+    ("fetched_at", ""),
+    ("created_at", ""),
+    ("updated_at", ""),
+    ("deleted_at", ""),
+    ("deleted_by", ""),
+)
+
+
+def _ensure_skills_table() -> None:
+    table = get_table("skills")
+    for column, example in SKILLS_COLUMNS:
+        if not table.has_column(column):
+            table.create_column_by_example(column, example)
+    _index(db, "skills", "idx_skills_user_name", ["user_uid", "name"], unique=True)
+    _index(
+        db,
+        "skills",
+        "idx_skills_user_live",
+        ["user_uid", "enabled"],
+        where="deleted_at IS NULL",
+    )
+
+
+NODES_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
+    "node_devices": (
+        ("uid", ""),
+        ("user_uid", ""),
+        ("label", ""),
+        ("kind", ""),
+        ("secret_hash", ""),
+        ("secret_prefix", ""),
+        ("platform", ""),
+        ("client_version", ""),
+        ("protocol_version", 0),
+        ("status", ""),
+        ("suspended_until", ""),
+        ("suspend_reason", ""),
+        ("country", ""),
+        ("asn", ""),
+        ("asn_class", ""),
+        ("class_override", ""),
+        ("ip_hash", ""),
+        ("last_seen_at", ""),
+        ("created_at", ""),
+        ("updated_at", ""),
+        ("deleted_at", ""),
+        ("deleted_by", ""),
+    ),
+    "node_settings": (
+        ("uid", ""),
+        ("user_uid", ""),
+        ("enabled", 0),
+        ("bandwidth_mb_day", 0),
+        ("max_concurrent", 0),
+        ("active_from", ""),
+        ("active_to", ""),
+        ("timezone", ""),
+        ("allowed_platforms", ""),
+        ("pause_on_battery", 0),
+        ("public_display", 0),
+        ("created_at", ""),
+        ("updated_at", ""),
+        ("deleted_at", ""),
+        ("deleted_by", ""),
+    ),
+    "node_usage_daily": (
+        ("uid", ""),
+        ("device_uid", ""),
+        ("user_uid", ""),
+        ("day", ""),
+        ("requests_ok", 0),
+        ("requests_failed", 0),
+        ("bytes_up", 0),
+        ("bytes_down", 0),
+        ("cache_hits_served", 0),
+        ("integrity_failures", 0),
+        ("created_at", ""),
+        ("updated_at", ""),
+        ("deleted_at", ""),
+        ("deleted_by", ""),
+    ),
+    "node_donors": (
+        ("uid", ""),
+        ("user_uid", ""),
+        ("requests_ok", 0),
+        ("bytes_total", 0),
+        ("first_donated_at", ""),
+        ("last_donated_at", ""),
+        ("xp_awarded_today", 0),
+        ("xp_day", ""),
+        ("created_at", ""),
+        ("updated_at", ""),
+        ("deleted_at", ""),
+        ("deleted_by", ""),
+    ),
+    "node_requests": (
+        ("uid", ""),
+        ("caller_kind", ""),
+        ("caller_id", ""),
+        ("url", ""),
+        ("method", ""),
+        ("adapter", ""),
+        ("mode", ""),
+        ("status", ""),
+        ("device_uid", ""),
+        ("result_status", 0),
+        ("result_bytes", 0),
+        ("result_body", ""),
+        ("result_headers", ""),
+        ("truncated", 0),
+        ("error", ""),
+        ("device_asn_class", ""),
+        ("shadow_of", ""),
+        ("exclude_device", ""),
+        ("created_at", ""),
+        ("assigned_at", ""),
+        ("finished_at", ""),
+    ),
+    "node_counters": (
+        ("name", ""),
+        ("value", 0),
+        ("updated_at", ""),
+    ),
+    "node_circuits": (
+        ("key", ""),
+        ("failures", 0),
+        ("state", ""),
+        ("opened_at", 0.0),
+        ("updated_at", ""),
+    ),
+    "node_cache": (
+        ("key", ""),
+        ("method", ""),
+        ("url", ""),
+        ("status", 0),
+        ("headers", ""),
+        ("body", ""),
+        ("size", 0),
+        ("stored_at", ""),
+        ("expires_at", ""),
+    ),
+    "node_caller_daily": (
+        ("uid", ""),
+        ("caller_kind", ""),
+        ("caller_id", ""),
+        ("day", ""),
+        ("requests", 0),
+        ("created_at", ""),
+        ("updated_at", ""),
+        ("deleted_at", ""),
+        ("deleted_by", ""),
+    ),
+    "node_events": (
+        ("uid", ""),
+        ("device_uid", ""),
+        ("user_uid", ""),
+        ("kind", ""),
+        ("detail", ""),
+        ("created_at", ""),
+    ),
+    "node_flags": (
+        ("uid", ""),
+        ("device_uid", ""),
+        ("user_uid", ""),
+        ("kind", ""),
+        ("detail", ""),
+        ("status", ""),
+        ("resolved_by", ""),
+        ("resolved_at", ""),
+        ("created_at", ""),
+        ("updated_at", ""),
+        ("deleted_at", ""),
+        ("deleted_by", ""),
+    ),
+}
+
+
+def _ensure_nodes_tables() -> None:
+    for table_name, columns in NODES_COLUMNS.items():
+        table = get_table(table_name)
+        for column, example in columns:
+            if not table.has_column(column):
+                table.create_column_by_example(column, example)
+    _index(db, "node_devices", "idx_node_devices_user", ["user_uid", "deleted_at"])
+    _index(db, "node_devices", "idx_node_devices_status", ["status", "last_seen_at"])
+    _index(db, "node_devices", "idx_node_devices_prefix", ["secret_prefix"])
+    _index(db, "node_settings", "idx_node_settings_user", ["user_uid"], unique=True)
+    _index(db, "node_donors", "idx_node_donors_user", ["user_uid"], unique=True)
+    _index(
+        db,
+        "node_usage_daily",
+        "idx_node_usage_device_day",
+        ["device_uid", "day"],
+        unique=True,
+    )
+    _index(db, "node_usage_daily", "idx_node_usage_day", ["day"])
+    _index(db, "node_usage_daily", "idx_node_usage_user", ["user_uid", "day"])
+    _index(
+        db,
+        "node_requests",
+        "idx_node_requests_open",
+        ["status", "created_at"],
+        where="status IN ('queued','assigned')",
+    )
+    _index(db, "node_requests", "idx_node_requests_created", ["created_at"])
+    _index(db, "node_requests", "idx_node_requests_shadow", ["shadow_of"], where="shadow_of != ''")
+    _index(db, "node_counters", "idx_node_counters_name", ["name"], unique=True)
+    _index(db, "node_circuits", "idx_node_circuits_key", ["key"], unique=True)
+    _index(db, "node_cache", "idx_node_cache_key", ["key"], unique=True)
+    _index(db, "node_cache", "idx_node_cache_expires", ["expires_at"])
+    _index(
+        db,
+        "node_caller_daily",
+        "idx_node_caller_day",
+        ["caller_kind", "caller_id", "day"],
+        unique=True,
+    )
+    _index(db, "node_caller_daily", "idx_node_caller_day_only", ["day"])
+    _index(db, "node_events", "idx_node_events_created", ["created_at"])
+    _index(db, "node_events", "idx_node_events_device", ["device_uid", "created_at"])
+    _index(
+        db,
+        "node_flags",
+        "idx_node_flags_open",
+        ["status", "created_at"],
+        where="deleted_at IS NULL",
+    )
+
+
+SYNC_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
+    "sync_links": (
+        ("uid", ""),
+        ("origin", ""),
+        ("kind", ""),
+        ("origin_uid", ""),
+        ("local_uid", ""),
+        ("fingerprint", ""),
+        ("origin_updated_at", ""),
+        ("first_synced_at", ""),
+        ("synced_at", ""),
+        ("ack_at", ""),
+    ),
+    "sync_pending": (
+        ("uid", ""),
+        ("origin", ""),
+        ("kind", ""),
+        ("origin_uid", ""),
+        ("missing_kind", ""),
+        ("missing_uid", ""),
+        ("record", ""),
+        ("attempts", 0),
+        ("created_at", ""),
+        ("expires_at", ""),
+    ),
+    "sync_candidates": (
+        ("uid", ""),
+        ("origin", ""),
+        ("post_uid", ""),
+        ("record", ""),
+        ("comment_count", 0),
+        ("created_at", ""),
+        ("expires_at", ""),
+    ),
+    "sync_identities": (
+        ("uid", ""),
+        ("origin", ""),
+        ("source_name", ""),
+        ("source_key", ""),
+        ("our_name", ""),
+        ("our_key", ""),
+        ("style_fingerprint", ""),
+        ("style_samples", ""),
+        ("ai_generated", 0),
+        ("created_at", ""),
+    ),
+}
+
+
+def _ensure_sync_tables() -> None:
+    for table_name, columns in SYNC_COLUMNS.items():
+        table = get_table(table_name)
+        for column, example in columns:
+            if not table.has_column(column):
+                table.create_column_by_example(column, example)
+    _index(
+        db,
+        "sync_links",
+        "idx_sync_links_origin",
+        ["origin", "kind", "origin_uid"],
+        unique=True,
+    )
+    _index(db, "sync_links", "idx_sync_links_local", ["kind", "local_uid"])
+    _index(db, "sync_pending", "idx_sync_pending_record", ["origin", "kind", "origin_uid"], unique=True)
+    _index(db, "sync_pending", "idx_sync_pending_missing", ["origin", "missing_kind", "missing_uid"])
+    _index(db, "sync_pending", "idx_sync_pending_expires", ["expires_at"])
+    _index(db, "sync_candidates", "idx_sync_candidates_post", ["origin", "post_uid"], unique=True)
+    _index(db, "sync_candidates", "idx_sync_candidates_expires", ["expires_at"])
+    _index(
+        db,
+        "sync_identities",
+        "idx_sync_identities_source",
+        ["origin", "source_key"],
+        unique=True,
+    )
+    _index(db, "sync_identities", "idx_sync_identities_our", ["our_key"], unique=True)
+
+
+COMMUNITY_COLUMNS: dict[str, tuple[tuple[str, object], ...]] = {
+    "communities": (
+        ("uid", ""),
+        ("slug", ""),
+        ("name", ""),
+        ("origin", ""),
+        ("url", ""),
+        ("query", ""),
+        ("status", ""),
+        ("description", ""),
+        ("rules", ""),
+        ("language", ""),
+        ("favicon", ""),
+        ("user_count", 0),
+        ("schema", ""),
+        ("watermark", ""),
+        ("enabled", 1),
+        ("interval_seconds", 0),
+        ("next_run_at", ""),
+        ("last_run_at", ""),
+        ("last_status", ""),
+        ("last_error", ""),
+        ("run_count", 0),
+        ("failure_count", 0),
+        ("totals", ""),
+        ("learned_at", ""),
+        ("created_at", ""),
+        ("created_by", ""),
+        ("login_username", ""),
+        ("login_email", ""),
+        ("login_password", ""),
+        ("login_totp_secret", ""),
+        ("schema_previous", ""),
+        ("sync_options", ""),
+        ("claimed_at", ""),
+        ("deleted_at", ""),
+        ("deleted_by", ""),
+    ),
+    "community_runs": (
+        ("uid", ""),
+        ("community_uid", ""),
+        ("started_at", ""),
+        ("finished_at", ""),
+        ("status", ""),
+        ("dry_run", 0),
+        ("trigger", ""),
+        ("collected", ""),
+        ("pushed", ""),
+        ("errors", ""),
+        ("log", ""),
+        ("tally", ""),
+        ("verdict", ""),
+        ("headline", ""),
+        ("items_stored", 0),
+        ("items_dropped", 0),
+    ),
+    "community_run_items": (
+        ("uid", ""),
+        ("run_uid", ""),
+        ("community_uid", ""),
+        ("seq", 0),
+        ("rank", 0),
+        ("kind", ""),
+        ("outcome", ""),
+        ("reason", ""),
+        ("origin_uid", ""),
+        ("local_uid", ""),
+        ("label", ""),
+        ("source_url", ""),
+        ("detail", ""),
+        ("created_at", ""),
+    ),
+}
+
+
+def _ensure_community_tables() -> None:
+    for table_name, columns in COMMUNITY_COLUMNS.items():
+        table = get_table(table_name)
+        for column, example in columns:
+            if not table.has_column(column):
+                table.create_column_by_example(column, example)
+    _index(db, "communities", "idx_communities_slug", ["slug"], unique=True, where="deleted_at IS NULL")
+    _index(db, "communities", "idx_communities_due", ["status", "enabled", "next_run_at"])
+    _index(db, "community_runs", "idx_community_runs_community", ["community_uid", "started_at"])
+    _index(db, "community_run_items", "idx_community_items_run_rank", ["run_uid", "rank", "seq"])
+    _index(db, "community_run_items", "idx_community_items_run_outcome", ["run_uid", "outcome", "kind", "seq"])
+    _index(db, "community_run_items", "idx_community_items_community_rank", ["community_uid", "rank", "created_at DESC", "seq"])
+    _index(db, "community_run_items", "idx_community_items_community_outcome", ["community_uid", "outcome", "created_at DESC", "seq"])
+    _index(db, "community_run_items", "idx_community_items_run_seq", ["run_uid", "seq"], unique=True)
+    _ensure_ledger_search()
+
+
+LEDGER_SEARCH_TABLE = "community_run_items_fts"
+LEDGER_SEARCH_COLUMNS = ("label", "origin_uid", "detail")
+
+
+def _ensure_ledger_search() -> None:
+    try:
+        exists = bool(list(db.query("SELECT name FROM sqlite_master WHERE name = :name", name=LEDGER_SEARCH_TABLE)))
+        columns = ", ".join(LEDGER_SEARCH_COLUMNS)
+        new_columns = ", ".join(f"new.{column}" for column in LEDGER_SEARCH_COLUMNS)
+        old_columns = ", ".join(f"old.{column}" for column in LEDGER_SEARCH_COLUMNS)
+        with db:
+            db.query(
+                f"CREATE VIRTUAL TABLE IF NOT EXISTS {LEDGER_SEARCH_TABLE} USING fts5({columns}, "
+                "content='community_run_items', content_rowid='id', tokenize='trigram')"
+            )
+            db.query(
+                "CREATE TRIGGER IF NOT EXISTS community_run_items_fts_insert AFTER INSERT ON community_run_items BEGIN "
+                f"INSERT INTO {LEDGER_SEARCH_TABLE}(rowid, {columns}) VALUES (new.id, {new_columns}); END"
+            )
+            db.query(
+                "CREATE TRIGGER IF NOT EXISTS community_run_items_fts_delete AFTER DELETE ON community_run_items BEGIN "
+                f"INSERT INTO {LEDGER_SEARCH_TABLE}({LEDGER_SEARCH_TABLE}, rowid, {columns}) VALUES ('delete', old.id, {old_columns}); END"
+            )
+            if not exists:
+                db.query(f"INSERT INTO {LEDGER_SEARCH_TABLE}({LEDGER_SEARCH_TABLE}) VALUES ('rebuild')")
+    except Exception as exc:
+        logger.warning("Ledger full-text search is unavailable, the text filter will scan: %s", exc)
+
+
 def _ensure_moderation_tables() -> None:
     for table_name, columns in MODERATION_COLUMNS.items():
         table = get_table(table_name)
@@ -1953,6 +2662,12 @@ def migrate_ai_gateway_settings() -> None:
         if get_setting(key, "") == OLD_GATEWAY_URL:
             set_setting(key, INTERNAL_GATEWAY_URL)
             logger.info(f"Migrated {key} to the internal gateway")
+    if get_setting("news_ai_url", "") == INTERNAL_GATEWAY_URL:
+        set_setting("news_ai_url", AQUALITY_NEWS_GRADING_URL)
+        logger.info("Migrated news_ai_url to the free aquality grading model")
+    if get_setting("news_ai_model", "") in ("molodetz", ""):
+        set_setting("news_ai_model", AQUALITY_NEWS_GRADING_MODEL)
+        logger.info("Migrated news_ai_model to aquality")
     if get_setting("bot_model", "") == "deepseek-chat":
         set_setting("bot_model", "molodetz")
         logger.info("Migrated bot_model to molodetz")
@@ -1967,6 +2682,26 @@ def migrate_ai_gateway_settings() -> None:
     migrate_retired_image_gateway()
 
 
+def _ensure_users_uid() -> int:
+    if "users" not in db.tables:
+        return 0
+    users = get_table("users")
+    if not users.has_column("uid"):
+        users.create_column_by_example("uid", "")
+    import uuid_utils
+
+    updated = 0
+    for user in users.find():
+        if not user.get("uid"):
+            users.update(
+                {"id": user["id"], "uid": str(uuid_utils.uuid7())}, ["id"]
+            )
+            updated += 1
+    if updated:
+        logger.info("Backfilled uid for %s user(s)", updated)
+    return updated
+
+
 def backfill_api_keys() -> int:
     users = get_table("users")
     if not users.has_column("api_key"):
@@ -2021,6 +2756,10 @@ def backfill_api_keys() -> int:
         users.create_column_by_example("suspension_reason", "")
     if not users.has_column("deletion_requested_at"):
         users.create_column_by_example("deletion_requested_at", "")
+    if not users.has_column("active_conversation_uid"):
+        users.create_column_by_example("active_conversation_uid", "")
+    if not users.has_column("active_conversation_at"):
+        users.create_column_by_example("active_conversation_at", "")
     with db:
         db.query(
             "UPDATE users SET ai_modifier_enabled = 1 WHERE ai_modifier_enabled IS NULL"

devplacepy/database/skills.py

@@ -0,0 +1,141 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+from sqlalchemy.exc import IntegrityError
+
+from devplacepy.services.skills import limits
+
+from .core import _now_iso, db, get_table
+from .soft_delete import soft_delete
+
+
+class SkillLimitError(ValueError):
+    pass
+
+
+def count_skills(user_uid):
+    if "skills" not in db.tables:
+        return 0
+    rows = db.query(
+        "SELECT COUNT(*) AS c FROM skills WHERE user_uid=:u AND deleted_at IS NULL",
+        u=user_uid,
+    )
+    return next(iter(rows))["c"]
+
+
+def list_skills(user_uid):
+    if "skills" not in db.tables:
+        return []
+    return list(
+        db.query(
+            "SELECT * FROM skills WHERE user_uid=:u AND deleted_at IS NULL ORDER BY name",
+            u=user_uid,
+        )
+    )
+
+
+def get_skill_row(user_uid, name):
+    if "skills" not in db.tables:
+        return None
+    rows = db.query(
+        "SELECT * FROM skills WHERE user_uid=:u AND name=:n AND deleted_at IS NULL",
+        u=user_uid,
+        n=name,
+    )
+    return next(iter(rows), None)
+
+
+def enabled_skill_summaries(user_uid):
+    if "skills" not in db.tables:
+        return []
+    rows = db.query(
+        "SELECT name, description FROM skills "
+        "WHERE user_uid=:u AND enabled=1 AND deleted_at IS NULL ORDER BY name LIMIT :n",
+        u=user_uid,
+        n=limits.PROMPT_MAX_SKILLS,
+    )
+    return [(row["name"], row["description"]) for row in rows]
+
+
+def upsert_skill(user_uid, fields):
+    table = get_table("skills")
+    name = fields["name"]
+    existing = table.find_one(user_uid=user_uid, name=name)
+    now = _now_iso()
+    values = {**fields, "updated_at": now, "fetched_at": now}
+    if existing is None:
+        if count_skills(user_uid) >= limits.SKILLS_PER_USER_MAX:
+            raise SkillLimitError(
+                f"You can have at most {limits.SKILLS_PER_USER_MAX} skills; delete one first."
+            )
+        from devplacepy.utils import generate_uid
+
+        row = {
+            **values,
+            "uid": generate_uid(),
+            "user_uid": user_uid,
+            "enabled": 1,
+            "created_at": now,
+            "deleted_at": None,
+            "deleted_by": None,
+        }
+        try:
+            table.insert(row)
+            return get_skill_row(user_uid, name), "install"
+        except IntegrityError:
+            existing = table.find_one(user_uid=user_uid, name=name)
+    revived = existing.get("deleted_at") is not None
+    if revived and count_skills(user_uid) >= limits.SKILLS_PER_USER_MAX:
+        raise SkillLimitError(
+            f"You can have at most {limits.SKILLS_PER_USER_MAX} skills; delete one first."
+        )
+    update = {"id": existing["id"], **values, "deleted_at": None, "deleted_by": None}
+    if revived:
+        update["enabled"] = 1
+    table.update(update, ["id"])
+    return get_skill_row(user_uid, name), "update"
+
+
+def set_skill_enabled(user_uid, name, enabled):
+    row = get_skill_row(user_uid, name)
+    if row is None:
+        return None
+    get_table("skills").update(
+        {"id": row["id"], "enabled": 1 if enabled else 0, "updated_at": _now_iso()},
+        ["id"],
+    )
+    return get_skill_row(user_uid, name)
+
+
+def delete_skill(user_uid, name, deleted_by):
+    row = get_skill_row(user_uid, name)
+    if row is None:
+        return False
+    soft_delete("skills", deleted_by, uid=row["uid"])
+    return True
+
+
+def skill_view(row, *, detail=False):
+    view = {
+        "uid": row["uid"],
+        "name": row["name"],
+        "description": row["description"],
+        "enabled": bool(row.get("enabled")),
+        "source_url": row.get("source_url") or "",
+        "license": row.get("license") or "",
+        "compatibility": row.get("compatibility") or "",
+        "allowed_tools": row.get("allowed_tools") or "",
+        "sha256": row.get("sha256") or "",
+        "file_count": len(json.loads(row.get("files") or "{}")),
+        "fetched_at": row.get("fetched_at"),
+        "created_at": row.get("created_at"),
+        "updated_at": row.get("updated_at"),
+    }
+    if detail:
+        view["metadata"] = json.loads(row.get("metadata") or "{}")
+        view["body"] = row.get("body") or ""
+        view["files"] = json.loads(row.get("files") or "{}")
+        view["skipped"] = json.loads(row.get("skipped") or "[]")
+        view["warnings"] = json.loads(row.get("warnings") or "[]")
+    return view

devplacepy/database/soft_delete.py

@@ -16,6 +16,15 @@ SOFT_DELETE_TABLES = [
     "votes",
     "reactions",
     "bookmarks",
+    "notes",
+    "skills",
+    "communities",
+    "node_devices",
+    "node_settings",
+    "node_usage_daily",
+    "node_donors",
+    "node_flags",
+    "node_caller_daily",
     "follows",
     "poll_votes",
     "polls",
@@ -51,6 +60,9 @@ SOFT_DELETE_TABLES = [
     "quiz_options",
     "quiz_attempts",
     "quiz_answers",
+    "opinion_wars",
+    "opinion_war_fighters",
+    "opinion_war_events",
     "content_reports",
     "moderation_actions",
     "content_maturity",

devplacepy/database/sync.py

@@ -0,0 +1,360 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import text
+
+from devplacepy.utils.text import generate_uid
+
+from .core import _now_iso, db, get_table
+
+
+def get_link(origin: str, kind: str, origin_uid: str) -> dict | None:
+    return get_table("sync_links").find_one(
+        origin=origin, kind=kind, origin_uid=origin_uid
+    )
+
+
+def get_links_by_origin_uids(
+    origin: str, kind: str, origin_uids: list[str]
+) -> dict[str, dict]:
+    uids = [uid for uid in dict.fromkeys(origin_uids) if uid]
+    if not uids:
+        return {}
+    table = get_table("sync_links")
+    column = table.table.columns.origin_uid
+    rows = table.find(column.in_(uids), origin=origin, kind=kind)
+    return {row["origin_uid"]: row for row in rows}
+
+
+def claim_link(
+    origin: str,
+    kind: str,
+    origin_uid: str,
+    local_uid: str,
+    fingerprint: str,
+    origin_updated_at: str,
+) -> bool:
+    now = _now_iso()
+    with db:
+        result = db.executable.execute(
+            text(
+                "INSERT OR IGNORE INTO sync_links (uid, origin, kind, origin_uid, "
+                "local_uid, fingerprint, origin_updated_at, first_synced_at, synced_at) "
+                "VALUES (:uid, :origin, :kind, :origin_uid, :local_uid, :fingerprint, "
+                ":origin_updated_at, :now, :now)"
+            ),
+            {
+                "uid": generate_uid(),
+                "origin": origin,
+                "kind": kind,
+                "origin_uid": origin_uid,
+                "local_uid": local_uid,
+                "fingerprint": fingerprint,
+                "origin_updated_at": origin_updated_at,
+                "now": now,
+            },
+        )
+        return result.rowcount == 1
+
+
+def release_link(origin: str, kind: str, origin_uid: str, local_uid: str) -> None:
+    get_table("sync_links").delete(
+        origin=origin, kind=kind, origin_uid=origin_uid, local_uid=local_uid
+    )
+
+
+def advance_link(link_uid: str, fingerprint: str, origin_updated_at: str) -> None:
+    get_table("sync_links").update(
+        {
+            "uid": link_uid,
+            "fingerprint": fingerprint,
+            "origin_updated_at": origin_updated_at,
+            "synced_at": _now_iso(),
+        },
+        ["uid"],
+    )
+
+
+def username_taken(username: str) -> bool:
+    return get_table("users").find_one(username=username) is not None
+
+
+def count_links(origin: str) -> int:
+    return get_table("sync_links").count(origin=origin)
+
+
+def identity_map(origin: str) -> dict[str, dict]:
+    return {row["source_key"]: row for row in get_table("sync_identities").find(origin=origin)}
+
+
+def identity_by_source(origin: str, source_key: str) -> dict | None:
+    return get_table("sync_identities").find_one(origin=origin, source_key=source_key)
+
+
+def identity_name_taken(our_key: str) -> bool:
+    return get_table("sync_identities").find_one(our_key=our_key) is not None
+
+
+def claim_identity(
+    origin: str,
+    source_name: str,
+    source_key: str,
+    our_name: str,
+    style_fingerprint: str,
+    ai_generated: bool,
+) -> bool:
+    with db:
+        result = db.executable.execute(
+            text(
+                "INSERT OR IGNORE INTO sync_identities (uid, origin, source_name, source_key, "
+                "our_name, our_key, style_fingerprint, style_samples, ai_generated, created_at) "
+                "VALUES (:uid, :origin, :source_name, :source_key, :our_name, :our_key, "
+                ":style_fingerprint, '[]', :ai_generated, :now)"
+            ),
+            {
+                "uid": generate_uid(),
+                "origin": origin,
+                "source_name": source_name,
+                "source_key": source_key,
+                "our_name": our_name,
+                "our_key": our_name.lower(),
+                "style_fingerprint": style_fingerprint,
+                "ai_generated": int(ai_generated),
+                "now": _now_iso(),
+            },
+        )
+        return result.rowcount == 1
+
+
+def save_style_samples(identity_uid: str, samples_json: str) -> None:
+    get_table("sync_identities").update({"uid": identity_uid, "style_samples": samples_json}, ["uid"])
+
+
+PENDING_TABLE = "sync_pending"
+
+
+def park_pending(
+    origin: str,
+    kind: str,
+    origin_uid: str,
+    missing_kind: str,
+    missing_uid: str,
+    record_json: str,
+    *,
+    attempts: int,
+    ttl_days: int,
+    cap: int,
+) -> bool:
+    now = datetime.now(timezone.utc)
+    with db:
+        result = db.executable.execute(
+            text(
+                "INSERT INTO sync_pending (uid, origin, kind, origin_uid, missing_kind, missing_uid, "
+                "record, attempts, created_at, expires_at) "
+                "SELECT :uid, :origin, :kind, :origin_uid, :missing_kind, :missing_uid, :record, "
+                ":attempts, :now, :expires "
+                "WHERE (SELECT COUNT(*) FROM sync_pending WHERE origin = :origin) < :cap "
+                "ON CONFLICT(origin, kind, origin_uid) DO UPDATE SET "
+                "missing_kind = excluded.missing_kind, missing_uid = excluded.missing_uid, "
+                "record = excluded.record, attempts = excluded.attempts, expires_at = excluded.expires_at"
+            ),
+            {
+                "uid": generate_uid(),
+                "origin": origin,
+                "kind": kind,
+                "origin_uid": origin_uid,
+                "missing_kind": missing_kind,
+                "missing_uid": missing_uid,
+                "record": record_json,
+                "attempts": int(attempts),
+                "now": now.isoformat(),
+                "expires": (now + timedelta(days=ttl_days)).isoformat(),
+                "cap": int(cap),
+            },
+        )
+        return result.rowcount >= 1
+
+
+def pending_exists(origin: str, kind: str, origin_uid: str) -> bool:
+    return get_table(PENDING_TABLE).find_one(origin=origin, kind=kind, origin_uid=origin_uid) is not None
+
+
+def ready_pending(limit: int, origin: str = "") -> list[dict]:
+    clause = "AND p.origin = :origin " if origin else ""
+    rows = db.query(
+        "SELECT p.* FROM sync_pending p JOIN sync_links l ON l.origin = p.origin "
+        "AND l.kind = p.missing_kind AND l.origin_uid = p.missing_uid "
+        f"WHERE 1 = 1 {clause}ORDER BY p.created_at, p.id LIMIT :n",
+        n=int(limit),
+        **({"origin": origin} if origin else {}),
+    )
+    return [dict(row) for row in rows]
+
+
+def delete_pending(uid: str) -> int:
+    with db:
+        return db.executable.execute(text("DELETE FROM sync_pending WHERE uid = :uid"), {"uid": uid}).rowcount
+
+
+def expire_pending(max_attempts: int) -> int:
+    with db:
+        return db.executable.execute(
+            text("DELETE FROM sync_pending WHERE expires_at <= :now OR attempts >= :attempts"),
+            {"now": _now_iso(), "attempts": int(max_attempts)},
+        ).rowcount
+
+
+def pending_counts() -> dict[str, int]:
+    rows = db.query("SELECT origin, COUNT(*) AS n FROM sync_pending GROUP BY origin ORDER BY n DESC LIMIT 100")
+    return {row["origin"]: int(row["n"]) for row in rows}
+
+
+STAR_TABLES = {"post": "posts", "project": "projects"}
+
+
+def recompute_stars(target_type: str, target_uid: str) -> int | None:
+    table = STAR_TABLES.get(target_type)
+    if table is None:
+        return None
+    with db:
+        db.executable.execute(
+            text(
+                f"UPDATE {table} SET stars = (SELECT COALESCE(SUM(value), 0) FROM votes "
+                "WHERE target_uid = :uid AND target_type = :type AND deleted_at IS NULL) "
+                "WHERE uid = :uid"
+            ),
+            {"uid": target_uid, "type": target_type},
+        )
+    row = get_table(table).find_one(uid=target_uid)
+    return int(row["stars"] or 0) if row else None
+
+
+EXPORT_CHANGED: dict[str, str] = {
+    "users": "created_at",
+    "projects": "COALESCE(deleted_at, updated_at, created_at)",
+    "posts": "COALESCE(deleted_at, updated_at, created_at)",
+    "comments": "COALESCE(deleted_at, updated_at, created_at)",
+    "polls": "COALESCE(deleted_at, created_at)",
+    "follows": "COALESCE(deleted_at, created_at)",
+    "attachments": "COALESCE(deleted_at, created_at)",
+}
+
+
+def ensure_export_indexes() -> None:
+    from .core import _index
+
+    for table, expression in EXPORT_CHANGED.items():
+        if table in db.tables:
+            _index(db, table, f"idx_{table}_changed", [expression, "uid"])
+
+
+def deletion_rows(origin: str, kind: str, table: str, after_id: int, limit: int) -> list[dict]:
+    if table not in db.tables:
+        return []
+    soft = db[table].has_column("deleted_at")
+    columns = "t.deleted_at AS deleted_at, t.deleted_by AS deleted_by" if soft else "NULL AS deleted_at, NULL AS deleted_by"
+    deleted = "OR (t.deleted_at IS NOT NULL AND COALESCE(t.deleted_by, '') != :actor)" if soft else ""
+    rows = db.query(
+        f"SELECT l.id AS link_id, l.kind AS kind, l.origin_uid AS origin_uid, t.uid AS local_uid, {columns} "
+        f"FROM sync_links l LEFT JOIN {table} t ON t.uid = l.local_uid "
+        "WHERE l.origin = :origin AND l.kind = :kind AND l.id > :after "
+        "AND COALESCE(l.ack_at, '') = '' "
+        f"AND (t.uid IS NULL {deleted}) "
+        "ORDER BY l.id LIMIT :n",
+        origin=origin,
+        kind=kind,
+        after=int(after_id),
+        n=int(limit),
+        **({"actor": "sync"} if soft else {}),
+    )
+    return [dict(row) for row in rows]
+
+
+def acknowledge_links(origin: str, items: list[tuple[str, str]]) -> int:
+    stamp = _now_iso()
+    done = 0
+    with db:
+        for kind, origin_uid in items:
+            done += db.executable.execute(
+                text(
+                    "UPDATE sync_links SET ack_at = :now WHERE origin = :origin AND kind = :kind "
+                    "AND origin_uid = :origin_uid AND COALESCE(ack_at, '') = ''"
+                ),
+                {"now": stamp, "origin": origin, "kind": kind, "origin_uid": origin_uid},
+            ).rowcount
+    return done
+
+
+def rename_identity(identity_uid: str, our_name: str) -> bool:
+    try:
+        with db:
+            result = db.executable.execute(
+                text("UPDATE sync_identities SET our_name = :name, our_key = :key WHERE uid = :uid"),
+                {"name": our_name, "key": our_name.lower(), "uid": identity_uid},
+            )
+        return result.rowcount == 1
+    except Exception:
+        return False
+
+
+def linked_rows_containing(origin: str, kind: str, table: str, columns: tuple[str, ...], needle: str, limit: int) -> list[dict]:
+    pattern = "%" + needle.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
+    matches = " OR ".join(f"t.{column} LIKE :pattern ESCAPE '\\'" for column in columns)
+    selected = ", ".join(f"t.{column} AS {column}" for column in columns)
+    rows = db.query(
+        f"SELECT t.uid AS uid, {selected} FROM sync_links l JOIN {table} t ON t.uid = l.local_uid "
+        f"WHERE l.origin = :origin AND l.kind = :kind AND ({matches}) ORDER BY l.id LIMIT :limit",
+        origin=origin,
+        kind=kind,
+        pattern=pattern,
+        limit=int(limit),
+    )
+    return [dict(row) for row in rows]
+
+
+def load_candidates(origin: str) -> list[dict]:
+    expire_candidates()
+    return [dict(row) for row in get_table("sync_candidates").find(origin=origin, order_by=["id"])]
+
+
+def save_candidate(origin: str, post_uid: str, record_json: str, comment_count: int, *, ttl_days: int, cap: int) -> bool:
+    now = datetime.now(timezone.utc)
+    with db:
+        result = db.executable.execute(
+            text(
+                "INSERT INTO sync_candidates (uid, origin, post_uid, record, comment_count, created_at, expires_at) "
+                "SELECT :uid, :origin, :post_uid, :record, :count, :now, :expires "
+                "WHERE (SELECT COUNT(*) FROM sync_candidates WHERE origin = :origin) < :cap "
+                "ON CONFLICT(origin, post_uid) DO UPDATE SET record = excluded.record, "
+                "comment_count = excluded.comment_count"
+            ),
+            {
+                "uid": generate_uid(),
+                "origin": origin,
+                "post_uid": post_uid,
+                "record": record_json,
+                "count": int(comment_count),
+                "now": now.isoformat(),
+                "expires": (now + timedelta(days=ttl_days)).isoformat(),
+                "cap": int(cap),
+            },
+        )
+        return result.rowcount >= 1
+
+
+def delete_candidate(origin: str, post_uid: str) -> int:
+    with db:
+        return db.executable.execute(
+            text("DELETE FROM sync_candidates WHERE origin = :origin AND post_uid = :post_uid"),
+            {"origin": origin, "post_uid": post_uid},
+        ).rowcount
+
+
+def expire_candidates() -> int:
+    with db:
+        return db.executable.execute(
+            text("DELETE FROM sync_candidates WHERE expires_at <= :now"), {"now": _now_iso()}
+        ).rowcount

devplacepy/database/users.py

@@ -15,8 +15,6 @@ def get_users_by_uids(uids):
 
 
 _admins_cache = TTLCache(ttl=300, max_size=4)
-# The primary administrator must be an account that can actually authenticate, so scan a
-# few of the earliest admins and skip any that are soft-deleted or deactivated.
 PRIMARY_ADMIN_CANDIDATES = 50
 
 
@@ -32,6 +30,9 @@ def get_admin_uids():
         return list(cached)
     if "users" not in db.tables:
         return []
+    users = db["users"]
+    if "uid" not in users.columns or "role" not in users.columns:
+        return []
     rows = db.query("SELECT uid FROM users WHERE role = 'Admin'")
     uids = [row["uid"] for row in rows]
     _admins_cache.set("uids", uids)
@@ -92,6 +93,9 @@ def get_primary_admin_uid():
         return cached or None
     if "users" not in db.tables:
         return None
+    users = db["users"]
+    if "uid" not in users.columns or "role" not in users.columns:
+        return None
     rows = list(
         db.query(
             "SELECT * FROM users WHERE role = 'Admin' "
@@ -114,15 +118,18 @@ def search_users_by_username(q, *, exclude_uid=None, limit=10):
         return []
     if exclude_uid is not None:
         rows = db.query(
-            "SELECT uid, username FROM users WHERE username LIKE :q AND uid != :me LIMIT :limit",
+            "SELECT uid, username, avatar_seed FROM users WHERE username LIKE :q AND uid != :me LIMIT :limit",
             q=f"%{q}%",
             me=exclude_uid,
             limit=limit,
         )
     else:
         rows = db.query(
-            "SELECT uid, username FROM users WHERE username LIKE :q LIMIT :limit",
+            "SELECT uid, username, avatar_seed FROM users WHERE username LIKE :q LIMIT :limit",
             q=f"%{q}%",
             limit=limit,
         )
-    return [{"uid": r["uid"], "username": r["username"]} for r in rows]
+    return [
+        {"uid": r["uid"], "username": r["username"], "avatar_seed": r["avatar_seed"]}
+        for r in rows
+    ]

devplacepy/dependencies.py

@@ -1,9 +1,4 @@
 # retoor <retoor@molodetz.nl>
-"""
-Generic FastAPI dependency that accepts JSON or form-encoded data,
-validated against a Pydantic model.
-"""
-
 import json
 import logging
 from typing import Any, TypeVar, get_origin
@@ -17,21 +12,10 @@ logger = logging.getLogger(__name__)
 
 _TModel = TypeVar("_TModel", bound=BaseModel)
 
-# Container origins recognised as sequence fields that may receive
-# multiple values from form data.
 _SEQUENCE_ORIGINS = frozenset({list, set, tuple, frozenset})
 
 
 def _formdata_to_dict(form: FormData, model: type[BaseModel]) -> dict[str, Any]:
-    """Convert FormData to a dict suitable for Pydantic validation.
-
-    * Sequence-typed model fields collect every submitted value via
-      ``getlist()``; a lone empty string is dropped (browsers emit empty
-      hidden inputs by default).
-    * Scalar fields use ``get()`` (the last value).
-    * Fields absent from the form are omitted so that Pydantic applies
-      the model default.
-    """
     body: dict[str, Any] = {}
     for field_name, field_info in model.model_fields.items():
         origin = get_origin(field_info.annotation)
@@ -50,8 +34,6 @@ def _formdata_to_dict(form: FormData, model: type[BaseModel]) -> dict[str, Any]:
 
 
 class _JsonOrForm:
-    """Internal callable that parses JSON or form data and validates."""
-
     def __init__(self, model: type[BaseModel]):
         self.model = model
 
@@ -70,7 +52,6 @@ class _JsonOrForm:
                         status_code=400, detail="JSON body must be an object"
                     )
                 return self.model.model_validate(body)
-            # Default: form-encoded (multipart or url-encoded)
             try:
                 form = await request.form()
             except Exception as exc:
@@ -85,11 +66,4 @@ class _JsonOrForm:
 
 
 def json_or_form(model: type[_TModel]) -> _JsonOrForm:
-    """Dependency factory: accept JSON or form-encoded data for a Pydantic model.
-
-    Usage:
-        @router.post("/create")
-        async def create(data: Annotated[PostForm, Depends(json_or_form(PostForm))]):
-            ...
-    """
     return _JsonOrForm(model)

devplacepy/docs_api/_shared.py

@@ -3,6 +3,7 @@
 VOTE_TARGETS = ["post", "comment", "gist", "project", "quiz"]
 REACTION_TARGETS = ["post", "comment", "gist", "project", "quiz"]
 BOOKMARK_TARGETS = ["post", "gist", "project", "news", "quiz"]
+NOTE_TARGETS = ["post", "gist", "project", "news"]
 COMMENT_TARGETS = ["post", "project", "news", "issue", "gist", "quiz"]
 PROJECT_TYPES = ["game", "game_asset", "software", "mobile_app", "website"]
 GIST_LANGUAGES = [
@@ -38,6 +39,7 @@ def field(
     example="",
     description="",
     options=None,
+    nullable=False,
 ):
     spec = {
         "name": name,
@@ -46,6 +48,7 @@ def field(
         "required": required,
         "example": example,
         "description": description,
+        "nullable": nullable,
     }
     if options:
         spec["options"] = list(options)

devplacepy/docs_api/groups/__init__.py

@@ -22,6 +22,11 @@ from . import (
     admin,
     game,
     quizzes,
+    battles,
+    skills,
+    nodes,
+    sync,
+    community,
 )
 
 ORDERED_GROUPS = [
@@ -46,4 +51,9 @@ ORDERED_GROUPS = [
     admin.GROUP,
     game.GROUP,
     quizzes.GROUP,
+    battles.GROUP,
+    skills.GROUP,
+    nodes.GROUP,
+    sync.GROUP,
+    community.GROUP,
 ]

devplacepy/docs_api/groups/admin.py

@@ -265,6 +265,29 @@ four ways to sign requests.
                     ),
                 ],
             ),
+            endpoint(
+                id="admin-issues-planning",
+                method="GET",
+                path="/admin/issues/planning",
+                title="Ticket planning report",
+                summary=(
+                    "Admin page listing every open Gitea ticket so an admin can pick a subset "
+                    "and generate a grouped, ordered planning document for a coding agent."
+                ),
+                auth="admin",
+                interactive=False,
+                sample_response={
+                    "configured": True,
+                    "tickets": [
+                        {"number": 42, "title": "Fix login redirect", "labels": ["bug"]},
+                    ],
+                    "tickets_error": False,
+                },
+                notes=[
+                    "`configured` is false when the issue tracker (Gitea) has not been set up in Services yet - `tickets` is then empty.",
+                    "`tickets_error` is true when the tracker is configured but the live fetch failed - retry shortly.",
+                ],
+            ),
             endpoint(
                 id="admin-ai-usage",
                 method="GET",
@@ -300,7 +323,7 @@ four ways to sign requests.
                     ),
                 ],
                 notes=[
-                    "TTFT and inter-token latency are not reported: the gateway forwards non-streaming to the upstream."
+                    "TTFT and inter-token latency (latency.ttft_ms/latency.inter_token_ms) are measured only for calls that requested streaming (stream: true); a zero count means no streaming calls occurred in the window, not that the metric is unavailable."
                 ],
             ),
             endpoint(
@@ -522,6 +545,23 @@ four ways to sign requests.
                         "Allow signups.",
                         ["1", "0"],
                     ),
+                    field(
+                        "sync_enabled",
+                        "form",
+                        "enum",
+                        False,
+                        "0",
+                        "Allow community sync.",
+                        ["1", "0"],
+                    ),
+                    field(
+                        "sync_origins",
+                        "form",
+                        "string",
+                        False,
+                        "other.example.org",
+                        "Comma separated origins allowed to sync; empty allows any.",
+                    ),
                     field(
                         "maintenance_mode",
                         "form",
@@ -613,6 +653,85 @@ four ways to sign requests.
                 auth="admin",
                 destructive=True,
             ),
+            endpoint(
+                id="admin-gateway-page",
+                method="GET",
+                path="/admin/gateway",
+                title="Gateway routing dashboard",
+                summary="Admin HTML page for managing AI gateway providers and per-model routing.",
+                auth="admin",
+                interactive=False,
+            ),
+            endpoint(
+                id="admin-gateway-providers",
+                method="GET",
+                path="/admin/gateway/providers",
+                title="List AI gateway providers",
+                summary="List every configured upstream provider plus the default provider summary.",
+                auth="admin",
+                interactive=True,
+            ),
+            endpoint(
+                id="admin-gateway-provider-set",
+                method="POST",
+                path="/admin/gateway/providers",
+                title="Create or update an AI gateway provider",
+                summary="Save an upstream provider (base URL, model, and API key) by name. Pass name to update an existing provider.",
+                auth="admin",
+                params=[
+                    field("name", "json", "string", True, "openrouter", "Provider name; existing name updates in place."),
+                    field("base_url", "json", "string", True, "https://openrouter.ai/api/v1", "Upstream chat completions base URL."),
+                    field("model", "json", "string", True, "x-ai/grok-4.3", "Default model for this provider."),
+                    field("api_key", "json", "string", False, "", "Upstream API key. Blank keeps the current key."),
+                ],
+            ),
+            endpoint(
+                id="admin-gateway-provider-delete",
+                method="DELETE",
+                path="/admin/gateway/providers/{name}",
+                title="Delete an AI gateway provider",
+                summary="Delete a configured upstream provider by name.",
+                auth="admin",
+                destructive=True,
+                params=[
+                    field("name", "path", "string", True, "openrouter", "Provider name."),
+                ],
+            ),
+            endpoint(
+                id="admin-gateway-models",
+                method="GET",
+                path="/admin/gateway/models",
+                title="List AI gateway model routes",
+                summary="List every source-to-target model route plus the configured provider names.",
+                auth="admin",
+                interactive=True,
+            ),
+            endpoint(
+                id="admin-gateway-model-set",
+                method="POST",
+                path="/admin/gateway/models",
+                title="Create or update an AI gateway model route",
+                summary="Route a source model name to a target model, optionally on a specific provider. Pass source_model to update an existing route.",
+                auth="admin",
+                params=[
+                    field("source_model", "json", "string", True, "gpt-4", "Model name callers request."),
+                    field("target_model", "json", "string", True, "x-ai/grok-4.3", "Model actually sent upstream."),
+                    field("provider", "json", "string", False, "openrouter", "Provider name to route through. Blank = the default provider."),
+                    field("fallback_model", "json", "string", False, "molodetz-pro", "Another already-configured source_model of the same kind, tried once automatically when this route fails after its own retries are exhausted. Blank = no fallback."),
+                ],
+            ),
+            endpoint(
+                id="admin-gateway-model-delete",
+                method="DELETE",
+                path="/admin/gateway/models/{source_model}",
+                title="Delete an AI gateway model route",
+                summary="Delete a source-to-target model route by source model name.",
+                auth="admin",
+                destructive=True,
+                params=[
+                    field("source_model", "path", "string", True, "gpt-4", "Source model name."),
+                ],
+            ),
             endpoint(
                 id="admin-gateway-quota-rules",
                 method="GET",
@@ -975,5 +1094,190 @@ four ways to sign requests.
                 destructive=True,
                 sample_response={"ok": True, "redirect": "/admin/game"},
             ),
+            endpoint(
+                id="admin-workspaces-page",
+                method="GET",
+                path="/admin/workspaces",
+                title="Workspaces dashboard",
+                summary="Admin HTML page listing every dev workspace across all projects, with owner, project, and moderation flags.",
+                auth="admin",
+                interactive=False,
+            ),
+            endpoint(
+                id="admin-workspaces-data",
+                method="GET",
+                path="/admin/workspaces/data",
+                title="List workspaces",
+                summary="Every dev workspace across all projects plus open moderation flags, as JSON.",
+                auth="admin",
+                interactive=True,
+                sample_response={"workspaces": [], "flags": []},
+            ),
+            endpoint(
+                id="admin-workspaces-suspend",
+                method="POST",
+                path="/admin/workspaces/{uid}/suspend",
+                title="Suspend a workspace",
+                summary="Suspend a workspace with a reason shown to its owner. The owner is notified.",
+                auth="admin",
+                destructive=True,
+                params=[
+                    field("uid", "path", "string", True, "INSTANCE_UID", "Workspace instance uid."),
+                    field("reason", "form", "string", True, "Excessive resource usage", "Shown to the workspace owner."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-unsuspend",
+                method="POST",
+                path="/admin/workspaces/{uid}/unsuspend",
+                title="Unsuspend a workspace",
+                summary="Lift a suspension. The owner is notified the workspace is available again.",
+                auth="admin",
+                params=[
+                    field("uid", "path", "string", True, "INSTANCE_UID", "Workspace instance uid."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-stop",
+                method="POST",
+                path="/admin/workspaces/{uid}/stop",
+                title="Stop a workspace",
+                summary="Stop the workspace container. Files and tunnels are kept.",
+                auth="admin",
+                destructive=True,
+                params=[
+                    field("uid", "path", "string", True, "INSTANCE_UID", "Workspace instance uid."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-start",
+                method="POST",
+                path="/admin/workspaces/{uid}/start",
+                title="Start a workspace",
+                summary="Resume a stopped workspace container.",
+                auth="admin",
+                params=[
+                    field("uid", "path", "string", True, "INSTANCE_UID", "Workspace instance uid."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-delete",
+                method="POST",
+                path="/admin/workspaces/{uid}/delete",
+                title="Delete a workspace",
+                summary="Remove a workspace and its tunnels.",
+                auth="admin",
+                destructive=True,
+                params=[
+                    field("uid", "path", "string", True, "INSTANCE_UID", "Workspace instance uid."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-flag",
+                method="POST",
+                path="/admin/workspaces/{uid}/flag",
+                title="Flag a workspace",
+                summary="Raise a moderation flag on a workspace. The owner is notified.",
+                auth="admin",
+                params=[
+                    field("uid", "path", "string", True, "INSTANCE_UID", "Workspace instance uid."),
+                    field("kind", "form", "string", False, "manual", "Flag kind."),
+                    field("severity", "form", "string", False, "warn", "Flag severity."),
+                    field("detail", "form", "string", False, "", "Detail shown to the owner."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-flag-resolve",
+                method="POST",
+                path="/admin/workspaces/flags/{flag_uid}/resolve",
+                title="Resolve or dismiss a workspace flag",
+                summary="Set a moderation flag's status.",
+                auth="admin",
+                params=[
+                    field("flag_uid", "path", "string", True, "FLAG_UID", "Flag uid."),
+                    field("status", "query", "string", False, "resolved", "resolved or dismissed."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-editor",
+                method="POST",
+                path="/admin/workspaces/{uid}/editor",
+                title="Set a workspace owner's editor preferences",
+                summary="Change the workspace owner's editor preferences on their behalf, or reset them. Subject to admin seniority.",
+                auth="admin",
+                params=[
+                    field("uid", "path", "string", True, "INSTANCE_UID", "Workspace instance uid."),
+                    field("theme", "form", "string", False, "devplace-dark", "devplace-dark, devplace-light or system."),
+                    field("layout", "form", "string", False, "standard", "standard, terminal-focus or zen."),
+                    field("panel_preset", "form", "string", False, "tall", "short, normal, tall or maximized."),
+                    field("font_size", "form", "integer", False, "14", "Editor font size in pixels."),
+                    field("terminal_font_size", "form", "integer", False, "13", "Terminal font size in pixels."),
+                    field("zoom_level", "form", "integer", False, "0", "Window zoom, -5 to 5."),
+                    field("boot_agent", "form", "string", False, "dpc", "dpc or none."),
+                    field("boot_shell", "form", "integer", False, "1", "1 opens a shell on boot, 0 skips it."),
+                    field("window_mode", "form", "string", False, "tab", "tab, window or fullscreen."),
+                    field("window_width", "form", "integer", False, "1600", "Editor window width in pixels."),
+                    field("window_height", "form", "integer", False, "1000", "Editor window height in pixels."),
+                    field("reset", "form", "boolean", False, "false", "Drop every preference for this owner."),
+                ],
+            ),
+            endpoint(
+                id="admin-workspaces-quota",
+                method="POST",
+                path="/admin/workspaces/quota",
+                title="Set a user's workspace quota override",
+                summary="Set or update a per-user override of the workspace count/disk/egress/idle/retention/CPU/memory limits.",
+                auth="admin",
+                params=[
+                    field("owner_id", "form", "string", True, "USER_UID", "Target user uid."),
+                    field("label", "form", "string", False, "", "Optional admin-facing note."),
+                    field("max_workspaces", "form", "int", False, "0", "0 = use the site default."),
+                    field("max_tunnels", "form", "int", False, "0", "0 = use the site default."),
+                    field("disk_quota_mb", "form", "int", False, "0", "0 = use the site default."),
+                    field("egress_quota_mb", "form", "int", False, "0", "0 = use the site default."),
+                    field("idle_stop_minutes", "form", "int", False, "0", "0 = use the site default."),
+                    field("retention_days", "form", "int", False, "0", "0 = use the site default."),
+                    field("cpu_millicores", "form", "int", False, "0", "0 = use the site default."),
+                    field("memory_mb", "form", "int", False, "0", "0 = use the site default."),
+                ],
+            ),
+            endpoint(
+                id="admin-trash-list",
+                method="GET",
+                path="/admin/trash",
+                title="Trash",
+                summary="Admin HTML page listing soft-deleted rows for one table, with restore/purge controls per row.",
+                auth="admin",
+                interactive=True,
+                params=[
+                    field("table", "query", "string", False, "posts", "Trash table key (posts, comments, gists, projects, news, awards, quizzes, project_files, attachments)."),
+                    field("page", "query", "int", False, "1", "Page number."),
+                ],
+            ),
+            endpoint(
+                id="admin-trash-restore",
+                method="POST",
+                path="/admin/trash/{table}/{uid}/restore",
+                title="Restore a soft-deleted row",
+                summary="Restore every row soft-deleted under the same event timestamp as the given row (a whole delete cascade at once).",
+                auth="admin",
+                params=[
+                    field("table", "path", "string", True, "posts", "Trash table key."),
+                    field("uid", "path", "string", True, "ROW_UID", "Row uid."),
+                ],
+            ),
+            endpoint(
+                id="admin-trash-purge",
+                method="POST",
+                path="/admin/trash/{table}/{uid}/purge",
+                title="Purge a soft-deleted row",
+                summary="Permanently delete every row soft-deleted under the same event timestamp as the given row, unlinking any attachment/project-file blobs.",
+                auth="admin",
+                destructive=True,
+                params=[
+                    field("table", "path", "string", True, "posts", "Trash table key."),
+                    field("uid", "path", "string", True, "ROW_UID", "Row uid."),
+                ],
+            ),
         ],
 }

devplacepy/docs_api/groups/auth.py

@@ -57,7 +57,20 @@ as a `422` with the shape `{ "fields": {...}, "messages": [...] }`.
                     "Signup is refused below the platform minimum age (`moderation_minimum_age`).",
                     "Accepting records the terms, privacy and activity-recording consents; "
                     "third-party AI processing stays off until it is granted separately.",
+                    "JSON callers receive the new account's `email`, `api_key`, and "
+                    "`is_admin` flag in `data`, so a script can authenticate its next "
+                    "request immediately.",
                 ],
+                sample_response={
+                    "ok": True,
+                    "redirect": "/feed",
+                    "data": {
+                        "username": "alice",
+                        "email": "alice@example.com",
+                        "api_key": "API_KEY",
+                        "is_admin": False,
+                    },
+                },
             ),
             endpoint(
                 id="auth-login",
@@ -85,6 +98,22 @@ as a `422` with the shape `{ "fields": {...}, "messages": [...] }`.
                     field("remember_me", "form", "string", False, "on", "Send 'on' to extend the session to the remember-me lifetime."),
                     field("next", "form", "string", False, "", "Redirect target after login."),
                 ],
+                notes=[
+                    "JSON callers receive the account's `email`, `api_key`, and "
+                    "`is_admin` flag in `data`, so a script can trade one email/password "
+                    "login for a reusable key. Failed logins return 401 and never "
+                    "include the key.",
+                ],
+                sample_response={
+                    "ok": True,
+                    "redirect": "/feed",
+                    "data": {
+                        "username": "alice",
+                        "email": "alice@example.com",
+                        "api_key": "API_KEY",
+                        "is_admin": False,
+                    },
+                },
             ),
             endpoint(
                 id="auth-forgot-password",

devplacepy/docs_api/groups/battles.py

@@ -0,0 +1,149 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.opinionwar import rules
+
+from .._shared import endpoint, field
+
+FILTER_KEYS = ["active", "ended", "mine"]
+
+SAMPLE_WAR = {
+    "uid": "0198f2c0-2222-7aaa-8bbb-000000000002",
+    "post_uid": "0198f2c0-3333-7aaa-8bbb-000000000003",
+    "post_url": "/posts/8bbb000000000003-tabs-or-spaces",
+    "post_title": "Tabs or spaces?",
+    "faction_a": "Tabs",
+    "faction_b": "Spaces",
+    "hp_a": 12548,
+    "hp_b": 7362,
+    "pct_a": 63,
+    "pct_b": 37,
+    "leader": "a",
+    "fighter_count": 42,
+    "status": "active",
+    "winner": "",
+    "ends_at": "2026-08-27T12:00:00+00:00",
+    "ends_in": "2d 14h 32m",
+    "last_seq": 87,
+    "fight_cost": rules.FIGHT_COST_COINS,
+    "top_contributors": [
+        {"username": "code_warrior", "faction": "a", "hp": 982},
+    ],
+    "recent_events": [
+        {"seq": 87, "kind": "fight", "message": "code_warrior dealt 300 HP for Tabs", "faction": "a"},
+    ],
+    "viewer": {
+        "faction": "a",
+        "hp": 256,
+        "rank": 7,
+        "can_fight": True,
+        "next_fight_at": "",
+    },
+}
+
+GROUP = {
+    "slug": "battles",
+    "title": "Opinion Wars",
+    "intro": f"""
+# Opinion Wars
+
+An Opinion War is a week-long two-faction battle attached to a post. The creator names
+exactly two factions when creating the post (the `war_faction_a` / `war_faction_b` fields
+on `POST /posts/create`); from that moment the battle runs for exactly
+{rules.WAR_DURATION_DAYS} days.
+
+Any signed-in member joins one of the two factions and may **fight** once every
+{rules.FIGHT_COOLDOWN_HOURS} hours per battle. A fight costs {rules.FIGHT_COST_COINS}
+Code Farm coins and deals deterministic, level-weighted damage for the fighter's faction:
+`{rules.BASE_DAMAGE} + {rules.LEVEL_DAMAGE_STEP} * min(level, {rules.LEVEL_DAMAGE_CAP})`
+HP, so a level 1 member deals {rules.damage_for(1)} HP and the bonus caps at
+{rules.damage_for(rules.LEVEL_DAMAGE_CAP)} HP. There is no randomness. Switching factions
+is allowed at any time; damage already dealt stays with the faction it was dealt to.
+
+When the week is over the faction with more HP wins. Resolution is evaluated lazily on
+read (no background clock): the first read after the deadline freezes the totals, awards
+XP (participation for every fighter with at least one fight, a bonus for the winning
+side, a bonus for the single top damage dealer) and notifies every fighter. Equal totals
+are a draw with participation XP only.
+
+Every battle keeps an ordered event log (kinds `join`, `switch`, `fight`, `lead_change`,
+`result`) replayable with the `after` cursor; live frames are also published on the
+pub/sub topic `public.battle.{{uid}}`.
+
+All endpoints negotiate HTML or JSON. POST bodies are form encoded. Action POSTs answer
+`{{"ok": true, "redirect": "...", "data": {{...}}}}`; a refused action (cooldown, missing
+coins, ended battle) answers `400` as `{{"error": {{"status": 400, "message": "..."}}}}`.
+""",
+    "endpoints": [
+        endpoint(
+            id="battles-list",
+            method="GET",
+            path="/battles",
+            title="Battle listing",
+            summary="Opinion Wars with HP totals, filter counts and the viewer's faction state.",
+            auth="public",
+            negotiation=True,
+            params=[
+                field("search", "query", "string", False, "tabs", "Match a faction name or the creator's username."),
+                field("filter", "query", "enum", False, "active", "Which battles to list.", options=FILTER_KEYS),
+                field("page", "query", "integer", False, "1", "1-based page number."),
+            ],
+            sample_response={"battles": [SAMPLE_WAR], "counts": {"active": 3, "ended": 12, "mine": 1}},
+        ),
+        endpoint(
+            id="battles-get",
+            method="GET",
+            path="/battles/{uid}",
+            title="Battle state",
+            summary="One battle's full serialized state, resolving it first when its week is over.",
+            auth="public",
+            params=[
+                field("uid", "path", "string", True, SAMPLE_WAR["uid"], "Battle uid."),
+            ],
+            sample_response=SAMPLE_WAR,
+        ),
+        endpoint(
+            id="battles-events",
+            method="GET",
+            path="/battles/{uid}/events",
+            title="Battle events",
+            summary="The ordered battle event log, replayable incrementally with the after cursor.",
+            auth="public",
+            params=[
+                field("uid", "path", "string", True, SAMPLE_WAR["uid"], "Battle uid."),
+                field("after", "query", "integer", False, "0", "Return only events with a seq greater than this."),
+                field("limit", "query", "integer", False, "500", "Maximum events to return."),
+            ],
+            sample_response={"events": SAMPLE_WAR["recent_events"], "status": "active"},
+        ),
+        endpoint(
+            id="battles-join",
+            method="POST",
+            path="/battles/{uid}/join",
+            title="Join or switch faction",
+            summary="Join faction a or b, or switch an existing fighter to the other side.",
+            auth="user",
+            encoding="form",
+            params=[
+                field("uid", "path", "string", True, SAMPLE_WAR["uid"], "Battle uid."),
+                field("faction", "form", "enum", True, "a", "Which side to join or switch to.", options=["a", "b"]),
+            ],
+            sample_response={"ok": True, "redirect": SAMPLE_WAR["post_url"], "data": {"war": SAMPLE_WAR}},
+        ),
+        endpoint(
+            id="battles-fight",
+            method="POST",
+            path="/battles/{uid}/fight",
+            title="Fight",
+            summary=(
+                f"Spend {rules.FIGHT_COST_COINS} Code Farm coins and deal level-weighted HP damage "
+                f"for your faction. Once per {rules.FIGHT_COOLDOWN_HOURS} hours per battle."
+            ),
+            auth="user",
+            encoding="form",
+            params=[
+                field("uid", "path", "string", True, SAMPLE_WAR["uid"], "Battle uid."),
+            ],
+            sample_response={"ok": True, "redirect": SAMPLE_WAR["post_url"], "data": {"war": SAMPLE_WAR, "damage": 300}},
+        ),
+    ],
+}

devplacepy/docs_api/groups/community.py

@@ -0,0 +1,233 @@
+# retoor <retoor@molodetz.nl>
+
+from .._shared import endpoint, field
+
+SLUG = field("slug", "path", "string", True, "other-example-org", "Community slug, as listed by GET /admin/communities.")
+ACCOUNT = [
+    field("login_username", "form", "string", False, "bot", "Username of an account on the community, for a login-protected community."),
+    field("login_email", "form", "string", False, "bot@example.com", "Email of that account, when the community logs in by email."),
+    field("login_password", "form", "string", False, "", "Password of that account. Stored for the collectors and never returned by any endpoint."),
+    field("login_totp_secret", "form", "string", False, "", "Authenticator (TOTP) secret of that account when it uses two-factor codes. Stored for the collectors and never returned by any endpoint."),
+]
+
+ITEM_SAMPLE = {
+    "uid": "COMMUNITY_UID",
+    "slug": "other-example-org",
+    "name": "Other Example",
+    "origin": "other.example.org",
+    "url": "https://other.example.org/",
+    "status": "ready",
+    "enabled": True,
+    "interval_seconds": 86400,
+    "run_count": 3,
+    "failure_count": 0,
+    "last_run_at": "2026-10-14T10:00:00+00:00",
+    "next_run_at": "2026-10-15T10:00:00+00:00",
+    "has_login": False,
+}
+
+GROUP = {
+    "slug": "community",
+    "title": "Community discovery",
+    "admin": True,
+    "intro": """
+# Community discovery
+
+Administrators add an external community by name or address. The system searches for it, detects
+its feeds (RSS, Atom, JSON or HTML lists, with JavaScript rendering when a page needs it), learns
+the structure once, records the community's description, rules and size, generates collectors and
+then runs them on a schedule, pushing users, posts and comments through the
+[community sync API](/docs/sync.html). Members never see these communities.
+
+- **Statuses.** `pending`, `learning`, `ready`, `login_required`, `failed` and `failing` (five failed
+  runs in a row: the community is disabled and needs a human).
+- **Login-protected communities.** Give `login_username` or `login_email` plus `login_password`; the
+  system finds the login form (or, for a page that needs JavaScript, drives it in the browser), logs
+  in before discovery and before every run. The password is never returned.
+- **Dry run.** `dry_run=true` on a run validates and logs everything and writes nothing.
+- **Sync log.** Every run records a verdict, exact counts per outcome and one row per record that needs an explanation (created, updated, skipped, rejected, failed, warnings, errors) with the reason in plain words. Read it with the sync log endpoint, per run or across the most recent runs.
+- **Asynchronous.** Adding, relearning and running return at once; read the result from the detail
+  endpoint.
+- **AI assistance.** For HTML pages whose structure the heuristics cannot read, the internal AI model
+  proposes selectors, and a proposal is only used when it matches the sampled pages.
+
+Every endpoint follows the shared [Conventions & Errors](/docs/conventions.html).
+""",
+    "endpoints": [
+        endpoint(
+            id="community-list",
+            method="GET",
+            path="/admin/communities",
+            title="List communities",
+            summary="Every community with status, schedule and last run, plus the state of the sync switches.",
+            auth="admin",
+            interactive=True,
+            sample_response={"items": [ITEM_SAMPLE], "sync_enabled": True, "service_enabled": True, "service_dry_run": True},
+        ),
+        endpoint(
+            id="community-add",
+            method="POST",
+            path="/admin/communities",
+            title="Add a community",
+            summary="Starts discovery in the background and redirects to the community.",
+            auth="admin",
+            encoding="form",
+            interactive=False,
+            params=[field("query", "form", "string", True, "https://forum.example.com", "Community name or address."), *ACCOUNT],
+            sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"},
+        ),
+        endpoint(
+            id="community-detail",
+            method="GET",
+            path="/admin/communities/{slug}",
+            title="Community detail",
+            summary="Metadata, learned feeds, totals and recent runs with their logs.",
+            auth="admin",
+            interactive=True,
+            params=[SLUG],
+            sample_response={
+                "community": ITEM_SAMPLE,
+                "rules": "Be kind.",
+                "totals": {"posts_created": 120},
+                "feeds": [{"id": "feed1", "type": "json_api", "url": "https://other.example.org/api/posts", "comments": "inline", "pagination": "page"}],
+                "runs": [{"status": "ok", "dry_run": 0, "trigger": "service", "collected": {"users": 4, "posts": 20, "comments": 31}}],
+                "busy": False,
+            },
+        ),
+        endpoint(
+            id="community-schema",
+            method="GET",
+            path="/admin/communities/{slug}/schema",
+            title="Learned schema",
+            summary="The complete learned schema of a community as JSON, editable by hand.",
+            auth="admin",
+            params=[SLUG],
+            sample_response={"community": ITEM_SAMPLE, "schema_text": "{...}", "has_previous": False, "errors": [], "notice": ""},
+        ),
+        endpoint(
+            id="community-schema-save",
+            method="POST",
+            path="/admin/communities/{slug}/schema",
+            title="Replace the learned schema",
+            summary="Validate a schema, check it against a small live sample, and only then store it. The previous schema is kept.",
+            auth="admin",
+            encoding="form",
+            destructive=True,
+            params=[SLUG, field("schema", "form", "string", True, "{...}", "The complete schema as a JSON object string.")],
+            sample_response={"ok": True, "redirect": "/admin/communities/SLUG/schema", "data": {"posts": 5, "comments": 12}},
+            notes=["`400` lists every structural problem; `422` means the live sample collected no post. Nothing is stored in either case. Audited as `admin.community.schema`."],
+        ),
+        endpoint(
+            id="community-schema-restore",
+            method="POST",
+            path="/admin/communities/{slug}/schema/restore",
+            title="Restore the previous schema",
+            summary="Swap back to the schema that was replaced by the last save.",
+            auth="admin",
+            encoding="form",
+            params=[SLUG],
+            sample_response={"ok": True, "redirect": "/admin/communities/SLUG/schema"},
+            notes=["`409` when there is no previous schema. Audited as `admin.community.schema_restore`."],
+        ),
+        endpoint(
+            id="community-ledger",
+            method="GET",
+            path="/admin/communities/{slug}/ledger",
+            title="Sync log",
+            summary="Every record a run handled, with the result and the reason, problems first. Without run it spans the most recent runs.",
+            auth="admin",
+            interactive=True,
+            params=[
+                SLUG,
+                field("run", "query", "string", False, "RUN_UID", "A run uid from the community detail. Empty lists the most recent runs."),
+                field("kind", "query", "string", False, "posts", "users, posts, comments or feed."),
+                field("outcome", "query", "string", False, "rejected", "created, updated, unchanged, stale, skipped, warning, rejected, failed or error."),
+                field("reason", "query", "string", False, "content_refused", "A reason key from options.reason."),
+                field("q", "query", "string", False, "", "Search the title, name, origin id and detail."),
+                field("page", "query", "integer", False, "1", "Page number, 50 records per page, at most 200 pages."),
+            ],
+            sample_response={
+                "community": ITEM_SAMPLE,
+                "scope": "run",
+                "run": {"uid": "RUN_UID", "status": "partial", "dry_run": False, "verdict": "attention", "headline": "Synced, with 1 rejected."},
+                "verdict": {"level": "attention", "title": "Look at this", "headline": "Synced, with 1 rejected."},
+                "tiles": [{"key": "created", "label": "Created", "tone": "good", "count": 99, "listed": 99}, {"key": "rejected", "label": "Rejected", "tone": "bad", "count": 1, "listed": 1}],
+                "items": [
+                    {
+                        "kind": "posts",
+                        "outcome": "rejected",
+                        "reason": "content_refused",
+                        "reason_title": "Content filter",
+                        "label": "A post the filter refused",
+                        "origin_uid": "h:4215bd02760d2fa7415c3a7952b0966f",
+                        "detail": "",
+                    }
+                ],
+                "pagination": {"page": 1, "per_page": 50, "total": 1, "total_pages": 1, "capped": False},
+                "filters": {"outcome": "rejected"},
+                "options": {"kind": ["posts"], "reason": [{"value": "content_refused", "title": "Content filter"}]},
+            },
+        ),
+        endpoint(
+            id="community-run",
+            method="POST",
+            path="/admin/communities/{slug}/run",
+            title="Run now",
+            summary="Collects and pushes in the background. A live run needs the sync_enabled setting.",
+            auth="admin",
+            encoding="form",
+            interactive=False,
+            params=[SLUG, field("dry_run", "form", "boolean", False, "false", "Validate and log, write nothing, keep the watermark.")],
+            sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"},
+        ),
+        endpoint(
+            id="community-relearn",
+            method="POST",
+            path="/admin/communities/{slug}/relearn",
+            title="Relearn",
+            summary="Learns the structure again with the stored account. A failed relearn keeps the working schema.",
+            auth="admin",
+            encoding="form",
+            interactive=False,
+            params=[SLUG],
+            sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"},
+        ),
+        endpoint(
+            id="community-update",
+            method="POST",
+            path="/admin/communities/{slug}/update",
+            title="Update schedule or account",
+            summary="Changes the interval, the enabled flag and the stored account. An empty password keeps the stored one.",
+            auth="admin",
+            encoding="form",
+            interactive=False,
+            params=[
+                SLUG,
+                field("interval_seconds", "form", "integer", True, "86400", "Seconds between runs, 900 to 2592000."),
+                field("enabled", "form", "boolean", False, "true", "Whether the scheduled service runs this community."),
+                *ACCOUNT,
+                field("clear_login", "form", "boolean", False, "false", "Remove the stored account."),
+                field("alias", "form", "boolean", False, "false", "Store authors under permanent pseudonyms and rewrite every mention. Refused once the origin has records under original names."),
+                field("translate_to", "form", "string", False, "en", "Language code to translate collected posts and comments into; empty turns it off."),
+                field("sample_percent", "form", "integer", False, "100", "Percentage of new posts accepted, 1 to 100, chosen deterministically per post."),
+                field("min_comments", "form", "integer", False, "0", "Minimum comments collected in the run for a new post to be accepted."),
+                field("topics", "form", "string", False, "devlog, rant", "Comma separated topic allow-list; empty accepts all."),
+                field("jitter_percent", "form", "integer", False, "0", "Randomise the next run time by up to this percentage of the interval, 0 to 50."),
+                field("collect_attachments", "form", "boolean", False, "false", "Download the images and PDFs linked from posts and comments (at most 4 per post, 40 and 25 MB per run) and store them as attachments."),
+            ],
+            sample_response={"ok": True, "redirect": "/admin/communities/other-example-org"},
+        ),
+        endpoint(
+            id="community-delete",
+            method="POST",
+            path="/admin/communities/{slug}/delete",
+            title="Delete a community",
+            summary="Moves the community to the trash and stops its collectors. Content already synchronised stays.",
+            auth="admin",
+            encoding="form",
+            interactive=False,
+            params=[SLUG],
+            sample_response={"ok": True, "redirect": "/admin/communities"},
+        ),
+    ],
+}

devplacepy/docs_api/groups/content.py

@@ -62,6 +62,75 @@ four ways to sign requests.
                         "Search post title, content, and author username.",
                     ),
                     field("before", "query", "string", False, "", "Pagination cursor."),
+                    field(
+                        "items[].poll",
+                        "response",
+                        "object",
+                        description="Present only when the post has a poll attached.",
+                        nullable=True,
+                    ),
+                    field(
+                        "items[].war",
+                        "response",
+                        "object",
+                        description="Present only when an Opinion War is running on the post.",
+                        nullable=True,
+                    ),
+                    field(
+                        "items[].project_link",
+                        "response",
+                        "object",
+                        description="Present only when the post is attached to a project.",
+                        nullable=True,
+                    ),
+                    field(
+                        "items[].maturity",
+                        "response",
+                        "string",
+                        description="Content maturity label; absent for unrated posts.",
+                        nullable=True,
+                    ),
+                    field(
+                        "items[].my_vote",
+                        "response",
+                        "integer",
+                        description="Viewer's own vote on the post, defaults to 0 (never null).",
+                        nullable=False,
+                    ),
+                    field(
+                        "items[].comment_count",
+                        "response",
+                        "integer",
+                        description="Total comment count, defaults to 0 (never null).",
+                        nullable=False,
+                    ),
+                ],
+            ),
+            endpoint(
+                id="topics-hub",
+                method="GET",
+                path="/topics",
+                title="Browse topics",
+                summary="The topics hub - every post topic with its live post count, linking to its own crawlable listing page.",
+                auth="public",
+                interactive=True,
+            ),
+            endpoint(
+                id="topics-list",
+                method="GET",
+                path="/topics/{topic}",
+                title="Browse one topic",
+                summary=(
+                    "A single topic's post listing, on its own permanent, crawlable URL (unlike /feed?topic=, "
+                    "whose canonical collapses back to /feed). Same author-interleaved pagination as the feed."
+                ),
+                auth="public",
+                interactive=True,
+                params=[
+                    field(
+                        "topic", "path", "enum", True, "devlog", "Topic key.", TOPICS
+                    ),
+                    field("before", "query", "string", False, "", "Pagination cursor."),
                 ],
             ),
             endpoint(
@@ -117,6 +186,22 @@ four ways to sign requests.
                         "",
                         "Repeat the field for each poll option, or send a single newline- or comma-separated string (2-6 options).",
                     ),
+                    field(
+                        "war_faction_a",
+                        "form",
+                        "string",
+                        False,
+                        "",
+                        "Optional Opinion War faction A name (max 30 chars). Both faction names start the week-long battle.",
+                    ),
+                    field(
+                        "war_faction_b",
+                        "form",
+                        "string",
+                        False,
+                        "",
+                        "Optional Opinion War faction B name (max 30 chars). Must differ from faction A.",
+                    ),
                 ],
                 notes=["Returns a `302` redirect to `/posts/{slug}` on success."],
             ),
@@ -136,7 +221,49 @@ four ways to sign requests.
                         True,
                         "POST_SLUG",
                         "Slug or UID of the post.",
-                    )
+                    ),
+                    field(
+                        "post.title",
+                        "response",
+                        "string",
+                        description="Optional title, blank when the author posted without one.",
+                        nullable=True,
+                    ),
+                    field(
+                        "post.image",
+                        "response",
+                        "string",
+                        description="Cover image URL; null unless one was attached.",
+                        nullable=True,
+                    ),
+                    field(
+                        "post.project_uid",
+                        "response",
+                        "string",
+                        description="Attached project uid; null unless the post is linked to a project.",
+                        nullable=True,
+                    ),
+                    field(
+                        "post.updated_at",
+                        "response",
+                        "string",
+                        description="ISO timestamp of the last edit; null until the post is edited for the first time.",
+                        nullable=True,
+                    ),
+                    field(
+                        "post.content",
+                        "response",
+                        "string",
+                        description="Post body, always present (minimum 10 characters at creation).",
+                        nullable=False,
+                    ),
+                    field(
+                        "post.slug",
+                        "response",
+                        "string",
+                        description="Always generated at creation, never null.",
+                        nullable=False,
+                    ),
                 ],
             ),
             endpoint(

devplacepy/docs_api/groups/game.py

@@ -53,7 +53,7 @@ client are on the [Code Farm guide](/docs/code-farm.html).
                 method="GET",
                 path="/game",
                 title="Code Farm page",
-                summary="The player's own farm: HUD, plot grid, shop, and leaderboard.",
+                summary="The player's own farm: server rack, workload picker, control terminal, HUD, shop, and leaderboard.",
                 auth="user",
                 negotiation=True,
                 sample_response={"ok": True, "farm": {"coins": 50, "level": 1, "plots": []}},
@@ -163,6 +163,19 @@ client are on the [Code Farm guide](/docs/code-farm.html).
                 params=[field("slot", "form", "integer", True, "0", "Plot slot index, 0-based.")],
                 sample_response={"ok": True, "farm": {"coins": 86}},
             ),
+            endpoint(
+                id="game-plot-stop",
+                method="POST",
+                path="/game/plot/stop",
+                title="Pause auto-replant on a plot",
+                summary="Needs the Continuous Delivery Mastery upgrade. With stop true, the next harvest of this plot does not replant, so you can choose a different crop. The pause clears itself when the plot is planted again. With stop false it cancels a pending pause. Rejected on an empty plot.",
+                auth="user",
+                params=[
+                    field("slot", "form", "integer", True, "0", "Plot slot index, 0-based."),
+                    field("stop", "form", "boolean", True, "true", "`true` to skip the replant after the next harvest, `false` to cancel."),
+                ],
+                sample_response={"ok": True, "farm": {"plots": [{"slot": 0, "can_stop_replant": True, "stop_after_harvest": True}]}},
+            ),
             endpoint(
                 id="game-buy-plot",
                 method="POST",
@@ -262,7 +275,7 @@ client are on the [Code Farm guide](/docs/code-farm.html).
                 method="POST",
                 path="/game/prestige",
                 title="Refactor (prestige)",
-                summary="Reset the farm at level 10+ for a permanent +25% coin bonus and earn Stars to spend on Legacy upgrades. Refactoring costs a coin fee that scales with prestige and current wealth (the farm state's refactor_cost); the fee funds the community treasury and a fraction of the remaining coins (10% base, up to 35% with the Golden Parachute Legacy upgrade) carries over. From prestige 50 onward, every 10 more prestige also earns a permanent Mastery point.",
+                summary="Reset the farm at level 10+ for a permanent +25% coin bonus and earn Stars to spend on Legacy upgrades. Refactoring costs a coin fee that scales with prestige and current wealth (the farm state's refactor_cost); the fee funds the community treasury and a fraction of the remaining coins (10% base, up to 60% with the Golden Parachute Legacy upgrade at level 10) carries over. Reaching prestige 50 earns a permanent Mastery point, then every 5 more prestige earns another.",
                 auth="user",
                 destructive=True,
                 sample_response={"ok": True, "farm": {"prestige": 1, "coins": 6550}},

devplacepy/docs_api/groups/gateway.py

@@ -77,9 +77,16 @@ managed by administrators on the **Gateway** page (`/admin/gateway`).
 
 ## Per-call cost and usage headers
 
-Every gateway response - chat, embeddings, and passthrough, on both success and error - carries
-`X-Gateway-*` response headers describing that single call, so a client can read its own token usage
-and dollar cost directly from the response with no extra request:
+Every NON-STREAMING gateway response - chat, embeddings, images, and passthrough, on both success
+and error - carries `X-Gateway-*` response headers describing that single call, so a client can read
+its own token usage and dollar cost directly from the response with no extra request. A streaming
+chat response (`"stream": true`) is the one exception: it carries only `X-Gateway-Model`,
+`X-Gateway-Backend`, and `X-App-Reference` - HTTP headers must be sent before the body, and cost/token
+counts for a streamed call are only known once the stream ends, so they cannot be response headers on
+that same response. The call is still fully metered server-side, and a client that sends
+`"stream_options": {"include_usage": true}` still receives the upstream's real `usage` object on the
+final SSE chunk, exactly as the underlying provider's own streaming API works - it is just not
+summarized into headers.
 
 | Header | Meaning |
 |--------|---------|
@@ -123,9 +130,17 @@ X-App-Reference: devplace-devii-v-1-0-0
 Administrators enable and configure this gateway under [Background Services](/docs/services.html)
 (the `openai` service).
 
-The gateway is exempt from rate limiting, but every other endpoint follows the shared
+The gateway is exempt from general rate limiting, but every other endpoint follows the shared
 [Conventions & Errors](/docs/conventions.html); see [Authentication](/docs/authentication.html)
 for signing DevPlace's own requests.
+
+A dedicated failed-authentication throttle protects the gateway from unauthenticated probing:
+an IP that repeatedly presents no credentials at all and fails authentication is answered `429`
+(with a `Retry-After` header) once it crosses a configurable threshold within a rolling window.
+This never affects a request that presents a valid key or session, even from an IP that has
+recently failed - a legitimate caller sharing a network address with a prior bad actor is never
+locked out. A `429` returned to a properly-authenticated call is a separate daily quota limit,
+not this throttle.
 """,
         "endpoints": [
             endpoint(
@@ -161,11 +176,20 @@ for signing DevPlace's own requests.
                         "false",
                         "Set true for a streamed SSE response.",
                     ),
+                    field(
+                        "think",
+                        "json",
+                        "string",
+                        False,
+                        "false",
+                        "Optional thinking override. Omitted: the gateway disables thinking (fast path). true / high / medium / low enables it; false disables it. DeepSeek-native `thinking.type` and OpenRouter `reasoning.effort` are also accepted.",
+                    ),
                 ],
                 notes=[
                     "Returns `503` when the gateway service is not running.",
-                    "Every response carries the `X-Gateway-*` token and dollar-cost headers (see Per-call cost and usage headers above), including the streamed SSE response.",
+                    "A non-streaming response carries the `X-Gateway-*` token and dollar-cost headers (see Per-call cost and usage headers above). A streaming response (`stream: true`) is forwarded from the upstream in real time and carries only `X-Gateway-Model`/`X-Gateway-Backend`/`X-App-Reference` - request `stream_options: {\"include_usage\": true}` to receive the real token usage on the final SSE chunk instead.",
                     "If `model` matches a configured model route it is forwarded to that route's provider, upstream model, and per-model pricing (with an optional vision model); otherwise it falls through to the default upstream (see Model routing and providers above).",
+                    "Thinking is disabled by default. Pass `think: true` (or `thinking: {\"type\": \"enabled\"}`) to turn it on for that call.",
                 ],
             ),
             endpoint(

devplacepy/docs_api/groups/lookups.py

@@ -34,7 +34,13 @@ four ways to sign requests.
                     )
                 ],
                 sample_response={
-                    "results": [{"uid": "8f14e45f-...", "username": "alice_test"}]
+                    "results": [
+                        {
+                            "uid": "8f14e45f-...",
+                            "username": "alice_test",
+                            "avatar_seed": None,
+                        }
+                    ]
                 },
             ),
             endpoint(
@@ -54,7 +60,13 @@ four ways to sign requests.
                     )
                 ],
                 sample_response={
-                    "results": [{"uid": "0cc175b9-...", "username": "bob_test"}]
+                    "results": [
+                        {
+                            "uid": "0cc175b9-...",
+                            "username": "bob_test",
+                            "avatar_seed": None,
+                        }
+                    ]
                 },
             ),
         ],

devplacepy/docs_api/groups/messaging.py

@@ -41,6 +41,14 @@ four ways to sign requests.
                         "",
                         "Jump to a conversation by username.",
                     ),
+                    field(
+                        "before",
+                        "query",
+                        "string",
+                        False,
+                        "",
+                        "ISO timestamp. When set, return the page of messages strictly older than this instant (for loading earlier history).",
+                    ),
                 ],
             ),
             endpoint(

devplacepy/docs_api/groups/nodes.py

@@ -0,0 +1,434 @@
+# retoor <retoor@molodetz.nl>
+
+from .._shared import endpoint, field
+
+DEVICE_SAMPLE = {
+    "uid": "DEVICE_UID",
+    "label": "Home laptop",
+    "kind": "native",
+    "platform": "linux/amd64",
+    "client_version": "0.1.0",
+    "status": "active",
+    "suspended_until": None,
+    "country": "NL",
+    "asn_class": "residential",
+    "last_seen_at": "2026-01-01T00:00:00+00:00",
+    "created_at": "2026-01-01T00:00:00+00:00",
+}
+
+GROUP = {
+    "slug": "nodes",
+    "title": "Community nodes",
+    "intro": """
+# Community nodes
+
+Members can donate a little of their internet connection to the DevPlace community. A donated
+**node** is a small program on the member's own machine that dials out to DevPlace and, when
+asked, fetches a public page from a short list of allowed websites on DevPlace's behalf. This is
+how search reaches sites that block data-centre addresses.
+
+- **Opt-in only.** Nobody is enrolled by default. Donating needs an explicit consent that only
+  the account holder can grant or withdraw.
+- **Not an open proxy.** A node refuses everything that is not on the published allowlist, every
+  port except 80 and 443, and every private or local address. The refusal is enforced on the
+  donor's machine as well as on the server.
+- **Counted and visible.** Requests and bytes relayed are counted by the server and shown to the
+  donor.
+
+The wire protocol, every limit and the allowlist are published as one machine-readable document
+at `/nodes/spec.json` and rendered on the [Node network protocol](/docs/nodes-protocol.html)
+page. Every endpoint follows the shared [Conventions & Errors](/docs/conventions.html).
+""",
+    "endpoints": [
+        endpoint(
+            id="nodes-spec",
+            method="GET",
+            path="/nodes/spec.json",
+            title="Node protocol specification",
+            summary="The single source of truth for the node protocol: version, frames, limits, allowlist and refusal codes.",
+            auth="public",
+            interactive=True,
+            sample_response={
+                "version": 1,
+                "min_supported": 1,
+                "open_proxy": False,
+                "allowed_ports": [80, 443],
+                "limits": {"max_devices_per_user": 5, "max_connected_nodes": 2000},
+                "frames": [
+                    {
+                        "type": "hello",
+                        "direction": "c2s",
+                        "max_bytes": 262144,
+                        "fields": [{"name": "protocol", "type": "int", "required": True}],
+                    }
+                ],
+                "destinations": [
+                    {
+                        "suffix": "reddit.com",
+                        "adapter": "reddit",
+                        "methods": ["GET"],
+                        "ports": [80, 443],
+                        "modes": ["tunnel", "fetch"],
+                        "max_rps": 0.5,
+                    }
+                ],
+                "refusals": [{"code": "host_not_allowed", "meaning": "destination is not on the allowlist"}],
+            },
+            notes=[
+                "`open_proxy` is always `false`. A change to that would be a protocol change, not a setting.",
+                "Clients and the server read the same numbers from here; a contract test fails when they drift.",
+            ],
+        ),
+        endpoint(
+            id="nodes-devices-list",
+            method="GET",
+            path="/nodes/devices",
+            title="List your devices",
+            summary="The devices you registered, with their status and when they last connected. Secrets are never listed.",
+            auth="user",
+            interactive=True,
+            sample_response={"devices": [DEVICE_SAMPLE], "max_devices": 5},
+        ),
+        endpoint(
+            id="nodes-devices-create",
+            method="POST",
+            path="/nodes/devices",
+            title="Register a device",
+            summary="Register one client install. The device secret is returned once and cannot be read again.",
+            auth="user",
+            ajax=True,
+            encoding="form",
+            params=[
+                field("label", "form", "string", False, "Home laptop", "A name only you see, up to 60 characters."),
+                field("kind", "form", "string", False, "native", "`native`, `extension` or `container`.", options=["native", "extension", "container"]),
+                field("platform", "form", "string", False, "linux/amd64", "Operating system and architecture of the client."),
+                field("client_version", "form", "string", False, "0.1.0", "Version of the client."),
+            ],
+            sample_response={"device": DEVICE_SAMPLE, "secret": "dpn_EXAMPLE_SECRET_SHOWN_ONCE"},
+            notes=[
+                "Refused with `403` and a `reason` (`kill_switch`, `no_consent`, `not_enabled`, `not_adult`, `deletion_pending`, `suspended`, `account_inactive`) unless donation is switched on, you are an adult, you have given the `bandwidth_sharing` consent and enabled donation for yourself.",
+                "Refused with `409` when you already have the maximum number of devices.",
+                "The response is `201` and is never cached.",
+            ],
+        ),
+        endpoint(
+            id="nodes-devices-rotate",
+            method="POST",
+            path="/nodes/devices/{device_uid}/rotate",
+            title="Rotate a device secret",
+            summary="Issue a new secret for one of your devices. The old secret stops working immediately.",
+            auth="user",
+            ajax=True,
+            params=[
+                field("device_uid", "path", "string", True, "DEVICE_UID", "Device uid."),
+            ],
+            sample_response={"uid": "DEVICE_UID", "secret": "dpn_EXAMPLE_SECRET_SHOWN_ONCE"},
+            notes=["`404` for a device that is not yours."],
+        ),
+        endpoint(
+            id="nodes-devices-delete",
+            method="POST",
+            path="/nodes/devices/{device_uid}/delete",
+            title="Delete a device",
+            summary="Remove a device. A connected node is disconnected on the next service tick. An administrator may remove any device.",
+            auth="user",
+            ajax=True,
+            destructive=True,
+            params=[
+                field("device_uid", "path", "string", True, "DEVICE_UID", "Device uid."),
+            ],
+            sample_response={"ok": True},
+        ),
+        endpoint(
+            id="nodes-ticket",
+            method="POST",
+            path="/nodes/ticket",
+            title="Exchange a device secret for a socket ticket",
+            summary="The node client calls this with its device secret to get a single-use ticket for `WS /nodes/ws?ticket=`.",
+            auth="user",
+            interactive=False,
+            sample_response={"ticket": "TICKET", "expires_in": 30, "ws_path": "/nodes/ws"},
+            notes=[
+                "Authenticated by `Authorization: Bearer dpn_...` with the device secret, not by a session. Anything else is `401`.",
+                "The ticket is valid for 30 seconds and can be redeemed once. The socket then speaks the frames published in the [protocol specification](/docs/nodes-protocol.html).",
+                "`403` with a `reason` when the device is suspended or its owner no longer passes the consent gate.",
+            ],
+        ),
+        endpoint(
+            id="nodes-me",
+            method="GET",
+            path="/nodes/me",
+            title="Your donation status",
+            summary="Whether you may donate right now (and the reason when not), your settings, your devices and what you have donated.",
+            auth="user",
+            interactive=True,
+            sample_response={
+                "consent": True,
+                "can_donate": True,
+                "reason": "",
+                "message": "",
+                "settings": {"enabled": True, "bandwidth_mb_day": 500, "max_concurrent": 2},
+                "platforms": ["reddit"],
+                "devices": [DEVICE_SAMPLE],
+                "max_devices": 5,
+                "donated": {"lifetime_requests": 12, "lifetime_bytes": 482113, "today_requests": 3},
+                "limits": {"max_donor_mb_per_day": 20480, "max_concurrent_per_node": 2, "default_donor_mb_per_day": 500},
+            },
+        ),
+        endpoint(
+            id="nodes-settings",
+            method="POST",
+            path="/nodes/settings",
+            title="Change your donation settings",
+            summary="Update bandwidth, concurrency, active hours, platforms and display. Unsent fields keep their value.",
+            auth="user",
+            ajax=True,
+            encoding="form",
+            params=[
+                field("enabled", "form", "boolean", False, "true", "Switch donation on or off. Switching on needs the `bandwidth_sharing` consent and is refused when sent by the Devii assistant."),
+                field("bandwidth_mb_day", "form", "integer", False, "500", "Daily bandwidth cap in megabytes."),
+                field("max_concurrent", "form", "integer", False, "2", "Simultaneous requests."),
+                field("active_from", "form", "string", False, "08:00", "Start of the active window, HH:MM."),
+                field("active_to", "form", "string", False, "22:00", "End of the active window, HH:MM."),
+                field("timezone", "form", "string", False, "Europe/Amsterdam", "IANA timezone of the window."),
+                field("allowed_platforms", "form", "string", False, "reddit", "Comma separated platforms you are willing to serve."),
+                field("pause_on_battery", "form", "boolean", False, "true", "Stop donating on battery power."),
+                field("public_display", "form", "boolean", False, "false", "Show your name on the donor leaderboard."),
+            ],
+            sample_response={"consent": True, "can_donate": True, "settings": {"enabled": True}},
+            notes=["`403` with `reason: no_consent` when switching on without the consent."],
+        ),
+        endpoint(
+            id="nodes-stats",
+            method="GET",
+            path="/nodes/stats",
+            title="Community donation totals",
+            summary="Aggregate numbers only: requests relayed, bytes relayed and number of donors. No per-member data.",
+            auth="public",
+            interactive=True,
+            sample_response={"requests": 1200, "bytes": 90311234, "donors": 7},
+        ),
+        endpoint(
+            id="nodes-leaderboard",
+            method="GET",
+            path="/nodes/leaderboard",
+            title="Donor leaderboard",
+            summary="The top donors who chose to be shown. Members are not listed unless they switched on public display.",
+            auth="public",
+            interactive=True,
+            sample_response={"donors": [{"username": "alice", "requests": 400, "bytes": 31200000}]},
+        ),
+        endpoint(
+            id="nodes-social-search",
+            method="GET",
+            path="/nodes/social/search",
+            title="Social search",
+            summary="Search public developer discussion on Hacker News, Stack Overflow, GitHub, dev.to, Mastodon and, when donors are online, Reddit.",
+            auth="user",
+            interactive=True,
+            params=[
+                field("q", "query", "string", True, "sqlite wal", "What to search for, up to 200 characters."),
+                field("platforms", "query", "string", False, "hackernews,github", "Comma separated subset. Empty means all."),
+                field("limit", "query", "integer", False, "8", "Hits per platform, 1 to 20."),
+            ],
+            sample_response={
+                "query": "sqlite wal",
+                "platforms": ["hackernews", "reddit"],
+                "hits": [
+                    {
+                        "platform": "hackernews",
+                        "title": "SQLite WAL mode explained",
+                        "url": "https://news.ycombinator.com/item?id=1",
+                        "snippet": "",
+                        "author": "someone",
+                        "score": 120,
+                        "comments": 33,
+                        "created_at": "2026-01-01T00:00:00Z",
+                        "via": "direct",
+                    }
+                ],
+                "unavailable": {"reddit": "no_nodes"},
+            },
+            notes=[
+                "Platforms that could not answer are listed under `unavailable` with a reason code; results are never invented for them.",
+                "Counted against a daily per-member quota (`429` when reached). Bluesky and Lobsters are not covered because they block unauthenticated server access.",
+            ],
+        ),
+        endpoint(
+            id="nodes-internal-fetch",
+            method="POST",
+            path="/nodes/internal/fetch",
+            title="Fetch a page through the donor network",
+            summary="Fetch one page from an allowlisted site through a donor node. Meant for agents and services that hold an API key.",
+            auth="user",
+            encoding="form",
+            params=[
+                field("url", "form", "string", True, "https://www.reddit.com/r/programming/.json", "Absolute URL on an allowlisted host."),
+                field("cache", "form", "boolean", False, "true", "Allow a cached copy."),
+                field("timeout", "form", "integer", False, "15", "Seconds to wait, 1 to 60."),
+            ],
+            sample_response={
+                "url": "https://www.reddit.com/r/programming/.json",
+                "status": 200,
+                "via": "fetch",
+                "truncated": False,
+                "elapsed_ms": 812,
+                "content_type": "application/json",
+                "text": "{...}",
+            },
+            notes=[
+                "`403` with `reason: host_not_allowed` for any host that is not on the allowlist, `503` with `reason: no_nodes` when no donor is online, `429` when the daily node quota is reached.",
+                "`via` says how the body arrived: `fetch` (relayed by a donor browser, treat as unverified) or `cache`.",
+            ],
+        ),
+        endpoint(
+            id="nodes-client-version",
+            method="GET",
+            path="/nodes/client-version",
+            title="Client versions and checksums",
+            summary="The protocol versions this server speaks and every downloadable client with its SHA-256.",
+            auth="public",
+            interactive=True,
+            sample_response={
+                "version": "0.1.0",
+                "protocol": 1,
+                "min_protocol": 1,
+                "artifacts": [{"name": "devplace-node-linux-amd64", "size": 6291456, "sha256": "ab12..."}],
+            },
+        ),
+        endpoint(
+            id="nodes-download",
+            method="GET",
+            path="/nodes/download/{artifact}",
+            title="Download a client",
+            summary="Download one client artifact listed by the client version endpoint. The SHA-256 is also sent as `X-Content-SHA256`.",
+            auth="public",
+            params=[
+                field("artifact", "path", "string", True, "devplace-node-linux-amd64", "Artifact name."),
+            ],
+            sample_response="(binary file)",
+        ),
+        endpoint(
+            id="admin-nodes",
+            method="GET",
+            path="/admin/nodes",
+            title="Node network overview",
+            summary="Pool, queue, device breakdowns, today's totals, open flags, recent devices, settings and the kill switch.",
+            auth="admin",
+            interactive=True,
+            sample_response={
+                "enabled": True,
+                "pool": {"online": True, "connected": 12, "capacity": 2000, "accepted": 40, "kicked": 3, "jobs_done": 310, "jobs_failed": 4, "open_circuits": 0, "heartbeat": "2026-01-01T00:00:00+00:00"},
+                "queue": {"depth": 2, "max_depth": 500, "queued": 1, "assigned": 1, "done": 30, "failed": 1, "expired": 0},
+                "devices_by_status": {"active": 14, "suspended": 1},
+                "devices_by_kind": {"native": 10, "extension": 5},
+                "devices_by_asn_class": {"unknown": 15},
+                "devices_by_country": {"unknown": 15},
+                "usage_today": {"requests_ok": 120, "requests_failed": 3, "bytes_up": 40000, "bytes_down": 9000000, "cache_hits_served": 0, "integrity_failures": 0},
+                "flags": [{"uid": "FLAG_UID", "kind": "refusal_mismatch", "detail": "", "status": "open", "device_uid": "DEVICE_UID", "device_label": "Home laptop", "user_uid": "USER_UID", "owner_username": "alice", "created_at": "2026-01-01T00:00:00+00:00"}],
+                "devices": [dict(DEVICE_SAMPLE, owner_uid="USER_UID", owner_username="alice", suspend_reason="")],
+                "settings": [{"key": "nodes_max_total", "label": "Maximum connected nodes", "kind": "int", "value": "2000", "default": "2000", "ceiling": 2000}],
+                "limits": {"max_devices_per_user": 5, "max_connected_nodes": 2000},
+            },
+            notes=[
+                "Per-destination circuit state lives in the memory of the worker that holds the sockets and is not readable from other workers; only the number of open circuits is reported.",
+                "`usage_today` is the current UTC day. Lists are capped at 100 rows.",
+            ],
+        ),
+        endpoint(
+            id="admin-nodes-killswitch",
+            method="POST",
+            path="/admin/nodes/killswitch",
+            title="Switch community donation on or off",
+            summary="The global kill switch. Switching it off disconnects every node within one reconcile tick.",
+            auth="admin",
+            encoding="form",
+            destructive=True,
+            params=[
+                field("enabled", "form", "boolean", False, "false", "True to switch on, false to switch off. Omit to flip the current state."),
+            ],
+            sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"enabled": False}},
+            notes=["Audited as `node.killswitch`."],
+        ),
+        endpoint(
+            id="admin-nodes-settings",
+            method="POST",
+            path="/admin/nodes/settings",
+            title="Update node network settings",
+            summary="Change the caps and quotas. Only the fields you send are changed; values above the ceiling are clamped to it.",
+            auth="admin",
+            encoding="form",
+            params=[
+                field("nodes_max_total", "form", "integer", False, "2000", "Maximum connected nodes, 0 to 2000."),
+                field("nodes_daily_quota_user", "form", "integer", False, "200", "Daily node requests per member, 0 to 10000."),
+                field("nodes_daily_quota_guest", "form", "integer", False, "0", "Daily node requests per guest, 0 to 1000."),
+                field("nodes_cache_ttl_seconds", "form", "integer", False, "600", "Result cache lifetime in seconds, 0 to 600."),
+                field("nodes_daily_social_user", "form", "integer", False, "300", "Daily social searches per member, 0 to 10000."),
+                field("nodes_social_via_nodes", "form", "string", False, "0", "`1` routes social search through nodes, `0` keeps it direct."),
+            ],
+            sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"changed": ["nodes_max_total"], "clamped": []}},
+            notes=["Audited as `node.admin.settings` with the old and new values."],
+        ),
+        endpoint(
+            id="admin-nodes-suspend",
+            method="POST",
+            path="/admin/nodes/devices/{device_uid}/suspend",
+            title="Suspend a donor device",
+            summary="Suspend one device for 1 to 10080 minutes. The device is disconnected on the next tick and cannot reconnect until the period ends.",
+            auth="admin",
+            encoding="form",
+            destructive=True,
+            params=[
+                field("device_uid", "path", "string", True, "DEVICE_UID", "Device uid."),
+                field("minutes", "form", "integer", True, "60", "Suspension length in minutes, 1 to 10080."),
+                field("reason", "form", "string", False, "Repeated refusals", "Reason, up to 500 characters."),
+            ],
+            sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"uid": "DEVICE_UID", "minutes": 60}},
+            notes=["Refused (audited as `denied`) when the device owner is a more senior administrator. Audited as `node.suspend`."],
+        ),
+        endpoint(
+            id="admin-nodes-class",
+            method="POST",
+            path="/admin/nodes/devices/{device_uid}/class",
+            title="Override a device's address class",
+            summary="Let one donor device serve jobs although its address is not residential or mobile.",
+            auth="admin",
+            encoding="form",
+            destructive=True,
+            params=[
+                field("device_uid", "path", "string", True, "DEVICE_UID", "Device uid."),
+                field("value", "form", "string", True, "allow", "`allow` to override, `clear` to remove the override."),
+            ],
+            sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"uid": "DEVICE_UID", "class_override": "allow"}},
+            notes=["Refused (audited as `denied`) when the device owner is a more senior administrator. Audited as `node.device.override`."],
+        ),
+        endpoint(
+            id="admin-nodes-unsuspend",
+            method="POST",
+            path="/admin/nodes/devices/{device_uid}/unsuspend",
+            title="Lift a device suspension",
+            summary="Return a suspended device to active.",
+            auth="admin",
+            encoding="form",
+            params=[
+                field("device_uid", "path", "string", True, "DEVICE_UID", "Device uid."),
+            ],
+            sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"uid": "DEVICE_UID"}},
+            notes=["`409` when the device is not suspended. Audited as `node.unsuspend`."],
+        ),
+        endpoint(
+            id="admin-nodes-flag-resolve",
+            method="POST",
+            path="/admin/nodes/flags/{flag_uid}/resolve",
+            title="Resolve a node flag",
+            summary="Mark an open integrity or refusal flag as handled.",
+            auth="admin",
+            encoding="form",
+            params=[
+                field("flag_uid", "path", "string", True, "FLAG_UID", "Flag uid."),
+            ],
+            sample_response={"ok": True, "redirect": "/admin/nodes", "data": {"uid": "FLAG_UID"}},
+            notes=["`409` when the flag is already resolved. Audited as `node.flag.resolve`."],
+        ),
+    ],
+}

devplacepy/docs_api/groups/profiles.py

@@ -65,6 +65,83 @@ four ways to sign requests.
                         "Profile tab.",
                         ["posts", "activity", "followers", "following", "media", "awards"],
                     ),
+                    field(
+                        "api_key",
+                        "response",
+                        "string",
+                        description="The profile's API key; null unless the viewer is the owner or an admin (see can_view_api_key).",
+                        nullable=True,
+                    ),
+                    field(
+                        "can_view_api_key",
+                        "response",
+                        "boolean",
+                        description="True when the viewer is the profile owner or an admin, i.e. when api_key holds the real key.",
+                        nullable=False,
+                    ),
+                    field(
+                        "profile_is_admin",
+                        "response",
+                        "boolean",
+                        description="True when the profile owner holds the Admin role. Always a boolean, never null.",
+                        nullable=False,
+                    ),
+                    field(
+                        "viewer_is_admin",
+                        "response",
+                        "boolean",
+                        description="True when the signed-in viewer holds the Admin role. Always a boolean, never null.",
+                        nullable=False,
+                    ),
+                    field(
+                        "is_owner",
+                        "response",
+                        "boolean",
+                        description="True when the viewer owns the profile. Always a boolean, never null.",
+                        nullable=False,
+                    ),
+                    field(
+                        "rank",
+                        "response",
+                        "integer",
+                        description="Leaderboard rank; null for a user with no stars yet.",
+                        nullable=True,
+                    ),
+                    field(
+                        "follow_pagination",
+                        "response",
+                        "object",
+                        description="Pagination metadata for the followers/following tabs; null on every other tab.",
+                        nullable=True,
+                    ),
+                    field(
+                        "profile_user.avatar_seed",
+                        "response",
+                        "string",
+                        description="Nullable users.avatar_seed override; null falls back to the username as the avatar seed.",
+                        nullable=True,
+                    ),
+                    field(
+                        "profile_user.bio",
+                        "response",
+                        "string",
+                        description="Optional profile bio; null when never set.",
+                        nullable=True,
+                    ),
+                    field(
+                        "is_following",
+                        "response",
+                        "boolean",
+                        description="Always a boolean, defaults to false, never null.",
+                        nullable=False,
+                    ),
+                    field(
+                        "profile_user.username",
+                        "response",
+                        "string",
+                        description="Always present, never null.",
+                        nullable=False,
+                    ),
                 ],
             ),
             endpoint(

devplacepy/docs_api/groups/push.py

@@ -14,13 +14,15 @@ implements the Web Push protocol: fetch the public VAPID key, then register a
 Push Notification service device token and is only offered when an administrator has
 configured it.
 
-`GET /push.json` lists the providers that currently accept registrations. A registration
-body without a `provider` field is a `webpush` registration, so existing clients need no
-change.
+`GET /push.json` lists the providers that currently accept registrations. When `apns` is
+active it includes `environment` (`production` or `sandbox`) so a native client can match
+its build. A registration body without a `provider` field is a `webpush` registration, so
+existing clients need no change. An APNs body may include a stable `client_id` so a later
+token rotation updates the same device instead of inserting another row.
 
-There is no server-side unsubscribe endpoint: unsubscription is handled entirely in the
-browser by calling `PushManager.unsubscribe()` on the subscription. The server stops delivering
-to a subscription once its push endpoint reports it as gone. These mirror the in-app
+`DELETE /push.json` removes a single registration by the same identity used to create it
+(`endpoint` for webpush, `token` or `client_id` for apns). The server also stops delivering to a
+subscription once its push endpoint reports it as gone. These mirror the in-app
 [Notifications](/docs/notifications.html) feed.
 
 Every endpoint follows the shared [Conventions & Errors](/docs/conventions.html) (auth, content
@@ -37,7 +39,10 @@ four ways to sign requests.
                 auth="public",
                 sample_response={
                     "publicKey": "BASE64_VAPID_KEY",
-                    "providers": {"webpush": {"publicKey": "BASE64_VAPID_KEY"}},
+                    "providers": {
+                        "webpush": {"publicKey": "BASE64_VAPID_KEY"},
+                        "apns": {"environment": "production"},
+                    },
                 },
             ),
             endpoint(
@@ -80,15 +85,75 @@ four ways to sign requests.
                         "string",
                         False,
                         "a1b2c3...",
-                        "Hexadecimal device token. Required for apns.",
+                        "Hexadecimal device token. Required for apns. Spaces and angle brackets are stripped.",
+                    ),
+                    field(
+                        "client_id",
+                        "json",
+                        "string",
+                        False,
+                        "vendor-uuid",
+                        "Stable per-device id for apns. When present, a new token updates this device instead of inserting a row.",
                     ),
                 ],
                 notes=[
                     'A webpush body is JSON: `{"endpoint": "...", "keys": {"p256dh": "...", "auth": "..."}}`.',
-                    'An APNs body is JSON: `{"provider": "apns", "token": "..."}`.',
+                    'An APNs body is JSON: `{"provider": "apns", "token": "...", "client_id": "..."}`. `client_id` is optional; token-only bodies keep working and revive a previously dead token.',
                     "A provider that is unknown, disabled or unconfigured returns 400.",
+                    "A newly created or revived registration is probed immediately. The response then includes `delivered` and, on failure, `error` with the provider reason. `registered` stays true so existing clients keep working.",
                 ],
-                sample_response={"registered": True},
+                sample_response={"registered": True, "delivered": True},
+            ),
+            endpoint(
+                id="push-unregister",
+                method="DELETE",
+                path="/push.json",
+                title="Unregister a subscription",
+                summary="Remove one push registration by its identity.",
+                auth="user",
+                encoding="json",
+                interactive=False,
+                params=[
+                    field(
+                        "provider",
+                        "json",
+                        "string",
+                        False,
+                        "webpush",
+                        "Provider the registration belongs to. Omit for webpush.",
+                    ),
+                    field(
+                        "endpoint",
+                        "json",
+                        "string",
+                        False,
+                        "https://fcm.googleapis.com/...",
+                        "Subscription endpoint URL. Identifies a webpush registration.",
+                    ),
+                    field(
+                        "token",
+                        "json",
+                        "string",
+                        False,
+                        "a1b2c3...",
+                        "Hexadecimal device token. Identifies an apns registration.",
+                    ),
+                    field(
+                        "client_id",
+                        "json",
+                        "string",
+                        False,
+                        "vendor-uuid",
+                        "Stable per-device id. Identifies an apns registration when set.",
+                    ),
+                ],
+                notes=[
+                    "Exactly one identity field is required: `endpoint` for webpush, `token` or `client_id` for apns.",
+                    "Matches the same identity priority as registration: `client_id`, then `token`, then `endpoint`.",
+                    "Idempotent: unregistering an unknown or already-removed identity still returns 200 with `unregistered: false`.",
+                    "Call this before logging out to stop delivery to the device that is logging out - the server has no way to know a browser tab or native app closed on its own.",
+                ],
+                sample_response={"unregistered": True},
             ),
         ],
 }

devplacepy/docs_api/groups/skills.py

@@ -0,0 +1,145 @@
+# retoor <retoor@molodetz.nl>
+
+from .._shared import endpoint, field
+
+SKILL_SAMPLE = {
+    "uid": "SKILL_UID",
+    "name": "seriousdevelopment",
+    "description": "Scaffold a new project or upgrade an existing one to a spec-first engineering bar.",
+    "enabled": True,
+    "source_url": "https://static.molodetz.nl/seriousdevelopment/SKILL.md",
+    "license": "",
+    "compatibility": "",
+    "allowed_tools": "",
+    "sha256": "SHA256_OF_BODY",
+    "file_count": 1,
+    "fetched_at": "2026-01-01T00:00:00+00:00",
+    "created_at": "2026-01-01T00:00:00+00:00",
+    "updated_at": "2026-01-01T00:00:00+00:00",
+}
+
+SKILL_DETAIL_SAMPLE = {
+    **SKILL_SAMPLE,
+    "metadata": {},
+    "body": "# Serious Development\n\nOne method for building software seriously.",
+    "files": {"assets/templates/SPEC.md": "# SPEC"},
+    "skipped": [{"path": "assets/img/hero.png", "reason": "binary or unsupported file type; only text files are stored"}],
+    "warnings": [],
+}
+
+GROUP = {
+        "slug": "skills",
+        "title": "Skills",
+        "intro": """
+# Skills
+
+A skill is an [Agent Skills](https://agentskills.io) package: a `SKILL.md` file (YAML frontmatter
+with a `name` and a `description`, then a markdown body) plus optional text files under
+`scripts/`, `references/` and `assets/`. Install one from a URL and Devii can use it.
+
+- **Private.** Your skills are visible only to you. An administrator may delete any skill.
+- **On demand.** Only the name and description of your *enabled* skills are added to Devii's
+  system prompt. Devii loads the full body and bundled files with `get_skill` when a request
+  matches. Installing, enabling, disabling or deleting a skill takes effect on the next turn.
+- **Untrusted text.** A skill is third-party text. It guides Devii within what you asked for; it
+  never bypasses a confirmation, never grants a tool (`allowed-tools` is stored, not honoured),
+  and its scripts are stored as text and never executed automatically.
+- **Install.** Pass a direct `.../SKILL.md` URL, or a directory URL (redirects to an index page
+  are reduced to their directory and `SKILL.md` is fetched from there). Bundled files are fetched
+  only when the body references them by relative path under `scripts/`, `references/` or
+  `assets/`, on the same origin, as text, within the size and count limits. Anything skipped is
+  reported with a reason. Installing the same `name` again updates it in place.
+
+Every endpoint follows the shared [Conventions & Errors](/docs/conventions.html) (auth, content
+negotiation, pagination, status codes); see [Authentication](/docs/authentication.html) for the
+four ways to sign requests.
+""",
+        "endpoints": [
+            endpoint(
+                id="skills-list",
+                method="GET",
+                path="/skills",
+                title="List your skills",
+                summary="Your installed skills. Returns an HTML page, or JSON with `Accept: application/json`.",
+                auth="user",
+                interactive=True,
+                sample_response={
+                    "items": [SKILL_SAMPLE],
+                    "skill_total": 1,
+                    "skill_limit": 50,
+                    "install_error": None,
+                },
+            ),
+            endpoint(
+                id="skills-detail",
+                method="GET",
+                path="/skills/{name}",
+                title="Get a skill",
+                summary="One of your skills with its full body, bundled files and the files that were skipped.",
+                auth="user",
+                interactive=True,
+                params=[
+                    field("name", "path", "string", True, "seriousdevelopment", "Skill name."),
+                ],
+                sample_response={"skill": SKILL_DETAIL_SAMPLE},
+                notes=["Returns `404` when you have no skill with that name; skills of other users are never visible."],
+            ),
+            endpoint(
+                id="skills-install",
+                method="POST",
+                path="/skills/install",
+                title="Install a skill",
+                summary="Fetch a skill from a URL and store it privately. Re-installing a name updates it in place.",
+                auth="user",
+                ajax=True,
+                encoding="form",
+                params=[
+                    field(
+                        "url",
+                        "form",
+                        "string",
+                        True,
+                        "https://static.molodetz.nl/seriousdevelopment",
+                        "A direct URL ending in SKILL.md, or a directory URL that resolves to one.",
+                    ),
+                ],
+                sample_response={"action": "install", "skill": SKILL_DETAIL_SAMPLE},
+                notes=[
+                    "`action` is `install` for a new skill and `update` when the name already existed.",
+                    "Fails with `400` and a reason for a missing or invalid frontmatter, an invalid name, a blocked address, an unreachable URL, or when you already have the maximum number of skills.",
+                    "Without `X-Requested-With: fetch` or `Accept: application/json` the browser is redirected to `/skills`.",
+                ],
+            ),
+            endpoint(
+                id="skills-enabled",
+                method="POST",
+                path="/skills/{name}/enabled",
+                title="Enable or disable a skill",
+                summary="Only enabled skills are listed in Devii's system prompt. Omit `enabled` to flip the current state.",
+                auth="user",
+                ajax=True,
+                encoding="form",
+                params=[
+                    field("name", "path", "string", True, "seriousdevelopment", "Skill name."),
+                    field("enabled", "form", "boolean", False, "true", "`true` to enable, `false` to disable; omit to toggle."),
+                ],
+                sample_response={"name": "seriousdevelopment", "enabled": False},
+            ),
+            endpoint(
+                id="skills-delete",
+                method="POST",
+                path="/skills/{name}/delete",
+                title="Delete a skill",
+                summary="Remove one of your skills. An administrator may remove another user's skill by passing `owner`.",
+                auth="user",
+                ajax=True,
+                destructive=True,
+                encoding="form",
+                params=[
+                    field("name", "path", "string", True, "seriousdevelopment", "Skill name."),
+                    field("owner", "form", "string", False, "", "Administrators only: username of the skill's owner."),
+                ],
+                sample_response={"deleted": True, "name": "seriousdevelopment"},
+            ),
+        ],
+}

devplacepy/docs_api/groups/social_actions.py

@@ -1,18 +1,29 @@
 # retoor <retoor@molodetz.nl>
 
-from .._shared import BOOKMARK_TARGETS, REACTION_TARGETS, VOTE_TARGETS, endpoint, field
+from .._shared import (
+    BOOKMARK_TARGETS,
+    NOTE_TARGETS,
+    REACTION_TARGETS,
+    VOTE_TARGETS,
+    endpoint,
+    field,
+)
 from devplacepy.constants import REACTION_EMOJI
 
 GROUP = {
         "slug": "social-actions",
-        "title": "Votes, Reactions, Bookmarks & Polls",
+        "title": "Votes, Reactions, Bookmarks, Notes & Polls",
         "intro": """
-# Votes, Reactions, Bookmarks & Polls
+# Votes, Reactions, Bookmarks, Notes & Polls
 
-Lightweight engagement actions. The POST endpoints here are **toggles** - sending the same
-action again removes it. They return JSON when called with `X-Requested-With: fetch` (sent
-automatically by the panels below); the [Conventions & Errors](/docs/conventions.html) page
-explains that header rule and the response envelope.
+Lightweight engagement actions. The vote/reaction/bookmark POST endpoints here are **toggles** -
+sending the same action again removes it. They return JSON when called with
+`X-Requested-With: fetch` (sent automatically by the panels below); the
+[Conventions & Errors](/docs/conventions.html) page explains that header rule and the response
+envelope.
+
+Personal notes are private, per-user annotations attached to a piece of content - only you can
+ever see your own notes.
 
 Every endpoint follows the shared [Conventions & Errors](/docs/conventions.html) (auth, content
 negotiation, pagination, status codes); see [Authentication](/docs/authentication.html) for the
@@ -24,7 +35,7 @@ four ways to sign requests.
                 method="POST",
                 path="/votes/{target_type}/{target_uid}",
                 title="Cast or toggle a vote",
-                summary="Upvote or downvote a target. Re-sending the same value removes the vote.",
+                summary="Upvote or downvote a target. Re-sending the same value, or sending 0, removes the vote.",
                 auth="user",
                 ajax=True,
                 encoding="form",
@@ -52,8 +63,8 @@ four ways to sign requests.
                         "enum",
                         True,
                         "1",
-                        "1 to upvote, -1 to downvote.",
-                        ["1", "-1"],
+                        "1 to upvote, -1 to downvote, 0 to retract your existing vote.",
+                        ["1", "-1", "0"],
                     ),
                 ],
                 sample_response={"net": 3, "up": 4, "down": 1, "value": 1},
@@ -151,6 +162,97 @@ four ways to sign requests.
                     "Bookmarks target posts, projects, gists, and news; see [Posts, Comments, Projects, Gists & News](/docs/content.html)."
                 ],
             ),
+            endpoint(
+                id="notes-set",
+                method="POST",
+                path="/notes/{target_type}/{target_uid}",
+                title="Add or update a personal note",
+                summary="Save a private note on a target. Only you can ever see it.",
+                auth="user",
+                ajax=True,
+                encoding="form",
+                params=[
+                    field(
+                        "target_type",
+                        "path",
+                        "enum",
+                        True,
+                        "post",
+                        "Type of content to annotate.",
+                        NOTE_TARGETS,
+                    ),
+                    field(
+                        "target_uid",
+                        "path",
+                        "string",
+                        True,
+                        "POST_UID",
+                        "UID of the target.",
+                    ),
+                    field(
+                        "content",
+                        "form",
+                        "string",
+                        True,
+                        "Remember to check this later.",
+                        "Note body, up to 4000 characters. Sending again on the same target replaces the note.",
+                    ),
+                ],
+                sample_response={"uid": "NOTE_UID", "content": "Remember to check this later."},
+            ),
+            endpoint(
+                id="notes-delete",
+                method="POST",
+                path="/notes/{target_type}/{target_uid}/delete",
+                title="Delete a personal note",
+                summary="Remove your private note from a target.",
+                auth="user",
+                ajax=True,
+                encoding="none",
+                params=[
+                    field(
+                        "target_type",
+                        "path",
+                        "enum",
+                        True,
+                        "post",
+                        "Type of content the note is on.",
+                        NOTE_TARGETS,
+                    ),
+                    field(
+                        "target_uid",
+                        "path",
+                        "string",
+                        True,
+                        "POST_UID",
+                        "UID of the target.",
+                    ),
+                ],
+                sample_response={"deleted": True},
+            ),
+            endpoint(
+                id="notes-saved",
+                method="GET",
+                path="/notes/saved",
+                title="View your personal notes",
+                summary="Render your saved notes. Returns an HTML page.",
+                auth="user",
+                interactive=True,
+                params=[
+                    field(
+                        "before",
+                        "query",
+                        "string",
+                        False,
+                        "",
+                        "Pagination cursor (created_at of the last item).",
+                    )
+                ],
+                notes=[
+                    "Notes target posts, projects, gists, and news; see [Posts, Comments, Projects, Gists & News](/docs/content.html).",
+                    "Nobody else can ever see your notes, not even the author of the content you annotated.",
+                ],
+            ),
             endpoint(
                 id="polls-vote",
                 method="POST",

devplacepy/docs_api/groups/sync.py

@@ -0,0 +1,288 @@
+# retoor <retoor@molodetz.nl>
+
+from .._shared import endpoint, field
+
+ENVELOPE_PARAMS = [
+    field("origin", "body", "string", True, "other.example.org", "Stable id of the sending community, 2-63 characters of a-z, 0-9, dots and hyphens. Lower-cased by the server."),
+    field("records", "body", "array", True, "[]", "1-50 records of the kind this endpoint takes. Each record is validated on its own."),
+    field("dry_run", "body", "boolean", False, "false", "Validate, normalize and report what would happen without writing anything."),
+]
+
+RESULT_SAMPLE = {
+    "origin": "other.example.org",
+    "kind": "posts",
+    "dry_run": False,
+    "total": 2,
+    "counts": {"created": 1, "updated": 0, "unchanged": 0, "stale": 0, "rejected": 0, "failed": 0, "deferred": 1},
+    "results": [
+        {"index": 0, "origin_uid": "p-1", "status": "created", "reason": "", "local_uid": "LOCAL_UID", "errors": [], "ignored": ["stars"], "needs_kind": "", "needs_uid": ""},
+        {"index": 1, "origin_uid": "p-2", "status": "deferred", "reason": "unknown_author", "local_uid": "", "errors": ["author u-9 is not synced yet, send users first"], "ignored": [], "needs_kind": "users", "needs_uid": "u-9"},
+    ],
+    "released": {"created": 3},
+}
+
+GROUP = {
+    "slug": "sync",
+    "title": "Community sync",
+    "admin": True,
+    "intro": """
+# Community sync
+
+Copy users, projects, posts, comments, polls, votes, reactions, follows, poll ballots and attachments from another community into this one, and pull this community's own public data out. The API is idempotent
+and safe to repeat: every record carries the id it has in the **origin** community
+(`origin_uid`), the server remembers which local row that id maps to, and a repeated record is
+an update or a no-op, never a duplicate.
+
+- **Validated and normalized here.** The sender posts raw data; the server fixes what can be
+  fixed (usernames, whitespace, control characters, unknown topics, timestamps) and rejects what
+  cannot, record by record. One bad record never fails the batch.
+- **Counters are never sent as numbers.** Stars, XP and levels cannot be verified across
+  communities, so they are ignored and listed under `ignored` in the result. A post or project
+  gets stars only from synced votes, and a vote needs a synced voter and a synced target.
+- **No side effects.** Synced content sends no notifications, awards no XP and runs no AI
+  correction. Content that matches the content filter is rejected.
+- **Order is forgiven, not required.** Send users, projects, posts, comments, polls, then the
+  engagement kinds. A record whose author, post, parent, target, poll or project is not synced
+  yet is answered `deferred` with `needs_kind` and `needs_uid`, parked for seven days (at most
+  5000 per origin) and applied by itself when that record arrives; the response then lists
+  `released` counts.
+- **Off by default.** An administrator enables it at `/admin/settings` (`sync_enabled`) and may
+  restrict senders with `sync_origins` (comma separated; empty allows any origin).
+- **JSON or form.** The body is JSON. A form-encoded body is accepted too, with `records` as a
+  JSON array string; Devii uses that form through the `sync_<kind>` tools.
+- **Pull feed.** `GET /sync/export/{kind}` pages out this community's own users, projects,
+  posts, comments, polls, follows and attachments. Votes, reactions and poll ballots are never
+  exported because who voted is not public, and rows that were themselves synced in are never
+  echoed back.
+- **Deletions are reported.** `GET /sync/deletions` lists synced records deleted or purged here
+  and `POST /sync/deletions/ack` stops them being reported.
+
+The accepted fields and every limit are published at `/sync/spec.json`. Every endpoint follows
+the shared [Conventions & Errors](/docs/conventions.html).
+""",
+    "endpoints": [
+        endpoint(
+            id="sync-spec",
+            method="GET",
+            path="/sync/spec.json",
+            title="Sync specification",
+            summary="Limits, statuses, refusal reasons and the JSON schema of each record kind.",
+            auth="public",
+            interactive=True,
+            sample_response={
+                "limits": {"max_batch_records": 50},
+                "kinds": ["users", "projects", "posts", "comments", "polls", "votes", "reactions", "follows", "poll_votes", "attachments"],
+                "statuses": ["created", "updated", "unchanged", "stale", "rejected", "failed", "deferred"],
+                "reasons": ["invalid", "unknown_author"],
+                "records": {"users": {"title": "UserRecord", "type": "object"}},
+            },
+        ),
+        endpoint(
+            id="sync-users",
+            method="POST",
+            path="/sync/users",
+            title="Upsert users",
+            summary="Create or update synced accounts. A synced account cannot sign in; it only owns the content that was synced for it.",
+            auth="admin",
+            encoding="json",
+            interactive=False,
+            params=ENVELOPE_PARAMS,
+            sample_response=RESULT_SAMPLE,
+            notes=[
+                "Record fields: `origin_uid`, `username`, `created_at`, optional `updated_at`, `bio`, `location`, `git_link`, `website`, `avatar_seed`, `deleted`.",
+                "A username that is invalid is normalized to letters, digits, `_` and `-`. A name already taken by a different account gets the origin as a suffix, so the result may differ from the one sent.",
+                "`deleted: true` deactivates the account. Email, password, role, XP and level are never accepted.",
+            ],
+        ),
+        endpoint(
+            id="sync-posts",
+            method="POST",
+            path="/sync/posts",
+            title="Upsert posts",
+            summary="Create or update posts. The author must already be synced.",
+            auth="admin",
+            encoding="json",
+            interactive=False,
+            params=ENVELOPE_PARAMS,
+            sample_response=RESULT_SAMPLE,
+            notes=[
+                "Record fields: `origin_uid`, `author_uid` (the author's `origin_uid`), `content`, `created_at`, optional `title`, `topic`, `updated_at`, `deleted`.",
+                "An unknown `topic` becomes `random`. Content shorter than 10 characters is rejected.",
+            ],
+        ),
+        endpoint(
+            id="sync-comments",
+            method="POST",
+            path="/sync/comments",
+            title="Upsert comments",
+            summary="Create or update comments on synced posts. The author, the post and any parent comment must already be synced.",
+            auth="admin",
+            encoding="json",
+            interactive=False,
+            params=ENVELOPE_PARAMS,
+            sample_response=RESULT_SAMPLE,
+            notes=[
+                "Record fields: `origin_uid`, `author_uid`, `post_uid` (the post's `origin_uid`), `content`, `created_at`, optional `parent_uid`, `updated_at`, `deleted`.",
+                "A parent that belongs to a different post is rejected with `parent_mismatch`.",
+            ],
+        ),
+        endpoint(
+            id="sync-projects",
+            method="POST",
+            path="/sync/projects",
+            title="Upsert projects",
+            summary="Create or update public, writable projects. The author must already be synced.",
+            auth="admin",
+            encoding="json",
+            interactive=False,
+            params=ENVELOPE_PARAMS,
+            sample_response=RESULT_SAMPLE,
+            notes=[
+                "Record fields: `origin_uid`, `author_uid`, `title`, `description`, `created_at`, optional `project_type` (game, game_asset, software, mobile_app, website), `status`, `platforms`, `website_url`, `repo_url`, `updated_at`, `deleted`.",
+                "Posts link to a project with `project_uid`.",
+            ],
+        ),
+        endpoint(
+            id="sync-polls",
+            method="POST",
+            path="/sync/polls",
+            title="Upsert polls",
+            summary="Create or update the poll of a synced post (one poll per post, 2 to 6 options).",
+            auth="admin",
+            encoding="json",
+            interactive=False,
+            params=ENVELOPE_PARAMS,
+            sample_response=RESULT_SAMPLE,
+            notes=[
+                "Record fields: `origin_uid`, `author_uid`, `post_uid`, `question`, `options` (a list of `{origin_uid, label}`), `created_at`, optional `deleted`.",
+                "Editing the options adds, renames and removes them; ballots for a removed option stay.",
+            ],
+        ),
+        endpoint(
+            id="sync-votes",
+            method="POST",
+            path="/sync/votes",
+            title="Upsert votes",
+            summary="Create, change or clear a vote of a synced user on a synced post, comment or project.",
+            auth="admin",
+            encoding="json",
+            interactive=False,
+            params=ENVELOPE_PARAMS,
+            sample_response=RESULT_SAMPLE,
+            notes=[
+                "Record fields: `origin_uid`, `voter_uid`, `target_kind` (post, comment, project), `target_uid`, `value` (1 or -1), `created_at`, optional `deleted` to clear.",
+                "A second origin id for the same voter and target revives the same row. Stars of posts and projects are recomputed from the live synced votes.",
+            ],
+        ),
+        endpoint(
+            id="sync-reactions",
+            method="POST",
+            path="/sync/reactions",
+            title="Upsert reactions",
+            summary="Add or remove an emoji reaction of a synced user.",
+            auth="admin",
+            encoding="json",
+            interactive=False,
+            params=ENVELOPE_PARAMS,
+            sample_response=RESULT_SAMPLE,
+            notes=["Record fields: `origin_uid`, `author_uid`, `target_kind`, `target_uid`, `emoji` (one emoji), `created_at`, optional `deleted`."],
+        ),
+        endpoint(
+            id="sync-follows",
+            method="POST",
+            path="/sync/follows",
+            title="Upsert follows",
+            summary="Create or remove a follow between two synced users. No notification, XP or badge is produced.",
+            auth="admin",
+            encoding="json",
+            interactive=False,
+            params=ENVELOPE_PARAMS,
+            sample_response=RESULT_SAMPLE,
+            notes=["Record fields: `origin_uid`, `follower_uid`, `following_uid`, `created_at`, optional `deleted`. Following yourself is rejected with `self_follow`."],
+        ),
+        endpoint(
+            id="sync-poll-votes",
+            method="POST",
+            path="/sync/poll_votes",
+            title="Upsert poll ballots",
+            summary="Record a synced user's ballot in a synced poll (one per voter; changing the option updates it).",
+            auth="admin",
+            encoding="json",
+            interactive=False,
+            params=ENVELOPE_PARAMS,
+            sample_response=RESULT_SAMPLE,
+            notes=["Record fields: `origin_uid`, `voter_uid`, `poll_uid`, `option_uid`, `created_at`, optional `deleted`. An option that does not exist in that poll is rejected with `unknown_option`."],
+        ),
+        endpoint(
+            id="sync-attachments",
+            method="POST",
+            path="/sync/attachments",
+            title="Upsert attachments",
+            summary="Store a file for a synced post, comment or project. The sender downloads the file and sends its bytes.",
+            auth="admin",
+            encoding="json",
+            interactive=False,
+            params=ENVELOPE_PARAMS,
+            sample_response=RESULT_SAMPLE,
+            notes=[
+                "Record fields: `origin_uid`, `author_uid`, `target_kind`, `target_uid`, `filename`, `content_b64` (strict base64, at most 5 MB), `created_at`, optional `deleted`.",
+                "The normal upload rules apply (allowed types, size). An attachment cannot change: a known `origin_uid` with different bytes is rejected with `immutable`; only a tombstone or a restore is accepted.",
+            ],
+        ),
+        endpoint(
+            id="sync-export",
+            method="GET",
+            path="/sync/export/{kind}",
+            title="Export own records",
+            summary="Page out this community's users, projects, posts, comments, polls, follows or attachments in sync record form.",
+            auth="admin",
+            encoding="json",
+            interactive=False,
+            params=[
+                field("kind", "path", "string", True, "posts", "users, projects, posts, comments, polls, follows or attachments."),
+                field("since", "query", "string", False, "", "Only records changed after this ISO 8601 instant. Users have no change time, so for them it filters by sign-up time."),
+                field("cursor", "query", "string", False, "", "The `next_cursor` of the previous page."),
+                field("limit", "query", "integer", False, "50", "Records per page, 1 to 50."),
+            ],
+            sample_response={"kind": "posts", "count": 1, "has_more": False, "next_cursor": "", "records": [{"origin_uid": "POST_UID", "author_uid": "USER_UID", "content": "...", "deleted": False}]},
+            notes=[
+                "Only public data: no email, password hash, role or API key; private projects are left out; attachments carry a `url` instead of bytes.",
+                "Votes, reactions and poll ballots are never exported. Rows that arrived by sync are never exported, so two communities cannot echo each other's data.",
+                "Soft-deleted rows appear as records with `deleted: true`. Audited as `sync.export`.",
+            ],
+        ),
+        endpoint(
+            id="sync-deletions",
+            method="GET",
+            path="/sync/deletions",
+            title="List local deletions",
+            summary="Synced records of one origin that were deleted or purged on this side and not yet acknowledged.",
+            auth="admin",
+            encoding="json",
+            interactive=False,
+            params=[
+                field("origin", "query", "string", True, "other.example.org", "The sending community's origin id."),
+                field("cursor", "query", "string", False, "", "The `next_cursor` of the previous page."),
+                field("limit", "query", "integer", False, "50", "Entries per page, 1 to 50."),
+            ],
+            sample_response={"origin": "other.example.org", "count": 1, "has_more": False, "next_cursor": "", "items": [{"kind": "posts", "origin_uid": "p-1", "by": "member", "deleted_at": "2026-10-16T10:00:00+00:00"}]},
+            notes=["`by` is `purged` (the row is gone), `system` or `member`. A deletion made by the sync itself is never listed."],
+        ),
+        endpoint(
+            id="sync-deletions-ack",
+            method="POST",
+            path="/sync/deletions/ack",
+            title="Acknowledge local deletions",
+            summary="Tell this community that the origin has seen the deletions, so they are no longer listed or reported.",
+            auth="admin",
+            encoding="json",
+            interactive=False,
+            params=[
+                field("origin", "body", "string", True, "other.example.org", "The sending community's origin id."),
+                field("items", "body", "array", True, "[]", "1-50 objects `{kind, origin_uid}`."),
+            ],
+            sample_response={"origin": "other.example.org", "acknowledged": 1},
+            notes=["After acknowledging, a re-sent record answers `unchanged` with reason `acknowledged_deleted`. Audited as `sync.deletions.ack`."],
+        ),
+    ],
+}

devplacepy/docs_api/groups/tools.py

@@ -226,12 +226,43 @@ status and report.
                         {"title": "Invention", "detail": "...", "confidence": 0.8, "citations": [1]}
                     ],
                     "sources": [{"url": "https://example.com", "title": "Example", "source": "httpx"}],
+                    "follow_up_questions": ["Who else worked on the transistor?", "How did it replace vacuum tubes?"],
                     "chat_ws_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/chat",
                     "export_md_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/export.md",
                     "export_json_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/export.json",
                     "export_pdf_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/export.pdf",
                 },
             ),
+            endpoint(
+                id="tools-deepsearch-history",
+                method="GET",
+                path="/tools/deepsearch/history",
+                title="My DeepSearch history",
+                summary="List the caller's own past DeepSearch runs, newest first, with a link back to each report/chat. Member history is account-bound; guest history is scoped to the visitor's address. Negotiates HTML or JSON.",
+                auth="public",
+                params=[
+                    field("limit", "query", "integer", False, "20", "Maximum sessions to return (1-100)."),
+                ],
+                sample_response={
+                    "sessions": [
+                        {
+                            "uid": "DEEPSEARCH_JOB_UID",
+                            "query": "history of the transistor",
+                            "status": "done",
+                            "score": 78,
+                            "confidence": 0.72,
+                            "page_count": 11,
+                            "chunk_count": 240,
+                            "summary": "The transistor was invented at Bell Labs in 1947...",
+                            "reopen_url": "/tools/deepsearch/DEEPSEARCH_JOB_UID/session",
+                            "chat_available": True,
+                            "available": True,
+                            "created_at": "2026-06-14T10:00:00+00:00",
+                            "completed_at": "2026-06-14T10:01:40+00:00",
+                        }
+                    ]
+                },
+            ),
             endpoint(
                 id="tools-isslop-run",
                 method="POST",

devplacepy/docs_api/groups/uploads.py

@@ -87,6 +87,31 @@ four ways to sign requests.
                     "mime_type": "image/png",
                 },
             ),
+            endpoint(
+                id="uploads-generate-image",
+                method="POST",
+                path="/uploads/generate-image",
+                title="Generate an image",
+                summary="Generate an AI image from a prompt with the gateway's default image model and store it as an attachment.",
+                auth="user",
+                params=[
+                    field("prompt", "form", "string", True, "A lighthouse at dusk", "Description of the image, 1-2000 characters."),
+                    field("size", "form", "string", False, "1024x1024", "Optional WIDTHxHEIGHT; defaults to 1024x1024."),
+                ],
+                notes=[
+                    "The model is never chosen by the caller: the request goes to the internal gateway as `molodetz-img-small`, so the admin's image route decides the model and price, metered to your account.",
+                    "Returns `201` with the same body as Upload a file, `502` when the provider fails or returns no image, `413` if the result exceeds the upload limit.",
+                ],
+                sample_response={
+                    "uid": "ATTACHMENT_UID",
+                    "filename": "generated.png",
+                    "url": "/static/uploads/attachments/ab/cd/ATTACHMENT_UID.png",
+                    "size": 802816,
+                    "is_image": True,
+                    "is_video": False,
+                    "mime_type": "image/png",
+                },
+            ),
             endpoint(
                 id="uploads-list",
                 method="GET",

devplacepy/docs_api/groups/workspaces.py

@@ -37,7 +37,10 @@ arrives as a `workspace` notification and states exactly what happens next and w
             title="Read workspace",
             summary=(
                 "State, quota usage, idle countdown, tunnels and open moderation flags "
-                "for your workspace on this project."
+                "for your workspace on this project. The phase (stopped, starting, ready, "
+                "stopping, crashed, suspended) is derived from the desired state, the "
+                "container status and a live probe of the editor port, so editor_ready "
+                "is true only when the editor will actually open."
             ),
             auth="user",
             params=[
@@ -51,7 +54,12 @@ arrives as a `workspace` notification and states exactly what happens next and w
                 "editor_url": "/projects/my-project/containers/instances/INSTANCE_UID/code/",
                 "workspace": {
                     "uid": "INSTANCE_UID",
+                    "owner_uid": "USER_UID",
                     "status": "running",
+                    "desired_state": "running",
+                    "phase": "ready",
+                    "phase_label": "Ready",
+                    "editor_ready": True,
                     "suspended": False,
                     "tunnel_name": "brave-otter",
                     "primary_url": "https://brave-otter.tunnel.pravda.education",
@@ -131,7 +139,7 @@ arrives as a `workspace` notification and states exactly what happens next and w
                     "terminal_font_size": 13,
                     "zoom_level": 0,
                     "layout": "standard",
-                    "panel_preset": "tall",
+                    "panel_preset": "normal",
                     "boot_agent": "dpc",
                     "boot_shell": True,
                     "window_mode": "tab",

devplacepy/docs_api/negotiation.py

@@ -37,6 +37,7 @@ _PAGE_RESPONSES = {
     "issues-detail": schemas.IssueDetailOut,
     "issues-attachments-list": schemas.IssueAttachmentsOut,
     "bookmarks-saved": schemas.SavedOut,
+    "notes-saved": schemas.NotesOut,
     "admin-users": schemas.AdminUsersOut,
     "admin-news-list": schemas.AdminNewsOut,
     "admin-settings-get": schemas.AdminSettingsOut,

devplacepy/docs_api/services_group.py

@@ -117,6 +117,25 @@ def build_services_group(services, base):
             )
         )
     control_endpoints = [
+        endpoint(
+            id="services-page",
+            method="GET",
+            path="/admin/services",
+            title="Services dashboard",
+            summary="Admin HTML index of every registered background service, its status, and controls.",
+            auth="admin",
+            interactive=False,
+        ),
+        endpoint(
+            id="services-detail-page",
+            method="GET",
+            path="/admin/services/{name}",
+            title="Service detail page",
+            summary="Admin HTML detail page for a single service: overview, configuration form, and logs.",
+            auth="admin",
+            interactive=False,
+            params=[field("name", "path", "string", True, "news", "Service name.")],
+        ),
         endpoint(
             id="services-data",
             method="GET",

devplacepy/docs_export.py

@@ -31,8 +31,8 @@ def _params_table(params: list) -> str:
     if not params:
         return ""
     rows = [
-        "| Name | In | Type | Required | Description |",
-        "|------|----|------|----------|-------------|",
+        "| Name | In | Type | Required | Nullable | Description |",
+        "|------|----|------|----------|----------|-------------|",
     ]
     for p in params:
         desc = (p.get("description", "") or "").replace("|", "\\|")
@@ -43,7 +43,8 @@ def _params_table(params: list) -> str:
             desc = f"{desc} Allowed: {allowed}.".strip()
         rows.append(
             f"| `{p['name']}` | {p['location']} | {p['type']} | "
-            f"{'yes' if p['required'] else 'no'} | {desc} |"
+            f"{'yes' if p['required'] else 'no'} | "
+            f"{'yes' if p.get('nullable') else 'no'} | {desc} |"
         )
     return "\n".join(rows)
 

devplacepy/errors.py

@@ -0,0 +1,154 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+
+from fastapi import Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse, RedirectResponse
+
+from devplacepy import happy404
+from devplacepy.responses import json_error, wants_json
+from devplacepy.schemas import ValidationErrorOut
+from devplacepy.seo import base_seo_context
+from devplacepy.services.moderation.screening import ContentRefused
+from devplacepy.templating import templates
+from devplacepy.utils import safe_next
+
+logger = logging.getLogger(__name__)
+
+
+async def not_found(request: Request, exc):
+    if wants_json(request):
+        return json_error(404, "Not found")
+    happy_response = happy404.render(request)
+    if happy_response is not None:
+        return happy_response
+    seo_ctx = base_seo_context(
+        request,
+        title="Not Found - DevPlace",
+        description="The page you requested does not exist.",
+        robots="noindex",
+    )
+    return templates.TemplateResponse(
+        request,
+        "error.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "error_code": 404,
+            "error_message": "Page not found",
+        },
+        status_code=404,
+    )
+
+
+async def server_error(request: Request, exc):
+    logger.exception("500 error on %s %s", request.method, request.url.path)
+    if wants_json(request):
+        return json_error(500, "Internal server error")
+    seo_ctx = base_seo_context(
+        request,
+        title="Server Error - DevPlace",
+        description="Something went wrong.",
+        robots="noindex",
+    )
+    return templates.TemplateResponse(
+        request,
+        "error.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "error_code": 500,
+            "error_message": "Internal server error",
+        },
+        status_code=500,
+    )
+
+
+async def content_refused(request: Request, exc: ContentRefused):
+    logger.info("content refused on %s %s: %s", request.method, request.url.path, exc.message)
+    if wants_json(request):
+        return json_error(400, exc.message, categories=list(exc.categories))
+    seo_ctx = base_seo_context(
+        request,
+        title="Content not published - DevPlace",
+        description=exc.message,
+        robots="noindex",
+    )
+    return templates.TemplateResponse(
+        request,
+        "error.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "error_code": 400,
+            "error_message": exc.message,
+        },
+        status_code=400,
+    )
+
+
+_AUTH_FORM_PAGES = {
+    "/auth/signup": ("signup.html", "Join DevPlace"),
+    "/auth/login": ("login.html", "Sign In"),
+    "/auth/forgot-password": ("forgot_password.html", "Reset Password"),
+}
+
+_FRIENDLY_ERRORS = {
+    ("username", "too_short"): "Username must be between 3 and 32 characters",
+    ("username", "too_long"): "Username must be between 3 and 32 characters",
+    ("password", "too_short"): "Password must be at least 6 characters",
+}
+
+
+def _friendly_error(err):
+    field = err["loc"][-1] if err.get("loc") else ""
+    key = (field, err.get("type", "").replace("string_", ""))
+    if key in _FRIENDLY_ERRORS:
+        return _FRIENDLY_ERRORS[key]
+    msg = err.get("msg", "Invalid input")
+    prefix = "Value error, "
+    return msg[len(prefix) :] if msg.startswith(prefix) else msg
+
+
+async def on_validation_error(request: Request, exc: RequestValidationError):
+    errors = [_friendly_error(e) for e in exc.errors()]
+    if wants_json(request):
+        fields: dict = {}
+        for raw, friendly in zip(exc.errors(), errors):
+            name = raw["loc"][-1] if raw.get("loc") else "_"
+            fields.setdefault(str(name), []).append(friendly)
+        return JSONResponse(
+            ValidationErrorOut(fields=fields, messages=errors).model_dump(mode="json"),
+            status_code=422,
+        )
+    path = request.url.path
+    page = _AUTH_FORM_PAGES.get(path)
+    if page is None and path.startswith("/auth/reset-password/"):
+        page = ("reset_password.html", "Set New Password")
+    if page:
+        template_name, title = page
+        context = {
+            **base_seo_context(request, title=title, robots="noindex,nofollow"),
+            "request": request,
+            "errors": errors,
+        }
+        try:
+            form = await request.form()
+            context.update({k: v for k, v in form.items() if isinstance(v, str)})
+        except Exception:
+            pass
+        if "token" in request.path_params:
+            context["token"] = request.path_params["token"]
+        return templates.TemplateResponse(
+            request, template_name, context, status_code=400
+        )
+    referer = safe_next(request.headers.get("referer"), "/feed")
+    return RedirectResponse(url=referer, status_code=303)
+
+
+def register(app):
+    app.exception_handler(404)(not_found)
+    app.exception_handler(500)(server_error)
+    app.exception_handler(ContentRefused)(content_refused)
+    app.exception_handler(RequestValidationError)(on_validation_error)

devplacepy/happy404.py

@@ -0,0 +1,70 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import logging
+import os
+import random
+
+from fastapi import Request
+from fastapi.responses import HTMLResponse
+
+from devplacepy.cache import TTLCache
+from devplacepy.content import load_detail
+from devplacepy.database import db, get_setting
+from devplacepy.routers.posts import post_page_context
+from devplacepy.templating import templates
+from devplacepy.utils import get_current_user
+
+logger = logging.getLogger("happy404")
+
+POOL_TTL_SECONDS = int(os.environ.get("DEVPLACE_HAPPY_404_POOL_TTL", "300"))
+POOL_SIZE = 100
+
+API_PATH_PREFIXES = ("/api", "/dbapi", "/openai", "/xmlrpc", "/swagger", "/openapi.json", "/nodes")
+
+_pool_cache = TTLCache(ttl=POOL_TTL_SECONDS, max_size=1)
+
+
+def _post_pool() -> list[str]:
+    cached = _pool_cache.get("slugs")
+    if cached is not None:
+        return cached
+    slugs: list[str] = []
+    if "posts" in db.tables:
+        rows = db.query(
+            "SELECT slug, uid FROM posts WHERE deleted_at IS NULL ORDER BY RANDOM() LIMIT :limit",
+            limit=POOL_SIZE,
+        )
+        slugs = [row["slug"] or row["uid"] for row in rows]
+    _pool_cache.set("slugs", slugs)
+    return slugs
+
+
+def _eligible(request: Request) -> bool:
+    if request.method != "GET":
+        return False
+    if request.url.path.startswith(API_PATH_PREFIXES):
+        return False
+    return get_setting("happy_404_enabled", "1") == "1"
+
+
+def render(request: Request) -> HTMLResponse | None:
+    try:
+        if not _eligible(request):
+            return None
+        pool = _post_pool()
+        if not pool:
+            return None
+        slug = random.choice(pool)
+        user = get_current_user(request)
+        detail = load_detail("posts", "post", slug, user)
+        if not detail:
+            return None
+        context = post_page_context(
+            request, user, detail, robots="noindex,nofollow"
+        )
+        return templates.TemplateResponse(request, "post.html", context)
+    except Exception as exc:  # noqa: BLE001 - a happy-404 bug must never break the 404 page
+        logger.warning("happy_404 render failed: %s", exc)
+        return None

devplacepy/lifespan.py

@@ -0,0 +1,154 @@
+# retoor <retoor@molodetz.nl>
+
+import fcntl
+import logging
+import os
+from contextlib import asynccontextmanager, contextmanager
+
+from fastapi import FastAPI
+
+from devplacepy.config import (
+    INIT_LOCK_FILE,
+    PORT,
+    SERVICE_LOCK_FILE,
+    ensure_data_dirs,
+)
+from devplacepy.database import init_db
+from devplacepy.services.acceptance.service import AcceptanceService
+from devplacepy.services.audit import AuditService
+from devplacepy.services.backup import BackupService
+from devplacepy.services.background import background
+from devplacepy.services.bot import BotsService
+from devplacepy.services.community.service import CommunityService
+from devplacepy.services.containers.service import ContainerService
+from devplacepy.services.containers.workspace_service import WorkspaceService
+from devplacepy.services.dbapi.service import DbApiJobService
+from devplacepy.services.devii import DeviiService
+from devplacepy.services.gitea.service import IssueTrackerService
+from devplacepy.services.jobs.award_service import AwardService
+from devplacepy.services.jobs.deepsearch.service import DeepsearchService
+from devplacepy.services.jobs.fork_service import ForkService
+from devplacepy.services.jobs.issue_create_service import IssueCreateService
+from devplacepy.services.jobs.isslop.service import IsslopService
+from devplacepy.services.jobs.planning_service import PlanningReportService
+from devplacepy.services.jobs.seo.service import SeoService
+from devplacepy.services.jobs.seo_meta_service import SeoMetaService
+from devplacepy.services.jobs.zip_service import ZipService
+from devplacepy.services.live_view_relay import LiveViewRelayService
+from devplacepy.services.manager import service_manager
+from devplacepy.services.moderation.service import ModerationService
+from devplacepy.services.news import NewsService
+from devplacepy.services.nodes.service import NodesService
+from devplacepy.services.notification_relay import NotificationRelayService
+from devplacepy.services.openai_gateway import GatewayService
+from devplacepy.services.opinionwar.service import OpinionWarService
+from devplacepy.services.presence_relay import PresenceRelayService
+from devplacepy.services.pubsub import PubSubService
+from devplacepy.services.push import PushService
+from devplacepy.services.telegram import TelegramService
+from devplacepy.services.telegram.outbox_service import TelegramOutboxService
+from devplacepy.services.xmlrpc import XmlrpcService
+
+logger = logging.getLogger(__name__)
+
+SERVICE_CLASSES = [
+    NewsService,
+    BotsService,
+    GatewayService,
+    DeviiService,
+    ZipService,
+    ForkService,
+    SeoService,
+    SeoMetaService,
+    AwardService,
+    BackupService,
+    DbApiJobService,
+    PubSubService,
+    NotificationRelayService,
+    OpinionWarService,
+    LiveViewRelayService,
+    PresenceRelayService,
+    DeepsearchService,
+    IsslopService,
+    IssueCreateService,
+    PlanningReportService,
+    IssueTrackerService,
+    ContainerService,
+    WorkspaceService,
+    XmlrpcService,
+    AuditService,
+    ModerationService,
+    AcceptanceService,
+    CommunityService,
+    NodesService,
+    PushService,
+    TelegramService,
+    TelegramOutboxService,
+]
+
+_service_lock_handle = None
+
+
+@contextmanager
+def init_lock():
+    handle = open(INIT_LOCK_FILE, "w")
+    try:
+        fcntl.flock(handle, fcntl.LOCK_EX)
+        yield
+    finally:
+        fcntl.flock(handle, fcntl.LOCK_UN)
+        handle.close()
+
+
+def acquire_service_lock() -> bool:
+    global _service_lock_handle
+    handle = open(SERVICE_LOCK_FILE, "w")
+    try:
+        fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except OSError:
+        handle.close()
+        return False
+    _service_lock_handle = handle
+    return True
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    ensure_data_dirs()
+    with init_lock():
+        init_db()
+        from devplacepy.push import ensure_certificates
+
+        ensure_certificates()
+        from devplacepy.services.openai_gateway import model_health
+
+        model_health.seed_from_ledger()
+    for service_cls in SERVICE_CLASSES:
+        service_manager.register(service_cls())
+    if not os.environ.get("DEVPLACE_DISABLE_SERVICES"):
+        await background.start()
+        if acquire_service_lock():
+            service_manager.set_lock_owner(True)
+            service_manager.supervise()
+            logger.info(f"Background services started in worker pid {os.getpid()}")
+        else:
+            logger.info(
+                f"Worker pid {os.getpid()} declined service lock; another worker owns background services"
+            )
+    from devplacepy.services.statistics.tracking import start_visit_flusher
+
+    start_visit_flusher()
+    logger.info(f"DevPlace started on port {PORT}")
+    yield
+    logger.info("Shutting down services...")
+    from devplacepy.services.statistics.tracking import flush_visits
+
+    flush_visits()
+    await service_manager.shutdown_all()
+    await background.stop()
+    from devplacepy.services.containers import forward
+
+    await forward.close_client()
+    from devplacepy import push
+
+    await push.shutdown_providers()

devplacepy/main.py

@@ -1,56 +1,26 @@
 # retoor <retoor@molodetz.nl>
 
-import asyncio
-import fcntl
 import logging
-import os
-import time
-from collections import defaultdict
-from contextlib import asynccontextmanager, contextmanager
-from pathlib import Path
-from fastapi import FastAPI, Request, WebSocket
-from fastapi.responses import HTMLResponse, RedirectResponse
-from fastapi.staticfiles import StaticFiles
-from fastapi.exceptions import RequestValidationError
-from starlette.middleware.gzip import GZipMiddleware
+
+from fastapi import FastAPI
+
+from devplacepy import errors as error_handlers
 from devplacepy.config import (
+    LOG_LEVEL,
     STATIC_DIR,
     STATIC_VERSION,
     UPLOADS_DIR,
-    PORT,
-    SERVICE_LOCK_FILE,
-    INIT_LOCK_FILE,
-    ensure_data_dirs,
 )
-from devplacepy.database import (
-    init_db,
-    get_table,
-    db,
-    refresh_snapshot,
-    get_users_by_uids,
-    get_comment_counts_by_post_uids,
-    get_vote_counts,
-    get_news_images_by_uids,
-    get_setting,
-    get_int_setting,
-    interleave_by_author,
-    get_user_post_count,
-    get_user_stars,
-    get_blocked_uids,
-    get_top_authors,
-    get_trending_topics,
-)
-from devplacepy.templating import templates, jinja_unread_count
-from devplacepy.cache import TTLCache
-from devplacepy.responses import respond, wants_json, json_error
-from devplacepy.schemas import LandingOut, ValidationErrorOut
-from fastapi.responses import JSONResponse
-from devplacepy.utils import get_current_user, time_ago, safe_next, client_ip
-from devplacepy.seo import base_seo_context, site_url, website_schema
+from devplacepy.lifespan import lifespan
+from devplacepy.middleware import register as register_middleware
 from devplacepy.routers import (
     auth,
+    battles,
+    nodes,
+    sync,
     feed,
     posts,
+    topics,
     comments,
     projects,
     profile,
@@ -74,6 +44,8 @@ from devplacepy.routers import (
     reactions,
     reports,
     bookmarks,
+    notes,
+    skills,
     polls,
     docs,
     openai_gateway,
@@ -88,231 +60,18 @@ from devplacepy.routers import (
     pubsub,
     game,
     workspaces,
+    landing,
+)
+from devplacepy.static_files import (
+    CachedStaticFiles,
+    FallbackStaticFiles,
+    UploadStaticFiles,
 )
-from devplacepy.services.manager import service_manager
-from devplacepy.services.background import background
-from devplacepy.services.news import NewsService
-from devplacepy.services.bot import BotsService
-from devplacepy.services.openai_gateway import GatewayService
-from devplacepy.services.devii import DeviiService
-from devplacepy.services.jobs.zip_service import ZipService
-from devplacepy.services.jobs.fork_service import ForkService
-from devplacepy.services.jobs.issue_create_service import IssueCreateService
-from devplacepy.services.jobs.planning_service import PlanningReportService
-from devplacepy.services.jobs.seo.service import SeoService
-from devplacepy.services.jobs.seo_meta_service import SeoMetaService
-from devplacepy.services.jobs.award_service import AwardService
-from devplacepy.services.backup import BackupService
-from devplacepy.services.dbapi.service import DbApiJobService
-from devplacepy.services.pubsub import PubSubService
-from devplacepy.services.notification_relay import NotificationRelayService
-from devplacepy.services.live_view_relay import LiveViewRelayService
-from devplacepy.services.presence_relay import PresenceRelayService
-from devplacepy.services import presence
-from devplacepy.services.correction import PENDING_SCOPE_KEY
-from devplacepy.services.jobs.deepsearch.service import DeepsearchService
-from devplacepy.services.jobs.isslop.service import IsslopService
-from devplacepy.services.gitea.service import IssueTrackerService
-from devplacepy.services.containers.service import ContainerService
-from devplacepy.services.containers.workspace_service import WorkspaceService
-from devplacepy.services.xmlrpc import XmlrpcService
-from devplacepy.services.audit import AuditService
-from devplacepy.services.moderation.service import ModerationService
-from devplacepy.services.moderation.screening import ContentRefused
-from devplacepy.services.acceptance.service import AcceptanceService
-from devplacepy.services.audit import record as audit
-from devplacepy.services.push import PushService
-from devplacepy.services.telegram import TelegramService
-from devplacepy.services.telegram.outbox_service import TelegramOutboxService
 
 logging.basicConfig(
-    level=logging.INFO,
+    level=getattr(logging, LOG_LEVEL, logging.INFO),
     format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
 )
-logger = logging.getLogger(__name__)
-
-_rate_limit_store = defaultdict(list)
-RATE_LIMIT = int(os.environ.get("DEVPLACE_RATE_LIMIT", "60"))
-RATE_WINDOW = 60
-WEB_WORKERS = max(1, int(os.environ.get("DEVPLACE_WEB_WORKERS", "1")))
-RATE_LIMIT_DISABLED = os.environ.get("DEVPLACE_DISABLE_RATE_LIMIT") == "1"
-
-HOT_SETTINGS_TTL = 2.0
-_hot_settings_value: dict = {}
-_hot_settings_at = 0.0
-_last_rate_sweep = 0.0
-RATE_SWEEP_INTERVAL = 60.0
-
-
-def _hot_settings() -> dict:
-    global _hot_settings_value, _hot_settings_at
-    now = time.monotonic()
-    if not _hot_settings_value or now - _hot_settings_at >= HOT_SETTINGS_TTL:
-        _hot_settings_value = {
-            "maintenance_mode": get_setting("maintenance_mode", "0"),
-            "rate_limit_per_minute": max(
-                1, get_int_setting("rate_limit_per_minute", RATE_LIMIT)
-            ),
-            "rate_limit_window_seconds": max(
-                1, get_int_setting("rate_limit_window_seconds", RATE_WINDOW)
-            ),
-        }
-        _hot_settings_at = now
-    return _hot_settings_value
-
-
-def _sweep_rate_limit_store(window_start: float) -> None:
-    stale = [
-        ip
-        for ip, timestamps in _rate_limit_store.items()
-        if not timestamps or timestamps[-1] <= window_start
-    ]
-    for ip in stale:
-        del _rate_limit_store[ip]
-
-
-def _worker_rate_limit(limit: int) -> int:
-    return max(1, -(-limit // WEB_WORKERS))
-
-
-_service_lock_handle = None
-
-
-@contextmanager
-def init_lock():
-    handle = open(INIT_LOCK_FILE, "w")
-    try:
-        fcntl.flock(handle, fcntl.LOCK_EX)
-        yield
-    finally:
-        fcntl.flock(handle, fcntl.LOCK_UN)
-        handle.close()
-
-
-def acquire_service_lock() -> bool:
-    global _service_lock_handle
-    handle = open(SERVICE_LOCK_FILE, "w")
-    try:
-        fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
-    except OSError:
-        handle.close()
-        return False
-    _service_lock_handle = handle
-    return True
-
-
-INLINE_MEDIA_EXTENSIONS = {
-    ".jpg",
-    ".jpeg",
-    ".png",
-    ".gif",
-    ".webp",
-    ".bmp",
-    ".tiff",
-    ".mp4",
-    ".webm",
-    ".ogv",
-    ".mov",
-    ".m4v",
-    ".mp3",
-}
-
-
-class UploadStaticFiles(StaticFiles):
-    async def get_response(self, path, scope):
-        response = await super().get_response(path, scope)
-        disposition = (
-            "inline"
-            if Path(path).suffix.lower() in INLINE_MEDIA_EXTENSIONS
-            else "attachment"
-        )
-        response.headers["Content-Disposition"] = disposition
-        response.headers["Cache-Control"] = "public, max-age=604800"
-        return response
-
-
-class CachedStaticFiles(StaticFiles):
-    async def get_response(self, path, scope):
-        response = await super().get_response(path, scope)
-        if Path(path).name == "service-worker.js":
-            response.headers["Cache-Control"] = "no-cache"
-        else:
-            response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
-        return response
-
-
-class FallbackStaticFiles(StaticFiles):
-    async def get_response(self, path, scope):
-        response = await super().get_response(path, scope)
-        if Path(path).name == "service-worker.js":
-            response.headers["Cache-Control"] = "no-cache"
-        else:
-            response.headers["Cache-Control"] = "public, max-age=3600"
-        return response
-
-
-@asynccontextmanager
-async def lifespan(app: FastAPI):
-    ensure_data_dirs()
-    with init_lock():
-        init_db()
-        from devplacepy.push import ensure_certificates
-
-        ensure_certificates()
-    service_manager.register(NewsService())
-    service_manager.register(BotsService())
-    service_manager.register(GatewayService())
-    service_manager.register(DeviiService())
-    service_manager.register(ZipService())
-    service_manager.register(ForkService())
-    service_manager.register(SeoService())
-    service_manager.register(SeoMetaService())
-    service_manager.register(AwardService())
-    service_manager.register(BackupService())
-    service_manager.register(DbApiJobService())
-    service_manager.register(PubSubService())
-    service_manager.register(NotificationRelayService())
-    service_manager.register(LiveViewRelayService())
-    service_manager.register(PresenceRelayService())
-    service_manager.register(DeepsearchService())
-    service_manager.register(IsslopService())
-    service_manager.register(IssueCreateService())
-    service_manager.register(PlanningReportService())
-    service_manager.register(IssueTrackerService())
-    service_manager.register(ContainerService())
-    service_manager.register(WorkspaceService())
-    service_manager.register(XmlrpcService())
-    service_manager.register(AuditService())
-    service_manager.register(ModerationService())
-    service_manager.register(AcceptanceService())
-    service_manager.register(PushService())
-    service_manager.register(TelegramService())
-    service_manager.register(TelegramOutboxService())
-    if not os.environ.get("DEVPLACE_DISABLE_SERVICES"):
-        await background.start()
-        if acquire_service_lock():
-            service_manager.set_lock_owner(True)
-            service_manager.supervise()
-            logger.info(f"Background services started in worker pid {os.getpid()}")
-        else:
-            logger.info(
-                f"Worker pid {os.getpid()} declined service lock; another worker owns background services"
-            )
-    from devplacepy.services.statistics.tracking import start_visit_flusher
-
-    start_visit_flusher()
-    logger.info(f"DevPlace started on port {PORT}")
-    yield
-    logger.info("Shutting down services...")
-    from devplacepy.services.statistics.tracking import flush_visits
-
-    flush_visits()
-    await service_manager.shutdown_all()
-    await background.stop()
-    from devplacepy.services.containers import forward
-
-    await forward.close_client()
-
 
 app = FastAPI(
     title="DevPlace",
@@ -333,141 +92,12 @@ app.mount(
 )
 app.mount("/static", FallbackStaticFiles(directory=str(STATIC_DIR)), name="static")
 
-
-@app.exception_handler(404)
-async def not_found(request: Request, exc):
-    if wants_json(request):
-        return json_error(404, "Not found")
-    seo_ctx = base_seo_context(
-        request,
-        title="Not Found - DevPlace",
-        description="The page you requested does not exist.",
-        robots="noindex",
-    )
-    return templates.TemplateResponse(
-        request,
-        "error.html",
-        {
-            **seo_ctx,
-            "request": request,
-            "error_code": 404,
-            "error_message": "Page not found",
-        },
-        status_code=404,
-    )
-
-
-@app.exception_handler(500)
-async def server_error(request: Request, exc):
-    logger.exception("500 error on %s %s", request.method, request.url.path)
-    if wants_json(request):
-        return json_error(500, "Internal server error")
-    seo_ctx = base_seo_context(
-        request,
-        title="Server Error - DevPlace",
-        description="Something went wrong.",
-        robots="noindex",
-    )
-    return templates.TemplateResponse(
-        request,
-        "error.html",
-        {
-            **seo_ctx,
-            "request": request,
-            "error_code": 500,
-            "error_message": "Internal server error",
-        },
-        status_code=500,
-    )
-
-
-@app.exception_handler(ContentRefused)
-async def content_refused(request: Request, exc: ContentRefused):
-    logger.info("content refused on %s %s: %s", request.method, request.url.path, exc.message)
-    if wants_json(request):
-        return json_error(400, exc.message, categories=list(exc.categories))
-    seo_ctx = base_seo_context(
-        request,
-        title="Content not published - DevPlace",
-        description=exc.message,
-        robots="noindex",
-    )
-    return templates.TemplateResponse(
-        request,
-        "error.html",
-        {
-            **seo_ctx,
-            "request": request,
-            "error_code": 400,
-            "error_message": exc.message,
-        },
-        status_code=400,
-    )
-
-
-_AUTH_FORM_PAGES = {
-    "/auth/signup": ("signup.html", "Join DevPlace"),
-    "/auth/login": ("login.html", "Sign In"),
-    "/auth/forgot-password": ("forgot_password.html", "Reset Password"),
-}
-
-_FRIENDLY_ERRORS = {
-    ("username", "too_short"): "Username must be between 3 and 32 characters",
-    ("username", "too_long"): "Username must be between 3 and 32 characters",
-    ("password", "too_short"): "Password must be at least 6 characters",
-}
-
-
-def _friendly_error(err):
-    field = err["loc"][-1] if err.get("loc") else ""
-    key = (field, err.get("type", "").replace("string_", ""))
-    if key in _FRIENDLY_ERRORS:
-        return _FRIENDLY_ERRORS[key]
-    msg = err.get("msg", "Invalid input")
-    prefix = "Value error, "
-    return msg[len(prefix) :] if msg.startswith(prefix) else msg
-
-
-@app.exception_handler(RequestValidationError)
-async def on_validation_error(request: Request, exc: RequestValidationError):
-    errors = [_friendly_error(e) for e in exc.errors()]
-    if wants_json(request):
-        fields: dict = {}
-        for raw, friendly in zip(exc.errors(), errors):
-            name = raw["loc"][-1] if raw.get("loc") else "_"
-            fields.setdefault(str(name), []).append(friendly)
-        return JSONResponse(
-            ValidationErrorOut(fields=fields, messages=errors).model_dump(mode="json"),
-            status_code=422,
-        )
-    path = request.url.path
-    page = _AUTH_FORM_PAGES.get(path)
-    if page is None and path.startswith("/auth/reset-password/"):
-        page = ("reset_password.html", "Set New Password")
-    if page:
-        template_name, title = page
-        context = {
-            **base_seo_context(request, title=title, robots="noindex,nofollow"),
-            "request": request,
-            "errors": errors,
-        }
-        try:
-            form = await request.form()
-            context.update({k: v for k, v in form.items() if isinstance(v, str)})
-        except Exception:
-            pass
-        if "token" in request.path_params:
-            context["token"] = request.path_params["token"]
-        return templates.TemplateResponse(
-            request, template_name, context, status_code=400
-        )
-    referer = safe_next(request.headers.get("referer"), "/feed")
-    return RedirectResponse(url=referer, status_code=303)
-
+error_handlers.register(app)
 
 app.include_router(auth.router, prefix="/auth")
 app.include_router(feed.router, prefix="/feed")
 app.include_router(posts.router, prefix="/posts")
+app.include_router(topics.router, prefix="/topics")
 app.include_router(comments.router, prefix="/comments")
 app.include_router(projects.router, prefix="/projects")
 app.include_router(profile.router, prefix="/profile")
@@ -477,6 +107,8 @@ app.include_router(votes.router, prefix="/votes")
 app.include_router(reactions.router, prefix="/reactions")
 app.include_router(reports.router, prefix="/reports")
 app.include_router(bookmarks.router, prefix="/bookmarks")
+app.include_router(notes.router, prefix="/notes")
+app.include_router(skills.router, prefix="/skills")
 app.include_router(polls.router, prefix="/polls")
 app.include_router(avatar.router, prefix="/avatar")
 app.include_router(awards.router, prefix="/awards")
@@ -504,364 +136,10 @@ app.include_router(dbapi.router, prefix="/dbapi")
 app.include_router(pubsub.router, prefix="/pubsub")
 app.include_router(game.router, prefix="/game")
 app.include_router(quizzes.router, prefix="/quizzes")
+app.include_router(battles.router, prefix="/battles")
+app.include_router(nodes.router, prefix="/nodes")
+app.include_router(sync.router, prefix="/sync")
 app.include_router(workspaces.router, prefix="/workspaces")
+app.include_router(landing.router)
 
-
-@app.middleware("http")
-async def refresh_db_snapshot(request: Request, call_next):
-    if not request.url.path.startswith(("/static", "/avatar")):
-        refresh_snapshot()
-    return await call_next(request)
-
-
-@app.middleware("http")
-async def await_pending_corrections(request: Request, call_next):
-    response = await call_next(request)
-    pending = request.scope.get(PENDING_SCOPE_KEY)
-    if pending:
-        await asyncio.gather(*pending, return_exceptions=True)
-    return response
-
-
-def _frame_ancestors() -> str:
-    from devplacepy.services.containers.workspace import naming
-
-    tunnel_domain = naming.domain()
-    if not tunnel_domain:
-        return "'self'"
-    return f"'self' https://*.{tunnel_domain}"
-
-
-@app.middleware("http")
-async def add_security_headers(request: Request, call_next):
-    response = await call_next(request)
-    if not response.headers.get("X-Robots-Tag"):
-        response.headers["X-Robots-Tag"] = "index, follow"
-    response.headers["X-Content-Type-Options"] = "nosniff"
-    response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
-    response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
-    if not request.url.path.startswith("/p/"):
-        response.headers["Content-Security-Policy"] = (
-            "object-src 'none'; base-uri 'self'; "
-            f"frame-ancestors {_frame_ancestors()}; form-action 'self'"
-        )
-    if request.url.path.startswith("/admin"):
-        response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0"
-        response.headers["Pragma"] = "no-cache"
-        response.headers["Expires"] = "0"
-    return response
-
-
-@app.middleware("http")
-async def rate_limit_middleware(request: Request, call_next):
-    if RATE_LIMIT_DISABLED:
-        return await call_next(request)
-    if request.method in (
-        "POST",
-        "PUT",
-        "DELETE",
-        "PATCH",
-    ) and not request.url.path.startswith(("/openai", "/xmlrpc")):
-        settings = _hot_settings()
-        limit = _worker_rate_limit(settings["rate_limit_per_minute"])
-        window = settings["rate_limit_window_seconds"]
-        ip = client_ip(request, default="unknown")
-        now = time.time()
-        window_start = now - window
-        global _last_rate_sweep
-        if now - _last_rate_sweep >= RATE_SWEEP_INTERVAL:
-            _sweep_rate_limit_store(window_start)
-            _last_rate_sweep = now
-        timestamps = [t for t in _rate_limit_store.get(ip, ()) if t > window_start]
-        if len(timestamps) >= limit:
-            _rate_limit_store[ip] = timestamps
-            audit.record(
-                request,
-                "security.rate_limit.block",
-                result="denied",
-                summary=f"request from {ip} blocked by rate limit",
-                metadata={"ip": ip, "limit": limit, "window_seconds": window},
-            )
-            retry_after = {"Retry-After": str(window)}
-            if wants_json(request):
-                response = json_error(429, "Rate limit exceeded. Try again later.")
-                response.headers["Retry-After"] = str(window)
-                return response
-            return HTMLResponse(
-                "Rate limit exceeded. Try again later.",
-                status_code=429,
-                headers=retry_after,
-            )
-        timestamps.append(now)
-        _rate_limit_store[ip] = timestamps
-    return await call_next(request)
-
-
-_MAINTENANCE_ALLOWED_PREFIXES = ("/static", "/avatar", "/auth", "/admin", "/openai")
-
-
-@app.middleware("http")
-async def maintenance_middleware(request: Request, call_next):
-    if _hot_settings()["maintenance_mode"] != "1":
-        return await call_next(request)
-    if request.url.path.startswith(_MAINTENANCE_ALLOWED_PREFIXES):
-        return await call_next(request)
-    user = get_current_user(request)
-    if user and user.get("role") == "Admin":
-        return await call_next(request)
-    message = get_setting(
-        "maintenance_message",
-        "DevPlace is undergoing scheduled maintenance. Please check back shortly.",
-    )
-    audit.record(
-        request,
-        "security.maintenance.block",
-        user=user,
-        result="denied",
-        summary="non-admin request blocked by maintenance mode",
-    )
-    if wants_json(request):
-        return json_error(503, message)
-    seo_ctx = base_seo_context(
-        request, title="Maintenance - DevPlace", description=message, robots="noindex"
-    )
-    return templates.TemplateResponse(
-        request,
-        "error.html",
-        {**seo_ctx, "request": request, "error_code": 503, "error_message": message},
-        status_code=503,
-    )
-
-
-_TERMS_ALLOWED_PREFIXES = (
-    "/static",
-    "/avatar",
-    "/auth",
-    "/docs",
-    "/reports",
-    "/block",
-    "/mute",
-    "/openai",
-)
-
-_TERMS_GATED_METHODS = ("POST", "PUT", "DELETE", "PATCH")
-
-
-def _terms_exempt(path: str) -> bool:
-    if path.startswith(_TERMS_ALLOWED_PREFIXES):
-        return True
-    return path.startswith("/profile/") and (
-        path.endswith("/delete") or path.endswith("/consent")
-    )
-
-
-@app.middleware("http")
-async def terms_acceptance_gate(request: Request, call_next):
-    if request.method not in _TERMS_GATED_METHODS or _terms_exempt(request.url.path):
-        return await call_next(request)
-    from devplacepy.routers.auth.terms import (
-        TERMS_ACCEPTANCE_CODE,
-        current_terms_version,
-        needs_acceptance,
-    )
-
-    user = get_current_user(request)
-    if not needs_acceptance(user):
-        return await call_next(request)
-    message = "Accept the updated Terms of Service to continue."
-    if wants_json(request):
-        return json_error(
-            403,
-            message,
-            code=TERMS_ACCEPTANCE_CODE,
-            redirect="/auth/accept-terms",
-            terms_version=current_terms_version(),
-        )
-    return RedirectResponse(url="/auth/accept-terms", status_code=303)
-
-
-@app.middleware("http")
-async def track_presence(request: Request, call_next):
-    path = request.url.path
-    if not path.startswith(("/static", "/avatar")):
-        user = get_current_user(request)
-        if user:
-            presence.touch(user["uid"])
-    return await call_next(request)
-
-
-@app.middleware("http")
-async def visit_statistics(request: Request, call_next):
-    from devplacepy.services.statistics.tracking import track_visit
-
-    response = await call_next(request)
-    track_visit(request, response.status_code)
-    return response
-
-
-@app.middleware("http")
-async def response_timing(request: Request, call_next):
-    start = time.perf_counter()
-    request.state.request_start = start
-    response = await call_next(request)
-    response.headers["X-Response-Time"] = f"{(time.perf_counter() - start) * 1000:.1f}ms"
-    return response
-
-
-class TunnelDispatchMiddleware:
-    def __init__(self, app):
-        self.app = app
-
-    @staticmethod
-    def _host(scope) -> str:
-        for key, value in scope.get("headers") or []:
-            if key == b"host":
-                return value.decode("latin-1")
-        return ""
-
-    async def __call__(self, scope, receive, send):
-        if scope["type"] not in ("http", "websocket"):
-            await self.app(scope, receive, send)
-            return
-        from devplacepy.routers import tunnel as tunnel_router
-        from devplacepy.services.containers.workspace import naming
-
-        try:
-            if not naming.is_tunnel_host(self._host(scope)):
-                await self.app(scope, receive, send)
-                return
-        except Exception:
-            await self.app(scope, receive, send)
-            return
-        path = (scope.get("path") or "/").lstrip("/")
-        if scope["type"] == "websocket":
-            websocket = WebSocket(scope, receive, send)
-            await tunnel_router.handle_ws(websocket, path)
-            return
-        request = Request(scope, receive)
-        response = await tunnel_router.handle_http(request, path)
-        await response(scope, receive, send)
-
-
-app.add_middleware(TunnelDispatchMiddleware)
-app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=5)
-
-
-_home_cache = TTLCache(ttl=int(os.environ.get("DEVPLACE_HOME_CACHE_TTL", "60")), max_size=4)
-
-
-def _landing_news():
-    cached = _home_cache.get("news")
-    if cached is not None:
-        return cached
-    articles = []
-    if "news" in db.tables:
-        news_table = get_table("news")
-        raw = list(
-            news_table.find(show_on_landing=1, order_by=["-synced_at"], _limit=6)
-        )
-        images_by_news = get_news_images_by_uids([a["uid"] for a in raw])
-        for a in raw:
-            articles.append(
-                {
-                    "uid": a["uid"],
-                    "slug": a.get("slug", ""),
-                    "title": a.get("title", ""),
-                    "description": (a.get("description", "") or "")[:250],
-                    "url": a.get("url", ""),
-                    "source_name": a.get("source_name", ""),
-                    "grade": a.get("grade", 0),
-                    "featured": a.get("featured", 0),
-                    "synced_at": a.get("synced_at", "") or "",
-                    "time_ago": time_ago(a["synced_at"]),
-                    "image_url": a.get("image_url", "") or images_by_news.get(a["uid"], ""),
-                }
-            )
-    _home_cache.set("news", articles)
-    return articles
-
-
-def _landing_recent_posts(blocked):
-    if not blocked:
-        cached = _home_cache.get("posts")
-        if cached is not None:
-            return cached
-    posts = []
-    if "posts" in db.tables:
-        posts_table = get_table("posts")
-        clauses = []
-        if blocked:
-            clauses.append(posts_table.table.columns.user_uid.notin_(blocked))
-        raw_posts = list(
-            posts_table.find(
-                *clauses, deleted_at=None, order_by=["-created_at"], _limit=6
-            )
-        )
-        raw_posts = interleave_by_author(raw_posts)
-        if raw_posts:
-            post_uids = [p["uid"] for p in raw_posts]
-            author_uids = [p["user_uid"] for p in raw_posts]
-            authors = get_users_by_uids(author_uids)
-            comment_counts = get_comment_counts_by_post_uids(post_uids)
-            upvotes, downvotes = get_vote_counts(post_uids)
-            for p in raw_posts:
-                posts.append(
-                    {
-                        "post": p,
-                        "author": authors.get(p["user_uid"]),
-                        "time_ago": time_ago(p["created_at"]),
-                        "comment_count": comment_counts.get(p["uid"], 0),
-                        "stars": upvotes.get(p["uid"], 0) - downvotes.get(p["uid"], 0),
-                        "slug": p.get("slug", "") or p["uid"],
-                    }
-                )
-    if not blocked:
-        _home_cache.set("posts", posts)
-    return posts
-
-
-@app.get("/")
-async def landing(request: Request):
-    user = get_current_user(request)
-
-    landing_articles = _landing_news()
-    blocked = get_blocked_uids(user["uid"]) if user else frozenset()
-    landing_posts = _landing_recent_posts(blocked)
-
-    base = site_url(request)
-    seo_ctx = base_seo_context(
-        request,
-        title="DevPlace - The Developer Social Network",
-        description="Track industry shifts. Discover bold releases. Share what you're building in an open environment built by developers, for developers.",
-        breadcrumbs=[],
-        schemas=[website_schema(base)],
-    )
-    user_xp = user.get("xp", 0) or 0 if user else 0
-    user_level = user.get("level", 1) or 1 if user else 1
-    xp_progress_pct = (user_xp % 100) if user_xp else 0
-    unread_count = jinja_unread_count(user["uid"]) if user else 0
-
-    top_contributors = get_top_authors(5) if not blocked else []
-    trending_topics = get_trending_topics(6) if not blocked else []
-
-    return respond(
-        request,
-        "landing.html",
-        {
-            **seo_ctx,
-            "request": request,
-            "user": user,
-            "is_authenticated": bool(user),
-            "user_post_count": get_user_post_count(user["uid"]) if user else 0,
-            "user_stars": get_user_stars(user["uid"]) if user else 0,
-            "user_xp": user_xp,
-            "user_level": user_level,
-            "xp_progress_pct": xp_progress_pct,
-            "unread_count": unread_count,
-            "landing_articles": landing_articles,
-            "landing_posts": landing_posts,
-            "top_contributors": top_contributors,
-            "trending_topics": trending_topics,
-        },
-        model=LandingOut,
-    )
+register_middleware(app)

devplacepy/middleware/__init__.py

@@ -0,0 +1,34 @@
+# retoor <retoor@molodetz.nl>
+
+from starlette.middleware.gzip import GZipMiddleware
+
+from devplacepy.middleware.guards import (
+    maintenance_middleware,
+    rate_limit_middleware,
+    terms_acceptance_gate,
+)
+from devplacepy.middleware.observe import (
+    response_timing,
+    track_presence,
+    visit_statistics,
+)
+from devplacepy.middleware.pipeline import (
+    add_security_headers,
+    await_pending_corrections,
+    refresh_db_snapshot,
+)
+from devplacepy.middleware.tunnel import TunnelDispatchMiddleware
+
+
+def register(app):
+    app.middleware("http")(refresh_db_snapshot)
+    app.middleware("http")(await_pending_corrections)
+    app.middleware("http")(add_security_headers)
+    app.middleware("http")(rate_limit_middleware)
+    app.middleware("http")(maintenance_middleware)
+    app.middleware("http")(terms_acceptance_gate)
+    app.middleware("http")(track_presence)
+    app.middleware("http")(visit_statistics)
+    app.middleware("http")(response_timing)
+    app.add_middleware(TunnelDispatchMiddleware)
+    app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=5)

devplacepy/middleware/guards.py

@@ -0,0 +1,138 @@
+# retoor <retoor@molodetz.nl>
+
+import time
+
+from fastapi import Request
+from fastapi.responses import HTMLResponse, RedirectResponse
+
+from devplacepy.database import get_setting
+from devplacepy.middleware import state
+from devplacepy.responses import json_error, wants_json
+from devplacepy.seo import base_seo_context
+from devplacepy.services.audit import record as audit
+from devplacepy.templating import templates
+from devplacepy.utils import client_ip, get_current_user
+
+
+async def rate_limit_middleware(request: Request, call_next):
+    if state.RATE_LIMIT_DISABLED:
+        return await call_next(request)
+    if request.method in (
+        "POST",
+        "PUT",
+        "DELETE",
+        "PATCH",
+    ) and not request.url.path.startswith(("/openai", "/xmlrpc")):
+        settings = state._hot_settings()
+        limit = state._worker_rate_limit(settings["rate_limit_per_minute"])
+        window = settings["rate_limit_window_seconds"]
+        ip = client_ip(request, default="unknown")
+        now = time.time()
+        window_start = now - window
+        if now - state._last_rate_sweep >= state.RATE_SWEEP_INTERVAL:
+            state._sweep_rate_limit_store(window_start)
+            state._last_rate_sweep = now
+        timestamps = [t for t in state._rate_limit_store.get(ip, ()) if t > window_start]
+        if len(timestamps) >= limit:
+            state._rate_limit_store[ip] = timestamps
+            audit.record(
+                request,
+                "security.rate_limit.block",
+                result="denied",
+                summary=f"request from {ip} blocked by rate limit",
+                metadata={"ip": ip, "limit": limit, "window_seconds": window},
+            )
+            retry_after = {"Retry-After": str(window)}
+            if wants_json(request):
+                response = json_error(429, "Rate limit exceeded. Try again later.")
+                response.headers["Retry-After"] = str(window)
+                return response
+            return HTMLResponse(
+                "Rate limit exceeded. Try again later.",
+                status_code=429,
+                headers=retry_after,
+            )
+        timestamps.append(now)
+        state._rate_limit_store[ip] = timestamps
+    return await call_next(request)
+
+
+_MAINTENANCE_ALLOWED_PREFIXES = ("/static", "/avatar", "/auth", "/admin", "/openai")
+
+
+async def maintenance_middleware(request: Request, call_next):
+    if state._hot_settings()["maintenance_mode"] != "1":
+        return await call_next(request)
+    if request.url.path.startswith(_MAINTENANCE_ALLOWED_PREFIXES):
+        return await call_next(request)
+    user = get_current_user(request)
+    if user and user.get("role") == "Admin":
+        return await call_next(request)
+    message = get_setting(
+        "maintenance_message",
+        "DevPlace is undergoing scheduled maintenance. Please check back shortly.",
+    )
+    audit.record(
+        request,
+        "security.maintenance.block",
+        user=user,
+        result="denied",
+        summary="non-admin request blocked by maintenance mode",
+    )
+    if wants_json(request):
+        return json_error(503, message)
+    seo_ctx = base_seo_context(
+        request, title="Maintenance - DevPlace", description=message, robots="noindex"
+    )
+    return templates.TemplateResponse(
+        request,
+        "error.html",
+        {**seo_ctx, "request": request, "error_code": 503, "error_message": message},
+        status_code=503,
+    )
+
+
+_TERMS_ALLOWED_PREFIXES = (
+    "/static",
+    "/avatar",
+    "/auth",
+    "/docs",
+    "/reports",
+    "/block",
+    "/mute",
+    "/openai",
+)
+
+_TERMS_GATED_METHODS = ("POST", "PUT", "DELETE", "PATCH")
+
+
+def _terms_exempt(path: str) -> bool:
+    if path.startswith(_TERMS_ALLOWED_PREFIXES):
+        return True
+    return path.startswith("/profile/") and (
+        path.endswith("/delete") or path.endswith("/consent")
+    )
+
+
+async def terms_acceptance_gate(request: Request, call_next):
+    if request.method not in _TERMS_GATED_METHODS or _terms_exempt(request.url.path):
+        return await call_next(request)
+    from devplacepy.routers.auth.terms import (
+        TERMS_ACCEPTANCE_CODE,
+        current_terms_version,
+        needs_acceptance,
+    )
+
+    user = get_current_user(request)
+    if not needs_acceptance(user):
+        return await call_next(request)
+    message = "Accept the updated Terms of Service to continue."
+    if wants_json(request):
+        return json_error(
+            403,
+            message,
+            code=TERMS_ACCEPTANCE_CODE,
+            redirect="/auth/accept-terms",
+            terms_version=current_terms_version(),
+        )
+    return RedirectResponse(url="/auth/accept-terms", status_code=303)

devplacepy/middleware/observe.py

@@ -0,0 +1,33 @@
+# retoor <retoor@molodetz.nl>
+
+import time
+
+from fastapi import Request
+
+from devplacepy.services import presence
+from devplacepy.utils import get_current_user
+
+
+async def track_presence(request: Request, call_next):
+    path = request.url.path
+    if not path.startswith(("/static", "/avatar")):
+        user = get_current_user(request)
+        if user:
+            presence.touch(user["uid"])
+    return await call_next(request)
+
+
+async def visit_statistics(request: Request, call_next):
+    from devplacepy.services.statistics.tracking import track_visit
+
+    response = await call_next(request)
+    track_visit(request, response.status_code)
+    return response
+
+
+async def response_timing(request: Request, call_next):
+    start = time.perf_counter()
+    request.state.request_start = start
+    response = await call_next(request)
+    response.headers["X-Response-Time"] = f"{(time.perf_counter() - start) * 1000:.1f}ms"
+    return response

devplacepy/middleware/pipeline.py

@@ -0,0 +1,50 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+
+from fastapi import Request
+
+from devplacepy.database import refresh_snapshot
+from devplacepy.services.correction import PENDING_SCOPE_KEY
+
+
+async def refresh_db_snapshot(request: Request, call_next):
+    if not request.url.path.startswith(("/static", "/avatar")):
+        refresh_snapshot()
+    return await call_next(request)
+
+
+async def await_pending_corrections(request: Request, call_next):
+    response = await call_next(request)
+    pending = request.scope.get(PENDING_SCOPE_KEY)
+    if pending:
+        await asyncio.gather(*pending, return_exceptions=True)
+    return response
+
+
+def _frame_ancestors() -> str:
+    from devplacepy.services.containers.workspace import naming
+
+    tunnel_domain = naming.domain()
+    if not tunnel_domain:
+        return "'self'"
+    return f"'self' https://*.{tunnel_domain}"
+
+
+async def add_security_headers(request: Request, call_next):
+    response = await call_next(request)
+    if not response.headers.get("X-Robots-Tag"):
+        response.headers["X-Robots-Tag"] = "index, follow"
+    response.headers["X-Content-Type-Options"] = "nosniff"
+    response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
+    response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
+    if not request.url.path.startswith("/p/"):
+        response.headers["Content-Security-Policy"] = (
+            "object-src 'none'; base-uri 'self'; "
+            f"frame-ancestors {_frame_ancestors()}; form-action 'self'"
+        )
+    if request.url.path.startswith("/admin"):
+        response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate, max-age=0"
+        response.headers["Pragma"] = "no-cache"
+        response.headers["Expires"] = "0"
+    return response

devplacepy/middleware/state.py

@@ -0,0 +1,50 @@
+# retoor <retoor@molodetz.nl>
+
+import os
+import time
+from collections import defaultdict
+
+from devplacepy.database import get_int_setting, get_setting
+
+_rate_limit_store = defaultdict(list)
+RATE_LIMIT = int(os.environ.get("DEVPLACE_RATE_LIMIT", "60"))
+RATE_WINDOW = 60
+WEB_WORKERS = max(1, int(os.environ.get("DEVPLACE_WEB_WORKERS", "1")))
+RATE_LIMIT_DISABLED = os.environ.get("DEVPLACE_DISABLE_RATE_LIMIT") == "1"
+
+HOT_SETTINGS_TTL = 2.0
+_hot_settings_value: dict = {}
+_hot_settings_at = 0.0
+_last_rate_sweep = 0.0
+RATE_SWEEP_INTERVAL = 60.0
+
+
+def _hot_settings() -> dict:
+    global _hot_settings_value, _hot_settings_at
+    now = time.monotonic()
+    if not _hot_settings_value or now - _hot_settings_at >= HOT_SETTINGS_TTL:
+        _hot_settings_value = {
+            "maintenance_mode": get_setting("maintenance_mode", "0"),
+            "rate_limit_per_minute": max(
+                1, get_int_setting("rate_limit_per_minute", RATE_LIMIT)
+            ),
+            "rate_limit_window_seconds": max(
+                1, get_int_setting("rate_limit_window_seconds", RATE_WINDOW)
+            ),
+        }
+        _hot_settings_at = now
+    return _hot_settings_value
+
+
+def _sweep_rate_limit_store(window_start: float) -> None:
+    stale = [
+        ip
+        for ip, timestamps in _rate_limit_store.items()
+        if not timestamps or timestamps[-1] <= window_start
+    ]
+    for ip in stale:
+        del _rate_limit_store[ip]
+
+
+def _worker_rate_limit(limit: int) -> int:
+    return max(1, -(-limit // WEB_WORKERS))

devplacepy/middleware/tunnel.py

@@ -0,0 +1,38 @@
+# retoor <retoor@molodetz.nl>
+
+from fastapi import Request, WebSocket
+
+
+class TunnelDispatchMiddleware:
+    def __init__(self, app):
+        self.app = app
+
+    @staticmethod
+    def _host(scope) -> str:
+        for key, value in scope.get("headers") or []:
+            if key == b"host":
+                return value.decode("latin-1")
+        return ""
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] not in ("http", "websocket"):
+            await self.app(scope, receive, send)
+            return
+        from devplacepy.routers import tunnel as tunnel_router
+        from devplacepy.services.containers.workspace import naming
+
+        try:
+            if not naming.is_tunnel_host(self._host(scope)):
+                await self.app(scope, receive, send)
+                return
+        except Exception:
+            await self.app(scope, receive, send)
+            return
+        path = (scope.get("path") or "/").lstrip("/")
+        if scope["type"] == "websocket":
+            websocket = WebSocket(scope, receive, send)
+            await tunnel_router.handle_ws(websocket, path)
+            return
+        request = Request(scope, receive)
+        response = await tunnel_router.handle_http(request, path)
+        await response(scope, receive, send)

devplacepy/models/__init__.py

@@ -0,0 +1,289 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.models.shared import (
+    normalize_european_date,
+    normalize_website_url,
+    normalize_poll_options,
+    _declared_age,
+    normalize_index_list,
+)
+from devplacepy.models.auth import (
+    SignupForm,
+    LoginForm,
+    ForgotPasswordForm,
+    ResetPasswordForm,
+)
+from devplacepy.models.content import (
+    PostForm,
+    WarJoinForm,
+    PostEditForm,
+    CommentForm,
+    CommentEditForm,
+    NoteForm,
+    VoteForm,
+    ReactionForm,
+    PollVoteForm,
+)
+from devplacepy.models.projects import (
+    ProjectForm,
+    ProjectEditForm,
+    ProjectFlagForm,
+    ProjectScreenshotsForm,
+    ForkForm,
+    ProjectFileWriteForm,
+    ProjectFileMkdirForm,
+    ProjectFileMoveForm,
+    ProjectFileDeleteForm,
+    ProjectFileReplaceLinesForm,
+    ProjectFileInsertLinesForm,
+    ProjectFileDeleteLinesForm,
+    ProjectFileAppendForm,
+)
+from devplacepy.models.profile import (
+    CustomizationToggleForm,
+    AwardGiveForm,
+    NotificationPrefForm,
+    AiCorrectionForm,
+    AiModifierForm,
+    InteractionsForm,
+    TelegramPairForm,
+    ProfileForm,
+    ConsentForm,
+    MaturityForm,
+    MaturePreferenceForm,
+    AccountDeleteForm,
+)
+from devplacepy.models.containers import (
+    ContainerInstanceForm,
+    ContainerAdminCreateForm,
+    ContainerEditForm,
+    ContainerExecForm,
+    ContainerScheduleForm,
+    TunnelForm,
+    EditorPrefsForm,
+)
+from devplacepy.models.community import (
+    SkillInstallForm,
+    SkillEnabledForm,
+    SkillDeleteForm,
+    MessageForm,
+    GistForm,
+    GistEditForm,
+    IssueForm,
+    IssueCommentForm,
+    IssueAttachmentForm,
+    IssueStatusForm,
+)
+from devplacepy.models.uploads import (
+    UploadUrlForm,
+    GenerateImageForm,
+    AttachmentRenameForm,
+)
+from devplacepy.models.tools import (
+    PlanningForm,
+    SeoRunForm,
+    SEO_URL_PATTERN,
+    ISSLOP_URL_PATTERN,
+    ISSLOP_SINGLE_SLASH_PATTERN,
+    ISSLOP_SCHEME_PATTERN,
+    IsslopRunForm,
+    DEEPSEARCH_MIN_DEPTH,
+    DEEPSEARCH_MAX_DEPTH,
+    DEEPSEARCH_DEFAULT_DEPTH,
+    DEEPSEARCH_MIN_PAGES,
+    DEEPSEARCH_MAX_PAGES,
+    DEEPSEARCH_DEFAULT_PAGES,
+    DEEPSEARCH_MIN_QUERY,
+    DEEPSEARCH_MAX_QUERY,
+    DEEPSEARCH_MAX_MESSAGE,
+    DeepsearchRunForm,
+    DeepsearchChatForm,
+)
+from devplacepy.models.quizzes import (
+    QUIZ_KINDS,
+    QUIZ_OPTION_KINDS,
+    QuizForm,
+    QuizQuestionForm,
+    QuizReorderForm,
+    QuizAnswerForm,
+    QuizDocumentOption,
+    QuizDocumentQuestion,
+    QuizDocumentSettings,
+    QuizDocument,
+    QuizImportForm,
+)
+from devplacepy.models.game import (
+    GamePlantForm,
+    GameSlotForm,
+    GamePlotStopForm,
+    GamePerkForm,
+    GameQuestForm,
+    GameLegacyForm,
+    GameInfraForm,
+    GameCosmeticForm,
+    GameMasteryForm,
+    GameEraStartForm,
+)
+from devplacepy.models.admin import (
+    CommunityAddForm,
+    CommunitySchemaForm,
+    CommunityUpdateForm,
+    CommunityRunForm,
+    BackupRunForm,
+    BackupScheduleForm,
+    NotificationDefaultForm,
+    AdminRoleForm,
+    AdminPasswordForm,
+    AdminSettingsForm,
+    WorkspaceQuotaForm,
+    WorkspaceFlagForm,
+    WorkspaceSuspendForm,
+    ReportForm,
+    ReportStatusForm,
+    ModerationDecisionForm,
+    SuspensionForm,
+    BanForm,
+    NodeSettingsForm,
+    NodeFetchForm,
+    NODE_SUSPEND_MIN_MINUTES,
+    NODE_SUSPEND_MAX_MINUTES,
+    NODE_SETTING_FORM_MAX,
+    AdminNodeKillswitchForm,
+    AdminNodeSettingsForm,
+    AdminNodeClassForm,
+    AdminNodeSuspendForm,
+    NodeDeviceForm,
+)
+
+__all__ = [
+    "normalize_european_date",
+    "normalize_website_url",
+    "normalize_poll_options",
+    "_declared_age",
+    "normalize_index_list",
+    "SignupForm",
+    "LoginForm",
+    "ForgotPasswordForm",
+    "ResetPasswordForm",
+    "PostForm",
+    "WarJoinForm",
+    "PostEditForm",
+    "CommentForm",
+    "CommentEditForm",
+    "NoteForm",
+    "VoteForm",
+    "ReactionForm",
+    "PollVoteForm",
+    "ProjectForm",
+    "ProjectEditForm",
+    "ProjectFlagForm",
+    "ProjectScreenshotsForm",
+    "ForkForm",
+    "ProjectFileWriteForm",
+    "ProjectFileMkdirForm",
+    "ProjectFileMoveForm",
+    "ProjectFileDeleteForm",
+    "ProjectFileReplaceLinesForm",
+    "ProjectFileInsertLinesForm",
+    "ProjectFileDeleteLinesForm",
+    "ProjectFileAppendForm",
+    "CustomizationToggleForm",
+    "AwardGiveForm",
+    "NotificationPrefForm",
+    "AiCorrectionForm",
+    "AiModifierForm",
+    "InteractionsForm",
+    "TelegramPairForm",
+    "ProfileForm",
+    "ConsentForm",
+    "MaturityForm",
+    "MaturePreferenceForm",
+    "AccountDeleteForm",
+    "ContainerInstanceForm",
+    "ContainerAdminCreateForm",
+    "ContainerEditForm",
+    "ContainerExecForm",
+    "ContainerScheduleForm",
+    "TunnelForm",
+    "EditorPrefsForm",
+    "SkillInstallForm",
+    "SkillEnabledForm",
+    "SkillDeleteForm",
+    "MessageForm",
+    "GistForm",
+    "GistEditForm",
+    "IssueForm",
+    "IssueCommentForm",
+    "IssueAttachmentForm",
+    "IssueStatusForm",
+    "UploadUrlForm",
+    "GenerateImageForm",
+    "AttachmentRenameForm",
+    "PlanningForm",
+    "SeoRunForm",
+    "SEO_URL_PATTERN",
+    "ISSLOP_URL_PATTERN",
+    "ISSLOP_SINGLE_SLASH_PATTERN",
+    "ISSLOP_SCHEME_PATTERN",
+    "IsslopRunForm",
+    "DEEPSEARCH_MIN_DEPTH",
+    "DEEPSEARCH_MAX_DEPTH",
+    "DEEPSEARCH_DEFAULT_DEPTH",
+    "DEEPSEARCH_MIN_PAGES",
+    "DEEPSEARCH_MAX_PAGES",
+    "DEEPSEARCH_DEFAULT_PAGES",
+    "DEEPSEARCH_MIN_QUERY",
+    "DEEPSEARCH_MAX_QUERY",
+    "DEEPSEARCH_MAX_MESSAGE",
+    "DeepsearchRunForm",
+    "DeepsearchChatForm",
+    "QUIZ_KINDS",
+    "QUIZ_OPTION_KINDS",
+    "QuizForm",
+    "QuizQuestionForm",
+    "QuizReorderForm",
+    "QuizAnswerForm",
+    "QuizDocumentOption",
+    "QuizDocumentQuestion",
+    "QuizDocumentSettings",
+    "QuizDocument",
+    "QuizImportForm",
+    "GamePlantForm",
+    "GameSlotForm",
+    "GamePlotStopForm",
+    "GamePerkForm",
+    "GameQuestForm",
+    "GameLegacyForm",
+    "GameInfraForm",
+    "GameCosmeticForm",
+    "GameMasteryForm",
+    "GameEraStartForm",
+    "CommunityAddForm",
+    "CommunitySchemaForm",
+    "CommunityUpdateForm",
+    "CommunityRunForm",
+    "BackupRunForm",
+    "BackupScheduleForm",
+    "NotificationDefaultForm",
+    "AdminRoleForm",
+    "AdminPasswordForm",
+    "AdminSettingsForm",
+    "WorkspaceQuotaForm",
+    "WorkspaceFlagForm",
+    "WorkspaceSuspendForm",
+    "ReportForm",
+    "ReportStatusForm",
+    "ModerationDecisionForm",
+    "SuspensionForm",
+    "BanForm",
+    "NodeSettingsForm",
+    "NodeFetchForm",
+    "NODE_SUSPEND_MIN_MINUTES",
+    "NODE_SUSPEND_MAX_MINUTES",
+    "NODE_SETTING_FORM_MAX",
+    "AdminNodeKillswitchForm",
+    "AdminNodeSettingsForm",
+    "AdminNodeClassForm",
+    "AdminNodeSuspendForm",
+    "NodeDeviceForm",
+]

devplacepy/models/admin.py

@@ -0,0 +1,299 @@
+# retoor <retoor@molodetz.nl>
+
+from typing import Literal, Optional
+from urllib.parse import urlsplit
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+
+class CommunityAddForm(BaseModel):
+    query: str = Field(min_length=2, max_length=200)
+    login_username: str = Field(default="", max_length=200)
+    login_email: str = Field(default="", max_length=254)
+    login_password: str = Field(default="", max_length=500)
+    login_totp_secret: str = Field(default="", max_length=200)
+
+    @field_validator("query")
+    @classmethod
+    def trimmed(cls, value):
+        return value.strip()
+
+
+class CommunitySchemaForm(BaseModel):
+    schema_text: str = Field(min_length=2, max_length=300_000, alias="schema")
+
+    model_config = {"populate_by_name": True}
+
+
+class CommunityUpdateForm(BaseModel):
+    interval_seconds: int = Field(ge=900, le=2592000)
+    enabled: bool = False
+    login_username: str = Field(default="", max_length=200)
+    login_email: str = Field(default="", max_length=254)
+    login_password: str = Field(default="", max_length=500)
+    login_totp_secret: str = Field(default="", max_length=200)
+    clear_login: bool = False
+    alias: Optional[bool] = None
+    translate_to: Optional[str] = Field(default=None, max_length=12)
+    sample_percent: Optional[int] = Field(default=None, ge=1, le=100)
+    min_comments: Optional[int] = Field(default=None, ge=0, le=1000)
+    topics: Optional[str] = Field(default=None, max_length=500)
+    jitter_percent: Optional[int] = Field(default=None, ge=0, le=50)
+    collect_attachments: Optional[bool] = None
+
+
+class CommunityRunForm(BaseModel):
+    dry_run: bool = False
+
+
+class BackupRunForm(BaseModel):
+    target: Literal["database", "uploads", "keys", "full"] = "full"
+
+
+class BackupScheduleForm(BaseModel):
+    name: str = Field(min_length=1, max_length=120)
+    target: Literal["database", "uploads", "keys", "full"] = "full"
+    kind: Literal["interval", "cron"] = "interval"
+    every_seconds: int = Field(default=86400, ge=60)
+    cron: str = Field(default="", max_length=120)
+    keep_last: int = Field(default=7, ge=0, le=1000)
+
+    @model_validator(mode="after")
+    def _validate_schedule(self) -> "BackupScheduleForm":
+        if self.kind == "cron" and not self.cron.strip():
+            raise ValueError("A cron schedule requires a cron expression")
+        return self
+
+
+class NotificationDefaultForm(BaseModel):
+    notification_type: str = Field(min_length=1, max_length=40)
+    channel: Literal["in_app", "push", "telegram"]
+    value: bool = False
+
+
+class AdminRoleForm(BaseModel):
+    role: Literal["member", "admin"]
+
+
+class AdminPasswordForm(BaseModel):
+    password: str = Field(min_length=6, max_length=128)
+
+
+class AdminSettingsForm(BaseModel):
+    site_name: str = Field(default="", max_length=200)
+    site_description: str = Field(default="", max_length=500)
+    site_tagline: str = Field(default="", max_length=500)
+    site_url: str = Field(default="", max_length=300)
+    ios_app_url: str = Field(default="", max_length=300)
+    max_upload_size_mb: str = Field(default="", max_length=10)
+    allowed_file_types: str = Field(default="", max_length=1000)
+    max_attachments_per_resource: str = Field(default="", max_length=10)
+    rate_limit_per_minute: str = Field(default="", max_length=10)
+    rate_limit_window_seconds: str = Field(default="", max_length=10)
+    session_max_age_days: str = Field(default="", max_length=10)
+    session_remember_days: str = Field(default="", max_length=10)
+    registration_open: str = Field(default="", max_length=1)
+    maintenance_mode: str = Field(default="", max_length=1)
+    maintenance_message: str = Field(default="", max_length=300)
+    happy_404_enabled: str = Field(default="", max_length=1)
+    docs_search_mode: str = Field(default="", max_length=20)
+    outbound_proxy_url: str = Field(default="", max_length=500)
+    moderation_sla_hours: str = Field(default="", max_length=10)
+    moderation_filter_mode: str = Field(default="", max_length=10)
+    moderation_minimum_age: str = Field(default="", max_length=3)
+    moderation_mature_default_hidden: str = Field(default="", max_length=1)
+    account_deletion_grace_hours: str = Field(default="", max_length=10)
+    contact_email: str = Field(default="", max_length=200)
+    contact_phone: str = Field(default="", max_length=60)
+    contact_address: str = Field(default="", max_length=500)
+    terms_version: str = Field(default="", max_length=20)
+    privacy_version: str = Field(default="", max_length=20)
+    guidelines_version: str = Field(default="", max_length=20)
+    ai_third_party_provider: str = Field(default="", max_length=120)
+    correction_model: str = Field(default="", max_length=120)
+    modifier_model: str = Field(default="", max_length=120)
+    quiz_grading_model: str = Field(default="", max_length=120)
+    extra_head: str = Field(default="", max_length=50000)
+    sync_enabled: str = Field(default="", max_length=1)
+    sync_origins: str = Field(default="", max_length=1000)
+
+    @field_validator("sync_origins")
+    @classmethod
+    def validate_sync_origins(cls, value):
+        from devplacepy.services.sync.normalize import normalize_origin
+
+        items = [item for item in value.replace("\n", ",").split(",") if item.strip()]
+        return ",".join(dict.fromkeys(normalize_origin(item) for item in items))
+
+    @field_validator("moderation_filter_mode")
+    @classmethod
+    def validate_filter_mode(cls, value):
+        from devplacepy.services.moderation.rules import FILTER_MODES
+
+        if value and value not in FILTER_MODES:
+            raise ValueError(f"Filter mode must be one of {', '.join(FILTER_MODES)}")
+        return value
+
+    @field_validator("outbound_proxy_url")
+    @classmethod
+    def validate_outbound_proxy_url(cls, value):
+        text = value.strip()
+        if not text:
+            return text
+        parsed = urlsplit(text)
+        if parsed.scheme not in ("http", "https", "socks5", "socks5h") or not parsed.hostname:
+            raise ValueError("Proxy URL must be http(s):// or socks5(h):// with a host, e.g. http://user:pass@host:port")
+        return text
+
+
+class WorkspaceQuotaForm(BaseModel):
+    owner_id: str = Field(default="", max_length=36)
+    label: str = Field(default="", max_length=64)
+    max_workspaces: int = Field(default=0, ge=0, le=100)
+    max_tunnels: int = Field(default=0, ge=0, le=100)
+    disk_quota_mb: int = Field(default=0, ge=0)
+    egress_quota_mb: int = Field(default=0, ge=0)
+    idle_stop_minutes: int = Field(default=0, ge=0)
+    retention_days: int = Field(default=0, ge=0)
+    cpu_millicores: int = Field(default=0, ge=0, le=64000)
+    memory_mb: int = Field(default=0, ge=0, le=1048576)
+
+
+class WorkspaceFlagForm(BaseModel):
+    kind: str = Field(default="manual", max_length=40)
+    severity: str = Field(default="warn", max_length=16)
+    detail: str = Field(default="", max_length=500)
+
+
+class WorkspaceSuspendForm(BaseModel):
+    reason: str = Field(default="", max_length=500)
+
+
+class ReportForm(BaseModel):
+    reason: str = Field(min_length=1, max_length=40)
+    detail: str = Field(default="", max_length=2000)
+
+    @field_validator("reason")
+    @classmethod
+    def known_reason(cls, value):
+        from devplacepy.database.moderation import REPORT_REASONS
+
+        if value not in REPORT_REASONS:
+            raise ValueError("Unknown report reason")
+        return value
+
+
+class ReportStatusForm(BaseModel):
+    status: str = Field(default="acknowledged", max_length=20)
+
+    @field_validator("status")
+    @classmethod
+    def known_status(cls, value):
+        from devplacepy.database.moderation import REPORT_STATUSES
+
+        if value not in REPORT_STATUSES:
+            raise ValueError("Unknown report status")
+        return value
+
+
+class ModerationDecisionForm(BaseModel):
+    action: str = Field(min_length=1, max_length=30)
+    reason: str = Field(default="", max_length=200)
+    notes: str = Field(default="", max_length=2000)
+    duration_hours: int = Field(default=24, ge=1, le=8760)
+
+    @field_validator("action")
+    @classmethod
+    def known_action(cls, value):
+        from devplacepy.database.moderation import MODERATION_ACTIONS
+
+        if value not in MODERATION_ACTIONS:
+            raise ValueError("Unknown moderation action")
+        return value
+
+
+class SuspensionForm(BaseModel):
+    reason: str = Field(default="", max_length=200)
+    duration_hours: int = Field(default=24, ge=1, le=8760)
+
+
+class BanForm(BaseModel):
+    reason: str = Field(default="", max_length=200)
+
+
+class NodeSettingsForm(BaseModel):
+    enabled: Optional[bool] = None
+    bandwidth_mb_day: Optional[int] = Field(default=None, ge=1, le=20 * 1024)
+    max_concurrent: Optional[int] = Field(default=None, ge=1, le=16)
+    active_from: Optional[str] = Field(default=None, max_length=5)
+    active_to: Optional[str] = Field(default=None, max_length=5)
+    timezone: Optional[str] = Field(default=None, max_length=64)
+    allowed_platforms: Optional[list[str]] = Field(default=None, max_length=16)
+    pause_on_battery: Optional[bool] = None
+    public_display: Optional[bool] = None
+
+    @field_validator("allowed_platforms", mode="before")
+    @classmethod
+    def split_platforms(cls, value):
+        if isinstance(value, str):
+            return [item.strip() for item in value.split(",") if item.strip()]
+        return value
+
+
+class NodeFetchForm(BaseModel):
+    url: str = Field(max_length=2048)
+    cache: bool = True
+    timeout: int = Field(default=15, ge=1, le=60)
+
+
+NODE_SUSPEND_MIN_MINUTES = 1
+
+
+NODE_SUSPEND_MAX_MINUTES = 10080
+
+
+NODE_SETTING_FORM_MAX = 1_000_000
+
+
+class AdminNodeKillswitchForm(BaseModel):
+    enabled: Optional[bool] = None
+
+
+class AdminNodeSettingsForm(BaseModel):
+    nodes_max_total: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX)
+    nodes_daily_quota_user: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX)
+    nodes_daily_quota_guest: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX)
+    nodes_cache_ttl_seconds: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX)
+    nodes_social_via_nodes: Optional[Literal["0", "1"]] = None
+    nodes_daily_social_user: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX)
+    nodes_allow_unknown_class: Optional[Literal["0", "1"]] = None
+    nodes_redundancy_percent: Optional[int] = Field(default=None, ge=0, le=NODE_SETTING_FORM_MAX)
+
+
+class AdminNodeClassForm(BaseModel):
+    value: Literal["allow", "clear"]
+
+
+class AdminNodeSuspendForm(BaseModel):
+    minutes: int = Field(ge=NODE_SUSPEND_MIN_MINUTES, le=NODE_SUSPEND_MAX_MINUTES)
+    reason: str = Field(default="", max_length=500)
+
+    @field_validator("reason")
+    @classmethod
+    def trimmed(cls, value):
+        return value.strip()
+
+
+class NodeDeviceForm(BaseModel):
+    label: str = Field(default="", max_length=60)
+    kind: str = Field(default="native", max_length=20)
+    platform: str = Field(default="", max_length=40)
+    client_version: str = Field(default="", max_length=40)
+
+    @field_validator("kind")
+    @classmethod
+    def known_kind(cls, value):
+        from devplacepy.services.nodes.spec import DEVICE_KINDS
+
+        if value not in DEVICE_KINDS:
+            raise ValueError("Unknown device kind")
+        return value

devplacepy/models/auth.py

@@ -0,0 +1,95 @@
+# retoor <retoor@molodetz.nl>
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+from devplacepy.models.shared import _declared_age
+
+
+class SignupForm(BaseModel):
+    username: str = Field(min_length=3, max_length=32)
+    email: str = Field(min_length=1, max_length=255)
+    password: str = Field(min_length=6, max_length=128)
+    confirm_password: str = Field(min_length=1, max_length=128)
+    birth_date: str = Field(min_length=1, max_length=20)
+    accept_terms: str = Field(default="")
+
+    @property
+    def age_band(self) -> str:
+        from devplacepy.database.moderation import age_band_for
+
+        return age_band_for(_declared_age(self.birth_date))
+
+    @field_validator("birth_date")
+    @classmethod
+    def old_enough(cls, value):
+        from devplacepy.database import minimum_age
+
+        minimum = minimum_age()
+        if _declared_age(value) < minimum:
+            raise ValueError(f"You must be at least {minimum} years old to join")
+        return value
+
+    @field_validator("accept_terms")
+    @classmethod
+    def terms_accepted(cls, value):
+        if value.strip().lower() not in ("1", "on", "true", "yes"):
+            raise ValueError(
+                "You must accept the Terms of Service and Community Guidelines"
+            )
+        return value
+
+    @field_validator("username")
+    @classmethod
+    def username_chars(cls, value):
+        if not value.isascii() or not all(
+            c.isalnum() or c in ("-", "_") for c in value
+        ):
+            raise ValueError(
+                "Username can only contain letters, numbers, hyphens, and underscores"
+            )
+        from devplacepy.services.moderation.filter import classify
+
+        if classify(value).verdict == "block":
+            raise ValueError("That username breaks the community guidelines")
+        return value
+
+    @field_validator("email")
+    @classmethod
+    def email_has_at(cls, value):
+        if "@" not in value:
+            raise ValueError("Valid email is required")
+        return value
+
+    @model_validator(mode="after")
+    def passwords_match(self):
+        if self.password != self.confirm_password:
+            raise ValueError("Passwords do not match")
+        return self
+
+
+class LoginForm(BaseModel):
+    email: str = Field(min_length=1, max_length=255)
+    password: str = Field(min_length=1, max_length=128)
+    remember_me: str = ""
+    next: str = ""
+
+
+class ForgotPasswordForm(BaseModel):
+    email: str = Field(min_length=1, max_length=255)
+
+    @field_validator("email")
+    @classmethod
+    def email_has_at(cls, value):
+        if "@" not in value:
+            raise ValueError("Valid email is required")
+        return value
+
+
+class ResetPasswordForm(BaseModel):
+    password: str = Field(min_length=6, max_length=128)
+    confirm_password: str = Field(min_length=1, max_length=128)
+
+    @model_validator(mode="after")
+    def passwords_match(self):
+        if self.password != self.confirm_password:
+            raise ValueError("Passwords do not match")
+        return self

devplacepy/models/community.py

@@ -0,0 +1,57 @@
+# retoor <retoor@molodetz.nl>
+
+from typing import Literal, Optional
+from pydantic import BaseModel, Field
+
+
+class SkillInstallForm(BaseModel):
+    url: str = Field(min_length=1, max_length=2000)
+
+
+class SkillEnabledForm(BaseModel):
+    enabled: Optional[bool] = None
+
+
+class SkillDeleteForm(BaseModel):
+    owner: str = Field(default="", max_length=32)
+
+
+class MessageForm(BaseModel):
+    content: str = Field(min_length=0, max_length=2000)
+    receiver_uid: str = Field(min_length=1, max_length=36)
+    attachment_uids: list[str] = []
+    client_id: Optional[str] = Field(default=None, max_length=64)
+
+
+class GistForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(default="", max_length=5000)
+    source_code: str = Field(min_length=1, max_length=400000)
+    language: str = Field(default="plaintext", max_length=50)
+    attachment_uids: list[str] = []
+
+
+class GistEditForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(default="", max_length=5000)
+    source_code: str = Field(min_length=1, max_length=400000)
+    language: str = Field(default="plaintext", max_length=50)
+
+
+class IssueForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(min_length=1, max_length=5000)
+    attachment_uids: list[str] = []
+
+
+class IssueCommentForm(BaseModel):
+    body: str = Field(min_length=1, max_length=5000)
+    attachment_uids: list[str] = []
+
+
+class IssueAttachmentForm(BaseModel):
+    attachment_uids: list[str] = []
+
+
+class IssueStatusForm(BaseModel):
+    status: Literal["open", "closed"]

devplacepy/models/containers.py

@@ -0,0 +1,78 @@
+# retoor <retoor@molodetz.nl>
+
+from typing import Optional
+from pydantic import BaseModel, Field, field_validator
+
+
+class ContainerInstanceForm(BaseModel):
+    name: str = Field(min_length=1, max_length=64)
+    boot_command: str = Field(default="", max_length=500)
+    boot_language: str = Field(default="none", max_length=10)
+    boot_script: str = Field(default="", max_length=100000)
+    run_as_uid: str = Field(default="", max_length=36)
+    start_on_boot: bool = False
+    env: str = Field(default="", max_length=10000)
+    cpu_limit: str = Field(default="", max_length=16)
+    mem_limit: str = Field(default="", max_length=16)
+    ports: str = Field(default="", max_length=500)
+    volumes: str = Field(default="", max_length=2000)
+    restart_policy: str = Field(default="never", max_length=20)
+    autostart: bool = True
+    ingress_slug: str = Field(default="", max_length=64)
+    ingress_port: Optional[int] = Field(default=None, ge=1, le=65535)
+
+    @field_validator("ingress_port", mode="before")
+    @classmethod
+    def _blank_ingress_port(cls, value):
+        if value is None or (isinstance(value, str) and not value.strip()):
+            return None
+        return value
+
+
+class ContainerAdminCreateForm(ContainerInstanceForm):
+    project_slug: str = Field(min_length=1, max_length=128)
+
+
+class ContainerEditForm(BaseModel):
+    run_as_uid: str = Field(default="", max_length=36)
+    boot_language: str = Field(default="none", max_length=10)
+    boot_script: str = Field(default="", max_length=100000)
+    boot_command: str = Field(default="", max_length=500)
+    restart_policy: str = Field(default="never", max_length=20)
+    start_on_boot: bool = False
+    cpu_limit: str = Field(default="", max_length=16)
+    mem_limit: str = Field(default="", max_length=16)
+
+
+class ContainerExecForm(BaseModel):
+    command: str = Field(min_length=1, max_length=2000)
+
+
+class ContainerScheduleForm(BaseModel):
+    action: str = Field(max_length=10)
+    kind: str = Field(max_length=10)
+    cron: str = Field(default="", max_length=120)
+    run_at: str = Field(default="", max_length=40)
+    delay_seconds: Optional[int] = Field(default=None, ge=1)
+    every_seconds: Optional[int] = Field(default=None, ge=1)
+    max_runs: Optional[int] = Field(default=None, ge=1)
+
+
+class TunnelForm(BaseModel):
+    label: str = Field(default="", max_length=64)
+    container_port: int = Field(default=0, ge=0, le=65535)
+
+
+class EditorPrefsForm(BaseModel):
+    font_size: int = Field(default=0, ge=0, le=48)
+    terminal_font_size: int = Field(default=0, ge=0, le=48)
+    zoom_level: int = Field(default=-99, ge=-99, le=5)
+    theme: str = Field(default="", max_length=32)
+    layout: str = Field(default="", max_length=32)
+    panel_preset: str = Field(default="", max_length=32)
+    window_mode: str = Field(default="", max_length=32)
+    window_width: int = Field(default=0, ge=0, le=7680)
+    window_height: int = Field(default=0, ge=0, le=4320)
+    boot_agent: str = Field(default="", max_length=32)
+    boot_shell: int = Field(default=-1, ge=-1, le=1)
+    reset: bool = False

devplacepy/models/content.py

@@ -0,0 +1,128 @@
+# retoor <retoor@molodetz.nl>
+
+from typing import Literal
+from pydantic import BaseModel, Field, field_validator, model_validator
+from devplacepy.constants import TOPICS
+from devplacepy.rendering import is_single_emoji
+from devplacepy.models.shared import normalize_poll_options
+
+
+class PostForm(BaseModel):
+    content: str = Field(min_length=10, max_length=125000)
+    title: str = Field(default="", max_length=500)
+    topic: str = "random"
+    project_uid: str = Field(default="", max_length=36)
+    attachment_uids: list[str] = []
+    poll_question: str = Field(default="", max_length=200)
+    poll_options: list[str] = []
+    war_faction_a: str = Field(default="", max_length=30)
+    war_faction_b: str = Field(default="", max_length=30)
+
+    @field_validator("poll_options")
+    @classmethod
+    def poll_options_max_length(cls, value):
+        if value is None:
+            return value
+        for opt in value:
+            if isinstance(opt, str) and len(opt) > 200:
+                raise ValueError("Each poll option must be 200 characters or fewer")
+        return value
+
+    @field_validator("topic")
+    @classmethod
+    def valid_topic(cls, value):
+        return value if value in TOPICS else "random"
+
+    @field_validator("poll_options", mode="before")
+    @classmethod
+    def split_poll_options(cls, value):
+        return normalize_poll_options(value)
+
+
+class WarJoinForm(BaseModel):
+    faction: str
+
+    @field_validator("faction")
+    @classmethod
+    def valid_faction(cls, value):
+        if value not in ("a", "b"):
+            raise ValueError("Faction must be a or b")
+        return value
+
+
+class PostEditForm(BaseModel):
+    content: str = Field(min_length=10, max_length=125000)
+    title: str = Field(default="", max_length=500)
+    topic: str = "random"
+    poll_question: str = Field(default="", max_length=200)
+    poll_options: list[str] = []
+
+    @field_validator("poll_options")
+    @classmethod
+    def poll_options_max_length(cls, value):
+        if value is None:
+            return value
+        for opt in value:
+            if isinstance(opt, str) and len(opt) > 200:
+                raise ValueError("Each poll option must be 200 characters or fewer")
+        return value
+
+    @field_validator("topic")
+    @classmethod
+    def valid_topic(cls, value):
+        return value if value in TOPICS else "random"
+
+    @field_validator("poll_options", mode="before")
+    @classmethod
+    def split_poll_options(cls, value):
+        return normalize_poll_options(value)
+
+
+class CommentForm(BaseModel):
+    content: str = Field(min_length=3, max_length=125000)
+    target_uid: str = Field(default="", max_length=36)
+    post_uid: str = Field(default="", max_length=36)
+    target_type: Literal["post", "project", "news", "issue", "gist", "quiz"] = "post"
+    parent_uid: str = Field(default="", max_length=36)
+    attachment_uids: list[str] = []
+
+    @model_validator(mode="after")
+    def require_target(self):
+        if not (self.target_uid or self.post_uid):
+            raise ValueError("A target is required")
+        return self
+
+
+class CommentEditForm(BaseModel):
+    content: str = Field(min_length=3, max_length=125000)
+
+
+class NoteForm(BaseModel):
+    content: str = Field(min_length=1, max_length=4000)
+
+
+class VoteForm(BaseModel):
+    value: int
+
+    @field_validator("value")
+    @classmethod
+    def valid_value(cls, value):
+        if value not in (1, -1, 0):
+            raise ValueError("value must be 1, -1 or 0")
+        return value
+
+
+class ReactionForm(BaseModel):
+    emoji: str = Field(min_length=1, max_length=16)
+
+    @field_validator("emoji")
+    @classmethod
+    def valid_emoji(cls, value):
+        reaction = (value or "").strip()
+        if not is_single_emoji(reaction):
+            raise ValueError("Reaction must be a single emoji")
+        return reaction
+
+
+class PollVoteForm(BaseModel):
+    option_uid: str = Field(min_length=1, max_length=36)

devplacepy/models/game.py

@@ -0,0 +1,47 @@
+# retoor <retoor@molodetz.nl>
+
+from pydantic import BaseModel, Field
+
+
+class GamePlantForm(BaseModel):
+    slot: int = Field(ge=0, le=64)
+    crop: str = Field(min_length=1, max_length=40)
+
+
+class GameSlotForm(BaseModel):
+    slot: int = Field(ge=0, le=64)
+
+
+class GamePlotStopForm(BaseModel):
+    slot: int = Field(ge=0, le=64)
+    stop: bool
+
+
+class GamePerkForm(BaseModel):
+    perk: str = Field(min_length=1, max_length=40)
+
+
+class GameQuestForm(BaseModel):
+    quest: str = Field(min_length=1, max_length=40)
+    scope: str = Field(default="daily", min_length=1, max_length=10)
+
+
+class GameLegacyForm(BaseModel):
+    key: str = Field(min_length=1, max_length=40)
+
+
+class GameInfraForm(BaseModel):
+    key: str = Field(min_length=1, max_length=40)
+
+
+class GameCosmeticForm(BaseModel):
+    key: str = Field(min_length=1, max_length=40)
+
+
+class GameMasteryForm(BaseModel):
+    key: str = Field(min_length=1, max_length=40)
+
+
+class GameEraStartForm(BaseModel):
+    name: str = Field(min_length=1, max_length=60)
+    duration_days: int = Field(default=28, ge=1, le=180)

devplacepy/models/profile.py

@@ -0,0 +1,86 @@
+# retoor <retoor@molodetz.nl>
+
+from typing import Literal
+from pydantic import BaseModel, Field, field_validator
+from devplacepy.config import (
+    DEFAULT_CORRECTION_PROMPT,
+    DEFAULT_MODIFIER_PROMPT,
+)
+
+
+class CustomizationToggleForm(BaseModel):
+    value: bool = False
+
+
+class AwardGiveForm(BaseModel):
+    description: str = Field(min_length=1, max_length=125)
+
+
+class NotificationPrefForm(BaseModel):
+    notification_type: str = Field(min_length=1, max_length=40)
+    channel: Literal["in_app", "push", "telegram"]
+    value: bool = False
+
+
+class AiCorrectionForm(BaseModel):
+    enabled: bool = False
+    sync: bool = False
+    prompt: str = Field(default=DEFAULT_CORRECTION_PROMPT, max_length=20000)
+
+
+class AiModifierForm(BaseModel):
+    enabled: bool = False
+    sync: bool = False
+    prompt: str = Field(default=DEFAULT_MODIFIER_PROMPT, max_length=20000)
+
+
+class InteractionsForm(BaseModel):
+    enabled: bool = True
+    reset: bool = False
+
+
+class TelegramPairForm(BaseModel):
+    action: str = Field(default="request", max_length=16)
+
+
+class ProfileForm(BaseModel):
+    bio: str = Field(default="", max_length=500)
+    location: str = Field(default="", max_length=200)
+    git_link: str = Field(default="", max_length=500)
+    website: str = Field(default="", max_length=500)
+
+
+class ConsentForm(BaseModel):
+    kind: str = Field(min_length=1, max_length=40)
+    granted: str = Field(default="0", max_length=5)
+
+    @field_validator("kind")
+    @classmethod
+    def known_kind(cls, value):
+        from devplacepy.database.moderation import CONSENT_KINDS
+
+        if value not in CONSENT_KINDS:
+            raise ValueError("Unknown consent kind")
+        return value
+
+
+class MaturityForm(BaseModel):
+    level: str = Field(default="general", max_length=20)
+
+    @field_validator("level")
+    @classmethod
+    def known_level(cls, value):
+        from devplacepy.database.moderation import MATURITY_LEVELS
+
+        if value not in MATURITY_LEVELS:
+            raise ValueError("Unknown maturity level")
+        return value
+
+
+class MaturePreferenceForm(BaseModel):
+    mature_opt_in: str = Field(default="0", max_length=5)
+
+
+class AccountDeleteForm(BaseModel):
+    password: str = Field(min_length=1, max_length=128)
+    confirm_text: str = Field(default="", max_length=40)

devplacepy/models/projects.py

@@ -0,0 +1,113 @@
+# retoor <retoor@molodetz.nl>
+
+from typing import Literal
+from pydantic import BaseModel, Field, field_validator
+from devplacepy.models.shared import normalize_european_date, normalize_website_url
+
+
+class ProjectForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(min_length=1, max_length=5000)
+    release_date: str = ""
+    demo_date: str = ""
+    project_type: Literal["game", "game_asset", "software", "mobile_app", "website"] = (
+        "software"
+    )
+    platforms: str = Field(default="", max_length=500)
+    status: str = Field(default="In Development", max_length=100)
+    website_url: str = Field(default="", max_length=500)
+    repo_url: str = Field(default="", max_length=500)
+    cover_attachment_uid: str = Field(default="", max_length=64)
+    logo_attachment_uid: str = Field(default="", max_length=64)
+    is_private: bool = False
+    attachment_uids: list[str] = []
+
+    @field_validator("release_date", "demo_date", mode="before")
+    @classmethod
+    def normalize_dates(cls, value):
+        return normalize_european_date(value)
+
+    @field_validator("website_url", "repo_url")
+    @classmethod
+    def valid_link_url(cls, value):
+        return normalize_website_url(value)
+
+
+class ProjectEditForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(min_length=1, max_length=5000)
+    release_date: str = ""
+    demo_date: str = ""
+    project_type: Literal["game", "game_asset", "software", "mobile_app", "website"] = (
+        "software"
+    )
+    platforms: str = Field(default="", max_length=500)
+    status: str = Field(default="In Development", max_length=100)
+    website_url: str = Field(default="", max_length=500)
+    repo_url: str = Field(default="", max_length=500)
+    cover_attachment_uid: str = Field(default="", max_length=64)
+    logo_attachment_uid: str = Field(default="", max_length=64)
+
+    @field_validator("release_date", "demo_date", mode="before")
+    @classmethod
+    def normalize_dates(cls, value):
+        return normalize_european_date(value)
+
+    @field_validator("website_url", "repo_url")
+    @classmethod
+    def valid_link_url(cls, value):
+        return normalize_website_url(value)
+
+
+class ProjectFlagForm(BaseModel):
+    value: bool = False
+
+
+class ProjectScreenshotsForm(BaseModel):
+    attachment_uids: list[str] = []
+
+
+class ForkForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+
+
+class ProjectFileWriteForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    content: str = Field(default="", max_length=400000)
+
+
+class ProjectFileMkdirForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+
+
+class ProjectFileMoveForm(BaseModel):
+    from_path: str = Field(min_length=1, max_length=1024)
+    to_path: str = Field(min_length=1, max_length=1024)
+
+
+class ProjectFileDeleteForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+
+
+class ProjectFileReplaceLinesForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    start: int = Field(ge=1)
+    end: int = Field(ge=0)
+    content: str = Field(default="", max_length=400000)
+
+
+class ProjectFileInsertLinesForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    at: int = Field(ge=1)
+    content: str = Field(default="", max_length=400000)
+
+
+class ProjectFileDeleteLinesForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    start: int = Field(ge=1)
+    end: int = Field(ge=1)
+
+
+class ProjectFileAppendForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    content: str = Field(default="", max_length=400000)

devplacepy/models/quizzes.py

@@ -0,0 +1,247 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+from typing import Literal
+from pydantic import BaseModel, Field, field_validator, model_validator
+from devplacepy.config import (
+    QUIZ_ANSWER_MAX_CHARS,
+    QUIZ_MAX_OPTIONS,
+    QUIZ_MAX_QUESTIONS,
+    QUIZ_MAX_TIME_LIMIT_SECONDS,
+)
+from devplacepy.models.shared import normalize_poll_options, normalize_index_list
+
+
+QUIZ_KINDS = (
+    "single_choice",
+    "multiple_choice",
+    "true_false",
+    "free_text",
+    "fill_blank",
+    "numeric",
+    "ordering",
+    "matching",
+)
+
+
+QUIZ_OPTION_KINDS = frozenset(
+    {"single_choice", "multiple_choice", "fill_blank", "ordering", "matching"}
+)
+
+
+class QuizForm(BaseModel):
+    title: str = Field(min_length=3, max_length=200)
+    description: str = Field(default="", max_length=5000)
+    shuffle_questions: bool = False
+    shuffle_options: bool = False
+    reveal_answers: bool = False
+    allow_review: bool = True
+    time_limit_seconds: int = Field(default=0, ge=0, le=QUIZ_MAX_TIME_LIMIT_SECONDS)
+    pass_percent: int = Field(default=0, ge=0, le=100)
+
+
+class QuizQuestionForm(BaseModel):
+    kind: Literal[QUIZ_KINDS]
+    prompt: str = Field(min_length=1, max_length=2000)
+    explanation: str = Field(default="", max_length=2000)
+    points: int = Field(default=1, ge=1, le=100)
+    media_attachment_uid: str = Field(default="", max_length=36)
+    correct_boolean: bool = False
+    expected_answer: str = Field(default="", max_length=2000)
+    grading_criteria: str = Field(default="", max_length=2000)
+    numeric_value: float = 0.0
+    numeric_tolerance: float = Field(default=0.0, ge=0.0)
+    case_sensitive: bool = False
+    options: list[str] = []
+    match_values: list[str] = []
+    correct_indexes: list[int] = []
+
+    @field_validator("options", "match_values", mode="before")
+    @classmethod
+    def split_lists(cls, value):
+        return normalize_poll_options(value)
+
+    @field_validator("correct_indexes", mode="before")
+    @classmethod
+    def split_indexes(cls, value):
+        return normalize_index_list(value)
+
+    @field_validator("options", "match_values")
+    @classmethod
+    def bounded_options(cls, value):
+        if len(value) > QUIZ_MAX_OPTIONS:
+            raise ValueError(f"A question takes at most {QUIZ_MAX_OPTIONS} options")
+        for entry in value:
+            if len(entry) > 500:
+                raise ValueError("Each option must be 500 characters or fewer")
+        return value
+
+    @model_validator(mode="after")
+    def kind_requirements(self):
+        options = [option for option in self.options if option.strip()]
+        if self.kind in QUIZ_OPTION_KINDS and not options:
+            raise ValueError("This question type needs at least one option")
+        if self.kind in ("single_choice", "multiple_choice") and len(options) < 2:
+            raise ValueError("Choice questions need at least two options")
+        if self.kind == "single_choice" and len(self.correct_indexes) != 1:
+            raise ValueError("A single choice question needs exactly one correct option")
+        if self.kind == "multiple_choice" and not self.correct_indexes:
+            raise ValueError("A multiple choice question needs at least one correct option")
+        if self.kind in ("fill_blank", "matching") and len(self.match_values) < len(options):
+            raise ValueError("Every option needs an accepted answer")
+        if self.kind == "matching" and len(options) < 2:
+            raise ValueError("A matching question needs at least two pairs")
+        if self.kind == "ordering" and len(options) < 2:
+            raise ValueError("An ordering question needs at least two items")
+        if self.kind == "free_text" and not (
+            self.expected_answer.strip() or self.grading_criteria.strip()
+        ):
+            raise ValueError("A free text question needs a reference answer or grading criteria")
+        return self
+
+    def option_rows(self) -> list[dict]:
+        rows = []
+        for index, label in enumerate(self.options):
+            if not label.strip():
+                continue
+            match_value = (
+                self.match_values[index] if index < len(self.match_values) else ""
+            )
+            rows.append(
+                {
+                    "label": label.strip(),
+                    "match_value": match_value.strip(),
+                    "is_correct": index in set(self.correct_indexes),
+                }
+            )
+        return rows
+
+
+class QuizReorderForm(BaseModel):
+    order: list[str] = []
+
+    @field_validator("order", mode="before")
+    @classmethod
+    def split_order(cls, value):
+        return normalize_poll_options(value)
+
+    @model_validator(mode="after")
+    def require_order(self):
+        if not self.order:
+            raise ValueError("The new question order is required")
+        return self
+
+
+class QuizAnswerForm(BaseModel):
+    question_uid: str = Field(min_length=1, max_length=36)
+    answer_text: str = Field(default="", max_length=QUIZ_ANSWER_MAX_CHARS)
+    option_uids: list[str] = []
+    blanks: list[str] = []
+    matches: list[str] = []
+
+    @field_validator("option_uids", "blanks", "matches", mode="before")
+    @classmethod
+    def split_lists(cls, value):
+        return normalize_poll_options(value)
+
+    def submission(self) -> dict:
+        if self.blanks:
+            return {
+                "answer_text": json.dumps(self.blanks, ensure_ascii=False),
+                "option_uids": self.option_uids,
+            }
+        if self.matches:
+            pairs = dict(zip(self.option_uids, self.matches))
+            return {
+                "answer_text": json.dumps(pairs, ensure_ascii=False),
+                "option_uids": self.option_uids,
+            }
+        return {"answer_text": self.answer_text, "option_uids": self.option_uids}
+
+
+class QuizDocumentOption(BaseModel):
+    label: str = Field(default="", max_length=500)
+    match_value: str = Field(default="", max_length=500)
+    is_correct: bool = False
+
+
+class QuizDocumentQuestion(BaseModel):
+    kind: Literal[QUIZ_KINDS]
+    prompt: str = Field(min_length=1, max_length=2000)
+    explanation: str = Field(default="", max_length=2000)
+    points: int = Field(default=1, ge=1, le=100)
+    media_attachment_uid: str = Field(default="", max_length=36)
+    correct_boolean: bool = False
+    expected_answer: str = Field(default="", max_length=2000)
+    grading_criteria: str = Field(default="", max_length=2000)
+    numeric_value: float = 0.0
+    numeric_tolerance: float = Field(default=0.0, ge=0.0)
+    case_sensitive: bool = False
+    options: list[QuizDocumentOption] = Field(default_factory=list, max_length=QUIZ_MAX_OPTIONS)
+
+    @model_validator(mode="after")
+    def kind_requirements(self):
+        labelled = [option for option in self.options if option.label.strip()]
+        if self.kind in ("single_choice", "multiple_choice") and len(labelled) < 2:
+            raise ValueError("Choice questions need at least two options")
+        if self.kind == "single_choice" and sum(
+            1 for option in labelled if option.is_correct
+        ) != 1:
+            raise ValueError("A single choice question needs exactly one correct option")
+        if self.kind == "multiple_choice" and not any(
+            option.is_correct for option in labelled
+        ):
+            raise ValueError("A multiple choice question needs at least one correct option")
+        if self.kind in ("ordering", "matching") and len(labelled) < 2:
+            raise ValueError("This question type needs at least two entries")
+        if self.kind == "matching" and any(
+            not option.match_value.strip() for option in labelled
+        ):
+            raise ValueError("Every matching pair needs a right-hand value")
+        if self.kind == "fill_blank" and any(
+            not option.match_value.strip() for option in labelled
+        ):
+            raise ValueError("Every blank needs an accepted answer")
+        if self.kind == "free_text" and not (
+            self.expected_answer.strip() or self.grading_criteria.strip()
+        ):
+            raise ValueError("A free text question needs a reference answer or grading criteria")
+        return self
+
+
+class QuizDocumentSettings(BaseModel):
+    shuffle_questions: bool = False
+    shuffle_options: bool = False
+    reveal_answers: bool = False
+    allow_review: bool = True
+    time_limit_seconds: int = Field(default=0, ge=0, le=QUIZ_MAX_TIME_LIMIT_SECONDS)
+    pass_percent: int = Field(default=0, ge=0, le=100)
+
+
+class QuizDocument(BaseModel):
+    title: str = Field(min_length=3, max_length=200)
+    description: str = Field(default="", max_length=5000)
+    settings: QuizDocumentSettings = Field(default_factory=QuizDocumentSettings)
+    questions: list[QuizDocumentQuestion] = Field(
+        default_factory=list, max_length=QUIZ_MAX_QUESTIONS
+    )
+
+    @model_validator(mode="after")
+    def require_questions(self):
+        if not self.questions:
+            raise ValueError("A quiz document needs at least one question")
+        return self
+
+
+class QuizImportForm(BaseModel):
+    document: QuizDocument
+
+    @field_validator("document", mode="before")
+    @classmethod
+    def parse_document(cls, value):
+        if isinstance(value, str):
+            try:
+                return json.loads(value)
+            except ValueError as exc:
+                raise ValueError("document must be valid JSON") from exc
+        return value

devplacepy/models/shared.py

@@ -0,0 +1,75 @@
+# retoor <retoor@molodetz.nl>
+
+from datetime import datetime
+from urllib.parse import urlparse
+
+
+def normalize_european_date(value):
+    if not value:
+        return ""
+    text = str(value).strip()
+    if not text:
+        return ""
+    for fmt in ("%d/%m/%Y", "%Y-%m-%d"):
+        try:
+            return datetime.strptime(text, fmt).strftime("%Y-%m-%d")
+        except ValueError:
+            continue
+    raise ValueError("Date must be in DD/MM/YYYY format")
+
+
+def normalize_website_url(value):
+    if not value:
+        return ""
+    text = str(value).strip()
+    if not text:
+        return ""
+    if not text.lower().startswith(("http://", "https://")):
+        text = f"https://{text}"
+    parsed = urlparse(text)
+    if parsed.scheme not in ("http", "https") or not parsed.hostname or "." not in parsed.hostname:
+        raise ValueError("Website must be a valid http(s) URL")
+    return text
+
+
+def normalize_poll_options(value):
+    if value is None:
+        return []
+    if isinstance(value, str):
+        value = [value]
+    if not isinstance(value, list):
+        return value
+    if len(value) == 1 and isinstance(value[0], str):
+        single = value[0]
+        separator = "\n" if "\n" in single else ("," if "," in single else "")
+        if separator:
+            return [part.strip() for part in single.split(separator) if part.strip()]
+    return value
+
+
+def _declared_age(birth_date: str) -> int:
+    from datetime import date
+
+    from devplacepy.database.moderation import years_between
+
+    normalized = normalize_european_date(birth_date)
+    if not normalized:
+        raise ValueError("Date of birth is required")
+    born = date.fromisoformat(normalized)
+    today = date.today()
+    if born > today:
+        raise ValueError("Date of birth cannot be in the future")
+    return years_between(born, today)
+
+
+def normalize_index_list(value):
+    parts = normalize_poll_options(value)
+    if not isinstance(parts, list):
+        return []
+    indexes = []
+    for part in parts:
+        try:
+            indexes.append(int(str(part).strip()))
+        except (TypeError, ValueError):
+            continue
+    return indexes

devplacepy/models/tools.py

@@ -0,0 +1,119 @@
+# retoor <retoor@molodetz.nl>
+
+import re
+from typing import Literal
+from pydantic import BaseModel, Field, field_validator
+
+
+class PlanningForm(BaseModel):
+    numbers: str = Field(default="", max_length=4096)
+
+
+class SeoRunForm(BaseModel):
+    url: str = Field(min_length=3, max_length=2000)
+    mode: Literal["url", "sitemap"] = "url"
+    max_pages: int = Field(default=10, ge=1, le=50)
+
+    @field_validator("url")
+    @classmethod
+    def url_scheme(cls, value):
+        text = value.strip()
+        if not text:
+            raise ValueError("A URL is required")
+        if "://" in text:
+            scheme = text.split("://", 1)[0]
+            if scheme not in ("http", "https"):
+                raise ValueError(f"Only http and https URLs are allowed; got '{scheme}://'")
+        else:
+            text = f"https://{text}"
+        if not SEO_URL_PATTERN.match(text):
+            raise ValueError("URL must be a valid http or https source location")
+        return text
+
+
+SEO_URL_PATTERN = re.compile(r"^https?://[a-zA-Z0-9][\w./:@~^?&#%=;-]*$")
+
+
+ISSLOP_URL_PATTERN = re.compile(r"^(https?://|git://|ssh://|git@)[\w./:@~^-]+$", re.IGNORECASE)
+
+
+ISSLOP_SINGLE_SLASH_PATTERN = re.compile(r"^(https?|git|ssh):/(?!/)", re.IGNORECASE)
+
+
+ISSLOP_SCHEME_PATTERN = re.compile(r"^[a-z][a-z0-9+.-]*://", re.IGNORECASE)
+
+
+class IsslopRunForm(BaseModel):
+    url: str = Field(min_length=4, max_length=2048)
+
+    @field_validator("url")
+    @classmethod
+    def url_scheme(cls, value):
+        text = value.strip()
+        text = ISSLOP_SINGLE_SLASH_PATTERN.sub(lambda match: f"{match.group(1)}://", text)
+        if not ISSLOP_SCHEME_PATTERN.match(text) and not text.startswith("git@"):
+            text = f"https://{text}"
+        if not ISSLOP_URL_PATTERN.match(text):
+            raise ValueError("URL must be an http(s), git or ssh source location")
+        return text
+
+
+DEEPSEARCH_MIN_DEPTH = 1
+
+
+DEEPSEARCH_MAX_DEPTH = 4
+
+
+DEEPSEARCH_DEFAULT_DEPTH = 2
+
+
+DEEPSEARCH_MIN_PAGES = 1
+
+
+DEEPSEARCH_MAX_PAGES = 30
+
+
+DEEPSEARCH_DEFAULT_PAGES = 12
+
+
+DEEPSEARCH_MIN_QUERY = 3
+
+
+DEEPSEARCH_MAX_QUERY = 500
+
+
+DEEPSEARCH_MAX_MESSAGE = 2000
+
+
+class DeepsearchRunForm(BaseModel):
+    query: str = Field(min_length=DEEPSEARCH_MIN_QUERY, max_length=DEEPSEARCH_MAX_QUERY)
+    depth: int = Field(
+        default=DEEPSEARCH_DEFAULT_DEPTH,
+        ge=DEEPSEARCH_MIN_DEPTH,
+        le=DEEPSEARCH_MAX_DEPTH,
+    )
+    max_pages: int = Field(
+        default=DEEPSEARCH_DEFAULT_PAGES,
+        ge=DEEPSEARCH_MIN_PAGES,
+        le=DEEPSEARCH_MAX_PAGES,
+    )
+
+    @field_validator("query")
+    @classmethod
+    def query_present(cls, value):
+        text = value.strip()
+        if not text:
+            raise ValueError("A research question is required")
+        return text
+
+
+class DeepsearchChatForm(BaseModel):
+    message: str = Field(min_length=1, max_length=DEEPSEARCH_MAX_MESSAGE)
+
+    @field_validator("message")
+    @classmethod
+    def message_present(cls, value):
+        text = value.strip()
+        if not text:
+            raise ValueError("A message is required")
+        return text

devplacepy/models/uploads.py

@@ -0,0 +1,18 @@
+# retoor <retoor@molodetz.nl>
+
+from typing import Optional
+from pydantic import BaseModel, Field
+
+
+class UploadUrlForm(BaseModel):
+    url: str = Field(min_length=1, max_length=2048)
+    filename: Optional[str] = Field(default=None, max_length=255)
+
+
+class GenerateImageForm(BaseModel):
+    prompt: str = Field(min_length=1, max_length=2000)
+    size: Optional[str] = Field(default=None, max_length=16)
+
+
+class AttachmentRenameForm(BaseModel):
+    filename: str = Field(min_length=1, max_length=255)

devplacepy/project_files.py

@@ -1,7 +1,9 @@
 # retoor <retoor@molodetz.nl>
 
 import logging
+import os
 import shutil
+import time
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -17,6 +19,8 @@ MAX_FILES_PER_PROJECT = 5000
 MAX_PATH_LENGTH = 1024
 MAX_SEGMENT_LENGTH = 255
 MAX_DEPTH = 32
+_MISSING_BLOB_LOG_SECONDS = 60
+_missing_blob_logged: dict[str, tuple[int, float]] = {}
 
 TEXT_EXTENSIONS = {
     ".txt",
@@ -477,6 +481,7 @@ def delete_node(project_uid: str, raw_path: str, deleted_by: str = "system") ->
 def soft_delete_all_project_files(project_uid: str, deleted_by: str) -> None:
     if "project_files" not in db.tables:
         return
+    clear_sync_state(project_uid)
     stamp = _now()
     for row in _table().find(project_uid=project_uid, deleted_at=None):
         _table().update(
@@ -486,6 +491,7 @@ def soft_delete_all_project_files(project_uid: str, deleted_by: str) -> None:
 
 
 _SYNC_CLOCK_SKEW_SECONDS = 1.0
+SYNC_STATE_TABLE = "project_file_sync_state"
 
 
 def _epoch_of(iso_timestamp) -> float:
@@ -497,6 +503,48 @@ def _epoch_of(iso_timestamp) -> float:
         return 0.0
 
 
+def _sync_state_table():
+    return get_table(SYNC_STATE_TABLE)
+
+
+def _load_sync_manifest(project_uid: str) -> dict:
+    if SYNC_STATE_TABLE not in db.tables:
+        return {}
+    return {
+        row["path"]: {"db_epoch": row["db_epoch"], "fs_epoch": row["fs_epoch"]}
+        for row in _sync_state_table().find(project_uid=project_uid)
+    }
+
+
+def _save_sync_manifest(project_uid: str, old: dict, new: dict) -> None:
+    table = _sync_state_table()
+    for path in old:
+        if path not in new:
+            table.delete(project_uid=project_uid, path=path)
+    for path, entry in new.items():
+        if old.get(path) == entry:
+            continue
+        table.upsert(
+            {"project_uid": project_uid, "path": path, **entry},
+            ["project_uid", "path"],
+        )
+
+
+def clear_sync_state(project_uid: str) -> None:
+    if SYNC_STATE_TABLE not in db.tables:
+        return
+    _sync_state_table().delete(project_uid=project_uid)
+
+
+def _delete_db_row_for_sync(row: dict, deleted_by: str) -> None:
+    _table().update(
+        {"uid": row["uid"], "deleted_at": _now(), "deleted_by": deleted_by},
+        ["uid"],
+    )
+    if row.get("is_binary"):
+        _unlink_blob(row)
+
+
 def _file_records(project_uid: str) -> dict:
     records: dict = {}
     for row in _table().find(project_uid=project_uid, deleted_at=None):
@@ -510,7 +558,7 @@ def _walk_workspace_files(src, skip):
     for root, dirs, files in src.walk():
         dirs[:] = [d for d in sorted(dirs) if d not in skip]
         for name in sorted(files):
-            if name in skip:
+            if name in skip or Path(name).suffix in IMPORT_SKIP_EXTENSIONS:
                 continue
             full = Path(root) / name
             if full.is_symlink() or not full.is_file():
@@ -536,71 +584,176 @@ def _workspace_records(workspace) -> dict:
 
 
 def sync_dir_bidirectional(project_uid: str, workspace, user: dict) -> dict:
+    empty = {
+        "exported": 0,
+        "imported": 0,
+        "deleted_in_project": 0,
+        "deleted_in_workspace": 0,
+    }
     if "project_files" not in db.tables:
-        return {"exported": 0, "imported": 0}
+        return empty
     readonly = is_readonly(project_uid)
     dest = Path(workspace).resolve()
     dest.mkdir(parents=True, exist_ok=True)
     db_files = _file_records(project_uid)
     fs_files = _workspace_records(dest)
-    exported = 0
-    imported = 0
+    manifest = _load_sync_manifest(project_uid)
+    new_manifest: dict = {}
+    counts = dict(empty)
+    missing_blobs = 0
 
-    for path, row in db_files.items():
+    for path in set(manifest) | set(db_files) | set(fs_files):
+        row = db_files.get(path)
         fs_full = fs_files.get(path)
-        if fs_full is None:
-            _export_node(row, dest)
-            exported += 1
-            continue
-        try:
-            fs_mtime = fs_full.stat().st_mtime
-        except OSError:
-            continue
-        db_mtime = _epoch_of(row.get("updated_at"))
-        if db_mtime >= fs_mtime - _SYNC_CLOCK_SKEW_SECONDS:
-            _export_node(row, dest)
-            exported += 1
-        elif not readonly:
-            if _import_file(project_uid, user, path, fs_full):
-                imported += 1
+        entry = manifest.get(path)
 
-    if not readonly:
-        for path, fs_full in fs_files.items():
-            if path in db_files:
+        if row is not None and fs_full is not None:
+            try:
+                fs_epoch = fs_full.stat().st_mtime
+            except OSError:
                 continue
-            if _import_file(project_uid, user, path, fs_full):
-                imported += 1
+            db_epoch = _epoch_of(row.get("updated_at"))
+            if (
+                entry is not None
+                and abs(db_epoch - entry["db_epoch"]) <= _SYNC_CLOCK_SKEW_SECONDS
+                and abs(fs_epoch - entry["fs_epoch"]) <= _SYNC_CLOCK_SKEW_SECONDS
+            ):
+                new_manifest[path] = entry
+                continue
+            if db_epoch >= fs_epoch - _SYNC_CLOCK_SKEW_SECONDS or readonly:
+                recorded = _record_export(row, dest)
+                if recorded:
+                    counts["exported"] += 1
+                    new_manifest[path] = recorded
+                elif row.get("is_binary"):
+                    missing_blobs += 1
+            else:
+                recorded = _record_import(project_uid, user, path, fs_full, fs_epoch)
+                if recorded:
+                    counts["imported"] += 1
+                    new_manifest[path] = recorded
+            continue
 
-    return {"exported": exported, "imported": imported}
+        if row is not None and fs_full is None:
+            db_epoch = _epoch_of(row.get("updated_at"))
+            if (
+                entry is None
+                or readonly
+                or db_epoch > entry["db_epoch"] + _SYNC_CLOCK_SKEW_SECONDS
+            ):
+                recorded = _record_export(row, dest)
+                if recorded:
+                    counts["exported"] += 1
+                    new_manifest[path] = recorded
+                elif row.get("is_binary"):
+                    missing_blobs += 1
+            else:
+                _delete_db_row_for_sync(row, user["uid"])
+                counts["deleted_in_project"] += 1
+            continue
+
+        if row is None and fs_full is not None:
+            try:
+                fs_epoch = fs_full.stat().st_mtime
+            except OSError:
+                continue
+            if entry is None:
+                if not readonly:
+                    recorded = _record_import(project_uid, user, path, fs_full, fs_epoch)
+                    if recorded:
+                        counts["imported"] += 1
+                        new_manifest[path] = recorded
+                continue
+            if not readonly and fs_epoch > entry["fs_epoch"] + _SYNC_CLOCK_SKEW_SECONDS:
+                recorded = _record_import(project_uid, user, path, fs_full, fs_epoch)
+                if recorded:
+                    counts["imported"] += 1
+                    new_manifest[path] = recorded
+                continue
+            try:
+                fs_full.unlink()
+                counts["deleted_in_workspace"] += 1
+            except OSError:
+                pass
+            continue
+
+    _save_sync_manifest(project_uid, manifest, new_manifest)
+    _warn_missing_blobs(project_uid, missing_blobs, "while syncing")
+    return counts
 
 
-def _export_node(row: dict, dest: Path) -> None:
+def _record_export(row: dict, dest: Path):
+    target = _export_node(row, dest)
+    if target is None:
+        return None
+    db_epoch = _epoch_of(row.get("updated_at"))
+    try:
+        fs_epoch = target.stat().st_mtime
+    except OSError:
+        fs_epoch = db_epoch
+    return {"db_epoch": db_epoch, "fs_epoch": fs_epoch}
+
+
+def _record_import(project_uid: str, user: dict, path: str, fs_full: Path, fs_epoch: float):
+    imported = _import_file(project_uid, user, path, fs_full)
+    if imported is None:
+        return None
+    return {"db_epoch": _epoch_of(imported.get("updated_at")), "fs_epoch": fs_epoch}
+
+
+def _copy_blob(src: Path, target: Path) -> bool:
+    try:
+        shutil.copyfile(src, target)
+        return True
+    except (FileNotFoundError, OSError):
+        return False
+
+
+def _warn_missing_blobs(project_uid: str, missing: int, action: str) -> None:
+    if missing <= 0:
+        return
+    now = time.monotonic()
+    previous = _missing_blob_logged.get(project_uid)
+    if (
+        previous is not None
+        and previous[0] == missing
+        and (now - previous[1]) < _MISSING_BLOB_LOG_SECONDS
+    ):
+        return
+    _missing_blob_logged[project_uid] = (missing, now)
+    logger.warning(
+        "Skipped %s missing blob file(s) %s project %s",
+        missing,
+        action,
+        project_uid,
+    )
+
+
+def _export_node(row: dict, dest: Path):
     target = (dest / row["path"]).resolve()
     if target != dest and not target.is_relative_to(dest):
-        return
+        return None
     target.parent.mkdir(parents=True, exist_ok=True)
     if target.is_symlink():
         target.unlink()
     if row.get("is_binary") and row.get("stored_name") and row.get("directory"):
         src = PROJECT_FILES_DIR / row["directory"] / row["stored_name"]
-        try:
-            shutil.copyfile(src, target)
-        except (FileNotFoundError, OSError):
-            logger.warning("Blob file missing during export: %s", src)
+        if not _copy_blob(src, target):
+            return None
     else:
         target.write_text(row.get("content") or "", encoding="utf-8")
+    return target
 
 
-def _import_file(project_uid: str, user: dict, path: str, fs_full: Path) -> bool:
+def _import_file(project_uid: str, user: dict, path: str, fs_full: Path):
     try:
         data = fs_full.read_bytes()
     except OSError:
-        return False
+        return None
     try:
-        store_upload(project_uid, user, _parent_of(path), _name_of(path), data)
-        return True
+        return store_upload(project_uid, user, _parent_of(path), _name_of(path), data)
     except ProjectFileError:
-        return False
+        return None
 
 
 def node_to_dict(row: dict) -> dict:
@@ -662,6 +815,36 @@ IMPORT_SKIP_NAMES = {
     ".DS_Store",
     ".idea",
     ".cache",
+    "build",
+    "dist",
+    "target",
+    "out",
+    "bin",
+    "obj",
+    ".next",
+    ".nuxt",
+    ".gradle",
+    ".tox",
+    "cmake-build-debug",
+    "cmake-build-release",
+}
+# Compiled/build-artifact extensions, regenerated wholesale on every build -
+# never worth importing/syncing regardless of which directory they land in
+# (unlike IMPORT_SKIP_NAMES, matched by suffix rather than exact name; see
+# _walk_workspace_files). A missing exclusion here is what let an unlocked
+# concurrent sync (see api._sync_dir_bidirectional_locked) leak millions of
+# orphaned blobs from an actively-compiling container workspace.
+IMPORT_SKIP_EXTENSIONS = {
+    ".o",
+    ".obj",
+    ".pyc",
+    ".pyo",
+    ".class",
+    ".so",
+    ".dylib",
+    ".dll",
+    ".a",
+    ".exe",
 }
 SYNC_SKIP_NAMES = IMPORT_SKIP_NAMES | {
     ".devplace_boot.py",
@@ -704,6 +887,7 @@ def export_to_dir(project_uid: str, subpath: str, dest_dir) -> int:
         rows = list(_table().find(project_uid=project_uid, deleted_at=None))
         strip = ""
     written = 0
+    missing = 0
     for row in sorted(rows, key=lambda r: r["path"]):
         relative = row["path"][len(strip) :].lstrip("/") if strip else row["path"]
         target = (dest / relative).resolve()
@@ -717,14 +901,13 @@ def export_to_dir(project_uid: str, subpath: str, dest_dir) -> int:
             target.unlink()
         if row.get("is_binary") and row.get("stored_name") and row.get("directory"):
             src = PROJECT_FILES_DIR / row["directory"] / row["stored_name"]
-            try:
-                shutil.copyfile(src, target)
-            except (FileNotFoundError, OSError):
-                logger.warning("Blob file missing: %s", src)
+            if not _copy_blob(src, target):
+                missing += 1
                 continue
         else:
             target.write_text(row.get("content") or "", encoding="utf-8")
         written += 1
+    _warn_missing_blobs(project_uid, missing, "during export of")
     return written
 
 
@@ -875,7 +1058,66 @@ def append_lines(project_uid: str, raw_path: str, content: str) -> dict:
 def delete_all_project_files(project_uid: str) -> None:
     if "project_files" not in db.tables:
         return
+    clear_sync_state(project_uid)
     for row in _table().find(project_uid=project_uid):
         if row.get("is_binary"):
             _unlink_blob(row)
     _table().delete(project_uid=project_uid)
+
+
+def purge_soft_deleted_project_files(*, dry_run: bool = False) -> tuple[int, int]:
+    if "project_files" not in db.tables:
+        return 0, 0
+    table = _table()
+    rows = list(table.find(table.table.columns.deleted_at.isnot(None), is_binary=1))
+    removed = 0
+    freed = 0
+    for row in rows:
+        directory = row.get("directory")
+        stored_name = row.get("stored_name")
+        if directory and stored_name:
+            path = PROJECT_FILES_DIR / directory / stored_name
+            try:
+                freed += path.stat().st_size
+            except OSError:
+                pass
+            else:
+                if not dry_run:
+                    _unlink_blob(row)
+        if not dry_run:
+            table.delete(id=row["id"])
+        removed += 1
+    return removed, freed
+
+
+def sweep_orphan_project_file_blobs(*, dry_run: bool = False) -> tuple[int, int]:
+    if not PROJECT_FILES_DIR.exists():
+        return 0, 0
+    referenced = set()
+    if "project_files" in db.tables:
+        for row in _table().find(is_binary=1):
+            directory = row.get("directory")
+            stored_name = row.get("stored_name")
+            if directory and stored_name:
+                referenced.add((directory, stored_name))
+    removed = 0
+    freed = 0
+    base = str(PROJECT_FILES_DIR)
+    for root, _dirs, files in os.walk(base):
+        directory = os.path.relpath(root, base)
+        for name in files:
+            if (directory, name) in referenced:
+                continue
+            file_path = os.path.join(root, name)
+            try:
+                size = os.path.getsize(file_path)
+            except OSError:
+                continue
+            if not dry_run:
+                try:
+                    os.unlink(file_path)
+                except OSError:
+                    continue
+            removed += 1
+            freed += size
+    return removed, freed

devplacepy/push/CLAUDE.md

@@ -2,33 +2,35 @@ This file documents `devplacepy/push/` - push notification delivery and its prov
 
 ## What this package is
 
-One delivery library behind one public surface. `devplacepy.push` exports `notify_user`, `register`, `ensure_certificates`, `public_key_standard_b64` and the Web Push crypto helpers; every caller in the codebase (`main.py`, `utils/notifications.py`, `routers/push.py`) imports only those names. Everything else is internal to the package.
+One delivery library behind one public surface. `devplacepy.push` exports `notify_user`, `notify_registration`, `register`, `unregister`, `ensure_certificates`, `public_key_standard_b64` and the Web Push crypto helpers; every caller in the codebase (`main.py`, `utils/notifications.py`, `routers/push.py`) imports only those names. Everything else is internal to the package.
 
 | Module | Role |
 |---|---|
 | `providers/base.py` | `PushProvider` protocol, the `Delivery` outcome and the three outcome constants |
 | `providers/webpush.py` | VAPID key material, `aesgcm` payload encryption, the Web Push provider |
-| `providers/apns.py` | Apple Push Notification service provider (token based, HTTP/2) |
+| `providers/apns.py` | Apple Push Notification service provider (token based, dedicated HTTP/2 client) |
 | `providers/__init__.py` | `PROVIDERS` registry, `get`, `active`, `is_active`, `admin_fields`, `client_config` |
-| `store.py` | Every `push_registration` read and write |
-| `delivery.py` | `notify_user`: group by provider, one shared client, one prepared body per provider |
+| `store.py` | Every `push_registration` read and write, including identity upsert and revive |
+| `delivery.py` | `notify_user` / `notify_registration`: group by provider, one client per provider, one prepared body per provider |
 
 The admin configuration surface lives in `devplacepy/services/push/service.py` (`PushService`), not here.
 
 ## Adding a provider
 
-1. Write `providers/<name>.py` with a `PushProvider` subclass: `name`, `label`, `config_fields`, `is_configured`, `parse_registration`, `prepare`, `deliver`, optionally `client_config`.
+1. Write `providers/<name>.py` with a `PushProvider` subclass: `name`, `label`, `config_fields`, `is_configured`, `parse_registration`, `prepare`, `deliver`, optionally `client_config`, `stamp_registration`, `delivery_client`.
 2. Add one entry to `PROVIDERS` in `providers/__init__.py`.
 
 That is the whole change. The registration route, the delivery loop, the admin page, the audit record, the metrics and the docs are written against the protocol and need no edit. The `Enabled` toggle (`push_<name>_enabled`) comes from the base class, so a provider never declares its own.
 
+The default `delivery_client` is `stealth_async_client`. Override it only when the destination is a first-party API that must not see Chrome impersonation, PRIORITY frames, extra browser headers, or the outbound proxy. APNs is that case.
+
 ## Invariants
 
-- **Zero cost for the request.** Delivery is reached only through `utils/notifications.py` `_schedule_push`, a fire-and-forget task. Never make a route await `notify_user`, and never add a queue or a table to this path.
+- **Zero cost for the request, except the welcome probe.** Delivery of real notifications is reached only through `utils/notifications.py` `_schedule_push`, a fire-and-forget task. `POST /push.json` awaits `notify_registration` only for a newly created or revived row, so the client can see `delivered` / `error`. Never add a queue or a table to this path.
 - **`deliver` never raises.** Return `Delivery(REJECTED, detail)` instead. `delivery.py` guards anyway, but a raising provider costs a log line per subscription.
 - **A provider that is not configured is inert, never an error.** `is_configured()` is false, `is_active()` is false, the delivery loop skips it, and `POST /push.json` refuses a registration for it with 400. Nothing else in the platform notices.
-- **`DEAD` is the only outcome that touches the database.** It soft-deletes the registration (`deleted_at`), exactly like a `404`/`410` Web Push endpoint always did. `REJECTED` keeps the row.
-- **Every insert writes `deleted_at: None`,** and every read filters `deleted_at IS NULL`. `push_registration` deliberately stays out of `SOFT_DELETE_TABLES` (no `deleted_by`, not restorable from Trash) - a dead device token has no owner action to undo.
+- **`DEAD` is the only outcome that touches the database.** It soft-deletes the registration (`deleted_at`), exactly like a `404`/`410` Web Push endpoint always did - unless `Delivery.dead_before` names an instant the row was proven live again after (APNs 410 `timestamp` vs. `registered_at`, see "APNs specifics"), in which case the delete is skipped. `REJECTED` keeps the row. The Apple/Web Push reason is logged at WARNING (`Push dead via ...: <detail>`); do not drop `outcome.detail`.
+- **Every insert writes `deleted_at: None`,** and every read of the live set filters `deleted_at IS NULL`. Identity lookups for upsert/revive intentionally ignore `deleted_at` so a client that re-registers a previously dead token comes back live. `push_registration` deliberately stays out of `SOFT_DELETE_TABLES` (no `deleted_by`, not restorable from Trash) - a dead device token has no owner action to undo.
 - **A row without a provider is a Web Push row.** `store.provider_of` resolves `None`/`""` to `DEFAULT_PROVIDER`, so a row written by an old worker during a deploy still delivers. `init_db` backfills the column once with a single converging `UPDATE`.
 
 ## Storage
@@ -40,13 +42,39 @@ That is the whole change. The registration route, the delivery loop, the admin p
 | `provider` | `webpush` | `apns` |
 | `endpoint`, `key_auth`, `key_p256dh` | set | `NULL` |
 | `token` | `NULL` | device token |
+| `client_id` | unused | optional stable per-device id |
+| `environment` | unused | `production` or `sandbox`, stamped server-side at register |
+| `registered_at` | stamped on insert/merge | stamped on insert/merge |
 
-Deduplication is generic: `store.register` looks up `user_uid` + `provider` + exactly the fields the provider's `parse_registration` returned, so a provider never writes its own identity rule.
+`registered_at` (ISO, both providers) is stamped on insert and on every `_merge` that actually changes a row (including revival). It exists solely to arbitrate the dead-token race described below - it is not a general "last seen" field.
+
+`store.register` returns `RegistrationWrite(record, created, revived)`. Identity, in order:
+
+1. `user_uid` + `provider` + `client_id` (live or dead), if `client_id` is present.
+2. `user_uid` + `provider` + `token` (or `endpoint` for Web Push), live or dead.
+3. Exact live match on the remaining fields.
+4. Insert.
+
+A match **updates** the row (token, client_id, environment) and clears `deleted_at` if it was dead. Token rotation with the same `client_id` therefore replaces the token on one row. Sibling live rows that share the new token, the previous token, or the same `client_id` are marked dead so one device cannot accumulate duplicates. A body without `client_id` still works: the same token revives, a new token inserts. `push.update` is a real update, not a no-op of an identical POST.
+
+## Unregistering
+
+`DELETE /push.json` (`routers/push.py` `push_unregister`) removes exactly the one registration named by the caller - never every registration for the user, mirroring how `POST /push.json` creates or updates exactly one row. `store.unregister(user_uid, provider, fields)` looks the row up with the **same identity priority as `register`**: `client_id`, then `token`, then `endpoint`. A match stamps `deleted_at` (the same bespoke soft-delete `mark_dead` already uses for a dead token - `push_registration` has no `deleted_by` and is not restorable from Trash by design, see "Invariants" above), so a later re-`register` with the same identity revives the row exactly like a dead token does, rather than accumulating a duplicate. No match (unknown identity, already-removed, or wrong owner) is a no-op that still returns `200 {"unregistered": false}` - the call is idempotent so a client can fire it speculatively at logout without first checking whether a subscription exists. The route validates that at least one identity field is a non-empty string before calling the store, refusing `400` on a body carrying none, exactly like `POST /push.json` refuses an unparseable registration body.
+
+**Client wiring.** `static/js/PushManager.js` intercepts every `a[href="/auth/logout"]` click, resolves the current `PushManager.getSubscription()` (only ever a webpush one - browser JS has no access to a native APNs device token), calls `DELETE /push.json` with `{endpoint}`, then `subscription.unsubscribe()` client-side, then navigates to the logout link's `href`. This closes the webpush half of the "token stays registered after logout" gap. **A native (iOS/Android) client integrating APNs must call `DELETE /push.json` with `{provider: "apns", token}` (or `client_id`) itself before or alongside its own logout** - there is no way for server-side `GET /auth/logout` (a plain, bodyless navigation) or this web frontend to know about, let alone unregister, a native app's device token.
+
+**No Devii action.** Unlike most mutating endpoints, `push.json` (register OR unregister) has no Devii catalog entry - a push identity (a webpush subscription endpoint/keys or an APNs device token) is private per-device browser/OS state that Devii cannot obtain, generate, or usefully ask the user to paste into a chat turn. This mirrors `POST /push.json` already having no Devii action for the identical reason.
 
 ## APNs specifics
 
-- `POST https://{host}/3/device/{token}` over HTTP/2, host from `push_apns_environment` (`api.push.apple.com` or `api.sandbox.push.apple.com`; an unrecognised value falls back to production). HTTP/2 comes from `stealth_async_client` because the origin is `https` - the cleartext downgrade in `curl_transport` does not apply.
-- Provider token: `ES256`, header `kid` = key id, claims `iss` = team id and `iat`. Cached per credential fingerprint for 45 minutes, so a worker signs at most one token per 45 minutes; Apple refuses tokens regenerated faster than every 20 minutes. Editing any credential changes the fingerprint and takes effect on the next delivery, with no restart.
+- **Persistent HTTP/2 connection, not one per notification.** `apns.cached_client(timeout)` lazily creates ONE module-level `httpx.AsyncClient(http2=True, ...)` and reuses it across every `notify_user`/`notify_registration` call for the lifetime of the process, following Apple's explicit guidance to keep the connection open rather than repeatedly opening/closing (`sending-notification-requests-to-apns`). `ApnsProvider.closes_delivery_client()` returns `False` so `delivery.py` never closes it after a batch (Web Push still opens/closes per call via `stealth_async_client`, `closes_delivery_client()` defaulting `True` on the base class). Closed once, gracefully, in `main.py`'s shutdown via `push.shutdown_providers()` -> `ApnsProvider.aclose()` -> `apns.close_client()`, mirroring the identical `services/containers/forward.py` `client()`/`close_client()` pattern. Not stealth, not curl_cffi Chrome impersonation, not the outbound proxy - Apple's provider API is a first-party HTTP/2 service; browser PRIORITY frames, HPACK indexing of `:path`/`authorization`, extra `sec-ch-ua` headers, and a scraping proxy all violate that contract. This is the second documented exception to the stealth-only outbound rule (the other is container reverse-proxy forwarding). If the admin-configured delivery timeout changes, the cached client is rebuilt with the new timeout on next use and the old one is left for GC (not explicitly closed) - a deliberate, rare-path simplification.
+- `POST https://{host}/3/device/{token}` over HTTP/2. Host comes from the **row's** `environment` if set, else `push_apns_environment` (`api.push.apple.com` or `api.sandbox.push.apple.com`; an unrecognised value falls back to production). Stamping environment per row lets a sandbox debug token and a production TestFlight token coexist.
+- `GET /push.json` advertises `providers.apns.environment` when APNs is active so a native client can refuse to register a sandbox token against production.
+- `parse_registration` requires a hexadecimal `token` (64-200 digits after stripping spaces and `<>`) and optionally `client_id` (string, max 128). A `client_id` that is not a string is a 400, not a silent drop. Server stamps `environment`; the client cannot pick the host.
+- **Provider token (JWT):** `ES256`, header `kid` = key id, claims `iss` = team id and `iat`. Cached per credential fingerprint for 45 minutes (Apple refuses tokens regenerated faster than every 20 minutes, and requires a fresh one at least once an hour). **Shared cross-worker via `site_settings` key `push_apns_shared_token`** (JSON `{fingerprint, token, issued_at}`, read/written through the existing `get_setting`/`set_setting` cache-version machinery, propagating to sibling workers within ~1s like every other settings key) - a worker that finds no valid in-process cache checks the shared copy before signing a new one, so `make prod`'s multiple uvicorn workers converge on presenting Apple the SAME token instead of each independently re-signing on its own clock (which could otherwise interleave closer together than Apple's per-credential minimum spacing, worst case when workers boot near-simultaneously). Editing any credential changes the fingerprint and takes effect on the next delivery, with no restart.
+- **A rejected provider token is evicted immediately, not left to expire from cache.** A 403 status, or any reason in `AUTH_REASONS` (`InvalidProviderToken`, `ExpiredProviderToken`, `BadCertificate`, `BadCertificateEnvironment`, `Forbidden`, `MissingProviderToken` - all provider-credential-shaped per Apple's reason table, never about a specific device), calls `invalidate_provider_token()`: clears the in-process cache AND the shared DB copy, so the very next delivery attempt (any worker) re-signs instead of retrying the same rejected token for up to 45 minutes.
 - A `.p8` that does not parse is cached as a failure for the same window, so a misconfiguration costs one error log per window rather than one parse per notification.
-- `410`, or any status carrying reason `BadDeviceToken`, `Unregistered`, `ExpiredToken`, `DeviceTokenNotForTopic` or `TopicDisallowed`, is `DEAD`. Everything else is `REJECTED`.
+- **`DEAD_REASONS` is deliberately narrow: only `BadDeviceToken` (400), `Unregistered` (410) and `ExpiredToken` (410).** These are the only reasons in Apple's documented table that describe the TOKEN itself as permanently invalid. `DeviceTokenNotForTopic` and `TopicDisallowed` are 400 errors about the **topic/provisioning matching the connection**, not the token - they fire identically for every token when `push_apns_topic` is misconfigured or the certificate/entitlements don't match, so treating them as dead would soft-delete the entire APNs subscriber base (a table deliberately kept OUT of `SOFT_DELETE_TABLES`, hence unrestorable) on the first delivery after a one-field admin typo. Never add a topic/provisioning-shaped reason to `DEAD_REASONS`.
+- **A 410's `timestamp` is honored before deleting.** Apple's 410 body carries `{"reason": "Unregistered", "timestamp": <ms epoch>}` - the instant Apple last confirmed the token dead, which can trail real device state by "several days" per Apple engineering guidance (410 delivery is intentionally non-deterministic; do not use it to infer app-uninstall timing). `apns._dead_before` converts it to ISO and it rides on `Delivery.dead_before`; `store.mark_dead(id, dead_before)` compares it against the row's `registered_at` and **skips the delete** (logs at INFO instead) if the registration was re-registered/revived after that instant - closing the race where an in-flight delivery against a stale row state would otherwise undo a concurrent revival. `dead_before` is only ever set for a 410; the `BadDeviceToken`/`ExpiredToken` paths pass `None` (unconditional delete, as before), since those reasons carry no `timestamp`.
 - The shared payload dict (`title`, `message`, `icon`, `url`) is translated once per batch into `aps.alert` plus the custom `url`/`icon` keys, mirroring what `service-worker.js` does for Web Push. `thread-id` mirrors the service worker's notification `tag`.
+- `POST /push.json` probes a created or revived row immediately via `notify_registration`. The JSON is `{registered: true, delivered: bool, error?: string}`. `registered` stays true so existing clients keep working; `delivered`/`error` surface Apple's reason instead of killing the row silently from the client's point of view. The row is still marked dead on a `DEAD` outcome (subject to the `dead_before` guard above) so later `notify_user` calls do not keep hitting a known-bad token.

devplacepy/push/__init__.py

@@ -1,6 +1,7 @@
 # retoor <retoor@molodetz.nl>
 
-from devplacepy.push.delivery import notify_user
+from devplacepy.push.delivery import notify_registration, notify_user
+from devplacepy.push.providers import shutdown as shutdown_providers
 from devplacepy.push.providers.webpush import (
     browser_base64,
     create_notification_authorization,
@@ -12,7 +13,7 @@ from devplacepy.push.providers.webpush import (
     hkdf,
     public_key_standard_b64,
 )
-from devplacepy.push.store import register
+from devplacepy.push.store import register, unregister
 
 __all__ = [
     "browser_base64",
@@ -23,7 +24,10 @@ __all__ = [
     "generate_private_key",
     "generate_public_key",
     "hkdf",
+    "notify_registration",
     "notify_user",
     "public_key_standard_b64",
     "register",
+    "shutdown_providers",
+    "unregister",
 ]

devplacepy/push/delivery.py

@@ -3,9 +3,9 @@
 import logging
 from typing import Any
 
-from devplacepy import stealth
 from devplacepy.database import get_int_setting
 from devplacepy.push import providers, store
+from devplacepy.push.providers.base import Delivery
 
 logger = logging.getLogger(__name__)
 
@@ -29,6 +29,10 @@ def group_by_provider(
     return grouped
 
 
+def _open_client(provider, timeout: float):
+    return provider.delivery_client(timeout)
+
+
 async def notify_user(user_uid: str, payload: dict[str, Any]) -> None:
     registrations = store.active_for_user(user_uid)
     if not registrations:
@@ -36,48 +40,86 @@ async def notify_user(user_uid: str, payload: dict[str, Any]) -> None:
         return
 
     grouped = group_by_provider(registrations)
-    async with stealth.stealth_async_client(timeout=timeout_seconds()) as client:
-        for name, rows in grouped.items():
-            provider = providers.PROVIDERS.get(name)
-            if provider is None:
-                logger.warning(
-                    "Unknown push provider %s on %s subscriptions of user %s",
-                    name,
-                    len(rows),
-                    user_uid,
-                )
-                continue
-            if not providers.is_active(provider):
-                logger.debug(
-                    "Push provider %s is not active; skipping %s subscriptions",
-                    name,
-                    len(rows),
-                )
-                continue
-            try:
-                prepared = provider.prepare(payload)
-            except Exception as exc:
-                logger.error("Push provider %s could not build a payload: %s", name, exc)
-                continue
+    timeout = timeout_seconds()
+    for name, rows in grouped.items():
+        provider = providers.PROVIDERS.get(name)
+        if provider is None:
+            logger.warning(
+                "Unknown push provider %s on %s subscriptions of user %s",
+                name,
+                len(rows),
+                user_uid,
+            )
+            continue
+        if not providers.is_active(provider):
+            logger.debug(
+                "Push provider %s is not active; skipping %s subscriptions",
+                name,
+                len(rows),
+            )
+            continue
+        try:
+            prepared = provider.prepare(payload)
+        except Exception as exc:
+            logger.error("Push provider %s could not build a payload: %s", name, exc)
+            continue
+        try:
+            client = _open_client(provider, timeout)
+        except Exception as exc:
+            logger.error("Push provider %s could not open a client: %s", name, exc)
+            continue
+        try:
             for registration in rows:
                 await _deliver_one(provider, client, registration, prepared, user_uid)
+        finally:
+            if provider.closes_delivery_client():
+                await client.aclose()
 
 
-async def _deliver_one(provider, client, registration, prepared, user_uid) -> None:
+async def notify_registration(
+    registration: dict[str, Any], payload: dict[str, Any]
+) -> Delivery:
+    name = store.provider_of(registration)
+    provider = providers.PROVIDERS.get(name)
+    if provider is None:
+        return Delivery(providers.REJECTED, f"unknown provider {name}")
+    if not providers.is_active(provider):
+        return Delivery(providers.REJECTED, f"provider {name} is not active")
+    try:
+        prepared = provider.prepare(payload)
+    except Exception as exc:
+        return Delivery(providers.REJECTED, str(exc))
+    user_uid = registration.get("user_uid") or ""
+    try:
+        client = _open_client(provider, timeout_seconds())
+    except Exception as exc:
+        return Delivery(providers.REJECTED, str(exc))
+    try:
+        return await _deliver_one(provider, client, registration, prepared, user_uid)
+    finally:
+        if provider.closes_delivery_client():
+            await client.aclose()
+
+
+async def _deliver_one(provider, client, registration, prepared, user_uid) -> Delivery:
     try:
         outcome = await provider.deliver(client, registration, prepared)
     except Exception as exc:
         logger.error("Push provider %s raised for %s: %s", provider.name, user_uid, exc)
-        return
+        return Delivery(providers.REJECTED, str(exc))
     if outcome.status == providers.ACCEPTED:
         logger.debug("Push delivered to %s via %s", user_uid, provider.name)
-        return
+        return outcome
     if outcome.status == providers.DEAD:
+        logger.warning(
+            "Push dead via %s for %s: %s", provider.name, user_uid, outcome.detail
+        )
         try:
-            store.mark_dead(registration["id"])
+            store.mark_dead(registration["id"], outcome.dead_before)
         except Exception as exc:
             logger.error("Could not soft-delete push subscription: %s", exc)
-        return
+        return outcome
     logger.warning(
         "Push rejected by %s for %s: %s", provider.name, user_uid, outcome.detail
     )
+    return outcome

devplacepy/push/providers/__init__.py

@@ -35,6 +35,7 @@ __all__ = [
     "get",
     "is_active",
     "names",
+    "shutdown",
 ]
 
 
@@ -74,3 +75,11 @@ def _client_config(provider: PushProvider) -> dict[str, Any]:
     except Exception as exc:
         logger.error("Push provider %s failed to describe itself: %s", provider.name, exc)
         return {}
+
+
+async def shutdown() -> None:
+    for provider in PROVIDERS.values():
+        try:
+            await provider.aclose()
+        except Exception as exc:
+            logger.error("Push provider %s failed to close: %s", provider.name, exc)

devplacepy/push/providers/apns.py

@@ -5,13 +5,14 @@ import json
 import logging
 import string
 import time
+from datetime import datetime, timezone
 from typing import Any
 
 import httpx
 import jwt
 
 from devplacepy.config import SECONDS_PER_DAY
-from devplacepy.database import get_setting
+from devplacepy.database import get_setting, set_setting
 from devplacepy.push.providers.base import (
     ACCEPTED,
     DEAD,
@@ -44,20 +45,27 @@ ENVIRONMENT_OPTIONS = [
 TOKEN_REFRESH_SECONDS = 45 * 60
 TOKEN_MIN_LENGTH = 64
 TOKEN_MAX_LENGTH = 200
+CLIENT_ID_MAX_LENGTH = 128
 THREAD_ID = "devplace-notification"
 PUSH_TYPE = "alert"
 PRIORITY = "10"
-DEAD_REASONS = frozenset(
+SHARED_TOKEN_KEY = "push_apns_shared_token"
+
+DEAD_REASONS = frozenset({"BadDeviceToken", "ExpiredToken", "Unregistered"})
+
+AUTH_REASONS = frozenset(
     {
-        "BadDeviceToken",
-        "DeviceTokenNotForTopic",
-        "ExpiredToken",
-        "Unregistered",
-        "TopicDisallowed",
+        "BadCertificate",
+        "BadCertificateEnvironment",
+        "ExpiredProviderToken",
+        "Forbidden",
+        "InvalidProviderToken",
+        "MissingProviderToken",
     }
 )
 
 _token_state: dict[str, Any] = {}
+_client: httpx.AsyncClient | None = None
 
 
 def _setting(key: str) -> str:
@@ -70,13 +78,78 @@ def _environment() -> str:
 
 
 def host() -> str:
-    return HOSTS[_environment()]
+    return host_for(_environment())
+
+
+def host_for(environment: str | None) -> str:
+    value = (environment or "").strip() or _environment()
+    return HOSTS[value] if value in HOSTS else HOSTS[DEFAULT_ENVIRONMENT]
+
+
+def gateway_client(timeout: float) -> httpx.AsyncClient:
+    return httpx.AsyncClient(
+        http2=True,
+        timeout=timeout,
+        trust_env=False,
+        verify=True,
+        follow_redirects=False,
+    )
+
+
+def cached_client(timeout: float) -> httpx.AsyncClient:
+    global _client
+    if _client is None or _client.is_closed:
+        _client = gateway_client(timeout)
+    return _client
+
+
+async def close_client() -> None:
+    global _client
+    if _client is not None and not _client.is_closed:
+        await _client.aclose()
+    _client = None
+
+
+def _normalize_token(token: str) -> str:
+    return "".join(
+        character.lower() for character in token if character in string.hexdigits
+    )
 
 
 def _fingerprint(team_id: str, key_id: str, auth_key: str) -> str:
     return hashlib.sha256(f"{team_id}:{key_id}:{auth_key}".encode("utf-8")).hexdigest()
 
 
+def _read_shared_token() -> dict[str, Any] | None:
+    raw = get_setting(SHARED_TOKEN_KEY, "")
+    if not raw:
+        return None
+    try:
+        data = json.loads(raw)
+    except ValueError:
+        return None
+    if (
+        not isinstance(data, dict)
+        or not isinstance(data.get("fingerprint"), str)
+        or not isinstance(data.get("token"), str)
+        or not isinstance(data.get("issued_at"), int)
+    ):
+        return None
+    return data
+
+
+def _write_shared_token(fingerprint: str, token: str, issued_at: int) -> None:
+    set_setting(
+        SHARED_TOKEN_KEY,
+        json.dumps({"fingerprint": fingerprint, "token": token, "issued_at": issued_at}),
+    )
+
+
+def invalidate_provider_token() -> None:
+    _token_state.pop("current", None)
+    set_setting(SHARED_TOKEN_KEY, "")
+
+
 def provider_token(team_id: str, key_id: str, auth_key: str) -> str:
     fingerprint = _fingerprint(team_id, key_id, auth_key)
     issued_at = int(time.time())
@@ -89,6 +162,21 @@ def provider_token(team_id: str, key_id: str, auth_key: str) -> str:
         if state["token"] is None:
             raise ValueError(state["error"])
         return state["token"]
+
+    shared = _read_shared_token()
+    if (
+        shared
+        and shared["fingerprint"] == fingerprint
+        and issued_at - shared["issued_at"] < TOKEN_REFRESH_SECONDS
+    ):
+        _token_state["current"] = {
+            "token": shared["token"],
+            "error": "",
+            "issued_at": shared["issued_at"],
+            "fingerprint": fingerprint,
+        }
+        return shared["token"]
+
     try:
         token = jwt.encode(
             {"iss": team_id, "iat": issued_at},
@@ -112,6 +200,7 @@ def provider_token(team_id: str, key_id: str, auth_key: str) -> str:
         "issued_at": issued_at,
         "fingerprint": fingerprint,
     }
+    _write_shared_token(fingerprint, token, issued_at)
     return token
 
 
@@ -125,6 +214,22 @@ def _reason(response: httpx.Response) -> str:
     return ""
 
 
+def _dead_before(response: httpx.Response) -> str | None:
+    try:
+        body = response.json()
+    except ValueError:
+        return None
+    if not isinstance(body, dict):
+        return None
+    raw = body.get("timestamp")
+    if not isinstance(raw, (int, float)) or isinstance(raw, bool):
+        return None
+    try:
+        return datetime.fromtimestamp(raw / 1000, tz=timezone.utc).isoformat()
+    except (OverflowError, OSError, ValueError):
+        return None
+
+
 class ApnsProvider(PushProvider):
     name = "apns"
     label = PROVIDER_LABEL
@@ -181,16 +286,41 @@ class ApnsProvider(PushProvider):
             and _setting(TOPIC_KEY)
         )
 
+    def client_config(self) -> dict[str, Any]:
+        return {"environment": _environment()}
+
+    def stamp_registration(self, fields: dict[str, Any]) -> dict[str, Any]:
+        return {**fields, "environment": _environment()}
+
+    def delivery_client(self, timeout: float) -> httpx.AsyncClient:
+        return cached_client(timeout)
+
+    def closes_delivery_client(self) -> bool:
+        return False
+
+    async def aclose(self) -> None:
+        await close_client()
+
     def parse_registration(self, body: dict[str, Any]) -> dict[str, Any] | None:
         token = body.get("token")
         if not isinstance(token, str):
             return None
-        token = token.strip()
+        token = _normalize_token(token)
         if not TOKEN_MIN_LENGTH <= len(token) <= TOKEN_MAX_LENGTH:
             return None
-        if any(character not in string.hexdigits for character in token):
+        fields: dict[str, Any] = {"token": token}
+        client_id = body.get("client_id")
+        if client_id is None:
+            return fields
+        if not isinstance(client_id, str):
             return None
-        return {"token": token}
+        client_id = client_id.strip()
+        if not client_id:
+            return fields
+        if len(client_id) > CLIENT_ID_MAX_LENGTH:
+            return None
+        fields["client_id"] = client_id
+        return fields
 
     def prepare(self, payload: dict[str, Any]) -> str:
         return json.dumps(
@@ -228,7 +358,7 @@ class ApnsProvider(PushProvider):
         try:
             headers = self.headers()
             response = await client.post(
-                f"https://{host()}/3/device/{token}",
+                f"https://{host_for(registration.get('environment'))}/3/device/{token}",
                 headers=headers,
                 content=prepared.encode("utf-8"),
             )
@@ -238,6 +368,9 @@ class ApnsProvider(PushProvider):
             return Delivery(ACCEPTED)
         reason = _reason(response)
         detail = f"{response.status_code} {reason}".strip()
+        if response.status_code == 403 or reason in AUTH_REASONS:
+            invalidate_provider_token()
         if response.status_code == 410 or reason in DEAD_REASONS:
-            return Delivery(DEAD, detail)
+            dead_before = _dead_before(response) if response.status_code == 410 else None
+            return Delivery(DEAD, detail, dead_before)
         return Delivery(REJECTED, detail)

devplacepy/push/providers/base.py

@@ -18,6 +18,7 @@ REJECTED = "rejected"
 class Delivery:
     status: str
     detail: str = ""
+    dead_before: str | None = None
 
 
 class PushProvider(ABC):
@@ -51,6 +52,20 @@ class PushProvider(ABC):
     def client_config(self) -> dict[str, Any]:
         return {}
 
+    def stamp_registration(self, fields: dict[str, Any]) -> dict[str, Any]:
+        return fields
+
+    def delivery_client(self, timeout: float) -> httpx.AsyncClient:
+        from devplacepy import stealth
+
+        return stealth.stealth_async_client(timeout=timeout)
+
+    def closes_delivery_client(self) -> bool:
+        return True
+
+    async def aclose(self) -> None:
+        return None
+
     @abstractmethod
     def is_configured(self) -> bool: ...
 

devplacepy/push/store.py

@@ -1,6 +1,7 @@
 # retoor <retoor@molodetz.nl>
 
 import logging
+from dataclasses import dataclass
 from datetime import datetime, timezone
 from typing import Any
 
@@ -13,6 +14,17 @@ logger = logging.getLogger(__name__)
 TABLE = "push_registration"
 
 
+@dataclass(frozen=True)
+class RegistrationWrite:
+    record: dict[str, Any]
+    created: bool
+    revived: bool
+
+    @property
+    def probe(self) -> bool:
+        return self.created or self.revived
+
+
 def table():
     return get_table(TABLE)
 
@@ -25,31 +37,185 @@ def active_for_user(user_uid: str) -> list[dict[str, Any]]:
     return list(table().find(user_uid=user_uid, deleted_at=None))
 
 
+def _filled(fields: dict[str, Any]) -> dict[str, Any]:
+    filled: dict[str, Any] = {}
+    for key, value in fields.items():
+        if value is None:
+            continue
+        if isinstance(value, str):
+            value = value.strip()
+            if not value:
+                continue
+        filled[key] = value
+    return filled
+
+
+def _prefer_live(rows: list[dict[str, Any]]) -> dict[str, Any] | None:
+    if not rows:
+        return None
+    live = [row for row in rows if not row.get("deleted_at")]
+    return (live or rows)[-1]
+
+
+def _lookup(user_uid: str, provider: str, **identity: Any) -> dict[str, Any] | None:
+    return _prefer_live(
+        list(table().find(user_uid=user_uid, provider=provider, **identity))
+    )
+
+
+def _with_id(record: dict[str, Any]) -> dict[str, Any]:
+    if record.get("id"):
+        return record
+    found = table().find_one(uid=record.get("uid"))
+    return found or record
+
+
+def _retire_duplicates(
+    user_uid: str,
+    provider: str,
+    keep_id: int,
+    fields: dict[str, Any],
+    previous_token: str | None = None,
+) -> None:
+    token = fields.get("token")
+    client_id = fields.get("client_id")
+    for row in table().find(user_uid=user_uid, provider=provider, deleted_at=None):
+        if row["id"] == keep_id:
+            continue
+        if client_id and row.get("client_id") == client_id:
+            mark_dead(row["id"])
+            continue
+        if token and row.get("token") == token:
+            mark_dead(row["id"])
+            continue
+        if previous_token and row.get("token") == previous_token:
+            mark_dead(row["id"])
+
+
+def _merge(
+    existing: dict[str, Any], fields: dict[str, Any]
+) -> RegistrationWrite:
+    revived = bool(existing.get("deleted_at"))
+    patch: dict[str, Any] = {"id": existing["id"]}
+    if revived:
+        patch["deleted_at"] = None
+    changed = revived
+    for key, value in fields.items():
+        if existing.get(key) != value:
+            patch[key] = value
+            changed = True
+    if not changed:
+        return RegistrationWrite(_with_id(existing), False, False)
+    patch["registered_at"] = datetime.now(timezone.utc).isoformat()
+    previous_token = existing.get("token")
+    table().update(patch, ["id"])
+    merged = {**existing, **patch}
+    if revived:
+        merged["deleted_at"] = None
+    merged = _with_id(merged)
+    _retire_duplicates(
+        existing["user_uid"],
+        existing["provider"],
+        merged["id"],
+        fields,
+        previous_token=previous_token if previous_token != fields.get("token") else None,
+    )
+    logger.info(
+        "Updated %s push subscription for user %s%s",
+        existing.get("provider"),
+        existing.get("user_uid"),
+        " (revived)" if revived else "",
+    )
+    return RegistrationWrite(merged, False, revived)
+
+
 def register(
     user_uid: str, provider: str, fields: dict[str, Any]
-) -> tuple[dict[str, Any], bool]:
+) -> RegistrationWrite:
+    fields = _filled(fields)
     registrations = table()
-    existing = registrations.find_one(
+    client_id = fields.get("client_id")
+    token = fields.get("token")
+    endpoint = fields.get("endpoint")
+
+    if client_id:
+        existing = _lookup(user_uid, provider, client_id=client_id)
+        if existing:
+            return _merge(existing, fields)
+
+    if token:
+        existing = _lookup(user_uid, provider, token=token)
+        if existing:
+            return _merge(existing, fields)
+
+    if endpoint:
+        existing = _lookup(user_uid, provider, endpoint=endpoint)
+        if existing:
+            return _merge(existing, fields)
+
+    live = registrations.find_one(
         user_uid=user_uid, provider=provider, deleted_at=None, **fields
     )
-    if existing:
-        logger.debug("Push subscription already registered for user %s", user_uid)
-        return existing, False
+    if live:
+        return RegistrationWrite(_with_id(live), False, False)
 
+    now = datetime.now(timezone.utc).isoformat()
     record = {
         "uid": generate_uid(),
         "user_uid": user_uid,
         "provider": provider,
-        "created_at": datetime.now(timezone.utc).isoformat(),
+        "created_at": now,
+        "registered_at": now,
         "deleted_at": None,
         **fields,
     }
-    registrations.insert(record)
+    inserted = registrations.insert(record)
+    if isinstance(inserted, int):
+        record["id"] = inserted
+    record = _with_id(record)
+    if record.get("id"):
+        _retire_duplicates(user_uid, provider, record["id"], fields)
     logger.info("Registered %s push subscription for user %s", provider, user_uid)
-    return record, True
+    return RegistrationWrite(record, True, False)
 
 
-def mark_dead(registration_id: int) -> None:
+def unregister(user_uid: str, provider: str, fields: dict[str, Any]) -> bool:
+    fields = _filled(fields)
+    client_id = fields.get("client_id")
+    token = fields.get("token")
+    endpoint = fields.get("endpoint")
+
+    row = None
+    if client_id:
+        row = _lookup(user_uid, provider, client_id=client_id)
+    if row is None and token:
+        row = _lookup(user_uid, provider, token=token)
+    if row is None and endpoint:
+        row = _lookup(user_uid, provider, endpoint=endpoint)
+    if row is None or row.get("deleted_at"):
+        return False
+
+    table().update(
+        {"id": row["id"], "deleted_at": datetime.now(timezone.utc).isoformat()},
+        ["id"],
+    )
+    logger.info("Unregistered %s push subscription for user %s", provider, user_uid)
+    return True
+
+
+def mark_dead(registration_id: int, dead_before: str | None = None) -> None:
+    if dead_before is not None:
+        row = table().find_one(id=registration_id)
+        registered_at = row.get("registered_at") if row else None
+        if registered_at and registered_at > dead_before:
+            logger.info(
+                "Skipped marking push subscription id=%s dead: registered again at %s "
+                "after the provider confirmed it dead at %s",
+                registration_id,
+                registered_at,
+                dead_before,
+            )
+            return
     table().update(
         {"id": registration_id, "deleted_at": datetime.now(timezone.utc).isoformat()},
         ["id"],

devplacepy/rendering.py

@@ -70,6 +70,26 @@ _TOKEN_RE = re.compile(
 _SCHEME_RE = re.compile(r"^([a-z][a-z0-9+.\-]*):", re.I)
 _ALLOWED_SCHEMES = {"http", "https", "mailto", "tel"}
 
+_TRAILING_PUNCT = ".,;:!?"
+_TRAILING_CLOSERS = {")": "(", "]": "[", "}": "{"}
+
+
+def _split_trailing_punct(url: str) -> tuple[str, str]:
+    trailing: list[str] = []
+    while url:
+        char = url[-1]
+        if char in _TRAILING_PUNCT:
+            trailing.append(char)
+            url = url[:-1]
+            continue
+        opener = _TRAILING_CLOSERS.get(char)
+        if opener is not None and url.count(char) > url.count(opener):
+            trailing.append(char)
+            url = url[:-1]
+            continue
+        break
+    return url, "".join(reversed(trailing))
+
 _YOUTUBE_ALLOW = (
     "accelerometer; autoplay; clipboard-write; encrypted-media; "
     "gyroscope; picture-in-picture"
@@ -112,6 +132,40 @@ _content_markdown = mistune.create_markdown(
     plugins=["strikethrough", "table"],
 )
 
+_structure_markdown = mistune.create_markdown(
+    renderer=None,
+    plugins=["strikethrough", "table"],
+)
+
+_STRUCTURE_TOKEN_KEYS = {
+    "block_code": "code_fences",
+    "list_item": "list_items",
+    "heading": "headers",
+    "link": "links",
+}
+
+
+def _count_structure_tokens(tokens, signature: dict[str, int]) -> None:
+    for token in tokens or []:
+        key = _STRUCTURE_TOKEN_KEYS.get(token.get("type"))
+        if key:
+            signature[key] += 1
+        children = token.get("children")
+        if children:
+            _count_structure_tokens(children, signature)
+
+
+def markdown_structure_signature(text: str) -> dict[str, int]:
+    signature = {key: 0 for key in _STRUCTURE_TOKEN_KEYS.values()}
+    if not text or not text.strip():
+        return signature
+    try:
+        tokens = _structure_markdown(text)
+    except (TypeError, ValueError):
+        return signature
+    _count_structure_tokens(tokens, signature)
+    return signature
+
 
 def _normalize_dashes(text: str) -> str:
     text = text.replace("\u2014", "-")
@@ -183,7 +237,10 @@ def _transform_text(text: str) -> str:
         if match.start() > pos:
             out.append(html.escape(_mask_emails(text[pos:match.start()])))
         if match.group("url"):
-            out.append(_embed_url(match.group("url")))
+            url, trailing = _split_trailing_punct(match.group("url"))
+            out.append(_embed_url(url))
+            if trailing:
+                out.append(html.escape(trailing))
         else:
             user = match.group("mention")
             out.append(
@@ -335,3 +392,21 @@ def content_preview(text, length: int = 60) -> str:
     if len(plain) <= length:
         return plain
     return plain[:length].rstrip() + "..."
+
+
+def safe_truncate(text, length: int = 300) -> str:
+    if not text:
+        return ""
+    text_str = str(text)
+    if len(text_str) <= length:
+        return text_str
+    cutoff = length
+    for match in _TOKEN_RE.finditer(text_str):
+        if match.start() < cutoff < match.end():
+            cutoff = match.start()
+            break
+    if cutoff > 0 and not text_str[cutoff - 1].isspace() and not text_str[cutoff].isspace():
+        boundary = text_str.rfind(" ", 0, cutoff)
+        if boundary > 0:
+            cutoff = boundary
+    return text_str[:cutoff].rstrip()

devplacepy/routers/CLAUDE.md

@@ -11,15 +11,17 @@ Prefixes are wired in `main.py`:
 | `/auth` | auth/ package - one leaf per flow (`signup`, `login`, `logout`, `forgotpassword`, `resetpassword`) |
 | `/feed` | feed.py |
 | `/posts` | posts.py |
+| `/topics` | topics.py - crawlable per-topic category index pages (`GET /topics` hub, `GET /topics/{topic}` per-topic post listing over the same `TOPICS` set as the feed sidebar filter). Reuses `feed.py`'s `get_feed_posts`/`enrich_post_cards` so a topic page is a fully-enriched `_post_card.html` listing, not a stripped-down duplicate. Unlike `/feed?topic=X` (whose canonical strips the query string back to bare `/feed`, so it is never indexed as a distinct page), each `/topics/{topic}` page has its own canonical URL, unique title/description, breadcrumbs, and a sitemap entry - see "SEO implementation" below |
 | `/comments` | comments.py |
 | `/projects` | projects/ package - `index.py` (listing/detail/create/delete plus owner-only visibility toggles `POST /{slug}/private` and `POST /{slug}/readonly`, and async `POST /{slug}/fork`), `files.py`, and the `containers/` subpackage (below). `main.py` mounts the whole `/projects` tree from this one package. See `routers/projects/CLAUDE.md` for the deep detail on this tree |
 | `/projects/{slug}/files` | projects/files.py - per-project virtual filesystem (CRUD dirs/files, upload, inline edit, plus line-range ops: `lines` read, `replace-lines`, `insert-lines`, `delete-lines`, `append` for large text files) |
 | `/profile` | profile/ package - `index.py` (page/search/followers/update/api-key; the **Posts** tab shows only the 10 most recent posts, while the sidebar `posts_count` stat always uses the real `count()` so it is never capped to the rendered 10), `customization.py`, `notifications.py`, `ai_correction.py` (`POST /{username}/ai-correction`, owner-or-admin AI content correction toggle + prompt), `ai_modifier.py` (`POST /{username}/ai-modifier`, owner-or-admin AI modifier toggle + prompt), `interactions.py` (`POST /{username}/interactions`, owner-or-admin Devii interactive widgets preference; admin default via `devii_interactions_default`), `telegram.py` (`POST /{username}/telegram`, owner-or-admin Telegram pairing-code request/unpair), `usage.py` (`_ai_quota`, re-exported from the package) |
-| `/messages` | messages.py - real-time DM chat. `GET ""` (page; the conversation sidebar is one windowed SQL aggregate and an opened thread loads only the newest `CONVERSATION_MESSAGE_LIMIT` = 500 messages - keep these queries bounded), `GET /search`, `GET /conversations` (flat JSON conversation list, same `get_conversations` helper as the page), `POST /send` (no-JS fallback, also accepts an attachment-only empty-content message), `POST /ws-ticket` (issues a 30s single-use WS auth ticket so a browser WebSocket can authenticate cross-context, since a native `WebSocket` cannot set an `Authorization`/`X-API-KEY` header), and `WS /messages/ws` (NOT lock-owner gated - accepts on every worker; closes `1008` for guests). The WS adds live bidirectional delivery, typing, and read receipts on top of the existing `messages` table - presence is NOT part of this WS (it is the generic `PresenceManager`/pub/sub mechanism, see `devplacepy/services/CLAUDE.md`). Both the WS `send` and HTTP `POST /send` route through the shared `services/messaging/persist.py` `persist_message` choke point (identical audit/notification/mention) and then through `_finalize_and_broadcast`, which awaits any pending SYNC AI correction/modifier futures (DM content runs through both, so typing `@ai <instruction>` executes live), broadcasts the FINAL corrected/modified content with an additive `ai_processed` frame flag (the WS path passes `request=websocket` and awaits directly since HTTP middleware does not run for websockets; the frontend swaps the sender's optimistic bubble to the authoritative content). Same-worker delivery is instant via the per-worker `message_hub`; cross-worker delivery is filled by `services/messaging/relay.py` `message_relay` (per-worker ~1s DB poll on the `messages.id` watermark, deduped against the hub's delivered set). See `devplacepy/services/messaging/CLAUDE.md` |
+| `/messages` | messages.py - real-time DM chat. `GET ""` (page; the conversation sidebar is one windowed SQL aggregate and an opened thread loads only the newest `CONVERSATION_MESSAGE_LIMIT` = 500 messages - keep these queries bounded), `GET /search`, `GET /conversations` (flat JSON conversation list, same `get_conversations` helper as the page), `POST /send` (no-JS fallback, also accepts an attachment-only empty-content message), `POST /ws-ticket` (issues a 30s single-use WS auth ticket so a browser WebSocket can authenticate cross-context, since a native `WebSocket` cannot set an `Authorization`/`X-API-KEY` header), and `WS /messages/ws` (NOT lock-owner gated - accepts on every worker; closes `1008` for guests). The WS adds live bidirectional delivery, typing, and read receipts on top of the existing `messages` table - presence is NOT part of this WS (it is the generic `PresenceManager`/pub/sub mechanism, see `devplacepy/services/CLAUDE.md`). Both the WS `send` and HTTP `POST /send` route through the shared `services/messaging/persist.py` `persist_message` choke point (identical audit/notification/mention) and then through `_finalize_and_broadcast`, which broadcasts the persisted row immediately and then applies SYNC AI correction/modifier (HTTP awaits so the JSON body is final; WS schedules it so the receive loop never blocks). An in-place rewrite stamps `messages.updated_at` and emits a second `ai_processed` frame; other workers pick it up from `message_relay._tick_updates`. Same-worker delivery is instant via the per-worker `message_hub`; cross-worker delivery is filled by `services/messaging/relay.py` `message_relay` (per-worker ~1s DB poll on the `messages.id` watermark, plus `updated_at` for revisions, new rows deduped against the hub's delivered set). See `devplacepy/services/messaging/CLAUDE.md` |
 | `/notifications` | notifications.py |
 | `/votes` | votes.py |
 | `/reactions` | reactions.py - emoji reaction toggle on any target: `POST /reactions/{target_type}/{target_uid}` (optimistic `ReactionBar`) |
 | `/bookmarks` | bookmarks.py - bookmark/favorite toggle: `GET /bookmarks/saved` (the viewer's saved list) and `POST /bookmarks/{target_type}/{target_uid}` (toggle a bookmark) |
+| `/notes` | notes.py - private per-user annotation on a target: `GET /notes/saved` (the viewer's personal notes list), `POST /notes/{target_type}/{target_uid}` (add/replace the note), `POST /notes/{target_type}/{target_uid}/delete` (remove it) |
 | `/polls` | polls.py - poll voting: `POST /polls/{poll_uid}/vote` |
 | `/avatar` | avatar.py |
 | `/follow` | follow.py |
@@ -38,17 +40,21 @@ Prefixes are wired in `main.py`:
 | `/devii` | devii.py - WebSocket terminal (`/devii/ws`), page, `/devii/usage`, `/devii/session` |
 | `/zips` | zips.py - generic zip job status (`/zips/{uid}`) and archive download (`/zips/{uid}/download`); enqueued from `/projects/{slug}/zip` and `/projects/{slug}/files/zip` |
 | `/forks` | forks.py - fork job status (`/forks/{uid}`); enqueued from `/projects/{slug}/fork`. When done its `project_url` points at the new forked project |
-| `/tools` | tools/ package - public developer tools surface. `index.py` (`/tools` landing) plus `seo.py` (**SEO Diagnostics**): `GET /tools/seo` page, `POST /tools/seo/run` (enqueue `seo` job, per-owner one-active-job cap), `GET /tools/seo/{uid}` (`SeoJobOut`), `GET /tools/seo/{uid}/report` (HTML+JSON `SeoReportOut`), `WS /tools/seo/{uid}/ws` (live progress, lock-owner gated, close `4013` retry), `GET /tools/seo/{uid}/screenshot/{n}`. Also `deepsearch.py` (**DeepSearch**): `GET /tools/deepsearch` page, `POST /tools/deepsearch/run` (enqueue `deepsearch` job, per-owner one-active-job cap; resolves the user `api_key` into the payload, guests use the internal key), `GET /tools/deepsearch/{uid}` (`DeepsearchJobOut`), `GET /tools/deepsearch/{uid}/session` (HTML+JSON `DeepsearchSessionOut`), `WS /tools/deepsearch/{uid}/ws` (live progress, lock-owner gated, `4013` retry), `WS /tools/deepsearch/{uid}/chat` (grounded RAG chat over the session collection), `POST /tools/deepsearch/{uid}/{pause|resume|cancel}`, `GET /tools/deepsearch/{uid}/export.{md,json,pdf}`. The shared owner helper is `routers/tools/_shared.py` `owner_for`. Also `isslop.py` (**AI Usage Analyzer**): `GET /tools/isslop` page, `POST /tools/isslop/run` (enqueue `isslop` job, per-owner one-active-job cap; owner = user uid or the shared `DEVII_GUEST_COOKIE` guest identity, minted when absent), `GET /tools/isslop/list` (owner history; a signed-in request first claims any guest-cookie analyses via `store.claim_guest_analyses` - a move, never a copy), `GET /tools/isslop/{uid}` (`IsslopAnalysisOut`), `GET /tools/isslop/{uid}/events` (persisted ordered event trail, `?after=SEQ` incremental poll; live frames also publish on pub/sub `public.isslop.{uid}` - the DB trail is the source of truth, pub/sub the fast path), `GET /tools/isslop/{uid}/report` (HTML+JSON `IsslopReportOut`; live `<dp-isslop-run>` while running, server-rendered report via `render_content` when done), `GET /tools/isslop/{uid}/report.md`, `GET /tools/isslop/{uid}/badge.svg` (embeddable SVG authenticity badge). Analyses/reports/badges are permanent public capability URLs (`IsslopService.cleanup` never deletes them; only the job row is swept). Surfaced by a collapsible **Tools** header dropdown (`base.html`, visible to all) |
-| `/projects/{slug}/containers` | projects/containers/ subpackage - admin per-project container manager (`instances.py` for creation/lifecycle/exec/logs/metrics/sync plus the exec websocket, `schedules.py` for cron/interval/once schedules, shared helpers in `_shared.py`). Every instance runs the shared `ppy` image. Discoverable from the project detail page (admin-only **Containers** button) and from the admin index |
+| `/tools` | tools/ package - public developer tools surface. `index.py` (`/tools` landing) plus `seo.py` (**SEO Diagnostics**): `GET /tools/seo` page, `POST /tools/seo/run` (enqueue `seo` job, per-owner one-active-job cap), `GET /tools/seo/{uid}` (`SeoJobOut`), `GET /tools/seo/{uid}/report` (HTML+JSON `SeoReportOut`), `WS /tools/seo/{uid}/ws` (live progress, lock-owner gated, close `4013` retry), `GET /tools/seo/{uid}/screenshot/{n}`. Also `deepsearch.py` (**DeepSearch**): `GET /tools/deepsearch` page, `POST /tools/deepsearch/run` (enqueue `deepsearch` job, per-owner one-active-job cap; resolves the user `api_key` into the payload, guests use the internal key), `GET /tools/deepsearch/history` (HTML+JSON `DeepsearchHistoryOut`, owner's own past sessions newest-first with a `reopen_url` back into the session/chat routes - `database.list_deepsearch_sessions`, the `deepsearch_sessions` table already persists past runs independently of job retention), `GET /tools/deepsearch/{uid}` (`DeepsearchJobOut`), `GET /tools/deepsearch/{uid}/session` (HTML+JSON `DeepsearchSessionOut`), `WS /tools/deepsearch/{uid}/ws` (live progress, lock-owner gated, `4013` retry), `WS /tools/deepsearch/{uid}/chat` (grounded RAG chat over the session collection, reconnectable on any completed session within its job's retention window), `POST /tools/deepsearch/{uid}/{pause|resume|cancel}`, `GET /tools/deepsearch/{uid}/export.{md,json,pdf}`. The shared owner helper is `routers/tools/_shared.py` `owner_for`. Also `isslop.py` (**AI Usage Analyzer**): `GET /tools/isslop` page, `POST /tools/isslop/run` (enqueue `isslop` job, per-owner one-active-job cap; owner = user uid or the shared `DEVII_GUEST_COOKIE` guest identity, minted when absent), `GET /tools/isslop/list` (owner history; a signed-in request first claims any guest-cookie analyses via `store.claim_guest_analyses` - a move, never a copy), `GET /tools/isslop/{uid}` (`IsslopAnalysisOut`), `GET /tools/isslop/{uid}/events` (persisted ordered event trail, `?after=SEQ` incremental poll; live frames also publish on pub/sub `public.isslop.{uid}` - the DB trail is the source of truth, pub/sub the fast path), `GET /tools/isslop/{uid}/report` (HTML+JSON `IsslopReportOut`; live `<dp-isslop-run>` while running, server-rendered report via `render_content` when done), `GET /tools/isslop/{uid}/report.md`, `GET /tools/isslop/{uid}/badge.svg` (embeddable SVG authenticity badge). Analyses/reports/badges are permanent public capability URLs (`IsslopService.cleanup` never deletes them; only the job row is swept). Reachable by direct URL only; the former **Tools** header dropdown in `base.html` was removed |
+| `/projects/{slug}/containers` | projects/containers/ subpackage - admin per-project container manager (`instances.py` for creation/lifecycle/exec/logs/metrics/sync plus the exec websocket, `schedules.py` for cron/interval/once schedules, shared helpers in `_shared.py`). Every instance runs the shared `ppy` image. Reachable by direct URL and from the admin index; the former project detail page **Containers** button was removed |
 | `/admin/containers` | admin/containers.py - admin **Containers** manager: `/admin/containers` lists every instance across all projects with inline actions (start/stop/restart/terminal/edit/delete) plus a create modal (project search-select, run-as user search-select, boot language + source editor, restart policy, start-on-boot, env/ports/limits/ingress); `/admin/containers/{uid}` is the per-instance detail page (lifecycle, live logs/metrics, PTY terminal, schedules, ingress, sync, status history) and `/admin/containers/{uid}/edit` edits run-as user, boot language/script/command, restart policy, start-on-boot, and limits. `POST /admin/containers/create`, `/{uid}/edit`, `/{uid}/{start,stop,restart,pause,resume}`, `/{uid}/sync`, `/{uid}/delete` call `api.*` directly under `require_admin` (no docker/exec backend duplicated); `GET /admin/containers/projects/search` and `/users/search` back the create/edit search-selects. The lifecycle and detail views stay layered over the per-project `/projects/{slug}/containers/instances/{uid}/...` endpoints (the instance carries its `project_uid`) |
 | `/p/{slug}` | proxy.py - public ingress reverse proxy (HTTP + WebSocket) to a running container instance's published host port, opt-in per instance via `ingress_slug` |
 | `/xmlrpc` | xmlrpc.py - reverse-proxies XML-RPC calls to the forking XML-RPC bridge (`services/xmlrpc/`, supervised by `XmlrpcService` on loopback `config.XMLRPC_PORT`), which generates one XML-RPC method per documented REST endpoint from `docs_api.API_GROUPS` (`posts.create`, `feed.list`, ...). One struct of named params per call; auth via in-band `api_key`, `X-API-KEY`/`Bearer` header, or `http://user:pass@host/xmlrpc` Basic. Full introspection + `system.multicall`; REST errors become XML-RPC faults. Exempt from the rate limiter (enforced on the forwarded internal hop). See `devplacepy/services/xmlrpc/CLAUDE.md` |
 | `/api` | devrant/ package - devRant-compatible REST protocol (`auth.py`, `rants.py`, `comments.py`, `notifs.py`). Translates devRant requests onto DevPlace data: rants<->posts, comments/votes onto the native engagement layer, devRant integer ids onto each table's auto-increment `id`. Token auth via `devrant_tokens`. See `routers/devrant/CLAUDE.md` for the deep detail on this tree |
 | `/dbapi` | dbapi/ package - **primary-administrator-only, strictly READ-ONLY** generic database API over `dataset` (never inserts/updates/deletes/restores; the write surface was removed because it bypassed every per-route admin safeguard). `tables.py` (`GET /tables`, `GET /{table}/schema`), `crud.py` (read only: `GET /{table}` + `GET /{table}/{key}/{value}`, `?include_deleted`), `query.py` (`POST /query` hard SELECT-only validated read-only run; `POST /query/async` + `GET /query/{uid}` + `WS /query/{uid}/ws` via `DbApiJobService` kind `dbquery`), `nl.py` (`POST /nl` natural-language->SQL via the AI gateway, returns validated SELECT, `execute=true` runs it read-only). **Auth = the PRIMARY administrator only** (the earliest-created Admin, `utils.is_primary_admin` / `database.get_primary_admin_uid` - the same identity that gates backup downloads) by session/api_key (`services/dbapi/policy.py`); every other administrator is refused like a member, and there is **no internal-key path** - the gateway `internal_gateway_key()` is NOT accepted (no service-to-service access). SQL validated by `services/dbapi/validate.py` (sqlglot classify + suspicious-flag + read-only `EXPLAIN` dry-run). Devii tools `db_*` are read-only (list/get/query/nl; no write tools) and flagged `requires_primary_admin=True`, so they are added to the LLM tool list ONLY for the primary administrator (every other session never sees them and Devii is unaware they exist). `services/pubsub/policy.py` resolves its own admin/internal actor and does NOT reuse `dbapi.policy.caller_for`, so the primary-admin restriction does not leak onto the pub/sub bus. See `devplacepy/services/dbapi/CLAUDE.md` |
 | `/quizzes` | quizzes/ package - **Quizzes**: `index.py` (hub, create, import, detail, export, edit, delete, publish, per-quiz leaderboard, cross-quiz scoreboard), `questions.py` (builder page + question CRUD + reorder), `attempts.py` (start/resume, play, answer, finish, results). The hub `GET /quizzes` is the approved three-column `.feed-layout` reused verbatim from `/feed`: filters + search left (`?filter=all|todo|done|mine|drafts`, `?search=`, `?page=`), the quiz list centre with the **New quiz** and **Create quiz with Devii** actions, and the cross-quiz **scoreboard** rail right (score per user, best attempt per quiz, 15s display cache). Publishing is terminal - every write on a published quiz is a 400 and there is no unpublish route. See `devplacepy/services/quiz/CLAUDE.md` |
-| `/game` | game/ package - the **Code Farm** idle game (`index.py` + `farm.py`): `GET /game` (page), `GET /game/state`, `GET /game/leaderboard?board=`, `POST /game/{plant,harvest,buy-plot,upgrade,fertilize,daily,grant,perk,prestige,legacy,mastery,quests/claim}`, `POST /game/{defense/upgrade,defense/downgrade,infrastructure/buy,cosmetics/buy,cosmetics/equip}`, plus social `GET /game/farm/{username}` and `POST /game/farm/{username}/{water,steal}`. See `devplacepy/services/game/CLAUDE.md` |
-| (none) | push.py - push + PWA: `GET /push.json` (VAPID public key + the providers accepting registrations), `POST /push.json` (register with any active provider; a body without `provider` is a `webpush` body, so existing clients are unchanged; an unknown, disabled or unconfigured provider is a 400), `GET /service-worker.js`, `GET /manifest.json`. Provider protocol and delivery live in `devplacepy/push/` - see `devplacepy/push/CLAUDE.md` |
+| `/battles` | battles.py - **Opinion Wars**, week-long two-faction battles attached to posts: `GET ""` (listing, quizzes-style filters active/ended/mine + search), `GET /{uid}` (JSON-only full state, runs lazy resolution), `GET /{uid}/events?after=` (durable event trail replay), `POST /{uid}/join` (join or switch faction), `POST /{uid}/fight` (spend 25 Code Farm coins, deal level-weighted damage, 24h cooldown). The battle card renders on the parent post (no battle detail HTML page). See `devplacepy/services/opinionwar/CLAUDE.md` |
+| `/game` | game/ package - the **Code Farm** idle game (`index.py` + `farm.py`): `GET /game` (page), `GET /game/state`, `GET /game/leaderboard?board=`, `POST /game/{plant,harvest,plot/stop,buy-plot,upgrade,fertilize,daily,grant,perk,prestige,legacy,mastery,quests/claim}`, `POST /game/{defense/upgrade,defense/downgrade,infrastructure/buy,cosmetics/buy,cosmetics/equip}`, plus social `GET /game/farm/{username}` and `POST /game/farm/{username}/{water,steal}`. See `devplacepy/services/game/CLAUDE.md` |
+| (none) | push.py - push + PWA: `GET /push.json` (VAPID public key + the providers accepting registrations; `apns` includes `environment` when active), `POST /push.json` (register with any active provider; a body without `provider` is a `webpush` body; an APNs body is `token` plus optional `client_id` for device-stable upsert; created/revived rows are probed and the JSON may include `delivered`/`error`; an unknown, disabled or unconfigured provider is a 400), `DELETE /push.json` (unregister exactly the one registration named by `endpoint`/`token`/`client_id`, same identity priority as registration; idempotent, always 200 `{unregistered}`; wired into `PushManager.js`'s logout-link interceptor so a webpush subscription is dropped before the browser navigates to `/auth/logout`), `GET /service-worker.js`, `GET /manifest.json`. Provider protocol and delivery live in `devplacepy/push/` - see `devplacepy/push/CLAUDE.md` |
 | (none) | docs.py (docs/ package) - the documentation site (prose pages + API reference). See `routers/docs/CLAUDE.md` for the deep detail on this tree (`DOCS_PAGES`, audience tiers, prose rendering pipeline) |
+| `/skills` | skills.py - per-user Agent Skills: `GET ""` (list), `GET /{name}` (detail), `POST /install` (fetch a `SKILL.md` from a URL, update in place), `POST /{name}/enabled`, `POST /{name}/delete` (owner, or an administrator naming `owner`). Private to the owner, `noindex`. See `devplacepy/services/skills/CLAUDE.md` |
+| `/nodes` | nodes/ package - community node network. `spec.py` (`GET /nodes/spec.json`, public, `SPEC.describe()`), `devices.py` (`GET/POST /nodes/devices`, `POST /nodes/devices/{uid}/rotate|delete`, `require_user_api`, secret returned once, `no-store`), `ticket.py` (`POST /nodes/ticket`, authenticated by `Authorization: Bearer dpn_...`, never a session), `ws.py` (`WS /nodes/ws?ticket=`, lock-owner gated, `4013` retry). Needs its own nginx upgrade block. The administrator side is `admin/nodes.py` (`GET /admin/nodes`, `POST /admin/nodes/{killswitch,settings,devices/{uid}/suspend,devices/{uid}/unsuspend,flags/{uid}/resolve}`). See `devplacepy/services/nodes/CLAUDE.md` |
+| `/sync` | sync/ package - community sync. `spec.py` (`GET /sync/spec.json`, public), `users.py`, `posts.py`, `comments.py` (`POST /sync/{kind}`, administrator API key, batch of up to 50 records, per-record results, `dry_run`). Shared auth, kill switch (`sync_enabled`), origin allowlist (`sync_origins`) and audit in `_shared.py`. See `devplacepy/services/sync/CLAUDE.md` |
 | `/pubsub` | pubsub.py - **database-free** publish/subscribe bus. `WS /pubsub/ws` (subscribe/unsubscribe/publish frames, `foo.*` wildcards), `POST /pubsub/publish` + `GET /pubsub/topics` (admin/internal). Lock-owner-gated WS (`4013` retry) so subscribers converge on one worker; topic authz in `services/pubsub/policy.py` (`user.{uid}.*` private, `public.*` shared, admin/internal anywhere, guests opt-in). In-process `services.pubsub.publish(topic, data)` for backends; frontend `app.pubsub`. See `devplacepy/services/pubsub/CLAUDE.md` |
 | (none) | seo.py - `/robots.txt`, `/sitemap.xml` |
 
@@ -68,7 +74,7 @@ Response models live in `devplacepy/schemas.py` (Pydantic v2, `extra="ignore"`,
 
 ## FastAPI patterns
 
-- **All routes are async.** Form data is validated via a typed Pydantic body param: `data: Annotated[SomeForm, Form()]` (models in `models.py`). Read raw `await request.form()` only when also handling an uploaded file (a separate `File()` param would embed the model under its parameter name).
+- **All routes are async.** Form data is validated via a typed Pydantic body param: `data: Annotated[SomeForm, Form()]` (models in `models/`, one module per domain). Read raw `await request.form()` only when also handling an uploaded file (a separate `File()` param would embed the model under its parameter name).
 - **Return `RedirectResponse(url=..., status_code=302)`** for redirects - always pass `status_code` as an integer literal, never `status.HTTP_302_FOUND`.
 - **Return `templates.TemplateResponse("name.html", {...})`** from `devplacepy.templating` to render.
 - **Never create your own `Jinja2Templates` instance.** Import the shared one: `from devplacepy.templating import templates`.
@@ -90,6 +96,10 @@ The `comments` table uses `(target_type, target_uid)` so the same `_comment_sect
 
 Every post card on the feed has an inline comment form (`.feed-comment-form`) beneath the post actions. It posts to `/comments/create` like the detail page comment form. Tests must scope Post button clicks to `#create-post-modal button.btn-primary:has-text('Post')` to avoid matching the inline comment's Post button.
 
+### Comment permalinks
+
+Every comment's action bar (`_comment.html`) includes a "Copy link" button built on the existing `id="comment-{uid}"` anchor (already used by notifications, the profile Activity tab, and `NotificationManager.js`'s scroll-to-highlight) - no new route or URL scheme. It is a plain `data-share="#comment-{{ item.comment['uid'] }}"` button: `DomUtils.initShareButtons` resolves the relative hash against `window.location.href` at click time (so it always copies the exact page the viewer is on, canonical or not), copies it via `navigator.clipboard.writeText`, and flashes "Copied!" on the button (`Toast.flash`) - the same mechanism the gist detail page's Share button already uses.
+
 ### Comment editing
 
 A comment's owner (only the owner, never an admin) sees an inline "Edit" button (`data-action='edit'`) in `_comment.html`. `CommentManager.toggleEditForm` swaps the `.comment-text` for a textarea seeded from its `data-raw` attribute (the raw markdown, since `contentRenderer.applyTo` overwrites `textContent` on first render), posts via `Http.send` to `POST /comments/edit/{comment_uid}`, then re-renders the new body in place with `contentRenderer.applyTo`. The route (`content.edit_comment_record`) is `is_owner`-only, writes `content` + `updated_at`, records the `comment.edit` audit event, and branches on `wants_json`: JSON clients get `CommentEditOut{uid, content, url, updated_at}`, the no-JS form falls back to a redirect to the comment anchor. Edits are NOT soft-delete related (the body is overwritten in place). Devii tool: `edit_comment` (owner-only, no confirm). Scope test Edit clicks to `.comment-action-btn:has-text('Edit')`.
@@ -145,8 +155,9 @@ A dedicated `/gists` page for sharing code snippets. Uses `gists` table (auto-cr
 
 - Source code rendered in `<pre><code class="language-xxx">` block on detail page
 - Syntax highlighting handled by existing `highlight.js` loaded globally in `base.html`
-- Copy button uses `navigator.clipboard.writeText()`
+- Copy button uses `navigator.clipboard.writeText()` via the shared `data-copy` handler (`DomUtils.initClipboardCopy`)
 - Cards in listing show language badge, title, truncated description, author, star count
+- **Raw/rendered toggle for `language == "markdown"`:** unlike `"markdown_rendered"` (always rendered) and every other language (always raw), the plain `"markdown"` gist dual-renders server-side in `gist_detail.html` - the existing raw `<pre>` block plus a `render_content(gist['source_code'], ...)` block, the second one starting `hidden`. A `View rendered`/`View raw` button next to Copy uses the generic `data-view-toggle`/`data-view-toggle-alt` (+ the two `-label`/`-label-alt` pairs) attribute pair (`DomUtils.initViewToggles`, see `static/js/CLAUDE.md`) to swap the `hidden` class between the two blocks and its own label - no fetch, no new endpoint.
 
 ### Sitemap
 
@@ -156,7 +167,7 @@ A dedicated `/gists` page for sharing code snippets. Uses `gists` table (auto-cr
 
 ### Politics category
 
-The `politics` topic is available as a feed filter sidebar item (icon `&#x1F3DB;`) and post topic. It defines the CSS variable `--topic-politics: #00bcd4` in `variables.css` and the `.badge-politics` class in `base.css`. It is validated like every topic via the canonical `TOPICS` list in `constants.py` (consumed by `models.py` `valid_topic`, `templating.py` `TOPICS` global, `routers/posts.py`, `docs_api`). Unlike the former `signals` topic, `politics` is also part of the bot fleet's rotation - it is in `services/bot/config.py` `CATEGORIES`/`FEED_TOPICS`, carries a modest per-persona weight in `PERSONA_CATEGORY_WEIGHTS`, and has a writing instruction in `services/bot/llm.py` `category_extras`.
+The `politics` topic is available as a feed filter sidebar item (icon `&#x1F3DB;`) and post topic. It defines the CSS variable `--topic-politics: #00bcd4` in `variables.css` and the `.badge-politics` class in `base.css`. It is validated like every topic via the canonical `TOPICS` list in `constants.py` (consumed by `models/` `valid_topic`, `templating.py` `TOPICS` global, `routers/posts.py`, `docs_api`). Unlike the former `signals` topic, `politics` is also part of the bot fleet's rotation - it is in `services/bot/config.py` `CATEGORIES`/`FEED_TOPICS`, carries a modest per-persona weight in `PERSONA_CATEGORY_WEIGHTS`, and has a writing instruction in `services/bot/llm.py` `category_extras`.
 
 ### Date format
 
@@ -189,7 +200,7 @@ News articles have an internal detail page at `/news/{slug}` with full comment s
 
 ### Home page (`GET /`)
 
-The home route (`main.py` `landing()`) never redirects - it renders `templates/landing.html` for everyone, branching on `user`:
+The home route (`routers/landing.py` `landing()`) never redirects - it renders `templates/landing.html` for everyone, branching on `user`:
 
 - **Guests** get the marketing hero (`Join DevPlace Free` CTA + features grid).
 - **Signed-in users** get a personalized dashboard hero (`.dashboard-welcome`): avatar, "Welcome back, {username}", quicklink buttons (`.dashboard-btn`, with `New Post` -> `/feed` as `.dashboard-btn-primary`, plus Code Farm/Projects/Gists), and a Posts/Stars/Level stat strip (`.dashboard-stats`, `user_post_count` + the user dict's `stars`/`level`). Styles live in the `.dashboard-*` classes in `static/css/landing.css`.
@@ -228,7 +239,7 @@ Every public listing card that shows the last few comments must render them with
 Articles can be toggled to appear on the landing page via `/admin/news/{uid}/landing`:
 
 - **`show_on_landing`** field on `news` table
-- Landing route (`main.py` `GET /`) fetches up to 6 articles with `show_on_landing=1`
+- Landing route (`routers/landing.py` `GET /`) fetches up to 6 articles with `show_on_landing=1`
 - Rendered as a 3-column card grid with image, source, title, date (responsive -> 1 column on mobile)
 - Toggleable individually from the admin news table
 
@@ -249,6 +260,14 @@ All SEO features are implemented across the following locations:
 - `devplacepy/seo.py` - JSON-LD schema generators (WebSite, BreadcrumbList, DiscussionForumPosting, ProfilePage, SoftwareApplication), meta description truncation, schema combiner, sitemap XML generator
 - `routers/seo.py` - robots.txt and sitemap.xml routes
 
+### DiscussionForumPosting nested comments
+
+`discussion_forum_posting(post, author, comment_count, star_count, base_url, comments=None)` embeds up to `MAX_SCHEMA_COMMENTS` (20) of the post's comments as nested `comment: [{"@type": "Comment", "text", "author", "datePublished"}, ...]` entities - not just the aggregate `CommentAction` `InteractionCounter`, which stays for the total count. `seo.comment_schema_list(comment_tree, base_url)` flattens the already-loaded comment tree (`content.load_detail`'s `detail["comments"]`, the same nested `{comment, author, children}` shape `_comment.html` renders) depth-first up to the cap - it does not re-query the database. `posts.py::view_post` is the only call site; a future post-like discussion surface (project/gist/news comments) can reuse `comment_schema_list` the same way once/if it gets a `DiscussionForumPosting` schema of its own.
+
+### Topic category pages (`/topics`)
+
+`routers/topics.py` gives the feed's `TOPICS` filter (`constants.py`) real, independently-crawlable pages instead of only a `?topic=` query param (whose canonical collapses back to bare `/feed` - see `base_seo_context`, `canonical = f"{base}{request.url.path}"`, which drops the query string on purpose). `GET /topics` is a hub linking every topic (with a live post count); `GET /topics/{topic}` is a full post listing for that topic, built from the exact same `get_feed_posts`/`enrich_post_cards` pair `feed.py` uses (`enrich_post_cards` was extracted out of `feed_page` specifically so this page is not a second, drifting copy of the attachments/reactions/bookmarks/poll/war enrichment loop). Each topic page gets its own canonical URL, unique title/description, breadcrumbs (Home > Topics > {label}), a `rel=next` link when paginated (`list_page_seo`/`next_page_url`, same mechanism as `/feed`/`/news`), and a real crawlable `_load_more.html` link (not JS-only infinite scroll) for reaching older posts. Both `/topics` and every `/topics/{topic}` are in `sitemap.xml`.
+
 ### SEO template context
 - Every router passes `page_title`, `meta_description`, `meta_robots`, `canonical_url`, `og_title`, `og_description`, `og_image`, `og_type`, `breadcrumbs`, `page_schema` via `base_seo_context()`
 - Auth pages: `noindex,nofollow`
@@ -265,15 +284,22 @@ All SEO features are implemented across the following locations:
 - `profile.html` - username rendered as `<h1 class="profile-name">`
 - `messages.html` - `<h1 class="sr-only">Messages</h1>`
 - `projects.html` - `<h1>Projects</h1>`
-- `post.html` - post title as `<h1>`, "Related Discussions" as `<h3>`
+- `post.html` - post title as `<h1>`; "Gists from {author}", "Projects from {author}", and "Related Discussions" are `.sidebar-heading` labels in the left column (see "Post page layout" below), not `<h3>`
 
 ### Post slugs
 - Slug generated on post creation via `slugify()` and stored in `posts.slug` column
 - Posts can be looked up by slug or UUID
 - Minimum content validation: post body >= 10 chars, comment >= 3 chars
 
-### Related posts
-- `templates/post.html` - "Related Discussions" widget at bottom of post page (queried by matching topic)
+### Post page layout (three columns, mirrors `/feed`)
+
+`post.html` reuses `/feed`'s exact layout building blocks rather than inventing new ones, wrapped in `.post-page-layout` (`static/css/post.css`, `grid-template-columns: var(--sidebar-width) minmax(0, 1fr) 280px`, collapsing to one column at 1024px):
+
+- **Left column** - `<aside class="post-page-sidebar">`, sticky, holding up to three separate `.sidebar-card` blocks (never merged into one card - each is its own bordered panel): "Gists from {author}" and "Projects from {author}" (`content.get_user_sidebar_gists`/`get_user_sidebar_projects`, up to 5 each, most-recently-modified first, private projects filtered via `can_view_project`), and "Related Discussions" (same-topic posts). Every card title is a `.sidebar-heading` div (the same class `feed.html`'s left sidebar uses for "Topics"/"Resources"/"Online now" - `.sidebar-card .sidebar-heading` in `sidebar.css`, so it only styles correctly nested inside a `.sidebar-card`, never bare). Each list item reuses the plain `.related-list`/`.related-link`/`.related-title`/`.related-meta` classes (`base.css`) with `content_preview()` for the ellipsis-truncated description.
+- **Middle column** - `.post-page` (`max-width: 720px`): the post article, comments.
+- **Right column** - `<aside class="feed-right">`, the exact same class `feed.html` uses for its Daily Topic widget (sticky, hidden below 1024px via `feed.css`'s own media query - no post-page-specific override needed). Holds the "Featured" cards (`database.get_featured_topics`, up to 3, cached pool + per-request random sample): each is a `.daily-topic-card` with its own `.daily-topic-label` ("Featured") - matching the single Daily Topic card's internal label, since there is no section-level heading here (a `.sidebar-heading` div placed directly in `.feed-right` would NOT be styled, as noted above - the fix used is a per-card label instead of a bare heading).
+
+All three columns are populated by `routers/posts.py` `post_page_context()`, the single context builder shared by the real `/posts/{slug}` route and `happy404.render()` (see the root `CLAUDE.md`), so a decoy happy-404 post page renders with the identical sidebar/featured layout as a real one.
 
 ### Performance
 - `loading="lazy"` on all avatar images
@@ -300,9 +326,14 @@ All SEO features are implemented across the following locations:
 - `POST /bookmarks/{target_type}/{target_uid}` toggles a `bookmarks` row; `GET /bookmarks/saved` renders the personal list (`saved.html`). Target types: `post`, `gist`, `project`, `news`.
 - `_bookmark_button.html` takes `_type`, `_uid`, `_bookmarked`; `BookmarkManager.js` swaps the label/`bookmarked` class from the JSON `{saved}`. Batch state via `get_user_bookmarks(user_uid, target_type, uids)`.
 
+### Personal notes
+- A `notes` row is a private, per-user text annotation on a target (`post`, `gist`, `project`, `news` - the four detail pages, unlike bookmarks' listing-card coverage, since a note is read/written on the full content view, not skimmed from a card). `POST /notes/{target_type}/{target_uid}` (`NoteForm{content}`, max 4000 chars) creates or replaces the caller's own note on that target (revives a soft-deleted row rather than duplicating it, exactly like bookmarks); `POST /notes/{target_type}/{target_uid}/delete` soft-deletes it; `GET /notes/saved` renders the personal notes list (`notes.html`), mirroring `saved.html` but including each note's body.
+- There is no read/edit access for anyone but the author - the route only ever looks up `user_uid=user["uid"]`, so there is no "someone else's note" to view or moderate. `database/moderation.py` lists `notes` in `UNREPORTABLE_TABLES` ("private to the owner") for that reason.
+- `_note_button.html` takes `_type`, `_uid`, `_note` (the current content or `None`) and renders a button that opens a small inline textarea editor (not a modal - the body is short and the surrounding action bar has no room for a full dialog); `NoteManager.js` (`app.notes`) wires open/cancel/save/delete and swaps the button label/`has-note` class from the JSON `{content}` / `{deleted}` response, extending the shared `OptimisticAction` base like the other engagement controllers. Batch/single state via `get_user_notes(user_uid, target_type, uids)` (`database/engagement.py`), wired into `content.load_detail`/`detail_context` (post/gist/project) and `routers/news.py`'s detail route as `note_content` on the page context and the matching `*DetailOut` schema.
+
 ### Polls
 - A poll rides on a post (one `polls` row keyed by `post_uid`, options in `poll_options`, one-per-user votes in `poll_votes`). Created in `posts.py:create_poll` when `poll_question` plus >= 2 non-empty `poll_options` are submitted (capped at 6). Both `create_post` and `edit_post` accept the poll fields; `edit_post` only attaches a poll when the post has **none** yet (it never replaces an existing poll). The builders live in the create-post modal (`feed.html`) and the edit-post modal (`post.html`, rendered only when the post has no poll) using `data-poll-toggle` / `data-poll-add-option`.
-- `poll_options` accepts either repeated form fields (the web builders, which preserve commas inside an option label) **or** a single newline- or comma-separated string (the API/Devii path). `models.py:normalize_poll_options` splits a lone delimited element - applied as a `mode="before"` validator on `PostForm`/`PostEditForm` - so the documented "one per line or comma separated" agent format actually produces a multi-option poll instead of a single dropped option.
+- `poll_options` accepts either repeated form fields (the web builders, which preserve commas inside an option label) **or** a single newline- or comma-separated string (the API/Devii path). `models/shared.py:normalize_poll_options` splits a lone delimited element - applied as a `mode="before"` validator on `PostForm`/`PostEditForm` - so the documented "one per line or comma separated" agent format actually produces a multi-option poll instead of a single dropped option.
 - `POST /polls/{poll_uid}/vote` toggles the voter's choice and returns the full poll dict: voting on a new option switches, voting the **same** option again **retracts** (one vote per user, like reactions). Results (bars + `%`) are **always shown** to everyone; the chosen option gets `.chosen` + a checkmark. Guests see read-only bars with a "Log in to vote" hint (options `disabled`). Batch reads via `get_polls_by_post_uids(post_uids, user)` / `get_poll_for_post(post_uid, user)`.
 - Composer poll builder (`feed.html`, `PollManager.js`): poll inputs are `disabled` until "Add poll" is opened (so a never-opened or "Remove poll"-collapsed builder submits **nothing**); a capture-phase `submit` listener blocks an incomplete poll (question + >= 2 options) with an inline error instead of silently dropping it - capture phase + `stopImmediatePropagation()` is required so it pre-empts `FormManager`'s bubble-phase submit handlers.
 

devplacepy/routers/admin/__init__.py

@@ -7,6 +7,7 @@ from devplacepy.routers.admin import (
     auditlog,
     backups,
     bots,
+    communities,
     containers,
     workspaces,
     devii_tasks,
@@ -16,6 +17,7 @@ from devplacepy.routers.admin import (
     media,
     moderation,
     news,
+    nodes,
     notifications,
     services,
     settings,
@@ -40,6 +42,8 @@ router.include_router(issues.router)
 router.include_router(auditlog.router)
 router.include_router(backups.router)
 router.include_router(bots.router)
+router.include_router(communities.router)
+router.include_router(nodes.router)
 router.include_router(gateway_configs.router)
 router.include_router(devii_tasks.router)
 router.include_router(game.router)

devplacepy/routers/admin/backups.py

@@ -1,5 +1,6 @@
 # retoor <retoor@molodetz.nl>
 
+import asyncio
 import logging
 from pathlib import Path
 from typing import Annotated
@@ -61,11 +62,12 @@ def _targets() -> list[dict]:
         for key, meta in store.BACKUP_TARGETS.items()
     ]
 
-def _dashboard(can_download: bool) -> dict:
+async def _dashboard(can_download: bool) -> dict:
     backups = [_backup_payload(row, can_download) for row in store.list_backups()]
     schedules = store.list_schedules()
+    storage = await asyncio.to_thread(store.compute_storage_stats)
     return {
-        "storage": store.compute_storage_stats(),
+        "storage": storage,
         "backups": backups,
         "schedules": schedules,
         "targets": _targets(),
@@ -77,7 +79,7 @@ def _dashboard(can_download: bool) -> dict:
 @router.get("/backups", response_class=HTMLResponse)
 async def admin_backups(request: Request):
     admin = require_admin(request)
-    data = _dashboard(is_primary_admin(admin))
+    data = await _dashboard(is_primary_admin(admin))
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
@@ -89,6 +91,7 @@ async def admin_backups(request: Request):
             {"name": "Backups", "url": "/admin/backups"},
         ],
         schemas=[website_schema(base)],
+        robots="noindex,nofollow",
     )
     return respond(
         request,
@@ -106,7 +109,7 @@ async def admin_backups(request: Request):
 @router.get("/backups/data")
 async def admin_backups_data(request: Request):
     admin = require_admin(request)
-    data = _dashboard(is_primary_admin(admin))
+    data = await _dashboard(is_primary_admin(admin))
     return JSONResponse(BackupDashboardOut.model_validate(data).model_dump(mode="json"))
 
 @router.post("/backups/run")

devplacepy/routers/admin/communities.py

@@ -0,0 +1,494 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+import json
+import logging
+from datetime import datetime
+from typing import Annotated
+from urllib.parse import urlencode
+
+from fastapi import APIRouter, Form, Request
+from fastapi.responses import HTMLResponse
+
+from devplacepy.database import get_setting, get_table, get_users_by_uids, resolve_object_url
+from devplacepy.database.sync import count_links, identity_map
+from devplacepy.responses import action_result, json_error, respond
+from devplacepy.schemas import AdminCommunitiesOut, AdminCommunityLedgerOut, AdminCommunityOut, AdminCommunitySchemaOut
+from devplacepy.models import CommunityAddForm, CommunityRunForm, CommunitySchemaForm, CommunityUpdateForm
+from devplacepy.seo import base_seo_context, site_url, website_schema
+from devplacepy.services.audit import record as audit
+from devplacepy.services.community.discoverer import slug_for_query
+from devplacepy.services.community.files import files
+from devplacepy.services.community.ledger import (
+    LEDGER_KINDS,
+    LEVEL_TITLES,
+    OUTCOME_BY_KEY,
+    OUTCOMES,
+    Ledger,
+    explain,
+    tally_of,
+    verdict as run_verdict,
+)
+from devplacepy.services.community.ledger_store import ledger_store
+from devplacepy.services.community.runner import TRIGGER_ADMIN, CommunityRunner
+from devplacepy.services.community.service import DRY_RUN_KEY
+from devplacepy.services.community.schemacheck import validate_schema
+from devplacepy.services.community.spec import STATUS_LEARNING, STATUS_READY
+from devplacepy.services.community.store import iso, now, store
+from devplacepy.services.sync.gate import sync_enabled
+from devplacepy.services.sync.options import SyncOptions
+from devplacepy.utils import not_found, require_admin
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+background_jobs: set[asyncio.Task] = set()
+LIST_URL = "/admin/communities"
+
+
+async def spawn(coroutine) -> None:
+    task = asyncio.create_task(coroutine)
+    background_jobs.add(task)
+    task.add_done_callback(background_jobs.discard)
+    await asyncio.sleep(0)
+
+
+def service_flags() -> dict:
+    return {
+        "service_enabled": (get_setting("service_community_enabled", "0") or "0") == "1",
+        "service_dry_run": (get_setting(DRY_RUN_KEY, "1") or "1") == "1",
+    }
+
+
+def public(row: dict) -> dict:
+    view = {key: value for key, value in row.items() if key not in ("login_password", "login_totp_secret")}
+    view["has_login"] = bool(row.get("login_password"))
+    view["has_totp"] = bool(row.get("login_totp_secret"))
+    return view
+
+
+def require_community(slug: str) -> dict:
+    row = store.get(slug)
+    if row is None:
+        raise not_found("Unknown community")
+    return row
+
+
+def credentials_of(data) -> dict:
+    return {"username": data.login_username.strip(), "email": data.login_email.strip(), "password": data.login_password, "totp": data.login_totp_secret.strip()}
+
+
+def seo(request: Request, title: str, path: str, crumb: str):
+    base = site_url(request)
+    return base_seo_context(
+        request,
+        title=f"{title} - Admin",
+        description="External communities synchronised into this instance.",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Admin", "url": "/admin"},
+            {"name": crumb, "url": path},
+        ],
+        schemas=[website_schema(base)],
+        robots="noindex,nofollow",
+    )
+
+
+FILTER_KEYS = ("kind", "outcome", "reason", "q")
+
+
+def seconds_between(start: str | None, end: str | None) -> int | None:
+    try:
+        return max(0, round((datetime.fromisoformat(end) - datetime.fromisoformat(start)).total_seconds()))
+    except (TypeError, ValueError):
+        return None
+
+
+def run_view(run: dict) -> dict:
+    view = dict(run)
+    view["dry_run"] = bool(run.get("dry_run"))
+    for column in ("items_stored", "items_dropped"):
+        view[column] = int(run.get(column) or 0)
+    for column in ("verdict", "headline"):
+        view[column] = run.get(column) or ""
+    if not view.get("tally") and view.get("pushed"):
+        view["tally"] = tally_of(view["pushed"], Ledger())
+    view["duration_seconds"] = seconds_between(run.get("started_at"), run.get("finished_at"))
+    if not view.get("verdict"):
+        outcome = run_verdict(run.get("status", ""), view["dry_run"], view.get("tally") or {})
+        view["verdict"], view["headline"] = outcome["level"], outcome["headline"]
+    return view
+
+
+def local_links(rows: list[dict]) -> dict[str, str]:
+    wanted: dict[str, list[str]] = {"posts": [], "users": [], "comments": []}
+    for row in rows:
+        if row.get("local_uid") and row.get("kind") in wanted:
+            wanted[row["kind"]].append(row["local_uid"])
+    links: dict[str, str] = {}
+    if wanted["posts"]:
+        table = get_table("posts")
+        for found in table.find(table.table.columns.uid.in_(wanted["posts"]), deleted_at=None):
+            links[found["uid"]] = f"/posts/{found.get('slug') or found['uid']}"
+    for uid, found in get_users_by_uids(wanted["users"]).items():
+        links[uid] = f"/profile/{found['username']}"
+    if wanted["comments"]:
+        table = get_table("comments")
+        for found in table.find(table.table.columns.uid.in_(wanted["comments"]), deleted_at=None):
+            target = resolve_object_url(found["target_type"], found["target_uid"])
+            links[found["uid"]] = f"{target}#comment-{found['uid']}" if target else ""
+    return links
+
+
+def item_view(row: dict, links: dict[str, str]) -> dict:
+    view = dict(row)
+    view["tone"] = OUTCOME_BY_KEY[row["outcome"]].tone if row.get("outcome") in OUTCOME_BY_KEY else "muted"
+    why = explain(row.get("reason", ""))
+    view.update(reason_title=why["title"], reason_text=why["text"], retried=why["retried"])
+    view["local_url"] = links.get(row.get("local_uid") or "", "")
+    return view
+
+
+def ledger_options(scope: dict) -> dict:
+    facets = ledger_store.facets(**scope)
+    return {
+        "kind": [kind for kind in LEDGER_KINDS if kind in facets["kind"]],
+        "outcome": [{"value": outcome.key, "label": outcome.label} for outcome in OUTCOMES],
+        "reason": [{"value": reason, "title": explain(reason)["title"]} for reason in facets["reason"]],
+    }
+
+
+def feed_summary(feed: dict) -> dict:
+    return {
+        "id": feed["id"],
+        "type": feed["type"],
+        "url": feed["url"],
+        "list_path": feed.get("list_path"),
+        "item_selector": feed.get("item_selector"),
+        "post_fields": feed["post"]["fields"],
+        "author_fields": (feed.get("author") or {}).get("fields", {}),
+        "comments": (feed.get("comments") or {}).get("mode", "none"),
+        "comments_reason": (feed.get("comments") or {}).get("reason", ""),
+        "pagination": (feed.get("pagination") or {}).get("style", "none"),
+    }
+
+
+@router.get("/communities", response_class=HTMLResponse)
+async def admin_communities(request: Request):
+    admin = require_admin(request)
+    return respond(
+        request,
+        "admin_communities.html",
+        {
+            **seo(request, "Communities", LIST_URL, "Communities"),
+            "request": request,
+            "user": admin,
+            "items": [public(row) for row in store.list()],
+            "sync_enabled": sync_enabled(),
+            **service_flags(),
+            "admin_section": "communities",
+        },
+        model=AdminCommunitiesOut,
+    )
+
+
+@router.post("/communities")
+async def admin_community_add(request: Request, data: Annotated[CommunityAddForm, Form()]):
+    admin = require_admin(request)
+    slug = slug_for_query(data.query)
+    if not slug:
+        return json_error(400, "Enter a community name or address")
+    existing = store.get(slug)
+    if existing is None:
+        secrets = credentials_of(data)
+        existing = store.create(
+            slug,
+            data.query,
+            admin["uid"],
+            name=data.query,
+            status=STATUS_LEARNING,
+            claimed_at=iso(now()),
+            login_username=secrets["username"],
+            login_email=secrets["email"],
+            login_password=secrets["password"],
+            login_totp_secret=secrets["totp"],
+        )
+        await spawn(CommunityRunner().discover(data.query, admin["uid"], relearn_slug=slug))
+        audit.record(
+            request,
+            "admin.community.add",
+            user=admin,
+            target_type="community",
+            target_uid=existing["uid"],
+            target_label=slug,
+            metadata={"query": data.query},
+            summary=f"{admin['username']} added community {data.query!r}",
+        )
+    return action_result(request, f"{LIST_URL}/{existing['slug']}")
+
+
+@router.get("/communities/{slug}", response_class=HTMLResponse)
+async def admin_community(request: Request, slug: str):
+    admin = require_admin(request)
+    row = require_community(slug)
+    store.reap_interrupted()
+    source = files.read(row["slug"])
+    return respond(
+        request,
+        "admin_community.html",
+        {
+            **seo(request, row["name"] or slug, f"{LIST_URL}/{slug}", "Communities"),
+            "request": request,
+            "user": admin,
+            "community": public(row),
+            "rules": row.get("rules") or "",
+            "favicon": row.get("favicon") or "",
+            "totals": row.get("totals") or {},
+            "feeds": [feed_summary(feed) for feed in (row.get("schema") or {}).get("feeds", [])],
+            "runs": [run_view(run) for run in store.run_summaries(row["uid"])],
+            "collectors_path": str(files.collectors_path(row["slug"])) if source else "",
+            "collectors_lines": len(source.splitlines()),
+            "busy": store.busy(row["uid"]),
+            "sync_enabled": sync_enabled(),
+            "admin_section": "communities",
+        },
+        model=AdminCommunityOut,
+    )
+
+
+@router.get("/communities/{slug}/ledger", response_class=HTMLResponse)
+async def admin_community_ledger(
+    request: Request,
+    slug: str,
+    run: str = "",
+    kind: str = "",
+    outcome: str = "",
+    reason: str = "",
+    q: str = "",
+    page: int = 1,
+):
+    admin = require_admin(request)
+    row = require_community(slug)
+    run_row = None
+    if run:
+        found = store.run(run)
+        if found is None or found["community_uid"] != row["uid"]:
+            raise not_found("Unknown run")
+        run_row = run_view(found)
+    scope = {"run_uid": run_row["uid"]} if run_row else {"community_uid": row["uid"]}
+    filters = {key: value for key, value in zip(FILTER_KEYS, (kind, outcome, reason, q)) if value}
+    rows, pagination = ledger_store.page(filters, page, **scope)
+    links = local_links(rows)
+    listed = ledger_store.counts(**scope)
+    exact = (run_row or {}).get("tally") or listed
+    path = f"{LIST_URL}/{slug}/ledger"
+    query = urlencode({**({"run": run_row["uid"]} if run_row else {}), **filters})
+    return respond(
+        request,
+        "admin_community_ledger.html",
+        {
+            **seo(request, f"{row['name'] or slug} sync log", path, "Communities"),
+            "request": request,
+            "user": admin,
+            "community": public(row),
+            "scope": "run" if run_row else "community",
+            "run": run_row,
+            "run_errors": (run_row or {}).get("errors") or [],
+            "run_log": (run_row or {}).get("log") or "",
+            "busy": (run_row["status"] == "running") if run_row else store.busy(row["uid"]),
+            "verdict": {"level": run_row["verdict"], "title": LEVEL_TITLES.get(run_row["verdict"], ""), "headline": run_row["headline"]} if run_row else {},
+            "tiles": [
+                {"key": o.key, "label": o.label, "tone": o.tone, "help": o.help, "count": int(exact.get(o.key, 0)), "listed": int(listed.get(o.key, 0))}
+                for o in OUTCOMES
+            ],
+            "items": [item_view(item, links) for item in rows],
+            "pagination": pagination,
+            "pagination_query": f"{query}&" if query else "",
+            "filters": filters,
+            "options": ledger_options(scope),
+            "admin_section": "communities",
+        },
+        model=AdminCommunityLedgerOut,
+    )
+
+
+@router.post("/communities/{slug}/run")
+async def admin_community_run(request: Request, slug: str, data: Annotated[CommunityRunForm, Form()]):
+    admin = require_admin(request)
+    row = require_community(slug)
+    if store.busy(row["uid"]):
+        return json_error(409, "A run is already in progress for this community")
+    await spawn(CommunityRunner().run(row, dry_run=data.dry_run, trigger=TRIGGER_ADMIN))
+    audit.record(
+        request,
+        "admin.community.run",
+        user=admin,
+        target_type="community",
+        target_uid=row["uid"],
+        target_label=slug,
+        metadata={"dry_run": data.dry_run},
+        summary=f"{admin['username']} started a {'dry run' if data.dry_run else 'run'} of community {slug}",
+    )
+    return action_result(request, f"{LIST_URL}/{slug}")
+
+
+@router.post("/communities/{slug}/relearn")
+async def admin_community_relearn(request: Request, slug: str):
+    admin = require_admin(request)
+    row = require_community(slug)
+    if store.busy(row["uid"]):
+        return json_error(409, "A run is already in progress for this community")
+    await spawn(CommunityRunner().discover(row["query"] or row["url"], admin["uid"], relearn_slug=slug))
+    audit.record(
+        request,
+        "admin.community.relearn",
+        user=admin,
+        target_type="community",
+        target_uid=row["uid"],
+        target_label=slug,
+        summary=f"{admin['username']} started relearning community {slug}",
+    )
+    return action_result(request, f"{LIST_URL}/{slug}")
+
+
+@router.post("/communities/{slug}/update")
+async def admin_community_update(request: Request, slug: str, data: Annotated[CommunityUpdateForm, Form()]):
+    admin = require_admin(request)
+    row = require_community(slug)
+    changes = {"interval_seconds": data.interval_seconds, "enabled": int(data.enabled)}
+    if data.clear_login:
+        changes.update(login_username="", login_email="", login_password="", login_totp_secret="")
+    else:
+        for column, value in (("login_username", data.login_username.strip()), ("login_email", data.login_email.strip()), ("login_password", data.login_password), ("login_totp_secret", data.login_totp_secret.strip())):
+            if value:
+                changes[column] = value
+    submitted = {name: getattr(data, name) for name in SyncOptions.__dataclass_fields__ if getattr(data, name) is not None}
+    if submitted:
+        try:
+            options = SyncOptions.from_row(row).merged(**submitted)
+        except ValueError as exc:
+            return json_error(400, str(exc))
+        if options.alias and not SyncOptions.from_row(row).alias and not identity_map(row["origin"]) and count_links(row["origin"]):
+            return json_error(400, "This community already has records stored under their original names; aliasing cannot be switched on afterwards")
+        changes["sync_options"] = options.to_dict()
+    store.update(row["uid"], **changes)
+    audit.record(
+        request,
+        "admin.community.update",
+        user=admin,
+        target_type="community",
+        target_uid=row["uid"],
+        target_label=slug,
+        metadata={"interval_seconds": data.interval_seconds, "enabled": data.enabled, "sync_options": changes.get("sync_options"), "login_changed": data.clear_login or bool(data.login_password or data.login_username or data.login_email or data.login_totp_secret)},
+        summary=f"{admin['username']} updated community {slug}",
+    )
+    return action_result(request, f"{LIST_URL}/{slug}")
+
+
+def schema_text(schema) -> str:
+    return json.dumps(schema, indent=2, ensure_ascii=False)
+
+
+def schema_page(request: Request, admin: dict, row: dict, text: str, errors: list[str], notice: str, status_code: int):
+    context = {
+        **seo(request, f"{row['name'] or row['slug']} schema", f"{LIST_URL}/{row['slug']}/schema", "Communities"),
+        "request": request,
+        "user": admin,
+        "community": public(row),
+        "schema_text": text,
+        "has_previous": bool(row.get("schema_previous")),
+        "errors": errors,
+        "notice": notice,
+        "admin_section": "communities",
+    }
+    response = respond(request, "admin_community_schema.html", context, model=AdminCommunitySchemaOut)
+    response.status_code = status_code
+    return response
+
+
+@router.get("/communities/{slug}/schema", response_class=HTMLResponse)
+async def admin_community_schema(request: Request, slug: str):
+    admin = require_admin(request)
+    row = require_community(slug)
+    return schema_page(request, admin, row, schema_text(row.get("schema") or {}), [], "", 200)
+
+
+@router.post("/communities/{slug}/schema")
+async def admin_community_schema_save(request: Request, slug: str, data: Annotated[CommunitySchemaForm, Form()]):
+    admin = require_admin(request)
+    row = require_community(slug)
+    if store.busy(row["uid"]):
+        return json_error(409, "A run is already in progress for this community")
+    try:
+        schema = json.loads(data.schema_text)
+    except ValueError as exc:
+        return schema_page(request, admin, row, data.schema_text, [f"The text is not valid JSON: {exc}"], "", 400)
+    errors = validate_schema(schema)
+    if errors:
+        return schema_page(request, admin, row, data.schema_text, errors, "", 400)
+    verdict = await CommunityRunner().verify_schema(row, schema)
+    if not verdict.ok:
+        failed = ["The schema was checked against the live source and was not saved."] + verdict.problems[:10]
+        return schema_page(request, admin, row, data.schema_text, failed, "", 422)
+    store.update(
+        row["uid"],
+        schema=schema,
+        schema_previous=row.get("schema") or {},
+        status=STATUS_READY,
+        last_error="",
+        failure_count=0,
+        enabled=1,
+    )
+    files.write(row["slug"], schema)
+    audit.record(
+        request,
+        "admin.community.schema",
+        user=admin,
+        target_type="community",
+        target_uid=row["uid"],
+        target_label=slug,
+        metadata={"feeds": len(schema["feeds"]), "sample_posts": verdict.posts, "sample_comments": verdict.comments},
+        summary=f"{admin['username']} replaced the learned schema of community {slug}",
+    )
+    return action_result(request, f"{LIST_URL}/{slug}/schema", data={"posts": verdict.posts, "comments": verdict.comments})
+
+
+@router.post("/communities/{slug}/schema/restore")
+async def admin_community_schema_restore(request: Request, slug: str):
+    admin = require_admin(request)
+    row = require_community(slug)
+    previous = row.get("schema_previous") or {}
+    if not previous:
+        return json_error(409, "There is no previous schema to restore")
+    errors = validate_schema(previous)
+    if errors:
+        return schema_page(request, admin, row, schema_text(previous), errors, "", 400)
+    store.update(row["uid"], schema=previous, schema_previous=row.get("schema") or {})
+    files.write(row["slug"], previous)
+    audit.record(
+        request,
+        "admin.community.schema_restore",
+        user=admin,
+        target_type="community",
+        target_uid=row["uid"],
+        target_label=slug,
+        summary=f"{admin['username']} restored the previous schema of community {slug}",
+    )
+    return action_result(request, f"{LIST_URL}/{slug}/schema")
+
+
+@router.post("/communities/{slug}/delete")
+async def admin_community_delete(request: Request, slug: str):
+    admin = require_admin(request)
+    row = require_community(slug)
+    store.delete(row["uid"], admin["uid"])
+    audit.record(
+        request,
+        "admin.community.delete",
+        user=admin,
+        target_type="community",
+        target_uid=row["uid"],
+        target_label=slug,
+        summary=f"{admin['username']} deleted community {slug}",
+    )
+    return action_result(request, LIST_URL)

devplacepy/routers/admin/containers.py

@@ -409,7 +409,7 @@ async def container_instance_page(request: Request, uid: str):
             "events": store.list_events(inst["uid"]),
             "schedules": store.list_schedules(inst["uid"]),
             "stats": api.instance_stats(inst["uid"]),
-            "runtime": api.instance_runtime(inst),
+            "runtime": await api.instance_runtime(inst),
             "can_manage": can_manage,
             "admin_section": "containers",
         },

devplacepy/routers/admin/devii_tasks.py

@@ -139,6 +139,7 @@ async def admin_devii_tasks(request: Request, state: str = "active"):
             {"name": "Devii tasks", "url": "/admin/devii-tasks"},
         ],
         schemas=[website_schema(base)],
+        robots="noindex,nofollow",
     )
     return respond(
         request,

devplacepy/routers/admin/gateway_configs.py

@@ -1,21 +1,32 @@
 # retoor <retoor@molodetz.nl>
 
 import logging
+from typing import Optional
 
 from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse, JSONResponse
+from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
 from pydantic import ValidationError
 
+from devplacepy.responses import json_error, respond, wants_json
+from devplacepy.schemas import (
+    AdminGatewayModelFormOut,
+    AdminGatewayOut,
+    AdminGatewayProviderFormOut,
+    AdminGatewayQuotaFormOut,
+)
 from devplacepy.seo import base_seo_context, site_url, website_schema
 from devplacepy.services.audit import record as audit
 from devplacepy.services.manager import service_manager
-from devplacepy.services.openai_gateway import quota, routing
+from devplacepy.services.openai_gateway import model_stats_query, quota, routing
 from devplacepy.templating import templates
-from devplacepy.utils import require_admin
+from devplacepy.utils import not_found, require_admin
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
+INDEX_URL = "/admin/gateway"
+TABS = ("models", "providers", "quota", "stats")
+
 
 def _default_provider_summary() -> dict:
     svc = service_manager.get_service("openai")
@@ -32,10 +43,81 @@ def _default_provider_summary() -> dict:
     }
 
 
+def _quota_defaults_summary() -> dict:
+    svc = service_manager.get_service("openai")
+    cfg = svc.get_config() if svc is not None else {}
+    return {
+        "user": cfg.get(quota.FIELD_DEFAULT_USER, 0.0),
+        "admin": cfg.get(quota.FIELD_DEFAULT_ADMIN, 0.0),
+        "guest": cfg.get(quota.FIELD_DEFAULT_GUEST, 0.0),
+        "internal": cfg.get(quota.FIELD_DEFAULT_INTERNAL, 0.0),
+        "key": cfg.get(quota.FIELD_DEFAULT_KEY, 0.0),
+    }
+
+
+def _rule_label(rule: dict) -> str:
+    return quota.scope_label(rule, fallback=rule.get("uid", ""))
+
+
+def _breadcrumbs(*extra: dict) -> list[dict]:
+    trail = [
+        {"name": "Home", "url": "/feed"},
+        {"name": "Admin", "url": "/admin"},
+        {"name": "Gateway", "url": INDEX_URL},
+    ]
+    return trail + list(extra)
+
+
+def _seo(request: Request, title: str, breadcrumbs: list[dict]) -> dict:
+    base = site_url(request)
+    return base_seo_context(
+        request,
+        title=title,
+        description="Manage OpenAI gateway providers, per-model routing, and quota rules.",
+        robots="noindex,nofollow",
+        breadcrumbs=breadcrumbs,
+        schemas=[website_schema(base)],
+    )
+
+
+def _first_error(exc: ValidationError) -> str:
+    return exc.errors()[0].get("msg", "Invalid input")
+
+
 def _validation_error(exc: ValidationError) -> JSONResponse:
-    first = exc.errors()[0]
-    message = first.get("msg", "Invalid input")
-    return JSONResponse({"ok": False, "error": message}, status_code=400)
+    return JSONResponse({"ok": False, "error": _first_error(exc)}, status_code=400)
+
+
+def _bool_str(value) -> str:
+    if isinstance(value, bool):
+        return "1" if value else "0"
+    return "1" if str(value).strip().lower() in ("1", "true", "on", "yes") else "0"
+
+
+def _blank_to_none(value):
+    value = "" if value is None else str(value).strip()
+    return value or None
+
+
+def _time_to_minutes(value: str) -> Optional[int]:
+    value = (value or "").strip()
+    if not value or ":" not in value:
+        return None
+    hours_str, _, minutes_str = value.partition(":")
+    try:
+        return int(hours_str) * 60 + int(minutes_str)
+    except ValueError:
+        return None
+
+
+def _minutes_to_time(value) -> str:
+    if value is None or value == "":
+        return ""
+    try:
+        total = int(value)
+    except (TypeError, ValueError):
+        return ""
+    return f"{(total // 60) % 24:02d}:{total % 60:02d}"
 
 
 async def _payload(request: Request) -> dict:
@@ -50,35 +132,54 @@ async def _payload(request: Request) -> dict:
     return {key: value for key, value in form.items()}
 
 
-@router.get("/gateway", response_class=HTMLResponse)
-async def gateway_config_page(request: Request):
-    admin = require_admin(request)
-    base = site_url(request)
-    seo_ctx = base_seo_context(
-        request,
-        title="Gateway routing - Admin",
-        description="Manage OpenAI gateway providers and per-model routing.",
-        robots="noindex,nofollow",
-        breadcrumbs=[
-            {"name": "Home", "url": "/feed"},
-            {"name": "Admin", "url": "/admin"},
-            {"name": "Gateway", "url": "/admin/gateway"},
-        ],
-        schemas=[website_schema(base)],
-    )
+def _error_response(
+    request: Request, template: str, context: dict, message: str, status_code: int = 400
+):
+    if wants_json(request):
+        return json_error(status_code, message)
     return templates.TemplateResponse(
+        request,
+        template,
+        {**context, "request": request, "error": message},
+        status_code=status_code,
+    )
+
+
+# --- Index page (tabs: models / providers / quota) --------------------------
+
+
+@router.get("/gateway", response_class=HTMLResponse)
+async def gateway_config_page(request: Request, tab: str = "models"):
+    admin = require_admin(request)
+    if tab not in TABS:
+        tab = "models"
+    quota_rules = quota.quota_rule_store.list()
+    for rule in quota_rules:
+        rule["spent_24h_usd"] = round(
+            quota.spent_24h(rule["owner_kind"], rule["owner_id"], rule["app_reference"]), 6
+        )
+    return respond(
         request,
         "admin_gateway.html",
         {
-            **seo_ctx,
-            "request": request,
+            **_seo(request, "Gateway routing - Admin", _breadcrumbs()),
             "user": admin,
             "admin_section": "gateway",
+            "tab": tab,
+            "providers": routing.provider_store.list(),
+            "models": routing.model_store.list(),
+            "quota_rules": quota_rules,
             "default_provider": _default_provider_summary(),
+            "quota_defaults": _quota_defaults_summary(),
+            "stats_ranges": list(model_stats_query.RANGE_SECONDS.keys()),
         },
+        model=AdminGatewayOut,
     )
 
 
+# --- Providers: JSON API (Devii + programmatic clients) ---------------------
+
+
 @router.get("/gateway/providers")
 async def list_providers(request: Request):
     require_admin(request)
@@ -92,13 +193,41 @@ async def list_providers(request: Request):
 
 
 @router.post("/gateway/providers")
-async def save_provider(request: Request):
+async def save_provider_json(request: Request):
     admin = require_admin(request)
     body = await _payload(request)
     try:
         payload = routing.ProviderIn(**body)
     except ValidationError as exc:
         return _validation_error(exc)
+    saved = _save_provider(request, admin, payload)
+    return JSONResponse({"ok": True, "provider": saved})
+
+
+@router.delete("/gateway/providers/{name}")
+async def delete_provider(request: Request, name: str):
+    admin = require_admin(request)
+    if not _remove_provider(request, admin, name):
+        return JSONResponse({"ok": False, "error": "Provider not found"}, status_code=404)
+    return JSONResponse({"ok": True})
+
+
+# --- Providers: backend-rendered pages ---------------------------------------
+
+
+def _provider_form_values(data: Optional[dict] = None) -> dict:
+    data = data or {}
+    return {
+        "name": str(data.get("name", "")),
+        "base_url": str(data.get("base_url", "")),
+        "api_key": str(data.get("api_key", "")),
+        "is_active": _bool_str(data.get("is_active", True)),
+        "client_profile": str(data.get("client_profile", "")),
+        "protocol": str(data.get("protocol", "chat_completions")),
+    }
+
+
+def _save_provider(request: Request, admin: dict, payload: "routing.ProviderIn") -> dict:
     saved = routing.provider_store.set(payload)
     audit.record(
         request,
@@ -109,25 +238,130 @@ async def save_provider(request: Request):
         target_label=payload.name,
         summary=f"admin {admin['username']} saved gateway provider {payload.name}",
     )
-    return JSONResponse({"ok": True, "provider": saved})
+    return saved
 
 
-@router.delete("/gateway/providers/{name}")
-async def delete_provider(request: Request, name: str):
-    admin = require_admin(request)
+def _remove_provider(request: Request, admin: dict, name: str) -> bool:
     existed = routing.provider_store.remove(name)
-    if not existed:
-        return JSONResponse({"ok": False, "error": "Provider not found"}, status_code=404)
-    audit.record(
+    if existed:
+        audit.record(
+            request,
+            "gateway.provider.delete",
+            user=admin,
+            target_type="gateway_provider",
+            target_uid=name,
+            target_label=name,
+            summary=f"admin {admin['username']} deleted gateway provider {name}",
+        )
+    return existed
+
+
+@router.get("/gateway/providers/new", response_class=HTMLResponse)
+async def new_provider_page(request: Request):
+    admin = require_admin(request)
+    return respond(
         request,
-        "gateway.provider.delete",
-        user=admin,
-        target_type="gateway_provider",
-        target_uid=name,
-        target_label=name,
-        summary=f"admin {admin['username']} deleted gateway provider {name}",
+        "admin_gateway_provider_form.html",
+        {
+            **_seo(request, "Add provider - Gateway - Admin", _breadcrumbs({"name": "Add provider", "url": "/admin/gateway/providers/new"})),
+            "user": admin,
+            "admin_section": "gateway",
+            "is_edit": False,
+            "form": _provider_form_values(),
+            "error": None,
+        },
+        model=AdminGatewayProviderFormOut,
     )
-    return JSONResponse({"ok": True})
+
+
+@router.post("/gateway/providers/new")
+async def create_provider_page(request: Request):
+    admin = require_admin(request)
+    data = dict(await request.form())
+    try:
+        payload = routing.ProviderIn(**data)
+    except ValidationError as exc:
+        return _error_response(
+            request,
+            "admin_gateway_provider_form.html",
+            {
+                **_seo(request, "Add provider - Gateway - Admin", _breadcrumbs({"name": "Add provider", "url": "/admin/gateway/providers/new"})),
+                "user": admin,
+                "admin_section": "gateway",
+                "is_edit": False,
+                "form": _provider_form_values(data),
+            },
+            _first_error(exc),
+        )
+    _save_provider(request, admin, payload)
+    return RedirectResponse(url=f"{INDEX_URL}?tab=providers", status_code=302)
+
+
+@router.get("/gateway/providers/{name}/edit", response_class=HTMLResponse)
+async def edit_provider_page(request: Request, name: str):
+    admin = require_admin(request)
+    provider = routing.provider_store.get(name)
+    if provider is None:
+        raise not_found("Provider not found")
+    return respond(
+        request,
+        "admin_gateway_provider_form.html",
+        {
+            **_seo(
+                request,
+                f"Edit {provider['name']} - Gateway - Admin",
+                _breadcrumbs({"name": f"Edit {provider['name']}", "url": f"/admin/gateway/providers/{provider['name']}/edit"}),
+            ),
+            "user": admin,
+            "admin_section": "gateway",
+            "is_edit": True,
+            "form": _provider_form_values(provider),
+            "error": None,
+        },
+        model=AdminGatewayProviderFormOut,
+    )
+
+
+@router.post("/gateway/providers/{name}/edit")
+async def edit_provider_save(request: Request, name: str):
+    admin = require_admin(request)
+    existing = routing.provider_store.get(name)
+    if existing is None:
+        raise not_found("Provider not found")
+    data = dict(await request.form())
+    data["name"] = existing["name"]
+    try:
+        payload = routing.ProviderIn(**data)
+    except ValidationError as exc:
+        return _error_response(
+            request,
+            "admin_gateway_provider_form.html",
+            {
+                **_seo(
+                    request,
+                    f"Edit {existing['name']} - Gateway - Admin",
+                    _breadcrumbs({"name": f"Edit {existing['name']}", "url": f"/admin/gateway/providers/{existing['name']}/edit"}),
+                ),
+                "user": admin,
+                "admin_section": "gateway",
+                "is_edit": True,
+                "form": _provider_form_values({**data, "name": existing["name"]}),
+            },
+            _first_error(exc),
+        )
+    _save_provider(request, admin, payload)
+    return RedirectResponse(url=f"{INDEX_URL}?tab=providers", status_code=302)
+
+
+@router.post("/gateway/providers/{name}/delete")
+async def delete_provider_page(request: Request, name: str):
+    admin = require_admin(request)
+    if not _remove_provider(request, admin, name):
+        raise not_found("Provider not found")
+    return RedirectResponse(url=f"{INDEX_URL}?tab=providers", status_code=302)
+
+
+# --- Model routes: JSON API (Devii + programmatic clients) ------------------
 
 
 @router.get("/gateway/models")
@@ -142,14 +376,25 @@ async def list_models(request: Request):
     )
 
 
-@router.post("/gateway/models")
-async def save_model(request: Request):
-    admin = require_admin(request)
-    body = await _payload(request)
-    try:
-        payload = routing.ModelRouteIn(**body)
-    except ValidationError as exc:
-        return _validation_error(exc)
+@router.get("/gateway/provider-models")
+async def provider_models(request: Request, provider: str = ""):
+    require_admin(request)
+    models = await routing.fetch_provider_models(provider)
+    if models is None:
+        return json_error(404, "No model list available for this provider")
+    return JSONResponse({"provider": provider, "models": models})
+
+
+def _check_fallback(payload: "routing.ModelRouteIn") -> Optional[str]:
+    if not payload.fallback_model:
+        return None
+    fallback_route = routing.model_store.get(payload.fallback_model)
+    if fallback_route is None or fallback_route.kind != payload.kind:
+        return "Fallback model must be an existing model route of the same kind"
+    return None
+
+
+def _save_model(request: Request, admin: dict, payload: "routing.ModelRouteIn") -> dict:
     saved = routing.model_store.set(payload)
     audit.record(
         request,
@@ -163,41 +408,274 @@ async def save_model(request: Request):
             f"{payload.source_model} -> {payload.target_model}"
         ),
     )
+    return saved
+
+
+def _remove_model(request: Request, admin: dict, source_model: str) -> bool:
+    existed = routing.model_store.remove(source_model)
+    if existed:
+        audit.record(
+            request,
+            "gateway.model.delete",
+            user=admin,
+            target_type="gateway_model",
+            target_uid=source_model,
+            target_label=source_model,
+            summary=f"admin {admin['username']} deleted gateway model route {source_model}",
+        )
+    return existed
+
+
+@router.post("/gateway/models")
+async def save_model_json(request: Request):
+    admin = require_admin(request)
+    body = await _payload(request)
+    try:
+        payload = routing.ModelRouteIn(**body)
+    except ValidationError as exc:
+        return _validation_error(exc)
+    fallback_error = _check_fallback(payload)
+    if fallback_error:
+        return JSONResponse({"ok": False, "error": fallback_error}, status_code=400)
+    saved = _save_model(request, admin, payload)
     return JSONResponse({"ok": True, "model": saved})
 
 
-@router.delete("/gateway/models/{source_model}")
+@router.delete("/gateway/models/{source_model:path}")
 async def delete_model(request: Request, source_model: str):
     admin = require_admin(request)
-    existed = routing.model_store.remove(source_model)
-    if not existed:
+    if not _remove_model(request, admin, source_model):
         return JSONResponse({"ok": False, "error": "Model route not found"}, status_code=404)
-    audit.record(
-        request,
-        "gateway.model.delete",
-        user=admin,
-        target_type="gateway_model",
-        target_uid=source_model,
-        target_label=source_model,
-        summary=f"admin {admin['username']} deleted gateway model route {source_model}",
-    )
     return JSONResponse({"ok": True})
 
 
-def _quota_defaults_summary() -> dict:
-    svc = service_manager.get_service("openai")
-    cfg = svc.get_config() if svc is not None else {}
+# --- Model routes: backend-rendered pages ------------------------------------
+
+MODEL_FORM_DEFAULTS = {
+    "source_model": "",
+    "provider": "",
+    "target_model": "",
+    "kind": "chat",
+    "vision_provider": "",
+    "vision_model": "",
+    "context_window": "0",
+    "price_cache_hit_per_m": "0",
+    "price_cache_miss_per_m": "0",
+    "price_output_per_m": "0",
+    "price_input_per_m": "0",
+    "context_tier_threshold_tokens": "0",
+    "price_cache_hit_per_m_tier2": "",
+    "price_cache_miss_per_m_tier2": "",
+    "price_output_per_m_tier2": "",
+    "price_input_per_m_tier2": "",
+    "off_peak_start": "",
+    "off_peak_end": "",
+    "off_peak_discount_pct": "0",
+    "fallback_model": "",
+    "is_active": "1",
+}
+
+
+def _model_form_values_from_route(route: dict) -> dict:
+    def _num(key):
+        return str(route.get(key, 0))
+
+    def _opt_num(key):
+        value = route.get(key)
+        return "" if value is None else str(value)
+
     return {
-        "user": cfg.get(quota.FIELD_DEFAULT_USER, 0.0),
-        "admin": cfg.get(quota.FIELD_DEFAULT_ADMIN, 0.0),
-        "guest": cfg.get(quota.FIELD_DEFAULT_GUEST, 0.0),
-        "internal": cfg.get(quota.FIELD_DEFAULT_INTERNAL, 0.0),
-        "key": cfg.get(quota.FIELD_DEFAULT_KEY, 0.0),
+        "source_model": route.get("source_model", ""),
+        "provider": route.get("provider", ""),
+        "target_model": route.get("target_model", ""),
+        "kind": route.get("kind", "chat"),
+        "vision_provider": route.get("vision_provider", ""),
+        "vision_model": route.get("vision_model", ""),
+        "context_window": _num("context_window"),
+        "price_cache_hit_per_m": _num("price_cache_hit_per_m"),
+        "price_cache_miss_per_m": _num("price_cache_miss_per_m"),
+        "price_output_per_m": _num("price_output_per_m"),
+        "price_input_per_m": _num("price_input_per_m"),
+        "context_tier_threshold_tokens": _num("context_tier_threshold_tokens"),
+        "price_cache_hit_per_m_tier2": _opt_num("price_cache_hit_per_m_tier2"),
+        "price_cache_miss_per_m_tier2": _opt_num("price_cache_miss_per_m_tier2"),
+        "price_output_per_m_tier2": _opt_num("price_output_per_m_tier2"),
+        "price_input_per_m_tier2": _opt_num("price_input_per_m_tier2"),
+        "off_peak_start": _minutes_to_time(route.get("off_peak_start_minute")),
+        "off_peak_end": _minutes_to_time(route.get("off_peak_end_minute")),
+        "off_peak_discount_pct": _num("off_peak_discount_pct"),
+        "fallback_model": route.get("fallback_model", ""),
+        "is_active": _bool_str(route.get("is_active", True)),
     }
 
 
-def _rule_label(rule: dict) -> str:
-    return quota.scope_label(rule, fallback=rule.get("uid", ""))
+def _model_form_values_from_submission(data: dict) -> dict:
+    values = dict(MODEL_FORM_DEFAULTS)
+    for key in values:
+        if key in data:
+            values[key] = str(data[key])
+    values["is_active"] = _bool_str(data.get("is_active", "1"))
+    return values
+
+
+def _model_payload_kwargs(data: dict) -> dict:
+    def _num(key, default="0"):
+        value = data.get(key, default)
+        return value if str(value).strip() != "" else default
+
+    def _opt_num(key):
+        value = str(data.get(key, "")).strip()
+        return value or None
+
+    return {
+        "source_model": data.get("source_model", ""),
+        "provider": data.get("provider", ""),
+        "target_model": data.get("target_model", ""),
+        "kind": data.get("kind", "chat"),
+        "vision_provider": data.get("vision_provider", ""),
+        "vision_model": data.get("vision_model", ""),
+        "context_window": _num("context_window"),
+        "price_cache_hit_per_m": _num("price_cache_hit_per_m"),
+        "price_cache_miss_per_m": _num("price_cache_miss_per_m"),
+        "price_output_per_m": _num("price_output_per_m"),
+        "price_input_per_m": _num("price_input_per_m"),
+        "context_tier_threshold_tokens": _num("context_tier_threshold_tokens"),
+        "price_cache_hit_per_m_tier2": _opt_num("price_cache_hit_per_m_tier2"),
+        "price_cache_miss_per_m_tier2": _opt_num("price_cache_miss_per_m_tier2"),
+        "price_output_per_m_tier2": _opt_num("price_output_per_m_tier2"),
+        "price_input_per_m_tier2": _opt_num("price_input_per_m_tier2"),
+        "off_peak_start_minute": _time_to_minutes(data.get("off_peak_start", "")),
+        "off_peak_end_minute": _time_to_minutes(data.get("off_peak_end", "")),
+        "off_peak_discount_pct": _num("off_peak_discount_pct"),
+        "fallback_model": data.get("fallback_model", ""),
+        "is_active": data.get("is_active", "1"),
+    }
+
+
+def _fallback_groups(exclude_source: str = "") -> list[dict]:
+    by_kind: dict[str, list[str]] = {}
+    for route in routing.model_store.list():
+        if route["source_model"] == exclude_source:
+            continue
+        by_kind.setdefault(route["kind"], []).append(route["source_model"])
+    return [{"kind": kind, "options": names} for kind, names in sorted(by_kind.items())]
+
+
+def _model_form_context(
+    request: Request,
+    admin: dict,
+    *,
+    is_edit: bool,
+    form: dict,
+    exclude_source: str = "",
+    title: str,
+    crumb_url: str,
+) -> dict:
+    return {
+        **_seo(request, title, _breadcrumbs({"name": title.split(" - ")[0], "url": crumb_url})),
+        "user": admin,
+        "admin_section": "gateway",
+        "is_edit": is_edit,
+        "form": form,
+        "providers": routing.provider_store.list(),
+        "fallback_groups": _fallback_groups(exclude_source),
+    }
+
+
+@router.get("/gateway/models/new", response_class=HTMLResponse)
+async def new_model_page(request: Request):
+    admin = require_admin(request)
+    context = _model_form_context(
+        request,
+        admin,
+        is_edit=False,
+        form=_model_form_values_from_submission({}),
+        title="Add model route - Gateway - Admin",
+        crumb_url="/admin/gateway/models/new",
+    )
+    return respond(request, "admin_gateway_model_form.html", {**context, "error": None}, model=AdminGatewayModelFormOut)
+
+
+@router.post("/gateway/models/new")
+async def create_model_page(request: Request):
+    admin = require_admin(request)
+    data = dict(await request.form())
+    context = _model_form_context(
+        request,
+        admin,
+        is_edit=False,
+        form=_model_form_values_from_submission(data),
+        exclude_source=str(data.get("source_model", "")),
+        title="Add model route - Gateway - Admin",
+        crumb_url="/admin/gateway/models/new",
+    )
+    try:
+        payload = routing.ModelRouteIn(**_model_payload_kwargs(data))
+    except ValidationError as exc:
+        return _error_response(request, "admin_gateway_model_form.html", context, _first_error(exc))
+    fallback_error = _check_fallback(payload)
+    if fallback_error:
+        return _error_response(request, "admin_gateway_model_form.html", context, fallback_error)
+    _save_model(request, admin, payload)
+    return RedirectResponse(url=f"{INDEX_URL}?tab=models", status_code=302)
+
+
+@router.get("/gateway/models/{source_model:path}/edit", response_class=HTMLResponse)
+async def edit_model_page(request: Request, source_model: str):
+    admin = require_admin(request)
+    route = routing.model_store.get(source_model)
+    if route is None:
+        raise not_found("Model route not found")
+    route_dict = route.__dict__.copy()
+    context = _model_form_context(
+        request,
+        admin,
+        is_edit=True,
+        form=_model_form_values_from_route(route_dict),
+        exclude_source=source_model,
+        title=f"Edit {source_model} - Gateway - Admin",
+        crumb_url=f"/admin/gateway/models/{source_model}/edit",
+    )
+    return respond(request, "admin_gateway_model_form.html", {**context, "error": None}, model=AdminGatewayModelFormOut)
+
+
+@router.post("/gateway/models/{source_model:path}/edit")
+async def edit_model_save(request: Request, source_model: str):
+    admin = require_admin(request)
+    existing = routing.model_store.get(source_model)
+    if existing is None:
+        raise not_found("Model route not found")
+    data = dict(await request.form())
+    data["source_model"] = source_model
+    context = _model_form_context(
+        request,
+        admin,
+        is_edit=True,
+        form=_model_form_values_from_submission(data),
+        exclude_source=source_model,
+        title=f"Edit {source_model} - Gateway - Admin",
+        crumb_url=f"/admin/gateway/models/{source_model}/edit",
+    )
+    try:
+        payload = routing.ModelRouteIn(**_model_payload_kwargs(data))
+    except ValidationError as exc:
+        return _error_response(request, "admin_gateway_model_form.html", context, _first_error(exc))
+    fallback_error = _check_fallback(payload)
+    if fallback_error:
+        return _error_response(request, "admin_gateway_model_form.html", context, fallback_error)
+    _save_model(request, admin, payload)
+    return RedirectResponse(url=f"{INDEX_URL}?tab=models", status_code=302)
+
+
+@router.post("/gateway/models/{source_model:path}/delete")
+async def delete_model_page(request: Request, source_model: str):
+    admin = require_admin(request)
+    if not _remove_model(request, admin, source_model):
+        raise not_found("Model route not found")
+    return RedirectResponse(url=f"{INDEX_URL}?tab=models", status_code=302)
+
+
+# --- Quota rules: JSON API (Devii + programmatic clients) -------------------
 
 
 @router.get("/gateway/quota-rules")
@@ -217,15 +695,9 @@ async def list_quota_rules(request: Request):
     )
 
 
-@router.post("/gateway/quota-rules")
-async def save_quota_rule(request: Request):
-    admin = require_admin(request)
-    body = await _payload(request)
-    uid = str(body.pop("uid", "") or "").strip() or None
-    try:
-        payload = quota.QuotaRuleIn(**body)
-    except ValidationError as exc:
-        return _validation_error(exc)
+def _save_quota_rule(
+    request: Request, admin: dict, payload: "quota.QuotaRuleIn", uid: Optional[str] = None
+) -> dict:
     saved = quota.quota_rule_store.set(payload, uid=uid, created_by=admin["uid"])
     audit.record(
         request,
@@ -243,6 +715,19 @@ async def save_quota_rule(request: Request):
             "is_active": saved["is_active"],
         },
     )
+    return saved
+
+
+@router.post("/gateway/quota-rules")
+async def save_quota_rule_json(request: Request):
+    admin = require_admin(request)
+    body = await _payload(request)
+    uid = str(body.pop("uid", "") or "").strip() or None
+    try:
+        payload = quota.QuotaRuleIn(**body)
+    except ValidationError as exc:
+        return _validation_error(exc)
+    saved = _save_quota_rule(request, admin, payload, uid=uid)
     return JSONResponse({"ok": True, "rule": saved})
 
 
@@ -292,3 +777,220 @@ async def delete_quota_rule(request: Request, uid: str):
         summary=f"admin {admin['username']} deleted gateway quota rule ({label})",
     )
     return JSONResponse({"ok": True})
+
+
+# --- Quota rules: backend-rendered pages -------------------------------------
+
+QUOTA_FORM_DEFAULTS = {
+    "owner_kind": "",
+    "owner_id": "",
+    "app_reference": "",
+    "limit_usd": "0",
+    "is_active": "1",
+    "label": "",
+}
+
+
+def _quota_form_values(data: Optional[dict] = None) -> dict:
+    if not data:
+        return dict(QUOTA_FORM_DEFAULTS)
+    values = dict(QUOTA_FORM_DEFAULTS)
+    for key in values:
+        if key in data and data[key] is not None:
+            values[key] = str(data[key])
+    values["is_active"] = _bool_str(data.get("is_active", "1"))
+    return values
+
+
+@router.get("/gateway/quota-rules/new", response_class=HTMLResponse)
+async def new_quota_rule_page(request: Request):
+    admin = require_admin(request)
+    return respond(
+        request,
+        "admin_gateway_quota_form.html",
+        {
+            **_seo(request, "Add quota rule - Gateway - Admin", _breadcrumbs({"name": "Add quota rule", "url": "/admin/gateway/quota-rules/new"})),
+            "user": admin,
+            "admin_section": "gateway",
+            "is_edit": False,
+            "form": _quota_form_values(),
+            "error": None,
+        },
+        model=AdminGatewayQuotaFormOut,
+    )
+
+
+@router.post("/gateway/quota-rules/new")
+async def create_quota_rule_page(request: Request):
+    admin = require_admin(request)
+    data = dict(await request.form())
+    try:
+        payload = quota.QuotaRuleIn(
+            owner_kind=_blank_to_none(data.get("owner_kind")),
+            owner_id=_blank_to_none(data.get("owner_id")),
+            app_reference=_blank_to_none(data.get("app_reference")),
+            limit_usd=data.get("limit_usd", "0") or "0",
+            is_active=data.get("is_active", "1"),
+            label=data.get("label", ""),
+        )
+    except ValidationError as exc:
+        return _error_response(
+            request,
+            "admin_gateway_quota_form.html",
+            {
+                **_seo(request, "Add quota rule - Gateway - Admin", _breadcrumbs({"name": "Add quota rule", "url": "/admin/gateway/quota-rules/new"})),
+                "user": admin,
+                "admin_section": "gateway",
+                "is_edit": False,
+                "form": _quota_form_values(data),
+            },
+            _first_error(exc),
+        )
+    _save_quota_rule(request, admin, payload)
+    return RedirectResponse(url=f"{INDEX_URL}?tab=quota", status_code=302)
+
+
+@router.get("/gateway/quota-rules/{uid}/edit", response_class=HTMLResponse)
+async def edit_quota_rule_page(request: Request, uid: str):
+    admin = require_admin(request)
+    rule = quota.quota_rule_store.get(uid)
+    if rule is None:
+        raise not_found("Quota rule not found")
+    rule_dict = rule.as_dict()
+    return respond(
+        request,
+        "admin_gateway_quota_form.html",
+        {
+            **_seo(
+                request,
+                "Edit quota rule - Gateway - Admin",
+                _breadcrumbs({"name": "Edit quota rule", "url": f"/admin/gateway/quota-rules/{uid}/edit"}),
+            ),
+            "user": admin,
+            "admin_section": "gateway",
+            "is_edit": True,
+            "uid": uid,
+            "form": _quota_form_values(rule_dict),
+            "error": None,
+        },
+        model=AdminGatewayQuotaFormOut,
+    )
+
+
+@router.post("/gateway/quota-rules/{uid}/edit")
+async def edit_quota_rule_save(request: Request, uid: str):
+    admin = require_admin(request)
+    existing = quota.quota_rule_store.get(uid)
+    if existing is None:
+        raise not_found("Quota rule not found")
+    data = dict(await request.form())
+    try:
+        payload = quota.QuotaRuleIn(
+            owner_kind=_blank_to_none(data.get("owner_kind")),
+            owner_id=_blank_to_none(data.get("owner_id")),
+            app_reference=_blank_to_none(data.get("app_reference")),
+            limit_usd=data.get("limit_usd", "0") or "0",
+            is_active=data.get("is_active", "1"),
+            label=data.get("label", ""),
+        )
+    except ValidationError as exc:
+        return _error_response(
+            request,
+            "admin_gateway_quota_form.html",
+            {
+                **_seo(
+                    request,
+                    "Edit quota rule - Gateway - Admin",
+                    _breadcrumbs({"name": "Edit quota rule", "url": f"/admin/gateway/quota-rules/{uid}/edit"}),
+                ),
+                "user": admin,
+                "admin_section": "gateway",
+                "is_edit": True,
+                "uid": uid,
+                "form": _quota_form_values(data),
+            },
+            _first_error(exc),
+        )
+    _save_quota_rule(request, admin, payload, uid=uid)
+    return RedirectResponse(url=f"{INDEX_URL}?tab=quota", status_code=302)
+
+
+@router.post("/gateway/quota-rules/{uid}/delete")
+async def delete_quota_rule_page(request: Request, uid: str):
+    admin = require_admin(request)
+    existing = quota.quota_rule_store.get(uid)
+    label = _rule_label(existing.as_dict()) if existing else uid
+    if not quota.quota_rule_store.remove(uid):
+        raise not_found("Quota rule not found")
+    audit.record(
+        request,
+        "gateway.quota_rule.delete",
+        user=admin,
+        target_type="gateway_quota_rule",
+        target_uid=uid,
+        target_label=label,
+        summary=f"admin {admin['username']} deleted gateway quota rule ({label})",
+    )
+    return RedirectResponse(url=f"{INDEX_URL}?tab=quota", status_code=302)
+
+
+@router.post("/gateway/quota-rules/{uid}/reset")
+async def reset_quota_rule_page(request: Request, uid: str):
+    admin = require_admin(request)
+    existing = quota.quota_rule_store.get(uid)
+    if existing is None:
+        raise not_found("Quota rule not found")
+    scope = quota.reset(
+        quota.QuotaResetIn(
+            owner_kind=existing.owner_kind,
+            owner_id=existing.owner_id,
+            app_reference=existing.app_reference,
+        ),
+        created_by=admin["uid"],
+    )
+    label = quota.scope_label(scope, fallback="every caller")
+    audit.record(
+        request,
+        "gateway.quota.reset",
+        user=admin,
+        target_type="gateway_quota",
+        target_uid=scope["uid"],
+        target_label=label,
+        summary=f"admin {admin['username']} reset the gateway 24h spend for {label}",
+        metadata={
+            "owner_kind": scope["owner_kind"],
+            "owner_id": scope["owner_id"],
+            "app_reference": scope["app_reference"],
+            "reset_at": scope["reset_at"],
+        },
+    )
+    return RedirectResponse(url=f"{INDEX_URL}?tab=quota", status_code=302)
+
+
+# --- Stats: JSON API for the Stats tab's charts ------------------------------
+
+
+@router.get("/gateway/stats/data")
+async def gateway_stats_data(request: Request, range: str = "24h"):
+    require_admin(request)
+    try:
+        return JSONResponse(model_stats_query.compute_summary(range))
+    except ValueError:
+        return json_error(400, f"Unknown range: {range!r}")
+
+
+@router.get("/gateway/stats/models")
+async def gateway_stats_models(request: Request):
+    require_admin(request)
+    return JSONResponse({"models": model_stats_query.list_known_models()})
+
+
+@router.get("/gateway/stats/model/{provider}/{model:path}")
+async def gateway_stats_model_detail(
+    request: Request, provider: str, model: str, range: str = "24h"
+):
+    require_admin(request)
+    try:
+        return JSONResponse(model_stats_query.compute_model_detail(provider, model, range))
+    except ValueError:
+        return json_error(400, f"Unknown range: {range!r}")

devplacepy/routers/admin/issues.py

@@ -5,11 +5,12 @@ import logging
 from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse
 
+from devplacepy.responses import respond
+from devplacepy.schemas import AdminIssuesPlanningOut
 from devplacepy.seo import base_seo_context
 from devplacepy.services.gitea import runtime
 from devplacepy.services.gitea.config import gitea_config
 from devplacepy.services.gitea.planning import collect_open_issues
-from devplacepy.templating import templates
 from devplacepy.utils import require_admin
 
 logger = logging.getLogger(__name__)
@@ -65,4 +66,4 @@ async def admin_issues_planning(request: Request):
         "tickets": tickets,
         "tickets_error": tickets_error,
     }
-    return templates.TemplateResponse(request, "admin_issues_planning.html", context)
+    return respond(request, "admin_issues_planning.html", context, model=AdminIssuesPlanningOut)