Update

.coveragerc

@@ -0,0 +1,12 @@
+# retoor <retoor@molodetz.nl>
+[run]
+source = devplacepy
+parallel = true
+sigterm = true
+omit =
+    tests/*
+    sitecustomize.py
+
+[report]
+show_missing = true
+skip_covered = false

.env.example

@@ -0,0 +1,43 @@
+# Copy to .env and adjust. Loaded by docker-compose (env_file) and by the app
+# at startup (python-dotenv). .env is git-ignored; this example is committed.
+
+# Session signing key. CHANGE THIS for any real deployment.
+SECRET_KEY=change-me
+
+# Database. Leave unset to use the shared project-root devplace.db (the Docker
+# app container bind-mounts ./ to /app, so it reads and writes the same file as
+# `make dev`). Set only to point at a different SQLite file.
+# DEVPLACE_DATABASE_URL=sqlite:////app/devplace.db
+
+# Persistent runtime data (zip archives, container workspaces). Lives OUTSIDE the
+# package and is never served via /static. Defaults to <repo>/var. The docker
+# daemon must be able to bind-mount this dir for container /app mounts; point it
+# at a persistent volume in production.
+# DEVPLACE_DATA_DIR=/var/lib/devplace
+
+# Container Manager (admin-only, enabled via docker-compose.containers.yml).
+# Host the /p/<slug> ingress proxy dials to reach a published container port.
+# On the host: 127.0.0.1 (default). Containerized app reaching host ports:
+# host.docker.internal.
+# DEVPLACE_CONTAINER_PROXY_HOST=host.docker.internal
+# GID of /var/run/docker.sock on the host (getent group docker | cut -d: -f3),
+# so the UID-1000 app can use the socket.
+# DOCKER_GID=999
+
+# Public origin for absolute URLs (SEO, canonical links, push). Empty = derive
+# from the request.
+DEVPLACE_SITE_URL=
+
+# Host port the nginx front door binds.
+PORT=10500
+
+# nginx upload ceiling. Must be >= the admin-configurable max_upload_size_mb.
+NGINX_MAX_BODY_SIZE=50m
+
+# Optional nginx micro-cache for proxied GETs.
+NGINX_CACHE_ENABLED=false
+NGINX_CACHE_MAX_SIZE=1g
+
+# Run the app container as this host user so shared files keep dev ownership.
+DEVPLACE_UID=1000
+DEVPLACE_GID=1000

.gitea/workflows/test.yaml

@@ -18,17 +18,34 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install -e .
-          pip install playwright
+          pip install -e ".[dev]"
           python -m playwright install chromium --with-deps
 
-      - name: Run integration tests
+      - name: Run integration tests with coverage
+        env:
+          COVERAGE_PROCESS_START: ${{ github.workspace }}/.coveragerc
+          PLAYWRIGHT_HEADLESS: "1"
         run: |
-          python -m pytest tests/ -v --tb=line -x
+          python -m coverage run -m pytest tests/ -n auto --dist loadfile --tb=line
+
+      - name: Build coverage report
+        if: always()
+        run: |
+          python -m coverage combine
+          python -m coverage report
+          python -m coverage html
+
+      - name: Publish coverage HTML
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-html
+          path: htmlcov/
 
       - name: Upload test screenshots
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: failure-screenshots
           path: /tmp/devplace_test_screenshots/
+

.gitignore

@@ -1,7 +1,31 @@
+.cache
+.local
+.devplace_bots/
 __pycache__/
 *.py[cod]
 *.egg-info/
 .env
+agents/reports/
 devplace.db*
+devplace-services.lock
+devplace-init.lock
+.vapid.lock
+notification-private.pem
+notification-private.pkcs8.pem
+notification-public.pem
 .pytest_cache/
 .opencode
+devii_*.db
+devii_*.db-shm
+devii_*.db-wal
+devii.log
+webdata/
+# Uploaded/downloaded files - never track in git
+devplacepy/static/uploads/
+# Runtime data dir (container workspaces, zip artifacts) - never inside the package
+var/
+
+# coverage
+.coverage
+.coverage.*
+htmlcov/

Dockerfile

@@ -1,22 +1,40 @@
-FROM python:3.11-slim
+FROM python:3.13-slim
 
 WORKDIR /app
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    curl \
+    curl ca-certificates \
     && rm -rf /var/lib/apt/lists/*
 
+# Optional: the docker CLI so the (admin-only) container manager can drive the host
+# docker daemon. Off by default; the container compose override turns it on.
+ARG INSTALL_DOCKER_CLI=false
+RUN if [ "$INSTALL_DOCKER_CLI" = "true" ]; then \
+        install -m 0755 -d /etc/apt/keyrings && \
+        curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && \
+        chmod a+r /etc/apt/keyrings/docker.asc && \
+        echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bookworm stable" > /etc/apt/sources.list.d/docker.list && \
+        apt-get update && apt-get install -y --no-install-recommends docker-ce-cli && \
+        rm -rf /var/lib/apt/lists/* ; \
+    fi
+
 COPY pyproject.toml .
 
 COPY devplacepy/ devplacepy/
 
-RUN pip install --no-cache-dir .
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
 
-RUN mkdir -p /app/data /app/devplacepy/static/uploads/attachments
+RUN pip install --no-cache-dir ".[bots]" \
+    && python -m playwright install --with-deps chromium \
+    && chmod -R a+rx /ms-playwright
+
+RUN mkdir -p /app/devplacepy/static/uploads/attachments
 
 EXPOSE 10500
 
+ENV DEVPLACE_WEB_WORKERS=2
+
 HEALTHCHECK --interval=30s --timeout=10s --retries=3 --start-period=10s \
     CMD curl -f http://localhost:10500/ || exit 1
 
-CMD ["uvicorn", "devplacepy.main:app", "--host", "0.0.0.0", "--port", "10500", "--workers", "4", "--backlog", "8192"]
+CMD ["uvicorn", "devplacepy.main:app", "--host", "0.0.0.0", "--port", "10500", "--workers", "2", "--backlog", "8192", "--proxy-headers", "--forwarded-allow-ips", "*"]

Makefile

@@ -5,45 +5,77 @@ LOCUST_DB ?= $(LOCUST_DB_DIR)/datastore.db
 LOCUST_USERS ?= 20
 LOCUST_SPAWN_RATE ?= 5
 LOCUST_RUN_TIME ?= 120s
+DEVPLACE_RATE_LIMIT ?= 1000000
+TEST_WORKERS ?= auto
+TEST_DIST ?= loadfile
 
-.PHONY: install dev clean test test-headed demo locust locust-headless
+.PHONY: install dev clean test test-serial test-headed test-headed-serial coverage coverage-headed coverage-html locust locust-headless
 
 install:
 	pip install -e .
 
 dev:
-	uvicorn devplacepy.main:app --reload --host 0.0.0.0 --port 10500 --backlog 4096
+	uvicorn devplacepy.main:app --reload --reload-dir devplacepy --host 0.0.0.0 --port 10500 --backlog 4096
 
 prod:
-	uvicorn devplacepy.main:app --host 0.0.0.0 --port 10500 --workers 2 --backlog 8192
+	DEVPLACE_WEB_WORKERS=2 uvicorn devplacepy.main:app --host 0.0.0.0 --port 10500 --workers 2 --backlog 8192 --proxy-headers --forwarded-allow-ips '*'
 
 test:
-	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/test_landing.py tests/test_auth.py tests/test_feed.py tests/test_post.py tests/test_profile.py tests/test_projects.py tests/test_messages.py tests/test_notifications.py tests/test_services.py tests/test_seo.py -v --tb=line -x
+	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/ -n $(TEST_WORKERS) --dist $(TEST_DIST) --tb=line -x
+
+test-serial:
+	PLAYWRIGHT_HEADLESS=1 python -m pytest tests/ -p no:xdist -v --tb=line -x
 
 test-headed:
-	PLAYWRIGHT_HEADLESS=0 python -m pytest tests/test_landing.py tests/test_auth.py tests/test_feed.py tests/test_post.py tests/test_profile.py tests/test_projects.py tests/test_messages.py tests/test_notifications.py tests/test_services.py -v --tb=line -x
+	PLAYWRIGHT_HEADLESS=0 python -m pytest tests/ -n $(TEST_WORKERS) --dist $(TEST_DIST) --tb=line -x
 
-demo:
-	PLAYWRIGHT_HEADLESS=0 python -m pytest tests/test_demo.py -v -s --tb=line -x
+test-headed-serial:
+	PLAYWRIGHT_HEADLESS=0 python -m pytest tests/ -p no:xdist -v --tb=line -x
+
+coverage:
+	rm -f .coverage .coverage.*
+	COVERAGE_PROCESS_START=$(CURDIR)/.coveragerc PLAYWRIGHT_HEADLESS=1 \
+		python -m coverage run -m pytest tests/ -n $(TEST_WORKERS) --dist $(TEST_DIST) --tb=line
+	python -m coverage combine
+	python -m coverage report
+
+coverage-headed:
+	rm -f .coverage .coverage.*
+	COVERAGE_PROCESS_START=$(CURDIR)/.coveragerc PLAYWRIGHT_HEADLESS=0 \
+		python -m coverage run -m pytest tests/ -p no:xdist --tb=line
+	python -m coverage combine
+	python -m coverage report
+
+coverage-html: coverage
+	python -m coverage html
+	@echo "Report written to htmlcov/index.html"
 
 locust:
 	export DEVPLACE_DATABASE_URL="sqlite:///$(LOCUST_DB)"; \
+	export DEVPLACE_RATE_LIMIT=$(DEVPLACE_RATE_LIMIT); \
+	fuser -k $(LOCUST_PORT)/tcp 2>/dev/null || true; \
+	sleep 1; \
 	mkdir -p $(LOCUST_DB_DIR); \
 	rm -f $(LOCUST_DB); \
 	uvicorn devplacepy.main:app --host 127.0.0.1 --port $(LOCUST_PORT) --backlog 8192 > /tmp/devplace_locust_server.log 2>&1 & \
 	PID=$$!; \
-	while ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
+	while kill -0 $$PID 2>/dev/null && ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
+	if ! kill -0 $$PID 2>/dev/null; then echo "Server failed to start (port $(LOCUST_PORT) busy?). See /tmp/devplace_locust_server.log"; exit 1; fi; \
 	locust --host http://127.0.0.1:$(LOCUST_PORT) --web-port $(LOCUST_WEB_PORT); \
 	kill $$PID 2>/dev/null || true; \
 	rm -rf $(LOCUST_DB_DIR)
 
 locust-headless:
 	export DEVPLACE_DATABASE_URL="sqlite:///$(LOCUST_DB)"; \
+	export DEVPLACE_RATE_LIMIT=$(DEVPLACE_RATE_LIMIT); \
+	fuser -k $(LOCUST_PORT)/tcp 2>/dev/null || true; \
+	sleep 1; \
 	mkdir -p $(LOCUST_DB_DIR); \
 	rm -f $(LOCUST_DB); \
 	uvicorn devplacepy.main:app --host 127.0.0.1 --port $(LOCUST_PORT) --backlog 8192 > /tmp/devplace_locust_server.log 2>&1 & \
 	PID=$$!; \
-	while ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
+	while kill -0 $$PID 2>/dev/null && ! curl -s http://127.0.0.1:$(LOCUST_PORT)/ > /dev/null 2>&1; do sleep 0.5; done; \
+	if ! kill -0 $$PID 2>/dev/null; then echo "Server failed to start (port $(LOCUST_PORT) busy?). See /tmp/devplace_locust_server.log"; exit 1; fi; \
 	locust --host http://127.0.0.1:$(LOCUST_PORT) --headless -u $(LOCUST_USERS) -r $(LOCUST_SPAWN_RATE) --run-time $(LOCUST_RUN_TIME) --html=$(LOCUST_DB_DIR)/report.html; \
 	kill $$PID 2>/dev/null || true; \
 	rm -rf $(LOCUST_DB_DIR)
@@ -54,19 +86,105 @@ clean:
 	rm -rf devplacepy.egg-info
 	rm -rf .venv
 
-.PHONY: docker-build docker-up docker-down docker-logs docker-clean
+# Container Manager works out of the box: the overlay installs the docker CLI in
+# the image and mounts the host socket. DOCKER_GID is read straight from the
+# socket so the UID-1000 app can use it; the data dir is the project's own var/
+# at its real host path, so the DooD bind-mount (host == container path) holds
+# with no /srv dir and no sudo.
+COMPOSE := docker compose -f docker-compose.yml -f docker-compose.containers.yml
+DEVPLACE_DATA_DIR ?= $(CURDIR)/var
+DOCKER_GID ?= $(shell stat -c '%g' /var/run/docker.sock 2>/dev/null)
+export DEVPLACE_DATA_DIR
+export DOCKER_GID
 
-docker-build:
-	docker compose build
+.PHONY: docker-build docker-up docker-down docker-logs docker-clean docker-prep ppy
 
-docker-up:
-	docker compose up -d
+# Build the single shared container image every instance runs. Build once;
+# rebuild only when ppy.Dockerfile, the sudo shim, or pagent change.
+ppy:
+	docker build --network=host -f ppy.Dockerfile -t ppy:latest devplacepy/services/containers/files
+
+docker-prep:
+	mkdir -p $(DEVPLACE_DATA_DIR)
+
+docker-build: docker-prep
+	$(COMPOSE) build
+
+docker-up: docker-prep
+	$(COMPOSE) up -d
 
 docker-down:
-	docker compose down
+	$(COMPOSE) down
 
 docker-logs:
-	docker compose logs -f
+	$(COMPOSE) logs -f
 
 docker-clean:
-	docker compose down -v
+	$(COMPOSE) down -v
+
+docker-bup: docker-build docker-up
+
+deploy:
+	git checkout production
+	git merge master
+	git push origin production
+
+# ---------------------------------------------------------------------------
+# Code validation. Dependency-free, no external tools: Python ast+py_compile,
+# JavaScript node --check, CSS brace balance, HTML/Jinja template parse.
+# ---------------------------------------------------------------------------
+.PHONY: validate
+
+validate:
+	python -m agents.validator .
+
+# ---------------------------------------------------------------------------
+# Maintenance agent fleet (see agents.md). Default mode is fix; pass CHECK=1
+# for read-only reporting with a non-zero exit on findings.
+#   make audit-agent          # autonomous fix
+#   make audit-agent CHECK=1   # report only
+#   make agents-all            # run the whole fleet
+#   make maestro               # talk to the conductor; it runs the rest
+# ---------------------------------------------------------------------------
+CHECK ?=
+AGENT_MODE := $(if $(CHECK),--check,--fix)
+
+.PHONY: security-agent audit-agent devii-agent docs-agent fanout-agent \
+        dry-agent style-agent frontend-agent seo-agent test-agent \
+        agents-all maestro
+
+security-agent:
+	python -m agents.security $(AGENT_MODE)
+
+audit-agent:
+	python -m agents.audit $(AGENT_MODE)
+
+devii-agent:
+	python -m agents.devii $(AGENT_MODE)
+
+docs-agent:
+	python -m agents.docs $(AGENT_MODE)
+
+fanout-agent:
+	python -m agents.fanout $(AGENT_MODE)
+
+dry-agent:
+	python -m agents.dry $(AGENT_MODE)
+
+style-agent:
+	python -m agents.style $(AGENT_MODE)
+
+frontend-agent:
+	python -m agents.frontend $(AGENT_MODE)
+
+seo-agent:
+	python -m agents.seo $(AGENT_MODE)
+
+test-agent:
+	python -m agents.test_coverage $(AGENT_MODE)
+
+agents-all:
+	python -m agents.orchestrator $(AGENT_MODE)
+
+maestro:
+	python -m agents.maestro

README.md

@@ -1,4 +1,4 @@
-# DevPlace — The Developer Social Network
+# DevPlace - The Developer Social Network
 
 Server-rendered social network for developers. FastAPI backend serving Jinja2 templates with ES6 interactivity. Avatar generation via Multiavatar (local, no network). SQLite via `dataset` with WAL mode and concurrency tuning.
 
@@ -11,7 +11,6 @@ make install        # pip install -e .
 make dev            # uvicorn --reload on port 10500
 make test           # Playwright integration + unit tests, headless, fail-fast
 make test-headed    # same tests in visible browser
-make demo           # full-journey GUI demo (headed)
 ```
 
 Open `http://localhost:10500`.
@@ -20,13 +19,13 @@ Open `http://localhost:10500`.
 
 | Layer | Technology |
 |-------|-----------|
-| Backend | Python 3.13+, FastAPI, Uvicorn (single worker) |
+| Backend | Python 3.13+, FastAPI, Uvicorn (multi-worker in production) |
 | Templates | Jinja2 (server-side rendered) |
 | Frontend | Pure ES6 JavaScript, one class per file |
 | Database | SQLite via `dataset` (auto-sync schema, `uid` PKs, WAL mode, 30s busy timeout) |
-| Auth | Session cookies, SHA256+SALT via passlib |
+| Auth | Session cookie, API key (`X-API-KEY`/Bearer), or HTTP Basic; PBKDF2-SHA256 via passlib |
 | Avatars | Multiavatar (local SVG generation, no external API, <5ms) |
-| Validation | `hawk` (Python/JS/CSS/HTML) |
+| Coverage | `coverage.py` (`.coveragerc`, subprocess-aware) |
 | Load testing | Locust (locustfile.py) |
 
 ## Project structure
@@ -38,9 +37,10 @@ devplacepy/
   database.py         # dataset connection, index creation
   templating.py       # Shared Jinja2 environment + globals
   avatar.py           # Multiavatar generation, URL builder
-  utils.py            # Password hashing, session mgmt, time_ago
+  utils.py            # Password hashing, session mgmt, time_ago, notification hook
   models.py           # Pydantic schemas
-  routers/            # One file per domain (auth, feed, posts, ...)
+  push.py             # Web push crypto, VAPID keys, encrypt/send/register
+  routers/            # One file per domain (auth, feed, posts, push, ...)
   templates/          # Jinja2 HTML templates
   static/css/         # Page-specific CSS files
   static/js/          # Application.js (ES6 module)
@@ -56,59 +56,555 @@ devplacepy/
 | `/news` | Developer news listing, detail page with comments |
 | `/posts` | Post detail, creation |
 | `/comments` | Comment creation, deletion |
-| `/projects` | Project listing, creation |
-| `/profile` | Profile view, editing |
+| `/projects` | Project listing, creation, owner editing (`POST /projects/edit/{slug}`), and per-project visibility toggles: `POST /projects/{slug}/private` (owner-only visibility) and `POST /projects/{slug}/readonly` (immutable files) |
+| `/projects/{slug}/files` | Per-project filesystem: directory and file CRUD, upload, inline editing, and line-range operations (`lines` read, `replace-lines`, `insert-lines`, `delete-lines`, `append`) for surgical edits to large text files (public read, owner write; all writes refused while the project is read-only) |
+| `/zips` | Zip job status (`/zips/{uid}`) and archive download (`/zips/{uid}/download`); archives are queued via `/projects/{slug}/zip` and `/projects/{slug}/files/zip` |
+| `/forks` | Fork job status (`/forks/{uid}`); forks are queued via `/projects/{slug}/fork`. Any signed-in user can fork a project they can view into a new project they own; once the job finishes the response carries the new project URL |
+| `/projects/{slug}/containers` | Admin per-project container manager: Dockerfile CRUD with immutable versions, async image builds, and container instance creation. Reachable from the project page via the admin-only **Containers** button |
+| `/admin/containers` | Admin **Containers** section: a list of every container instance across all projects (`/admin/containers`), each linking to a dedicated instance detail page (`/admin/containers/{uid}`) with lifecycle controls, live logs and metrics, an interactive terminal, schedules, ingress, and workspace sync |
+| `/p/{slug}` | Public ingress proxy (HTTP + WebSocket) to a running container instance's published port, opt-in per instance via `ingress_slug` |
+| `/profile` | Profile view, editing, and a public **Media** tab (`?tab=media`) showing every attachment a user uploaded, newest first |
+| `/media` | Per-attachment soft delete and restore: `POST /media/{uid}/delete` (owner or admin), `POST /media/{uid}/restore` (admin) |
+| `/uploads` | File upload endpoints: `POST /uploads/upload` (multipart), `POST /uploads/upload-url` (from URL); served at `/static/uploads/` |
+| `/admin/trash` | Admin **Trash**: review, restore, and permanently purge soft-deleted content (posts, comments, gists, projects, news, project files, attachments) across the platform |
+| `/notifications` | Notification list, mark read, live unread counts (`/notifications/counts`) |
 | `/messages` | Direct messaging |
-| `/notifications` | Notification list, mark read |
 | `/votes` | Upvote/downvote on posts, comments, projects |
+| `/reactions` | Emoji reactions on posts, comments, gists, projects |
+| `/bookmarks` | Save/unsave content; `/bookmarks/saved` personal list |
+| `/polls` | Vote on post-attached polls |
 | `/follow` | Follow/unfollow users |
+| `/leaderboard` | Contributor ranking by total stars earned |
 | `/avatar` | Multiavatar proxy with in-memory cache |
 | `/bugs` | Bug reports listing, creation |
-| `/services` | Background service monitoring (status, logs) |
+| `/admin/services` | Background service management (start/stop, config, status, logs) |
 | `/admin` | Admin panel (user management, news curation, settings) |
+| `/docs` | Developer documentation site with a complete, interactive HTTP API reference |
+| `/openai` | OpenAI-compatible LLM gateway service (`/openai/v1/chat/completions`, `/openai/v1/*`) |
+| `/devii` | Devii agentic assistant: WebSocket terminal (`/devii/ws`), standalone page, usage (`/devii/usage`), session bootstrap |
 | `(none)` | `/robots.txt`, `/sitemap.xml` (SEO) |
+| `(none)` | `/push.json` (VAPID key + subscribe), `/service-worker.js`, `/manifest.json` (push + PWA) |
+
+## Gamification
+
+Member progression is driven by activity and peer recognition.
+
+- **Stars** are the net vote score (`upvotes - downvotes`) on a post, project, or gist. A member's total stars is the sum across all their content and is the basis for ranking.
+- **XP and levels.** Members earn XP for contributing: posting (10), commenting (2), publishing a project (15) or gist (5), receiving an upvote (5), and gaining a follower (5). Each level requires 100 XP (`level = 1 + xp // 100`). The profile shows the current level and progress to the next.
+- **Badges** are awarded once per milestone: first post/comment/project/gist, 10 posts (Prolific), 25 stars (Rising Star), 100 stars (Star Author), 10 followers (Popular), a 7-day activity streak (On Fire), and reaching levels 5 and 10. Badges render with an icon and description on the profile.
+- **Leaderboard** (`/leaderboard`) ranks the top 50 members by total stars (single page, no pagination); a member's own rank is shown on their profile.
+- **Contribution heatmap and streaks.** Each profile shows a 12-month activity heatmap and the current/longest daily streak, derived from post/comment/gist/project timestamps (no extra storage).
+- **Social graph listings.** Each profile has Followers and Following tabs that paginate the follow graph (25 per page) and show a follow/unfollow control for each person. The same data is available as JSON at `GET /profile/{username}/followers` and `GET /profile/{username}/following`.
+- **Media gallery.** Each profile has a public Media tab: a responsive, paginated grid of every attachment that user uploaded across posts, projects, gists, comments, messages, bugs, and news, newest first. Images open in the shared lightbox. The owner can delete their own media and an admin can delete anyone's; deletion is a **soft delete** that hides the item everywhere (including its parent object) while preserving the file and the relation, so an admin can restore it from the **Media** trash at `/admin/media`.
+- **Reward notifications** fire when a member levels up or earns a badge.
+
+## Engagement
+
+- **Emoji reactions** - a fixed palette of reactions on posts, comments, gists, and projects, separate from voting and carrying no ranking weight.
+- **Polls** - a post can carry a poll (question plus up to six options); results appear as live bars once the viewer votes, one vote per member. A poll can be attached when the post is created or added later by editing a post that has none.
+- **Bookmarks** - save posts, gists, projects, and news to a personal list at `/bookmarks/saved`.
+- **Private projects** - an owner can mark a project private so it is visible only to them (and administrators) and excluded from listings, profiles, search, the sitemap, and zip access. Set at creation or toggled later from the project page.
+- **Read-only projects** - an owner can mark a project read-only, making its entire virtual filesystem immutable: every write, edit, line-edit, move, delete, and upload is refused from all paths (the web UI, the HTTP API, the Devii agent, and container workspace sync) until read-only is turned off. Devii may toggle read-only only after the user explicitly confirms.
+
+XP awards are wired at the existing content-creation, vote, and follow hook points in the routers and centralized in `award_xp()` / `check_milestone_badges()` (`devplacepy/utils.py`). Existing accounts have their XP and levels backfilled once from prior activity at startup (`init_db()`).
+
+## Admin: Audit Log
+
+Every state-changing action on the platform is recorded to an append-only audit log: account and authentication events, content and engagement, projects and files, news curation, admin governance, background services, container operations, ingress traffic, AI gateway calls, the Devii agent, the CLI, gamification side-effects, and security denials. Each row captures who acted, in which role, from where (origin, IP, user-agent), on which objects, what changed (old to new value), a sanitised summary, event-specific metadata, and whether the action succeeded, failed, or was denied. A companion table links every related object with a labelled relation. The complete event catalogue lives in `events.md`.
+
+The log is **administrator-only**. `/admin/audit-log` is a paginated, filterable list (by search text, category, event key, role, origin, result, and date range) styled like the rest of the admin panel; `/admin/audit-log/{uid}` is the per-event detail with the full row, pretty-printed metadata, and the related-object graph. Both negotiate HTML or JSON. Financial figures are recorded for completeness but only surfaced on this admin-only view. Recording is best-effort and never blocks the audited action. When the Devii agent performs a mutation it reuses the same event key with `origin=devii`/`via_agent=1`. Rows older than `audit_log_retention_days` (default 90) are pruned daily by the **Audit retention** background service.
 
 ## Configuration
 
 | Env var | Default | Purpose |
 |---------|---------|---------|
 | `DEVPLACE_DATABASE_URL` | `sqlite:///devplace.db` | Database connection string |
+| `DEVPLACE_DATA_DIR` | `<repo>/var` | Persistent runtime data dir, outside the package and not served via `/static` (zip archives, container workspaces). Point at a volume in production |
 | `SECRET_KEY` | hardcoded fallback | Session signing key |
+| `DEVPLACE_VAPID_SUB` | `mailto:retoor@molodetz.nl` | Contact address in the VAPID JWT `sub` claim |
+| `DEVPLACE_INTERNAL_BASE_URL` | `http://localhost:10500` | Base URL the platform's own services dial for the AI gateway |
+| `DEEPSEEK_API_KEY` / `OPENROUTER_API_KEY` | unset | Upstream provider keys; migrated into the gateway settings on first boot |
+
+### Runtime settings
+
+Operational behavior is tunable live from `/admin/settings` (stored in `site_settings`, no redeploy). The Operational group covers:
+
+| Setting | Default | Effect |
+|---------|---------|--------|
+| `site_url` | empty | Public origin for absolute links (SEO, container ingress `/p/<slug>`); empty derives from the request or `DEVPLACE_SITE_URL` |
+| `rate_limit_per_minute` | `60` | Mutating requests allowed per IP per window |
+| `rate_limit_window_seconds` | `60` | Rolling window for the request limit |
+| `session_max_age_days` | `7` | Standard session cookie lifetime |
+| `session_remember_days` | `30` | Remember-me session cookie lifetime |
+| `registration_open` | `1` | When `0`, new sign-ups are rejected |
+| `maintenance_mode` | `0` | When `1`, non-admins see the maintenance page; admins retain access |
+| `maintenance_message` | scheduled-maintenance text | Message shown during maintenance |
+| `customization_enabled` | `1` | When `0`, no user CSS/JS customization is injected on any page |
+| `customization_js_enabled` | `1` | When `0`, user custom CSS is still served but custom JavaScript is suppressed |
+| `audit_log_retention_days` | `90` | Audit rows older than this are pruned daily by the Audit retention service; `0` disables pruning |
+
+Numeric values are floored to safe minimums so an invalid entry cannot lock out writes or stall services. Consumers read via `get_setting`/`get_int_setting`, which fall back to these defaults when a row is absent.
+
+## Authentication
+
+The website uses a `session` cookie. For automation, every page and action also
+accepts three header-based methods, resolved centrally in `get_current_user`
+(`utils.py`) so they work everywhere with no per-route changes:
+
+- **API key** - `X-API-KEY: <key>`
+- **Bearer** - `Authorization: Bearer <key>`
+- **HTTP Basic** - `Authorization: Basic base64(username-or-email:password)`
+
+Every user has an `api_key` (a uuid7), set at signup and backfilled for existing
+users on startup (and via `devplace apikey backfill`). The key is shown on the
+profile page to its owner (and to admins for any user) with a live **Regenerate**
+button; regenerating invalidates the old key immediately. Invalid credentials
+return `401`; requests with no credentials behave as anonymous (browser redirect to
+login). Full details and copy-paste `curl` examples live at `/docs/authentication.html`.
+
+```bash
+curl -H "X-API-KEY: <your-key>" https://your-host/notifications
+curl -u <username>:<password> -X POST https://your-host/follow/<username>
+```
+
+CLI: `devplace apikey get <username>` / `reset <username>` / `backfill`.
+
+## Content negotiation (HTML or JSON)
+
+Every endpoint that renders a page or returns a redirect also speaks JSON, so anything the
+website does is automatable from the same URLs. A request gets JSON when it sends
+`Accept: application/json` or `Content-Type: application/json`; a normal browser navigation
+(`Accept: text/html`) always gets HTML, so existing behaviour is unchanged (the legacy
+`X-Requested-With: fetch` AJAX header still drives the four engagement endpoints only). JSON responses are defined by Pydantic models in `devplacepy/schemas.py` and built
+from the same context the templates use (sensitive user fields like `email`/`api_key`/
+`password_hash` are never exposed). Page GETs return the page payload; form actions return a
+uniform envelope `{ "ok": true, "redirect": "…", "data": {…} }`; errors return
+`{ "error": { "status", "message" } }` (validation → `422` with a `fields` map, unauthenticated
+JSON → `401`, non-admin → `403`). The core lives in `devplacepy/responses.py`
+(`wants_json`, `respond`, `action_result`). Full details: `/docs/conventions.html`.
+
+```bash
+curl -H "Accept: application/json" https://your-host/feed
+curl -H "Accept: application/json" -X POST -d "content=hi&title=T&topic=devlog" https://your-host/posts/create
+```
+
+## Documentation site
+
+`/docs` serves a server-rendered docs site with a feed-style left sidebar
+(`routers/docs.py`). `/docs/index.html` is the start page and `/docs/authentication.html`
+documents the auth methods; both are curated markdown pages rendered through the shared
+content renderer (markdown + highlight.js) with the current host and, when logged in, your
+own API key and username filled in.
+
+Every other endpoint is documented from a single source of truth, the `API_GROUPS` registry
+in `devplacepy/docs_api.py`. Each group becomes a reference page listing its endpoints with
+copy-paste cURL, JavaScript, and Python examples and an interactive "try it" panel
+(`static/js/ApiTester.js`) that executes the real call from the browser with your API key
+pre-filled. Each panel has a response-format selector (defaulting to JSON) that drives the
+`Accept` header for both the live request and the generated snippets, an always-visible
+**Expected** tab showing the modeled response, and a **Live response** tab for the real
+result. To document a new endpoint, add an entry to `docs_api.py`; the page, sidebar link,
+examples, runner, and expected-response sample are generated automatically. FastAPI's
+built-in Swagger is moved to `/swagger` so `/docs` belongs to this site. ReDoc is at `/redoc` and the raw schema at `/openapi.json`.
+
+Operator pages are admin-only: `/docs/admin.html` (user/news/settings administration) and
+`/docs/services.html` (Background Services) are hidden from the sidebar and return 404 for
+non-admins. The Background Services page is generated live from the service registry
+(`build_services_group()` over `service_manager.describe_all()`), so every registered service
+and its full configuration are documented automatically - including future services.
 
 ## Background Services
 
-`devplacepy/services/` provides a generic async service framework. Services start on server boot, run at configurable intervals, and are gracefully cancelled on shutdown.
+`devplacepy/services/` provides a generic async service framework. `/admin/services` is a slim index listing each service with its description and live status; clicking a service opens its detail page where everything for that service lives - start/stop, enable-on-boot, run-now, clear logs, adjustable log size, and live status/metrics - organized into **Overview / Configuration / Logs** tabs (configuration grouped into sections). State is stored in the database (desired state in `site_settings`, observed state in the `service_state` table), so control and status are correct even with multiple workers where only one runs the services.
 
 ### Service framework
 
-- **`BaseService`** — abstract class with async run loop, `deque(maxlen=20)` log buffer, `name`, `interval_seconds`
-- **`ServiceManager`** — singleton that registers, starts, and stops all services
-- **`NewsService`** — fetches news from `news.app.molodetz.nl/api`, grades with AI, stores articles >= threshold
+- **`ConfigField`** - declarative parameter spec (type, default, validation, secret) a service uses to declare its editable settings
+- **`BaseService`** - abstract class with a reconciling run loop that honors the persisted `enabled`/command/interval state, plus `config_fields`, `get_config()`, `describe()`, and a log buffer
+- **`ServiceManager`** - singleton: `register`, `describe_all`, `set_enabled`, `send_command`, `save_config`, `supervise`, `shutdown_all`
+- **`NewsService`** - fetches news from `news.app.molodetz.nl/api`, grades with AI, stores articles >= threshold
+- **`BotsService`** - Playwright fleet of AI personas that browse and interact with a DevPlace instance, with live cost/usage metrics (opt-in; install the `bots` extra)
+- **`GatewayService`** - OpenAI-compatible LLM gateway at `/openai/v1/*`, forwarding to DeepSeek (default) with optional vision augmentation; the single point of truth for AI that every other service routes through (enabled by default)
+- **`JobService` / `ZipService` / `ForkService`** - generic async job framework (`services/jobs/`) for heavy, blocking work run off the request path; `ZipService` builds project zip archives in a subprocess, `ForkService` copies a project into a new project owned by the forking user
+- **`ContainerService`** - the admin container manager (`services/containers/`): a reconciling supervisor for container instances, all running one shared prebuilt image
+
+### Container manager (admin only)
+
+`services/containers/` runs supervised container instances from the web UI, the HTTP API, and Devii. It drives the `docker` CLI via async subprocesses behind a pluggable `Backend` interface (a `DockerCliBackend` plus a `FakeBackend` for tests). **There is no in-app image building.** Every instance runs ONE shared prebuilt image, `ppy:latest` (override `DEVPLACE_CONTAINER_IMAGE`), built once with **`make ppy`** from `ppy.Dockerfile`: a Python + Playwright base with a broad set of common libraries preinstalled, the `pravda` (uid 1000) user, the sudo superclone, and `pagent` at `/usr/bin/pagent.py` all baked in. Creating an instance is then an instant `docker run` (it fails fast with a clear error if the `ppy` image has not been built yet). `ContainerService` reconciles desired instance state against `docker ps` each tick (containers are labeled `devplace.instance=<uid>`, so orphans are reaped and no state is lost), applies restart policies, fires cron/interval/one-time schedules, and samples metrics. The container's `/app` is bind-mounted to a persistent project workspace (materialized from the project files) and can be synced back; projects that need extra packages use runtime `pip install` (pravda owns the site-packages, no sudo needed) or `apt install` directly (the `pravda` user runs `apt`/`dpkg` through a fakeroot wrapper, so system packages install without root) or add the library to `ppy.Dockerfile` and rerun `make ppy`. A running instance can be **published** with an `ingress_slug`, making its service reachable (HTTP and WebSocket) at `/p/<slug>` through DevPlace. The manager is reached two ways: the admin **Containers** sidebar entry (`/admin/containers`) lists every instance across all projects and opens a dedicated detail page per instance, and each project page carries an admin-only **Containers** button to its own instance manager.
+
+**Security:** this requires mounting the Docker socket, which grants host root. Every run, exec, lifecycle, and schedule operation is administrator-only; `--privileged` is never used and all docker calls are argument-list subprocesses. The service is disabled by default; an admin enables **Containers** on `/admin/services`. CLI: `devplace containers list | reconcile | prune | prune-builds | gc-workspaces` (`prune-builds` is a one-time cleanup that removes legacy per-project images and the old dockerfiles/builds tables).
+
+**Runtime data** (container workspaces and zip archives) lives in `DEVPLACE_DATA_DIR` (default `var/`), **outside the package and never served via `/static`**. The docker daemon must be able to bind-mount the data dir for `/app`.
+
+### Async job framework and zip downloads
+
+`services/jobs/` is the standard way to run blocking work asynchronously and hand the caller a result URL. A shared `jobs` table is the queue (discriminated by `kind`); `queue.enqueue()` inserts a `pending` row from any worker, the lock-owning worker processes jobs in `JobService.run_once` (reap, recover orphans, refill up to a concurrency limit, prune expired), and status is polled from the database. Retention is built in: each job service deletes its own expired artifacts via a `cleanup` hook (default 7 days, admin-configurable). To add a kind, subclass `JobService`, set `kind`, and implement `process()` and `cleanup()`.
+
+`ZipService` archives a project (or a file/folder subtree) into `{crc32}.{slug}.zip` using the standalone `zip_worker` subprocess, with a CRC32 over the output. The frontend `app.zipDownloader` (any `data-zip-download` element, plus the files context menu) enqueues, polls `/zips/{uid}`, and downloads from `/zips/{uid}/download`. The job uid is the capability token, so anyone with the link can download. CLI: `devplace zips prune` (delete expired) / `devplace zips clear` (delete all).
+
+`ForkService` copies a project into a new project owned by the forking user. The **Fork** button on the project page (any signed-in user) prompts for a name; the job creates the destination project, duplicates the entire virtual filesystem, and records a directional `project_forks` relation so each fork shows a "Forked from X" link. The frontend `app.projectForker` enqueues, polls `/forks/{uid}`, and redirects to the new project once it is done; on failure the partially created project is rolled back. The forked project is permanent, so retention removes only the job tracking row. CLI: `devplace forks prune` / `devplace forks clear` (job rows only).
 
 ### Adding a service
 
-Create a class extending `BaseService`, override `run_once()`, register in `main.py`:
+Create a class extending `BaseService`, declare its `config_fields`, override `run_once()` (read parameters via `self.get_config()`), and register in `main.py`:
 ```python
 service_manager.register(YourService())
 ```
-The service appears at `/services` with live status and log tail.
+The service appears at `/admin/services` with controls, a generated config form, live status, and the log tail - no template, JS, or router changes.
 
 ### News service
 
 Fetches news from a configurable API, grades each article via AI, stores ALL articles in the `news` table (nothing is silently skipped). Articles with grade >= the configurable threshold are auto-published; the rest go to draft. Admin can manually publish/draft, toggle for landing page appearance, and delete. Images are extracted from article URLs.
 
-Configuration via admin site settings:
+Configuration on the Services tab (`/admin/services`):
 
-| Setting key | Default | Purpose |
-|-------------|---------|---------|
+| Parameter | Default | Purpose |
+|-----------|---------|---------|
 | `news_grade_threshold` | `7` | Minimum AI grade for auto-publish |
 | `news_api_url` | `https://news.app.molodetz.nl/api` | News source |
-| `news_ai_url` | `https://openai.app.molodetz.nl/v1/chat/completions` | AI grading endpoint |
-| `news_ai_model` | `molodetz` | AI model |
+| `news_ai_url` | `http://localhost:10500/openai/v1/chat/completions` | AI grading endpoint (the internal gateway) |
+| `news_ai_model` | `molodetz` | Generic model name; the gateway maps it to the real model |
+| `news_ai_key` | internal key | AI API key (`NEWS_AI_KEY` env, then the auto-generated gateway internal key) |
+| `news_service_interval` | `3600` | Seconds between fetch cycles (min 60) |
 
 News articles have detail pages at `/news/{slug}` with full comment support (same component as posts/projects). The landing page can display curated articles toggled from admin.
 
-CLI: `devplace news clear` — delete all news from local database.
+CLI: `devplace news clear` - delete all news from local database.
+
+### Bots service
+
+A Playwright-driven fleet of AI personas (`devplacepy/services/bot/`) that browse and interact with a DevPlace instance: posting, commenting, voting, reacting, creating gists/projects, filing bugs, following, and messaging. It is the former standalone `dpbot.py`, refactored into a package and managed entirely from the Services tab. Disabled by default.
+
+Each persona has its own voice and its own interests: post titles are written in the persona's voice rather than copied from the source headline, news topics and post categories are weighted by personality (so different personas react to different stories), shared code snippets pass a non-triviality quality gate, and bots discuss each other's posts in threaded conversations rather than reacting in isolation.
+
+Optionally (`bot_ai_decisions`), each bot is also given a unique AI-generated identity card (name, backstory, interests, temperament, verbosity, contrarianness, generosity, rhythm) and lets that identity decide every on-page action through the LLM instead of fixed probabilities: on each page the bot is shown the menu of actions actually available and the model returns an ordered plan. Behaviour then follows personality rather than chance, while every content-quality gate is retained. See `aibots.md` for the design and cost model.
+
+Install the extra and the browser binary, then enable it on `/admin/services`:
+
+```bash
+pip install -e ".[bots]"
+playwright install chromium
+```
+
+Configuration on the Services tab:
+
+| Parameter | Default | Purpose |
+|-----------|---------|---------|
+| `bot_fleet_size` | `1` | Number of concurrent bot accounts |
+| `bot_headless` | enabled | Run Chromium headless |
+| `bot_max_actions` | `0` | Stop a bot after N actions (0 = forever) |
+| `bot_base_url` | `https://pravda.education` | Target DevPlace instance |
+| `bot_api_url` | `http://localhost:10500/openai/v1/chat/completions` | LLM endpoint (the internal gateway) |
+| `bot_news_api` | `https://news.app.molodetz.nl/api` | Article source |
+| `bot_model` | `molodetz` | Generic model name; the gateway maps it to the real model |
+| `bot_api_key` | internal key | LLM key (defaults to the auto-generated gateway internal key) |
+| `bot_input_cost_per_1m` / `bot_output_cost_per_1m` | `0.27` / `1.10` | Token pricing for live cost tracking |
+| `bot_max_per_article` | `2` | How many bots may post about one article, each from a different angle |
+| `bot_article_ttl_days` | `7` | How long an article stays covered before it can be posted again |
+| `bot_gist_min_lines` | `6` | Reject generated snippets shorter than this many non-empty lines |
+| `bot_action_pause_min_seconds` / `bot_action_pause_max_seconds` | `5` / `30` | Idle pause window a bot takes after each action |
+| `bot_break_scale` | `1.0` | Multiplier on between-session breaks (below 1 = more active and costlier) |
+| `bot_ai_decisions` | disabled | Let each bot's AI-generated identity decide every action via the LLM instead of fixed probabilities (one decision call per page); see `aibots.md` |
+| `bot_decision_temperature` | `0.4` | Sampling temperature for the per-page decision call |
+
+The Services tab shows live fleet metrics (bots running, total cost, LLM calls, tokens, posts/comments/votes) and a per-bot breakdown that update on the page poll. Each bot keeps a stable identity and cumulative cost in `~/.devplace_bots/state_slot{N}.json`.
+
+### LLM gateway service
+
+`GatewayService` (`devplacepy/services/openai_gateway/`) exposes an OpenAI-compatible
+endpoint at `/openai/v1/chat/completions` (plus a transparent `/openai/v1/*`
+passthrough). It forwards to the configured upstream (DeepSeek by default) and, when a
+request carries image content, first describes the image(s) via a vision model
+(OpenRouter/Gemma by default) and inlines the description as text - so a vision-less
+upstream can still answer. Enabled by default and configurable on `/admin/services`.
+
+It is the **single point of truth for AI**: the news, bots, and guest Devii sessions call this
+gateway by default instead of an external provider, send the generic model name `molodetz`,
+and authenticate with an auto-generated internal key. The real provider URLs, models, and keys
+live only here, so providers and backends are switched in one place. `DEEPSEEK_API_KEY` and
+`OPENROUTER_API_KEY` are migrated into the editable settings on first boot, and the value in use
+is shown. The internal base URL the other services dial is `DEVPLACE_INTERNAL_BASE_URL`
+(default `http://localhost:10500`).
+
+Authenticated users may call the gateway with their own API key (the `Allow users` setting,
+on by default), and Devii operating a signed-in user authenticates its LLM calls with that
+user's key. Every call is therefore attributed to the user, so their full AI spend (through
+Devii or direct API calls) is tracked and limitable per user. Administrators see a per-user
+"AI usage (last 24h)" panel on each profile page - request volume, success and error rates,
+token totals, cost, average latency and throughput, a per-model breakdown, a 30-day cost
+projection, and a quota bar - served from `GET /admin/users/{uid}/ai-usage`. A non-admin user
+sees only one figure on their own profile: the percentage of their daily AI quota used so far.
+The same rule governs the Devii assistant: monetary figures (cost, pricing, spend, limit) are
+disclosed only to administrators, while members and guests can see only the percentage of their
+24-hour quota used. The `/devii/usage` endpoint returns that percentage and the day's turn count to
+everyone, and includes dollar figures only for administrators.
+
+Configuration on the Services tab:
+
+| Parameter | Default | Purpose |
+|-----------|---------|---------|
+| `gateway_upstream_url` | `https://api.deepseek.com/chat/completions` | Where requests are forwarded |
+| `gateway_model` | `deepseek-v4-flash` | Real model sent upstream (what `molodetz` maps to); 1M-token context, 384K max output |
+| `gateway_force_model` | on | Override the client-requested model (and the `molodetz` alias) |
+| `gateway_api_key` | migrated from env | Upstream key, shown and editable (auto-migrated from `DEEPSEEK_API_KEY`/`OPENROUTER_API_KEY`) |
+| `gateway_instances` | `4` | Max concurrent upstream forwards per worker (pool + semaphore) |
+| `gateway_timeout` | `300` | Upstream timeout (seconds); minimum five minutes; also bounds the vision describe-image call |
+| `gateway_vision_enabled` / `_url` / `_model` / `_key` / `_cache_size` | on / OpenRouter / Gemma / env / 256 | Image-description augmentation |
+| `gateway_require_auth` | on | When off, the gateway is open |
+| `gateway_allow_admins` / `gateway_allow_users` | on / off | Which DevPlace users may call it (any auth scheme) |
+| `gateway_access_key` | empty | A standalone key (sent as `X-API-KEY`/Bearer) that always grants access |
+| `gateway_internal_key` | auto (uuid4) | Auto-generated on boot; DevPlace's own services authenticate with this. Clear and restart to rotate |
+| `gateway_price_cache_hit_per_m` / `_cache_miss_per_m` / `_output_per_m` | 0.0028 / 0.14 / 0.28 | Chat cost per 1M tokens, used when the upstream returns no native cost (DeepSeek) |
+| `gateway_vision_price_input_per_m` / `_output_per_m` | 0 / 0 | Vision cost per 1M tokens, used only when the vision upstream returns no native cost |
+| `gateway_max_retries` / `gateway_retry_backoff_ms` | 2 / 250 | Retry attempts and linear backoff on timeout, connection error, or upstream 5xx |
+| `gateway_circuit_threshold` / `gateway_circuit_cooldown_seconds` | 5 / 30 | Consecutive failures before the circuit breaker opens, and its cooldown |
+| `gateway_usage_retention_hours` | 720 | How long per-call usage rows are kept before pruning (30 days) |
+| `gateway_model_context_map` | JSON map | Model to max-context-tokens map for context-window utilization tracking |
+
+Callers authenticate with the static `gateway_access_key`, the auto-generated
+`gateway_internal_key` (used by the platform's own services), or - when allowed - their
+DevPlace credentials via any scheme (`X-API-KEY`, Bearer, Basic). The endpoint is
+exempt from the per-IP rate limit and the maintenance gate, and its concurrency is
+bounded by `gateway_instances`; scale further with uvicorn workers. The Services tab
+shows live request/error/latency/vision-call counters.
+
+#### Usage and cost metrics
+
+Every upstream call (chat, vision, and passthrough) is recorded to `gateway_usage_ledger`
+with its tokens, cost, latency breakdown, status, and caller. Cost is taken from the
+upstream native `cost` field when present (OpenRouter) and computed from the configured
+per-million pricing otherwise (DeepSeek, which reports no cost). The admin **AI usage**
+page (`/admin/ai-usage`) reports per-hour and 24h metrics - request volume and throughput,
+token usage with averages and percentiles (p50/p90/p95/p99), latency (upstream round-trip,
+gateway overhead, semaphore queue wait, connection establishment), error rates by category,
+cost (per model, per caller, input vs output, projected monthly burn, caching savings),
+caller behavior, and an hourly breakdown - divided and combined. The gateway also retries
+transient upstream failures and opens a circuit breaker after repeated failures. Time to
+first token and inter-token latency are not reported because the gateway forwards
+non-streaming to the upstream.
+
+```bash
+curl -H "X-API-KEY: <gateway_access_key>" \
+  -d '{"messages":[{"role":"user","content":"hi"}]}' \
+  https://your-host/openai/v1/chat/completions
+```
+
+### Devii assistant service
+
+`DeviiService` (`devplacepy/services/devii/`) is an in-platform agentic assistant exposed
+as a WebSocket terminal at `/devii/ws`, reachable from the **Devii** item in the user
+dropdown menu and embedded on the documentation page for guests. It runs a ReAct loop
+(planning, reflection, a verification gate, self-learning memory, autonomous tasks) over a
+declarative tool catalog. A signed-in user's Devii operates their **own** DevPlace account -
+the agent authenticates to this instance with the user's `api_key` - while guests get a
+sandbox session. Disabled by default; enable and configure it on `/admin/services`.
+
+Sessions are durable. A logged-in user's conversation is keyed by user uid, shared live
+across every tab, window, and device (each turn is broadcast to all connected sockets), and
+persisted to `devii_conversations` so it survives a server restart - on reconnect the prior
+conversation is rehydrated. Guests get a non-persistent session keyed by a `devii_guest`
+cookie. To keep one logical session and one cost/broadcast source per user, the `/devii/ws`
+endpoint is served only by the worker that holds the background-service lock; other workers
+close the socket so the client reconnects to the owning worker.
+
+Spend is capped per owner over a rolling 24 hours. Every turn appends a row to
+`devii_usage_ledger` (the authoritative source for the cap - the in-memory cost tracker is
+display-only) and an audit row to `devii_turns`. The cap is checked before each turn.
+
+Configuration on the Services tab:
+
+| Parameter | Default | Purpose |
+|-----------|---------|---------|
+| `devii_ai_url` | `https://openai.app.molodetz.nl/v1/chat/completions` | OpenAI-compatible reasoning endpoint |
+| `devii_ai_model` | `molodetz` | Model name |
+| `devii_ai_key` | env fallback (`DEVII_AI_KEY`) | AI API key |
+| `devii_base_url` | this instance's origin | Platform Devii drives via each user's API key |
+| `devii_plan_required` / `devii_verify_required` | on / on | Enforce plan-first and verify-after-mutation |
+| `devii_max_iterations` | `40` | Tool-loop iterations per turn |
+| `devii_timeout` | `300` | Read timeout (seconds) for each AI chat-completions call; minimum five minutes |
+| `devii_fetch_timeout` | `300` | Read timeout (seconds) for the fetch tool; minimum five minutes |
+| `devii_user_daily_usd` | `1.0` | Rolling 24h spend cap per signed-in user |
+| `devii_guest_daily_usd` | `0.05` | Rolling 24h spend cap per guest |
+| `devii_admin_daily_usd` | `0.0` | Rolling 24h spend cap per administrator (`0` = unlimited; admins are exempt by default) |
+| `devii_guests_enabled` | on | Allow anonymous guests |
+| `devii_price_cache_hit` / `_cache_miss` / `_output` | `0.0028` / `0.14` / `0.28` | USD per 1M tokens (drives the cap) |
+| `devii_rsearch_enabled` | on | Enable the external web search tools (`rsearch_*`) |
+| `devii_rsearch_url` | `https://rsearch.app.molodetz.nl` | Base URL of the web search service those tools call |
+| `devii_rsearch_timeout` | `300` | Read timeout (seconds) for `rsearch_*` calls; web-grounded answers can take minutes; minimum five minutes |
+
+Beyond the platform tools, Devii has external **web** tools. `fetch_url` reads a web page;
+`http_request` makes an arbitrary HTTP call (GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS) to any
+external REST/JSON API with custom headers and a JSON, form, or raw body, returning the full
+response (non-2xx is returned rather than raised so API errors are readable). Both carry an SSRF
+guard (private and loopback addresses are refused) and a size cap. `attach_url` downloads a public
+URL on the server and stores it as a real attachment through the same pipeline as a direct upload,
+returning a uid Devii then passes in `attachment_uids` when creating a post, project, gist, comment,
+bug, or message - so a user can ask Devii to attach an image straight from the internet. Devii also
+has external **web search** tools - `rsearch` (web/image search), `rsearch_answer` (a web-grounded
+AI answer with sources), `rsearch_chat` (direct AI chat), and `rsearch_describe_image` (vision).
+These reach an external public service rather than this platform, so platform tools are always
+preferred; Devii uses them only when the user explicitly asks to search the web or an outside
+source. They are gated by `devii_rsearch_enabled` and the service URL is configurable via
+`devii_rsearch_url`.
+
+A `devii` console script ships the same agent as an interactive terminal:
+
+```bash
+pip install -e .
+devii --api-key <your DevPlace api_key> --base-url https://your-host
+devii -p "List my unread notifications as a bullet list."   # one-shot
+```
+
+### Site customization (per-user CSS/JS)
+
+Each user can reshape the site to taste by injecting their own **CSS** (look) and
+**JavaScript** (behaviour), scoped either to a single **page type** (every post page, every
+project page) or **globally** (every page). Customizations apply only to that user's own
+sessions; a signed-out guest gets customizations scoped to their `devii_guest` session cookie.
+This is the user's own code running in their own browser, like a userscript, so it never affects
+anyone else's view.
+
+It is configured conversationally through Devii. Ask for a change and Devii previews it live in
+your browser, then before saving it asks whether to apply it to the current page type or the
+whole site, and only persists once you confirm. Overrides are stored in the `user_customizations`
+table (one row per owner, scope, and language; cached with cross-worker invalidation) and injected
+server-side: custom CSS is the last `<style>` in `<head>` so it overrides the design system, and
+custom JavaScript runs after the application boots with access to the global `app`. Devii tools:
+`customize_list`, `customize_get`, `customize_set_css`, `customize_set_js`, and `customize_reset`
+(restores the default look and behaviour). Operators can disable the feature site-wide with the
+`customization_enabled` / `customization_js_enabled` settings.
+
+Your profile page carries two toggle buttons that suppress your customizations without deleting
+them: one for the site-wide (global) CSS and JS, one for the per-page-type CSS and JS. Disabling
+keeps the stored code and re-enables instantly, which is the quick way to recover a page broken by
+your own custom code. You see the toggles only on your own profile; an administrator can also
+operate them on any user's profile (or call the Devii `customize_set_enabled` tool) to recover a
+user who locked themselves out.
+
+### User-defined tools (vibe new functionality)
+
+Users can invent new Devii capabilities by describing them in natural language, with no code. Telling
+Devii "when I say woeii, search the web for a cat picture matching my description and set it as my
+page background" makes it call `tool_create` and store a virtual tool. The tool then appears in
+Devii's own toolset, and on the next turn "woeii orange" calls it: the handler re-runs Devii itself
+(a self-evaluating sub-agent) on the stored prompt plus the user's input, with full tool access, and
+returns the result. Each tool takes a single free-form `input` string; the stored prompt directs what
+to do with it.
+
+Virtual tools are owner-scoped (persistent in the DB for signed-in users in `devii_virtual_tools`,
+session-only for guests) and never run in anyone else's session. The full CRUD is exposed as Devii
+tools: `tool_create`, `tool_list`, `tool_get`, `tool_update` (including enable/disable), and
+`tool_delete`. A general `eval(prompt)` tool runs Devii on any prompt and returns the result; it is
+the same engine that powers virtual tools. Self-evaluation depth is bounded so a tool cannot loop by
+calling itself.
+
+## Maintenance agents
+
+DevPlace ships a fleet of autonomous AI maintenance agents under `agents/` that
+keep the codebase consistent. Each agent owns one quality dimension, sweeps the
+whole repository for problems in that dimension, and either reports them or fixes
+them and verifies the build. The internals are documented in `AGENTS.md`; the
+public, newbie-friendly guide is at `/docs/maintenance-agents.html` and
+`/docs/maintenance-usage.html`.
+
+Every agent runs in two modes: `--fix` (default, autonomous) or `--check`
+(report only, non-zero exit on findings). The agents share one proven engine
+(`agents/agent.py`) and verify their own work with `agents/validator.py`, a
+dependency-free validator (Python `ast`/`py_compile`, JavaScript `node --check`,
+CSS brace balance, HTML and Jinja templates parsed with Jinja2).
+
+| Agent | Dimension |
+|-------|-----------|
+| `maestro` | Conversational conductor; runs the right agent or the whole fleet for you |
+| `security` | Route authorization, private-data and admin guards, input validation |
+| `audit` | Every state change leaves an audit-log entry |
+| `devii` | Devii capability parity and role-gated tool visibility |
+| `docs` | CLAUDE.md / AGENTS.md / README / `/docs` accuracy and role-aware visibility |
+| `fanout` | A feature is wired across every layer (form, schema, Devii tool, docs, SEO) |
+| `dry` | Duplication removed; shared helpers reused |
+| `style` | Naming, headers, typing, formatting rules |
+| `frontend` | ES6 modules, web components, CSS structure |
+| `seo` | Search metadata and sitemap coverage |
+| `test` | Integration-test coverage (writes tests, never runs the suite) |
+
+```bash
+make maestro                 # talk to the conductor; it runs the rest
+make validate                # validate the whole codebase (no external tools)
+make audit-agent             # fix one dimension autonomously
+make audit-agent CHECK=1     # report only
+make agents-all              # run the whole fleet
+make agents-all CHECK=1      # CI gate across the fleet
+```
+
+Every run is legible live: each agent prints a start banner (icon, name, a
+memorable codename, what it will do, and its report path), every line is prefixed
+with the wall-clock time and elapsed duration, each AI call streams its running
+dollar cost, and every file edit streams a unified diff. Reports land in
+`agents/reports/` as `<agent>-<codename>-<date>.{json,md}` (the orchestrator writes
+`fleet-<codename>-<date>.json`). Safety is enforced in code, not prompts: agents
+cannot edit anything under `agents/`, cannot shell out, and cannot modify more than
+a fixed number of files per run, and a run that exhausts its step budget is marked
+`incomplete` and exits non-zero rather than looking clean.
+
+## Push notifications & PWA
+
+Authenticated users can receive native web push notifications, and the site is an
+installable Progressive Web App. Push uses only standard libraries (`cryptography`,
+`PyJWT`, `httpx`) against the Web Push Protocol - no third-party push wrapper.
+
+### Events
+
+Every event that already produces an in-app notification also sends a web push,
+because both share a single funnel - `create_notification()` in `utils.py`:
+
+| Event | Recipient |
+|-------|-----------|
+| Direct message received | receiver |
+| Comment on your post | post author |
+| Reply to your comment | comment author |
+| `@mention` in any content | mentioned user |
+| Upvote on your content | content owner |
+| New follower | followed user |
+
+`create_notification` schedules delivery as a fire-and-forget async task, so a dead
+subscription or push-service error never blocks the triggering request. Delivery
+(`push.notify_user`) iterates a user's subscriptions, encrypts the payload
+(legacy `aesgcm` content encoding), and POSTs to each endpoint; subscriptions that
+return `404`/`410` are soft-deleted.
+
+### VAPID keys
+
+The server identity is three PEM files generated once at startup in the repository
+root: `notification-private.pem`, `notification-private.pkcs8.pem`,
+`notification-public.pem`. They are git-ignored.
+
+**These keys are the application's identity to the push services. If they are lost or
+regenerated, every existing subscription becomes permanently undeliverable.** Persist
+them across deployments and back them up; do not regenerate them.
+
+### Opt-in
+
+The browser requires the first permission prompt to originate from a user gesture, so
+opt-in is exposed as a button in both the top navigation (bell-with-slash icon) and on
+the `/notifications` page. After opt-in, the subscription is refreshed silently on
+every page load. `PushManager.js` owns registration, subscription, and the opt-in UI.
+
+### PWA
+
+`manifest.json` (192/512 and maskable icons), `service-worker.js`, and an install
+button (`PwaInstaller.js`) make the app installable. The service worker uses a
+network-first strategy for navigations and falls back to `static/offline.html` when
+offline. Installation requires a secure origin (HTTPS, or `localhost` for development).
+
+| File | Role |
+|------|------|
+| `devplacepy/push.py` | VAPID keys, payload encryption, send, register |
+| `devplacepy/routers/push.py` | `/push.json`, `/service-worker.js`, `/manifest.json` |
+| `static/js/PushManager.js` | Service-worker registration + subscribe + opt-in UI |
+| `static/js/PwaInstaller.js` | `beforeinstallprompt` capture + install button |
+| `static/service-worker.js` | Receives push, shows notification, offline fallback |
+| `static/manifest.json` | PWA manifest (icons, display, theme) |
+| `static/offline.html` | Offline fallback page |
 
 ## Database
 
@@ -123,48 +619,122 @@ PRAGMA temp_store=MEMORY;      -- temp tables in memory
 PRAGMA mmap_size=268435456;    -- 256MB memory map for reads
 ```
 
-All indexes are created via `CREATE INDEX IF NOT EXISTS` wrapped in try/except — safe to run on every startup regardless of table state.
+All indexes are created via `CREATE INDEX IF NOT EXISTS` wrapped in try/except - safe to run on every startup regardless of table state.
+
+SQLite is synchronous by design and will never be made async. It is more than fast enough for this platform: the database is a local file with WAL, a 30s busy timeout, and a 256MB memory map, so queries are sub-millisecond. The synchronous database layer is a deliberate architectural decision, not a limitation, and is not subject to change.
+
+### Soft delete and data retention
+
+Removing a record is a **soft delete**, not a physical one: it stamps `deleted_at` (timestamp) and `deleted_by` (actor) instead of erasing the row, and every list/count read filters `deleted_at IS NULL`, so the item disappears from the product while remaining recoverable. This covers user content (posts, comments, gists, projects, news, project files, attachments), engagement toggles (votes, reactions, bookmarks, follows, poll votes - which revive the same row when re-toggled), sessions (logout), container instances, and per-owner Devii/customization data. Deletions cascade with a shared timestamp so the whole event restores or purges as a unit. Garbage-collection operations (job retention, metrics ring buffers, usage-ledger pruning, expired-session cleanup) stay hard deletes - that is the stage that frees storage. Administrators review, restore, and permanently purge soft-deleted content from **Trash** at `/admin/trash`; the columns are indexed (`idx_<table>_deleted`).
+
+A member may delete only their own content, but an **administrator may delete any member's** post, comment, gist, project, project file, or attachment - the owner-or-admin check lives on each delete endpoint, so it applies equally to the web UI and to the Devii assistant (which acts purely through the platform API as the signed-in user). When an admin's Devii is asked to delete something it requires explicit confirmation before each deletion, and the result is the same soft delete, restorable from Trash.
 
 ## Testing
 
-- **148 tests** across 14 files: Playwright integration + unit tests
-- Playwright (NOT pytest-playwright plugin — conflicts, uninstall it)
-- Server starts as subprocess on port 10501 with isolated temp database
+- **849+ tests** across 63 files: Playwright integration + in-process unit tests
+- Playwright (NOT pytest-playwright plugin - conflicts, uninstall it)
+- Runs in parallel via pytest-xdist (`make test` = `-n auto --dist loadfile`); each worker is an isolated uvicorn subprocess on port 10501+ with its own temp database and `DEVPLACE_DATA_DIR`
+- `loadfile` keeps every test in a file on one worker so session fixtures and ordering hold; cap workers with `make test TEST_WORKERS=4`
+- `make test-serial` runs everything in one process for debugging
 - Test users `alice_test` / `bob_test` seeded via HTTP at session start
 - Tests stop at first failure (`-x` flag)
 - Failure screenshots auto-save to `/tmp/devplace_test_screenshots/`
-- Headed mode: `PLAYWRIGHT_HEADLESS=0 make test`
+- Headed mode: `make test-headed` (parallel, one Chromium window per worker); `make test-headed-serial` for a single window
+- Cap headed windows the same way: `make test-headed TEST_WORKERS=2`
 
 ### Key test patterns
 
-- Every `page.goto()` and `page.wait_for_url()` uses `wait_until="domcontentloaded"` — avatar images don't block test execution
+- Every `page.goto()` and `page.wait_for_url()` uses `wait_until="domcontentloaded"` - avatar images don't block test execution
 - Session-scoped browser context with per-test cookie clearing
 - `page.locator(...).wait_for(state="visible")` preferred over bare selectors
 
 ## Avatars
 
-Uses [Multiavatar](https://github.com/multiavatar/multiavatar-python) — generates deterministic SVG avatars locally from a seed string (the username). No external API calls, no network dependency. Generation takes <5ms. Results are cached in-memory (cleared on server restart).
+Uses [Multiavatar](https://github.com/multiavatar/multiavatar-python) - generates deterministic SVG avatars locally from a seed string (the username). No external API calls, no network dependency. Generation takes <5ms. Results are cached in-memory (cleared on server restart).
 
 Avatar URL format: `/avatar/multiavatar/{username}?size={size}`
 
+## Deployment
+
+Production runs on a single host via Docker Compose: an **app** container (Uvicorn, 2 workers) behind an **nginx** container that terminates HTTP, serves static assets, and proxies the rest.
+
+### Shared database and uploads
+
+The app container bind-mounts the host project directory (`./` to `/app`) and runs as the host user (`DEVPLACE_UID`/`DEVPLACE_GID`, default `1000`). Because `config.py` resolves the database to an absolute path under the project root, the container reads and writes the **same `devplace.db`** as `make dev`, plus the same `static/uploads/`, `devii_lessons.db`, `devii_tasks.db`, VAPID keys, and `devplace-services.lock`. SQLite WAL mode allows the concurrent access, and the file lock (`fcntl.flock` on `devplace-services.lock`) guarantees only one process - dev or prod - runs the background services (news, bots, Devii hub), so they never double-fire.
+
+This requires prod and dev to be on the **same host/filesystem**; SQLite is a local-file database and cannot be shared across machines. No `DEVPLACE_DATABASE_URL` override is set, so nothing diverges.
+
+### First deploy
+
+```bash
+cp .env.example .env        # set SECRET_KEY; adjust PORT, DEVPLACE_SITE_URL
+make docker-build           # build the image (installs the docker CLI)
+make docker-up              # start (creates ./var, mounts the socket)
+```
+
+Open `http://<host>:${PORT}` (default 10500). `make docker-logs` tails output; `make docker-down` stops.
+
+`make docker-build` and `make docker-up` always apply the `docker-compose.containers.yml` overlay, so the admin-only Container Manager works with no extra setup. Mounting the docker socket grants the app **root on the host**; the feature itself stays disabled until an admin enables **Containers** on `/admin/services` (and the `ppy` image is built once with `make ppy`), but treat the whole deployment as trusted-admins-only. Always update through the make targets - a plain `docker compose up -d` drops the overlay and silently removes the socket and CLI.
+
+### Updating
+
+```bash
+git pull
+make docker-up              # restart with new code (bind-mounted, no rebuild)
+make docker-build && \
+  make docker-up            # only when dependencies in pyproject.toml change
+```
+
+The `make deploy` target fast-forwards the `production` branch (`git checkout production && git merge master && git push origin production`); pull that branch on the server.
+
+### Container Manager wiring (what the overlay does)
+
+The Container Manager drives the host Docker daemon, so `make docker-build`/`make docker-up` apply `docker-compose.containers.yml` automatically. The make targets derive everything the overlay needs - `DOCKER_GID` from the socket and `DEVPLACE_DATA_DIR` as the project's own `./var` at its real host path - so no `sudo`, no `/srv` dir, and no manual `.env` editing are required. (Override `DEVPLACE_DATA_DIR` or `DOCKER_GID` on the make command line to point elsewhere.)
+
+What the overlay (`docker-compose.containers.yml`) changes:
+
+- **Docker CLI in the image** via the `INSTALL_DOCKER_CLI=true` build arg (the base image stays lean).
+- **Docker socket** mounted into the app container. This grants the app **root on the host** - every build/run/exec is admin-only, `--privileged` is never used, and all docker calls are argument-list subprocesses, but treat the whole feature as trusted-admins-only.
+- **Socket permissions:** the app runs as UID 1000, so the overlay adds the host `docker` group via `group_add`. `make` reads the gid straight from `/var/run/docker.sock` (`stat -c '%g'`), the exact group that owns the socket.
+- **Data dir at a consistent path (critical).** When the app (in its container) runs `docker run -v <path>:/app`, the daemon resolves `<path>` against the **host**, not the app container. So the workspace/data dir must be mounted at the **same absolute path** on host and in the container - the make targets set `DEVPLACE_DATA_DIR` to the project's `./var` (an absolute host path) and mount it at that identical path on both sides. (Build contexts go through the docker API as a tarball, so they can stay in the container's temp dir - only the `/app` bind mount needs path consistency.)
+- **Ingress reach:** published container ports live on the **host**, so the overlay sets `DEVPLACE_CONTAINER_PROXY_HOST=host.docker.internal` (with `extra_hosts: host-gateway`) so the `/p/<slug>` proxy can reach them. On a bare-metal `make prod` deploy the app is already on the host, so the default `127.0.0.1` works and no overlay is needed (just install the docker CLI and run the services).
+
+Then enable **Container builds** and **Containers** on `/admin/services`. Builds default to `--network=host` (configurable on the service) so pip can reach PyPI; set the build network to empty to use the docker default.
+
+### nginx specifics
+
+- **WebSockets:** `/devii/ws` (the Devii terminal) and `/p/` (container ingress) are proxied with `Upgrade`/`Connection` headers and 1h timeouts; `/p/` also disables buffering for streaming.
+- **Uploads:** `/static/uploads/` is served with `X-Content-Type-Options: nosniff` and a `Content-Disposition` that mirrors the app's `UploadStaticFiles` control: known-safe media (images, video, audio) is served `inline` so it embeds and plays in the page, while every other type is forced to `attachment` so it downloads instead of rendering (stored-XSS defense). SVG is deliberately excluded from the inline set.
+- **Upload size:** `NGINX_MAX_BODY_SIZE` (default `50m`) must be **>= the admin-configurable `max_upload_size_mb`** (Admin -> Settings), or large uploads are rejected with HTTP 413 before reaching the app.
+- **Micro-cache:** off by default; enable with `NGINX_CACHE_ENABLED=true`.
+
+### Bare-metal alternative
+
+`make prod` runs the same app without containers (`uvicorn ... --workers 2 --proxy-headers`) from the project root, sharing the identical database and files. Note it binds port 10500, so it conflicts with the Docker front door on the same port - run one, or set a different `PORT`.
+
+### Multi-worker safety
+
+The app runs correctly under multiple Uvicorn workers (independent processes sharing only the DB and lock files). Cross-worker invalidation of the auth and settings caches is coordinated through a `cache_state` version table (propagation bounded to ~2s); the rate limiter enforces `ceil(limit / DEVPLACE_WEB_WORKERS)` per process so the aggregate honours the configured value - **set `DEVPLACE_WEB_WORKERS` to match `--workers`** (done in `make prod` and the Dockerfile). First-boot DB init and VAPID key generation are `flock`-serialized. Full detail and the design rules are in the admin docs at `/docs/production-concurrency.html`.
+
 ## CI/CD
 
 Gitea Actions workflow at `.gitea/workflows/test.yaml`:
-- Runs on push/PR to main
-- Sets up Python 3.13, installs dependencies + Playwright
-- Validates all source files with `hawk`
-- Runs all tests with fail-fast
-- Uploads failure screenshots as artifacts
+- Runs on push/PR to `master`
+- Sets up Python 3.13, installs dependencies + Playwright Chromium
+- Runs the full test suite under coverage in parallel (`coverage run -m pytest -n auto --dist loadfile`); `.coveragerc` `parallel=true` lets the per-worker `.coverage.*` files combine
+- Builds and publishes the coverage HTML as an artifact on every run
+- Uploads failure screenshots as artifacts when a test fails
+
+Changes are promoted through automated DTAP streets: Development (`make dev`), Test (the CI suite + coverage on `master`), Acceptance (the `master` to `production` promotion via `make deploy`), and Production (the Docker Compose stack on the host). Only CI-green `master` commits reach `production`.
 
 ## Feature workflow
 
 1. Implement the feature (router + template + CSS + JS)
-2. `hawk .` — validate all source files
-3. `make test` — run all tests (fail-fast)
+2. Validate each touched language (Python compiles/imports, JS parses, CSS and HTML balance)
+3. `make test` - run all tests (fail-fast)
 4. Add Playwright tests in `tests/test_*.py` for new functionality
-5. For visual features: `falcon take --output /tmp/verify.png && falcon describe /tmp/verify.png`
-6. Update `AGENTS.md` and `README.md` if new conventions were introduced
+5. Update `AGENTS.md` and `README.md` if new conventions were introduced
 
 ## License
 
-MIT — DevPlace
+MIT - DevPlace

agents/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

agents/__main__.py

@@ -0,0 +1,54 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import sys
+
+from .fleet import ordered_agents
+
+
+def _usage() -> str:
+    names = ", ".join(ordered_agents())
+    return (
+        "usage: python -m agents <command> [options]\n\n"
+        "commands:\n"
+        f"  <agent>      run one agent: {names}\n"
+        "  all          run the whole fleet (orchestrator)\n"
+        "  maestro      open the conversational conductor\n\n"
+        "examples:\n"
+        "  python -m agents security --check\n"
+        "  python -m agents all --check\n"
+        "  python -m agents maestro \"is my audit coverage complete?\"\n"
+    )
+
+
+def main() -> None:
+    if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"):
+        sys.stdout.write(_usage())
+        sys.exit(0)
+    command = sys.argv[1]
+    rest = sys.argv[2:]
+    sys.argv = [f"agents.{command}", *rest]
+    if command == "all":
+        from .orchestrator import main as run
+
+        run()
+        return
+    if command == "maestro":
+        from .maestro import main as run
+
+        run()
+        return
+    if command in ordered_agents():
+        from .fleet import REGISTRY
+        from .base import cli_main
+
+        sys.argv[0] = f"agents.{command}"
+        cli_main(REGISTRY[command]())
+        return
+    sys.stderr.write(f"unknown command '{command}'\n\n{_usage()}")
+    sys.exit(2)
+
+
+if __name__ == "__main__":
+    main()

agents/audit.py

@@ -0,0 +1,49 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .base import MaintenanceAgent, cli_main
+
+
+class AuditAgent(MaintenanceAgent):
+    name = "audit"
+    description = "audit log coverage maintainer"
+
+    def mandate(self) -> str:
+        return (
+            "Guarantee that every state-changing action emits a correct audit record, that the event catalogue is "
+            "complete, and that denials and failures are logged with the right result.\n\n"
+            "DETECT:\n"
+            "- Any mutation lacking an audit record on its success path is an error. A mutation is a @router.post / "
+            "@router.put / @router.delete, a .insert / .update / .delete DB write, or a background-service, scheduler, or "
+            "CLI state change. The record is audit.record(request, ...) in request contexts or audit.record_system(...) "
+            "in request-less contexts.\n"
+            "- Guard and denial branches missing result=\"denied\", and failure branches missing result=\"failure\", are errors.\n"
+            "- Event keys used in code but absent from events.md are errors; a new domain not mapped in "
+            "services/audit/categories.py category_for is an error.\n"
+            "- Double-counting is an error: the HTTP path and the Devii agent path for the same mutation must be disjoint "
+            "(dispatcher _audit_mechanic covers the agent path; the route covers the HTTP path). A record gated on the "
+            "action (so a logging failure would block it) is an error; recording is best-effort and never raises.\n\n"
+            "FIX: add the recorder call at the mutation point with the correct event key, origin, via_agent, and result, "
+            "never gating the action on it; extend events.md with the new key in the right domain; extend category_for for "
+            "a new domain; route the call through the existing DRY choke point (content.py, the project_files.py helpers, "
+            "routers/containers.py _audit_instance, the Devii dispatcher _audit_mechanic) rather than scattering call sites."
+        )
+
+    def scope_units(self) -> list[tuple[str, str]]:
+        return [
+            ("routers", "devplacepy/routers/*.py - every mutating route has audit.record on success and result on denial"),
+            ("content-choke", "devplacepy/content.py create/edit/delete record at the choke point"),
+            ("project-files", "devplacepy/project_files.py file/dir mutations recorded; read-only guard records denied"),
+            ("containers", "devplacepy/routers/containers.py _audit_instance covers lifecycle/exec/schedule"),
+            ("services", "devplacepy/services/* (news, jobs, containers, devii) use record_system with origin"),
+            ("catalogue", "events.md keys vs code keys; services/audit/categories.py category_for domain coverage"),
+        ]
+
+
+def main() -> None:
+    cli_main(AuditAgent())
+
+
+if __name__ == "__main__":
+    main()

agents/base.py

@@ -0,0 +1,278 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import logging
+import sys
+from datetime import datetime
+from typing import Optional
+
+from . import core
+from .agent import (
+    AgentState,
+    MarkdownRenderer,
+    begin_run_cost,
+    clear_protected_trees,
+    clear_write_budget,
+    close_http_client,
+    cost_session_total,
+    current_run_cost,
+    end_run_cost,
+    format_usd,
+    install_timestamps,
+    protect_agents,
+    react_loop,
+    reset_tool_scope,
+    set_shell_restricted,
+    set_tool_scope,
+    set_write_budget,
+    usd_str,
+    _with_datetime,
+)
+
+DEFAULT_MAX_ITER = 180
+WRITE_BUDGET = 20
+
+_RUN_LOCK = asyncio.Lock()
+
+AGENT_ICONS = {
+    "security": "\U0001F6E1",
+    "audit": "\U0001F4CB",
+    "devii": "\U0001F6E0",
+    "docs": "\U0001F4DA",
+    "fanout": "\U0001F310",
+    "dry": "♻",
+    "style": "\U0001F3A8",
+    "frontend": "\U0001F5A5",
+    "seo": "\U0001F50D",
+    "test": "\U0001F9EA",
+}
+
+MAINT_HEADER = """You are {name}, an autonomous maintenance agent for the DevPlace codebase, a FastAPI plus Jinja2 platform using the dataset library over SQLite, with pure ES6 module JavaScript on the frontend. You enforce exactly ONE quality dimension across the repository and nothing else.
+
+ABSOLUTE EXCLUSION (NON-NEGOTIABLE): The `agents/` directory is the maintenance fleet's OWN source code. You MUST NOT read it for analysis, grep it, scan it, report on it, or modify it under any circumstances. It deliberately contains the very patterns you hunt for (em-dash characters, forbidden-name examples, destructive-command strings, HTML entities) as DETECTION DATA, not as violations. A "violation" found in `agents/` is never real. Always exclude `agents/` from every grep and glob (the search root is the application code, not the tooling). The engine also hard-blocks any write under `agents/`, so an attempt to fix something there will fail by design. Treat `agents/` as if it does not exist.
+
+REPOSITORY LAYOUT (do NOT waste tool calls rediscovering this): all application code lives under the `devplacepy/` package. There is NO top-level `static/`, `routers/`, `templates/`, or `services/` directory; they are `devplacepy/static/` (with `js/`, `css/`, `vendor/`), `devplacepy/routers/`, `devplacepy/templates/`, and `devplacepy/services/`. Tests live in the top-level `tests/`. Packaging is the top-level `pyproject.toml` and `Makefile`. Start your investigation directly inside `devplacepy/` (and `tests/` for the test dimension); never probe top-level `static`/`routers` first.
+
+OPERATING PROTOCOL
+1. PLAN FIRST. Your VERY FIRST tool call MUST be plan() with goal, steps, success_criteria, and confidence. success_criteria MUST include "every finding confirmed against source and survives refutation" and "no regression: all consumers of any changed code verified".
+2. INVESTIGATE BEFORE CONCLUDING. Use grep, glob_files, list_dir, find_symbol, read_file, read_lines, and retrieve to gather evidence. Never assume a violation; confirm it against the source.
+3. TOOL DISCIPLINE. Use the structured grep tool for pattern detection; do NOT shell out with run_command to scan, lint, or loop over files (run_command is for verify only). For a grep-able check (a character, a name, a header line) grep; do not read a whole file, and never read a large file (over ~400 lines) just to look for a pattern - grep it or read_lines the relevant range. Never repeat a grep or re-read a file you already read this run.
+4. ONE report_finding PER ISSUE. Every confirmed issue MUST be recorded with report_finding(dimension, severity, message, file, line, rule). Do not describe issues only in prose.
+5. SHARD WIDE. The codebase is large. Work through the scope units below one at a time. You MAY call delegate(task) to investigate an independent unit in a fresh context and fold its result back in.
+6. STAY IN YOUR LANE. Only this dimension. Record an unrelated problem at most as a single info finding. Respect "refactor only what you touch": do NOT mass-rewrite pre-existing files for a cosmetic rule they never followed; that is noise, not maintenance.
+
+ACCURACY AND SAFETY DOCTRINE (NON-NEGOTIABLE - this is the standard you are held to; ZERO fault tolerance)
+A. EVIDENCE OVER SUSPICION. Be SURE before you act. Confirm every finding against the actual source: read the exact line(s) AND enough surrounding context (the whole function, the class, the caller, the contract) to understand INTENT, not just the matched text. A grep hit is a lead, never a verdict. If you are not certain something is a real violation of THIS dimension, you do not yet know enough - keep investigating until you are. Never guess, never pattern-match blindly.
+B. ELIMINATE FALSE POSITIVES - actively try to DISPROVE every candidate before recording it. A candidate becomes a finding only after it SURVIVES a genuine refutation attempt. Before it stands, rule out each of: it is intentional and required (a framework, protocol, contract, or external API shape demands exactly this); it is DATA, not authored prose (a value being matched, replaced, transformed, parsed, sanitized, or a deliberate test fixture - editing it would negate the code's whole purpose); it is generated, vendored, or third-party; it is already correct under a known exemption (for example, @tool docstrings are required for the tool schema and the mandatory file header is allowed). When you discard a candidate as a false positive, note the one-line reason in your reasoning. A WRONG finding is worse than a missed one; a nonsensical or no-op "fix" (one that changes nothing real, or only re-encodes the same thing) is a defect, not a fix.
+C. CROSS-REFERENCE BEFORE EVERY CHANGE (mandatory impact analysis). No file stands alone. BEFORE you edit anything, find every place that depends on what you are about to touch and confirm the change is safe for ALL of them - use grep and find_symbol across the whole `devplacepy/` tree (and `tests/`): a renamed or moved symbol -> every caller and import; a changed route path or method -> every template link, fetch()/Http call, Devii action, and docs entry; a changed Pydantic or JSON field -> every producer and consumer (handler context keys, respond(model=...), templates, JS, API docs); a changed template name, block, or context key -> every render() call and every template include; a changed CSS class or JS export -> every user. If a change would break even ONE consumer, it is not a valid fix as-is: either fix the entire reference set coherently in the same run, or record the finding fixed=false with the blocking reason. NEVER leave the codebase half-migrated.
+D. ZERO DEGRADATION (hard gate). A fix must NEVER reduce functionality, weaken a check or validation, drop a capability, change observable behavior, or make the code worse just to satisfy a rule. If the only fix you can see would do any of that, DO NOT apply it - record it fixed=false with the reason and the safe path forward. Closing a finding is never worth a regression. Functionality and quality are not negotiable against tidiness.
+E. DIG DEEP, DO NOT GIVE UP. When something is unclear, gather more evidence (read the callers, the tests, the docs, retrieve related files, delegate an independent investigation) instead of guessing or bailing. Pursue the ROOT CAUSE, not the surface symptom. Do not fabricate a fix to look finished, and do not abandon a confirmed real issue because it is hard - if you genuinely cannot fix it safely this run, record it precisely (fixed=false) so it is never lost.
+F. VERIFY YOUR OWN WORK. After each edit, RE-READ the changed region to confirm the edit is exactly what you intended and is complete and correct, then re-check the consumers from C. Mark a finding fixed=true ONLY after the change is in place, the references still hold, and (in fix mode) the build validator passes. Match the surrounding code: reuse existing helpers and patterns, prefer the minimal idiomatic root-cause change, and leave every file at least as good as you found it.
+
+{mode_rules}
+
+OBEY THE RULES YOU ENFORCE: never write comments or docstrings in source you author, never use an em-dash (use a hyphen), keep the `retoor <retoor@molodetz.nl>` header as the first line of any file you create, never run the test suite, never perform any git write operation.
+
+DIMENSION MANDATE
+{mandate}
+"""
+
+CHECK_RULES = "MODE: CHECK (read-only). You MUST NOT modify any file; the writing tools are not available to you. Apply the Accuracy Doctrine in full: confirm each candidate against the source with enough context to read intent (A), and actively rule out false positives before recording it (B) - a wrong finding is worse than a missed one. Record every issue that survives refutation with report_finding (fixed=false), and state confidence honestly; never guess. For a structural issue, note in the message which consumers a future fix must update (C). End with a one-line summary."
+
+FIX_RULES = "MODE: FIX (autonomous). For every issue: first reconfirm it is real and survives refutation (Doctrine A, B), then run the cross-reference impact check across all consumers (Doctrine C), then apply a minimal, idiomatic root-cause fix, then re-read the changed region and re-check the consumers (Doctrine F), then record it with report_finding (fixed=true). You MUST read an existing file (read_file, or read_lines for a range) before editing it. Prefer edit_file for surgical changes; for LARGE files, read_lines the target range and edit by line number with replace_lines/insert_lines/delete_lines instead of pasting big strings - it is far more reliable. NEVER apply a fix that would break a consumer, change behavior, or otherwise degrade the code (Doctrine D): when a finding cannot be fixed safely this run, record it fixed=false with the precise blocking reason and the safe path - do not force it and do not fake it. After all edits, call verify('python -m agents.validator .') and ensure it passes. End with a one-line summary."
+
+
+class MaintenanceAgent:
+    name: str = "maintenance"
+    description: str = ""
+
+    def mandate(self) -> str:
+        raise NotImplementedError
+
+    def scope_units(self) -> list[tuple[str, str]]:
+        return []
+
+    def system_prompt(self, mode: str) -> str:
+        rules = CHECK_RULES if mode == "check" else FIX_RULES
+        return MAINT_HEADER.format(name=self.name, mode_rules=rules, mandate=self.mandate())
+
+    def task_prompt(self, mode: str, scope: Optional[str], seed_findings: Optional[list[dict]] = None) -> str:
+        if mode == "fix" and seed_findings:
+            listed = "\n".join(
+                f"- {f.get('file','')}:{f.get('line') or '?'} [{f.get('rule') or f.get('dimension','')}] {f.get('message','')}"
+                for f in seed_findings[:60]
+            )
+            return (
+                f"Dimension: {self.description}\n\n"
+                f"FIX THESE {len(seed_findings)} findings from a prior check. Do NOT re-scan the whole codebase; "
+                f"go straight to these targets - but treat each as a CANDIDATE, not gospel: a seeded finding may be "
+                f"stale (already fixed) or a false positive. For EACH finding: read_file the target and RECONFIRM it is "
+                f"still real and survives refutation (Doctrine A, B); if it is already resolved or was a false positive, "
+                f"record it fixed=false with that reason and move on (never apply a no-op or nonsensical edit just to "
+                f"close it). If it is real, run the cross-reference impact check (Doctrine C), apply a minimal idiomatic "
+                f"root-cause fix, re-read the change and re-check consumers (Doctrine F), and record it fixed=true. NEVER "
+                f"apply a fix that breaks a consumer or degrades the code (Doctrine D); record fixed=false with the reason "
+                f"instead. When all edits are done call verify('python -m agents.validator .') once and ensure it passes. "
+                f"Exclude the `agents/` directory.\n\n{listed}\n\n"
+                f"Finish with a single line: 'N findings (E errors, W warnings), M fixed.'"
+            )
+        units = self.scope_units()
+        if scope:
+            units = [unit for unit in units if unit[0] == scope] or units
+        listed = "\n".join(f"- {label}: {focus}" for label, focus in units)
+        action = (
+            "record each confirmed issue"
+            if mode == "check"
+            else "for each issue confirm it, impact-check its consumers, fix it at the root, re-verify, then record it"
+        )
+        return (
+            f"Dimension: {self.description}\n\n"
+            f"Scope units to sweep ({mode} mode):\n{listed}\n\n"
+            f"Apply the Accuracy and Safety Doctrine to every candidate (confirm against source, refute false positives, "
+            f"cross-reference before any change, never degrade, verify your work). Be efficient: investigate, then "
+            f"{action} via report_finding. Do not re-read a file you already read or "
+            f"repeat a grep. Record findings as you go rather than saving them all for the end. "
+            f"Exclude the `agents/` directory from every grep and glob; it is the fleet's own source and is off-limits. "
+            f"Finish with a single line: 'N findings (E errors, W warnings), M fixed.'"
+        )
+
+    async def run(
+        self,
+        mode: str,
+        scope: Optional[str],
+        max_iter: int,
+        renderer: Optional[MarkdownRenderer],
+        seed_findings: Optional[list[dict]] = None,
+    ) -> dict:
+        if mode == "check":
+            return await self._execute(mode, scope, max_iter, renderer, seed_findings)
+        async with _RUN_LOCK:
+            return await self._execute(mode, scope, max_iter, renderer, seed_findings)
+
+    async def _execute(
+        self,
+        mode: str,
+        scope: Optional[str],
+        max_iter: int,
+        renderer: Optional[MarkdownRenderer],
+        seed_findings: Optional[list[dict]],
+    ) -> dict:
+        writes_enabled = mode != "check"
+        core.reset_findings()
+        budget = WRITE_BUDGET
+        if writes_enabled:
+            protect_agents()
+            set_shell_restricted(True)
+            if seed_findings:
+                distinct = {f.get("file") for f in seed_findings if f.get("file")}
+                budget = max(WRITE_BUDGET, len(distinct) + 2)
+            set_write_budget(budget)
+        scope_token = set_tool_scope(core.worker_tool_names(mode))
+        cost_token = begin_run_cost()
+        started = datetime.now()
+        codename = core.report_codename()
+        if renderer is not None:
+            icon = AGENT_ICONS.get(self.name, "\U0001F916")
+            scope_note = f" (scope: {scope})" if scope else ""
+            if mode == "fix" and seed_findings:
+                plan_line = f"fix {len(seed_findings)} confirmed finding(s) from the last check, then verify the build"
+            elif mode == "fix":
+                plan_line = f"scan{scope_note}, fix each issue, and verify the build (up to {budget} files this run)"
+            else:
+                plan_line = f"scan{scope_note} read-only and report each issue (no files changed)"
+            report_file = core.report_path(self.name, codename, started)
+            renderer.print(
+                f"\n{icon} **{self.name} agent** `{codename}` starting -- {self.description}.\n"
+                f"I will {plan_line}.\n"
+                f"Report -> `{report_file}.json`"
+            )
+        messages = [
+            {"role": "system", "content": _with_datetime(self.system_prompt(mode))},
+            {"role": "user", "content": self.task_prompt(mode, scope, seed_findings)},
+        ]
+        state = AgentState()
+        try:
+            final = await react_loop(
+                messages=messages,
+                tools_payload=core.payloads_for(mode),
+                state=state,
+                max_iterations=max_iter,
+                renderer=renderer,
+            )
+            run_cost = current_run_cost()["cost"]
+        finally:
+            end_run_cost(cost_token)
+            reset_tool_scope(scope_token)
+            if writes_enabled:
+                clear_protected_trees()
+                clear_write_budget()
+                set_shell_restricted(False)
+        finished = datetime.now()
+        incomplete = final is None or state.iteration >= max_iter
+        session_cost = cost_session_total()["cost"]
+        cost = {"run_usd": usd_str(run_cost), "session_usd": usd_str(session_cost)}
+        report = core.write_reports(
+            self.name, mode, started, finished, cost=cost, incomplete=incomplete, codename=codename
+        )
+        summary = report["summary"]
+        if incomplete:
+            exit_code = 1
+        elif mode == "check":
+            exit_code = 1 if summary["total"] > 0 else 0
+        else:
+            exit_code = 1 if summary["unfixed"] > 0 else 0
+        result = {
+            "name": self.name,
+            "mode": mode,
+            "summary": summary,
+            "cost": cost,
+            "incomplete": incomplete,
+            "json": report["json"],
+            "items": report["items"],
+            "exit_code": exit_code,
+        }
+        if renderer is not None:
+            status = "  ⚠ INCOMPLETE (hit the iteration limit; results are partial)" if incomplete else ""
+            renderer.print(
+                f"\n**{self.name}** [{mode}] -> {summary['total']} findings "
+                f"({summary['errors']} errors, {summary['warnings']} warnings), "
+                f"{summary['fixed']} fixed.{status}   \U0001F4B0 {format_usd(run_cost)} this run "
+                f"({format_usd(session_cost)} session). Report: `{report['json']}`"
+            )
+        return result
+
+
+def build_parser(name: str, description: str) -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog=f"agents.{name}", description=description)
+    mode = parser.add_mutually_exclusive_group()
+    mode.add_argument("--fix", dest="mode", action="store_const", const="fix", help="Autonomously fix findings (default)")
+    mode.add_argument("--check", dest="mode", action="store_const", const="check", help="Report only; non-zero exit on findings")
+    parser.set_defaults(mode="fix")
+    parser.add_argument("--scope", default=None, help="Restrict to one scope unit label")
+    parser.add_argument("--max-iter", type=int, default=DEFAULT_MAX_ITER, help="Maximum agent iterations")
+    parser.add_argument("--no-color", action="store_true", help="Disable ANSI colour output")
+    parser.add_argument("-v", "--verbose", action="store_true", help="Enable debug logging")
+    return parser
+
+
+async def _amain(agent: MaintenanceAgent, argv: Optional[list[str]] = None) -> int:
+    install_timestamps()
+    args = build_parser(agent.name, agent.description).parse_args(argv)
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+    renderer = MarkdownRenderer(use_color=not args.no_color)
+    try:
+        result = await agent.run(args.mode, args.scope, args.max_iter, renderer)
+        return result["exit_code"]
+    finally:
+        await close_http_client()
+
+
+def cli_main(agent: MaintenanceAgent) -> None:
+    try:
+        sys.exit(asyncio.run(_amain(agent)))
+    except KeyboardInterrupt:
+        sys.exit(130)

agents/core/__init__.py

@@ -0,0 +1,278 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import contextvars
+import json
+import random
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Optional
+
+CODENAME_ADJECTIVES = (
+    "brave", "calm", "clever", "swift", "gentle", "bright", "bold", "cosmic", "lucky", "mighty",
+    "noble", "quiet", "rapid", "sunny", "witty", "eager", "jolly", "keen", "merry", "proud",
+    "sleek", "spry", "vivid", "zesty", "amber", "azure", "coral", "fuzzy", "golden", "happy",
+    "snappy", "cozy", "breezy", "plucky", "dapper", "nimble", "quirky", "steady", "tidy", "wise",
+)
+CODENAME_ANIMALS = (
+    "otter", "badger", "panda", "koala", "heron", "tiger", "gecko", "raven", "moose", "bison",
+    "crane", "dingo", "ferret", "marmot", "walrus", "puffin", "ibis", "lemur", "tapir", "quokka",
+    "narwhal", "ocelot", "wombat", "gibbon", "meerkat", "mongoose", "capybara", "axolotl", "pangolin", "armadillo",
+)
+
+
+def report_codename() -> str:
+    return f"{random.choice(CODENAME_ADJECTIVES)}-{random.choice(CODENAME_ANIMALS)}"
+
+from ..agent import (
+    tool,
+    react_loop,
+    AgentState,
+    get_tool,
+    get_tool_payloads,
+    MarkdownRenderer,
+    close_http_client,
+    _with_datetime,
+)
+
+REPORTS_DIR = Path(__file__).resolve().parent.parent / "reports"
+
+WRITE_TOOLS = (
+    "create_file",
+    "write_file",
+    "edit_file",
+    "patch_file",
+    "replace_lines",
+    "insert_lines",
+    "delete_lines",
+)
+WEB_TOOLS = (
+    "web_search",
+    "deep_search",
+    "ai_search",
+    "fetch_url",
+    "download_file",
+    "download_files",
+    "describe_image",
+)
+SWARM_TOOLS = (
+    "spawn",
+    "swarm_wait",
+    "swarm_result",
+    "swarm_tail",
+    "swarm_kill",
+    "swarm_cleanup",
+)
+SEVERITIES = ("error", "warning", "info")
+
+_findings_var: contextvars.ContextVar[Optional[list]] = contextvars.ContextVar("findings", default=None)
+
+
+def reset_findings() -> None:
+    _findings_var.set([])
+
+
+def findings() -> list[dict[str, Any]]:
+    current = _findings_var.get()
+    return list(current) if current is not None else []
+
+
+@tool
+async def report_finding(
+    dimension: str,
+    severity: str,
+    message: str,
+    file: str = "",
+    line: int = 0,
+    rule: str = "",
+    fixed: bool = False,
+    fix_summary: str = "",
+):
+    """Record one maintenance finding for the run report. Call once per confirmed issue.
+    dimension: The check that produced this finding.
+    severity: One of error, warning, info.
+    message: Human-readable description of the issue.
+    file: Repository-relative path of the offending file.
+    line: 1-based line number when known, else 0.
+    rule: Short machine rule id, e.g. mutation-without-record.
+    fixed: True if this run already applied a fix.
+    fix_summary: One line describing the fix when fixed is true.
+    """
+    severity_value = severity if severity in SEVERITIES else "warning"
+    current = _findings_var.get()
+    if current is None:
+        current = []
+        _findings_var.set(current)
+    current.append(
+        {
+            "severity": severity_value,
+            "dimension": dimension,
+            "file": file,
+            "line": int(line) if line else 0,
+            "rule": rule,
+            "message": message,
+            "fixed": bool(fixed),
+            "fix_summary": fix_summary,
+        }
+    )
+    return json.dumps({"status": "success", "recorded": True, "total_findings": len(current)})
+
+
+WORKER_READ_TOOLS = (
+    "read_file",
+    "read_lines",
+    "list_dir",
+    "glob_files",
+    "grep",
+    "find_symbol",
+    "retrieve",
+)
+WORKER_REASONING_TOOLS = (
+    "plan",
+    "reflect",
+    "delegate",
+    "report_finding",
+    "get_current_isodate",
+)
+
+
+def worker_tool_names(mode: str) -> tuple[str, ...]:
+    names = WORKER_READ_TOOLS + WORKER_REASONING_TOOLS
+    if mode != "check":
+        names = names + WRITE_TOOLS + ("verify",)
+    return names
+
+
+def payloads_for(mode: str) -> list[dict[str, Any]]:
+    return payloads_named(worker_tool_names(mode))
+
+
+def payloads_named(names: tuple[str, ...]) -> list[dict[str, Any]]:
+    out: list[dict[str, Any]] = []
+    for name in names:
+        func = get_tool(name)
+        if func is not None:
+            out.append(func._tool_payload)
+    return out
+
+
+def summarize(items: list[dict[str, Any]]) -> dict[str, int]:
+    errors = sum(1 for f in items if f["severity"] == "error")
+    warnings = sum(1 for f in items if f["severity"] == "warning")
+    fixed = sum(1 for f in items if f["fixed"])
+    return {
+        "total": len(items),
+        "errors": errors,
+        "warnings": warnings,
+        "fixed": fixed,
+        "unfixed": len(items) - fixed,
+    }
+
+
+def _md_report(name: str, mode: str, summary: dict[str, int], items: list[dict[str, Any]]) -> str:
+    lines = [
+        f"# {name} report ({mode})",
+        "",
+        f"- total: {summary['total']}",
+        f"- errors: {summary['errors']}",
+        f"- warnings: {summary['warnings']}",
+        f"- fixed: {summary['fixed']}",
+        f"- unfixed: {summary['unfixed']}",
+        "",
+        "## Findings",
+        "",
+    ]
+    if not items:
+        lines.append("None.")
+        return "\n".join(lines)
+    by_file: dict[str, list[dict[str, Any]]] = {}
+    for finding in items:
+        by_file.setdefault(finding["file"] or "(repository)", []).append(finding)
+    for path, group in sorted(by_file.items()):
+        lines.append(f"### {path}")
+        for finding in group:
+            location = f":{finding['line']}" if finding["line"] else ""
+            mark = "fixed" if finding["fixed"] else finding["severity"]
+            label = finding["rule"] or finding["dimension"]
+            lines.append(f"- [{mark}] {label}{location}: {finding['message']}")
+            if finding["fixed"] and finding["fix_summary"]:
+                lines.append(f"  fix: {finding['fix_summary']}")
+        lines.append("")
+    return "\n".join(lines)
+
+
+REPORTS_KEEP = 25
+
+
+def report_path(name: str, codename: str, started: datetime) -> Path:
+    stamp = started.strftime("%Y%m%d-%H%M%S")
+    return REPORTS_DIR / f"{name}-{codename}-{stamp}"
+
+
+def prune_reports(prefix: str, keep: int = REPORTS_KEEP) -> None:
+    existing = sorted(
+        REPORTS_DIR.glob(f"{prefix}-*.json"),
+        key=lambda path: path.stat().st_mtime,
+        reverse=True,
+    )
+    for stale in existing[keep:]:
+        for companion in (stale, stale.with_suffix(".md")):
+            try:
+                companion.unlink()
+            except OSError:
+                pass
+
+
+def write_reports(
+    name: str,
+    mode: str,
+    started: datetime,
+    finished: datetime,
+    cost: Optional[dict[str, Any]] = None,
+    incomplete: bool = False,
+    codename: Optional[str] = None,
+) -> dict[str, Any]:
+    REPORTS_DIR.mkdir(parents=True, exist_ok=True)
+    items = findings()
+    summary = summarize(items)
+    codename = codename or report_codename()
+    stamp = started.strftime("%Y%m%d-%H%M%S")
+    base = REPORTS_DIR / f"{name}-{codename}-{stamp}"
+    payload = {
+        "agent": name,
+        "codename": codename,
+        "mode": mode,
+        "started_at": started.isoformat(),
+        "finished_at": finished.isoformat(),
+        "incomplete": incomplete,
+        "summary": summary,
+        "cost": cost or {},
+        "findings": items,
+    }
+    base.with_suffix(".json").write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    base.with_suffix(".md").write_text(_md_report(name, mode, summary, items), encoding="utf-8")
+    prune_reports(name)
+    return {"summary": summary, "json": str(base.with_suffix(".json")), "items": items}
+
+
+__all__ = [
+    "tool",
+    "react_loop",
+    "AgentState",
+    "MarkdownRenderer",
+    "close_http_client",
+    "_with_datetime",
+    "report_finding",
+    "reset_findings",
+    "findings",
+    "payloads_for",
+    "payloads_named",
+    "worker_tool_names",
+    "summarize",
+    "write_reports",
+    "report_codename",
+    "report_path",
+    "prune_reports",
+    "REPORTS_DIR",
+]

agents/core/cost.py

@@ -0,0 +1,13 @@
+# retoor <retoor@molodetz.nl>
+
+from ..agent import (  # noqa: F401
+    cost_session_total,
+    cost_record,
+    set_cost_stream,
+    format_usd,
+    usd_str,
+    PER_MILLION,
+    PRICE_CACHE_HIT_PER_M,
+    PRICE_CACHE_MISS_PER_M,
+    PRICE_OUTPUT_PER_M,
+)

agents/core/exec_tools.py

@@ -0,0 +1,3 @@
+# retoor <retoor@molodetz.nl>
+
+from ..agent import run_command, verify, stream_subprocess  # noqa: F401

agents/core/fs_tools.py

@@ -0,0 +1,3 @@
+# retoor <retoor@molodetz.nl>
+
+from ..agent import read_file, read_lines, write_file, create_file, edit_file, patch_file, list_dir, glob_files, grep, find_symbol  # noqa: F401

agents/core/llm.py

@@ -0,0 +1,3 @@
+# retoor <retoor@molodetz.nl>
+
+from ..agent import Backend, get_backends, vision_available, llm_call, _call_backend, MODEL, DEEPSEEK_MODEL  # noqa: F401

agents/core/loop.py

@@ -0,0 +1,3 @@
+# retoor <retoor@molodetz.nl>
+
+from ..agent import AgentState, react_loop, compact_messages, context_size, find_compaction_split  # noqa: F401

agents/core/prompts.py

@@ -0,0 +1,3 @@
+# retoor <retoor@molodetz.nl>
+
+from ..agent import SYSTEM_PROMPT, SUB_AGENT_SYSTEM_PROMPT, system_message, _with_datetime  # noqa: F401

agents/core/render.py

@@ -0,0 +1,3 @@
+# retoor <retoor@molodetz.nl>
+
+from ..agent import MarkdownRenderer  # noqa: F401

agents/core/runner.py

@@ -0,0 +1,3 @@
+# retoor <retoor@molodetz.nl>
+
+from ..agent import run_once, interactive, amain, main  # noqa: F401

agents/core/swarm.py

@@ -0,0 +1,3 @@
+# retoor <retoor@molodetz.nl>
+
+from ..agent import delegate, spawn, swarm_wait, swarm_result, swarm_tail, swarm_kill, swarm_cleanup, retrieve, CorpusIndex, get_corpus_index  # noqa: F401

agents/core/tools.py

@@ -0,0 +1,3 @@
+# retoor <retoor@molodetz.nl>
+
+from ..agent import tool, get_tool, get_tool_payloads, execute_tool_call, coerce_tool_args, validate_tool_args, _build_function_payload  # noqa: F401

agents/core/web_tools.py

@@ -0,0 +1,3 @@
+# retoor <retoor@molodetz.nl>
+
+from ..agent import web_search, deep_search, ai_search, fetch_url, download_file, download_files, describe_image  # noqa: F401

agents/devii.py

@@ -0,0 +1,47 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .base import MaintenanceAgent, cli_main
+
+
+class DeviiAgent(MaintenanceAgent):
+    name = "devii"
+    description = "Devii capability and role-gated tool-list maintainer"
+
+    def mandate(self) -> str:
+        return (
+            "Guarantee that Devii can perform, via REST, everything the site offers to the logged-in user's role, and "
+            "that the tool list presented to a given user exposes only the tools that role may call. A non-admin must not "
+            "even see that admin tools exist.\n\n"
+            "DETECT:\n"
+            "- Enumerate every REST route across devplacepy/routers/*.py and diff against CATALOG.by_name(). Every route a "
+            "user could reasonably ask Devii to perform has a corresponding Action. A user-facing capability with no Devii "
+            "action is a finding.\n"
+            "- Each Action's requires_auth and requires_admin flags exactly match its route's guard. An admin-guarded route "
+            "exposed as a non-admin Devii action is a security-grade error; a public route wrongly marked requires_auth=True "
+            "is a capability gap.\n"
+            "- Catalog.tool_schemas_for(authenticated, is_admin) withholds an admin tool's schema from a non-admin, and the "
+            "dispatcher still raises AuthRequiredError if a non-admin names it. Confirm both halves hold for every action; a "
+            "tool whose schema leaks to the wrong role is an error.\n"
+            "- Irreversible Devii actions are in CONFIRM_REQUIRED.\n\n"
+            "FIX: add the missing Action in the correct handler module with the right method, path, requires_auth, and "
+            "requires_admin; correct a misaligned auth flag. Never grant a member an admin capability to close a parity gap; "
+            "an admin-only capability with no member action is left admin-only. Hand new-tool documentation to the docs agent."
+        )
+
+    def scope_units(self) -> list[tuple[str, str]]:
+        return [
+            ("route-parity", "devplacepy/routers/*.py routes vs services/devii/actions/catalog.py CATALOG.by_name()"),
+            ("flag-alignment", "each Action requires_auth/requires_admin matches the route guard"),
+            ("role-visibility", "services/devii/actions/spec.py tool_schemas_for: no admin schema reaches a non-admin"),
+            ("dispatch-guard", "services/devii/actions/dispatcher.py AuthRequiredError on requires_admin; CONFIRM_REQUIRED"),
+        ]
+
+
+def main() -> None:
+    cli_main(DeviiAgent())
+
+
+if __name__ == "__main__":
+    main()

agents/docs.py

@@ -0,0 +1,48 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .base import MaintenanceAgent, cli_main
+
+
+class DocsAgent(MaintenanceAgent):
+    name = "docs"
+    description = "documentation coverage and role-aware show/hide maintainer"
+
+    def mandate(self) -> str:
+        return (
+            "Keep CLAUDE.md, AGENTS.md, README.md, and the /docs pages in exact agreement with the source, and keep "
+            "role-based visibility consistent so admin material is shown to admins and hidden from members and guests at "
+            "both the page and the section level.\n\n"
+            "DETECT:\n"
+            "- Every public or authenticated REST route has a docs_api.endpoint() entry in the correct group, with params "
+            "and a sample_response. A documented route whose params drifted from the actual Form model is an error.\n"
+            "- Every prose page's factual claims match the code (routes, env vars, defaults, behavior). A stale claim is an error.\n"
+            "- README.md reflects current routes, env vars, dependencies, and user-visible features. AGENTS.md has a domain "
+            "section for every mechanic. CLAUDE.md changes only for a new architectural rule.\n"
+            "- Page-level role gating: admin-only pages carry \"admin\": True in their DOCS_PAGES entry; the router filters "
+            "the sidebar to visible_pages and 404s a non-admin requesting an admin page, while docs_search still indexes admin "
+            "pages for admins. An admin page missing the flag, or a member page wrongly flagged admin, is an error.\n"
+            "- Section-level role gating: prose templates receive the user context via docs_prose.render_prose and gate admin "
+            "sections with Jinja {% if user %} / {% if user.role == 'admin' %}. Unguarded admin material on a public page is an error.\n\n"
+            "FIX: add or repair the endpoint() entry, rewrite the stale prose, add the missing README.md / AGENTS.md section, "
+            "add the \"admin\": True flag, or wrap the leaking section in the correct Jinja guard. The source is authoritative; "
+            "correct the docs to match the code, never the reverse."
+        )
+
+    def scope_units(self) -> list[tuple[str, str]]:
+        return [
+            ("api-docs", "devplacepy/docs_api.py endpoint() coverage vs routers/*.py routes"),
+            ("page-gating", "devplacepy/routers/docs.py DOCS_PAGES admin flag; visible_pages filter; docs_search indexing"),
+            ("section-gating", "templates/docs/*.html Jinja {% if user.role == 'admin' %} on admin sections"),
+            ("readme", "README.md reflects current routes, env vars, dependencies, features"),
+            ("agents-md", "AGENTS.md has a domain section for every mechanic; CLAUDE.md only for new rules"),
+        ]
+
+
+def main() -> None:
+    cli_main(DocsAgent())
+
+
+if __name__ == "__main__":
+    main()

agents/dry.py

@@ -0,0 +1,43 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .base import MaintenanceAgent, cli_main
+
+
+class DryAgent(MaintenanceAgent):
+    name = "dry"
+    description = "duplication and reuse enforcement"
+
+    def mandate(self) -> str:
+        return (
+            "Eliminate duplicated logic and re-implementations of the canonical shared utilities.\n\n"
+            "DETECT:\n"
+            "- Backend: inline N+1 loops where a batch helper exists (get_users_by_uids, get_comment_counts_by_post_uids, "
+            "get_vote_counts, load_comments, build_pagination, _in_clause); per-router Jinja2Templates instead of the shared "
+            "templating.templates; inline avatar or user links instead of the _avatar_link.html / _user_link.html partials.\n"
+            "- Frontend: hand-rolled fetch instead of Http; bespoke polling instead of Poller; bespoke job polling instead of "
+            "JobPoller; click-to-POST controllers not extending OptimisticAction; floating windows not extending FloatingWindow.\n"
+            "- General: blocks of duplicated logic that should be extracted into a shared helper.\n\n"
+            "FIX: replace the call site with the existing utility, or extract a new shared helper and route the duplicate "
+            "call sites through it; extractions follow the project's small-files structure and must not change behavior. When "
+            "similarity is below a confidence threshold, record an info finding for human review rather than auto-extracting."
+        )
+
+    def scope_units(self) -> list[tuple[str, str]]:
+        return [
+            ("batch-helpers", "routers/*.py use database.py batch helpers, not inline N+1 loops"),
+            ("templates", "every router imports templating.templates, never its own Jinja2Templates"),
+            ("partials", "_avatar_link.html / _user_link.html reused, not inline avatar/user markup"),
+            ("frontend-http", "static/js/*.js use Http, not hand-rolled fetch"),
+            ("frontend-poll", "static/js/*.js use Poller / JobPoller, not bespoke loops"),
+            ("frontend-base", "controllers extend OptimisticAction; windows extend FloatingWindow"),
+        ]
+
+
+def main() -> None:
+    cli_main(DryAgent())
+
+
+if __name__ == "__main__":
+    main()

agents/fanout.py

@@ -0,0 +1,46 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .base import MaintenanceAgent, cli_main
+
+
+class FeatureFanoutAgent(MaintenanceAgent):
+    name = "fanout"
+    description = "cross-layer feature completeness checker"
+
+    def mandate(self) -> str:
+        return (
+            "Enforce the 'Anatomy of a feature' checklist: for each route, every layer of the fan-out exists and agrees.\n\n"
+            "DETECT, for each route:\n"
+            "- Input has a models.py Form model declared as data: Annotated[SomeForm, Form()] (or a documented raw-form "
+            "exception for file uploads).\n"
+            "- If the route serves JSON via respond(..., model=XOut), every context key the route returns exists on XOut. "
+            "A key returned but absent from the schema is silently dropped and is an error.\n"
+            "- The route returns HTML and JSON through respond (or pure JSON via JSONResponse) consistently.\n"
+            "- A services/devii/actions/catalog.py Action exists if the route is something a user could ask Devii to do.\n"
+            "- A docs_api.py entry exists for every public or authenticated endpoint.\n"
+            "- Public pages build base_seo_context.\n"
+            "- README.md and AGENTS.md mention the feature.\n\n"
+            "FIX: add the missing Form, add the missing key to the *Out schema, switch the handler to respond, or flag the "
+            "responsible specialist's layer. When a layer is intentionally absent (an internal route with no public docs, a "
+            "route Devii should never call), record an info finding with the rationale rather than fabricating the layer."
+        )
+
+    def scope_units(self) -> list[tuple[str, str]]:
+        return [
+            ("forms", "devplacepy/models.py Form model exists for each mutating route input"),
+            ("schemas", "devplacepy/schemas.py *Out has every key returned by respond(model=XOut)"),
+            ("respond", "routers/*.py serve HTML+JSON via respond consistently"),
+            ("devii-action", "services/devii/actions/catalog.py Action exists for user-facing routes"),
+            ("api-docs", "devplacepy/docs_api.py entry for each public/auth endpoint"),
+            ("seo-readme", "seo.py base_seo_context for public pages; README/AGENTS mention the feature"),
+        ]
+
+
+def main() -> None:
+    cli_main(FeatureFanoutAgent())
+
+
+if __name__ == "__main__":
+    main()

agents/fleet.py

@@ -0,0 +1,47 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .audit import AuditAgent
+from .devii import DeviiAgent
+from .docs import DocsAgent
+from .dry import DryAgent
+from .fanout import FeatureFanoutAgent
+from .frontend import FrontendAgent
+from .security import SecurityAgent
+from .seo import SeoAgent
+from .style import StyleAgent
+from .test_coverage import TestAgent
+
+ORDER = [
+    "style",
+    "dry",
+    "security",
+    "audit",
+    "devii",
+    "seo",
+    "frontend",
+    "fanout",
+    "docs",
+    "test",
+]
+
+REGISTRY = {
+    "style": StyleAgent,
+    "dry": DryAgent,
+    "security": SecurityAgent,
+    "audit": AuditAgent,
+    "devii": DeviiAgent,
+    "seo": SeoAgent,
+    "frontend": FrontendAgent,
+    "fanout": FeatureFanoutAgent,
+    "docs": DocsAgent,
+    "test": TestAgent,
+}
+
+
+def ordered_agents(only: list[str] | None = None) -> list[str]:
+    if not only:
+        return list(ORDER)
+    wanted = {name.strip() for name in only if name.strip()}
+    return [name for name in ORDER if name in wanted]

agents/frontend.py

@@ -0,0 +1,42 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .base import MaintenanceAgent, cli_main
+
+
+class FrontendAgent(MaintenanceAgent):
+    name = "frontend"
+    description = "ES6, component, and CSS consistency"
+
+    def mandate(self) -> str:
+        return (
+            "Keep the frontend conformant to the project's strict ES6 and component rules.\n\n"
+            "DETECT:\n"
+            "- One class per ES6 module, instantiated and reachable via the global app, with Application.js as the root.\n"
+            "- Custom dp- components extend Component, self-register via customElements.define at the bottom of their file, "
+            "render into the light DOM (no shadow root so global CSS applies), and inject their own CSS <link> on "
+            "instantiation if absent.\n"
+            "- CSS uses variables (the design tokens), and pages are responsive down to very small phones.\n"
+            "- CDN scripts in templates/base.html use defer or type=\"module\" so the Playwright domcontentloaded wait does "
+            "not time out.\n\n"
+            "FIX: split a multi-class module, add the missing customElements.define, remove a shadow root, add the dynamic CSS "
+            "link injection, replace a hard-coded color with a token, or add defer to a CDN script. Never introduce a JS "
+            "framework, NPM, or a build step. Visual judgement is out of scope for auto-fix and is recorded as a finding."
+        )
+
+    def scope_units(self) -> list[tuple[str, str]]:
+        return [
+            ("one-class", "static/js/*.js one class per module, instantiated on app"),
+            ("components", "static/js/components/*.js extend Component, define, light DOM, CSS link injection"),
+            ("css-tokens", "static/css/*.css use design-token variables; responsive to small phones"),
+            ("cdn-scripts", "templates/base.html CDN scripts use defer or type=module"),
+        ]
+
+
+def main() -> None:
+    cli_main(FrontendAgent())
+
+
+if __name__ == "__main__":
+    main()

agents/maestro.py

@@ -0,0 +1,351 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import re
+import sys
+from typing import Any, Optional
+
+from . import core
+from .agent import (
+    AgentState,
+    MarkdownRenderer,
+    close_http_client,
+    get_tool,
+    install_timestamps,
+    mark_orchestration_tools,
+    react_loop,
+    tool,
+    _with_datetime,
+)
+from .fleet import REGISTRY, ordered_agents
+from .orchestrator import run_fleet
+
+MAESTRO_AGENT_MAX_ITER = 180
+MAX_RETURNED_FINDINGS = 12
+
+_LAST_RESULTS: dict[str, dict[str, Any]] = {}
+_LAST_CHECK: dict[str, list[dict[str, Any]]] = {}
+_RENDERER: Optional[MarkdownRenderer] = None
+
+
+def _set_renderer(renderer: MarkdownRenderer) -> None:
+    global _RENDERER
+    _RENDERER = renderer
+
+
+async def _dispatch(name: str, mode: str, scope: str) -> str:
+    if name not in REGISTRY:
+        return json.dumps({"status": "error", "error": f"Unknown agent '{name}'. Known: {ordered_agents()}"})
+    run_mode = "fix" if str(mode).lower() == "fix" else "check"
+    renderer = _RENDERER
+    seed = _LAST_CHECK.get(name) if run_mode == "fix" else None
+    agent = REGISTRY[name]()
+    result = await agent.run(run_mode, scope or None, MAESTRO_AGENT_MAX_ITER, renderer=renderer, seed_findings=seed)
+    _LAST_RESULTS[name] = result
+    if run_mode == "check":
+        _LAST_CHECK[name] = result["items"]
+    items = result["items"][:MAX_RETURNED_FINDINGS]
+    return json.dumps(
+        {
+            "status": "success",
+            "agent": name,
+            "mode": run_mode,
+            "summary": result["summary"],
+            "incomplete": result.get("incomplete", False),
+            "seeded_from_check": bool(seed),
+            "cost": result.get("cost"),
+            "report": result["json"],
+            "findings": items,
+        }
+    )
+
+
+@tool
+async def list_agents():
+    """List the maintenance agents Maestro can run, with their dimension."""
+    return json.dumps(
+        {"status": "success", "agents": [{"name": name, "dimension": REGISTRY[name].description} for name in ordered_agents()]}
+    )
+
+
+@tool
+async def read_report(agent: str):
+    """Return the latest in-memory result for one agent without re-running it.
+    agent: The agent name, e.g. security, audit, devii.
+    """
+    result = _LAST_RESULTS.get(agent)
+    if result is None:
+        return json.dumps({"status": "error", "error": f"No result yet for '{agent}'. Run it first."})
+    return json.dumps({"status": "success", "agent": agent, "summary": result["summary"], "findings": result["items"][:MAX_RETURNED_FINDINGS]})
+
+
+@tool
+async def run_security(mode: str = "check", scope: str = ""):
+    """Run the data and role security agent. mode: check (default) or fix. scope: optional unit label."""
+    return await _dispatch("security", mode, scope)
+
+
+@tool
+async def run_audit(mode: str = "check", scope: str = ""):
+    """Run the audit-log coverage agent. mode: check (default) or fix. scope: optional unit label."""
+    return await _dispatch("audit", mode, scope)
+
+
+@tool
+async def run_devii(mode: str = "check", scope: str = ""):
+    """Run the Devii capability and role-gated tool-list agent. mode: check (default) or fix."""
+    return await _dispatch("devii", mode, scope)
+
+
+@tool
+async def run_docs(mode: str = "check", scope: str = ""):
+    """Run the documentation coverage and role-aware show/hide agent. mode: check (default) or fix."""
+    return await _dispatch("docs", mode, scope)
+
+
+@tool
+async def run_fanout(mode: str = "check", scope: str = ""):
+    """Run the cross-layer feature completeness agent. mode: check (default) or fix."""
+    return await _dispatch("fanout", mode, scope)
+
+
+@tool
+async def run_dry(mode: str = "check", scope: str = ""):
+    """Run the duplication and reuse agent. mode: check (default) or fix."""
+    return await _dispatch("dry", mode, scope)
+
+
+@tool
+async def run_style(mode: str = "check", scope: str = ""):
+    """Run the coding-rule compliance agent. mode: check (default) or fix."""
+    return await _dispatch("style", mode, scope)
+
+
+@tool
+async def run_frontend(mode: str = "check", scope: str = ""):
+    """Run the ES6, component, and CSS consistency agent. mode: check (default) or fix."""
+    return await _dispatch("frontend", mode, scope)
+
+
+@tool
+async def run_seo(mode: str = "check", scope: str = ""):
+    """Run the SEO and sitemap coverage agent. mode: check (default) or fix."""
+    return await _dispatch("seo", mode, scope)
+
+
+@tool
+async def run_tests(mode: str = "check", scope: str = ""):
+    """Run the integration-test coverage agent. It writes tests but never runs the suite. mode: check (default) or fix."""
+    return await _dispatch("test", mode, scope)
+
+
+@tool
+async def run_fleet_tool(mode: str = "check"):
+    """Run the whole maintenance fleet in dependency order via the orchestrator. mode: check (default) or fix."""
+    run_mode = "fix" if str(mode).lower() == "fix" else "check"
+    per_agent: list[dict[str, Any]] = []
+    fleet_items: list[dict[str, Any]] = []
+
+    def _record(name: str, result: dict[str, Any]) -> None:
+        _LAST_RESULTS[name] = result
+        if run_mode == "check":
+            _LAST_CHECK[name] = result["items"]
+        fleet_items.extend(result["items"])
+        per_agent.append(
+            {
+                "agent": name,
+                "summary": result["summary"],
+                "incomplete": result.get("incomplete", False),
+                "cost": result.get("cost"),
+                "report": result["json"],
+            }
+        )
+
+    code = await run_fleet(run_mode, None, MAESTRO_AGENT_MAX_ITER, renderer=_RENDERER, on_result=_record)
+    totals = {key: sum(a["summary"][key] for a in per_agent) for key in ("total", "errors", "warnings", "fixed", "unfixed")}
+    incomplete_agents = [a["agent"] for a in per_agent if a["incomplete"]]
+    _LAST_RESULTS["fleet"] = {"summary": totals, "items": fleet_items, "json": ""}
+    return json.dumps(
+        {
+            "status": "success",
+            "mode": run_mode,
+            "fleet_exit_code": code,
+            "totals": totals,
+            "incomplete_agents": incomplete_agents,
+            "agents": per_agent,
+            "note": "Per-agent results are cached. Summarize directly from this payload and use read_report(agent) for detail; do NOT re-run agents to gather results.",
+        }
+    )
+
+
+mark_orchestration_tools(
+    "list_agents",
+    "read_report",
+    "run_security",
+    "run_audit",
+    "run_devii",
+    "run_docs",
+    "run_fanout",
+    "run_dry",
+    "run_style",
+    "run_frontend",
+    "run_seo",
+    "run_tests",
+    "run_fleet_tool",
+)
+
+
+MAESTRO_TOOLS = (
+    "list_agents",
+    "read_report",
+    "run_security",
+    "run_audit",
+    "run_devii",
+    "run_docs",
+    "run_fanout",
+    "run_dry",
+    "run_style",
+    "run_frontend",
+    "run_seo",
+    "run_tests",
+    "run_fleet_tool",
+    "read_file",
+    "read_lines",
+    "grep",
+    "glob_files",
+    "list_dir",
+    "retrieve",
+    "get_current_isodate",
+)
+
+MAESTRO_PROMPT = """You are Maestro, the single conversational conductor of the DevPlace maintenance fleet. The operator talks to you and you master the rest: you decide which specialist agent or combination answers a request, run it, and explain the result in plain language.
+
+THE FLEET (run each via its run_* tool):
+- run_security: data and role security
+- run_audit: audit-log coverage
+- run_devii: Devii capability and role-gated tool list
+- run_docs: documentation coverage and role-aware show/hide
+- run_fanout: cross-layer feature completeness
+- run_dry: duplication and reuse
+- run_style: coding-rule compliance
+- run_frontend: ES6, component, and CSS consistency
+- run_seo: SEO and sitemap coverage
+- run_tests: integration-test coverage (writes tests, never runs the suite)
+- run_fleet_tool: the whole fleet in dependency order
+
+ROUTING: map the request to the right dimension and call its tool. Use list_agents if unsure. For an "everything" request use run_fleet_tool EXACTLY ONCE. Clarify an ambiguous request before running anything.
+
+NEVER RE-RUN TO GATHER RESULTS: run_fleet_tool already runs every agent once and returns a complete payload (per-agent summaries, totals, incomplete_agents) AND caches each agent's result. After it returns, summarize straight from that payload; use read_report(agent) or read_report("fleet") for detail. Do NOT call the individual run_* tools again after a fleet run, and do NOT run the fleet a second time, unless the operator explicitly asks you to re-run a specific agent.
+
+MODE POLICY: a question defaults to check (read-only). Run fix ONLY when the operator explicitly asks to fix, and CONFIRM before any fix run that writes, especially run_fleet_tool in fix mode. Never silently edit in answer to a question. When the operator asks to fix issues you just found in a check, call the SAME agent's run_* tool with mode=fix exactly once; it is automatically seeded with the confirmed findings and fixes them directly (do not re-run check first, and do not call it more than once).
+
+HONESTY ABOUT RESULTS (critical): the tool result includes `incomplete` and `seeded_from_check`. The agents DO have full write access (read_file, edit_file, write_file, verify) in fix mode. If a fix run reports `incomplete: true`, it ran out of its iteration budget before finishing - say exactly that and offer to run it again. If it fixed 0 of N seeded findings, report that plainly and show the operator the findings; do NOT invent a reason. NEVER claim you or the agents lack write access or tools - that is false. Never describe a run as clean or successful when `incomplete` is true.
+
+COST: every run streams its live cost after each AI call (per call and running total, with money emojis) and the tool result returns a `cost` field ({run_usd, session_usd}). When you summarize a run, mention what it cost in dollars so the operator sees how expensive each procedure was.
+
+EXPLAINING: after a run, summarize the findings in plain language and reference the report path. Use read_report to answer follow-ups without re-running. Use read_file, grep, and retrieve to ground an explanation in the actual source. Every claim you make must be backed by a tool result or a direct source read; never invent a finding.
+"""
+
+
+def _payload() -> list[dict[str, Any]]:
+    out: list[dict[str, Any]] = []
+    for name in MAESTRO_TOOLS:
+        func = get_tool(name)
+        if func is not None:
+            out.append(func._tool_payload)
+    return out
+
+
+async def _turn(messages: list[dict[str, Any]], renderer: MarkdownRenderer) -> None:
+    _set_renderer(renderer)
+    state = AgentState()
+    await react_loop(
+        messages=messages,
+        tools_payload=_payload(),
+        state=state,
+        max_iterations=60,
+        renderer=renderer,
+    )
+
+
+_OWN_OUTPUT_PREFIX = re.compile(r"^\d{1,2}:\d{2}:\d{2}\s+\+\d{1,2}:\d{2}:\d{2}")
+
+
+def _has_terminal_control(raw: str) -> bool:
+    return any(ch == "\x1b" or (ord(ch) < 32 and ch != "\t") for ch in raw)
+
+
+def _drain_stdin() -> None:
+    try:
+        if not sys.stdin.isatty():
+            return
+        import termios
+
+        termios.tcflush(sys.stdin.fileno(), termios.TCIFLUSH)
+    except Exception:  # noqa: BLE001
+        pass
+
+
+def _prompt_line() -> str:
+    _drain_stdin()
+    return input("\n> ")
+
+
+async def interactive(renderer: MarkdownRenderer) -> None:
+    renderer.print("# Maestro\nThe conductor. Ask about any quality dimension, or say `exit` to quit.")
+    messages: list[dict[str, Any]] = [{"role": "system", "content": _with_datetime(MAESTRO_PROMPT)}]
+    loop = asyncio.get_event_loop()
+    while True:
+        try:
+            raw = await loop.run_in_executor(None, _prompt_line)
+        except (EOFError, KeyboardInterrupt):
+            break
+        if _has_terminal_control(raw):
+            continue
+        line = raw.strip()
+        if not line or _OWN_OUTPUT_PREFIX.match(line):
+            continue
+        if line.lower() in ("exit", "quit"):
+            break
+        messages.append({"role": "user", "content": line})
+        await _turn(messages, renderer)
+
+
+async def _amain(argv: Optional[list[str]] = None) -> int:
+    install_timestamps()
+    parser = argparse.ArgumentParser(prog="agents.maestro", description="Maestro, the conversational maintenance conductor")
+    parser.add_argument("prompt", nargs="?", help="One-shot request; omit for interactive mode")
+    parser.add_argument("--no-color", action="store_true")
+    parser.add_argument("-v", "--verbose", action="store_true")
+    args = parser.parse_args(argv)
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+    renderer = MarkdownRenderer(use_color=not args.no_color)
+    try:
+        if args.prompt:
+            messages = [
+                {"role": "system", "content": _with_datetime(MAESTRO_PROMPT)},
+                {"role": "user", "content": args.prompt},
+            ]
+            await _turn(messages, renderer)
+        else:
+            await interactive(renderer)
+        return 0
+    finally:
+        await close_http_client()
+
+
+def main() -> None:
+    try:
+        sys.exit(asyncio.run(_amain()))
+    except KeyboardInterrupt:
+        sys.exit(130)
+
+
+if __name__ == "__main__":
+    main()

agents/orchestrator.py

@@ -0,0 +1,124 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import sys
+from datetime import datetime
+from typing import Callable, Optional
+
+from . import core
+from .agent import MarkdownRenderer, close_http_client, cost_session_total, format_usd, get_http_client, install_timestamps, usd_str
+from .fleet import REGISTRY, ordered_agents
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="agents.orchestrator", description="Run the maintenance fleet")
+    mode = parser.add_mutually_exclusive_group()
+    mode.add_argument("--fix", dest="mode", action="store_const", const="fix")
+    mode.add_argument("--check", dest="mode", action="store_const", const="check")
+    parser.set_defaults(mode="fix")
+    parser.add_argument("--only", default=None, help="Comma-separated subset of agent names")
+    parser.add_argument("--max-iter", type=int, default=120)
+    parser.add_argument("--no-color", action="store_true")
+    parser.add_argument("-v", "--verbose", action="store_true")
+    return parser
+
+
+async def run_fleet(
+    mode: str,
+    only: Optional[str],
+    max_iter: int,
+    renderer: Optional[MarkdownRenderer],
+    on_result: Optional[Callable[[str, dict], None]] = None,
+) -> int:
+    names = ordered_agents(only.split(",") if only else None)
+    started = datetime.now()
+    await get_http_client()
+
+    async def run_one(name: str) -> dict:
+        agent = REGISTRY[name]()
+        if renderer is not None:
+            renderer.print(f"\n## {name}")
+        result = await agent.run(mode, None, max_iter, renderer)
+        if on_result is not None:
+            on_result(name, result)
+        return {
+            "name": name,
+            "summary": result["summary"],
+            "incomplete": result.get("incomplete", False),
+            "exit_code": result["exit_code"],
+            "json": result["json"],
+        }
+
+    if mode == "check":
+        if renderer is not None:
+            renderer.print(f"\n# Fleet [check] running {len(names)} agents concurrently (read-only)")
+        results = list(await asyncio.gather(*(run_one(name) for name in names)))
+    else:
+        results = [await run_one(name) for name in names]
+    finished = datetime.now()
+
+    totals = {"total": 0, "errors": 0, "warnings": 0, "fixed": 0, "unfixed": 0}
+    for result in results:
+        for key in totals:
+            totals[key] += result["summary"][key]
+    exit_code = max((result["exit_code"] for result in results), default=0)
+
+    session_cost = cost_session_total()["cost"]
+    core.REPORTS_DIR.mkdir(parents=True, exist_ok=True)
+    codename = core.report_codename()
+    stamp = started.strftime("%Y%m%d-%H%M%S")
+    fleet_payload = {
+        "agent": "fleet",
+        "codename": codename,
+        "mode": mode,
+        "started_at": started.isoformat(),
+        "finished_at": finished.isoformat(),
+        "totals": totals,
+        "cost": {"session_usd": usd_str(session_cost)},
+        "agents": results,
+    }
+    fleet_json = core.REPORTS_DIR / f"fleet-{codename}-{stamp}.json"
+    fleet_json.write_text(json.dumps(fleet_payload, indent=2), encoding="utf-8")
+    core.prune_reports("fleet")
+
+    if renderer is not None:
+        rows = "\n".join(
+            f"- {result['name']}: {result['summary']['total']} findings "
+            f"({result['summary']['errors']} err), {result['summary']['fixed']} fixed"
+            for result in results
+        )
+        renderer.print(
+            f"\n# Fleet [{mode}] complete\n{rows}\n\n"
+            f"Totals: {totals['total']} findings, {totals['errors']} errors, {totals['fixed']} fixed.   "
+            f"\U0001F4B0 {format_usd(session_cost)} total session cost. "
+            f"Report: `{fleet_json}`"
+        )
+    return exit_code
+
+
+async def _amain(argv: Optional[list[str]] = None) -> int:
+    install_timestamps()
+    args = _parser().parse_args(argv)
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+    renderer = MarkdownRenderer(use_color=not args.no_color)
+    try:
+        return await run_fleet(args.mode, args.only, args.max_iter, renderer)
+    finally:
+        await close_http_client()
+
+
+def main() -> None:
+    try:
+        sys.exit(asyncio.run(_amain()))
+    except KeyboardInterrupt:
+        sys.exit(130)
+
+
+if __name__ == "__main__":
+    main()

agents/reports/seo-20260612-005119.json

@@ -0,0 +1,14 @@
+{
+  "agent": "seo",
+  "mode": "check",
+  "started_at": "2026-06-12T00:51:19.794707",
+  "finished_at": "2026-06-12T00:52:22.621735",
+  "summary": {
+    "total": 0,
+    "errors": 0,
+    "warnings": 0,
+    "fixed": 0,
+    "unfixed": 0
+  },
+  "findings": []
+}

agents/reports/seo-20260612-005119.md

@@ -0,0 +1,11 @@
+# seo report (check)
+
+- total: 0
+- errors: 0
+- warnings: 0
+- fixed: 0
+- unfixed: 0
+
+## Findings
+
+None.

agents/reports/seo-20260612-013402.json

@@ -0,0 +1,18 @@
+{
+  "agent": "seo",
+  "mode": "check",
+  "started_at": "2026-06-12T01:34:02.164037",
+  "finished_at": "2026-06-12T01:34:16.966318",
+  "summary": {
+    "total": 0,
+    "errors": 0,
+    "warnings": 0,
+    "fixed": 0,
+    "unfixed": 0
+  },
+  "cost": {
+    "run_usd": "0.00151031",
+    "session_usd": "0.00151031"
+  },
+  "findings": []
+}

agents/reports/seo-20260612-013402.md

@@ -0,0 +1,11 @@
+# seo report (check)
+
+- total: 0
+- errors: 0
+- warnings: 0
+- fixed: 0
+- unfixed: 0
+
+## Findings
+
+None.

agents/reports/style-20260612-005257.json

@@ -0,0 +1,135 @@
+{
+  "agent": "style",
+  "mode": "check",
+  "started_at": "2026-06-12T00:52:57.094561",
+  "finished_at": "2026-06-12T00:54:21.002389",
+  "summary": {
+    "total": 12,
+    "errors": 7,
+    "warnings": 5,
+    "fixed": 0,
+    "unfixed": 12
+  },
+  "findings": [
+    {
+      "severity": "error",
+      "dimension": "coding-rule-compliance",
+      "file": "devplacepy/services/bot/llm.py",
+      "line": 105,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in string literal \u2014 must be hyphen instead",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "coding-rule-compliance",
+      "file": "agents/agent.py",
+      "line": 30,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in docstring \u2014 must be hyphen instead",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "coding-rule-compliance",
+      "file": "agents/agent.py",
+      "line": 1732,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in string literal \u2014 must be hyphen instead",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "coding-rule-compliance",
+      "file": "agents/agent.py",
+      "line": 2217,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in instruction string \u2014 must be hyphen instead",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "coding-rule-compliance",
+      "file": "agents/agent.py",
+      "line": 2250,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in string literal \u2014 must be hyphen instead",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "coding-rule-compliance",
+      "file": "agents/agent.py",
+      "line": 2293,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in string literal \u2014 must be hyphen instead",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "coding-rule-compliance",
+      "file": "agents/style.py",
+      "line": 38,
+      "rule": "em-dash",
+      "message": "HTML entity — and — used in rule definition \u2014 must use hyphen instead",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "coding-rule-compliance",
+      "file": "review.md",
+      "line": 1,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used throughout documentation prose in multiple lines",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "coding-rule-compliance",
+      "file": "notifs.md",
+      "line": 256,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in documentation prose",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "coding-rule-compliance",
+      "file": "devplace.md",
+      "line": 1,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in documentation prose throughout multiple lines",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "coding-rule-compliance",
+      "file": "visuals.md",
+      "line": 8,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in documentation prose throughout multiple lines",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "coding-rule-compliance",
+      "file": "session-ses_1eae.md",
+      "line": 37,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) and — HTML entity used throughout session log prose",
+      "fixed": false,
+      "fix_summary": ""
+    }
+  ]
+}

agents/reports/style-20260612-005257.md

@@ -0,0 +1,37 @@
+# style report (check)
+
+- total: 12
+- errors: 7
+- warnings: 5
+- fixed: 0
+- unfixed: 12
+
+## Findings
+
+### agents/agent.py
+- [error] em-dash:30: Em-dash character (U+2014) used in docstring — must be hyphen instead
+- [error] em-dash:1732: Em-dash character (U+2014) used in string literal — must be hyphen instead
+- [error] em-dash:2217: Em-dash character (U+2014) used in instruction string — must be hyphen instead
+- [error] em-dash:2250: Em-dash character (U+2014) used in string literal — must be hyphen instead
+- [error] em-dash:2293: Em-dash character (U+2014) used in string literal — must be hyphen instead
+
+### agents/style.py
+- [error] em-dash:38: HTML entity — and — used in rule definition — must use hyphen instead
+
+### devplace.md
+- [warning] em-dash:1: Em-dash character (U+2014) used in documentation prose throughout multiple lines
+
+### devplacepy/services/bot/llm.py
+- [error] em-dash:105: Em-dash character (U+2014) used in string literal — must be hyphen instead
+
+### notifs.md
+- [warning] em-dash:256: Em-dash character (U+2014) used in documentation prose
+
+### review.md
+- [warning] em-dash:1: Em-dash character (U+2014) used throughout documentation prose in multiple lines
+
+### session-ses_1eae.md
+- [warning] em-dash:37: Em-dash character (U+2014) and — HTML entity used throughout session log prose
+
+### visuals.md
+- [warning] em-dash:8: Em-dash character (U+2014) used in documentation prose throughout multiple lines

agents/reports/style-20260612-005746.json

@@ -0,0 +1,495 @@
+{
+  "agent": "style",
+  "mode": "check",
+  "started_at": "2026-06-12T00:57:46.956032",
+  "finished_at": "2026-06-12T01:01:12.646284",
+  "summary": {
+    "total": 48,
+    "errors": 13,
+    "warnings": 32,
+    "fixed": 0,
+    "unfixed": 48
+  },
+  "findings": [
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/utils.py",
+      "line": 434,
+      "rule": "forbidden-suffix-_info",
+      "message": "Function name 'badge_info' uses forbidden suffix '_info'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/push.py",
+      "line": 246,
+      "rule": "forbidden-suffix-_info",
+      "message": "Variable name 'notification_info' uses forbidden suffix '_info'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/routers/services.py",
+      "line": 44,
+      "rule": "forbidden-suffix-_data",
+      "message": "Function name 'services_data' uses forbidden suffix '_data'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/routers/services.py",
+      "line": 84,
+      "rule": "forbidden-suffix-_data",
+      "message": "Function name 'service_detail_data' uses forbidden suffix '_data'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/routers/containers.py",
+      "line": 112,
+      "rule": "forbidden-suffix-_data",
+      "message": "Function name 'containers_data' uses forbidden suffix '_data'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/routers/containers_admin.py",
+      "line": 70,
+      "rule": "forbidden-suffix-_data",
+      "message": "Function name 'containers_index_data' uses forbidden suffix '_data'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/routers/admin.py",
+      "line": 109,
+      "rule": "forbidden-suffix-_data",
+      "message": "Function name 'admin_ai_usage_data' uses forbidden suffix '_data'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/services/devii/actions/catalog.py",
+      "line": 1046,
+      "rule": "forbidden-suffix-_data",
+      "message": "Action name 'admin_services_data' uses forbidden suffix '_data'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/routers/gists.py",
+      "line": 91,
+      "rule": "forbidden-suffix-_data",
+      "message": "Variable name 'gists_data' uses forbidden suffix '_data'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/services/jobs/base.py",
+      "line": 109,
+      "rule": "forbidden-suffix-_data",
+      "message": "Parameter name 'result_data' uses forbidden suffix '_data'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/services/devii/session.py",
+      "line": 264,
+      "rule": "forbidden-prefix-best_",
+      "message": "Variable name 'best_rank' uses forbidden prefix 'best_'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/schemas.py",
+      "line": 156,
+      "rule": "forbidden-prefix-my_",
+      "message": "Field name 'my_choice' uses forbidden prefix 'my_' (public API field in PollOut schema)",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/schemas.py",
+      "line": 265,
+      "rule": "forbidden-prefix-my_",
+      "message": "Field name 'my_vote' uses forbidden prefix 'my_' (public API field repeated across multiple schemas)",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/database.py",
+      "line": 589,
+      "rule": "forbidden-prefix-my_",
+      "message": "Variable name 'my_placeholders' uses forbidden prefix 'my_'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/database.py",
+      "line": 589,
+      "rule": "forbidden-prefix-my_",
+      "message": "Variable name 'my_params' uses forbidden prefix 'my_'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/database.py",
+      "line": 646,
+      "rule": "forbidden-prefix-my_",
+      "message": "Variable name 'my_choice' uses forbidden prefix 'my_'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "forbidden-names",
+      "file": "devplacepy/database.py",
+      "line": 692,
+      "rule": "forbidden-prefix-my_",
+      "message": "Variable name 'my_votes' uses forbidden prefix 'my_'",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "headers",
+      "file": "devplacepy/main.py",
+      "line": 1,
+      "rule": "missing-retoor-header",
+      "message": "Missing mandatory retoor header as first line in devplacepy core module",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "headers",
+      "file": "devplacepy/database.py",
+      "line": 1,
+      "rule": "missing-retoor-header",
+      "message": "Missing mandatory retoor header as first line",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "headers",
+      "file": "devplacepy/schemas.py",
+      "line": 1,
+      "rule": "missing-retoor-header",
+      "message": "Missing mandatory retoor header as first line",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "headers",
+      "file": "devplacepy/utils.py",
+      "line": 1,
+      "rule": "missing-retoor-header",
+      "message": "Missing mandatory retoor header as first line",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "headers",
+      "file": "devplacepy/models.py",
+      "line": 1,
+      "rule": "missing-retoor-header",
+      "message": "Missing mandatory retoor header as first line",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "headers",
+      "file": "devplacepy/config.py",
+      "line": 1,
+      "rule": "missing-retoor-header",
+      "message": "Missing mandatory retoor header as first line",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "headers",
+      "file": "devplacepy/routers/admin.py",
+      "line": 1,
+      "rule": "missing-retoor-header",
+      "message": "Missing mandatory retoor header - many files in devplacepy/routers/ directory missing header",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "headers",
+      "file": "devplacepy/services/base.py",
+      "line": 1,
+      "rule": "missing-retoor-header",
+      "message": "Missing mandatory retoor header - many files in devplacepy/services/ directory missing header",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "headers",
+      "file": "devplacepy/services/bot/bot.py",
+      "line": 1,
+      "rule": "missing-retoor-header",
+      "message": "Missing mandatory retoor header - bot service files missing header",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "headers",
+      "file": "devplacepy/services/openai_gateway/gateway.py",
+      "line": 1,
+      "rule": "missing-retoor-header",
+      "message": "Missing mandatory retoor header - gateway service files missing header",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "headers",
+      "file": "devplacepy/static/js/ApiDocs.js",
+      "line": 1,
+      "rule": "missing-retoor-header",
+      "message": "Missing mandatory retoor header - many JS files in devplacepy/static/js/ missing header (e.g., ApiDocs.js, ApiKeyManager.js, ApiTester.js, Application.js, etc.)",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "headers",
+      "file": "devplacepy/static/css/base.css",
+      "line": 1,
+      "rule": "missing-retoor-header",
+      "message": "Missing mandatory retoor header - many CSS files missing header (e.g., base.css, variables.css, post.css, auth.css, etc.)",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "headers",
+      "file": "devplacepy/templates/base.html",
+      "line": 1,
+      "rule": "missing-retoor-header",
+      "message": "Missing mandatory retoor header - HTML template files missing header",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "typing",
+      "file": "devplacepy/cli.py",
+      "line": 7,
+      "rule": "missing-type-annotations",
+      "message": "Function '_audit_cli' missing type annotations for all parameters and return type",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "typing",
+      "file": "devplacepy/seo.py",
+      "line": 20,
+      "rule": "missing-type-annotations",
+      "message": "Function 'truncate' missing type annotations for 'text' parameter and return type",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "typing",
+      "file": "devplacepy/attachments.py",
+      "line": 109,
+      "rule": "missing-type-annotations",
+      "message": "Function 'allowed_extensions' missing return type annotation",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "typing",
+      "file": "devplacepy/templating.py",
+      "line": 21,
+      "rule": "missing-type-annotations",
+      "message": "Function 'guest_disabled' missing type annotations for 'user' parameter and return type",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "typing",
+      "file": "devplacepy/templating.py",
+      "line": 27,
+      "rule": "missing-type-annotations",
+      "message": "Function 'login_hint' missing type annotations for 'user' parameter and return type",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "typing",
+      "file": "devplacepy/main.py",
+      "line": 97,
+      "rule": "missing-type-annotations",
+      "message": "Function 'init_lock' missing return type annotation",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "typing",
+      "file": "devplacepy/database.py",
+      "line": 94,
+      "rule": "missing-type-annotations",
+      "message": "Function '_index' missing type annotations for all parameters",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "typing",
+      "file": "devplacepy/database.py",
+      "line": 103,
+      "rule": "missing-type-annotations",
+      "message": "Function 'init_db' missing return type annotation",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "typing",
+      "file": "devplacepy/seo.py",
+      "line": 36,
+      "rule": "missing-type-annotations",
+      "message": "Function 'site_url' missing type annotations for 'request' parameter and return type",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "typing",
+      "file": "devplacepy/cli.py",
+      "line": 7,
+      "rule": "missing-type-annotations-bulk",
+      "message": "Many functions in devplacepy/cli.py, seo.py, attachments.py, templating.py, docs_api.py, docs_examples.py, database.py lack full type annotations on parameters and return types",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "pathlib",
+      "file": "devplacepy/services/jobs/zip_worker.py",
+      "line": 22,
+      "rule": "use-pathlib",
+      "message": "Uses os.path.getsize, os.path.join, os.path.relpath, os.path.isdir instead of pathlib.Path equivalents",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "pathlib",
+      "file": "devplacepy/services/bot/registry.py",
+      "line": 47,
+      "rule": "use-pathlib",
+      "message": "Uses os.path.getsize instead of Path.stat().st_size",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "pathlib",
+      "file": "devplacepy/project_files.py",
+      "line": 528,
+      "rule": "use-pathlib",
+      "message": "Uses os.walk instead of Path.rglob or Path.iterdir",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "pathlib",
+      "file": "devplacepy/services/jobs/zip_worker.py",
+      "line": 18,
+      "rule": "use-pathlib",
+      "message": "Uses os.walk instead of pathlib.Path.rglob",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "warning",
+      "dimension": "magic-numbers",
+      "file": "devplacepy/routers/avatar.py",
+      "line": 12,
+      "rule": "magic-number",
+      "message": "Magic number 4096 used as cache max_size; should be a named constant",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "info",
+      "dimension": "magic-numbers",
+      "file": "devplacepy/attachments.py",
+      "line": 106,
+      "rule": "magic-number",
+      "message": "Magic number 1024*1024 used repeatedly for bytes conversion; should use a named constant like BYTES_PER_MB",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "info",
+      "dimension": "magic-numbers",
+      "file": "devplacepy/attachments.py",
+      "line": 499,
+      "rule": "magic-number",
+      "message": "Repeated literal 1024 in format_file_size; should use named constant",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "info",
+      "dimension": "version-pinning",
+      "file": "pyproject.toml",
+      "line": 0,
+      "rule": "no-pinning-convention",
+      "message": "No version pinning found in pyproject.toml - dependencies use unversioned constraints (which is the desired pattern)",
+      "fixed": false,
+      "fix_summary": ""
+    }
+  ]
+}

agents/reports/style-20260612-005746.md

@@ -0,0 +1,117 @@
+# style report (check)
+
+- total: 48
+- errors: 13
+- warnings: 32
+- fixed: 0
+- unfixed: 48
+
+## Findings
+
+### devplacepy/attachments.py
+- [warning] missing-type-annotations:109: Function 'allowed_extensions' missing return type annotation
+- [info] magic-number:106: Magic number 1024*1024 used repeatedly for bytes conversion; should use a named constant like BYTES_PER_MB
+- [info] magic-number:499: Repeated literal 1024 in format_file_size; should use named constant
+
+### devplacepy/cli.py
+- [warning] missing-type-annotations:7: Function '_audit_cli' missing type annotations for all parameters and return type
+- [warning] missing-type-annotations-bulk:7: Many functions in devplacepy/cli.py, seo.py, attachments.py, templating.py, docs_api.py, docs_examples.py, database.py lack full type annotations on parameters and return types
+
+### devplacepy/config.py
+- [error] missing-retoor-header:1: Missing mandatory retoor header as first line
+
+### devplacepy/database.py
+- [warning] forbidden-prefix-my_:589: Variable name 'my_placeholders' uses forbidden prefix 'my_'
+- [warning] forbidden-prefix-my_:589: Variable name 'my_params' uses forbidden prefix 'my_'
+- [warning] forbidden-prefix-my_:646: Variable name 'my_choice' uses forbidden prefix 'my_'
+- [warning] forbidden-prefix-my_:692: Variable name 'my_votes' uses forbidden prefix 'my_'
+- [error] missing-retoor-header:1: Missing mandatory retoor header as first line
+- [warning] missing-type-annotations:94: Function '_index' missing type annotations for all parameters
+- [warning] missing-type-annotations:103: Function 'init_db' missing return type annotation
+
+### devplacepy/main.py
+- [error] missing-retoor-header:1: Missing mandatory retoor header as first line in devplacepy core module
+- [warning] missing-type-annotations:97: Function 'init_lock' missing return type annotation
+
+### devplacepy/models.py
+- [error] missing-retoor-header:1: Missing mandatory retoor header as first line
+
+### devplacepy/project_files.py
+- [warning] use-pathlib:528: Uses os.walk instead of Path.rglob or Path.iterdir
+
+### devplacepy/push.py
+- [warning] forbidden-suffix-_info:246: Variable name 'notification_info' uses forbidden suffix '_info'
+
+### devplacepy/routers/admin.py
+- [warning] forbidden-suffix-_data:109: Function name 'admin_ai_usage_data' uses forbidden suffix '_data'
+- [error] missing-retoor-header:1: Missing mandatory retoor header - many files in devplacepy/routers/ directory missing header
+
+### devplacepy/routers/avatar.py
+- [warning] magic-number:12: Magic number 4096 used as cache max_size; should be a named constant
+
+### devplacepy/routers/containers.py
+- [warning] forbidden-suffix-_data:112: Function name 'containers_data' uses forbidden suffix '_data'
+
+### devplacepy/routers/containers_admin.py
+- [warning] forbidden-suffix-_data:70: Function name 'containers_index_data' uses forbidden suffix '_data'
+
+### devplacepy/routers/gists.py
+- [warning] forbidden-suffix-_data:91: Variable name 'gists_data' uses forbidden suffix '_data'
+
+### devplacepy/routers/services.py
+- [warning] forbidden-suffix-_data:44: Function name 'services_data' uses forbidden suffix '_data'
+- [warning] forbidden-suffix-_data:84: Function name 'service_detail_data' uses forbidden suffix '_data'
+
+### devplacepy/schemas.py
+- [warning] forbidden-prefix-my_:156: Field name 'my_choice' uses forbidden prefix 'my_' (public API field in PollOut schema)
+- [warning] forbidden-prefix-my_:265: Field name 'my_vote' uses forbidden prefix 'my_' (public API field repeated across multiple schemas)
+- [error] missing-retoor-header:1: Missing mandatory retoor header as first line
+
+### devplacepy/seo.py
+- [warning] missing-type-annotations:20: Function 'truncate' missing type annotations for 'text' parameter and return type
+- [warning] missing-type-annotations:36: Function 'site_url' missing type annotations for 'request' parameter and return type
+
+### devplacepy/services/base.py
+- [error] missing-retoor-header:1: Missing mandatory retoor header - many files in devplacepy/services/ directory missing header
+
+### devplacepy/services/bot/bot.py
+- [error] missing-retoor-header:1: Missing mandatory retoor header - bot service files missing header
+
+### devplacepy/services/bot/registry.py
+- [warning] use-pathlib:47: Uses os.path.getsize instead of Path.stat().st_size
+
+### devplacepy/services/devii/actions/catalog.py
+- [warning] forbidden-suffix-_data:1046: Action name 'admin_services_data' uses forbidden suffix '_data'
+
+### devplacepy/services/devii/session.py
+- [warning] forbidden-prefix-best_:264: Variable name 'best_rank' uses forbidden prefix 'best_'
+
+### devplacepy/services/jobs/base.py
+- [warning] forbidden-suffix-_data:109: Parameter name 'result_data' uses forbidden suffix '_data'
+
+### devplacepy/services/jobs/zip_worker.py
+- [warning] use-pathlib:22: Uses os.path.getsize, os.path.join, os.path.relpath, os.path.isdir instead of pathlib.Path equivalents
+- [warning] use-pathlib:18: Uses os.walk instead of pathlib.Path.rglob
+
+### devplacepy/services/openai_gateway/gateway.py
+- [error] missing-retoor-header:1: Missing mandatory retoor header - gateway service files missing header
+
+### devplacepy/static/css/base.css
+- [error] missing-retoor-header:1: Missing mandatory retoor header - many CSS files missing header (e.g., base.css, variables.css, post.css, auth.css, etc.)
+
+### devplacepy/static/js/ApiDocs.js
+- [error] missing-retoor-header:1: Missing mandatory retoor header - many JS files in devplacepy/static/js/ missing header (e.g., ApiDocs.js, ApiKeyManager.js, ApiTester.js, Application.js, etc.)
+
+### devplacepy/templates/base.html
+- [error] missing-retoor-header:1: Missing mandatory retoor header - HTML template files missing header
+
+### devplacepy/templating.py
+- [warning] missing-type-annotations:21: Function 'guest_disabled' missing type annotations for 'user' parameter and return type
+- [warning] missing-type-annotations:27: Function 'login_hint' missing type annotations for 'user' parameter and return type
+
+### devplacepy/utils.py
+- [warning] forbidden-suffix-_info:434: Function name 'badge_info' uses forbidden suffix '_info'
+- [error] missing-retoor-header:1: Missing mandatory retoor header as first line
+
+### pyproject.toml
+- [info] no-pinning-convention: No version pinning found in pyproject.toml - dependencies use unversioned constraints (which is the desired pattern)

agents/reports/style-20260612-010202.json

@@ -0,0 +1,95 @@
+{
+  "agent": "style",
+  "mode": "check",
+  "started_at": "2026-06-12T01:02:02.351939",
+  "finished_at": "2026-06-12T01:03:54.924366",
+  "summary": {
+    "total": 8,
+    "errors": 8,
+    "warnings": 0,
+    "fixed": 0,
+    "unfixed": 8
+  },
+  "findings": [
+    {
+      "severity": "error",
+      "dimension": "em-dash",
+      "file": "agents/agent.py",
+      "line": 30,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in docstring \u2014 must be hyphen instead",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "em-dash",
+      "file": "agents/agent.py",
+      "line": 1732,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in string literal \u2014 must be hyphen instead",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "em-dash",
+      "file": "agents/agent.py",
+      "line": 2217,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in instruction string \u2014 must be hyphen instead",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "em-dash",
+      "file": "agents/agent.py",
+      "line": 2250,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in string literal \u2014 must be hyphen instead",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "em-dash",
+      "file": "agents/agent.py",
+      "line": 2293,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in string literal \u2014 must be hyphen instead",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "em-dash",
+      "file": "devplacepy/services/bot/llm.py",
+      "line": 105,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in string literal as replace target \u2014 must be hyphen or unicode escape instead",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "em-dash",
+      "file": "tests/test_demo.py.bak",
+      "line": 36,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in comments throughout file \u2014 must be hyphen instead",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "em-dash",
+      "file": "tests/test_demo.py.bak",
+      "line": 277,
+      "rule": "em-dash",
+      "message": "Em-dash character (U+2014) used in string literal \u2014 must be hyphen instead",
+      "fixed": false,
+      "fix_summary": ""
+    }
+  ]
+}

agents/reports/style-20260612-010202.md

@@ -0,0 +1,23 @@
+# style report (check)
+
+- total: 8
+- errors: 8
+- warnings: 0
+- fixed: 0
+- unfixed: 8
+
+## Findings
+
+### agents/agent.py
+- [error] em-dash:30: Em-dash character (U+2014) used in docstring — must be hyphen instead
+- [error] em-dash:1732: Em-dash character (U+2014) used in string literal — must be hyphen instead
+- [error] em-dash:2217: Em-dash character (U+2014) used in instruction string — must be hyphen instead
+- [error] em-dash:2250: Em-dash character (U+2014) used in string literal — must be hyphen instead
+- [error] em-dash:2293: Em-dash character (U+2014) used in string literal — must be hyphen instead
+
+### devplacepy/services/bot/llm.py
+- [error] em-dash:105: Em-dash character (U+2014) used in string literal as replace target — must be hyphen or unicode escape instead
+
+### tests/test_demo.py.bak
+- [error] em-dash:36: Em-dash character (U+2014) used in comments throughout file — must be hyphen instead
+- [error] em-dash:277: Em-dash character (U+2014) used in string literal — must be hyphen instead

agents/reports/style-20260612-010235.json

@@ -0,0 +1,14 @@
+{
+  "agent": "style",
+  "mode": "check",
+  "started_at": "2026-06-12T01:02:35.851370",
+  "finished_at": "2026-06-12T01:05:34.005737",
+  "summary": {
+    "total": 0,
+    "errors": 0,
+    "warnings": 0,
+    "fixed": 0,
+    "unfixed": 0
+  },
+  "findings": []
+}

agents/reports/style-20260612-010235.md

@@ -0,0 +1,11 @@
+# style report (check)
+
+- total: 0
+- errors: 0
+- warnings: 0
+- fixed: 0
+- unfixed: 0
+
+## Findings
+
+None.

agents/reports/style-20260612-010550.json

@@ -0,0 +1,125 @@
+{
+  "agent": "style",
+  "mode": "check",
+  "started_at": "2026-06-12T01:05:50.761027",
+  "finished_at": "2026-06-12T01:07:16.324649",
+  "summary": {
+    "total": 11,
+    "errors": 6,
+    "warnings": 0,
+    "fixed": 0,
+    "unfixed": 11
+  },
+  "findings": [
+    {
+      "severity": "error",
+      "dimension": "em-dash",
+      "file": "agents/agent.py",
+      "line": 30,
+      "rule": "no-em-dash",
+      "message": "Em-dash character (U+2014) used in comment text instead of a hyphen.",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "em-dash",
+      "file": "agents/agent.py",
+      "line": 1732,
+      "rule": "no-em-dash",
+      "message": "Em-dash character (U+2014) used in advice string literal instead of a hyphen.",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "em-dash",
+      "file": "agents/agent.py",
+      "line": 2217,
+      "rule": "no-em-dash",
+      "message": "Em-dash character (U+2014) used in comment text instead of a hyphen.",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "em-dash",
+      "file": "agents/agent.py",
+      "line": 2250,
+      "rule": "no-em-dash",
+      "message": "Em-dash character (U+2014) used in f-string instead of a hyphen.",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "em-dash",
+      "file": "agents/agent.py",
+      "line": 2293,
+      "rule": "no-em-dash",
+      "message": "Em-dash character (U+2014) used in argparse description string instead of a hyphen.",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "error",
+      "dimension": "em-dash",
+      "file": "devplacepy/services/bot/llm.py",
+      "line": 105,
+      "rule": "no-em-dash",
+      "message": "Em-dash character (U+2014) used in string literal for text.replace() instead of a hyphen or escape sequence.",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "info",
+      "dimension": "em-dash",
+      "file": "tests/test_demo.py.bak",
+      "line": 36,
+      "rule": "no-em-dash",
+      "message": "Backup test file contains 15 em-dash characters (U+2014) in comments and string literals.",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "info",
+      "dimension": "em-dash",
+      "file": "review.md",
+      "line": 1,
+      "rule": "no-em-dash",
+      "message": "Documentation file review.md uses em-dash characters throughout (standard English prose, not code).",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "info",
+      "dimension": "em-dash",
+      "file": "notifs.md",
+      "line": 256,
+      "rule": "no-em-dash",
+      "message": "Documentation file notifs.md uses em-dash characters throughout (standard English prose, not code).",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "info",
+      "dimension": "em-dash",
+      "file": "devplace.md",
+      "line": 1,
+      "rule": "no-em-dash",
+      "message": "Documentation file devplace.md uses em-dash characters throughout (standard English prose, not code).",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "info",
+      "dimension": "em-dash",
+      "file": "visuals.md",
+      "line": 8,
+      "rule": "no-em-dash",
+      "message": "Documentation file visuals.md uses em-dash characters throughout (standard English prose, not code).",
+      "fixed": false,
+      "fix_summary": ""
+    }
+  ]
+}

agents/reports/style-20260612-010550.md

@@ -0,0 +1,34 @@
+# style report (check)
+
+- total: 11
+- errors: 6
+- warnings: 0
+- fixed: 0
+- unfixed: 11
+
+## Findings
+
+### agents/agent.py
+- [error] no-em-dash:30: Em-dash character (U+2014) used in comment text instead of a hyphen.
+- [error] no-em-dash:1732: Em-dash character (U+2014) used in advice string literal instead of a hyphen.
+- [error] no-em-dash:2217: Em-dash character (U+2014) used in comment text instead of a hyphen.
+- [error] no-em-dash:2250: Em-dash character (U+2014) used in f-string instead of a hyphen.
+- [error] no-em-dash:2293: Em-dash character (U+2014) used in argparse description string instead of a hyphen.
+
+### devplace.md
+- [info] no-em-dash:1: Documentation file devplace.md uses em-dash characters throughout (standard English prose, not code).
+
+### devplacepy/services/bot/llm.py
+- [error] no-em-dash:105: Em-dash character (U+2014) used in string literal for text.replace() instead of a hyphen or escape sequence.
+
+### notifs.md
+- [info] no-em-dash:256: Documentation file notifs.md uses em-dash characters throughout (standard English prose, not code).
+
+### review.md
+- [info] no-em-dash:1: Documentation file review.md uses em-dash characters throughout (standard English prose, not code).
+
+### tests/test_demo.py.bak
+- [info] no-em-dash:36: Backup test file contains 15 em-dash characters (U+2014) in comments and string literals.
+
+### visuals.md
+- [info] no-em-dash:8: Documentation file visuals.md uses em-dash characters throughout (standard English prose, not code).

agents/reports/style-20260612-010804.json

@@ -0,0 +1,85 @@
+{
+  "agent": "style",
+  "mode": "fix",
+  "started_at": "2026-06-12T01:08:04.361327",
+  "finished_at": "2026-06-12T01:10:20.066323",
+  "summary": {
+    "total": 7,
+    "errors": 5,
+    "warnings": 0,
+    "fixed": 5,
+    "unfixed": 2
+  },
+  "findings": [
+    {
+      "severity": "error",
+      "dimension": "coding-rule-compliance",
+      "file": "agents/agent.py",
+      "line": 30,
+      "rule": "no-em-dash",
+      "message": "Em-dash character (U+2014) used instead of hyphen in docstring",
+      "fixed": true,
+      "fix_summary": "Replaced em-dash with hyphen in docstring about Chrome-accurate header order"
+    },
+    {
+      "severity": "error",
+      "dimension": "coding-rule-compliance",
+      "file": "agents/agent.py",
+      "line": 1732,
+      "rule": "no-em-dash",
+      "message": "Em-dash character (U+2014) used instead of hyphen in advice string literal",
+      "fixed": true,
+      "fix_summary": "Replaced em-dash with hyphen in plan confidence advice string"
+    },
+    {
+      "severity": "error",
+      "dimension": "coding-rule-compliance",
+      "file": "agents/agent.py",
+      "line": 2217,
+      "rule": "no-em-dash",
+      "message": "Em-dash character (U+2014) used instead of hyphen in SYSTEM_PROMPT string",
+      "fixed": true,
+      "fix_summary": "Replaced em-dash with hyphen in operating protocol text"
+    },
+    {
+      "severity": "error",
+      "dimension": "coding-rule-compliance",
+      "file": "agents/agent.py",
+      "line": 2250,
+      "rule": "no-em-dash",
+      "message": "Em-dash character (U+2014) used instead of hyphen in _with_datetime() f-string",
+      "fixed": true,
+      "fix_summary": "Replaced em-dash with hyphen in datetime message string"
+    },
+    {
+      "severity": "error",
+      "dimension": "coding-rule-compliance",
+      "file": "agents/agent.py",
+      "line": 2293,
+      "rule": "no-em-dash",
+      "message": "Em-dash character (U+2014) used instead of hyphen in argparse description",
+      "fixed": true,
+      "fix_summary": "Replaced em-dash with hyphen in argument parser description"
+    },
+    {
+      "severity": "info",
+      "dimension": "coding-rule-compliance",
+      "file": "devplacepy/services/bot/llm.py",
+      "line": 105,
+      "rule": "no-em-dash",
+      "message": "Em-dash characters in llm.py replace() pattern are intentional data processing logic; not changed",
+      "fixed": false,
+      "fix_summary": ""
+    },
+    {
+      "severity": "info",
+      "dimension": "coding-rule-compliance",
+      "file": "devplacepy/static/uploads/attachments/1a/64/1a640917-f599-4fce-8dc4-2babdb25f716.py",
+      "line": 3,
+      "rule": "no-em-dash",
+      "message": "Em-dash in user-uploaded attachment file; not project source",
+      "fixed": false,
+      "fix_summary": ""
+    }
+  ]
+}

agents/reports/style-20260612-010804.md

@@ -0,0 +1,27 @@
+# style report (fix)
+
+- total: 7
+- errors: 5
+- warnings: 0
+- fixed: 5
+- unfixed: 2
+
+## Findings
+
+### agents/agent.py
+- [fixed] no-em-dash:30: Em-dash character (U+2014) used instead of hyphen in docstring
+  fix: Replaced em-dash with hyphen in docstring about Chrome-accurate header order
+- [fixed] no-em-dash:1732: Em-dash character (U+2014) used instead of hyphen in advice string literal
+  fix: Replaced em-dash with hyphen in plan confidence advice string
+- [fixed] no-em-dash:2217: Em-dash character (U+2014) used instead of hyphen in SYSTEM_PROMPT string
+  fix: Replaced em-dash with hyphen in operating protocol text
+- [fixed] no-em-dash:2250: Em-dash character (U+2014) used instead of hyphen in _with_datetime() f-string
+  fix: Replaced em-dash with hyphen in datetime message string
+- [fixed] no-em-dash:2293: Em-dash character (U+2014) used instead of hyphen in argparse description
+  fix: Replaced em-dash with hyphen in argument parser description
+
+### devplacepy/services/bot/llm.py
+- [info] no-em-dash:105: Em-dash characters in llm.py replace() pattern are intentional data processing logic; not changed
+
+### devplacepy/static/uploads/attachments/1a/64/1a640917-f599-4fce-8dc4-2babdb25f716.py
+- [info] no-em-dash:3: Em-dash in user-uploaded attachment file; not project source

agents/security.py

@@ -0,0 +1,54 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .base import MaintenanceAgent, cli_main
+
+
+class SecurityAgent(MaintenanceAgent):
+    name = "security"
+    description = "data and role security checker"
+
+    def mandate(self) -> str:
+        return (
+            "Guarantee that every state-changing action is correctly authorized, every private resource is gated by "
+            "the single canonical predicate, every file mutation is read-only-guarded, and the input and output "
+            "boundaries are sanitized.\n\n"
+            "DETECT:\n"
+            "- Every @router.post / @router.put / @router.delete has the correct guard: require_user for member writes, "
+            "require_admin for admin writes, or an explicit ownership comparison resource[\"user_uid\"] == user[\"uid\"] "
+            "before edit and delete. A POST with no guard is an error.\n"
+            "- Every private-project read surface flows through content.can_view_project(project, user) and none "
+            "re-implements the owner-or-admin check inline. Surfaces: project detail, project_files._load_viewable_project, "
+            "zip enqueue, listing, profile project list, sitemap.\n"
+            "- Every file-mutating entrypoint in project_files.py calls project_files._guard_writable(project_uid).\n"
+            "- Devii irreversible or destructive actions are present in the dispatcher CONFIRM_REQUIRED set, and "
+            "destructive shell commands match dispatcher.DESTRUCTIVE_COMMAND.\n"
+            "- Input is Pydantic-validated with explicit max lengths (models.py Form models); uploads and downloads are "
+            "slugified; path traversal is blocked with pathlib, never string joins.\n"
+            "- Passwords are hashed with pbkdf2_sha256 via passlib; no plaintext or weak path exists.\n"
+            "- Capability URLs (zip and fork status and download) stay scoped only by the unguessable uuid7.\n"
+            "- The XSS control is intact: DOMPurify.sanitize runs on raw marked output in static/js/components/ContentRenderer.js "
+            "and fails closed; seo.py _json_ld_dumps escapes <, >, & in JSON-LD.\n\n"
+            "FIX: insert the missing guard, route the read through can_view_project, add _guard_writable at the top of the "
+            "mutating function, add the action to the confirm set, add the missing max length or validator, or restore the "
+            "sanitize step. Never weaken a guard to make a finding disappear; a deliberately public read is an info finding."
+        )
+
+    def scope_units(self) -> list[tuple[str, str]]:
+        return [
+            ("routers", "devplacepy/routers/*.py - guard on every POST/PUT/DELETE; ownership before edit/delete"),
+            ("project-visibility", "devplacepy/content.py can_view_project used at every private read surface"),
+            ("project-files", "devplacepy/project_files.py _guard_writable on every mutating entrypoint"),
+            ("devii-confirm", "devplacepy/services/devii/actions/dispatcher.py CONFIRM_REQUIRED and DESTRUCTIVE_COMMAND"),
+            ("input-validation", "devplacepy/models.py max lengths; path traversal via pathlib; slugify on upload/download"),
+            ("xss", "static/js/components/ContentRenderer.js DOMPurify; devplacepy/seo.py _json_ld_dumps escaping"),
+        ]
+
+
+def main() -> None:
+    cli_main(SecurityAgent())
+
+
+if __name__ == "__main__":
+    main()

agents/seo.py

@@ -0,0 +1,40 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .base import MaintenanceAgent, cli_main
+
+
+class SeoAgent(MaintenanceAgent):
+    name = "seo"
+    description = "SEO and sitemap coverage"
+
+    def mandate(self) -> str:
+        return (
+            "Ensure every public page is correctly described for search and indexed where appropriate.\n\n"
+            "DETECT:\n"
+            "- Every public page builds base_seo_context(request, ...) and merges it into the template response.\n"
+            "- The right JSON-LD schema is emitted (WebSite, BreadcrumbList, DiscussionForumPosting, ProfilePage, "
+            "SoftwareApplication).\n"
+            "- meta_robots is set, and the noindex rules hold (auth, messages, notifications are noindex,nofollow; profiles "
+            "with fewer than two posts are noindex,follow).\n"
+            "- Indexable public pages appear in the routers/seo.py sitemap.\n\n"
+            "FIX: add the missing base_seo_context call, the JSON-LD schema, the robots directive, or the sitemap entry. "
+            "Never index a private or auth-gated page."
+        )
+
+    def scope_units(self) -> list[tuple[str, str]]:
+        return [
+            ("seo-context", "public page routes build seo.base_seo_context"),
+            ("json-ld", "the correct JSON-LD schema is emitted per page type"),
+            ("robots", "meta_robots set; noindex rules for auth/messages/notifications/thin profiles"),
+            ("sitemap", "indexable public pages appear in routers/seo.py sitemap"),
+        ]
+
+
+def main() -> None:
+    cli_main(SeoAgent())
+
+
+if __name__ == "__main__":
+    main()

agents/style.py

@@ -0,0 +1,90 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .base import MaintenanceAgent, cli_main
+
+
+class StyleAgent(MaintenanceAgent):
+    name = "style"
+    description = "coding-rule compliance"
+
+    def mandate(self) -> str:
+        return (
+            "Enforce the explicit CLAUDE.md and AGENTS.md coding rules across all source.\n\n"
+            "DETECT (mostly deterministic, grep and AST):\n"
+            "- Forbidden naming prefixes and suffixes, CONTEXT-AWARE: the banned tokens are _new, _old, _current, _prev, "
+            "_next (outside iteration), _temp, _tmp, _v1/_v2/_v3, better_, best_, simple_, my_, the_, _data, _info, and the "
+            "rest of the forbidden list. This rule targets LAZY, RENAMEABLE VARIABLE AND HELPER names you own - it is NOT a "
+            "blind substring sweep, and most surface hits on _data/_info/_item/_val are FALSE POSITIVES. Run this exact "
+            "decision algorithm for EVERY candidate before recording it, and skip it the moment any test below fails:\n"
+            "    STEP 1 - IS IT A CONTRACT IDENTIFIER? Resolve what the name actually is. If it is a string that other code, "
+            "templates, the database, the API, or docs reference by that exact spelling, it is a CONTRACT and renaming it is "
+            "a breaking change (Doctrine C and D), NOT a style fix. Contract identifiers include: a Jinja template global or "
+            "filter (templates.env.globals[...] / env.filters[...], called as {{ name(...) }} in .html), a Devii action or "
+            "tool name=, a route path or endpoint, a DB table or column, a Pydantic or dataclass FIELD, a JSON response key, "
+            "an audit event key, a site_settings/config/env key, a CSS class, or a JS export. For ANY contract identifier: do "
+            "NOT flag it as a forbidden-name violation and NEVER rename it; at most record ONE info finding noting the "
+            "convention. (Examples that are contracts, hence NOT violations: the template global badge_info; the Devii action "
+            "admin_services_data.)\n"
+            "    STEP 2 - SUBSTANCE TEST (only for a genuinely local/private, freely-renameable name). Ask: is the trailing "
+            "(or leading) token a VAGUE PLACEHOLDER that adds zero information, so the name means exactly the same thing "
+            "without it? Real violations: users_new -> users_active, connection_old, my_config -> config, result_val -> "
+            "result, payload_obj -> payload, user_data -> user. It is a FALSE POSITIVE (do NOT flag) when: the token is the "
+            "actual domain noun or a real concept here (an audit event, a metrics sample, a request's data body of a "
+            "data endpoint, badge info as a real thing); OR the token is part of a larger real word or compound (data inside "
+            "metadata, info inside a normal word, next/prev as loop iterators); OR dropping it would collide with another "
+            "name in scope or lose genuine meaning; OR it matches a well-known external library/framework name.\n"
+            "    STEP 3 - CONFIDENCE GATE. Record a forbidden-name WARNING only if, after steps 1-2, you are CERTAIN it is a "
+            "renameable local name whose token is pure placeholder AND you can state the safe replacement and have checked "
+            "its references (Doctrine C). Otherwise drop it or record a single info finding. A wrong rename is a regression; "
+            "when in doubt, do not flag.\n"
+            "- No comments or docstrings in source files, EXCEPT the mandatory header and the docstrings that @tool functions "
+            "require for their schema (the proven agent engine convention).\n"
+            "- Em-dash, CONTEXT-AWARE (think before you touch it): the rule bans em-dashes (U+2014, and U+2013) "
+            "that WE authored as prose - in a comment, a docstring, a user-facing string or label or error message, "
+            "markdown or template copy. An em-dash that is DATA is NOT a violation and MUST be left exactly as is: when "
+            "the character is the target or source of a transformation (str.replace, str.maketrans, a regex character "
+            "class, a sanitizer or normaliser that converts typographic punctuation to ASCII), a parser literal, or a "
+            "test fixture that deliberately feeds an em-dash to exercise handling. Rewriting such a literal negates the "
+            "code's whole purpose (e.g. a `.replace(\"<em-dash>\", \"-\")` cleaner stops cleaning). When unsure whether "
+            "an occurrence is prose or data, read the surrounding lines; if it is operated on rather than displayed, "
+            "treat it as data and skip it (record at most one info finding, never an edit).\n"
+            "- Full typing coverage on Python function signatures and variables.\n"
+            "- pathlib instead of the os module for paths.\n"
+            "- A fixed-key dict that should be a dataclass.\n"
+            "- No version pinning anywhere (pyproject, requirements, or inline).\n"
+            "- The mandatory `retoor <retoor@molodetz.nl>` header on files you CREATE or are otherwise already editing. "
+            "Do NOT sweep the whole repo adding headers: many pre-existing application files were authored without one, "
+            "and mass-inserting headers into dozens of untouched files is exactly the noise the 'refactor only what you "
+            "touch' rule forbids. If files lack the header, record at most ONE info finding stating the count, and never "
+            "auto-edit a file solely to add a header.\n"
+            "- No magic numbers; named constants instead. No warnings.\n\n"
+            "FIX: rename the symbol to an intent-revealing name, strip the stray comment or docstring, replace a PROSE "
+            "em-dash with a literal ASCII hyphen (never with the escape backslash-u-2014, which is the SAME character and "
+            "fixes nothing, and never with an HTML entity inside non-HTML source, which injects literal text into code or a "
+            "string), leaving every data em-dash untouched, "
+            "add the type annotation, convert os.path to pathlib, convert the dict to a dataclass, remove the version pin, add "
+            "the header, or name the constant. Only touch code you are already editing for a finding; do not restyle untouched "
+            "code. A rename is auto-applied ONLY for a confirmed local/private name that passed the forbidden-name decision "
+            "algorithm, and ONLY after you grep every reference and update them in the same run (Doctrine C); a rename that "
+            "would touch a contract identifier or any public API symbol is reported, never auto-applied."
+        )
+
+    def scope_units(self) -> list[tuple[str, str]]:
+        return [
+            ("forbidden-names", "devplacepy/**/*.py forbidden naming prefixes/suffixes on renameable local names only - run the decision algorithm: contract identifiers (template globals, Devii action names, DB columns, schema fields, routes, CSS/JS/config keys) and meaningful domain tokens are false positives, not violations"),
+            ("headers", "retoor header on created/edited files only; one info finding for pre-existing files that lack it, never a mass sweep"),
+            ("em-dash", "prose em-dashes (comments, docstrings, user-facing strings, markdown) become hyphens; em-dashes that are DATA (replace/maketrans/regex targets, sanitizers, fixtures) are left untouched"),
+            ("typing", "Python function signatures and variables fully typed"),
+            ("pathlib", "pathlib over the os module; no magic numbers; no version pinning"),
+            ("frontend-style", "static/js and static/css naming and constants"),
+        ]
+
+
+def main() -> None:
+    cli_main(StyleAgent())
+
+
+if __name__ == "__main__":
+    main()

agents/test_coverage.py

@@ -0,0 +1,37 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+from .base import MaintenanceAgent, cli_main
+
+
+class TestAgent(MaintenanceAgent):
+    name = "test"
+    description = "integration-test coverage"
+
+    def mandate(self) -> str:
+        return (
+            "Keep integration-test coverage in step with the routes and features, writing tests that follow the project's "
+            "required patterns.\n\n"
+            "DETECT: routes and features with no corresponding test in tests/test_{area}.py. The project prefers integration "
+            "tests over unit tests and tests the interface and API.\n\n"
+            "FIX: write the missing integration test following the required patterns: every page.goto and page.wait_for_url "
+            "passes wait_until=\"domcontentloaded\"; selectors are scoped; a test that flips a global site_settings value "
+            "restores it in try/finally; the shared fixtures (alice, bob, app_server) are used.\n\n"
+            "HARD GUARDRAIL: write tests but NEVER run the suite, not the full suite and not a single file. Validate only by a "
+            "clean import of the new test module (python -c). Never weaken an existing test to make it pass."
+        )
+
+    def scope_units(self) -> list[tuple[str, str]]:
+        return [
+            ("coverage-gaps", "routers/*.py routes with no referencing test in tests/test_{area}.py"),
+            ("pattern-lint", "tests/*.py use domcontentloaded, scoped selectors, try/finally global restore, shared fixtures"),
+        ]
+
+
+def main() -> None:
+    cli_main(TestAgent())
+
+
+if __name__ == "__main__":
+    main()

agents/validator.py

@@ -0,0 +1,150 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import argparse
+import ast
+import py_compile
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional
+
+PYTHON_SUFFIXES = (".py",)
+JS_SUFFIXES = (".js", ".mjs")
+CSS_SUFFIXES = (".css",)
+HTML_SUFFIXES = (".html", ".htm")
+
+SKIP_DIRS = {
+    ".git",
+    "__pycache__",
+    "node_modules",
+    ".venv",
+    "venv",
+    "var",
+    "htmlcov",
+    "vendor",
+    ".egg-info",
+    "downloads",
+}
+
+
+class Issue:
+    def __init__(self, path: Path, message: str) -> None:
+        self.path = path
+        self.message = message
+
+
+def _check_python(path: Path) -> Optional[str]:
+    source = path.read_text(encoding="utf-8", errors="replace")
+    try:
+        ast.parse(source, filename=str(path))
+    except SyntaxError as error:
+        return f"SyntaxError: {error.msg} (line {error.lineno})"
+    try:
+        py_compile.compile(str(path), doraise=True)
+    except py_compile.PyCompileError as error:
+        return f"CompileError: {error.msg}"
+    return None
+
+
+def _check_js(path: Path, node: Optional[str]) -> Optional[str]:
+    if node is None:
+        return None
+    result = subprocess.run(
+        [node, "--check", str(path)],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        return f"node --check failed: {result.stderr.strip().splitlines()[-1] if result.stderr.strip() else 'syntax error'}"
+    return None
+
+
+def _check_braces(path: Path) -> Optional[str]:
+    text = path.read_text(encoding="utf-8", errors="replace")
+    depth = 0
+    for char in text:
+        if char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth < 0:
+                return "Unbalanced braces: unexpected '}'"
+    if depth != 0:
+        return f"Unbalanced braces: {depth} unclosed '{{'"
+    return None
+
+
+def _check_html(path: Path) -> Optional[str]:
+    try:
+        import jinja2
+    except ImportError:
+        return None
+    source = path.read_text(encoding="utf-8", errors="replace")
+    try:
+        jinja2.Environment(autoescape=True).parse(source)
+    except jinja2.TemplateSyntaxError as error:
+        return f"TemplateSyntaxError: {error.message} (line {error.lineno})"
+    return None
+
+
+def _iter_files(root: Path):
+    if root.is_file():
+        yield root
+        return
+    for path in root.rglob("*"):
+        if not path.is_file():
+            continue
+        if any(part in SKIP_DIRS or part.endswith(".egg-info") for part in path.parts):
+            continue
+        yield path
+
+
+def validate(root: Path) -> list[Issue]:
+    node = shutil.which("node")
+    issues: list[Issue] = []
+    for path in _iter_files(root):
+        suffix = path.suffix.lower()
+        message: Optional[str] = None
+        if suffix in PYTHON_SUFFIXES:
+            message = _check_python(path)
+        elif suffix in JS_SUFFIXES:
+            message = _check_js(path, node)
+        elif suffix in CSS_SUFFIXES:
+            message = _check_braces(path)
+        elif suffix in HTML_SUFFIXES:
+            message = _check_html(path)
+        if message is not None:
+            issues.append(Issue(path, message))
+    return issues
+
+
+def main(argv: Optional[list[str]] = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="agents.validator",
+        description="Validate Python, JavaScript, CSS, and HTML without external tools.",
+    )
+    parser.add_argument("path", nargs="?", default=".", help="File or directory to validate")
+    parser.add_argument("-q", "--quiet", action="store_true", help="Only print failures")
+    args = parser.parse_args(argv)
+
+    root = Path(args.path)
+    if not root.exists():
+        sys.stderr.write(f"path not found: {root}\n")
+        return 2
+
+    issues = validate(root)
+    if issues:
+        for issue in issues:
+            sys.stderr.write(f"FAIL {issue.path}: {issue.message}\n")
+        sys.stderr.write(f"\n{len(issues)} error(s)\n")
+        return 1
+    if not args.quiet:
+        sys.stdout.write(f"OK: {root} validated clean\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

devplacepy/__init__.py

@@ -0,0 +1,2 @@
+# retoor <retoor@molodetz.nl>
+

devplacepy/attachments.py

@@ -1,33 +1,145 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+import ipaddress
 import logging
-from datetime import datetime
+import socket
+from datetime import datetime, timezone
 from pathlib import Path
+from urllib.parse import urlparse
 from PIL import Image
 from io import BytesIO
-from devplacepy.database import get_table
+import httpx
+from devplacepy.database import get_table, db, get_setting
 from devplacepy.config import STATIC_DIR
 from devplacepy.utils import generate_uid
 
 logger = logging.getLogger(__name__)
-ATTACHMENTS_DIR = STATIC_DIR / "uploads" / "attachments"
+
+REMOTE_FETCH_TIMEOUT = 20.0
+REMOTE_FETCH_USER_AGENT = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+)
+
+UPLOADS_DIR = STATIC_DIR / "uploads"
+ATTACHMENTS_DIR = UPLOADS_DIR / "attachments"
 THUMBNAIL_SIZE = (200, 200)
 THUMBNAIL_QUALITY = 80
 IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".tiff"}
 THUMBNAIL_EXTENSIONS = IMAGE_EXTENSIONS - {".gif"}
+POST_IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp"}
+
+ALLOWED_UPLOAD_TYPES = {
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".png": "image/png",
+    ".gif": "image/gif",
+    ".webp": "image/webp",
+    ".bmp": "image/bmp",
+    ".tiff": "image/tiff",
+    ".pdf": "application/pdf",
+    ".zip": "application/zip",
+    ".mp4": "video/mp4",
+    ".webm": "video/webm",
+    ".ogv": "video/ogg",
+    ".mov": "video/quicktime",
+    ".m4v": "video/x-m4v",
+    ".mp3": "audio/mpeg",
+    ".txt": "text/plain",
+    ".py": "text/x-python",
+    ".js": "text/javascript",
+    ".css": "text/css",
+    ".md": "text/markdown",
+}
+
+MIME_TO_EXT = {
+    "image/jpeg": ".jpg",
+    "image/png": ".png",
+    "image/gif": ".gif",
+    "image/webp": ".webp",
+    "image/bmp": ".bmp",
+    "image/tiff": ".tiff",
+    "application/pdf": ".pdf",
+    "application/zip": ".zip",
+    "video/mp4": ".mp4",
+    "video/webm": ".webm",
+    "video/ogg": ".ogv",
+    "video/quicktime": ".mov",
+    "video/x-m4v": ".m4v",
+    "audio/mpeg": ".mp3",
+    "text/plain": ".txt",
+    "text/markdown": ".md",
+}
+
+FILE_ICONS = {
+    ".pdf": "\U0001f4c4",
+    ".zip": "\U0001f4e6",
+    ".gz": "\U0001f4e6",
+    ".tar": "\U0001f4e6",
+    ".rar": "\U0001f4e6",
+    ".7z": "\U0001f4e6",
+    ".mp4": "\U0001f3ac",
+    ".webm": "\U0001f3ac",
+    ".ogv": "\U0001f3ac",
+    ".mov": "\U0001f3ac",
+    ".m4v": "\U0001f3ac",
+    ".mp3": "\U0001f3b5",
+    ".py": "\U0001f4bb",
+    ".js": "\U0001f4bb",
+    ".ts": "\U0001f4bb",
+    ".html": "\U0001f4bb",
+    ".css": "\U0001f4bb",
+    ".json": "\U0001f4bb",
+    ".md": "\U0001f4bb",
+    ".csv": "\U0001f4ca",
+    ".xls": "\U0001f4ca",
+    ".xlsx": "\U0001f4ca",
+    ".doc": "\U0001f4dd",
+    ".docx": "\U0001f4dd",
+    ".txt": "\U0001f4c4",
+    ".exe": "\u2699",
+    ".bin": "\u2699",
+}
+DEFAULT_FILE_ICON = "\U0001f4ce"
 
-def _get_setting(key, default):
-    row = get_table("site_settings").find_one(key=key)
-    return row["value"] if row else default
 
 def _get_max_upload_bytes():
-    return int(_get_setting("max_upload_size_mb", "10")) * 1024 * 1024
+    return int(get_setting("max_upload_size_mb", "10")) * 1024 * 1024
+
+
+def allowed_extensions():
+    raw = get_setting("allowed_file_types", "").strip()
+    if raw:
+        return {
+            ext if ext.startswith(".") else f".{ext}"
+            for ext in (part.strip().lower() for part in raw.split(","))
+            if ext
+        }
+    return set(ALLOWED_UPLOAD_TYPES)
+
+
+def is_extension_allowed(ext):
+    return ext in allowed_extensions()
+
 
 def _directory_for(uid):
     return f"{uid[:2]}/{uid[2:4]}"
 
+
 def _detect_mime(file_bytes, original_filename):
     ext = Path(original_filename).suffix.lower()
-    m = {".jpg":"image/jpeg",".jpeg":"image/jpeg",".png":"image/png",".gif":"image/gif",".webp":"image/webp",".svg":"image/svg+xml",".bmp":"image/bmp",".tiff":"image/tiff",".pdf":"application/pdf",".zip":"application/zip",".mp4":"video/mp4",".mp3":"audio/mpeg",".txt":"text/plain",".py":"text/x-python",".js":"text/javascript",".html":"text/html",".css":"text/css",".md":"text/markdown"}
-    return m.get(ext, "application/octet-stream")
+    return ALLOWED_UPLOAD_TYPES.get(ext, "application/octet-stream")
+
+
+def _image_dimensions(file_bytes):
+    try:
+        img = Image.open(BytesIO(file_bytes))
+        return img.width, img.height
+    except Exception as e:
+        logger.warning(f"Could not read image dimensions: {e}")
+        return None, None
+
 
 def _generate_thumbnail(file_bytes, thumb_path):
     try:
@@ -45,95 +157,384 @@ def _generate_thumbnail(file_bytes, thumb_path):
         logger.warning(f"Thumbnail generation failed: {e}")
         return None
 
+
+def save_inline_image(file_bytes, original_filename):
+    if len(file_bytes) > _get_max_upload_bytes():
+        logger.warning(f"Inline image too large: {original_filename}")
+        return None
+    ext = Path(original_filename).suffix.lower()
+    if ext not in POST_IMAGE_EXTENSIONS:
+        logger.warning(f"Unsupported inline image type: {ext}")
+        return None
+    UPLOADS_DIR.mkdir(parents=True, exist_ok=True)
+    filename = f"{generate_uid()}{ext}"
+    (UPLOADS_DIR / filename).write_bytes(file_bytes)
+    logger.info(f"Inline image saved: {filename}")
+    return filename
+
+
+def delete_inline_image(filename):
+    if not filename:
+        return
+    try:
+        (UPLOADS_DIR / filename).unlink(missing_ok=True)
+    except Exception as e:
+        logger.warning(f"Failed to delete inline image {filename}: {e}")
+
+
 def store_attachment(file_bytes, original_filename, user_uid):
     if len(file_bytes) > _get_max_upload_bytes():
         return None
+    ext = Path(original_filename).suffix.lower()
+    if not is_extension_allowed(ext):
+        return None
+
     uid = generate_uid()
-    ext = Path(original_filename).suffix.lower() or ".bin"
     stored_name = f"{uid}{ext}"
     directory = _directory_for(uid)
     file_dir = ATTACHMENTS_DIR / directory
     file_dir.mkdir(parents=True, exist_ok=True)
     (file_dir / stored_name).write_bytes(file_bytes)
+
     mime = _detect_mime(file_bytes, original_filename)
-    is_img = mime.startswith("image/")
-    img_w, img_h = None, None
-    thumb = None
-    if is_img:
-        try:
-            img = Image.open(BytesIO(file_bytes))
-            img_w, img_h = img.width, img.height
-            if ext not in (".gif",):
-                thumb_name = f"{uid}_thumb.jpg"
-                r = _generate_thumbnail(file_bytes, file_dir / thumb_name)
-                if r: thumb = r
-        except: pass
-    get_table("attachments").insert({
-        "uid": uid, "target_type": "", "target_uid": "", "user_uid": user_uid,
-        "original_filename": original_filename, "stored_name": stored_name,
-        "directory": directory, "file_size": len(file_bytes), "mime_type": mime,
-        "image_width": img_w, "image_height": img_h,
-        "has_thumbnail": 1 if thumb else 0,
-        "created_at": datetime.utcnow().isoformat(),
-    })
-    return {"uid": uid, "original_filename": original_filename, "file_size": len(file_bytes), "mime_type": mime, "url": f"/static/uploads/attachments/{directory}/{stored_name}", "thumbnail_url": f"/static/uploads/attachments/{directory}/{thumb}" if thumb else None, "has_thumbnail": thumb is not None, "is_image": is_img}
+    is_image = mime.startswith("image/")
+    image_width, image_height = None, None
+    thumbnail = None
+    if is_image:
+        image_width, image_height = _image_dimensions(file_bytes)
+        if ext not in (".gif",):
+            thumbnail = _generate_thumbnail(file_bytes, file_dir / f"{uid}_thumb.jpg")
+
+    get_table("attachments").insert(
+        {
+            "uid": uid,
+            "target_type": "",
+            "target_uid": "",
+            "user_uid": user_uid,
+            "original_filename": original_filename,
+            "stored_name": stored_name,
+            "directory": directory,
+            "file_size": len(file_bytes),
+            "mime_type": mime,
+            "image_width": image_width,
+            "image_height": image_height,
+            "has_thumbnail": 1 if thumbnail else 0,
+            "thumbnail_name": thumbnail,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+            "deleted_at": None,
+            "deleted_by": None,
+        }
+    )
+    return {
+        "uid": uid,
+        "original_filename": original_filename,
+        "file_size": len(file_bytes),
+        "mime_type": mime,
+        "url": f"/static/uploads/attachments/{directory}/{stored_name}",
+        "thumbnail_url": f"/static/uploads/attachments/{directory}/{thumbnail}"
+        if thumbnail
+        else None,
+        "has_thumbnail": thumbnail is not None,
+        "is_image": is_image,
+        "is_video": mime.startswith("video/"),
+    }
+
+
+class RemoteFetchError(Exception):
+    def __init__(self, message, status=400):
+        super().__init__(message)
+        self.message = message
+        self.status = status
+
+
+async def _guard_public_url(url):
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https"):
+        raise RemoteFetchError("Only http and https URLs can be attached.", 400)
+    host = parsed.hostname
+    if not host:
+        raise RemoteFetchError("The URL has no host.", 400)
+    try:
+        infos = await asyncio.to_thread(socket.getaddrinfo, host, None)
+    except socket.gaierror as exc:
+        raise RemoteFetchError(f"Could not resolve host: {host}", 400) from exc
+    for info in infos:
+        address = ipaddress.ip_address(info[4][0])
+        if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped:
+            address = address.ipv4_mapped
+        if (
+            address.is_private
+            or address.is_loopback
+            or address.is_link_local
+            or address.is_reserved
+            or address.is_multicast
+            or address.is_unspecified
+        ):
+            raise RemoteFetchError(
+                f"Refusing to attach a private or local address ({address}).", 400
+            )
+
+
+def _resolve_remote_filename(final_url, content_type, override):
+    name = (override or "").strip() or Path(urlparse(final_url).path).name
+    ext = Path(name).suffix.lower()
+    if name and ext and is_extension_allowed(ext):
+        return name
+    base_mime = (content_type or "").split(";")[0].strip().lower()
+    mapped = MIME_TO_EXT.get(base_mime)
+    if mapped is None or not is_extension_allowed(mapped):
+        return None
+    stem = Path(name).stem or "download"
+    return f"{stem}{mapped}"
+
+
+async def fetch_remote_file(url, filename=None):
+    if "://" not in url:
+        url = "https://" + url
+    await _guard_public_url(url)
+    max_bytes = _get_max_upload_bytes()
+    try:
+        async with httpx.AsyncClient(
+            follow_redirects=True,
+            timeout=REMOTE_FETCH_TIMEOUT,
+            headers={"User-Agent": REMOTE_FETCH_USER_AGENT},
+        ) as client:
+            async with client.stream("GET", url) as response:
+                if response.status_code >= 400:
+                    raise RemoteFetchError(
+                        f"The remote server returned {response.status_code}.", 400
+                    )
+                final_url = str(response.url)
+                content_type = response.headers.get("content-type", "")
+                chunks = []
+                total = 0
+                async for chunk in response.aiter_bytes():
+                    chunks.append(chunk)
+                    total += len(chunk)
+                    if total > max_bytes:
+                        raise RemoteFetchError(
+                            f"The file exceeds the {max_bytes // (1024 * 1024)}MB limit.",
+                            413,
+                        )
+                data = b"".join(chunks)
+    except httpx.HTTPError as exc:
+        raise RemoteFetchError(f"Could not fetch {url}: {exc}", 400) from exc
+
+    name = _resolve_remote_filename(final_url, content_type, filename)
+    if name is None:
+        raise RemoteFetchError(
+            "Could not determine an allowed file type for the URL. Pass a filename "
+            "with an allowed extension.",
+            415,
+        )
+    return name, data
+
+
+async def store_attachment_from_url(url, user_uid, filename=None):
+    name, data = await fetch_remote_file(url, filename)
+    result = store_attachment(data, name, user_uid)
+    if result is None:
+        raise RemoteFetchError(
+            "The downloaded file is not an allowed type or exceeds the size limit.",
+            413,
+        )
+    return result
+
 
 def link_attachments(uids, target_type, target_uid):
-    if not uids: return
-    atts = get_table("attachments")
-    for auid in uids:
-        auid = auid.strip()
-        if not auid: continue
-        e = atts.find_one(uid=auid)
-        if e: atts.update({"id": e["id"], "uid": auid, "target_type": target_type, "target_uid": target_uid}, ["id"])
+    flat = [
+        uid.strip() for raw in uids or [] for uid in str(raw).split(",") if uid.strip()
+    ]
+    if not flat:
+        return
+    placeholders = ",".join(f":p{i}" for i in range(len(flat)))
+    params = {f"p{i}": uid for i, uid in enumerate(flat)}
+    db.query(
+        f"UPDATE attachments SET target_type=:tt, target_uid=:tu WHERE uid IN ({placeholders})",
+        tt=target_type,
+        tu=target_uid,
+        **params,
+    )
+
+
+def _unlink_attachment_files(row):
+    stored_name = row.get("stored_name", "")
+    directory = row.get("directory", "")
+    if not (stored_name and directory):
+        return
+    file_path = ATTACHMENTS_DIR / directory / stored_name
+    try:
+        file_path.unlink(missing_ok=True)
+    except Exception as e:
+        logger.warning(f"Failed to delete attachment file {file_path}: {e}")
+    for thumb_path in (ATTACHMENTS_DIR / directory).glob(
+        f"{Path(stored_name).stem}_thumb.*"
+    ):
+        try:
+            thumb_path.unlink(missing_ok=True)
+        except Exception as e:
+            logger.warning(f"Failed to delete thumbnail {thumb_path}: {e}")
+
+
+def _delete_attachment_row(row):
+    _unlink_attachment_files(row)
+    get_table("attachments").delete(id=row["id"])
+
 
 def delete_attachment(uid):
-    atts = get_table("attachments")
-    att = atts.find_one(uid=uid)
-    if not att: return
-    sn, d = att.get("stored_name",""), att.get("directory","")
-    if sn and d:
-        fp = ATTACHMENTS_DIR / d / sn
-        try: fp.unlink(missing_ok=True)
-        except: pass
-        for tp in (ATTACHMENTS_DIR / d).glob(f"{Path(sn).stem}_thumb.*"):
-            try: tp.unlink(missing_ok=True)
-            except: pass
-    atts.delete(id=att["id"])
+    row = get_table("attachments").find_one(uid=uid)
+    if row:
+        _delete_attachment_row(row)
 
-def delete_target_attachments(tt, tu):
-    for a in get_table("attachments").find(target_type=tt, target_uid=tu):
-        delete_attachment(a["uid"])
 
-def get_attachments(tt, tu):
-    try: rows = list(get_table("attachments").find(target_type=tt, target_uid=tu, order_by=["created_at"]))
-    except: return []
-    return [_r2a(r) for r in rows]
+def soft_delete_attachment(uid, deleted_by="system"):
+    row = get_table("attachments").find_one(uid=uid)
+    if not row or row.get("deleted_at"):
+        return None
+    get_table("attachments").update(
+        {
+            "uid": uid,
+            "deleted_at": datetime.now(timezone.utc).isoformat(),
+            "deleted_by": deleted_by,
+        },
+        ["uid"],
+    )
+    return row
 
-def get_attachments_batch(tt, uids):
-    if not uids: return {}
-    from devplacepy.database import db
-    if "attachments" not in db.tables: return {u:[] for u in uids}
-    ph = ",".join(f":p{i}" for i in range(len(uids)))
-    try:
-        rows = db.query(f"SELECT * FROM attachments WHERE target_type=:tt AND target_uid IN ({ph}) ORDER BY created_at", tt=tt, **{f"p{i}":u for i,u in enumerate(uids)})
-    except: return {u:[] for u in uids}
-    r = {u:[] for u in uids}
+
+def restore_attachment(uid):
+    row = get_table("attachments").find_one(uid=uid)
+    if not row or not row.get("deleted_at"):
+        return False
+    get_table("attachments").update(
+        {"uid": uid, "deleted_at": None, "deleted_by": None}, ["uid"]
+    )
+    return True
+
+
+def soft_delete_target_attachments(target_type, target_uid, deleted_by):
+    stamp = datetime.now(timezone.utc).isoformat()
+    for row in get_table("attachments").find(
+        target_type=target_type, target_uid=target_uid, deleted_at=None
+    ):
+        get_table("attachments").update(
+            {"uid": row["uid"], "deleted_at": stamp, "deleted_by": deleted_by}, ["uid"]
+        )
+
+
+def soft_delete_attachments_for(target_type, target_uids, deleted_by):
+    from devplacepy.database import soft_delete_in
+
+    soft_delete_in(
+        "attachments",
+        "target_uid",
+        target_uids,
+        deleted_by,
+        target_type=target_type,
+    )
+
+
+def delete_target_attachments(target_type, target_uid):
+    for row in get_table("attachments").find(
+        target_type=target_type, target_uid=target_uid
+    ):
+        _delete_attachment_row(row)
+
+
+def delete_attachments_for(target_type, target_uids):
+    uids = [uid for uid in target_uids if uid]
+    if not uids or "attachments" not in db.tables:
+        return
+    placeholders = ",".join(f":p{i}" for i in range(len(uids)))
+    params = {f"p{i}": uid for i, uid in enumerate(uids)}
+    rows = list(
+        db.query(
+            f"SELECT * FROM attachments WHERE target_type=:tt AND target_uid IN ({placeholders})",
+            tt=target_type,
+            **params,
+        )
+    )
+    if not rows:
+        return
     for row in rows:
-        if row["target_uid"] in r: r[row["target_uid"]].append(_r2a(row))
-    return r
+        _unlink_attachment_files(row)
+    ids = ",".join(str(row["id"]) for row in rows)
+    db.query(f"DELETE FROM attachments WHERE id IN ({ids})")
 
-def _r2a(r):
-    sn, d = r.get("stored_name",""), r.get("directory","")
-    ts = f"{Path(sn).stem}_thumb.jpg" if r.get("has_thumbnail") else None
-    return {"uid":r["uid"],"original_filename":r.get("original_filename",""),"file_size":r.get("file_size",0),"mime_type":r.get("mime_type",""),"url":f"/static/uploads/attachments/{d}/{sn}","thumbnail_url":f"/static/uploads/attachments/{d}/{ts}" if ts else None,"has_thumbnail":bool(r.get("has_thumbnail")),"is_image":r.get("mime_type","").startswith("image/")}
 
-def format_file_size(b):
-    if b < 1024: return f"{b} B"
-    if b < 1048576: return f"{b/1024:.1f} KB"
-    return f"{b/1048576:.1f} MB"
+def get_attachments(target_type, target_uid):
+    if "attachments" not in db.tables:
+        return []
+    rows = list(
+        get_table("attachments").find(
+            target_type=target_type,
+            target_uid=target_uid,
+            deleted_at=None,
+            order_by=["created_at"],
+        )
+    )
+    return [_row_to_attachment(r) for r in rows]
 
-def file_icon_emoji(fn):
-    ext = Path(fn).suffix.lower()
-    m = {".pdf":"\U0001F4C4",".zip":"\U0001F4E6",".gz":"\U0001F4E6",".tar":"\U0001F4E6",".rar":"\U0001F4E6",".7z":"\U0001F4E6",".mp4":"\U0001F3AC",".mp3":"\U0001F3B5",".py":"\U0001F4BB",".js":"\U0001F4BB",".ts":"\U0001F4BB",".html":"\U0001F4BB",".css":"\U0001F4BB",".json":"\U0001F4BB",".md":"\U0001F4BB",".csv":"\U0001F4CA",".xls":"\U0001F4CA",".xlsx":"\U0001F4CA",".doc":"\U0001F4DD",".docx":"\U0001F4DD",".txt":"\U0001F4C4",".exe":"\u2699",".bin":"\u2699"}
-    return m.get(ext, "\U0001F4CE")
+
+def get_attachments_batch(target_type, uids):
+    if not uids:
+        return {}
+    if "attachments" not in db.tables:
+        return {uid: [] for uid in uids}
+    placeholders = ",".join(f":p{i}" for i in range(len(uids)))
+    params = {f"p{i}": uid for i, uid in enumerate(uids)}
+    rows = db.query(
+        f"SELECT * FROM attachments WHERE target_type=:tt AND target_uid IN ({placeholders}) AND deleted_at IS NULL ORDER BY created_at",
+        tt=target_type,
+        **params,
+    )
+    result = {uid: [] for uid in uids}
+    for row in rows:
+        if row["target_uid"] in result:
+            result[row["target_uid"]].append(_row_to_attachment(row))
+    return result
+
+
+def _row_to_attachment(row):
+    stored_name = row.get("stored_name", "")
+    directory = row.get("directory", "")
+    thumb_name = None
+    if row.get("has_thumbnail"):
+        thumb_name = row.get("thumbnail_name")
+        if not thumb_name:
+            stem = Path(stored_name).stem
+            png = f"{stem}_thumb.png"
+            thumb_name = (
+                png
+                if (ATTACHMENTS_DIR / directory / png).exists()
+                else f"{stem}_thumb.jpg"
+            )
+    return {
+        "uid": row["uid"],
+        "original_filename": row.get("original_filename", ""),
+        "file_size": row.get("file_size", 0),
+        "mime_type": row.get("mime_type", ""),
+        "url": f"/static/uploads/attachments/{directory}/{stored_name}",
+        "thumbnail_url": f"/static/uploads/attachments/{directory}/{thumb_name}"
+        if thumb_name
+        else None,
+        "has_thumbnail": bool(row.get("has_thumbnail")),
+        "is_image": row.get("mime_type", "").startswith("image/"),
+        "is_video": row.get("mime_type", "").startswith("video/"),
+        "target_type": row.get("target_type", ""),
+        "target_uid": row.get("target_uid", ""),
+        "created_at": row.get("created_at", ""),
+    }
+
+
+def format_file_size(size):
+    if size < 1024:
+        return f"{size} B"
+    if size < 1048576:
+        return f"{size / 1024:.1f} KB"
+    return f"{size / 1048576:.1f} MB"
+
+
+def file_icon_emoji(filename):
+    ext = Path(filename).suffix.lower()
+    return FILE_ICONS.get(ext, DEFAULT_FILE_ICON)

devplacepy/avatar.py

@@ -1,3 +1,5 @@
+# retoor <retoor@molodetz.nl>
+
 import logging
 
 logger = logging.getLogger(__name__)
@@ -9,11 +11,13 @@ def avatar_url(style: str, seed: str, size: int = 128) -> str:
 
 def generate_avatar_svg(seed: str) -> str:
     try:
-        from multiavatar import multiavatar
-        svg = multiavatar(seed)
+        from multiavatar.multiavatar import multiavatar
+
+        svg = multiavatar(seed, None, None)
         if svg and svg.strip().startswith("<svg"):
             return svg
     except Exception as e:
+        logger.exception(e)
         logger.warning(f"Avatar generation failed for {seed}: {e}")
     initial = seed[:1].upper() if seed else "?"
     return (

devplacepy/cache.py

@@ -0,0 +1,40 @@
+# retoor <retoor@molodetz.nl>
+
+import time
+from collections import OrderedDict
+
+
+class TTLCache:
+    def __init__(self, ttl: int, max_size: int = 0):
+        self.ttl = ttl
+        self.max_size = max_size
+        self._store = OrderedDict()
+
+    def get(self, key):
+        entry = self._store.get(key)
+        if entry is None:
+            return None
+        value, expiry = entry
+        if time.time() >= expiry:
+            self._store.pop(key, None)
+            return None
+        self._store.move_to_end(key)
+        return value
+
+    def set(self, key, value):
+        self._store[key] = (value, time.time() + self.ttl)
+        self._store.move_to_end(key)
+        if self.max_size and len(self._store) > self.max_size:
+            self._store.popitem(last=False)
+
+    def pop(self, key):
+        self._store.pop(key, None)
+
+    def clear(self):
+        self._store.clear()
+
+    def items(self):
+        now = time.time()
+        return [
+            (key, value) for key, (value, expiry) in self._store.items() if now < expiry
+        ]

devplacepy/cli.py

@@ -1,6 +1,26 @@
+# retoor <retoor@molodetz.nl>
+
 import argparse
 import sys
 from devplacepy.database import get_table
+from devplacepy.utils import strip_html
+
+
+def _audit_cli(event_key, summary, metadata=None, target_type=None, target_uid=None, target_label=None, links=None):
+    from devplacepy.services.audit import record as audit
+
+    audit.record_system(
+        event_key,
+        actor_kind="cli",
+        actor_role="system",
+        origin="cli",
+        target_type=target_type,
+        target_uid=target_uid,
+        target_label=target_label,
+        summary=summary,
+        metadata=metadata,
+        links=links,
+    )
 
 
 def cmd_role_get(args):
@@ -24,61 +44,327 @@ def cmd_role_set(args):
         print(f"User '{args.username}' not found")
         sys.exit(1)
 
+    old_role = user.get("role")
     users.update({"uid": user["uid"], "role": role.capitalize()}, ["uid"])
+    from devplacepy.services.audit import record as audit
+
+    _audit_cli(
+        "cli.role.set",
+        f"CLI set role of user {args.username} from {old_role} to {role.capitalize()}",
+        metadata={"old": old_role, "new": role.capitalize()},
+        target_type="user",
+        target_uid=user["uid"],
+        target_label=args.username,
+        links=[audit.target("user", user["uid"], args.username)],
+    )
     print(f"User '{args.username}' role set to '{role}'")
 
 
+def cmd_apikey_get(args):
+    users = get_table("users")
+    user = users.find_one(username=args.username)
+    if not user:
+        print(f"User '{args.username}' not found")
+        sys.exit(1)
+    print(user.get("api_key", "") or "")
+
+
+def cmd_apikey_reset(args):
+    from devplacepy.utils import generate_uid, clear_user_cache
+
+    users = get_table("users")
+    user = users.find_one(username=args.username)
+    if not user:
+        print(f"User '{args.username}' not found")
+        sys.exit(1)
+    new_key = generate_uid()
+    users.update({"uid": user["uid"], "api_key": new_key}, ["uid"])
+    clear_user_cache(user["uid"])
+    from devplacepy.services.audit import record as audit
+
+    _audit_cli(
+        "cli.apikey.reset",
+        f"CLI regenerated the API key of user {args.username}",
+        target_type="user",
+        target_uid=user["uid"],
+        target_label=args.username,
+        links=[audit.target("user", user["uid"], args.username)],
+    )
+    print(new_key)
+
+
+def cmd_apikey_backfill(args):
+    from devplacepy.database import backfill_api_keys
+
+    updated = backfill_api_keys()
+    _audit_cli(
+        "cli.apikey.backfill",
+        f"CLI backfilled API keys for {updated} users",
+        metadata={"count": updated},
+    )
+    print(f"Assigned API keys to {updated} user(s) without one")
+
+
+def cmd_devii_reset_quota(args):
+    from devplacepy.database import db
+
+    table_name = "devii_usage_ledger"
+    if table_name not in db.tables:
+        print(f"Table '{table_name}' does not exist, nothing to reset")
+        return
+    table = db[table_name]
+    if args.all:
+        count = table.count()
+        table.delete()
+        _audit_cli("cli.devii.quota.reset", "CLI reset AI quota (all)", metadata={"scope": "all", "rows_removed": count})
+        print(f"Reset all AI quotas ({count} ledger rows deleted)")
+        return
+    if args.guests:
+        count = table.count(owner_kind="guest")
+        table.delete(owner_kind="guest")
+        _audit_cli("cli.devii.quota.reset", "CLI reset AI quota (guests)", metadata={"scope": "guests", "rows_removed": count})
+        print(f"Reset all guest AI quotas ({count} ledger rows deleted)")
+        return
+    if not args.username:
+        print("Provide a username, or --guests, or --all")
+        sys.exit(1)
+    user = get_table("users").find_one(username=args.username)
+    if not user:
+        print(f"User '{args.username}' not found")
+        sys.exit(1)
+    count = table.count(owner_kind="user", owner_id=user["uid"])
+    table.delete(owner_kind="user", owner_id=user["uid"])
+    from devplacepy.services.audit import record as audit
+
+    _audit_cli(
+        "cli.devii.quota.reset",
+        f"CLI reset AI quota for {args.username}",
+        metadata={"scope": "user", "rows_removed": count},
+        target_type="user",
+        target_uid=user["uid"],
+        target_label=args.username,
+        links=[audit.target("user", user["uid"], args.username)],
+    )
+    print(f"Reset AI quota for '{args.username}' ({count} ledger rows deleted)")
+
+
 def cmd_news_clear(args):
     from devplacepy.database import db
+
+    deleted = {}
     for table in ("news", "news_images", "news_sync"):
         if table in db.tables:
-            count = len(list(db[table].all()))
+            count = db[table].count()
             db[table].delete()
+            deleted[table] = count
             print(f"Deleted {count} rows from '{table}'")
         else:
             print(f"Table '{table}' does not exist, skipping")
+    _audit_cli("cli.news.clear", "CLI cleared all news data", metadata={"deleted": deleted})
     print("News data cleared")
 
 
-def cmd_attachments_prune(args):
+def cmd_news_sanitize(args):
     from devplacepy.database import db
-    from devplacepy.config import STATIC_DIR
-    import os
-    deleted_records = 0
-    deleted_files = 0
-    freed_bytes = 0
 
-    if "attachments" in db.tables:
-        orphans = list(db["attachments"].find(resource_uid=""))
-        orphans += list(db["attachments"].find(resource_type=""))
-        seen = set()
-        unique_orphans = []
-        for o in orphans:
-            if o["uid"] not in seen:
-                seen.add(o["uid"])
-                unique_orphans.append(o)
+    if "news" not in db.tables:
+        print("News table does not exist")
+        return
+    news_table = db["news"]
+    updated = 0
+    for row in news_table.all():
+        desc = (strip_html(row.get("description", "") or ""))[:5000]
+        content = (strip_html(row.get("content", "") or ""))[:10000]
+        if desc != row.get("description", "") or content != row.get("content", ""):
+            news_table.update(
+                {"id": row["id"], "description": desc, "content": content}, ["id"]
+            )
+            updated += 1
+    _audit_cli("cli.news.sanitize", f"CLI sanitized {updated} news articles", metadata={"count": updated})
+    print(f"Sanitized {updated} news article(s)")
 
-        for att in unique_orphans:
-            sp = att.get("storage_path", "")
-            if sp:
-                fp = STATIC_DIR / "uploads" / sp
-                try:
-                    if fp.exists():
-                        freed_bytes += fp.stat().st_size
-                        fp.unlink()
-                        deleted_files += 1
-                    parent = fp.parent
-                    if parent.exists() and not any(parent.iterdir()):
-                        parent.rmdir()
-                    grandparent = parent.parent
-                    if grandparent.exists() and not any(grandparent.iterdir()):
-                        grandparent.rmdir()
-                except Exception as e:
-                    print(f"  Error deleting {sp}: {e}")
-            db["attachments"].delete(id=att["id"])
-            deleted_records += 1
 
-    print(f"Pruned {deleted_records} orphan records, {deleted_files} files, {freed_bytes / 1024:.1f} KB freed")
+def cmd_attachments_prune(args):
+    from datetime import datetime, timezone, timedelta
+    from devplacepy.database import db
+    from devplacepy.attachments import delete_attachment
+
+    if "attachments" not in db.tables:
+        print("Attachments table does not exist")
+        return
+
+    cutoff = (datetime.now(timezone.utc) - timedelta(hours=args.hours)).isoformat()
+    orphans = [
+        att
+        for att in db["attachments"].find(target_type="", target_uid="")
+        if att.get("created_at", "") < cutoff
+    ]
+    for att in orphans:
+        delete_attachment(att["uid"])
+    _audit_cli(
+        "cli.attachments.prune",
+        f"CLI pruned {len(orphans)} orphan attachments",
+        metadata={"count": len(orphans), "hours": args.hours},
+    )
+    print(f"Pruned {len(orphans)} orphan attachment(s) older than {args.hours}h")
+
+
+def _remove_zip_artifacts(job):
+    import shutil
+    from pathlib import Path
+    from devplacepy.services.jobs.zip_service import STAGING_DIR
+
+    local_path = (job.get("result") or {}).get("local_path")
+    if local_path:
+        Path(local_path).unlink(missing_ok=True)
+    shutil.rmtree(STAGING_DIR / job["uid"], ignore_errors=True)
+
+
+def cmd_zips_prune(args):
+    from datetime import datetime, timezone
+    from devplacepy.services.jobs import queue
+
+    now = datetime.now(timezone.utc)
+    removed = 0
+    for job in queue.list_jobs(kind="zip", status=queue.DONE):
+        expires_at = job.get("expires_at")
+        if not expires_at:
+            continue
+        try:
+            expiry = datetime.fromisoformat(expires_at)
+        except (ValueError, TypeError):
+            continue
+        if expiry < now:
+            _remove_zip_artifacts(job)
+            get_table("jobs").delete(uid=job["uid"])
+            removed += 1
+    _audit_cli("cli.zips.prune", f"CLI pruned {removed} expired zip jobs", metadata={"count": removed})
+    print(f"Pruned {removed} expired zip job(s)")
+
+
+def cmd_zips_clear(args):
+    from devplacepy.services.jobs import queue
+
+    jobs = queue.list_jobs(kind="zip")
+    for job in jobs:
+        _remove_zip_artifacts(job)
+        get_table("jobs").delete(uid=job["uid"])
+    _audit_cli("cli.zips.clear", f"CLI cleared all zip jobs ({len(jobs)})", metadata={"count": len(jobs)})
+    print(f"Cleared {len(jobs)} zip job(s) and their archives")
+
+
+def cmd_forks_prune(args):
+    from datetime import datetime, timezone
+    from devplacepy.services.jobs import queue
+
+    now = datetime.now(timezone.utc)
+    removed = 0
+    for job in queue.list_jobs(kind="fork", status=queue.DONE):
+        expires_at = job.get("expires_at")
+        if not expires_at:
+            continue
+        try:
+            expiry = datetime.fromisoformat(expires_at)
+        except (ValueError, TypeError):
+            continue
+        if expiry < now:
+            get_table("jobs").delete(uid=job["uid"])
+            removed += 1
+    _audit_cli("cli.forks.prune", f"CLI pruned {removed} expired fork jobs", metadata={"count": removed})
+    print(f"Pruned {removed} expired fork job(s)")
+
+
+def cmd_forks_clear(args):
+    from devplacepy.services.jobs import queue
+
+    jobs = queue.list_jobs(kind="fork")
+    for job in jobs:
+        get_table("jobs").delete(uid=job["uid"])
+    _audit_cli("cli.forks.clear", f"CLI cleared all fork jobs ({len(jobs)})", metadata={"count": len(jobs)})
+    print(f"Cleared {len(jobs)} fork job(s)")
+
+
+def cmd_containers_list(args):
+    from devplacepy.services.containers import store
+
+    instances = store.all_instances()
+    if not instances:
+        print("No container instances")
+        return
+    for inst in instances:
+        print(
+            f"{inst['uid'][:8]} {inst.get('name', ''):24.24} {inst.get('status', ''):10} "
+            f"desired={inst.get('desired_state', '')} policy={inst.get('restart_policy', '')}"
+        )
+
+
+def cmd_containers_reconcile(args):
+    import asyncio
+    from devplacepy.services.containers.service import ContainerService
+
+    asyncio.run(ContainerService().run_once())
+    _audit_cli("cli.containers.reconcile", "CLI ran one container reconcile pass")
+    print("Reconcile pass complete")
+
+
+def cmd_containers_prune(args):
+    import asyncio
+    from devplacepy.services.containers.runtime import get_backend
+    from devplacepy.services.containers.service import ContainerService
+
+    async def run():
+        await ContainerService().run_once()
+        await get_backend().image_prune()
+
+    asyncio.run(run())
+    _audit_cli("cli.containers.prune", "CLI reaped orphan containers and dangling images")
+    print("Reaped orphans and pruned dangling images")
+
+
+def cmd_containers_prune_builds(args):
+    import asyncio
+    from devplacepy.database import db, get_table
+    from devplacepy.services.containers.runtime import get_backend
+
+    async def run():
+        backend = get_backend()
+        removed = 0
+        if "builds" in db.tables:
+            for build in list(get_table("builds").find()):
+                tag = build.get("image_tag")
+                if tag:
+                    await backend.remove_image(tag)
+                    removed += 1
+        for table in ("builds", "dockerfile_versions", "dockerfiles"):
+            if table in db.tables:
+                get_table(table).delete()
+        return removed
+
+    removed = asyncio.run(run())
+    _audit_cli("cli.containers.prune_builds", "CLI removed legacy images and build tables", metadata={"removed": removed})
+    print(
+        f"Removed {removed} legacy per-project image(s) and cleared the dockerfiles/builds tables"
+    )
+
+
+def cmd_containers_gc_workspaces(args):
+    import shutil
+    from pathlib import Path
+    from devplacepy import config
+    from devplacepy.services.containers import store
+
+    active = {inst["project_uid"] for inst in store.all_instances()}
+    base = Path(config.CONTAINER_WORKSPACES_DIR)
+    removed = 0
+    if base.is_dir():
+        for child in base.iterdir():
+            if child.is_dir() and child.name not in active:
+                shutil.rmtree(child, ignore_errors=True)
+                removed += 1
+    _audit_cli("cli.containers.gc_workspaces", f"CLI removed {removed} unused workspace dirs", metadata={"count": removed})
+    print(
+        f"Removed {removed} unused workspace director{'y' if removed == 1 else 'ies'}"
+    )
 
 
 def main():
@@ -97,16 +383,100 @@ def main():
     role_set.add_argument("role", choices=["member", "admin"])
     role_set.set_defaults(func=cmd_role_set)
 
+    apikey = sub.add_parser("apikey", help="Manage user API keys")
+    apikey_sub = apikey.add_subparsers(title="action", dest="action")
+    apikey_get = apikey_sub.add_parser("get", help="Print a user's API key")
+    apikey_get.add_argument("username")
+    apikey_get.set_defaults(func=cmd_apikey_get)
+    apikey_reset = apikey_sub.add_parser("reset", help="Regenerate a user's API key")
+    apikey_reset.add_argument("username")
+    apikey_reset.set_defaults(func=cmd_apikey_reset)
+    apikey_backfill = apikey_sub.add_parser(
+        "backfill", help="Assign API keys to users that lack one"
+    )
+    apikey_backfill.set_defaults(func=cmd_apikey_backfill)
+
     news = sub.add_parser("news", help="News management")
     news_sub = news.add_subparsers(title="action", dest="action")
-    news_clear = news_sub.add_parser("clear", help="Delete all news from local database")
+    news_clear = news_sub.add_parser(
+        "clear", help="Delete all news from local database"
+    )
     news_clear.set_defaults(func=cmd_news_clear)
+    news_sanitize = news_sub.add_parser(
+        "sanitize", help="Strip HTML from all existing news descriptions and content"
+    )
+    news_sanitize.set_defaults(func=cmd_news_sanitize)
 
     attachments = sub.add_parser("attachments", help="Attachment management")
     att_sub = attachments.add_subparsers(title="action", dest="action")
-    att_prune = att_sub.add_parser("prune", help="Remove orphaned attachment records and files")
+    att_prune = att_sub.add_parser(
+        "prune", help="Remove orphaned attachment records and files"
+    )
+    att_prune.add_argument(
+        "--hours",
+        type=int,
+        default=24,
+        help="Only prune orphans older than this many hours",
+    )
     att_prune.set_defaults(func=cmd_attachments_prune)
 
+    devii = sub.add_parser("devii", help="Devii assistant management")
+    devii_sub = devii.add_subparsers(title="action", dest="action")
+    devii_reset = devii_sub.add_parser(
+        "reset-quota", help="Reset the rolling 24h AI spend quota"
+    )
+    devii_reset.add_argument(
+        "username", nargs="?", help="Reset the quota for a single user"
+    )
+    devii_reset.add_argument(
+        "--guests", action="store_true", help="Reset every guest quota"
+    )
+    devii_reset.add_argument(
+        "--all", action="store_true", help="Reset every quota (users and guests)"
+    )
+    devii_reset.set_defaults(func=cmd_devii_reset_quota)
+
+    zips = sub.add_parser("zips", help="Zip archive job management")
+    zips_sub = zips.add_subparsers(title="action", dest="action")
+    zips_prune = zips_sub.add_parser(
+        "prune", help="Delete expired zip archives and their job rows"
+    )
+    zips_prune.set_defaults(func=cmd_zips_prune)
+    zips_clear = zips_sub.add_parser(
+        "clear", help="Delete every zip archive and job row"
+    )
+    zips_clear.set_defaults(func=cmd_zips_clear)
+
+    forks = sub.add_parser("forks", help="Fork job management")
+    forks_sub = forks.add_subparsers(title="action", dest="action")
+    forks_prune = forks_sub.add_parser(
+        "prune", help="Delete expired completed fork job rows (forked projects persist)"
+    )
+    forks_prune.set_defaults(func=cmd_forks_prune)
+    forks_clear = forks_sub.add_parser(
+        "clear", help="Delete every fork job row (forked projects persist)"
+    )
+    forks_clear.set_defaults(func=cmd_forks_clear)
+
+    containers = sub.add_parser("containers", help="Container manager")
+    containers_sub = containers.add_subparsers(title="action", dest="action")
+    containers_sub.add_parser("list", help="List container instances").set_defaults(
+        func=cmd_containers_list
+    )
+    containers_sub.add_parser("reconcile", help="Run one reconcile pass").set_defaults(
+        func=cmd_containers_reconcile
+    )
+    containers_sub.add_parser(
+        "prune", help="Reap orphan containers and dangling images"
+    ).set_defaults(func=cmd_containers_prune)
+    containers_sub.add_parser(
+        "prune-builds",
+        help="Remove legacy per-project images and clear the dockerfiles/builds tables",
+    ).set_defaults(func=cmd_containers_prune_builds)
+    containers_sub.add_parser(
+        "gc-workspaces", help="Remove workspace dirs with no instances"
+    ).set_defaults(func=cmd_containers_gc_workspaces)
+
     args = parser.parse_args()
     if hasattr(args, "func"):
         args.func(args)

devplacepy/config.py

@@ -1,3 +1,5 @@
+# retoor <retoor@molodetz.nl>
+
 from pathlib import Path
 from dotenv import load_dotenv
 from os import environ
@@ -7,7 +9,39 @@ load_dotenv()
 BASE_DIR = Path(__file__).resolve().parent.parent
 STATIC_DIR = BASE_DIR / "devplacepy" / "static"
 TEMPLATES_DIR = BASE_DIR / "devplacepy" / "templates"
-DATABASE_URL = environ.get("DEVPLACE_DATABASE_URL", f"sqlite:///{BASE_DIR / 'devplace.db'}")
+DATABASE_URL = environ.get(
+    "DEVPLACE_DATABASE_URL", f"sqlite:///{BASE_DIR / 'devplace.db'}"
+)
 SECRET_KEY = environ.get("SECRET_KEY", "devplace-secret-key-change-in-production")
-SESSION_MAX_AGE = 86400 * 7
+SECONDS_PER_DAY = 86400
+SESSION_MAX_AGE = SECONDS_PER_DAY * 7
+SESSION_MAX_AGE_REMEMBER = SECONDS_PER_DAY * 30
 PORT = 10500
+SITE_URL = environ.get("DEVPLACE_SITE_URL", "").rstrip("/")
+
+INTERNAL_BASE_URL = environ.get(
+    "DEVPLACE_INTERNAL_BASE_URL", f"http://localhost:{PORT}"
+).rstrip("/")
+INTERNAL_GATEWAY_URL = f"{INTERNAL_BASE_URL}/openai/v1/chat/completions"
+INTERNAL_MODEL = "molodetz"
+
+SERVICE_LOCK_FILE = BASE_DIR / "devplace-services.lock"
+INIT_LOCK_FILE = BASE_DIR / "devplace-init.lock"
+
+# Container data lives OUTSIDE the package (never served via /static, never watched by --reload).
+DATA_DIR = Path(environ.get("DEVPLACE_DATA_DIR", str(BASE_DIR / "var")))
+CONTAINER_WORKSPACES_DIR = DATA_DIR / "container_workspaces"
+# Every container instance runs this one prebuilt image (built once via `make ppy`).
+CONTAINER_IMAGE = environ.get("DEVPLACE_CONTAINER_IMAGE", "ppy:latest")
+# Override host the /p/<slug> ingress proxy dials, with the published host port,
+# instead of the container's own bridge IP. Set this only for topologies where
+# the app cannot route to the container network directly (containerized app via
+# docker socket: use host.docker.internal). Left empty, the proxy connects
+# straight to the container IP and container port, which avoids the host
+# port-publishing layer (docker-proxy / iptables DNAT / loopback) entirely.
+CONTAINER_PROXY_HOST = environ.get("DEVPLACE_CONTAINER_PROXY_HOST", "").strip()
+
+VAPID_PRIVATE_KEY_FILE = BASE_DIR / "notification-private.pem"
+VAPID_PRIVATE_KEY_PKCS8_FILE = BASE_DIR / "notification-private.pkcs8.pem"
+VAPID_PUBLIC_KEY_FILE = BASE_DIR / "notification-public.pem"
+VAPID_SUB = environ.get("DEVPLACE_VAPID_SUB", "mailto:retoor@molodetz.nl")

devplacepy/constants.py

@@ -1 +1,16 @@
+# retoor <retoor@molodetz.nl>
+
 TOPICS = ["devlog", "showcase", "question", "rant", "fun", "random", "signals"]
+
+REACTION_EMOJI = [
+    "\U0001f44d",
+    "❤️",
+    "\U0001f680",
+    "\U0001f389",
+    "\U0001f602",
+    "\U0001f440",
+    "\U0001f525",
+    "\U0001f92f",
+]
+
+DEVII_GUEST_COOKIE = "devii_guest"

devplacepy/content.py

@@ -0,0 +1,343 @@
+# retoor <retoor@molodetz.nl>
+import logging
+from typing import Any
+from datetime import datetime, timezone
+from fastapi.responses import RedirectResponse
+from devplacepy.attachments import (
+    soft_delete_attachments_for,
+    get_attachments,
+    link_attachments,
+)
+from devplacepy.database import (
+    get_table,
+    resolve_by_slug,
+    get_users_by_uids,
+    get_vote_counts,
+    get_user_votes,
+    get_reactions_by_targets,
+    get_user_bookmarks,
+    get_poll_for_post,
+    soft_delete,
+    soft_delete_in,
+    soft_delete_engagement,
+    soft_delete_fork_relations,
+    load_comments,
+    _now_iso,
+    db,
+)
+from devplacepy.utils import (
+    time_ago,
+    generate_uid,
+    make_combined_slug,
+    award_rewards,
+    create_mention_notifications,
+    is_admin,
+)
+from devplacepy.services.audit import record as audit
+
+CREATE_METADATA_KEYS = ("project_type", "is_private", "language", "topic", "status")
+
+BOOKMARKABLE_TYPES = {"post", "gist", "project", "news"}
+REACTABLE_TYPES = {"post", "comment", "gist", "project"}
+
+logger = logging.getLogger(__name__)
+
+
+def is_owner(item: dict | None, user: dict | None) -> bool:
+    return bool(item and user and item["user_uid"] == user["uid"])
+
+
+def can_view_project(project: dict | None, user: dict | None) -> bool:
+    if not project:
+        return False
+    if not project.get("is_private"):
+        return True
+    return is_owner(project, user) or is_admin(user)
+
+
+def canonical_redirect(
+    area: str, item: dict, requested: str
+) -> RedirectResponse | None:
+    canonical = item.get("slug") or item["uid"]
+    if requested == canonical:
+        return None
+    return RedirectResponse(url=f"/{area}/{canonical}", status_code=301)
+
+
+def first_image_url(item: dict, attachments: list | None) -> str | None:
+    inline = item.get("image")
+    if inline:
+        return f"/static/uploads/{inline}"
+    for attachment in attachments or []:
+        if attachment.get("is_image"):
+            return attachment["url"]
+    return None
+
+
+def create_content_item(
+    table_name: str,
+    target_type: str,
+    user: dict,
+    fields: dict,
+    slug_source: str,
+    xp: int,
+    badge: str,
+    mention_text: str,
+    attachment_uids: list | None,
+    request=None,
+) -> tuple[str, str]:
+    uid = generate_uid()
+    slug = make_combined_slug(slug_source, uid)
+    get_table(table_name).insert(
+        {
+            "uid": uid,
+            "user_uid": user["uid"],
+            "slug": slug,
+            "stars": 0,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+            "deleted_at": None,
+            "deleted_by": None,
+            **fields,
+        }
+    )
+    award_rewards(user["uid"], xp, badge)
+    if attachment_uids:
+        link_attachments(attachment_uids, target_type, uid)
+    create_mention_notifications(mention_text, user["uid"], f"/{table_name}/{slug}")
+    logger.info(f"{target_type} {uid} created by {user['username']}")
+    label = fields.get("title") or slug
+    links = [audit.target(target_type, uid, label)]
+    if fields.get("project_uid"):
+        links.append(audit.project(fields["project_uid"]))
+    metadata = {
+        key: fields[key] for key in CREATE_METADATA_KEYS if fields.get(key) is not None
+    }
+    if attachment_uids:
+        metadata["attachment_count"] = len(attachment_uids)
+    audit.record(
+        request,
+        f"{target_type}.create",
+        user=user,
+        target_type=target_type,
+        target_uid=uid,
+        target_label=label,
+        summary=f"{user['username']} created {target_type} {label}",
+        metadata=metadata or None,
+        links=links,
+    )
+    return uid, slug
+
+
+def detail_context(
+    request,
+    user: dict | None,
+    detail: dict,
+    key: str,
+    seo_ctx: dict,
+    extra: dict | None = None,
+) -> dict:
+    context = {
+        **seo_ctx,
+        "request": request,
+        "user": user,
+        key: detail["item"],
+        "author": detail["author"],
+        "is_owner": detail["is_owner"],
+        "star_count": detail["star_count"],
+        "my_vote": detail["my_vote"],
+        "time_ago": detail["time_ago"],
+        "comments": detail["comments"],
+        "attachments": detail["attachments"],
+        "reactions": detail.get("reactions", {"counts": {}, "mine": []}),
+        "bookmarked": detail.get("bookmarked", False),
+        "poll": detail.get("poll"),
+    }
+    if extra:
+        context.update(extra)
+    return context
+
+
+def edit_content_item(
+    request,
+    table_name: str,
+    user: dict,
+    slug: str,
+    update_fields: dict,
+    redirect_fail: str,
+    target_type: str | None = None,
+):
+    from devplacepy.responses import action_result, wants_json, json_error
+
+    kind = target_type or table_name.rstrip("s")
+    table = get_table(table_name)
+    item = resolve_by_slug(table, slug)
+    if not is_owner(item, user):
+        audit.record(
+            request,
+            f"{kind}.edit",
+            user=user,
+            result="denied",
+            target_type=kind,
+            target_uid=item["uid"] if item else slug,
+            target_label=item.get("title") if item else slug,
+            summary=f"{user['username']} denied editing {kind} {slug}",
+        )
+        if wants_json(request):
+            return json_error(403, "Not allowed")
+        return RedirectResponse(url=redirect_fail, status_code=302)
+    update_fields = {
+        **update_fields,
+        "updated_at": datetime.now(timezone.utc).isoformat(),
+    }
+    table.update({"uid": item["uid"], **update_fields}, ["uid"])
+    logger.info(f"{table_name} {item['uid']} edited by {user['username']}")
+    label = update_fields.get("title") or item.get("title") or item["uid"]
+    audit.record(
+        request,
+        f"{kind}.edit",
+        user=user,
+        target_type=kind,
+        target_uid=item["uid"],
+        target_label=label,
+        summary=f"{user['username']} edited {kind} {label}",
+        metadata={"changed_fields": sorted(update_fields.keys())},
+        links=[audit.target(kind, item["uid"], label)],
+    )
+    url = f"/{table_name}/{item['slug'] or item['uid']}"
+    return action_result(
+        request, url, data={"uid": item["uid"], "slug": item.get("slug"), "url": url}
+    )
+
+
+def delete_content_item(
+    request,
+    table_name: str,
+    target_type: str,
+    user: dict,
+    slug: str,
+    redirect_url: str,
+    inline_image_field: str | None = None,
+):
+    from devplacepy.responses import action_result, wants_json, json_error
+
+    table = get_table(table_name)
+    item = resolve_by_slug(table, slug)
+    if not item or not (is_owner(item, user) or is_admin(user)):
+        audit.record(
+            request,
+            f"{target_type}.delete",
+            user=user,
+            result="denied",
+            target_type=target_type,
+            target_uid=item["uid"] if item else slug,
+            target_label=item.get("title") if item else slug,
+            summary=f"{user['username']} denied deleting {target_type} {slug}",
+        )
+        if wants_json(request):
+            return json_error(403, "Not allowed")
+        return RedirectResponse(url=redirect_url, status_code=302)
+    item_label = item.get("title") or item["uid"]
+    actor = user["uid"]
+    stamp = _now_iso()
+    soft_delete_attachments_for(target_type, [item["uid"]], actor)
+    comment_uids = []
+    if "comments" in db.tables:
+        comments = get_table("comments")
+        comment_uids = [
+            comment["uid"]
+            for comment in comments.find(target_uid=item["uid"], deleted_at=None)
+        ]
+        soft_delete_attachments_for("comment", comment_uids, actor)
+        soft_delete("comments", actor, stamp=stamp, target_uid=item["uid"])
+    if "votes" in db.tables:
+        soft_delete("votes", actor, stamp=stamp, target_uid=item["uid"])
+        soft_delete_in(
+            "votes", "target_uid", comment_uids, actor, stamp=stamp, target_type="comment"
+        )
+    soft_delete_engagement(target_type, [item["uid"]], actor)
+    if comment_uids:
+        soft_delete_engagement("comment", comment_uids, actor)
+    if target_type == "project":
+        from devplacepy.project_files import soft_delete_all_project_files
+
+        soft_delete_all_project_files(item["uid"], actor)
+        soft_delete_fork_relations(item["uid"], actor)
+    soft_delete(table_name, actor, stamp=stamp, uid=item["uid"])
+    logger.info(f"{table_name} {item['uid']} soft-deleted by {user['username']}")
+    audit.record(
+        request,
+        f"{target_type}.delete",
+        user=user,
+        target_type=target_type,
+        target_uid=item["uid"],
+        target_label=item_label,
+        summary=f"{user['username']} deleted {target_type} {item_label}",
+        metadata={"comment_count": len(comment_uids)},
+        links=[audit.target(target_type, item["uid"], item_label)],
+    )
+    return action_result(request, redirect_url)
+
+
+def load_detail(
+    table_name: str, target_type: str, slug: str, user: dict | None
+) -> dict | None:
+    item = resolve_by_slug(get_table(table_name), slug)
+    if not item:
+        return None
+    author = get_users_by_uids([item["user_uid"]]).get(item["user_uid"])
+    ups, downs = get_vote_counts([item["uid"]])
+    reactions = (
+        get_reactions_by_targets(target_type, [item["uid"]], user).get(
+            item["uid"], {"counts": {}, "mine": []}
+        )
+        if target_type in REACTABLE_TYPES
+        else {"counts": {}, "mine": []}
+    )
+    bookmarked = (
+        bool(user)
+        and target_type in BOOKMARKABLE_TYPES
+        and item["uid"] in get_user_bookmarks(user["uid"], target_type, [item["uid"]])
+    )
+    return {
+        "item": item,
+        "author": author,
+        "is_owner": bool(user and user["uid"] == item["user_uid"]),
+        "star_count": ups.get(item["uid"], 0) - downs.get(item["uid"], 0),
+        "my_vote": get_user_votes(user["uid"], [item["uid"]]).get(item["uid"], 0)
+        if user
+        else 0,
+        "comments": load_comments(target_type, item["uid"], user),
+        "attachments": get_attachments(target_type, item["uid"]),
+        "time_ago": time_ago(item["created_at"]),
+        "reactions": reactions,
+        "bookmarked": bookmarked,
+        "poll": get_poll_for_post(item["uid"], user) if target_type == "post" else None,
+    }
+
+
+def enrich_items(
+    items: list,
+    key: str,
+    authors: dict,
+    extra_maps: dict[str, Any] | None = None,
+    ts_field: str = "created_at",
+    user: dict | None = None,
+) -> list:
+    extra_maps = extra_maps or {}
+    user_votes = (
+        get_user_votes(user["uid"], [item["uid"] for item in items]) if user else {}
+    )
+    enriched = []
+    for item in items:
+        entry = {
+            key: item,
+            "author": authors.get(item["user_uid"]),
+            "time_ago": time_ago(item[ts_field]),
+            "my_vote": user_votes.get(item["uid"], 0),
+        }
+        for name, source in extra_maps.items():
+            entry[name] = (
+                source(item) if callable(source) else source.get(item["uid"], 0)
+            )
+        enriched.append(entry)
+    return enriched

devplacepy/customization.py

@@ -0,0 +1,77 @@
+# retoor <retoor@molodetz.nl>
+
+from __future__ import annotations
+
+import json
+import logging
+
+from markupsafe import Markup
+from starlette.requests import Request
+
+from devplacepy.constants import DEVII_GUEST_COOKIE
+from devplacepy.database import get_custom_overrides, get_setting
+from devplacepy.utils import get_current_user
+
+logger = logging.getLogger("customization")
+
+EMPTY = Markup("")
+
+
+def page_type_for(request: Request) -> str:
+    route = request.scope.get("route")
+    path = getattr(route, "path", None)
+    if path:
+        return path
+    return request.url.path
+
+
+def owner_for(request: Request) -> tuple[str, str] | None:
+    user = get_current_user(request)
+    if user:
+        return "user", user["uid"]
+    guest = request.cookies.get(DEVII_GUEST_COOKIE)
+    if guest:
+        return "guest", guest
+    return None
+
+
+def _overrides_for(request: Request) -> dict:
+    if get_setting("customization_enabled", "1") != "1":
+        return {"css": "", "js": ""}
+    owner = owner_for(request)
+    if owner is None:
+        return {"css": "", "js": ""}
+    return get_custom_overrides(owner[0], owner[1], page_type_for(request))
+
+
+def custom_css_tag(request: Request) -> Markup:
+    try:
+        css = _overrides_for(request).get("css", "")
+        if not css.strip():
+            return EMPTY
+        safe = css.replace("</", "<\\/")
+        return Markup(f'<style id="user-custom-css">\n{safe}\n</style>')
+    except Exception as exc:  # noqa: BLE001 - a customization bug must never break page render
+        logger.warning("custom_css_tag failed: %s", exc)
+        return EMPTY
+
+
+def custom_js_tag(request: Request) -> Markup:
+    try:
+        if get_setting("customization_js_enabled", "1") != "1":
+            return EMPTY
+        code = _overrides_for(request).get("js", "")
+        if not code.strip():
+            return EMPTY
+        payload = json.dumps(code).replace("<", "\\u003c").replace(">", "\\u003e")
+        runner = (
+            f'<script type="application/json" id="user-custom-js-src">{payload}</script>'
+            "<script>(function(){try{"
+            'var src=document.getElementById("user-custom-js-src");'
+            "if(src){new Function(JSON.parse(src.textContent))();}"
+            '}catch(error){console.error("user custom js error",error);}})();</script>'
+        )
+        return Markup(runner)
+    except Exception as exc:  # noqa: BLE001 - a customization bug must never break page render
+        logger.warning("custom_js_tag failed: %s", exc)
+        return EMPTY

devplacepy/docs_examples.py

@@ -0,0 +1,83 @@
+# retoor <retoor@molodetz.nl>
+
+from typing import Any, Union, get_args, get_origin
+
+from pydantic import BaseModel
+
+_PLACEHOLDERS = {
+    "uid": "UID",
+    "username": "username",
+    "slug": "slug",
+    "url": "/path",
+    "email": "user@example.com",
+    "title": "Title",
+    "content": "text",
+    "description": "text",
+    "bio": "text",
+    "language": "python",
+    "topic": "random",
+    "status": "published",
+    "next_cursor": "2026-01-01T00:00:00+00:00",
+    "created_at": "2026-01-01T00:00:00+00:00",
+    "updated_at": "2026-01-01T00:00:00+00:00",
+    "synced_at": "2026-01-01T00:00:00+00:00",
+    "time_ago": "2 hours ago",
+}
+
+
+def _placeholder(name: str) -> str:
+    for token, value in _PLACEHOLDERS.items():
+        if token in name:
+            return value
+    return "string"
+
+
+def _unwrap_optional(annotation):
+    if get_origin(annotation) is Union:
+        args = [arg for arg in get_args(annotation) if arg is not type(None)]
+        return args[0] if args else Any
+    return annotation
+
+
+def _value(name, annotation, stack):
+    annotation = _unwrap_optional(annotation)
+    origin = get_origin(annotation)
+    if origin in (list, set, tuple):
+        args = get_args(annotation)
+        inner = args[0] if args else str
+        if isinstance(inner, type) and issubclass(inner, BaseModel) and inner in stack:
+            return []
+        return [_value(name, inner, stack)]
+    if origin is dict or annotation is dict:
+        return {}
+    if annotation in (list, set, tuple):
+        return []
+    if isinstance(annotation, type) and issubclass(annotation, BaseModel):
+        return _from_model(annotation, stack)
+    if annotation is bool:
+        return False
+    if annotation is int:
+        return 0
+    if annotation is float:
+        return 0.0
+    if annotation is Any:
+        return None
+    return _placeholder(name)
+
+
+def _from_model(model, stack):
+    if model in stack:
+        return {}
+    stack = stack | {model}
+    example = {}
+    for name, info in model.model_fields.items():
+        example[name] = _value(name, info.annotation, stack)
+    return example
+
+
+def schema_example(model):
+    return _from_model(model, frozenset())
+
+
+def action_example(redirect, data=None):
+    return {"ok": True, "redirect": redirect, "data": data}

devplacepy/docs_export.py

@@ -0,0 +1,191 @@
+# retoor <retoor@molodetz.nl>
+
+import base64
+import json
+import re
+from functools import lru_cache
+from pathlib import Path
+
+from devplacepy.config import STATIC_DIR
+from devplacepy import docs_api
+
+
+@lru_cache(maxsize=None)
+def _vendor(name: str) -> str:
+    return (Path(STATIC_DIR) / "vendor" / name).read_text(encoding="utf-8")
+
+
+def _prose_markdown(slug: str, ctx: dict) -> str:
+    from devplacepy.templating import templates
+
+    try:
+        rendered = templates.env.get_template(f"docs/{slug}.html").render(**ctx)
+    except Exception:
+        return ""
+    rendered = re.sub(r"^\s*<div[^>]*>", "", rendered.strip())
+    rendered = re.sub(r"</div>\s*$", "", rendered)
+    return rendered.strip()
+
+
+def _params_table(params: list) -> str:
+    if not params:
+        return ""
+    rows = [
+        "| Name | In | Type | Required | Description |",
+        "|------|----|------|----------|-------------|",
+    ]
+    for p in params:
+        desc = (p.get("description", "") or "").replace("|", "\\|")
+        if p["type"] == "enum" and p.get("options"):
+            allowed = ", ".join(str(option) for option in p["options"]).replace(
+                "|", "\\|"
+            )
+            desc = f"{desc} Allowed: {allowed}.".strip()
+        rows.append(
+            f"| `{p['name']}` | {p['location']} | {p['type']} | "
+            f"{'yes' if p['required'] else 'no'} | {desc} |"
+        )
+    return "\n".join(rows)
+
+
+def _endpoint_markdown(ep: dict) -> str:
+    lines = [
+        f"### `{ep['method']} {ep['path']}` - {ep['title']}",
+        "",
+        ep["summary"],
+        "",
+        f"*Minimal role:* {ep.get('min_role', ep['auth'])}",
+    ]
+    if ep.get("params"):
+        lines += ["", "**Parameters**", "", _params_table(ep["params"])]
+    for note in ep.get("notes", []) or []:
+        lines += ["", f"> {note}"]
+    if ep.get("sample_response") is not None:
+        lines += [
+            "",
+            "**Sample response**",
+            "",
+            "```json",
+            json.dumps(ep["sample_response"], indent=2),
+            "```",
+        ]
+    return "\n".join(lines)
+
+
+def _group_markdown(group: dict) -> str:
+    parts = [(group.get("intro", "") or "").strip(), ""]
+    for ep in group.get("endpoints", []) or []:
+        parts.append(_endpoint_markdown(ep))
+        parts.append("")
+    return "\n".join(parts).strip()
+
+
+def build_markdown(
+    is_admin: bool, base: str, username: str = "", api_key: str = ""
+) -> str:
+    from devplacepy.routers.docs import DOCS_PAGES
+    from devplacepy.services.manager import service_manager
+
+    user_ctx = {"role": "Admin"} if is_admin else None
+    replacements = {
+        "{{ base }}": base,
+        "{{ username }}": username or "YOUR_USERNAME",
+        "{{ api_key }}": api_key or "YOUR_API_KEY",
+    }
+    pages = [
+        p
+        for p in DOCS_PAGES
+        if (not p.get("admin") or is_admin) and p["kind"] != "live"
+    ]
+    slugs = {p["slug"] for p in pages}
+
+    out = [
+        "# DevPlace Documentation",
+        "",
+        f"Complete developer documentation for `{base}`.",
+        "",
+        "## Contents",
+        "",
+    ]
+    for page in pages:
+        out.append(f"- [{page['title']}](#doc-{page['slug']})")
+    out += ["", "---", ""]
+
+    for page in pages:
+        if page["kind"] == "prose":
+            body = _prose_markdown(page["slug"], {"base": base, "user": user_ctx})
+        elif page.get("dynamic"):
+            group = docs_api.build_services_group(service_manager.describe_all(), base)
+            body = _group_markdown(docs_api._substitute(group, replacements))
+        else:
+            group = docs_api.render_group(page["slug"], base, username, api_key) or {}
+            body = _group_markdown(group)
+        out += [f'<a id="doc-{page["slug"]}"></a>', body, "", "---", ""]
+
+    text = "\n".join(out).strip() + "\n"
+    return re.sub(
+        r"\(/docs/([a-z0-9-]+)\.html(?:#[^)]*)?\)",
+        lambda m: f"(#doc-{m.group(1)})" if m.group(1) in slugs else m.group(0),
+        text,
+    )
+
+
+_LAYOUT_CSS = """
+:root { color-scheme: light dark; }
+* { box-sizing: border-box; }
+body { margin: 0; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+  line-height: 1.65; color: #1f2328; background: #fff; }
+.doc-header { position: sticky; top: 0; z-index: 5; background: #0d1117; color: #fff;
+  padding: 0.9rem 1.5rem; font-weight: 700; letter-spacing: 0.02em; box-shadow: 0 1px 0 rgba(0,0,0,0.15); }
+.doc-header span { opacity: 0.6; font-weight: 400; }
+main.doc { max-width: 860px; margin: 0 auto; padding: 2rem 1.5rem 4rem; }
+main.doc h1 { font-size: 1.9rem; margin: 2.4rem 0 1rem; padding-bottom: 0.3rem; border-bottom: 2px solid #e1e4e8; }
+main.doc h1:first-child { margin-top: 0; }
+main.doc h2 { font-size: 1.4rem; margin: 2rem 0 0.8rem; }
+main.doc h3 { font-size: 1.08rem; margin: 1.6rem 0 0.5rem; }
+main.doc h3 code { background: #eef1f4; padding: 0.1em 0.4em; border-radius: 5px; font-size: 0.95em; }
+main.doc a { color: #0969da; text-decoration: none; }
+main.doc a:hover { text-decoration: underline; }
+main.doc code { font-family: "SF Mono", Monaco, Consolas, monospace; font-size: 0.9em; }
+main.doc :not(pre) > code { background: #eef1f4; padding: 0.12em 0.4em; border-radius: 5px; }
+main.doc pre { background: #282c34; border-radius: 8px; padding: 1rem; overflow-x: auto; }
+main.doc pre code { color: #abb2bf; font-size: 0.85rem; line-height: 1.5; }
+main.doc table { border-collapse: collapse; width: 100%; margin: 1rem 0; font-size: 0.92rem; }
+main.doc th, main.doc td { border: 1px solid #d0d7de; padding: 0.45rem 0.7rem; text-align: left; }
+main.doc th { background: #f6f8fa; }
+main.doc blockquote { margin: 0.8rem 0; padding: 0.2rem 1rem; border-left: 4px solid #d0d7de; color: #57606a; }
+main.doc hr { border: none; border-top: 1px solid #e1e4e8; margin: 2.5rem 0; }
+@media (prefers-color-scheme: dark) {
+  body { background: #0d1117; color: #c9d1d9; }
+  main.doc h1, main.doc h2 { border-color: #30363d; }
+  main.doc :not(pre) > code, main.doc h3 code { background: #21262d; }
+  main.doc th { background: #161b22; }
+  main.doc th, main.doc td, main.doc hr, main.doc blockquote { border-color: #30363d; }
+  main.doc a { color: #58a6ff; }
+}
+"""
+
+
+def build_html(markdown: str) -> str:
+    marked = _vendor("marked.umd.js")
+    hljs = _vendor("highlight.min.js")
+    hl_css = _vendor("atom-one-dark.min.css")
+    payload = base64.b64encode(markdown.encode("utf-8")).decode("ascii")
+    return (
+        '<!doctype html>\n<html lang="en">\n<head>\n<meta charset="utf-8">\n'
+        '<meta name="viewport" content="width=device-width, initial-scale=1">\n'
+        "<title>DevPlace Documentation</title>\n"
+        f"<style>{hl_css}</style>\n<style>{_LAYOUT_CSS}</style>\n"
+        f"<script>{marked}</script>\n<script>{hljs}</script>\n"
+        "</head>\n<body>\n"
+        '<header class="doc-header">DevPlace <span>Documentation</span></header>\n'
+        '<main id="doc" class="doc"></main>\n'
+        f'<script id="docmd" type="application/x-markdown-base64">{payload}</script>\n'
+        "<script>\n(function(){\n"
+        "  var raw = atob(document.getElementById('docmd').textContent);\n"
+        "  var md = decodeURIComponent(escape(raw));\n"
+        "  marked.setOptions({ gfm: true, breaks: false });\n"
+        "  document.getElementById('doc').innerHTML = marked.parse(md);\n"
+        "  document.querySelectorAll('#doc pre code').forEach(function(el){ try { hljs.highlightElement(el); } catch (e) {} });\n"
+        "})();\n</script>\n</body>\n</html>\n"
+    )

devplacepy/docs_live.py

@@ -0,0 +1,197 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import (
+    db,
+    get_table,
+    get_setting,
+    get_site_stats,
+    get_top_authors,
+    get_featured_news,
+    get_gist_languages,
+    get_daily_topic,
+    get_streaks,
+    get_user_stars,
+    get_user_rank,
+)
+from devplacepy.utils import format_date
+
+
+def _count(table: str, **criteria) -> int:
+    if table not in db.tables:
+        return 0
+    return get_table(table).count(**criteria)
+
+
+def _recent(table: str, user_uid: str, limit: int = 5) -> list:
+    if table not in db.tables:
+        return []
+    return list(
+        get_table(table).find(user_uid=user_uid, order_by=["-created_at"], _limit=limit)
+    )
+
+
+def _user_facts(user: dict) -> dict:
+    uid = user["uid"]
+
+    posts = _recent("posts", uid)
+    gists = _recent("gists", uid)
+    projects = _recent("projects", uid)
+    badges = (
+        [b.get("badge_name", "") for b in get_table("badges").find(user_uid=uid)]
+        if "badges" in db.tables
+        else []
+    )
+    languages = sorted(
+        {
+            g.get("language")
+            for g in (
+                get_table("gists").find(user_uid=uid) if "gists" in db.tables else []
+            )
+            if g.get("language")
+        }
+    )
+
+    return {
+        "username": user.get("username", ""),
+        "role": user.get("role", "Member"),
+        "member_since": format_date(user.get("created_at", ""))
+        if user.get("created_at")
+        else "",
+        "level": user.get("level", 1),
+        "xp": user.get("xp", 0),
+        "stars": get_user_stars(uid),
+        "rank": get_user_rank(uid),
+        "streak": get_streaks(uid),
+        "badges": badges,
+        "languages": languages,
+        "counts": {
+            "posts": _count("posts", user_uid=uid),
+            "gists": _count("gists", user_uid=uid),
+            "projects": _count("projects", user_uid=uid),
+            "comments": _count("comments", user_uid=uid),
+            "followers": _count("follows", following_uid=uid),
+            "following": _count("follows", follower_uid=uid),
+            "bookmarks": _count("bookmarks", user_uid=uid),
+            "unread_notifications": _count("notifications", user_uid=uid, read=False),
+            "unread_messages": _count("messages", receiver_uid=uid, read=False),
+        },
+        "recent_posts": [
+            {
+                "title": (
+                    p.get("title") or (p.get("content") or "")[:60] or "Untitled"
+                ),
+                "url": f"/posts/{p.get('slug') or p['uid']}",
+            }
+            for p in posts
+        ],
+        "recent_gists": [
+            {
+                "title": g.get("title", "Untitled"),
+                "language": g.get("language", ""),
+                "url": f"/gists/{g.get('slug') or g['uid']}",
+            }
+            for g in gists
+        ],
+        "recent_projects": [
+            {
+                "title": p.get("title", "Untitled"),
+                "status": p.get("status", ""),
+                "url": f"/projects/{p.get('slug') or p['uid']}",
+            }
+            for p in projects
+        ],
+    }
+
+
+def build_live_facts(user: dict | None) -> dict:
+    stats = get_site_stats()
+    facts = {
+        "site": {
+            "name": get_setting("site_name", "DevPlace"),
+            "tagline": get_setting("site_tagline", ""),
+            "total_members": stats.get("total_members", 0),
+            "posts_today": stats.get("posts_today", 0),
+            "total_projects": stats.get("total_projects", 0),
+            "total_gists": stats.get("total_gists", 0),
+        },
+        "top_authors": [
+            {"username": a.get("username", ""), "stars": a.get("stars", 0)}
+            for a in get_top_authors(10)
+        ],
+        "languages": sorted(get_gist_languages()),
+        "news": get_featured_news(5),
+        "topic": get_daily_topic(),
+        "user": _user_facts(user) if user else None,
+    }
+    return facts
+
+
+def live_text(facts: dict) -> str:
+    parts: list[str] = []
+    site = facts["site"]
+    parts += [
+        site["name"],
+        site["tagline"],
+        "members",
+        str(site["total_members"]),
+        "posts today",
+        str(site["posts_today"]),
+        "projects",
+        str(site["total_projects"]),
+        "gists",
+        str(site["total_gists"]),
+        "top authors leaderboard",
+    ]
+    parts += [f"{a['username']} {a['stars']} stars" for a in facts["top_authors"]]
+    parts += ["trending languages"] + facts["languages"]
+    parts += ["developer news"] + [n.get("title", "") for n in facts["news"]]
+    parts += [facts["topic"].get("title", "")]
+
+    user = facts.get("user")
+    if user:
+        counts = user["counts"]
+        parts += [
+            "your dashboard account stats",
+            user["username"],
+            user["role"],
+            "member since",
+            user["member_since"],
+            "level",
+            str(user["level"]),
+            "xp",
+            str(user["xp"]),
+            "stars",
+            str(user["stars"]),
+            "rank",
+            str(user["rank"]),
+            "current streak",
+            str(user["streak"].get("current", 0)),
+            "longest streak",
+            str(user["streak"].get("longest", 0)),
+            "posts",
+            str(counts["posts"]),
+            "gists",
+            str(counts["gists"]),
+            "projects",
+            str(counts["projects"]),
+            "comments",
+            str(counts["comments"]),
+            "followers",
+            str(counts["followers"]),
+            "following",
+            str(counts["following"]),
+            "bookmarks saved",
+            str(counts["bookmarks"]),
+            "unread notifications",
+            str(counts["unread_notifications"]),
+            "unread messages",
+            str(counts["unread_messages"]),
+            "badges",
+        ]
+        parts += user["badges"]
+        parts += ["languages you use"] + user["languages"]
+        parts += ["your recent posts"] + [p["title"] for p in user["recent_posts"]]
+        parts += ["your gists"] + [g["title"] for g in user["recent_gists"]]
+        parts += ["your projects"] + [p["title"] for p in user["recent_projects"]]
+
+    return " ".join(str(part) for part in parts if part)

devplacepy/docs_prose.py

@@ -0,0 +1,28 @@
+# retoor <retoor@molodetz.nl>
+
+import html
+import re
+
+import mistune
+
+_markdown = mistune.create_markdown(
+    escape=False,
+    hard_wrap=True,
+    plugins=["table", "strikethrough", "url"],
+)
+
+_RENDER_BLOCK = re.compile(
+    r'<div class="docs-content" data-render>(.*?)</div>', re.DOTALL
+)
+
+
+def _convert(match: re.Match) -> str:
+    source = html.unescape(match.group(1)).strip()
+    return f'<div class="docs-content">{_markdown(source)}</div>'
+
+
+def render_prose(slug: str, context: dict) -> str:
+    from devplacepy.templating import templates
+
+    rendered = templates.env.get_template(f"docs/{slug}.html").render(**context)
+    return _RENDER_BLOCK.sub(_convert, rendered, count=1)

devplacepy/docs_search.py

@@ -0,0 +1,207 @@
+# retoor <retoor@molodetz.nl>
+
+import html
+import math
+import re
+from pathlib import Path
+
+from devplacepy.config import TEMPLATES_DIR
+from devplacepy import docs_api
+
+K1 = 1.5
+B = 0.75
+_TOKEN = re.compile(r"[a-z0-9]+")
+_static_docs = None
+
+
+def _tokenize(text: str) -> list:
+    return [t for t in _TOKEN.findall(text.lower()) if len(t) > 1]
+
+
+def _demarkdown(text: str) -> str:
+    text = re.sub(r"(?m)^[ \t]*```.*$", " ", text)
+    text = re.sub(r"!\[([^\]]*)\]\([^)]*\)", r"\1", text)
+    text = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", text)
+    text = re.sub(r"(?m)^[ \t]{0,3}#{1,6}[ \t]*", "", text)
+    text = re.sub(r"(?m)^[ \t]{0,3}>[ \t]?", "", text)
+    text = re.sub(r"(?m)^[ \t]*(?:[-*+]|\d+\.)[ \t]+", "", text)
+    text = re.sub(
+        r"(?m)^[ \t]*\|?(?:[ \t]*:?-{2,}:?[ \t]*\|)+[ \t]*:?-{0,}:?[ \t]*$", " ", text
+    )
+    text = text.replace("|", " ")
+    text = re.sub(r"[`*~]", "", text)
+    return text
+
+
+def _strip(raw: str) -> str:
+    raw = re.sub(r"(?is)<script\b.*?</script>", " ", raw)
+    raw = re.sub(r"(?is)<style\b.*?</style>", " ", raw)
+    raw = re.sub(r"{#.*?#}", " ", raw, flags=re.S)
+    raw = re.sub(r"{%.*?%}", " ", raw, flags=re.S)
+    raw = re.sub(r"{{.*?}}", " ", raw, flags=re.S)
+    raw = html.unescape(raw)
+    raw = re.sub(r"<[^>]+>", " ", raw)
+    raw = _demarkdown(raw)
+    return re.sub(r"\s+", " ", raw).strip()
+
+
+def _prose_text(slug: str) -> str:
+    path = Path(TEMPLATES_DIR) / "docs" / f"{slug}.html"
+    try:
+        return _strip(path.read_text(encoding="utf-8"))
+    except OSError:
+        return ""
+
+
+def _group_text(group: dict) -> str:
+    parts = [group.get("intro", "")]
+    for ep in group.get("endpoints", []) or []:
+        parts += [
+            ep.get("method", ""),
+            ep.get("path", ""),
+            ep.get("title", ""),
+            ep.get("summary", ""),
+        ]
+        parts += [str(note) for note in ep.get("notes", []) or []]
+        for param in ep.get("params", []) or []:
+            parts += [param.get("name", ""), param.get("description", "")]
+    return _strip("\n".join(str(p) for p in parts))
+
+
+def _collect_static_docs() -> list:
+    global _static_docs
+    if _static_docs is not None:
+        return _static_docs
+    from devplacepy.routers.docs import DOCS_PAGES
+    from devplacepy.services.manager import service_manager
+
+    docs = []
+    for page in DOCS_PAGES:
+        if page["kind"] == "live":
+            continue
+        slug, title = page["slug"], page["title"]
+        if page["kind"] == "prose":
+            body = _prose_text(slug)
+        elif page.get("dynamic"):
+            body = _group_text(
+                docs_api.build_services_group(service_manager.describe_all(), "")
+            )
+        else:
+            body = _group_text(docs_api.get_group(slug) or {})
+        docs.append(
+            {
+                "slug": slug,
+                "title": title,
+                "admin": page.get("admin", False),
+                "text": f"{title}\n{body}",
+            }
+        )
+    _static_docs = docs
+    return docs
+
+
+def _live_docs(user) -> list:
+    from devplacepy import docs_live
+
+    facts = docs_live.build_live_facts(user)
+    return [
+        {
+            "slug": "dashboard",
+            "title": "Your dashboard",
+            "admin": False,
+            "text": "Your dashboard live data stats\n" + docs_live.live_text(facts),
+        }
+    ]
+
+
+def _build(docs: list) -> dict:
+    postings: dict = {}
+    doc_len: list = []
+    doc_text: list = []
+    df: dict = {}
+    for i, doc in enumerate(docs):
+        tokens = _tokenize(doc["text"])
+        doc_len.append(len(tokens) or 1)
+        doc_text.append(doc["text"])
+        freqs: dict = {}
+        for token in tokens:
+            freqs[token] = freqs.get(token, 0) + 1
+        for token, count in freqs.items():
+            postings.setdefault(token, []).append((i, count))
+            df[token] = df.get(token, 0) + 1
+    n = len(docs) or 1
+    avgdl = (sum(doc_len) / n) if doc_len else 1.0
+    idf = {t: math.log(1 + (n - d + 0.5) / (d + 0.5)) for t, d in df.items()}
+    return {
+        "docs": docs,
+        "postings": postings,
+        "doc_len": doc_len,
+        "doc_text": doc_text,
+        "avgdl": avgdl or 1.0,
+        "idf": idf,
+    }
+
+
+def get_index(user=None) -> dict:
+    return _build(list(_collect_static_docs()) + _live_docs(user))
+
+
+def reset_index() -> None:
+    global _static_docs
+    _static_docs = None
+
+
+def _snippet(text: str, terms: list, width: int = 280) -> str:
+    low = text.lower()
+    pos = -1
+    for term in terms:
+        found = low.find(term)
+        if found != -1 and (pos == -1 or found < pos):
+            pos = found
+    if pos == -1:
+        pos = 0
+    start = max(0, pos - width // 3)
+    end = min(len(text), start + width)
+    fragment = html.escape(text[start:end])
+    for term in sorted(set(terms), key=len, reverse=True):
+        fragment = re.sub(
+            r"(?i)(" + re.escape(html.escape(term)) + r")", r"<mark>\1</mark>", fragment
+        )
+    prefix = "… " if start > 0 else ""
+    suffix = " …" if end < len(text) else ""
+    return prefix + fragment + suffix
+
+
+def search(query: str, user=None, is_admin: bool = False, limit: int = 20) -> list:
+    terms = _tokenize(query or "")
+    if not terms:
+        return []
+    idx = get_index(user)
+    scores: dict = {}
+    for term in set(terms):
+        plist = idx["postings"].get(term)
+        if not plist:
+            continue
+        idf_t = idx["idf"][term]
+        for i, tf in plist:
+            dl = idx["doc_len"][i]
+            denom = tf + K1 * (1 - B + B * dl / idx["avgdl"])
+            scores[i] = scores.get(i, 0.0) + idf_t * (tf * (K1 + 1)) / denom
+    ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
+    results = []
+    for i, score in ranked:
+        doc = idx["docs"][i]
+        if doc["admin"] and not is_admin:
+            continue
+        results.append(
+            {
+                "slug": doc["slug"],
+                "title": doc["title"],
+                "score": round(score, 3),
+                "url": f"/docs/{doc['slug']}.html",
+                "snippet": _snippet(idx["doc_text"][i], terms),
+            }
+        )
+        if len(results) >= limit:
+            break
+    return results

devplacepy/main.py

@@ -1,19 +1,80 @@
-import asyncio
+# retoor <retoor@molodetz.nl>
+
+import fcntl
 import logging
 import os
 import time
 from collections import defaultdict
+from contextlib import contextmanager
+from pathlib import Path
 from fastapi import FastAPI, Request
 from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
-from devplacepy.config import STATIC_DIR, PORT
-from devplacepy.database import init_db, get_table, db, get_users_by_uids, get_comment_counts_by_post_uids, get_vote_counts
+from fastapi.exceptions import RequestValidationError
+from devplacepy.config import STATIC_DIR, PORT, SERVICE_LOCK_FILE, INIT_LOCK_FILE
+from devplacepy.database import (
+    init_db,
+    get_table,
+    db,
+    refresh_snapshot,
+    get_users_by_uids,
+    get_comment_counts_by_post_uids,
+    get_vote_counts,
+    get_news_images_by_uids,
+    get_setting,
+    get_int_setting,
+)
 from devplacepy.templating import templates
+from devplacepy.responses import respond, wants_json, json_error
+from devplacepy.schemas import LandingOut, ValidationErrorOut
+from fastapi.responses import JSONResponse
 from devplacepy.utils import get_current_user, time_ago
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, combine
-from devplacepy.routers import auth, feed, posts, comments, projects, profile, messages, notifications, votes, avatar, follow, admin, seo, bugs, news, gists, services as services_router, uploads
+from devplacepy.seo import base_seo_context, site_url, website_schema
+from devplacepy.routers import (
+    auth,
+    feed,
+    posts,
+    comments,
+    projects,
+    project_files,
+    profile,
+    messages,
+    notifications,
+    votes,
+    avatar,
+    follow,
+    admin,
+    seo,
+    bugs,
+    news,
+    gists,
+    services as services_router,
+    uploads,
+    media,
+    push,
+    leaderboard,
+    reactions,
+    bookmarks,
+    polls,
+    docs,
+    openai_gateway,
+    devii,
+    zips,
+    forks,
+    containers,
+    containers_admin,
+    proxy,
+)
 from devplacepy.services.manager import service_manager
 from devplacepy.services.news import NewsService
+from devplacepy.services.bot import BotsService
+from devplacepy.services.openai_gateway import GatewayService
+from devplacepy.services.devii import DeviiService
+from devplacepy.services.jobs.zip_service import ZipService
+from devplacepy.services.jobs.fork_service import ForkService
+from devplacepy.services.containers.service import ContainerService
+from devplacepy.services.audit import AuditService
+from devplacepy.services.audit import record as audit
 
 logging.basicConfig(
     level=logging.INFO,
@@ -22,42 +83,230 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 _rate_limit_store = defaultdict(list)
-RATE_LIMIT = 60
+RATE_LIMIT = int(os.environ.get("DEVPLACE_RATE_LIMIT", "60"))
 RATE_WINDOW = 60
+WEB_WORKERS = max(1, int(os.environ.get("DEVPLACE_WEB_WORKERS", "1")))
 
-app = FastAPI(title="DevPlace")
+
+def _worker_rate_limit(limit: int) -> int:
+    return max(1, -(-limit // WEB_WORKERS))
+
+
+_service_lock_handle = None
+
+
+@contextmanager
+def init_lock():
+    handle = open(INIT_LOCK_FILE, "w")
+    try:
+        fcntl.flock(handle, fcntl.LOCK_EX)
+        yield
+    finally:
+        fcntl.flock(handle, fcntl.LOCK_UN)
+        handle.close()
+
+
+def acquire_service_lock() -> bool:
+    global _service_lock_handle
+    handle = open(SERVICE_LOCK_FILE, "w")
+    try:
+        fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except OSError:
+        handle.close()
+        return False
+    _service_lock_handle = handle
+    return True
+
+
+INLINE_MEDIA_EXTENSIONS = {
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".gif",
+    ".webp",
+    ".bmp",
+    ".tiff",
+    ".mp4",
+    ".webm",
+    ".ogv",
+    ".mov",
+    ".m4v",
+    ".mp3",
+}
+
+
+class UploadStaticFiles(StaticFiles):
+    async def get_response(self, path, scope):
+        response = await super().get_response(path, scope)
+        disposition = (
+            "inline"
+            if Path(path).suffix.lower() in INLINE_MEDIA_EXTENSIONS
+            else "attachment"
+        )
+        response.headers["Content-Disposition"] = disposition
+        return response
+
+
+app = FastAPI(
+    title="DevPlace",
+    docs_url="/swagger",
+    redoc_url="/redoc",
+    openapi_url="/openapi.json",
+)
+app.mount(
+    "/static/uploads",
+    UploadStaticFiles(directory=str(STATIC_DIR / "uploads"), check_dir=False),
+    name="uploads",
+)
 app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
 
 
 @app.exception_handler(404)
 async def not_found(request: Request, exc):
-    seo_ctx = base_seo_context(request, title="Not Found — DevPlace", description="The page you requested does not exist.", robots="noindex")
-    return templates.TemplateResponse("error.html", {**seo_ctx, "request": request, "error_code": 404, "error_message": "Page not found"}, status_code=404)
+    if wants_json(request):
+        return json_error(404, "Not found")
+    seo_ctx = base_seo_context(
+        request,
+        title="Not Found - DevPlace",
+        description="The page you requested does not exist.",
+        robots="noindex",
+    )
+    return templates.TemplateResponse(
+        request,
+        "error.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "error_code": 404,
+            "error_message": "Page not found",
+        },
+        status_code=404,
+    )
 
 
 @app.exception_handler(500)
 async def server_error(request: Request, exc):
-    seo_ctx = base_seo_context(request, title="Server Error — DevPlace", description="Something went wrong.", robots="noindex")
-    return templates.TemplateResponse("error.html", {**seo_ctx, "request": request, "error_code": 500, "error_message": "Internal server error"}, status_code=500)
+    logger.exception("500 error on %s %s", request.method, request.url.path)
+    if wants_json(request):
+        return json_error(500, "Internal server error")
+    seo_ctx = base_seo_context(
+        request,
+        title="Server Error - DevPlace",
+        description="Something went wrong.",
+        robots="noindex",
+    )
+    return templates.TemplateResponse(
+        request,
+        "error.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "error_code": 500,
+            "error_message": "Internal server error",
+        },
+        status_code=500,
+    )
+
+
+_AUTH_FORM_PAGES = {
+    "/auth/signup": ("signup.html", "Join DevPlace"),
+    "/auth/login": ("login.html", "Sign In"),
+    "/auth/forgot-password": ("forgot_password.html", "Reset Password"),
+}
+
+_FRIENDLY_ERRORS = {
+    ("username", "too_short"): "Username must be between 3 and 32 characters",
+    ("username", "too_long"): "Username must be between 3 and 32 characters",
+    ("password", "too_short"): "Password must be at least 6 characters",
+}
+
+
+def _friendly_error(err):
+    field = err["loc"][-1] if err.get("loc") else ""
+    key = (field, err.get("type", "").replace("string_", ""))
+    if key in _FRIENDLY_ERRORS:
+        return _FRIENDLY_ERRORS[key]
+    msg = err.get("msg", "Invalid input")
+    prefix = "Value error, "
+    return msg[len(prefix) :] if msg.startswith(prefix) else msg
+
+
+@app.exception_handler(RequestValidationError)
+async def on_validation_error(request: Request, exc: RequestValidationError):
+    errors = [_friendly_error(e) for e in exc.errors()]
+    if wants_json(request):
+        fields: dict = {}
+        for raw, friendly in zip(exc.errors(), errors):
+            name = raw["loc"][-1] if raw.get("loc") else "_"
+            fields.setdefault(str(name), []).append(friendly)
+        return JSONResponse(
+            ValidationErrorOut(fields=fields, messages=errors).model_dump(mode="json"),
+            status_code=422,
+        )
+    path = request.url.path
+    page = _AUTH_FORM_PAGES.get(path)
+    if page is None and path.startswith("/auth/reset-password/"):
+        page = ("reset_password.html", "Set New Password")
+    if page:
+        template_name, title = page
+        context = {
+            **base_seo_context(request, title=title, robots="noindex,nofollow"),
+            "request": request,
+            "errors": errors,
+        }
+        try:
+            form = await request.form()
+            context.update({k: v for k, v in form.items() if isinstance(v, str)})
+        except Exception:
+            pass
+        if "token" in request.path_params:
+            context["token"] = request.path_params["token"]
+        return templates.TemplateResponse(
+            request, template_name, context, status_code=400
+        )
+    referer = request.headers.get("referer") or "/feed"
+    return RedirectResponse(url=referer, status_code=303)
+
 
 app.include_router(auth.router, prefix="/auth")
 app.include_router(feed.router, prefix="/feed")
 app.include_router(posts.router, prefix="/posts")
 app.include_router(comments.router, prefix="/comments")
 app.include_router(projects.router, prefix="/projects")
+app.include_router(project_files.router, prefix="/projects")
 app.include_router(profile.router, prefix="/profile")
 app.include_router(messages.router, prefix="/messages")
 app.include_router(notifications.router, prefix="/notifications")
 app.include_router(votes.router, prefix="/votes")
+app.include_router(reactions.router, prefix="/reactions")
+app.include_router(bookmarks.router, prefix="/bookmarks")
+app.include_router(polls.router, prefix="/polls")
 app.include_router(avatar.router, prefix="/avatar")
 app.include_router(follow.router, prefix="/follow")
+app.include_router(leaderboard.router, prefix="/leaderboard")
 app.include_router(admin.router, prefix="/admin")
 app.include_router(seo.router)
+app.include_router(push.router)
+app.include_router(docs.router)
+app.include_router(openai_gateway.router, prefix="/openai")
+app.include_router(devii.router, prefix="/devii")
 app.include_router(bugs.router, prefix="/bugs")
 app.include_router(gists.router, prefix="/gists")
 app.include_router(news.router, prefix="/news")
 app.include_router(services_router.router, prefix="/admin/services")
 app.include_router(uploads.router, prefix="/uploads")
+app.include_router(media.router, prefix="/media")
+app.include_router(zips.router, prefix="/zips")
+app.include_router(forks.router, prefix="/forks")
+app.include_router(containers.router, prefix="/projects")
+app.include_router(containers_admin.router, prefix="/admin/containers")
+app.include_router(proxy.router, prefix="/p")
+
+
+@app.middleware("http")
+async def refresh_db_snapshot(request: Request, call_next):
+    refresh_snapshot()
+    return await call_next(request)
 
 
 @app.middleware("http")
@@ -71,31 +320,111 @@ async def add_security_headers(request: Request, call_next):
 
 @app.middleware("http")
 async def rate_limit_middleware(request: Request, call_next):
-    if request.method in ("POST", "PUT", "DELETE", "PATCH"):
-        ip = request.client.host if request.client else "unknown"
+    if request.method in (
+        "POST",
+        "PUT",
+        "DELETE",
+        "PATCH",
+    ) and not request.url.path.startswith("/openai"):
+        limit = _worker_rate_limit(
+            max(1, get_int_setting("rate_limit_per_minute", RATE_LIMIT))
+        )
+        window = max(1, get_int_setting("rate_limit_window_seconds", RATE_WINDOW))
+        ip = request.headers.get("X-Real-IP") or (
+            request.client.host if request.client else "unknown"
+        )
         now = time.time()
-        window_start = now - RATE_WINDOW
+        window_start = now - window
         _rate_limit_store[ip] = [t for t in _rate_limit_store[ip] if t > window_start]
-        if len(_rate_limit_store[ip]) >= RATE_LIMIT:
-            return HTMLResponse("Rate limit exceeded. Try again later.", status_code=429)
+        if len(_rate_limit_store[ip]) >= limit:
+            audit.record(
+                request,
+                "security.rate_limit.block",
+                result="denied",
+                summary=f"request from {ip} blocked by rate limit",
+                metadata={"ip": ip, "limit": limit, "window_seconds": window},
+            )
+            retry_after = {"Retry-After": str(window)}
+            if wants_json(request):
+                response = json_error(429, "Rate limit exceeded. Try again later.")
+                response.headers["Retry-After"] = str(window)
+                return response
+            return HTMLResponse(
+                "Rate limit exceeded. Try again later.",
+                status_code=429,
+                headers=retry_after,
+            )
         _rate_limit_store[ip].append(now)
     return await call_next(request)
 
 
+_MAINTENANCE_ALLOWED_PREFIXES = ("/static", "/avatar", "/auth", "/admin", "/openai")
+
+
+@app.middleware("http")
+async def maintenance_middleware(request: Request, call_next):
+    if get_setting("maintenance_mode", "0") != "1":
+        return await call_next(request)
+    if request.url.path.startswith(_MAINTENANCE_ALLOWED_PREFIXES):
+        return await call_next(request)
+    user = get_current_user(request)
+    if user and user.get("role") == "Admin":
+        return await call_next(request)
+    message = get_setting(
+        "maintenance_message",
+        "DevPlace is undergoing scheduled maintenance. Please check back shortly.",
+    )
+    audit.record(
+        request,
+        "security.maintenance.block",
+        user=user,
+        result="denied",
+        summary="non-admin request blocked by maintenance mode",
+    )
+    if wants_json(request):
+        return json_error(503, message)
+    seo_ctx = base_seo_context(
+        request, title="Maintenance - DevPlace", description=message, robots="noindex"
+    )
+    return templates.TemplateResponse(
+        request,
+        "error.html",
+        {**seo_ctx, "request": request, "error_code": 503, "error_message": message},
+        status_code=503,
+    )
+
+
 @app.on_event("startup")
 async def startup():
-    init_db()
+    with init_lock():
+        init_db()
+        from devplacepy.push import ensure_certificates
+
+        ensure_certificates()
+    service_manager.register(NewsService())
+    service_manager.register(BotsService())
+    service_manager.register(GatewayService())
+    service_manager.register(DeviiService())
+    service_manager.register(ZipService())
+    service_manager.register(ForkService())
+    service_manager.register(ContainerService())
+    service_manager.register(AuditService())
     if not os.environ.get("DEVPLACE_DISABLE_SERVICES"):
-        news_service = NewsService()
-        service_manager.register(news_service)
-        asyncio.create_task(service_manager.start_all())
+        if acquire_service_lock():
+            service_manager.set_lock_owner(True)
+            service_manager.supervise()
+            logger.info(f"Background services started in worker pid {os.getpid()}")
+        else:
+            logger.info(
+                f"Worker pid {os.getpid()} declined service lock; another worker owns background services"
+            )
     logger.info(f"DevPlace started on port {PORT}")
 
 
 @app.on_event("shutdown")
 async def shutdown():
     logger.info("Shutting down services...")
-    await service_manager.stop_all()
+    await service_manager.shutdown_all()
 
 
 @app.get("/")
@@ -107,26 +436,25 @@ async def landing(request: Request):
     landing_articles = []
     if "news" in db.tables:
         news_table = get_table("news")
-        raw = list(news_table.find(order_by=["-synced_at"], _limit=6))
-        images_table = get_table("news_images") if "news_images" in db.tables else None
+        raw = list(
+            news_table.find(show_on_landing=1, order_by=["-synced_at"], _limit=6)
+        )
+        images_by_news = get_news_images_by_uids([a["uid"] for a in raw])
         for a in raw:
-            image_url = ""
-            if images_table:
-                img = images_table.find_one(news_uid=a["uid"])
-                if img:
-                    image_url = img["url"]
-            landing_articles.append({
-                "uid": a["uid"],
-                "slug": a.get("slug", ""),
-                "title": a.get("title", ""),
-                "description": (a.get("description", "") or "")[:250],
-                "url": a.get("url", ""),
-                "source_name": a.get("source_name", ""),
-                "grade": a.get("grade", 0),
-                "synced_at": a.get("synced_at", "") or "",
-                "time_ago": time_ago(a["synced_at"]),
-                "image_url": image_url,
-            })
+            landing_articles.append(
+                {
+                    "uid": a["uid"],
+                    "slug": a.get("slug", ""),
+                    "title": a.get("title", ""),
+                    "description": (a.get("description", "") or "")[:250],
+                    "url": a.get("url", ""),
+                    "source_name": a.get("source_name", ""),
+                    "grade": a.get("grade", 0),
+                    "synced_at": a.get("synced_at", "") or "",
+                    "time_ago": time_ago(a["synced_at"]),
+                    "image_url": images_by_news.get(a["uid"], ""),
+                }
+            )
 
     landing_posts = []
     if "posts" in db.tables:
@@ -139,25 +467,33 @@ async def landing(request: Request):
             comment_counts = get_comment_counts_by_post_uids(post_uids)
             upvotes, downvotes = get_vote_counts(post_uids)
             for p in raw_posts:
-                landing_posts.append({
-                    "post": p,
-                    "author": authors.get(p["user_uid"]),
-                    "time_ago": time_ago(p["created_at"]),
-                    "comment_count": comment_counts.get(p["uid"], 0),
-                    "stars": upvotes.get(p["uid"], 0) - downvotes.get(p["uid"], 0),
-                    "slug": p.get("slug", "") or p["uid"],
-                })
+                landing_posts.append(
+                    {
+                        "post": p,
+                        "author": authors.get(p["user_uid"]),
+                        "time_ago": time_ago(p["created_at"]),
+                        "comment_count": comment_counts.get(p["uid"], 0),
+                        "stars": upvotes.get(p["uid"], 0) - downvotes.get(p["uid"], 0),
+                        "slug": p.get("slug", "") or p["uid"],
+                    }
+                )
 
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
-        title="DevPlace — The Developer Social Network",
+        title="DevPlace - The Developer Social Network",
         description="Track industry shifts. Discover bold releases. Share what you're building in an open, uncensored environment.",
         breadcrumbs=[],
         schemas=[website_schema(base)],
     )
-    return templates.TemplateResponse("landing.html", {
-        **seo_ctx, "request": request,
-        "landing_articles": landing_articles,
-        "landing_posts": landing_posts,
-    })
+    return respond(
+        request,
+        "landing.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "landing_articles": landing_articles,
+            "landing_posts": landing_posts,
+        },
+        model=LandingOut,
+    )

devplacepy/models.py

@@ -1,49 +1,394 @@
-from pydantic import BaseModel, Field
-from typing import Optional
-from datetime import date
+# retoor <retoor@molodetz.nl>
+
+from datetime import datetime
+from typing import Literal, Optional
+from pydantic import BaseModel, Field, field_validator, model_validator
+from devplacepy.constants import TOPICS, REACTION_EMOJI
 
 
-class SignupRequest(BaseModel):
-    username: str = Field(min_length=3, max_length=32, pattern=r"^[a-zA-Z0-9_-]+$")
-    email: str = Field(min_length=5, max_length=255)
+def normalize_european_date(value):
+    if not value:
+        return ""
+    text = str(value).strip()
+    if not text:
+        return ""
+    for fmt in ("%d/%m/%Y", "%Y-%m-%d"):
+        try:
+            return datetime.strptime(text, fmt).strftime("%Y-%m-%d")
+        except ValueError:
+            continue
+    raise ValueError("Date must be in DD/MM/YYYY format")
+
+
+def normalize_poll_options(value):
+    if value is None:
+        return []
+    if isinstance(value, str):
+        value = [value]
+    if not isinstance(value, list):
+        return value
+    if len(value) == 1 and isinstance(value[0], str):
+        single = value[0]
+        separator = "\n" if "\n" in single else ("," if "," in single else "")
+        if separator:
+            return [part.strip() for part in single.split(separator) if part.strip()]
+    return value
+
+
+class SignupForm(BaseModel):
+    username: str = Field(min_length=3, max_length=32)
+    email: str = Field(min_length=1, max_length=255)
     password: str = Field(min_length=6, max_length=128)
-    confirm_password: str = Field(min_length=6, max_length=128)
+    confirm_password: str = Field(min_length=1, max_length=128)
+
+    @field_validator("username")
+    @classmethod
+    def username_chars(cls, value):
+        if not value.isascii() or not all(
+            c.isalnum() or c in ("-", "_") for c in value
+        ):
+            raise ValueError(
+                "Username can only contain letters, numbers, hyphens, and underscores"
+            )
+        return value
+
+    @field_validator("email")
+    @classmethod
+    def email_has_at(cls, value):
+        if "@" not in value:
+            raise ValueError("Valid email is required")
+        return value
+
+    @model_validator(mode="after")
+    def passwords_match(self):
+        if self.password != self.confirm_password:
+            raise ValueError("Passwords do not match")
+        return self
 
 
-class LoginRequest(BaseModel):
-    email: str = Field(min_length=5, max_length=255)
+class LoginForm(BaseModel):
+    email: str = Field(min_length=1, max_length=255)
+    password: str = Field(min_length=1, max_length=128)
+    remember_me: str = ""
+    next: str = ""
+
+
+class ForgotPasswordForm(BaseModel):
+    email: str = Field(min_length=1, max_length=255)
+
+    @field_validator("email")
+    @classmethod
+    def email_has_at(cls, value):
+        if "@" not in value:
+            raise ValueError("Valid email is required")
+        return value
+
+
+class ResetPasswordForm(BaseModel):
     password: str = Field(min_length=6, max_length=128)
-    remember_me: bool = False
+    confirm_password: str = Field(min_length=1, max_length=128)
+
+    @model_validator(mode="after")
+    def passwords_match(self):
+        if self.password != self.confirm_password:
+            raise ValueError("Passwords do not match")
+        return self
 
 
-class PostCreate(BaseModel):
-    content: str = Field(min_length=1, max_length=2000)
-    title: Optional[str] = Field(default=None, max_length=500)
-    topic: str = Field(pattern=r"^(devlog|showcase|question|rant|fun|random)$")
-    project_uid: Optional[str] = None
+class PostForm(BaseModel):
+    content: str = Field(min_length=10, max_length=125000)
+    title: str = Field(default="", max_length=500)
+    topic: str = "random"
+    project_uid: str = Field(default="", max_length=36)
+    attachment_uids: list[str] = []
+    poll_question: str = Field(default="", max_length=200)
+    poll_options: list[str] = []
+
+    @field_validator("poll_options")
+    @classmethod
+    def poll_options_max_length(cls, value):
+        if value is None:
+            return value
+        for opt in value:
+            if isinstance(opt, str) and len(opt) > 200:
+                raise ValueError("Each poll option must be 200 characters or fewer")
+        return value
+
+    @field_validator("topic")
+    @classmethod
+    def valid_topic(cls, value):
+        return value if value in TOPICS else "random"
+
+    @field_validator("poll_options", mode="before")
+    @classmethod
+    def split_poll_options(cls, value):
+        return normalize_poll_options(value)
 
 
-class CommentCreate(BaseModel):
-    content: str = Field(min_length=1, max_length=1000)
-    parent_uid: Optional[str] = None
+class PostEditForm(BaseModel):
+    content: str = Field(min_length=10, max_length=125000)
+    title: str = Field(default="", max_length=500)
+    topic: str = "random"
+    poll_question: str = Field(default="", max_length=200)
+    poll_options: list[str] = []
+
+    @field_validator("poll_options")
+    @classmethod
+    def poll_options_max_length(cls, value):
+        if value is None:
+            return value
+        for opt in value:
+            if isinstance(opt, str) and len(opt) > 200:
+                raise ValueError("Each poll option must be 200 characters or fewer")
+        return value
+
+    @field_validator("topic")
+    @classmethod
+    def valid_topic(cls, value):
+        return value if value in TOPICS else "random"
+
+    @field_validator("poll_options", mode="before")
+    @classmethod
+    def split_poll_options(cls, value):
+        return normalize_poll_options(value)
 
 
-class ProjectCreate(BaseModel):
+class PostEditForm(BaseModel):
+    content: str = Field(min_length=10, max_length=125000)
+    title: str = Field(default="", max_length=500)
+    topic: str = "random"
+    poll_question: str = Field(default="", max_length=200)
+    poll_options: list[str] = []
+
+    @field_validator("topic")
+    @classmethod
+    def valid_topic(cls, value):
+        return value if value in TOPICS else "random"
+
+    @field_validator("poll_options", mode="before")
+    @classmethod
+    def split_poll_options(cls, value):
+        return normalize_poll_options(value)
+
+
+class CommentForm(BaseModel):
+    content: str = Field(min_length=3, max_length=1000)
+    target_uid: str = Field(default="", max_length=36)
+    post_uid: str = Field(default="", max_length=36)
+    target_type: Literal["post", "project", "news", "bug", "gist"] = "post"
+    parent_uid: str = Field(default="", max_length=36)
+    attachment_uids: list[str] = []
+
+    @model_validator(mode="after")
+    def require_target(self):
+        if not (self.target_uid or self.post_uid):
+            raise ValueError("A target is required")
+        return self
+
+
+class ProjectForm(BaseModel):
     title: str = Field(min_length=1, max_length=200)
     description: str = Field(min_length=1, max_length=5000)
-    release_date: Optional[date] = None
-    demo_date: Optional[date] = None
-    project_type: str = Field(pattern=r"^(game|game_asset|software|mobile_app|website)$")
+    release_date: str = ""
+    demo_date: str = ""
+    project_type: Literal["game", "game_asset", "software", "mobile_app", "website"] = (
+        "software"
+    )
     platforms: str = Field(default="", max_length=500)
+    status: str = Field(default="In Development", max_length=100)
+    is_private: bool = False
+    attachment_uids: list[str] = []
+
+    @field_validator("release_date", "demo_date", mode="before")
+    @classmethod
+    def normalize_dates(cls, value):
+        return normalize_european_date(value)
 
 
-class MessageCreate(BaseModel):
+class ProjectEditForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(min_length=1, max_length=5000)
+    release_date: str = ""
+    demo_date: str = ""
+    project_type: Literal["game", "game_asset", "software", "mobile_app", "website"] = (
+        "software"
+    )
+    platforms: str = Field(default="", max_length=500)
+    status: str = Field(default="In Development", max_length=100)
+
+    @field_validator("release_date", "demo_date", mode="before")
+    @classmethod
+    def normalize_dates(cls, value):
+        return normalize_european_date(value)
+
+
+class ProjectFlagForm(BaseModel):
+    value: bool = False
+
+
+class CustomizationToggleForm(BaseModel):
+    value: bool = False
+
+
+class ForkForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+
+
+class UploadUrlForm(BaseModel):
+    url: str = Field(min_length=1, max_length=2048)
+    filename: Optional[str] = Field(default=None, max_length=255)
+
+
+class ProjectFileWriteForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    content: str = Field(default="", max_length=400000)
+
+
+class ProjectFileMkdirForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+
+
+class ProjectFileMoveForm(BaseModel):
+    from_path: str = Field(min_length=1, max_length=1024)
+    to_path: str = Field(min_length=1, max_length=1024)
+
+
+class ProjectFileDeleteForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+
+
+class ProjectFileReplaceLinesForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    start: int = Field(ge=1)
+    end: int = Field(ge=0)
+    content: str = Field(default="", max_length=400000)
+
+
+class ProjectFileInsertLinesForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    at: int = Field(ge=1)
+    content: str = Field(default="", max_length=400000)
+
+
+class ProjectFileDeleteLinesForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    start: int = Field(ge=1)
+    end: int = Field(ge=1)
+
+
+class ProjectFileAppendForm(BaseModel):
+    path: str = Field(min_length=1, max_length=1024)
+    content: str = Field(default="", max_length=400000)
+
+
+class ContainerInstanceForm(BaseModel):
+    name: str = Field(min_length=1, max_length=64)
+    boot_command: str = Field(default="", max_length=500)
+    env: str = Field(default="", max_length=10000)
+    cpu_limit: str = Field(default="", max_length=16)
+    mem_limit: str = Field(default="", max_length=16)
+    ports: str = Field(default="", max_length=500)
+    volumes: str = Field(default="", max_length=2000)
+    restart_policy: str = Field(default="never", max_length=20)
+    autostart: bool = True
+    ingress_slug: str = Field(default="", max_length=64)
+    ingress_port: Optional[int] = Field(default=None, ge=1, le=65535)
+
+
+class ContainerExecForm(BaseModel):
+    command: str = Field(min_length=1, max_length=2000)
+
+
+class ContainerScheduleForm(BaseModel):
+    action: str = Field(max_length=10)
+    kind: str = Field(max_length=10)
+    cron: str = Field(default="", max_length=120)
+    run_at: str = Field(default="", max_length=40)
+    delay_seconds: Optional[int] = Field(default=None, ge=1)
+    every_seconds: Optional[int] = Field(default=None, ge=1)
+    max_runs: Optional[int] = Field(default=None, ge=1)
+
+
+class MessageForm(BaseModel):
     content: str = Field(min_length=1, max_length=2000)
-    receiver_uid: str
+    receiver_uid: str = Field(min_length=1, max_length=36)
+    attachment_uids: list[str] = []
 
 
-class ProfileUpdate(BaseModel):
-    bio: Optional[str] = Field(default=None, max_length=500)
-    location: Optional[str] = Field(default=None, max_length=200)
-    git_link: Optional[str] = Field(default=None, max_length=500)
-    website: Optional[str] = Field(default=None, max_length=500)
+class ProfileForm(BaseModel):
+    bio: str = Field(default="", max_length=500)
+    location: str = Field(default="", max_length=200)
+    git_link: str = Field(default="", max_length=500)
+    website: str = Field(default="", max_length=500)
+
+
+class GistForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(default="", max_length=5000)
+    source_code: str = Field(min_length=1, max_length=400000)
+    language: str = Field(default="plaintext", max_length=50)
+    attachment_uids: list[str] = []
+
+
+class GistEditForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(default="", max_length=5000)
+    source_code: str = Field(min_length=1, max_length=400000)
+    language: str = Field(default="plaintext", max_length=50)
+
+
+class BugForm(BaseModel):
+    title: str = Field(min_length=1, max_length=200)
+    description: str = Field(min_length=1, max_length=5000)
+    attachment_uids: list[str] = []
+
+
+class VoteForm(BaseModel):
+    value: int
+
+    @field_validator("value")
+    @classmethod
+    def valid_value(cls, value):
+        if value not in (1, -1):
+            raise ValueError("value must be 1 or -1")
+        return value
+
+
+class ReactionForm(BaseModel):
+    emoji: str = Field(min_length=1, max_length=16)
+
+    @field_validator("emoji")
+    @classmethod
+    def valid_emoji(cls, value):
+        if value not in REACTION_EMOJI:
+            raise ValueError("Invalid reaction")
+        return value
+
+
+class PollVoteForm(BaseModel):
+    option_uid: str = Field(min_length=1, max_length=36)
+
+
+class AdminRoleForm(BaseModel):
+    role: Literal["member", "admin"]
+
+
+class AdminPasswordForm(BaseModel):
+    password: str = Field(min_length=6, max_length=128)
+
+
+class AdminSettingsForm(BaseModel):
+    site_name: str = Field(default="", max_length=200)
+    site_description: str = Field(default="", max_length=500)
+    site_tagline: str = Field(default="", max_length=500)
+    site_url: str = Field(default="", max_length=300)
+    max_upload_size_mb: str = Field(default="", max_length=10)
+    allowed_file_types: str = Field(default="", max_length=1000)
+    max_attachments_per_resource: str = Field(default="", max_length=10)
+    rate_limit_per_minute: str = Field(default="", max_length=10)
+    rate_limit_window_seconds: str = Field(default="", max_length=10)
+    session_max_age_days: str = Field(default="", max_length=10)
+    session_remember_days: str = Field(default="", max_length=10)
+    registration_open: str = Field(default="", max_length=1)
+    maintenance_mode: str = Field(default="", max_length=1)
+    maintenance_message: str = Field(default="", max_length=300)

devplacepy/project_files.py

@@ -0,0 +1,756 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+import shutil
+from datetime import datetime, timezone
+from pathlib import Path
+
+from devplacepy.database import get_table, db, get_int_setting
+from devplacepy.config import STATIC_DIR
+from devplacepy.utils import generate_uid
+from devplacepy.attachments import _directory_for, _detect_mime
+
+logger = logging.getLogger(__name__)
+
+PROJECT_FILES_DIR = STATIC_DIR / "uploads" / "project_files"
+MAX_TEXT_CHARS = 400_000
+MAX_FILES_PER_PROJECT = 5000
+MAX_PATH_LENGTH = 1024
+MAX_SEGMENT_LENGTH = 255
+MAX_DEPTH = 32
+
+TEXT_EXTENSIONS = {
+    ".txt",
+    ".md",
+    ".markdown",
+    ".rst",
+    ".log",
+    ".csv",
+    ".tsv",
+    ".py",
+    ".pyi",
+    ".ipynb",
+    ".js",
+    ".mjs",
+    ".cjs",
+    ".jsx",
+    ".ts",
+    ".tsx",
+    ".json",
+    ".jsonc",
+    ".yaml",
+    ".yml",
+    ".toml",
+    ".ini",
+    ".cfg",
+    ".conf",
+    ".env",
+    ".html",
+    ".htm",
+    ".xml",
+    ".svg",
+    ".css",
+    ".scss",
+    ".sass",
+    ".less",
+    ".c",
+    ".h",
+    ".cpp",
+    ".cc",
+    ".hpp",
+    ".hh",
+    ".cs",
+    ".java",
+    ".kt",
+    ".kts",
+    ".go",
+    ".rs",
+    ".rb",
+    ".php",
+    ".pl",
+    ".pm",
+    ".lua",
+    ".r",
+    ".dart",
+    ".swift",
+    ".scala",
+    ".clj",
+    ".ex",
+    ".exs",
+    ".erl",
+    ".hs",
+    ".ml",
+    ".vue",
+    ".svelte",
+    ".sh",
+    ".bash",
+    ".zsh",
+    ".fish",
+    ".ps1",
+    ".bat",
+    ".cmd",
+    ".sql",
+    ".graphql",
+    ".gql",
+    ".proto",
+    ".tf",
+    ".hcl",
+    ".dockerfile",
+    ".gitignore",
+    ".gitattributes",
+    ".editorconfig",
+    ".properties",
+}
+TEXT_FILENAMES = {
+    "dockerfile",
+    "makefile",
+    "license",
+    "readme",
+    "changelog",
+    "authors",
+    "procfile",
+    "rakefile",
+    "gemfile",
+    ".gitignore",
+    ".gitattributes",
+    ".env",
+    ".editorconfig",
+    ".dockerignore",
+    ".npmrc",
+    ".prettierrc",
+    ".babelrc",
+}
+
+
+class ProjectFileError(ValueError):
+    pass
+
+
+def is_readonly(project_uid: str) -> bool:
+    if "projects" not in db.tables:
+        return False
+    project = get_table("projects").find_one(uid=project_uid)
+    return bool(project and project.get("read_only"))
+
+
+def _guard_writable(project_uid: str) -> None:
+    if is_readonly(project_uid):
+        raise ProjectFileError("project is read-only; files cannot be modified")
+
+
+def normalize_path(raw) -> str:
+    if raw is None:
+        raise ProjectFileError("path is required")
+    text = str(raw).strip().replace("\\", "/")
+    if "\x00" in text:
+        raise ProjectFileError("path contains a null byte")
+    parts = []
+    for segment in text.split("/"):
+        segment = segment.strip()
+        if segment in ("", "."):
+            continue
+        if segment == "..":
+            raise ProjectFileError("path may not contain '..'")
+        if len(segment) > MAX_SEGMENT_LENGTH:
+            raise ProjectFileError("path segment is too long")
+        if any(ord(ch) < 32 for ch in segment):
+            raise ProjectFileError("path contains control characters")
+        parts.append(segment)
+    if not parts:
+        raise ProjectFileError("path is empty")
+    if len(parts) > MAX_DEPTH:
+        raise ProjectFileError("path is too deep")
+    normalized = "/".join(parts)
+    if len(normalized) > MAX_PATH_LENGTH:
+        raise ProjectFileError("path is too long")
+    return normalized
+
+
+def _name_of(path: str) -> str:
+    return path.rsplit("/", 1)[-1]
+
+
+def _parent_of(path: str) -> str:
+    return path.rsplit("/", 1)[0] if "/" in path else ""
+
+
+def is_text_name(name: str) -> bool:
+    lower = name.lower()
+    if lower in TEXT_FILENAMES:
+        return True
+    return Path(name).suffix.lower() in TEXT_EXTENSIONS
+
+
+def _table():
+    return get_table("project_files")
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _file_url(directory: str, stored_name: str) -> str:
+    return f"/static/uploads/project_files/{directory}/{stored_name}"
+
+
+def get_node(project_uid: str, path: str):
+    return _table().find_one(project_uid=project_uid, path=path, deleted_at=None)
+
+
+def _count(project_uid: str) -> int:
+    if "project_files" not in db.tables:
+        return 0
+    return _table().count(project_uid=project_uid, deleted_at=None)
+
+
+def _guard_capacity(project_uid: str) -> None:
+    if _count(project_uid) >= MAX_FILES_PER_PROJECT:
+        raise ProjectFileError(
+            f"project reached the {MAX_FILES_PER_PROJECT}-node limit"
+        )
+
+
+def _insert_dir(project_uid: str, user: dict, path: str) -> None:
+    _table().insert(
+        {
+            "uid": generate_uid(),
+            "project_uid": project_uid,
+            "deleted_at": None,
+            "deleted_by": None,
+            "user_uid": user["uid"],
+            "path": path,
+            "name": _name_of(path),
+            "parent_path": _parent_of(path),
+            "type": "dir",
+            "content": None,
+            "is_binary": 0,
+            "stored_name": None,
+            "directory": None,
+            "mime_type": None,
+            "size": 0,
+            "created_at": _now(),
+            "updated_at": _now(),
+        }
+    )
+
+
+def ensure_dirs(project_uid: str, user: dict, dir_path: str) -> None:
+    if not dir_path:
+        return
+    current = ""
+    for segment in dir_path.split("/"):
+        current = f"{current}/{segment}" if current else segment
+        existing = get_node(project_uid, current)
+        if existing is None:
+            _guard_capacity(project_uid)
+            _insert_dir(project_uid, user, current)
+        elif existing["type"] != "dir":
+            raise ProjectFileError(f"'{current}' exists and is a file")
+
+
+def make_dir(project_uid: str, user: dict, raw_path: str) -> dict:
+    _guard_writable(project_uid)
+    path = normalize_path(raw_path)
+    existing = get_node(project_uid, path)
+    if existing is not None:
+        if existing["type"] != "dir":
+            raise ProjectFileError(f"'{path}' exists and is a file")
+        return existing
+    ensure_dirs(project_uid, user, path)
+    return get_node(project_uid, path)
+
+
+def write_text_file(project_uid: str, user: dict, raw_path: str, content: str) -> dict:
+    _guard_writable(project_uid)
+    path = normalize_path(raw_path)
+    content = content or ""
+    if len(content) > MAX_TEXT_CHARS:
+        raise ProjectFileError(
+            f"file content exceeds the {MAX_TEXT_CHARS}-character limit"
+        )
+    ensure_dirs(project_uid, user, _parent_of(path))
+    existing = get_node(project_uid, path)
+    if existing is not None and existing["type"] == "dir":
+        raise ProjectFileError(f"'{path}' is a directory")
+    if existing is not None:
+        if existing.get("is_binary"):
+            _unlink_blob(existing)
+        _table().update(
+            {
+                "uid": existing["uid"],
+                "content": content,
+                "is_binary": 0,
+                "stored_name": None,
+                "directory": None,
+                "mime_type": "text/plain",
+                "size": len(content.encode("utf-8")),
+                "updated_at": _now(),
+            },
+            ["uid"],
+        )
+        return get_node(project_uid, path)
+    _guard_capacity(project_uid)
+    _table().insert(
+        {
+            "uid": generate_uid(),
+            "project_uid": project_uid,
+            "deleted_at": None,
+            "deleted_by": None,
+            "user_uid": user["uid"],
+            "path": path,
+            "name": _name_of(path),
+            "parent_path": _parent_of(path),
+            "type": "file",
+            "content": content,
+            "is_binary": 0,
+            "stored_name": None,
+            "directory": None,
+            "mime_type": "text/plain",
+            "size": len(content.encode("utf-8")),
+            "created_at": _now(),
+            "updated_at": _now(),
+        }
+    )
+    return get_node(project_uid, path)
+
+
+def _store_blob(data: bytes, filename: str) -> tuple[str, str, str]:
+    uid = generate_uid()
+    ext = Path(filename).suffix.lower()
+    stored_name = f"{uid}{ext}"
+    directory = _directory_for(uid)
+    file_dir = PROJECT_FILES_DIR / directory
+    file_dir.mkdir(parents=True, exist_ok=True)
+    (file_dir / stored_name).write_bytes(data)
+    return stored_name, directory, _detect_mime(data, filename)
+
+
+def _unlink_blob(node: dict) -> None:
+    stored_name = node.get("stored_name")
+    directory = node.get("directory")
+    if not (stored_name and directory):
+        return
+    try:
+        (PROJECT_FILES_DIR / directory / stored_name).unlink(missing_ok=True)
+    except OSError as exc:
+        logger.warning(
+            "Failed to delete project blob %s/%s: %s", directory, stored_name, exc
+        )
+
+
+def store_upload(
+    project_uid: str, user: dict, dir_path: str, filename: str, data: bytes
+) -> dict:
+    _guard_writable(project_uid)
+    max_bytes = get_int_setting("max_upload_size_mb", 10) * 1024 * 1024
+    if len(data) > max_bytes:
+        raise ProjectFileError(
+            f"file exceeds the {get_int_setting('max_upload_size_mb', 10)}MB limit"
+        )
+    base = _name_of(normalize_path(filename))
+    path = normalize_path(f"{dir_path}/{base}" if dir_path else base)
+
+    if is_text_name(base):
+        try:
+            text = data.decode("utf-8")
+            if len(text) <= MAX_TEXT_CHARS:
+                return write_text_file(project_uid, user, path, text)
+        except UnicodeDecodeError:
+            pass
+
+    ensure_dirs(project_uid, user, _parent_of(path))
+    existing = get_node(project_uid, path)
+    if existing is not None and existing["type"] == "dir":
+        raise ProjectFileError(f"'{path}' is a directory")
+    stored_name, directory, mime = _store_blob(data, base)
+    if existing is not None:
+        if existing.get("is_binary"):
+            _unlink_blob(existing)
+        _table().update(
+            {
+                "uid": existing["uid"],
+                "content": None,
+                "is_binary": 1,
+                "stored_name": stored_name,
+                "directory": directory,
+                "mime_type": mime,
+                "size": len(data),
+                "updated_at": _now(),
+            },
+            ["uid"],
+        )
+        return get_node(project_uid, path)
+    _guard_capacity(project_uid)
+    _table().insert(
+        {
+            "uid": generate_uid(),
+            "project_uid": project_uid,
+            "deleted_at": None,
+            "deleted_by": None,
+            "user_uid": user["uid"],
+            "path": path,
+            "name": base,
+            "parent_path": _parent_of(path),
+            "type": "file",
+            "content": None,
+            "is_binary": 1,
+            "stored_name": stored_name,
+            "directory": directory,
+            "mime_type": mime,
+            "size": len(data),
+            "created_at": _now(),
+            "updated_at": _now(),
+        }
+    )
+    return get_node(project_uid, path)
+
+
+def _descendants(project_uid: str, path: str) -> list:
+    prefix = f"{path}/"
+    return [
+        row
+        for row in _table().find(project_uid=project_uid, deleted_at=None)
+        if row["path"] == path or row["path"].startswith(prefix)
+    ]
+
+
+def move_node(project_uid: str, user: dict, raw_from: str, raw_to: str) -> dict:
+    _guard_writable(project_uid)
+    src_path = normalize_path(raw_from)
+    dst_path = normalize_path(raw_to)
+    if src_path == dst_path:
+        raise ProjectFileError("source and target are the same")
+    if dst_path == src_path or dst_path.startswith(f"{src_path}/"):
+        raise ProjectFileError("cannot move a node into itself")
+    src = get_node(project_uid, src_path)
+    if src is None:
+        raise ProjectFileError(f"'{src_path}' does not exist")
+    if get_node(project_uid, dst_path) is not None:
+        raise ProjectFileError(f"'{dst_path}' already exists")
+    ensure_dirs(project_uid, user, _parent_of(dst_path))
+
+    if src["type"] == "file":
+        _table().update(
+            {
+                "uid": src["uid"],
+                "path": dst_path,
+                "name": _name_of(dst_path),
+                "parent_path": _parent_of(dst_path),
+                "updated_at": _now(),
+            },
+            ["uid"],
+        )
+        return get_node(project_uid, dst_path)
+
+    for row in _descendants(project_uid, src_path):
+        new_path = dst_path + row["path"][len(src_path) :]
+        _table().update(
+            {
+                "uid": row["uid"],
+                "path": new_path,
+                "name": _name_of(new_path),
+                "parent_path": _parent_of(new_path),
+                "updated_at": _now(),
+            },
+            ["uid"],
+        )
+    return get_node(project_uid, dst_path)
+
+
+def delete_node(project_uid: str, raw_path: str, deleted_by: str = "system") -> None:
+    _guard_writable(project_uid)
+    path = normalize_path(raw_path)
+    node = get_node(project_uid, path)
+    if node is None:
+        raise ProjectFileError(f"'{path}' does not exist")
+    stamp = _now()
+    for row in _descendants(project_uid, path):
+        _table().update(
+            {"uid": row["uid"], "deleted_at": stamp, "deleted_by": deleted_by},
+            ["uid"],
+        )
+
+
+def soft_delete_all_project_files(project_uid: str, deleted_by: str) -> None:
+    if "project_files" not in db.tables:
+        return
+    stamp = _now()
+    for row in _table().find(project_uid=project_uid, deleted_at=None):
+        _table().update(
+            {"uid": row["uid"], "deleted_at": stamp, "deleted_by": deleted_by},
+            ["uid"],
+        )
+
+
+def node_to_dict(row: dict) -> dict:
+    is_binary = bool(row.get("is_binary"))
+    url = None
+    if is_binary and row.get("stored_name") and row.get("directory"):
+        url = _file_url(row["directory"], row["stored_name"])
+    return {
+        "uid": row["uid"],
+        "path": row["path"],
+        "name": row["name"],
+        "type": row["type"],
+        "is_binary": is_binary,
+        "mime_type": row.get("mime_type"),
+        "size": row.get("size", 0),
+        "url": url,
+        "updated_at": row.get("updated_at"),
+    }
+
+
+def list_files(project_uid: str) -> list:
+    if "project_files" not in db.tables:
+        return []
+    rows = sorted(
+        _table().find(project_uid=project_uid, deleted_at=None),
+        key=lambda r: r["path"],
+    )
+    return [node_to_dict(row) for row in rows]
+
+
+def read_file(project_uid: str, raw_path: str) -> dict:
+    path = normalize_path(raw_path)
+    node = get_node(project_uid, path)
+    if node is None:
+        raise ProjectFileError(f"'{path}' does not exist")
+    payload = node_to_dict(node)
+    payload["content"] = (
+        node.get("content")
+        if node["type"] == "file" and not node.get("is_binary")
+        else None
+    )
+    return payload
+
+
+IMPORT_SKIP_NAMES = {
+    ".git",
+    "node_modules",
+    "__pycache__",
+    ".venv",
+    "venv",
+    ".mypy_cache",
+    ".pytest_cache",
+    ".DS_Store",
+    ".idea",
+    ".cache",
+}
+IMPORT_MAX_FILE_BYTES = 25 * 1024 * 1024
+
+
+def import_from_dir(project_uid: str, src_dir, user: dict, *, skip_names=None) -> int:
+    _guard_writable(project_uid)
+    src = Path(src_dir).resolve()
+    if not src.is_dir():
+        raise ProjectFileError("source directory does not exist")
+    skip = set(skip_names) if skip_names is not None else set(IMPORT_SKIP_NAMES)
+    imported = 0
+    for root, dirs, files in src.walk():
+        dirs[:] = [d for d in sorted(dirs) if d not in skip]
+        for name in sorted(files):
+            if name in skip:
+                continue
+            full = Path(root) / name
+            if full.is_symlink() or not full.is_file():
+                continue
+            try:
+                if full.stat().st_size > IMPORT_MAX_FILE_BYTES:
+                    continue
+                relative = full.relative_to(src).as_posix()
+                path = normalize_path(relative)
+                data = full.read_bytes()
+            except (OSError, ProjectFileError):
+                continue
+            try:
+                store_upload(project_uid, user, _parent_of(path), _name_of(path), data)
+                imported += 1
+            except ProjectFileError:
+                continue
+    return imported
+
+
+def export_to_dir(project_uid: str, subpath: str, dest_dir) -> int:
+    _guard_writable(project_uid)
+    dest = Path(dest_dir).resolve()
+    dest.mkdir(parents=True, exist_ok=True)
+    if subpath:
+        base = normalize_path(subpath)
+        rows = _descendants(project_uid, base)
+        strip = _parent_of(base)
+    else:
+        rows = list(_table().find(project_uid=project_uid, deleted_at=None))
+        strip = ""
+    written = 0
+    for row in sorted(rows, key=lambda r: r["path"]):
+        relative = row["path"][len(strip) :].lstrip("/") if strip else row["path"]
+        target = (dest / relative).resolve()
+        if target != dest and not target.is_relative_to(dest):
+            raise ProjectFileError("path escapes export directory")
+        if row["type"] == "dir":
+            target.mkdir(parents=True, exist_ok=True)
+            continue
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if target.is_symlink() or target.is_file():
+            target.unlink()
+        if row.get("is_binary") and row.get("stored_name") and row.get("directory"):
+            shutil.copyfile(
+                PROJECT_FILES_DIR / row["directory"] / row["stored_name"], target
+            )
+        else:
+            target.write_text(row.get("content") or "", encoding="utf-8")
+        written += 1
+    return written
+
+
+def _load_text_node(project_uid: str, raw_path: str) -> tuple[dict, str]:
+    path = normalize_path(raw_path)
+    node = get_node(project_uid, path)
+    if node is None:
+        raise ProjectFileError(f"'{path}' does not exist")
+    if node["type"] != "file":
+        raise ProjectFileError(f"'{path}' is a directory, not a file")
+    if node.get("is_binary"):
+        raise ProjectFileError(f"'{path}' is a binary file and has no editable lines")
+    return node, node.get("content") or ""
+
+
+def _split_lines(content: str) -> tuple[list, bool]:
+    if content == "":
+        return [], False
+    trailing = content.endswith("\n")
+    body = content[:-1] if trailing else content
+    return body.split("\n"), trailing
+
+
+def _join_lines(lines: list, trailing: bool) -> str:
+    if not lines:
+        return ""
+    return "\n".join(lines) + ("\n" if trailing else "")
+
+
+def _split_content(content: str) -> tuple[list, bool]:
+    if not content:
+        return [], False
+    return _split_lines(content)
+
+
+def _tail_trailing(new_lines: list, tail: list, file_trailing: bool, content_trailing: bool) -> bool:
+    if not new_lines:
+        return False
+    if tail:
+        return file_trailing
+    return content_trailing or file_trailing
+
+
+def _save_text(project_uid: str, node: dict, new_content: str) -> dict:
+    if len(new_content) > MAX_TEXT_CHARS:
+        raise ProjectFileError(
+            f"file content exceeds the {MAX_TEXT_CHARS}-character limit"
+        )
+    _table().update(
+        {
+            "uid": node["uid"],
+            "content": new_content,
+            "is_binary": 0,
+            "stored_name": None,
+            "directory": None,
+            "mime_type": "text/plain",
+            "size": len(new_content.encode("utf-8")),
+            "updated_at": _now(),
+        },
+        ["uid"],
+    )
+    return get_node(project_uid, node["path"])
+
+
+def read_lines(project_uid: str, raw_path: str, start: int = 1, end=None) -> dict:
+    node, content = _load_text_node(project_uid, raw_path)
+    lines, _ = _split_lines(content)
+    total = len(lines)
+    start = max(1, int(start))
+    end = total if end is None else int(end)
+    if end < 0:
+        end = total
+    end = min(end, total)
+    selected = lines[start - 1 : end] if start <= total and end >= start else []
+    return {
+        "path": node["path"],
+        "start": start,
+        "end": end if selected else start - 1,
+        "total_lines": total,
+        "lines": selected,
+        "content": "\n".join(selected),
+    }
+
+
+def replace_lines(
+    project_uid: str, raw_path: str, start: int, end: int, content: str
+) -> dict:
+    _guard_writable(project_uid)
+    node, original = _load_text_node(project_uid, raw_path)
+    lines, file_trailing = _split_lines(original)
+    total = len(lines)
+    start = max(1, int(start))
+    end = int(end)
+    if start > total:
+        raise ProjectFileError(
+            f"start line {start} is past the end of the file ({total} lines)"
+        )
+    end = min(max(end, start - 1), total)
+    replacement, content_trailing = _split_content(content)
+    tail = lines[end:]
+    new_lines = lines[: start - 1] + replacement + tail
+    trailing = _tail_trailing(new_lines, tail, file_trailing, content_trailing)
+    return _save_text(project_uid, node, _join_lines(new_lines, trailing))
+
+
+def insert_lines(project_uid: str, raw_path: str, at: int, content: str) -> dict:
+    _guard_writable(project_uid)
+    node, original = _load_text_node(project_uid, raw_path)
+    lines, file_trailing = _split_lines(original)
+    total = len(lines)
+    at = max(1, int(at))
+    if at > total + 1:
+        at = total + 1
+    replacement, content_trailing = _split_content(content)
+    tail = lines[at - 1 :]
+    new_lines = lines[: at - 1] + replacement + tail
+    trailing = _tail_trailing(new_lines, tail, file_trailing, content_trailing)
+    return _save_text(project_uid, node, _join_lines(new_lines, trailing))
+
+
+def delete_lines(project_uid: str, raw_path: str, start: int, end: int) -> dict:
+    _guard_writable(project_uid)
+    node, original = _load_text_node(project_uid, raw_path)
+    lines, trailing = _split_lines(original)
+    total = len(lines)
+    start = max(1, int(start))
+    end = min(int(end), total)
+    if start > total or end < start:
+        raise ProjectFileError(
+            f"no lines in range {start}-{end} (file has {total} lines)"
+        )
+    new_lines = lines[: start - 1] + lines[end:]
+    return _save_text(
+        project_uid, node, _join_lines(new_lines, trailing if new_lines else False)
+    )
+
+
+def append_lines(project_uid: str, raw_path: str, content: str) -> dict:
+    _guard_writable(project_uid)
+    node, original = _load_text_node(project_uid, raw_path)
+    lines, file_trailing = _split_lines(original)
+    replacement, content_trailing = _split_content(content)
+    new_lines = lines + replacement
+    trailing = bool(new_lines) and (content_trailing or file_trailing)
+    return _save_text(project_uid, node, _join_lines(new_lines, trailing))
+
+
+def delete_all_project_files(project_uid: str) -> None:
+    if "project_files" not in db.tables:
+        return
+    for row in _table().find(project_uid=project_uid):
+        if row.get("is_binary"):
+            _unlink_blob(row)
+    _table().delete(project_uid=project_uid)

devplacepy/push.py

@@ -0,0 +1,299 @@
+# retoor <retoor@molodetz.nl>
+
+import base64
+import fcntl
+import json
+import logging
+import os
+import random
+import time
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urlparse
+
+import httpx
+import jwt
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from cryptography.hazmat.primitives.hashes import SHA256
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+
+from devplacepy.config import (
+    SECONDS_PER_DAY,
+    VAPID_PRIVATE_KEY_FILE,
+    VAPID_PRIVATE_KEY_PKCS8_FILE,
+    VAPID_PUBLIC_KEY_FILE,
+    VAPID_SUB,
+)
+from devplacepy.database import get_table
+from devplacepy.utils import generate_uid
+
+logger = logging.getLogger(__name__)
+
+JWT_LIFETIME_SECONDS = 60 * 60
+PUSH_TTL_SECONDS = str(SECONDS_PER_DAY)
+DEAD_SUBSCRIPTION_STATUSES = (404, 410)
+ACCEPTED_STATUSES = (200, 201)
+
+
+def generate_private_key() -> None:
+    if not VAPID_PRIVATE_KEY_FILE.exists():
+        private_key = ec.generate_private_key(ec.SECP256R1(), default_backend())
+        pem = private_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.TraditionalOpenSSL,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+        VAPID_PRIVATE_KEY_FILE.write_bytes(pem)
+        logger.info("Generated VAPID private key at %s", VAPID_PRIVATE_KEY_FILE)
+
+
+def generate_pkcs8_private_key() -> None:
+    if not VAPID_PRIVATE_KEY_PKCS8_FILE.exists():
+        private_key = serialization.load_pem_private_key(
+            VAPID_PRIVATE_KEY_FILE.read_bytes(),
+            password=None,
+            backend=default_backend(),
+        )
+        pem = private_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+        VAPID_PRIVATE_KEY_PKCS8_FILE.write_bytes(pem)
+        logger.info(
+            "Generated VAPID PKCS8 private key at %s", VAPID_PRIVATE_KEY_PKCS8_FILE
+        )
+
+
+def generate_public_key() -> None:
+    if not VAPID_PUBLIC_KEY_FILE.exists():
+        private_key = serialization.load_pem_private_key(
+            VAPID_PRIVATE_KEY_FILE.read_bytes(),
+            password=None,
+            backend=default_backend(),
+        )
+        pem = private_key.public_key().public_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PublicFormat.SubjectPublicKeyInfo,
+        )
+        VAPID_PUBLIC_KEY_FILE.write_bytes(pem)
+        logger.info("Generated VAPID public key at %s", VAPID_PUBLIC_KEY_FILE)
+
+
+def ensure_certificates() -> None:
+    if (
+        VAPID_PRIVATE_KEY_FILE.exists()
+        and VAPID_PRIVATE_KEY_PKCS8_FILE.exists()
+        and VAPID_PUBLIC_KEY_FILE.exists()
+    ):
+        return
+    lock_path = VAPID_PRIVATE_KEY_FILE.parent / ".vapid.lock"
+    with open(lock_path, "w") as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX)
+        try:
+            generate_private_key()
+            generate_pkcs8_private_key()
+            generate_public_key()
+        finally:
+            fcntl.flock(lock, fcntl.LOCK_UN)
+
+
+def hkdf(input_key: bytes, salt: bytes, info: bytes, length: int) -> bytes:
+    return HKDF(
+        algorithm=SHA256(),
+        length=length,
+        salt=salt,
+        info=info,
+        backend=default_backend(),
+    ).derive(input_key)
+
+
+def browser_base64(data: bytes) -> str:
+    return base64.urlsafe_b64encode(data).decode("utf-8").rstrip("=")
+
+
+_keys: dict[str, Any] = {}
+
+
+def _load_keys() -> dict[str, Any]:
+    if _keys:
+        return _keys
+    ensure_certificates()
+    private_key = serialization.load_pem_private_key(
+        VAPID_PRIVATE_KEY_FILE.read_bytes(), password=None, backend=default_backend()
+    )
+    public_key = serialization.load_pem_public_key(
+        VAPID_PUBLIC_KEY_FILE.read_bytes(), backend=default_backend()
+    )
+    uncompressed_point = public_key.public_bytes(
+        encoding=serialization.Encoding.X962,
+        format=serialization.PublicFormat.UncompressedPoint,
+    )
+    _keys["private_key_pem"] = private_key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.TraditionalOpenSSL,
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+    _keys["public_key_point"] = uncompressed_point
+    _keys["public_key_base64"] = browser_base64(uncompressed_point)
+    logger.debug("Loaded VAPID key material into cache")
+    return _keys
+
+
+def public_key_standard_b64() -> str:
+    point = _load_keys()["public_key_point"]
+    return base64.b64encode(point).decode("utf-8").rstrip("=")
+
+
+def create_notification_authorization(push_url: str) -> str:
+    target = urlparse(push_url)
+    audience = f"{target.scheme}://{target.netloc}"
+    issued_at = int(time.time())
+    return jwt.encode(
+        {
+            "sub": VAPID_SUB,
+            "aud": audience,
+            "exp": issued_at + JWT_LIFETIME_SECONDS,
+            "nbf": issued_at,
+            "iat": issued_at,
+            "jti": generate_uid(),
+        },
+        _load_keys()["private_key_pem"],
+        algorithm="ES256",
+    )
+
+
+def create_notification_info_with_payload(
+    endpoint: str, auth: str, p256dh: str, payload: str
+) -> dict[str, Any]:
+    message_private_key = ec.generate_private_key(ec.SECP256R1(), default_backend())
+    message_public_key_bytes = message_private_key.public_key().public_bytes(
+        encoding=serialization.Encoding.X962,
+        format=serialization.PublicFormat.UncompressedPoint,
+    )
+
+    salt = os.urandom(16)
+
+    user_key_bytes = base64.urlsafe_b64decode(p256dh + "==")
+    shared_secret = message_private_key.exchange(
+        ec.ECDH(),
+        ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256R1(), user_key_bytes),
+    )
+
+    encryption_key = hkdf(
+        shared_secret,
+        base64.urlsafe_b64decode(auth + "=="),
+        b"Content-Encoding: auth\x00",
+        32,
+    )
+
+    context = (
+        b"P-256\x00"
+        + len(user_key_bytes).to_bytes(2, "big")
+        + user_key_bytes
+        + len(message_public_key_bytes).to_bytes(2, "big")
+        + message_public_key_bytes
+    )
+
+    nonce = hkdf(encryption_key, salt, b"Content-Encoding: nonce\x00" + context, 12)
+    content_encryption_key = hkdf(
+        encryption_key, salt, b"Content-Encoding: aesgcm\x00" + context, 16
+    )
+
+    padding_length = random.randint(0, 16)
+    padding = padding_length.to_bytes(2, "big") + b"\x00" * padding_length
+    data = AESGCM(content_encryption_key).encrypt(
+        nonce, padding + payload.encode("utf-8"), None
+    )
+
+    return {
+        "headers": {
+            "Authorization": f"WebPush {create_notification_authorization(endpoint)}",
+            "Crypto-Key": f"dh={browser_base64(message_public_key_bytes)}; p256ecdsa={_load_keys()['public_key_base64']}",
+            "Encryption": f"salt={browser_base64(salt)}",
+            "Content-Encoding": "aesgcm",
+            "Content-Length": str(len(data)),
+            "Content-Type": "application/octet-stream",
+        },
+        "data": data,
+    }
+
+
+def _mark_subscription_dead(subscription_id: int) -> None:
+    get_table("push_registration").update(
+        {"id": subscription_id, "deleted_at": datetime.now(timezone.utc).isoformat()},
+        ["id"],
+    )
+    logger.info("Soft-deleted dead push subscription id=%s", subscription_id)
+
+
+async def notify_user(user_uid: str, payload: dict[str, Any]) -> None:
+    registrations = list(
+        get_table("push_registration").find(user_uid=user_uid, deleted_at=None)
+    )
+    if not registrations:
+        logger.debug("No active push subscriptions for user %s", user_uid)
+        return
+
+    body = json.dumps(payload)
+    async with httpx.AsyncClient(timeout=10.0) as client:
+        for subscription in registrations:
+            endpoint = subscription["endpoint"]
+            try:
+                notification_payload = create_notification_info_with_payload(
+                    endpoint,
+                    subscription["key_auth"],
+                    subscription["key_p256dh"],
+                    body,
+                )
+                headers = {**notification_payload["headers"], "TTL": PUSH_TTL_SECONDS}
+                response = await client.post(
+                    endpoint, headers=headers, content=notification_payload["data"]
+                )
+            except (httpx.HTTPError, ValueError) as exc:
+                logger.warning("Push error for %s via %s: %s", user_uid, endpoint, exc)
+                continue
+
+            if response.status_code in ACCEPTED_STATUSES:
+                logger.debug("Push delivered to %s via %s", user_uid, endpoint)
+            elif response.status_code in DEAD_SUBSCRIPTION_STATUSES:
+                _mark_subscription_dead(subscription["id"])
+            else:
+                logger.warning(
+                    "Push rejected (%s) for %s via %s",
+                    response.status_code,
+                    user_uid,
+                    endpoint,
+                )
+
+
+async def register(
+    user_uid: str, endpoint: str, key_auth: str, key_p256dh: str
+) -> tuple[dict[str, Any], bool]:
+    table = get_table("push_registration")
+    existing = table.find_one(
+        user_uid=user_uid,
+        endpoint=endpoint,
+        key_auth=key_auth,
+        key_p256dh=key_p256dh,
+        deleted_at=None,
+    )
+    if existing:
+        logger.debug("Push subscription already registered for user %s", user_uid)
+        return existing, False
+
+    record = {
+        "uid": generate_uid(),
+        "user_uid": user_uid,
+        "endpoint": endpoint,
+        "key_auth": key_auth,
+        "key_p256dh": key_p256dh,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "deleted_at": None,
+    }
+    table.insert(record)
+    logger.info("Registered push subscription for user %s", user_uid)
+    return record, True

devplacepy/responses.py

@@ -0,0 +1,54 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+
+from fastapi import Request
+from fastapi.responses import JSONResponse, RedirectResponse
+
+from devplacepy.schemas import ActionResult
+
+logger = logging.getLogger(__name__)
+
+
+def wants_json(request: Request) -> bool:
+    # NOTE: `X-Requested-With: fetch` is intentionally NOT a trigger here. The browser
+    # frontend sends it on form/engagement fetches and relies on the legacy HTML/redirect
+    # behaviour (the four engagement endpoints handle that header themselves). JSON is
+    # requested via the Accept or Content-Type header so existing flows are unchanged.
+    content_type = request.headers.get("content-type", "")
+    if content_type.startswith("application/json"):
+        return True
+    accept = request.headers.get("accept", "")
+    return "application/json" in accept and "text/html" not in accept
+
+
+def respond(request: Request, template: str, context: dict, *, model):
+    if wants_json(request):
+        try:
+            payload = model.model_validate(context).model_dump(mode="json")
+        except Exception as exc:
+            logger.warning("JSON serialization failed for %s: %s", template, exc)
+            return json_error(500, "Could not serialize response")
+        return JSONResponse(payload)
+    from devplacepy.templating import templates
+
+    return templates.TemplateResponse(request, template, context)
+
+
+def action_result(
+    request: Request, redirect_url: str, *, data=None, status_code: int = 302
+):
+    if wants_json(request):
+        return JSONResponse(
+            ActionResult(ok=True, redirect=redirect_url, data=data).model_dump(
+                mode="json"
+            )
+        )
+    return RedirectResponse(url=redirect_url, status_code=status_code)
+
+
+def json_error(status_code: int, message: str, **extra) -> JSONResponse:
+    return JSONResponse(
+        {"error": {"status": status_code, "message": message, **extra}},
+        status_code=status_code,
+    )

devplacepy/routers/__init__.py

@@ -0,0 +1,2 @@
+# retoor <retoor@molodetz.nl>
+

devplacepy/routers/admin.py

@@ -1,38 +1,172 @@
+# retoor <retoor@molodetz.nl>
+
+import json
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
-from devplacepy.database import get_table, db, build_pagination
-from devplacepy.templating import templates
-from devplacepy.utils import require_admin, hash_password, generate_uid, time_ago
+from typing import Annotated
+from fastapi import APIRouter, Request, Form
+from devplacepy.models import AdminRoleForm, AdminPasswordForm, AdminSettingsForm
+from fastapi.responses import HTMLResponse, JSONResponse
+from devplacepy.database import (
+    get_table,
+    build_pagination,
+    get_post_counts_by_user_uids,
+    get_news_images_by_uids,
+    clear_settings_cache,
+    get_platform_analytics,
+    get_deleted_media,
+    soft_delete,
+    restore,
+    purge,
+    list_deleted,
+    count_deleted,
+    restore_event,
+    purge_event,
+    get_users_by_uids,
+    resolve_object_url,
+    SOFT_DELETE_TABLES,
+    _now_iso,
+)
+from devplacepy.attachments import delete_attachment, _unlink_attachment_files
+from devplacepy.project_files import _unlink_blob
+from devplacepy.utils import (
+    require_admin,
+    hash_password,
+    generate_uid,
+    time_ago,
+    clear_user_cache,
+    get_current_user,
+    is_admin,
+    not_found,
+)
 from devplacepy.seo import base_seo_context, site_url, website_schema
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import (
+    AdminUsersOut,
+    AdminNewsOut,
+    AdminSettingsOut,
+    AdminMediaOut,
+    AdminTrashOut,
+    GatewayUsageOut,
+    AuditLogOut,
+    AuditEventOut,
+)
+from devplacepy.utils import not_found
+from devplacepy.database import get_users_by_uids
+from devplacepy.services.audit import query as audit_query
+from devplacepy.services.audit import record as audit
+from urllib.parse import urlencode
+from devplacepy.services.manager import service_manager
+from devplacepy.services.openai_gateway.analytics import (
+    build_analytics,
+    build_user_usage,
+)
+from devplacepy.services.openai_gateway.usage import pricing_from_cfg
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
+def _parse_metadata(raw: str | dict | None) -> dict | None:
+    if not raw:
+        return None
+    if isinstance(raw, dict):
+        return raw
+    try:
+        return json.loads(raw)
+    except (ValueError, TypeError):
+        return {"raw": str(raw)}
+
+
 @router.get("", response_class=HTMLResponse)
 async def admin_index(request: Request):
     require_admin(request)
-    return RedirectResponse(url="/admin/users", status_code=302)
+    return action_result(request, "/admin/users")
+
+
+@router.get("/analytics")
+async def admin_analytics(request: Request, top_n: int = 10):
+    user = get_current_user(request)
+    if not user:
+        return JSONResponse({"error": "Authentication required"}, status_code=401)
+    if not is_admin(user):
+        return JSONResponse({"error": "Admin access required"}, status_code=403)
+    return JSONResponse(get_platform_analytics(top_n))
+
+
+@router.get("/ai-usage", response_class=HTMLResponse)
+async def admin_ai_usage(request: Request):
+    admin = require_admin(request)
+    base = site_url(request)
+    seo_ctx = base_seo_context(
+        request,
+        title="AI usage - Admin",
+        description="AI gateway token usage, cost, latency, and reliability metrics.",
+        robots="noindex,nofollow",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Admin", "url": "/admin"},
+            {"name": "AI usage", "url": "/admin/ai-usage"},
+        ],
+        schemas=[website_schema(base)],
+    )
+    return respond(
+        request,
+        "ai_usage.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": admin,
+            "admin_section": "ai-usage",
+        },
+        model=GatewayUsageOut,
+    )
+
+
+@router.get("/ai-usage/data")
+async def admin_ai_usage_json(request: Request, hours: int = 48, top_n: int = 10):
+    user = get_current_user(request)
+    if not user:
+        return JSONResponse({"error": "Authentication required"}, status_code=401)
+    if not is_admin(user):
+        return JSONResponse({"error": "Admin access required"}, status_code=403)
+    service = service_manager.get_service("openai")
+    pricing = pricing_from_cfg(service.get_config()) if service is not None else None
+    return JSONResponse(build_analytics(hours, top_n=top_n, pricing=pricing))
+
+
+@router.get("/users/{uid}/ai-usage")
+async def admin_user_ai_usage(request: Request, uid: str, hours: int = 24):
+    user = get_current_user(request)
+    if not user:
+        return JSONResponse({"error": "Authentication required"}, status_code=401)
+    if not is_admin(user):
+        return JSONResponse({"error": "Admin access required"}, status_code=403)
+    service = service_manager.get_service("openai")
+    pricing = pricing_from_cfg(service.get_config()) if service is not None else None
+    return JSONResponse(build_user_usage(uid, hours=hours, pricing=pricing))
 
 
 @router.get("/users", response_class=HTMLResponse)
 async def admin_users(request: Request, page: int = 1):
     admin = require_admin(request)
     users_table = get_table("users")
-    total = len(list(users_table.all()))
+    total = users_table.count()
     pagination = build_pagination(page, total)
     offset = (pagination["page"] - 1) * pagination["per_page"]
-    page_users = list(users_table.find(order_by=["-created_at"], _limit=pagination["per_page"], _offset=offset))
+    page_users = list(
+        users_table.find(
+            order_by=["-created_at"], _limit=pagination["per_page"], _offset=offset
+        )
+    )
+    post_counts = get_post_counts_by_user_uids([u["uid"] for u in page_users])
     for u in page_users:
-        posts_count = len(list(get_table("posts").find(user_uid=u["uid"])))
-        u["posts_count"] = posts_count
+        u["posts_count"] = post_counts.get(u["uid"], 0)
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
-        title="Users — Admin",
+        title="Users - Admin",
         description="Manage DevPlace users.",
+        robots="noindex,nofollow",
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
             {"name": "Admin", "url": "/admin"},
@@ -40,56 +174,359 @@ async def admin_users(request: Request, page: int = 1):
         ],
         schemas=[website_schema(base)],
     )
-    return templates.TemplateResponse("admin_users.html", {
-        **seo_ctx,
-        "request": request,
-        "user": admin,
-        "users": page_users,
-        "pagination": pagination,
-        "admin_section": "users",
-    })
+    return respond(
+        request,
+        "admin_users.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": admin,
+            "users": page_users,
+            "pagination": pagination,
+            "admin_section": "users",
+        },
+        model=AdminUsersOut,
+    )
 
 
 @router.post("/users/{uid}/role")
-async def admin_user_role(request: Request, uid: str):
+async def admin_user_role(
+    request: Request, uid: str, data: Annotated[AdminRoleForm, Form()]
+):
     admin = require_admin(request)
-    form = await request.form()
-    role = form.get("role", "").strip().capitalize()
-    if role not in ("Member", "Admin"):
-        return RedirectResponse(url="/admin/users", status_code=302)
+    role = data.role.capitalize()
     if uid == admin["uid"]:
-        return RedirectResponse(url="/admin/users", status_code=302)
+        audit.record(
+            request,
+            "admin.user.role.change",
+            user=admin,
+            result="denied",
+            target_type="user",
+            target_uid=uid,
+            target_label=admin.get("username"),
+            summary=f"admin {admin['username']} cannot change their own role",
+            links=[audit.target("user", uid, admin.get("username"))],
+        )
+        return action_result(request, "/admin/users")
     users = get_table("users")
+    target_user = users.find_one(uid=uid)
+    old_role = target_user.get("role") if target_user else None
     users.update({"uid": uid, "role": role}, ["uid"])
+    clear_user_cache(uid)
     logger.info(f"Admin {admin['username']} set user {uid} role to {role}")
-    return RedirectResponse(url="/admin/users", status_code=302)
+    audit.record(
+        request,
+        "admin.user.role.change",
+        user=admin,
+        target_type="user",
+        target_uid=uid,
+        target_label=target_user.get("username") if target_user else None,
+        old_value=old_role,
+        new_value=role,
+        summary=f"admin {admin['username']} changed role of {target_user.get('username') if target_user else uid} to {role}",
+        links=[
+            audit.target(
+                "user", uid, target_user.get("username") if target_user else None
+            )
+        ],
+    )
+    return action_result(request, "/admin/users")
 
 
 @router.post("/users/{uid}/password")
-async def admin_user_password(request: Request, uid: str):
+async def admin_user_password(
+    request: Request, uid: str, data: Annotated[AdminPasswordForm, Form()]
+):
     admin = require_admin(request)
-    form = await request.form()
-    password = form.get("password", "")
-    if len(password) < 6:
-        return RedirectResponse(url="/admin/users", status_code=302)
     users = get_table("users")
-    users.update({"uid": uid, "password_hash": hash_password(password)}, ["uid"])
+    target_user = users.find_one(uid=uid)
+    users.update({"uid": uid, "password_hash": hash_password(data.password)}, ["uid"])
     logger.info(f"Admin {admin['username']} changed password for user {uid}")
-    return RedirectResponse(url="/admin/users", status_code=302)
+    audit.record(
+        request,
+        "admin.user.password.reset",
+        user=admin,
+        target_type="user",
+        target_uid=uid,
+        target_label=target_user.get("username") if target_user else None,
+        summary=f"admin {admin['username']} reset the password of {target_user.get('username') if target_user else uid}",
+        links=[
+            audit.target(
+                "user", uid, target_user.get("username") if target_user else None
+            )
+        ],
+    )
+    return action_result(request, "/admin/users")
 
 
 @router.post("/users/{uid}/toggle")
 async def admin_user_toggle(request: Request, uid: str):
     admin = require_admin(request)
     if uid == admin["uid"]:
-        return RedirectResponse(url="/admin/users", status_code=302)
+        audit.record(
+            request,
+            "admin.user.active.disable",
+            user=admin,
+            result="denied",
+            target_type="user",
+            target_uid=uid,
+            target_label=admin.get("username"),
+            summary=f"admin {admin['username']} cannot disable their own account",
+            links=[audit.target("user", uid, admin.get("username"))],
+        )
+        return action_result(request, "/admin/users")
     users = get_table("users")
     user = users.find_one(uid=uid)
     if user:
         new_state = not user.get("is_active", True)
         users.update({"uid": uid, "is_active": new_state}, ["uid"])
-        logger.info(f"Admin {admin['username']} {'disabled' if not new_state else 'enabled'} user {uid}")
-    return RedirectResponse(url="/admin/users", status_code=302)
+        clear_user_cache(uid)
+        logger.info(
+            f"Admin {admin['username']} {'disabled' if not new_state else 'enabled'} user {uid}"
+        )
+        audit.record(
+            request,
+            "admin.user.active.enable" if new_state else "admin.user.active.disable",
+            user=admin,
+            target_type="user",
+            target_uid=uid,
+            target_label=user.get("username"),
+            new_value=1 if new_state else 0,
+            summary=f"admin {admin['username']} {'enabled' if new_state else 'disabled'} user {user.get('username')}",
+            links=[audit.target("user", uid, user.get("username"))],
+        )
+    return action_result(request, "/admin/users")
+
+
+@router.post("/users/{uid}/reset-ai-quota")
+async def admin_user_reset_ai_quota(request: Request, uid: str):
+    admin = require_admin(request)
+    devii = service_manager.get_service("devii")
+    removed = devii.reset_quota("user", uid) if devii is not None else 0
+    logger.info(
+        f"Admin {admin['username']} reset AI quota for user {uid} ({removed} ledger rows)"
+    )
+    target_user = get_table("users").find_one(uid=uid)
+    audit.record(
+        request,
+        "admin.user.ai_quota.reset",
+        user=admin,
+        target_type="user",
+        target_uid=uid,
+        target_label=target_user.get("username") if target_user else None,
+        metadata={"rows_removed": removed},
+        summary=f"admin {admin['username']} reset the AI quota of {target_user.get('username') if target_user else uid}",
+        links=[
+            audit.target(
+                "user", uid, target_user.get("username") if target_user else None
+            )
+        ],
+    )
+    return action_result(request, "/admin/users")
+
+
+@router.get("/media", response_class=HTMLResponse)
+async def admin_media(request: Request, page: int = 1):
+    admin = require_admin(request)
+    media, pagination = get_deleted_media(page)
+    base = site_url(request)
+    seo_ctx = base_seo_context(
+        request,
+        title="Media - Admin",
+        description="Restore or permanently remove soft-deleted media.",
+        robots="noindex,nofollow",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Admin", "url": "/admin"},
+            {"name": "Media", "url": "/admin/media"},
+        ],
+        schemas=[website_schema(base)],
+    )
+    return respond(
+        request,
+        "admin_media.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": admin,
+            "media": media,
+            "pagination": pagination,
+            "admin_section": "media",
+        },
+        model=AdminMediaOut,
+    )
+
+
+@router.post("/media/{uid}/purge")
+async def admin_media_purge(request: Request, uid: str):
+    admin = require_admin(request)
+    att = get_table("attachments").find_one(uid=uid)
+    delete_attachment(uid)
+    logger.info(f"Admin {admin['username']} purged media {uid}")
+    audit.record(
+        request,
+        "attachment.delete",
+        user=admin,
+        target_type="attachment",
+        target_uid=uid,
+        target_label=att.get("filename") if att else None,
+        metadata={"purge": True},
+        summary=f"admin {admin['username']} purged attachment {att.get('filename') if att else uid}",
+        links=[audit.attachment_link(uid, att.get("filename") if att else None)],
+    )
+    return action_result(request, "/admin/media")
+
+
+TRASH_TABLES = [
+    {"key": "posts", "label": "Posts", "type": "post"},
+    {"key": "comments", "label": "Comments", "type": "comment"},
+    {"key": "gists", "label": "Gists", "type": "gist"},
+    {"key": "projects", "label": "Projects", "type": "project"},
+    {"key": "news", "label": "News", "type": "news"},
+    {"key": "project_files", "label": "Project files", "type": None},
+    {"key": "attachments", "label": "Attachments", "type": None},
+]
+_TRASH_KEYS = {entry["key"] for entry in TRASH_TABLES}
+_TRASH_TYPE = {entry["key"]: entry["type"] for entry in TRASH_TABLES}
+
+
+def _trash_label(table: str, row: dict) -> str:
+    for field in ("title", "name", "path", "original_filename", "question", "content"):
+        value = row.get(field)
+        if value:
+            return str(value)[:80]
+    return row.get("slug") or row.get("uid", "")
+
+
+def _trash_target_url(table: str, row: dict) -> str | None:
+    target_type = _TRASH_TYPE.get(table)
+    if target_type:
+        return resolve_object_url(target_type, row["uid"])
+    return None
+
+
+@router.get("/trash", response_class=HTMLResponse)
+async def admin_trash(request: Request, table: str = "posts", page: int = 1):
+    admin = require_admin(request)
+    if table not in _TRASH_KEYS:
+        table = "posts"
+    rows, pagination = list_deleted(table, page)
+    actor_ids = [r.get("deleted_by") for r in rows if r.get("deleted_by")]
+    actors = get_users_by_uids([a for a in actor_ids if a]) if actor_ids else {}
+    items = []
+    for row in rows:
+        actor = actors.get(row.get("deleted_by"))
+        items.append(
+            {
+                "uid": row["uid"],
+                "table": table,
+                "label": _trash_label(table, row),
+                "deleted_at": row.get("deleted_at"),
+                "deleted_by": actor["username"] if actor else row.get("deleted_by"),
+                "target_url": _trash_target_url(table, row),
+            }
+        )
+    tabs = [
+        {**entry, "count": count_deleted(entry["key"]), "active": entry["key"] == table}
+        for entry in TRASH_TABLES
+    ]
+    base = site_url(request)
+    seo_ctx = base_seo_context(
+        request,
+        title="Trash - Admin",
+        description="Restore or permanently remove soft-deleted content.",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Admin", "url": "/admin"},
+            {"name": "Trash", "url": "/admin/trash"},
+        ],
+        schemas=[website_schema(base)],
+    )
+    return respond(
+        request,
+        "admin_trash.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": admin,
+            "items": items,
+            "table": table,
+            "tables": tabs,
+            "pagination": pagination,
+            "admin_section": "trash",
+        },
+        model=AdminTrashOut,
+    )
+
+
+@router.post("/trash/{table}/{uid}/restore")
+async def admin_trash_restore(request: Request, table: str, uid: str):
+    admin = require_admin(request)
+    if table not in _TRASH_KEYS:
+        raise not_found("Unknown trash table")
+    row = get_table(table).find_one(uid=uid)
+    if row and row.get("deleted_at"):
+        restored = restore_event(row["deleted_at"])
+        logger.info(
+            f"Admin {admin['username']} restored {table} {uid} ({restored} rows)"
+        )
+    return action_result(request, f"/admin/trash?table={table}")
+
+
+@router.post("/trash/{table}/{uid}/purge")
+async def admin_trash_purge(request: Request, table: str, uid: str):
+    admin = require_admin(request)
+    if table not in _TRASH_KEYS:
+        raise not_found("Unknown trash table")
+    row = get_table(table).find_one(uid=uid)
+    if row and row.get("deleted_at"):
+        purged = purge_event(row["deleted_at"])
+        for purged_table, purged_rows in purged:
+            if purged_table == "attachments":
+                for attachment in purged_rows:
+                    _unlink_attachment_files(attachment)
+            elif purged_table == "project_files":
+                for node in purged_rows:
+                    if node.get("is_binary"):
+                        _unlink_blob(node)
+        logger.info(f"Admin {admin['username']} purged {table} {uid}")
+    return action_result(request, f"/admin/trash?table={table}")
+
+
+@router.post("/ai-quota/reset-guests")
+async def admin_reset_guest_ai_quota(request: Request):
+    admin = require_admin(request)
+    devii = service_manager.get_service("devii")
+    removed = devii.reset_guest_quotas() if devii is not None else 0
+    logger.info(
+        f"Admin {admin['username']} reset all guest AI quotas ({removed} ledger rows)"
+    )
+    audit.record(
+        request,
+        "admin.ai_quota.reset_guests",
+        user=admin,
+        metadata={"rows_removed": removed},
+        summary=f"admin {admin['username']} reset all guest AI quotas",
+    )
+    return action_result(request, "/admin/ai-usage")
+
+
+@router.post("/ai-quota/reset-all")
+async def admin_reset_all_ai_quota(request: Request):
+    admin = require_admin(request)
+    devii = service_manager.get_service("devii")
+    removed = devii.reset_all_quotas() if devii is not None else 0
+    logger.info(
+        f"Admin {admin['username']} reset ALL AI quotas ({removed} ledger rows)"
+    )
+    audit.record(
+        request,
+        "admin.ai_quota.reset_all",
+        user=admin,
+        metadata={"rows_removed": removed},
+        summary=f"admin {admin['username']} reset all AI quotas",
+    )
+    return action_result(request, "/admin/ai-usage")
 
 
 @router.get("/settings", response_class=HTMLResponse)
@@ -100,8 +537,9 @@ async def admin_settings(request: Request):
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
-        title="Settings — Admin",
+        title="Settings - Admin",
         description="Manage DevPlace site settings.",
+        robots="noindex,nofollow",
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
             {"name": "Admin", "url": "/admin"},
@@ -109,43 +547,176 @@ async def admin_settings(request: Request):
         ],
         schemas=[website_schema(base)],
     )
-    return templates.TemplateResponse("admin_settings.html", {
-        **seo_ctx,
-        "request": request,
-        "user": admin,
-        "settings": all_settings,
-        "admin_section": "settings",
-    })
+    return respond(
+        request,
+        "admin_settings.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": admin,
+            "settings": all_settings,
+            "admin_section": "settings",
+        },
+        model=AdminSettingsOut,
+    )
+
+
+@router.get("/audit-log", response_class=HTMLResponse)
+async def admin_audit_log(
+    request: Request,
+    page: int = 1,
+    event_key: str = "",
+    category: str = "",
+    actor_role: str = "",
+    actor_uid: str = "",
+    origin: str = "",
+    result: str = "",
+    q: str = "",
+    date_from: str = "",
+    date_to: str = "",
+):
+    admin = require_admin(request)
+    filters = {
+        "event_key": event_key,
+        "category": category,
+        "actor_role": actor_role,
+        "actor_uid": actor_uid,
+        "origin": origin,
+        "result": result,
+        "q": q,
+        "date_from": date_from,
+        "date_to": date_to,
+    }
+    active = {key: value for key, value in filters.items() if value}
+    rows, pagination = audit_query.list_events(filters, page)
+    user_uids = [
+        row["actor_uid"]
+        for row in rows
+        if row.get("actor_kind") == "user" and row.get("actor_uid")
+    ]
+    users_by_uid = get_users_by_uids(user_uids)
+    entries = []
+    for row in rows:
+        entry = dict(row)
+        entry["actor_user"] = users_by_uid.get(row.get("actor_uid"))
+        entry["time_ago"] = time_ago(row["created_at"]) if row.get("created_at") else ""
+        entry["metadata"] = _parse_metadata(row.get("metadata"))
+        entries.append(entry)
+    filter_qs = urlencode(active)
+    pagination_query = (filter_qs + "&") if filter_qs else ""
+    base = site_url(request)
+    seo_ctx = base_seo_context(
+        request,
+        title="Audit Log - Admin",
+        description="Platform audit trail of every state-changing action.",
+        robots="noindex,nofollow",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Admin", "url": "/admin"},
+            {"name": "Audit Log", "url": "/admin/audit-log"},
+        ],
+        schemas=[website_schema(base)],
+    )
+    return respond(
+        request,
+        "admin_audit_log.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": admin,
+            "entries": entries,
+            "pagination": pagination,
+            "pagination_query": pagination_query,
+            "filters": active,
+            "options": audit_query.filter_options(),
+            "admin_section": "audit-log",
+        },
+        model=AuditLogOut,
+    )
+
+
+@router.get("/audit-log/{uid}", response_class=HTMLResponse)
+async def admin_audit_event(request: Request, uid: str):
+    admin = require_admin(request)
+    data = audit_query.get_event_with_links(uid)
+    if data is None:
+        raise not_found("Audit event not found")
+    event = dict(data["event"])
+    if event.get("actor_kind") == "user" and event.get("actor_uid"):
+        event["actor_user"] = get_users_by_uids([event["actor_uid"]]).get(
+            event["actor_uid"]
+        )
+    event["metadata"] = _parse_metadata(event.get("metadata"))
+    event["metadata_pretty"] = (
+        json.dumps(event["metadata"], indent=2) if event["metadata"] else ""
+    )
+    event["time_ago"] = (
+        time_ago(event["created_at"]) if event.get("created_at") else ""
+    )
+    base = site_url(request)
+    seo_ctx = base_seo_context(
+        request,
+        title="Audit Event - Admin",
+        description="A single audit event with related objects.",
+        robots="noindex,nofollow",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Admin", "url": "/admin"},
+            {"name": "Audit Log", "url": "/admin/audit-log"},
+            {"name": "Event", "url": f"/admin/audit-log/{uid}"},
+        ],
+        schemas=[website_schema(base)],
+    )
+    return respond(
+        request,
+        "admin_audit_event.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": admin,
+            "event": event,
+            "links": data["links"],
+            "admin_section": "audit-log",
+        },
+        model=AuditEventOut,
+    )
 
 
 @router.get("/news", response_class=HTMLResponse)
 async def admin_news(request: Request, page: int = 1):
     admin = require_admin(request)
     news_table = get_table("news")
-    total = len(list(news_table.all()))
+    total = news_table.count(deleted_at=None)
     pagination = build_pagination(page, total)
     offset = (pagination["page"] - 1) * pagination["per_page"]
-    page_articles = list(news_table.find(order_by=["-synced_at"], _limit=pagination["per_page"], _offset=offset))
-    images_table = get_table("news_images")
+    page_articles = list(
+        news_table.find(
+            deleted_at=None,
+            order_by=["-synced_at"],
+            _limit=pagination["per_page"],
+            _offset=offset,
+        )
+    )
+    images_by_news = get_news_images_by_uids([a["uid"] for a in page_articles])
 
     enriched = []
     for a in page_articles:
-        has_image = False
-        if "news_images" in db.tables:
-            has_image = images_table.find_one(news_uid=a["uid"]) is not None
-        enriched.append({
-            "article": a,
-            "time_ago": time_ago(a["synced_at"]),
-            "synced_at": a.get("synced_at", ""),
-            "grade": a.get("grade", 0),
-            "has_image": has_image,
-        })
+        enriched.append(
+            {
+                "article": a,
+                "time_ago": time_ago(a["synced_at"]),
+                "synced_at": a.get("synced_at", ""),
+                "grade": a.get("grade", 0),
+                "has_image": a["uid"] in images_by_news,
+            }
+        )
 
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
-        title="News — Admin",
+        title="News - Admin",
         description="Manage DevPlace news articles.",
+        robots="noindex,nofollow",
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
             {"name": "Admin", "url": "/admin"},
@@ -153,14 +724,19 @@ async def admin_news(request: Request, page: int = 1):
         ],
         schemas=[website_schema(base)],
     )
-    return templates.TemplateResponse("admin_news.html", {
-        **seo_ctx,
-        "request": request,
-        "user": admin,
-        "articles": enriched,
-        "pagination": pagination,
-        "admin_section": "news",
-    })
+    return respond(
+        request,
+        "admin_news.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": admin,
+            "articles": enriched,
+            "pagination": pagination,
+            "admin_section": "news",
+        },
+        model=AdminNewsOut,
+    )
 
 
 @router.post("/news/{uid}/toggle")
@@ -170,9 +746,24 @@ async def admin_news_toggle(request: Request, uid: str):
     article = news_table.find_one(uid=uid)
     if article:
         current = article.get("featured", 0)
-        news_table.update({"uid": uid, "featured": 0 if current else 1}, ["uid"])
-        logger.info(f"Admin {admin['username']} toggled news {uid} featured={'off' if current else 'on'}")
-    return RedirectResponse(url="/admin/news", status_code=302)
+        new_value = 0 if current else 1
+        news_table.update({"uid": uid, "featured": new_value}, ["uid"])
+        logger.info(
+            f"Admin {admin['username']} toggled news {uid} featured={'off' if current else 'on'}"
+        )
+        audit.record(
+            request,
+            "news.featured.toggle",
+            user=admin,
+            target_type="news",
+            target_uid=uid,
+            target_label=article.get("title"),
+            old_value=current,
+            new_value=new_value,
+            summary=f"admin {admin['username']} toggled featured for news {article.get('title')}",
+            links=[audit.target("news", uid, article.get("title"))],
+        )
+    return action_result(request, "/admin/news")
 
 
 @router.post("/news/{uid}/publish")
@@ -185,7 +776,19 @@ async def admin_news_publish(request: Request, uid: str):
         new_status = "draft" if current == "published" else "published"
         news_table.update({"uid": uid, "status": new_status}, ["uid"])
         logger.info(f"Admin {admin['username']} set news {uid} status={new_status}")
-    return RedirectResponse(url="/admin/news", status_code=302)
+        audit.record(
+            request,
+            "news.publish.toggle",
+            user=admin,
+            target_type="news",
+            target_uid=uid,
+            target_label=article.get("title"),
+            old_value=current,
+            new_value=new_status,
+            summary=f"admin {admin['username']} set news {article.get('title')} to {new_status}",
+            links=[audit.target("news", uid, article.get("title"))],
+        )
+    return action_result(request, "/admin/news")
 
 
 @router.post("/news/{uid}/landing")
@@ -195,35 +798,83 @@ async def admin_news_landing(request: Request, uid: str):
     article = news_table.find_one(uid=uid)
     if article:
         current = article.get("show_on_landing", 0)
-        news_table.update({"uid": uid, "show_on_landing": 0 if current else 1}, ["uid"])
-        logger.info(f"Admin {admin['username']} toggled news {uid} landing={'on' if not current else 'off'}")
-    return RedirectResponse(url="/admin/news", status_code=302)
+        new_value = 0 if current else 1
+        news_table.update({"uid": uid, "show_on_landing": new_value}, ["uid"])
+        logger.info(
+            f"Admin {admin['username']} toggled news {uid} landing={'on' if not current else 'off'}"
+        )
+        audit.record(
+            request,
+            "news.landing.toggle",
+            user=admin,
+            target_type="news",
+            target_uid=uid,
+            target_label=article.get("title"),
+            old_value=current,
+            new_value=new_value,
+            summary=f"admin {admin['username']} toggled landing visibility for news {article.get('title')}",
+            links=[audit.target("news", uid, article.get("title"))],
+        )
+    return action_result(request, "/admin/news")
 
 
 @router.post("/news/{uid}/delete")
 async def admin_news_delete(request: Request, uid: str):
     admin = require_admin(request)
     news_table = get_table("news")
-    article = news_table.find_one(uid=uid)
+    article = news_table.find_one(uid=uid, deleted_at=None)
     if article:
-        images_table = get_table("news_images")
-        for img in images_table.find(news_uid=uid):
-            images_table.delete(id=img["id"])
-        news_table.delete(id=article["id"])
-        logger.info(f"Admin {admin['username']} deleted news {uid}")
-    return RedirectResponse(url="/admin/news", status_code=302)
+        stamp = _now_iso()
+        image_count = soft_delete("news_images", admin["uid"], stamp=stamp, news_uid=uid)
+        soft_delete("news", admin["uid"], stamp=stamp, uid=uid)
+        logger.info(f"Admin {admin['username']} soft-deleted news {uid}")
+        audit.record(
+            request,
+            "news.delete",
+            user=admin,
+            target_type="news",
+            target_uid=uid,
+            target_label=article.get("title"),
+            metadata={"image_count": image_count},
+            summary=f"admin {admin['username']} deleted news article {article.get('title')}",
+            links=[audit.target("news", uid, article.get("title"))],
+        )
+    return action_result(request, "/admin/news")
 
 
 @router.post("/settings")
-async def admin_settings_save(request: Request):
+async def admin_settings_save(
+    request: Request, data: Annotated[AdminSettingsForm, Form()]
+):
     admin = require_admin(request)
-    form = await request.form()
     settings = get_table("site_settings")
-    for key, value in form.multi_items():
+    changes = []
+    for key, value in data.model_dump().items():
         existing = settings.find_one(key=key)
         if existing:
+            if value == "":
+                continue
+            old_value = existing.get("value")
+            if str(old_value) == str(value):
+                continue
             settings.update({"id": existing["id"], "key": key, "value": value}, ["id"])
+            changes.append((key, old_value, value))
         else:
             settings.insert({"uid": generate_uid(), "key": key, "value": value})
+            changes.append((key, None, value))
+    clear_settings_cache()
     logger.info(f"Admin {admin['username']} updated settings")
-    return RedirectResponse(url="/admin/settings", status_code=302)
+    for key, old_value, new_value in changes:
+        audit.record(
+            request,
+            "admin.setting.update",
+            user=admin,
+            target_type="setting",
+            target_uid=key,
+            target_label=key,
+            old_value=old_value,
+            new_value=new_value,
+            summary=f"admin {admin['username']} updated setting {key}",
+            links=[audit.setting(key)],
+        )
+    return action_result(request, "/admin/settings")

devplacepy/routers/auth.py

@@ -1,13 +1,35 @@
+# retoor <retoor@molodetz.nl>
+
 import hashlib
 import secrets
 import logging
-from datetime import datetime, timedelta
-from fastapi import APIRouter, Request
-from fastapi.responses import RedirectResponse, HTMLResponse
-from devplacepy.database import get_table
+from typing import Annotated
+from datetime import datetime, timedelta, timezone
+from fastapi import APIRouter, Request, Form
+from fastapi.responses import HTMLResponse, JSONResponse
+from devplacepy.database import get_table, get_setting, get_int_setting, _now_iso
 from devplacepy.templating import templates
-from devplacepy.utils import hash_password, verify_password, create_session, generate_uid, get_current_user
+from devplacepy.utils import (
+    hash_password,
+    verify_password,
+    create_session,
+    generate_uid,
+    get_current_user,
+    award_badge,
+    clear_session_cache,
+    safe_next,
+)
 from devplacepy.seo import base_seo_context
+from devplacepy.models import (
+    SignupForm,
+    LoginForm,
+    ForgotPasswordForm,
+    ResetPasswordForm,
+)
+from devplacepy.config import SECONDS_PER_DAY
+from devplacepy.responses import respond, action_result, wants_json, json_error
+from devplacepy.schemas import AuthPageOut
+from devplacepy.services.audit import record as audit
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -17,50 +39,70 @@ router = APIRouter()
 async def signup_page(request: Request):
     user = get_current_user(request)
     if user:
-        return RedirectResponse(url="/feed", status_code=302)
+        return action_result(request, "/feed")
     seo_ctx = base_seo_context(
         request,
         title="Join DevPlace",
         description="Create your DevPlace account and start connecting with developers.",
         robots="noindex,nofollow",
     )
-    return templates.TemplateResponse("signup.html", {**seo_ctx, "request": request})
+    registration_closed = get_setting("registration_open", "1") != "1"
+    return respond(
+        request,
+        "signup.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "page": "signup",
+            "registration_closed": registration_closed,
+        },
+        model=AuthPageOut,
+    )
 
 
 @router.get("/login", response_class=HTMLResponse)
-async def login_page(request: Request):
+async def login_page(request: Request, next: str | None = None):
     user = get_current_user(request)
     if user:
-        return RedirectResponse(url="/feed", status_code=302)
+        return action_result(request, safe_next(next))
     seo_ctx = base_seo_context(
         request,
         title="Sign In",
         description="Sign in to DevPlace to connect with developers.",
         robots="noindex,nofollow",
     )
-    return templates.TemplateResponse("login.html", {**seo_ctx, "request": request})
+    return respond(
+        request,
+        "login.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "page": "login",
+            "next_url": safe_next(next, ""),
+        },
+        model=AuthPageOut,
+    )
 
 
 @router.post("/signup")
-async def signup(request: Request):
-    form = await request.form()
-    username = form.get("username", "").strip()
-    email = form.get("email", "").strip().lower()
-    password = form.get("password", "")
-    confirm_password = form.get("confirm_password", "")
+async def signup(request: Request, data: Annotated[SignupForm, Form()]):
+    username = data.username
+    email = data.email.strip().lower()
+    password = data.password
+
+    if get_setting("registration_open", "1") != "1":
+        if wants_json(request):
+            return json_error(403, "Registration is closed")
+        seo_ctx = base_seo_context(
+            request, title="Join DevPlace", robots="noindex,nofollow"
+        )
+        return templates.TemplateResponse(
+            request,
+            "signup.html",
+            {**seo_ctx, "request": request, "registration_closed": True},
+        )
 
     errors = []
-    if len(username) < 3 or len(username) > 32:
-        errors.append("Username must be between 3 and 32 characters")
-    if not username.isascii() or not all(c.isalnum() or c in ("-", "_") for c in username):
-        errors.append("Username can only contain letters, numbers, hyphens, and underscores")
-    if not email or "@" not in email or len(email) > 255:
-        errors.append("Valid email is required")
-    if len(password) < 6:
-        errors.append("Password must be at least 6 characters")
-    if password != confirm_password:
-        errors.append("Passwords do not match")
-
     users = get_table("users")
     if users.find_one(username=username):
         errors.append("Username already taken")
@@ -68,75 +110,133 @@ async def signup(request: Request):
         errors.append("Email already registered")
 
     if errors:
-        seo_ctx = base_seo_context(request, title="Join DevPlace", robots="noindex,nofollow")
+        if wants_json(request):
+            return json_error(400, "; ".join(errors), errors=errors)
+        seo_ctx = base_seo_context(
+            request, title="Join DevPlace", robots="noindex,nofollow"
+        )
         return templates.TemplateResponse(
+            request,
             "signup.html",
-            {**seo_ctx, "request": request, "errors": errors, "username": username, "email": email},
+            {
+                **seo_ctx,
+                "request": request,
+                "errors": errors,
+                "username": username,
+                "email": email,
+            },
         )
 
     uid = generate_uid()
-    is_first = len(list(users.all())) == 0
-    users.insert({
-        "uid": uid,
-        "username": username,
-        "email": email,
-        "password_hash": hash_password(password),
-        "bio": "",
-        "location": "",
-        "git_link": "",
-        "website": "",
-        "role": "Admin" if is_first else "Member",
-        "is_active": True,
-        "level": 1,
-        "xp": 0,
-        "stars": 0,
-        "created_at": datetime.utcnow().isoformat(),
-    })
+    is_first = users.count() == 0
+    users.insert(
+        {
+            "uid": uid,
+            "username": username,
+            "email": email,
+            "api_key": generate_uid(),
+            "password_hash": hash_password(password),
+            "bio": "",
+            "location": "",
+            "git_link": "",
+            "website": "",
+            "role": "Admin" if is_first else "Member",
+            "is_active": True,
+            "level": 1,
+            "xp": 0,
+            "stars": 0,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+    )
 
-    badges = get_table("badges")
-    badges.insert({
-        "uid": generate_uid(),
-        "user_uid": uid,
-        "badge_name": "Member",
-        "created_at": datetime.utcnow().isoformat(),
-    })
+    award_badge(uid, "Member")
 
-    token = create_session(uid)
-    response = RedirectResponse(url="/feed", status_code=302)
-    response.set_cookie(key="session", value=token, max_age=86400 * 7, httponly=True, samesite="lax")
+    max_age = max(1, get_int_setting("session_max_age_days", 7)) * SECONDS_PER_DAY
+    token = create_session(uid, max_age)
+    response = action_result(request, "/feed", data={"username": username})
+    response.set_cookie(
+        key="session", value=token, max_age=max_age, httponly=True, samesite="lax"
+    )
     logger.info(f"User {username} signed up")
+    role = "Admin" if is_first else "Member"
+    audit.record(
+        request,
+        "auth.signup",
+        user={"uid": uid, "username": username, "role": role},
+        target_type="user",
+        target_uid=uid,
+        target_label=username,
+        new_value=role,
+        metadata={"first_user": is_first},
+        summary=f"user {username} registered account",
+        links=[audit.target("user", uid, username)],
+    )
     return response
 
 
 @router.post("/login")
-async def login(request: Request):
-    form = await request.form()
-    email = form.get("email", "").strip().lower()
-    password = form.get("password", "")
-    remember_me = form.get("remember_me") == "on"
+async def login(request: Request, data: Annotated[LoginForm, Form()]):
+    email = data.email.strip().lower()
+    password = data.password
+    remember_me = data.remember_me == "on"
 
     errors = []
-    if not email or not password:
-        errors.append("Email and password are required")
-
     users = get_table("users")
-    user = users.find_one(email=email) if not errors else None
+    user = users.find_one(email=email)
 
     if not user or not verify_password(password, user["password_hash"]):
         errors.append("Invalid email or password")
 
     if errors:
+        audit.record(
+            request,
+            "auth.login.failure",
+            user=None,
+            actor_kind="guest",
+            result="failure",
+            metadata={"email": email},
+            summary=f"failed login attempt for {email}",
+        )
+        if wants_json(request):
+            return json_error(401, "; ".join(errors), errors=errors)
         seo_ctx = base_seo_context(request, title="Sign In", robots="noindex,nofollow")
         return templates.TemplateResponse(
+            request,
             "login.html",
-            {**seo_ctx, "request": request, "errors": errors, "email": email},
+            {
+                **seo_ctx,
+                "request": request,
+                "errors": errors,
+                "email": email,
+                "next_url": safe_next(data.next, ""),
+            },
         )
 
-    token = create_session(user["uid"])
-    max_age = 86400 * 30 if remember_me else 86400 * 7
-    response = RedirectResponse(url="/feed", status_code=302)
-    response.set_cookie(key="session", value=token, max_age=max_age, httponly=True, samesite="lax")
+    days = (
+        get_int_setting("session_remember_days", 30)
+        if remember_me
+        else get_int_setting("session_max_age_days", 7)
+    )
+    max_age = max(1, days) * SECONDS_PER_DAY
+    token = create_session(user["uid"], max_age)
+    response = action_result(
+        request, safe_next(data.next), data={"username": user["username"]}
+    )
+    response.set_cookie(
+        key="session", value=token, max_age=max_age, httponly=True, samesite="lax"
+    )
     logger.info(f"User {user['username']} logged in")
+    audit.record(
+        request,
+        "auth.login.success",
+        user=user,
+        target_type="user",
+        target_uid=user["uid"],
+        target_label=user["username"],
+        metadata={"remember_me": remember_me, "max_age_seconds": max_age},
+        summary=f"user {user['username']} logged in",
+        links=[audit.target("user", user["uid"], user["username"])],
+    )
     return response
 
 
@@ -147,22 +247,22 @@ async def forgot_password_page(request: Request):
         title="Reset Password",
         robots="noindex,nofollow",
     )
-    return templates.TemplateResponse("forgot_password.html", {**seo_ctx, "request": request})
+    return respond(
+        request,
+        "forgot_password.html",
+        {**seo_ctx, "request": request, "page": "forgot-password"},
+        model=AuthPageOut,
+    )
 
 
 @router.post("/forgot-password")
-async def forgot_password(request: Request):
-    form = await request.form()
-    email = form.get("email", "").strip().lower()
-    errors = []
-    if not email or "@" not in email:
-        errors.append("Valid email is required")
-
-    seo_ctx = base_seo_context(request, title="Reset Password", robots="noindex,nofollow")
-    if errors:
-        return templates.TemplateResponse("forgot_password.html", {
-            **seo_ctx, "request": request, "errors": errors,
-        })
+async def forgot_password(
+    request: Request, data: Annotated[ForgotPasswordForm, Form()]
+):
+    email = data.email.strip().lower()
+    seo_ctx = base_seo_context(
+        request, title="Reset Password", robots="noindex,nofollow"
+    )
 
     users = get_table("users")
     user = users.find_one(email=email)
@@ -170,19 +270,52 @@ async def forgot_password(request: Request):
         token = secrets.token_hex(32)
         token_hash = hashlib.sha256(token.encode()).hexdigest()
         resets = get_table("password_resets")
-        resets.insert({
-            "uid": generate_uid(),
-            "user_uid": user["uid"],
-            "token": token_hash,
-            "expires_at": (datetime.utcnow() + timedelta(hours=1)).isoformat(),
-            "used": False,
-            "created_at": datetime.utcnow().isoformat(),
-        })
+        resets.insert(
+            {
+                "uid": generate_uid(),
+                "user_uid": user["uid"],
+                "token": token_hash,
+                "expires_at": (
+                    datetime.now(timezone.utc) + timedelta(hours=1)
+                ).isoformat(),
+                "used": False,
+                "created_at": datetime.now(timezone.utc).isoformat(),
+            }
+        )
         logger.info(f"Password reset requested for {email}")
+        audit.record(
+            request,
+            "auth.password.forgot_request",
+            user=None,
+            actor_kind="guest",
+            target_type="user",
+            target_uid=user["uid"],
+            target_label=user["username"],
+            metadata={"email": email},
+            summary=f"password reset requested for {email}",
+            links=[audit.target("user", user["uid"], user["username"])],
+        )
+    else:
+        audit.record(
+            request,
+            "auth.password.forgot_request",
+            user=None,
+            actor_kind="guest",
+            result="failure",
+            metadata={"email": email},
+            summary=f"password reset requested for unknown email {email}",
+        )
 
-    return templates.TemplateResponse("forgot_password.html", {
-        **seo_ctx, "request": request, "sent": True,
-    })
+    return respond(
+        request,
+        "forgot_password.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "sent": True,
+        },
+        model=AuthPageOut,
+    )
 
 
 @router.get("/reset-password/{token}", response_class=HTMLResponse)
@@ -192,54 +325,110 @@ async def reset_password_page(request: Request, token: str):
         title="Set New Password",
         robots="noindex,nofollow",
     )
-    return templates.TemplateResponse("reset_password.html", {
-        **seo_ctx, "request": request, "token": token,
-    })
+    return respond(
+        request,
+        "reset_password.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "page": "reset-password",
+            "token": token,
+        },
+        model=AuthPageOut,
+    )
 
 
 @router.post("/reset-password/{token}")
-async def reset_password(request: Request, token: str):
-    form = await request.form()
-    password = form.get("password", "")
-    confirm = form.get("confirm_password", "")
-
+async def reset_password(
+    request: Request, token: str, data: Annotated[ResetPasswordForm, Form()]
+):
+    password = data.password
     errors = []
-    if len(password) < 6:
-        errors.append("Password must be at least 6 characters")
-    if password != confirm:
-        errors.append("Passwords do not match")
-
-    if errors:
-        seo_ctx = base_seo_context(request, title="Set New Password", robots="noindex,nofollow")
-        return templates.TemplateResponse("reset_password.html", {
-            **seo_ctx, "request": request, "token": token, "errors": errors,
-        })
 
     resets = get_table("password_resets")
     users = get_table("users")
     token_hash = hashlib.sha256(token.encode()).hexdigest()
     matched = resets.find_one(token=token_hash, used=False)
 
-    if not matched or datetime.fromisoformat(matched["expires_at"]) < datetime.utcnow():
-        errors.append("Invalid or expired reset token")
-        return templates.TemplateResponse("reset_password.html", {
-            "request": request, "token": token, "errors": errors,
-        })
+    expired = False
+    if matched:
+        expires = datetime.fromisoformat(matched["expires_at"])
+        if expires.tzinfo is None:
+            expires = expires.replace(tzinfo=timezone.utc)
+        expired = expires < datetime.now(timezone.utc)
 
-    users.update({"uid": matched["user_uid"], "password_hash": hash_password(password)}, ["uid"])
+    if not matched or expired:
+        errors.append("Invalid or expired reset token")
+        if wants_json(request):
+            return json_error(400, errors[0], errors=errors)
+        seo_ctx = base_seo_context(
+            request, title="Set New Password", robots="noindex,nofollow"
+        )
+        return templates.TemplateResponse(
+            request,
+            "reset_password.html",
+            {
+                **seo_ctx,
+                "request": request,
+                "token": token,
+                "errors": errors,
+            },
+        )
+
+    users.update(
+        {"uid": matched["user_uid"], "password_hash": hash_password(password)}, ["uid"]
+    )
     resets.update({"id": matched["id"], "used": True}, ["id"])
     logger.info(f"Password reset completed for user {matched['user_uid']}")
-    return RedirectResponse(url="/auth/login", status_code=302)
+    reset_user = users.find_one(uid=matched["user_uid"])
+    audit.record(
+        request,
+        "auth.password.reset_complete",
+        user=reset_user,
+        actor_kind="user",
+        target_type="user",
+        target_uid=matched["user_uid"],
+        target_label=reset_user.get("username") if reset_user else None,
+        summary=f"password reset completed for user {reset_user.get('username') if reset_user else matched['user_uid']}",
+        links=[
+            audit.target(
+                "user",
+                matched["user_uid"],
+                reset_user.get("username") if reset_user else None,
+            )
+        ],
+    )
+    return action_result(request, "/auth/login")
 
 
 @router.get("/logout")
 async def logout(request: Request):
+    user = get_current_user(request)
     token = request.cookies.get("session")
     if token:
         sessions = get_table("sessions")
-        session = sessions.find_one(session_token=token)
+        session = sessions.find_one(session_token=token, deleted_at=None)
         if session:
-            sessions.delete(id=session["id"])
-    response = RedirectResponse(url="/", status_code=302)
+            sessions.update(
+                {
+                    "id": session["id"],
+                    "deleted_at": _now_iso(),
+                    "deleted_by": session["user_uid"],
+                },
+                ["id"],
+            )
+        clear_session_cache(token)
+    if user:
+        audit.record(
+            request,
+            "auth.logout",
+            user=user,
+            target_type="user",
+            target_uid=user["uid"],
+            target_label=user.get("username"),
+            summary=f"user {user.get('username')} logged out",
+            links=[audit.target("user", user["uid"], user.get("username"))],
+        )
+    response = action_result(request, "/")
     response.delete_cookie("session")
     return response

devplacepy/routers/avatar.py

@@ -1,20 +1,31 @@
+# retoor <retoor@molodetz.nl>
+
+import hashlib
 import logging
 from fastapi import APIRouter, Request
 from fastapi.responses import Response
 from devplacepy.avatar import generate_avatar_svg
+from devplacepy.cache import TTLCache
+from devplacepy.config import SECONDS_PER_DAY
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
-_cache = {}
+_cache = TTLCache(ttl=SECONDS_PER_DAY, max_size=4096)
+_CACHE_CONTROL = f"public, max-age={SECONDS_PER_DAY}, immutable"
 
 
 @router.get("/{style}/{seed}")
 async def avatar_proxy(request: Request, style: str, seed: str, size: int = 128):
     cache_key = f"{seed}:{size}"
-    if cache_key in _cache:
-        return Response(content=_cache[cache_key], media_type="image/svg+xml")
+    etag = '"' + hashlib.md5(cache_key.encode("utf-8")).hexdigest() + '"'
+    headers = {"ETag": etag, "Cache-Control": _CACHE_CONTROL}
 
-    svg = generate_avatar_svg(seed)
-    _cache[cache_key] = svg
-    return Response(content=svg, media_type="image/svg+xml")
+    if request.headers.get("if-none-match") == etag:
+        return Response(status_code=304, headers=headers)
+
+    svg = _cache.get(cache_key)
+    if svg is None:
+        svg = generate_avatar_svg(seed)
+        _cache.set(cache_key, svg)
+    return Response(content=svg, media_type="image/svg+xml", headers=headers)

devplacepy/routers/bookmarks.py

@@ -0,0 +1,142 @@
+# retoor <retoor@molodetz.nl>
+import logging
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request
+from fastapi.responses import RedirectResponse, JSONResponse, HTMLResponse
+from devplacepy.database import get_table, db, paginate, resolve_object_url, _now_iso
+from devplacepy.utils import generate_uid, require_user, time_ago
+from devplacepy.seo import base_seo_context
+from devplacepy.responses import respond
+from devplacepy.schemas import SavedOut
+from devplacepy.services.audit import record as audit
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+BOOKMARKABLE: set[str] = {"post", "gist", "project", "news"}
+
+TABLE_BY_TYPE: dict[str, str] = {
+    "post": "posts",
+    "gist": "gists",
+    "project": "projects",
+    "news": "news",
+}
+
+LABEL_BY_TYPE: dict[str, str] = {
+    "post": "Post",
+    "gist": "Gist",
+    "project": "Project",
+    "news": "Article",
+}
+
+
+@router.get("/saved", response_class=HTMLResponse)
+async def saved_page(request: Request, before: str = None):
+    user = require_user(request)
+    bookmarks = get_table("bookmarks")
+    rows, next_cursor = paginate(bookmarks, before=before, user_uid=user["uid"])
+
+    uids_by_type: dict[str, list] = {}
+    for row in rows:
+        uids_by_type.setdefault(row["target_type"], []).append(row["target_uid"])
+
+    resolved: dict[tuple, dict] = {}
+    for target_type, uids in uids_by_type.items():
+        table_name = TABLE_BY_TYPE.get(target_type)
+        if not table_name or table_name not in db.tables:
+            continue
+        table = get_table(table_name)
+        clauses = [table.table.columns.uid.in_(uids)]
+        if table.has_column("deleted_at"):
+            clauses.append(table.table.columns.deleted_at.is_(None))
+        for obj in table.find(*clauses):
+            resolved[(target_type, obj["uid"])] = obj
+
+    items = []
+    for row in rows:
+        obj = resolved.get((row["target_type"], row["target_uid"]))
+        if not obj:
+            continue
+        title = obj.get("title") or (obj.get("content", "") or "")[:80] or "Untitled"
+        items.append(
+            {
+                "target_type": row["target_type"],
+                "type_label": LABEL_BY_TYPE.get(
+                    row["target_type"], row["target_type"].title()
+                ),
+                "title": title,
+                "url": resolve_object_url(row["target_type"], row["target_uid"]),
+                "time_ago": time_ago(row["created_at"]),
+            }
+        )
+
+    seo_ctx = base_seo_context(
+        request,
+        title="Saved",
+        description="Your saved content on DevPlace.",
+        robots="noindex,nofollow",
+    )
+    return respond(
+        request,
+        "saved.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "items": items,
+            "next_cursor": next_cursor,
+        },
+        model=SavedOut,
+    )
+
+
+@router.post("/{target_type}/{target_uid}")
+async def toggle_bookmark(request: Request, target_type: str, target_uid: str):
+    user = require_user(request)
+    if target_type not in BOOKMARKABLE:
+        return JSONResponse({"error": "Invalid target"}, status_code=400)
+
+    bookmarks = get_table("bookmarks")
+    existing = bookmarks.find_one(
+        user_uid=user["uid"], target_uid=target_uid, target_type=target_type
+    )
+    if existing and not existing.get("deleted_at"):
+        bookmarks.update(
+            {"id": existing["id"], "deleted_at": _now_iso(), "deleted_by": user["uid"]},
+            ["id"],
+        )
+        saved = False
+    elif existing:
+        bookmarks.update(
+            {"id": existing["id"], "deleted_at": None, "deleted_by": None}, ["id"]
+        )
+        saved = True
+    else:
+        bookmarks.insert(
+            {
+                "uid": generate_uid(),
+                "user_uid": user["uid"],
+                "target_uid": target_uid,
+                "target_type": target_type,
+                "created_at": datetime.now(timezone.utc).isoformat(),
+                "deleted_at": None,
+                "deleted_by": None,
+            }
+        )
+        saved = True
+
+    audit.record(
+        request,
+        "bookmark.add" if saved else "bookmark.remove",
+        user=user,
+        target_type=target_type,
+        target_uid=target_uid,
+        summary=f"{user['username']} {'bookmarked' if saved else 'removed bookmark from'} {target_type} {target_uid}",
+        links=[audit.target(target_type, target_uid)],
+    )
+
+    if request.headers.get("x-requested-with") == "fetch":
+        return JSONResponse({"saved": saved})
+    return RedirectResponse(
+        url=request.headers.get("Referer", "/feed"), status_code=302
+    )

devplacepy/routers/bugs.py

@@ -1,11 +1,24 @@
+# retoor <retoor@molodetz.nl>
+
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
-from devplacepy.database import get_table, load_comments, get_attachments_by_type
-from devplacepy.templating import templates
-from devplacepy.utils import generate_uid, require_user, time_ago, get_current_user, create_mention_notifications
-from devplacepy.seo import base_seo_context
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
+from fastapi.responses import HTMLResponse
+from devplacepy.models import BugForm
+from devplacepy.database import get_table, load_comments_by_target_uids
+from devplacepy.attachments import get_attachments_batch, link_attachments
+from devplacepy.utils import (
+    generate_uid,
+    require_user,
+    time_ago,
+    get_current_user,
+    create_mention_notifications,
+)
+from devplacepy.seo import base_seo_context, site_url, website_schema
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import BugsOut
+from devplacepy.services.audit import record as audit
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -18,21 +31,26 @@ async def bugs_page(request: Request):
     all_bugs = list(bugs_table.find(order_by=["-created_at"]))
 
     from devplacepy.database import get_users_by_uids
+
     uids = [b["user_uid"] for b in all_bugs]
     users_map = get_users_by_uids(uids)
 
-    bug_list = []
     bug_uids = [b["uid"] for b in all_bugs]
-    attachments_map = get_attachments_by_type("bug", bug_uids) if bug_uids else {}
+    attachments_map = get_attachments_batch("bug", bug_uids) if bug_uids else {}
+    comments_map = load_comments_by_target_uids("bug", bug_uids, user) if bug_uids else {}
+    bug_list = []
     for b in all_bugs:
-        bug_list.append({
-            "bug": b,
-            "author": users_map.get(b["user_uid"]),
-            "time_ago": time_ago(b["created_at"]),
-            "comments": load_comments("bug", b["uid"]),
-            "attachments": attachments_map.get(b["uid"], []),
-        })
+        bug_list.append(
+            {
+                "bug": b,
+                "author": users_map.get(b["user_uid"]),
+                "time_ago": time_ago(b["created_at"]),
+                "comments": comments_map.get(b["uid"], []),
+                "attachments": attachments_map.get(b["uid"], []),
+            }
+        )
 
+    base = site_url(request)
     seo_ctx = base_seo_context(
         request,
         title="Bug Reports",
@@ -41,38 +59,53 @@ async def bugs_page(request: Request):
             {"name": "Home", "url": "/feed"},
             {"name": "Bug Reports", "url": "/bugs"},
         ],
+        schemas=[website_schema(base)],
+    )
+    return respond(
+        request,
+        "bugs.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "bugs": bug_list,
+        },
+        model=BugsOut,
     )
-    return templates.TemplateResponse("bugs.html", {
-        **seo_ctx,
-        "request": request,
-        "user": user,
-        "bugs": bug_list,
-    })
 
 
 @router.post("/create")
-async def create_bug(request: Request):
+async def create_bug(request: Request, data: Annotated[BugForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    title = form.get("title", "").strip()
-    description = form.get("description", "").strip()
-
-    if not title or not description:
-        return RedirectResponse(url="/bugs", status_code=302)
+    title = data.title.strip()
+    description = data.description.strip()
 
     bugs_table = get_table("bug_reports")
     bug_uid = generate_uid()
-    bugs_table.insert({
-        "uid": bug_uid,
-        "user_uid": user["uid"],
-        "title": title,
-        "description": description,
-        "status": "open",
-        "created_at": datetime.utcnow().isoformat(),
-    })
+    bugs_table.insert(
+        {
+            "uid": bug_uid,
+            "user_uid": user["uid"],
+            "title": title,
+            "description": description,
+            "status": "open",
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+    )
 
-    attachment_uids = form.getlist("attachment_uids") if hasattr(form, "getlist") else []
+    if data.attachment_uids:
+        link_attachments(data.attachment_uids, "bug", bug_uid)
 
     create_mention_notifications(description, user["uid"], f"/bugs?highlight={bug_uid}")
     logger.info(f"Bug report created by {user['username']}: {title}")
-    return RedirectResponse(url="/bugs", status_code=302)
+    audit.record(
+        request,
+        "bug.create",
+        user=user,
+        target_type="bug",
+        target_uid=bug_uid,
+        target_label=title,
+        summary=f"{user['username']} reported a bug: {title}",
+        links=[audit.target("bug", bug_uid, title)],
+    )
+    return action_result(request, "/bugs", data={"uid": bug_uid})

devplacepy/routers/comments.py

@@ -1,51 +1,45 @@
+# retoor <retoor@molodetz.nl>
+
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
 from fastapi.responses import RedirectResponse
-from devplacepy.database import get_table, delete_attachments
-from devplacepy.utils import generate_uid, require_user, create_mention_notifications
+from devplacepy.database import (
+    get_table,
+    resolve_object_url,
+    soft_delete,
+    soft_delete_engagement,
+    _now_iso,
+)
+from devplacepy.attachments import link_attachments, soft_delete_attachments_for
+from devplacepy.content import is_owner
+from devplacepy.utils import (
+    generate_uid,
+    require_user,
+    create_mention_notifications,
+    create_notification,
+    award_rewards,
+    is_admin,
+    XP_COMMENT,
+)
+from devplacepy.models import CommentForm
+from devplacepy.responses import action_result, wants_json, json_error
+from devplacepy.services.audit import record as audit
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-def resolve_target_redirect(target_type, target_uid):
-    if target_type == "post":
-        post = resolve_by_slug(get_table("posts"), target_uid)
-        return f"/posts/{post['slug'] or post['uid']}" if post else "/feed"
-    if target_type == "project":
-        project = resolve_by_slug(get_table("projects"), target_uid)
-        return f"/projects/{project['slug'] or project['uid']}" if project else "/projects"
-    if target_type == "news":
-        article = resolve_by_slug(get_table("news"), target_uid)
-        if article:
-            return f"/news/{article.get('slug', '') or article['uid']}"
-        return "/news"
-    if target_type == "bug":
-        return f"/bugs?highlight={target_uid}"
-    if target_type == "gist":
-        gist = resolve_by_slug(get_table("gists"), target_uid)
-        return f"/gists/{gist['slug'] or gist['uid']}" if gist else "/gists"
-    return "/bugs"
-
-
 @router.post("/create")
-async def create_comment(request: Request):
+async def create_comment(request: Request, data: Annotated[CommentForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    content = form.get("content", "").strip()
-    target_uid = form.get("target_uid") or form.get("post_uid", "")
-    target_type = form.get("target_type", "post")
-    parent_uid = form.get("parent_uid", "")
+    content = data.content.strip()
+    target_uid = data.target_uid or data.post_uid
+    target_type = data.target_type
+    parent_uid = data.parent_uid
 
-    redirect_url = resolve_target_redirect(target_type, target_uid)
-
-    if not content or not target_uid:
-        return RedirectResponse(url=redirect_url, status_code=302)
-    if len(content) < 3:
-        return RedirectResponse(url=redirect_url, status_code=302)
-    if len(content) > 1000:
-        return RedirectResponse(url=redirect_url, status_code=302)
+    redirect_url = resolve_object_url(target_type, target_uid)
 
     comment_uid = generate_uid()
     insert = {
@@ -55,78 +49,103 @@ async def create_comment(request: Request):
         "user_uid": user["uid"],
         "content": content,
         "parent_uid": parent_uid or None,
-        "created_at": datetime.utcnow().isoformat(),
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "deleted_at": None,
+        "deleted_by": None,
     }
     if target_type == "post":
         insert["post_uid"] = target_uid
     get_table("comments").insert(insert)
 
-    attachment_uids = form.getlist("attachment_uids") if hasattr(form, "getlist") else []
-    for auid in attachment_uids:
-        if auid.strip():
-            get_table("attachments").update({"uid": auid.strip(), "resource_uid": insert["uid"], "resource_type": "comment"}, ["uid"])
+    if data.attachment_uids:
+        link_attachments(data.attachment_uids, "comment", comment_uid)
 
-    badges = get_table("badges")
-    existing = badges.find_one(user_uid=user["uid"], badge_name="First Comment")
-    if not existing:
-        badges.insert({
-            "uid": generate_uid(),
-            "user_uid": user["uid"],
-            "badge_name": "First Comment",
-            "created_at": datetime.utcnow().isoformat(),
-        })
+    award_rewards(user["uid"], XP_COMMENT, "First Comment")
+
+    comment_url = f"{redirect_url}#comment-{comment_uid}"
 
     if target_type == "post":
         if parent_uid:
-            comments_table = get_table("comments")
-            parent = comments_table.find_one(uid=parent_uid)
+            parent = get_table("comments").find_one(uid=parent_uid, deleted_at=None)
             if parent and parent["user_uid"] != user["uid"]:
-                notifications = get_table("notifications")
-                notifications.insert({
-                    "uid": generate_uid(),
-                    "user_uid": parent["user_uid"],
-                    "type": "reply",
-                    "message": f"{user['username']} replied to your comment",
-                    "related_uid": user["uid"],
-                    "read": False,
-                    "created_at": datetime.utcnow().isoformat(),
-                })
+                create_notification(
+                    parent["user_uid"],
+                    "reply",
+                    f"{user['username']} replied to your comment",
+                    user["uid"],
+                    comment_url,
+                )
         else:
             posts = get_table("posts")
             post = posts.find_one(uid=target_uid)
             if not post:
                 post = posts.find_one(slug=target_uid)
             if post and post["user_uid"] != user["uid"]:
-                notifications = get_table("notifications")
-                notifications.insert({
-                    "uid": generate_uid(),
-                    "user_uid": post["user_uid"],
-                    "type": "comment",
-                    "message": f"{user['username']} commented on your post",
-                    "related_uid": user["uid"],
-                    "read": False,
-                    "created_at": datetime.utcnow().isoformat(),
-                })
+                create_notification(
+                    post["user_uid"],
+                    "comment",
+                    f"{user['username']} commented on your post",
+                    user["uid"],
+                    comment_url,
+                )
 
-    create_mention_notifications(content, user["uid"], redirect_url)
+    create_mention_notifications(content, user["uid"], comment_url)
     logger.info(f"Comment by {user['username']} on {target_type} {target_uid}")
-    return RedirectResponse(url=redirect_url, status_code=302)
+    comment_links = [
+        audit.target("comment", comment_uid),
+        audit.parent(target_type, target_uid),
+    ]
+    if parent_uid:
+        comment_links.append(audit.link("parent_comment", "comment", parent_uid))
+    audit.record(
+        request,
+        f"comment.create.{target_type}",
+        user=user,
+        target_type="comment",
+        target_uid=comment_uid,
+        summary=f"{user['username']} commented on {target_type} {target_uid}: {content}",
+        links=comment_links,
+    )
+    return action_result(
+        request, comment_url, data={"uid": comment_uid, "url": comment_url}
+    )
 
 
 @router.post("/delete/{comment_uid}")
 async def delete_comment(request: Request, comment_uid: str):
     user = require_user(request)
     comments = get_table("comments")
-    comment = comments.find_one(uid=comment_uid)
-    if not comment:
-        return RedirectResponse(url="/feed", status_code=302)
-    if comment["user_uid"] != user["uid"]:
+    comment = comments.find_one(uid=comment_uid, deleted_at=None)
+    if not comment or not (is_owner(comment, user) or is_admin(user)):
+        audit.record(
+            request,
+            "comment.delete",
+            user=user,
+            result="denied",
+            target_type="comment",
+            target_uid=comment_uid,
+            summary=f"{user['username']} denied deleting comment {comment_uid}",
+        )
+        if wants_json(request):
+            return json_error(403, "Not allowed")
         return RedirectResponse(url="/feed", status_code=302)
     target_type = comment.get("target_type", "post")
     target_uid = comment.get("target_uid") or comment.get("post_uid", "")
-    delete_attachments("comment", comment_uid)
-    get_table("votes").delete(target_uid=comment_uid, target_type="comment")
-    comments.delete(id=comment["id"])
-    logger.info(f"Comment {comment_uid} deleted by {user['username']}")
-    redirect_url = resolve_target_redirect(target_type, target_uid)
-    return RedirectResponse(url=redirect_url, status_code=302)
+    actor = user["uid"]
+    stamp = _now_iso()
+    soft_delete_attachments_for("comment", [comment_uid], actor)
+    soft_delete("votes", actor, stamp=stamp, target_uid=comment_uid, target_type="comment")
+    soft_delete_engagement("comment", [comment_uid], actor)
+    soft_delete("comments", actor, stamp=stamp, uid=comment_uid)
+    logger.info(f"Comment {comment_uid} soft-deleted by {user['username']}")
+    audit.record(
+        request,
+        "comment.delete",
+        user=user,
+        target_type="comment",
+        target_uid=comment_uid,
+        summary=f"{user['username']} deleted a comment under {target_type} {target_uid}",
+        links=[audit.target("comment", comment_uid), audit.parent(target_type, target_uid)],
+    )
+    redirect_url = resolve_object_url(target_type, target_uid)
+    return action_result(request, redirect_url)

devplacepy/routers/containers.py

@@ -0,0 +1,478 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+import fcntl
+import json
+import logging
+import os
+import pty
+import re
+import signal
+import struct
+import termios
+from typing import Any, Annotated
+
+from fastapi import APIRouter, Form, Request, WebSocket, WebSocketDisconnect
+from fastapi.responses import HTMLResponse, JSONResponse
+
+from devplacepy.database import get_table, resolve_by_slug
+from devplacepy.models import (
+    ContainerExecForm,
+    ContainerInstanceForm,
+    ContainerScheduleForm,
+)
+from devplacepy.responses import action_result, json_error, respond
+from devplacepy.schemas import ContainersOut
+from devplacepy.seo import base_seo_context
+from devplacepy.utils import _user_from_session, is_admin, not_found, require_admin
+from devplacepy.services.manager import service_manager
+from devplacepy.services.containers import api, store
+from devplacepy.services.containers.api import ContainerError
+from devplacepy.services.containers.backend.base import WORKSPACE_MOUNT
+from devplacepy.services.containers.runtime import get_backend
+from devplacepy.services.devii.tasks.schedule import Schedule, from_iso
+from devplacepy.services.audit import record as audit
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+def _audit_instance(request: Request, user: dict, event_key: str, inst: dict, project: dict | None = None, summary: str | None = None, metadata: Any = None) -> None:
+    links = [audit.instance(inst["uid"], inst.get("name"))]
+    if project:
+        links.append(audit.project(project["uid"], project.get("title")))
+    audit.record(
+        request,
+        event_key,
+        user=user,
+        target_type="instance",
+        target_uid=inst["uid"],
+        target_label=inst.get("name"),
+        metadata=metadata,
+        summary=summary or f"{user['username']} {event_key} instance {inst.get('name')}",
+        links=links,
+    )
+
+
+def _project(project_slug: str) -> dict:
+    project = resolve_by_slug(get_table("projects"), project_slug)
+    if not project:
+        raise not_found("Project not found")
+    return project
+
+
+def _slug(project: dict) -> str:
+    return project["slug"] or project["uid"]
+
+
+def _fail(exc: ContainerError) -> JSONResponse:
+    return json_error(400, str(exc))
+
+
+def _instance(project: dict, uid: str) -> dict:
+    inst = store.get_instance(uid)
+    if not inst or inst["project_uid"] != project["uid"]:
+        raise not_found("Instance not found")
+    return inst
+
+
+@router.get("/{project_slug}/containers", response_class=HTMLResponse)
+async def containers_page(request: Request, project_slug: str):
+    user = require_admin(request)
+    project = _project(project_slug)
+    slug = _slug(project)
+    return respond(
+        request,
+        "containers.html",
+        {
+            **base_seo_context(
+                request,
+                title=f"Containers - {project.get('title', 'Project')}",
+                description="Container manager",
+                robots="noindex,nofollow",
+                breadcrumbs=[
+                    {"name": "Home", "url": "/feed"},
+                    {
+                        "name": project.get("title", "Project"),
+                        "url": f"/projects/{slug}",
+                    },
+                    {"name": "Containers", "url": f"/projects/{slug}/containers"},
+                ],
+            ),
+            "request": request,
+            "user": user,
+            "project": project,
+            "instances": store.list_instances(project["uid"]),
+        },
+        model=ContainersOut,
+    )
+
+
+@router.get("/{project_slug}/containers/data")
+async def containers_json(request: Request, project_slug: str):
+    require_admin(request)
+    project = _project(project_slug)
+    return JSONResponse(
+        {
+            "instances": store.list_instances(project["uid"]),
+        }
+    )
+
+
+# ---------------- instances ----------------
+
+
+@router.post("/{project_slug}/containers/instances")
+async def create_instance(
+    request: Request, project_slug: str, data: Annotated[ContainerInstanceForm, Form()]
+):
+    user = require_admin(request)
+    project = _project(project_slug)
+    try:
+        inst = await api.create_instance(
+            project,
+            name=data.name,
+            boot_command=data.boot_command,
+            env=data.env,
+            cpu_limit=data.cpu_limit,
+            mem_limit=data.mem_limit,
+            ports=data.ports,
+            volumes=data.volumes,
+            restart_policy=data.restart_policy,
+            autostart=data.autostart,
+            ingress_slug=data.ingress_slug,
+            ingress_port=data.ingress_port,
+            actor=("user", user["uid"]),
+        )
+    except ContainerError as exc:
+        return _fail(exc)
+    _audit_instance(
+        request, user, "container.instance.create", inst, project,
+        summary=f"admin {user['username']} created container instance {inst.get('name')} in project {project.get('title')}",
+        metadata={"ports": data.ports, "ingress_slug": data.ingress_slug, "restart_policy": data.restart_policy},
+    )
+    return action_result(
+        request, f"/projects/{_slug(project)}/containers", data={"instance": inst}
+    )
+
+
+@router.get("/{project_slug}/containers/instances/{uid}")
+async def instance_detail(request: Request, project_slug: str, uid: str):
+    require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    return JSONResponse(
+        {
+            "instance": inst,
+            "events": store.list_events(uid),
+            "schedules": store.list_schedules(uid),
+            "stats": api.instance_stats(uid),
+            "runtime": api.instance_runtime(inst),
+        }
+    )
+
+
+_ACTIONS = {
+    "start": store.DESIRED_RUNNING,
+    "stop": store.DESIRED_STOPPED,
+    "pause": store.DESIRED_PAUSED,
+    "resume": store.DESIRED_RUNNING,
+}
+
+
+@router.post("/{project_slug}/containers/instances/{uid}/delete")
+async def delete_instance(request: Request, project_slug: str, uid: str):
+    user = require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    api.mark_for_removal(inst, actor=("user", user["uid"]))
+    _audit_instance(
+        request, user, "container.instance.delete", inst, project,
+        summary=f"admin {user['username']} removed container instance {inst.get('name')}",
+    )
+    return action_result(
+        request, f"/projects/{_slug(project)}/containers", data={"uid": uid}
+    )
+
+
+@router.post("/{project_slug}/containers/instances/{uid}/exec")
+async def instance_exec(
+    request: Request,
+    project_slug: str,
+    uid: str,
+    data: Annotated[ContainerExecForm, Form()],
+):
+    user = require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    if not inst.get("container_id"):
+        return json_error(400, "instance is not running")
+    result = await get_backend().exec(
+        inst["container_id"], ["/bin/sh", "-c", data.command]
+    )
+    store.record_event(
+        inst,
+        "exec",
+        "user",
+        user["uid"],
+        {"command": data.command, "exit_code": result.exit_code},
+    )
+    _audit_instance(
+        request, user, "container.instance.exec", inst, project,
+        summary=f"admin {user['username']} ran a command in instance {inst.get('name')}",
+        metadata={"command": data.command, "exit_code": result.exit_code},
+    )
+    return JSONResponse({"exit_code": result.exit_code, "output": result.output})
+
+
+@router.get("/{project_slug}/containers/instances/{uid}/logs")
+async def instance_logs(request: Request, project_slug: str, uid: str, tail: int = 200):
+    require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    if not inst.get("container_id"):
+        return JSONResponse({"logs": ""})
+    lines: list = []
+
+    async def collect(line: str) -> None:
+        lines.append(line)
+
+    await get_backend().logs(
+        inst["container_id"], follow=False, tail=max(1, min(tail, 2000)), on_log=collect
+    )
+    return JSONResponse({"logs": "\n".join(lines)})
+
+
+@router.get("/{project_slug}/containers/instances/{uid}/metrics")
+async def instance_metrics(request: Request, project_slug: str, uid: str):
+    require_admin(request)
+    return JSONResponse(
+        {"metrics": store.recent_metrics(uid), "stats": api.instance_stats(uid)}
+    )
+
+
+@router.post("/{project_slug}/containers/instances/{uid}/sync")
+async def instance_sync(request: Request, project_slug: str, uid: str):
+    user = require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    try:
+        count = await api.sync_workspace(inst, user)
+    except ContainerError as exc:
+        return _fail(exc)
+    _audit_instance(
+        request, user, "container.instance.sync", inst, project,
+        summary=f"admin {user['username']} synced the workspace of instance {inst.get('name')}",
+        metadata={"files_imported": count},
+    )
+    return action_result(
+        request, f"/projects/{_slug(project)}/containers", data={"imported": count}
+    )
+
+
+@router.post("/{project_slug}/containers/instances/{uid}/schedules")
+async def create_schedule(
+    request: Request,
+    project_slug: str,
+    uid: str,
+    data: Annotated[ContainerScheduleForm, Form()],
+):
+    user = require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    try:
+        schedule = Schedule(
+            kind=data.kind,
+            cron=data.cron or None,
+            run_at=from_iso(data.run_at) if data.run_at else None,
+            delay_seconds=data.delay_seconds,
+            every_seconds=data.every_seconds,
+            max_runs=data.max_runs,
+        )
+        sched = api.add_schedule(inst, data.action, schedule)
+    except (ContainerError, ValueError) as exc:
+        return json_error(400, str(exc))
+    audit.record(
+        request,
+        "container.schedule.create",
+        user=user,
+        target_type="instance",
+        target_uid=inst["uid"],
+        target_label=inst.get("name"),
+        metadata={"action": data.action, "kind": data.kind, "cron": data.cron},
+        summary=f"admin {user['username']} scheduled {data.action} on instance {inst.get('name')}",
+        links=[audit.instance(inst["uid"], inst.get("name")), audit.schedule(sched.get("uid") if isinstance(sched, dict) else None)],
+    )
+    return action_result(
+        request, f"/projects/{_slug(project)}/containers", data={"schedule": sched}
+    )
+
+
+@router.post("/{project_slug}/containers/instances/{uid}/schedules/{sid}/delete")
+async def delete_schedule(request: Request, project_slug: str, uid: str, sid: str):
+    user = require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    store.delete_schedule(sid)
+    audit.record(
+        request,
+        "container.schedule.delete",
+        user=user,
+        target_type="instance",
+        target_uid=inst["uid"],
+        target_label=inst.get("name"),
+        metadata={"schedule_uid": sid},
+        summary=f"admin {user['username']} deleted a schedule on instance {inst.get('name')}",
+        links=[audit.instance(inst["uid"], inst.get("name")), audit.schedule(sid)],
+    )
+    return action_result(
+        request, f"/projects/{_slug(project)}/containers", data={"uid": sid}
+    )
+
+
+@router.post("/{project_slug}/containers/instances/{uid}/start")
+@router.post("/{project_slug}/containers/instances/{uid}/stop")
+@router.post("/{project_slug}/containers/instances/{uid}/pause")
+@router.post("/{project_slug}/containers/instances/{uid}/resume")
+@router.post("/{project_slug}/containers/instances/{uid}/restart")
+async def instance_action(request: Request, project_slug: str, uid: str):
+    user = require_admin(request)
+    project = _project(project_slug)
+    inst = _instance(project, uid)
+    actor = ("user", user["uid"])
+    action = request.url.path.rsplit("/", 1)[-1]
+    if action == "restart":
+        api.request_restart(inst, actor=actor)
+    else:
+        api.set_desired_state(inst, _ACTIONS[action], actor=actor)
+    _audit_instance(
+        request, user, f"container.instance.{action}", inst, project,
+        summary=f"admin {user['username']} {action} container instance {inst.get('name')}",
+    )
+    return action_result(
+        request,
+        f"/projects/{_slug(project)}/containers",
+        data={"uid": uid, "action": action},
+    )
+
+
+@router.websocket("/{project_slug}/containers/instances/{uid}/exec/ws")
+async def instance_exec_ws(websocket: WebSocket, project_slug: str, uid: str):
+    await websocket.accept()
+    user = _user_from_session(websocket)
+    if not user or not is_admin(user):
+        await websocket.close(code=1008)
+        return
+    if not service_manager.owns_lock():
+        await websocket.close(code=1013)
+        return
+    project = resolve_by_slug(get_table("projects"), project_slug)
+    inst = store.get_instance(uid)
+    if (
+        not project
+        or not inst
+        or inst["project_uid"] != project["uid"]
+        or not inst.get("container_id")
+    ):
+        await websocket.close(code=1011)
+        return
+
+    master, slave = pty.openpty()
+    session = (websocket.query_params.get("session") or "").strip()
+    if session and re.match(r"^[A-Za-z0-9_-]{1,64}$", session):
+        shell_cmd = (
+            f"if command -v tmux >/dev/null 2>&1; then exec tmux new-session -A -s {session}; "
+            "elif command -v bash >/dev/null 2>&1; then exec bash; else exec sh; fi"
+        )
+    else:
+        shell_cmd = (
+            "if command -v bash >/dev/null 2>&1; then exec bash; else exec sh; fi"
+        )
+    proc = await asyncio.create_subprocess_exec(
+        "docker",
+        "exec",
+        "-it",
+        "-w",
+        WORKSPACE_MOUNT,
+        inst["container_id"],
+        "/bin/sh",
+        "-c",
+        shell_cmd,
+        stdin=slave,
+        stdout=slave,
+        stderr=slave,
+        start_new_session=True,
+    )
+    os.close(slave)
+    loop = asyncio.get_event_loop()
+    audit.record(
+        websocket,
+        "container.instance.shell.open",
+        user=user,
+        target_type="instance",
+        target_uid=inst["uid"],
+        target_label=inst.get("name"),
+        metadata={"session": session or None},
+        summary=f"admin {user['username']} opened an interactive shell on instance {inst.get('name')}",
+        links=[audit.instance(inst["uid"], inst.get("name"))],
+    )
+
+    async def pump_out():
+        try:
+            while True:
+                data = await loop.run_in_executor(None, os.read, master, 4096)
+                if not data:
+                    break
+                await websocket.send_text(data.decode("utf-8", "replace"))
+        except Exception:
+            pass
+
+    def set_winsize(rows: int, cols: int) -> None:
+        try:
+            fcntl.ioctl(
+                master, termios.TIOCSWINSZ, struct.pack("HHHH", rows, cols, 0, 0)
+            )
+        except OSError:
+            pass
+        try:
+            proc.send_signal(signal.SIGWINCH)
+        except (ProcessLookupError, OSError, ValueError):
+            pass
+
+    reader = asyncio.create_task(pump_out())
+    try:
+        while True:
+            text = await websocket.receive_text()
+            if text[:1] == "{":
+                try:
+                    control = json.loads(text)
+                except ValueError:
+                    control = None
+                if isinstance(control, dict) and control.get("type") == "resize":
+                    rows = int(control.get("rows") or 0)
+                    cols = int(control.get("cols") or 0)
+                    if rows > 0 and cols > 0:
+                        await loop.run_in_executor(None, set_winsize, rows, cols)
+                    continue
+            await loop.run_in_executor(None, os.write, master, text.encode("utf-8"))
+    except WebSocketDisconnect:
+        pass
+    except Exception:
+        pass
+    finally:
+        reader.cancel()
+        try:
+            proc.kill()
+        except ProcessLookupError:
+            pass
+        os.close(master)
+        audit.record(
+            websocket,
+            "container.instance.shell.close",
+            user=user,
+            target_type="instance",
+            target_uid=inst["uid"],
+            target_label=inst.get("name"),
+            summary=f"admin {user['username']} closed the shell on instance {inst.get('name')}",
+            links=[audit.instance(inst["uid"], inst.get("name"))],
+        )

devplacepy/routers/containers_admin.py

@@ -0,0 +1,117 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+
+from fastapi import APIRouter, Request
+from fastapi.responses import HTMLResponse, JSONResponse
+
+from devplacepy.database import db, get_table
+from devplacepy.seo import base_seo_context, site_url, website_schema
+from devplacepy.services.containers import api, store
+from devplacepy.responses import respond
+from devplacepy.schemas import AdminContainersOut, AdminContainerInstanceOut
+from devplacepy.utils import not_found, require_admin
+
+router = APIRouter()
+
+
+def _project_index(project_uids: set) -> dict:
+    if not project_uids or "projects" not in db.tables:
+        return {}
+    table = get_table("projects")
+    rows = table.find(table.table.columns.uid.in_(list(project_uids)))
+    return {row["uid"]: row for row in rows}
+
+
+def _decorate(instances: list) -> list:
+    index = _project_index({inst["project_uid"] for inst in instances})
+    decorated = []
+    for inst in instances:
+        project = index.get(inst["project_uid"], {})
+        row = dict(inst)
+        row["project_title"] = project.get("title", "")
+        row["project_slug"] = project.get("slug") or project.get("uid") or ""
+        decorated.append(row)
+    decorated.sort(key=lambda r: r.get("created_at", ""), reverse=True)
+    return decorated
+
+
+@router.get("", response_class=HTMLResponse)
+async def containers_index(request: Request):
+    admin = require_admin(request)
+    instances = _decorate(store.all_instances())
+    base = site_url(request)
+    seo_ctx = base_seo_context(
+        request,
+        title="Containers - Admin",
+        description="Manage container instances across all projects.",
+        robots="noindex,nofollow",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Admin", "url": "/admin"},
+            {"name": "Containers", "url": "/admin/containers"},
+        ],
+        schemas=[website_schema(base)],
+    )
+
+    return respond(
+        request,
+        "containers_admin.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": admin,
+            "instances": instances,
+            "admin_section": "containers",
+        },
+        model=AdminContainersOut,
+    )
+
+
+@router.get("/data")
+async def containers_index_json(request: Request):
+    require_admin(request)
+    return JSONResponse({"instances": _decorate(store.all_instances())})
+
+
+@router.get("/{uid}", response_class=HTMLResponse)
+async def container_instance_page(request: Request, uid: str):
+    admin = require_admin(request)
+    inst = store.get_instance(uid)
+    if not inst:
+        raise not_found("Instance not found")
+    project = get_table("projects").find_one(uid=inst["project_uid"]) or {}
+    project_slug = project.get("slug") or project.get("uid") or ""
+    base = site_url(request)
+    seo_ctx = base_seo_context(
+        request,
+        title=f"{inst['name']} - Containers",
+        description=f"Container instance {inst['name']}.",
+        robots="noindex,nofollow",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Admin", "url": "/admin"},
+            {"name": "Containers", "url": "/admin/containers"},
+            {"name": inst["name"], "url": f"/admin/containers/{inst['uid']}"},
+        ],
+        schemas=[website_schema(base)],
+    )
+
+    return respond(
+        request,
+        "containers_instance.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": admin,
+            "instance": inst,
+            "project": project,
+            "project_slug": project_slug,
+            "events": store.list_events(inst["uid"]),
+            "schedules": store.list_schedules(inst["uid"]),
+            "stats": api.instance_stats(inst["uid"]),
+            "runtime": api.instance_runtime(inst),
+            "admin_section": "containers",
+        },
+        model=AdminContainerInstanceOut,
+    )

devplacepy/routers/devii.py

@@ -0,0 +1,288 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+
+import httpx
+import uuid_utils
+from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect
+from fastapi.responses import JSONResponse, RedirectResponse
+
+from devplacepy.constants import DEVII_GUEST_COOKIE
+from devplacepy.database import get_int_setting
+from devplacepy.seo import base_seo_context, site_url
+from devplacepy.services.manager import service_manager
+
+from devplacepy.responses import respond
+from devplacepy.schemas import DeviiPageOut
+from devplacepy.utils import (
+    _user_from_api_key,
+    _user_from_session,
+    get_current_user,
+    is_admin,
+    require_user,
+)
+from devplacepy.services.audit import record as audit
+
+logger = logging.getLogger("devii.router")
+
+router = APIRouter()
+
+GUEST_COOKIE = DEVII_GUEST_COOKIE
+GUEST_COOKIE_MAX_AGE = 31536000
+
+
+def _service():
+    return service_manager.get_service("devii")
+
+
+def _bearer_key(headers) -> str:
+    key = headers.get("x-api-key", "")
+    if key:
+        return key.strip()
+    scheme, _, credentials = headers.get("authorization", "").partition(" ")
+    if scheme.lower() == "bearer":
+        return credentials.strip()
+    return ""
+
+
+def _resolve_ws_owner(websocket: WebSocket):
+    user = _user_from_session(websocket)
+    if not user:
+        key = _bearer_key(websocket.headers)
+        if key:
+            user = _user_from_api_key(key)
+    if user:
+        return (
+            "user",
+            user["uid"],
+            user.get("username", ""),
+            user.get("api_key", ""),
+            is_admin(user),
+        )
+    guest_id = websocket.cookies.get(GUEST_COOKIE) or uuid_utils.uuid7().hex
+    return "guest", guest_id, "guest", "", False
+
+
+def _owner_from_request(request: Request):
+    user = get_current_user(request)
+    if user:
+        return "user", user["uid"], user.get("username", "")
+    guest_id = request.cookies.get(GUEST_COOKIE) or ""
+    return "guest", guest_id, "guest"
+
+
+@router.get("/")
+async def devii_page(request: Request):
+    user = get_current_user(request)
+    seo_ctx = base_seo_context(
+        request,
+        title="Devii",
+        description="Your personal AI development assistant on DevPlace.",
+        robots="noindex,nofollow",
+    )
+    response = respond(
+        request,
+        "devii.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+        },
+        model=DeviiPageOut,
+    )
+    if not user and not request.cookies.get(GUEST_COOKIE):
+        response.set_cookie(
+            GUEST_COOKIE,
+            uuid_utils.uuid7().hex,
+            httponly=True,
+            samesite="lax",
+            max_age=GUEST_COOKIE_MAX_AGE,
+        )
+    return response
+
+
+@router.get("/session")
+async def devii_session(request: Request):
+    user = get_current_user(request)
+    svc = _service()
+    payload = {
+        "loggedIn": bool(user),
+        "username": user.get("username") if user else "guest",
+        "enabled": bool(svc and svc.is_enabled()),
+        "guestsEnabled": bool(svc and svc.guests_enabled()),
+        "baseUrl": site_url(request),
+    }
+    response = JSONResponse(payload)
+    if not user and not request.cookies.get(GUEST_COOKIE):
+        response.set_cookie(
+            GUEST_COOKIE,
+            uuid_utils.uuid7().hex,
+            httponly=True,
+            samesite="lax",
+            max_age=GUEST_COOKIE_MAX_AGE,
+        )
+    return response
+
+
+def _validate_target(value: str) -> str:
+    if value.startswith("/") and not value.startswith("//"):
+        return value
+    return "/"
+
+
+@router.get("/adopt")
+async def devii_adopt(request: Request):
+    # The terminal's agent logged in; adopt the real session it minted into the browser so both
+    # share one session, then return to where the user was.
+    target = _validate_target(request.query_params.get("next", "/"))
+    response = RedirectResponse(target, status_code=303)
+    svc = _service()
+    if svc is None:
+        return response
+    owner_kind, owner_id, _ = _owner_from_request(request)
+    session = svc.hub().find(owner_kind, owner_id) if owner_id else None
+    token = session.take_pending_session() if session else None
+    if token:
+        max_age = max(1, get_int_setting("session_remember_days", 30)) * 86400
+        response.set_cookie(
+            "session", token, max_age=max_age, httponly=True, samesite="lax"
+        )
+    return response
+
+
+@router.get("/usage")
+async def devii_usage(request: Request):
+    svc = _service()
+    owner_kind, owner_id, _ = _owner_from_request(request)
+    if svc is None:
+        return JSONResponse({"used_pct": 0.0, "turns_today": 0})
+    spent = svc.spent_24h(owner_kind, owner_id) if owner_id else 0.0
+    limit = svc.daily_limit_for(owner_kind)
+    turns = svc.hub().ledger.turns_24h(owner_kind, owner_id) if owner_id else 0
+    used_pct = round(min(100.0, spent / limit * 100), 1) if limit > 0 else 0.0
+    payload = {
+        "used_pct": used_pct,
+        "turns_today": turns,
+        "owner_kind": owner_kind,
+    }
+    if is_admin(get_current_user(request)):
+        payload["spent_24h"] = round(spent, 6)
+        payload["limit"] = limit
+    return JSONResponse(payload)
+
+
+@router.post("/clippy/ai/chat")
+async def clippy_proxy(request: Request):
+    user = require_user(request)
+    svc = _service()
+    if svc is None or not svc.is_enabled():
+        return JSONResponse({"error": "Devii is unavailable"}, status_code=503)
+    cfg = svc.effective_config()
+    body = await request.body()
+    headers = {"Content-Type": "application/json"}
+    if cfg.get("devii_ai_key"):
+        headers["Authorization"] = f"Bearer {cfg['devii_ai_key']}"
+    async with httpx.AsyncClient(timeout=45.0) as client:
+        upstream = await client.post(cfg["devii_ai_url"], content=body, headers=headers)
+    audit.record(
+        request,
+        "ai.clippy.chat",
+        summary="clippy AI chat",
+        metadata={"status_code": upstream.status_code},
+    )
+    return JSONResponse(
+        content=upstream.json()
+        if upstream.headers.get("content-type", "").startswith("application/json")
+        else {"raw": upstream.text},
+        status_code=upstream.status_code,
+    )
+
+
+@router.websocket("/ws")
+async def devii_ws(websocket: WebSocket):
+    await websocket.accept()
+    svc = _service()
+    if svc is None or not svc.is_enabled():
+        await websocket.close(code=1013)
+        return
+    if not service_manager.owns_lock():
+        await websocket.close(code=4013)
+        return
+    owner_kind, owner_id, username, api_key, owner_is_admin = _resolve_ws_owner(
+        websocket
+    )
+    if owner_kind == "guest" and not svc.guests_enabled():
+        await websocket.send_json(
+            {"type": "error", "text": "Guest access to Devii is disabled."}
+        )
+        await websocket.close(code=1008)
+        return
+
+    session = svc.hub().get_or_create(
+        owner_kind,
+        owner_id,
+        username,
+        api_key,
+        svc.instance_base_url(),
+        is_admin=owner_is_admin,
+    )
+    session.attach(websocket)
+    try:
+        await session.send_bootstrap(websocket)
+        while True:
+            data = await websocket.receive_json()
+            kind = data.get("type")
+            if kind == "input":
+                text = str(data.get("text", "")).strip()
+                if not text:
+                    continue
+                command = text.lower().lstrip("/").strip()
+                if command == "stop":
+                    await session.stop()
+                    continue
+                if command == "reset":
+                    await session.reset()
+                    continue
+                limit = svc.daily_limit_for(owner_kind, owner_is_admin)
+                spent = svc.spent_24h(owner_kind, owner_id)
+                if limit > 0 and spent >= limit:
+                    audit.record_system(
+                        "ai.quota.exceeded",
+                        actor_kind=owner_kind if owner_kind == "guest" else "user",
+                        actor_uid=owner_id if owner_kind == "user" else None,
+                        actor_username=username,
+                        actor_role="admin" if owner_is_admin else ("guest" if owner_kind == "guest" else "member"),
+                        origin="devii",
+                        via_agent=1,
+                        result="denied",
+                        summary=f"AI request by {username or owner_id} blocked - 24h quota reached",
+                        metadata={"limit_usd": limit, "owner_kind": owner_kind},
+                    )
+                    await websocket.send_json(
+                        {
+                            "type": "error",
+                            "text": "Daily AI quota reached (100%). Try again later.",
+                        }
+                    )
+                    continue
+                session.spawn_turn(text)
+            elif kind == "reset":
+                await session.reset()
+            elif kind == "stop":
+                await session.stop()
+            elif kind == "clear":
+                await session.reset_conversation()
+            elif kind == "visibility":
+                session.set_visibility(
+                    websocket,
+                    bool(data.get("visible", True)),
+                    bool(data.get("focused", False)),
+                )
+            elif kind in ("avatar_result", "client_result"):
+                session.resolve_query(str(data.get("id", "")), data.get("result"))
+    except WebSocketDisconnect:
+        pass
+    except Exception:  # noqa: BLE001 - never let the socket loop crash the worker
+        logger.exception("Devii websocket loop failed for %s/%s", owner_kind, owner_id)
+    finally:
+        session.detach(websocket)

devplacepy/routers/docs.py

@@ -0,0 +1,632 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+from fastapi import APIRouter, Request
+from fastapi.responses import HTMLResponse, RedirectResponse, Response
+from devplacepy.templating import templates
+from devplacepy.utils import get_current_user, not_found, is_admin as user_is_admin
+from devplacepy.seo import base_seo_context, site_url, website_schema
+from devplacepy.docs_api import api_doc_pages, render_group, build_services_group
+from devplacepy.services.manager import service_manager
+from devplacepy import docs_search, docs_export, docs_live, docs_prose
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+SECTION_GENERAL = "General"
+SECTION_COMPONENTS = "Components"
+SECTION_STYLES = "Styles"
+SECTION_API = "API"
+SECTION_ADMIN = "Administration"
+SECTION_DEVII = "Devii internals"
+SECTION_BOTS = "Bots internals"
+SECTION_SERVICES = "Services"
+SECTION_ARCH = "Architecture"
+SECTION_TESTING = "Testing"
+SECTION_PROD = "Production"
+SECTION_MAINTENANCE = "Maintenance agents"
+
+DOCS_PAGES = [
+    # General - how to use the site and Devii (everyone)
+    {"slug": "index", "title": "Overview", "kind": "prose", "section": SECTION_GENERAL},
+    {
+        "slug": "devii",
+        "title": "Devii Assistant",
+        "kind": "prose",
+        "section": SECTION_GENERAL,
+    },
+    {
+        "slug": "dashboard",
+        "title": "Your dashboard",
+        "kind": "live",
+        "section": SECTION_GENERAL,
+    },
+    {
+        "slug": "media-gallery",
+        "title": "Media gallery",
+        "kind": "prose",
+        "section": SECTION_GENERAL,
+    },
+    # Maintenance agents - the self-maintaining quality fleet (public)
+    {
+        "slug": "maintenance-agents",
+        "title": "Maintenance agents",
+        "kind": "prose",
+        "section": SECTION_MAINTENANCE,
+    },
+    {
+        "slug": "maintenance-usage",
+        "title": "Running the agents",
+        "kind": "prose",
+        "section": SECTION_MAINTENANCE,
+    },
+    # Components - custom HTML web components with live examples (everyone)
+    {
+        "slug": "components",
+        "title": "Components overview",
+        "kind": "prose",
+        "section": SECTION_COMPONENTS,
+    },
+    {
+        "slug": "component-dp-avatar",
+        "title": "dp-avatar",
+        "kind": "prose",
+        "section": SECTION_COMPONENTS,
+    },
+    {
+        "slug": "component-dp-code",
+        "title": "dp-code",
+        "kind": "prose",
+        "section": SECTION_COMPONENTS,
+    },
+    {
+        "slug": "component-dp-content",
+        "title": "dp-content",
+        "kind": "prose",
+        "section": SECTION_COMPONENTS,
+    },
+    {
+        "slug": "component-dp-upload",
+        "title": "dp-upload",
+        "kind": "prose",
+        "section": SECTION_COMPONENTS,
+    },
+    {
+        "slug": "component-dp-toast",
+        "title": "dp-toast",
+        "kind": "prose",
+        "section": SECTION_COMPONENTS,
+    },
+    {
+        "slug": "component-dp-dialog",
+        "title": "dp-dialog",
+        "kind": "prose",
+        "section": SECTION_COMPONENTS,
+    },
+    {
+        "slug": "component-dp-context-menu",
+        "title": "dp-context-menu",
+        "kind": "prose",
+        "section": SECTION_COMPONENTS,
+    },
+    {
+        "slug": "component-dp-lightbox",
+        "title": "dp-lightbox",
+        "kind": "prose",
+        "section": SECTION_COMPONENTS,
+    },
+    {
+        "slug": "component-devii-terminal",
+        "title": "devii-terminal",
+        "kind": "prose",
+        "section": SECTION_COMPONENTS,
+    },
+    {
+        "slug": "component-devii-avatar",
+        "title": "devii-avatar",
+        "kind": "prose",
+        "section": SECTION_COMPONENTS,
+    },
+    {
+        "slug": "component-emoji-picker",
+        "title": "emoji-picker",
+        "kind": "prose",
+        "section": SECTION_COMPONENTS,
+    },
+    # Styles - the design system: colors, layout, responsiveness, and hard structural rules (everyone)
+    {
+        "slug": "styles",
+        "title": "Design system overview",
+        "kind": "prose",
+        "section": SECTION_STYLES,
+    },
+    {
+        "slug": "styles-colors",
+        "title": "Colors",
+        "kind": "prose",
+        "section": SECTION_STYLES,
+    },
+    {
+        "slug": "styles-layout",
+        "title": "Layout",
+        "kind": "prose",
+        "section": SECTION_STYLES,
+    },
+    {
+        "slug": "styles-responsiveness",
+        "title": "Responsiveness",
+        "kind": "prose",
+        "section": SECTION_STYLES,
+    },
+    {
+        "slug": "styles-consistency",
+        "title": "Consistency rules",
+        "kind": "prose",
+        "section": SECTION_STYLES,
+    },
+    # API - developer reference (everyone); admin-only groups are routed to Administration below
+    {
+        "slug": "authentication",
+        "title": "Authentication",
+        "kind": "prose",
+        "section": SECTION_API,
+    },
+    *[
+        {**page, "section": (SECTION_ADMIN if page.get("admin") else SECTION_API)}
+        for page in api_doc_pages()
+    ],
+    # Administration - operational guides (admins only)
+    {
+        "slug": "media-moderation",
+        "title": "Media moderation",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_ADMIN,
+    },
+    {
+        "slug": "soft-delete",
+        "title": "Soft delete and data retention",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_ADMIN,
+    },
+    # Devii internals - technical reference for the Devii assistant (admins only)
+    {
+        "slug": "audit-log",
+        "title": "Audit Log",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_ADMIN,
+    },
+    {
+        "slug": "devii-internals",
+        "title": "Devii internals",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_DEVII,
+    },
+    {
+        "slug": "devii-architecture",
+        "title": "Architecture and sessions",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_DEVII,
+    },
+    {
+        "slug": "devii-tools",
+        "title": "Tools and scopes",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_DEVII,
+    },
+    {
+        "slug": "devii-data",
+        "title": "Data and persistence",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_DEVII,
+    },
+    {
+        "slug": "devii-security",
+        "title": "Security and limits",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_DEVII,
+    },
+    {
+        "slug": "devii-config",
+        "title": "Configuration and CLI",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_DEVII,
+    },
+    # Bots internals - deep technical reference for the autonomous bot fleet (admins only)
+    {
+        "slug": "bots-internals",
+        "title": "Bots internals",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_BOTS,
+    },
+    {
+        "slug": "bots-architecture",
+        "title": "Architecture and lifecycle",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_BOTS,
+    },
+    {
+        "slug": "bots-personas",
+        "title": "Personas and voice",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_BOTS,
+    },
+    {
+        "slug": "bots-content",
+        "title": "Content generation and quality",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_BOTS,
+    },
+    {
+        "slug": "bots-engagement",
+        "title": "Engagement and interaction",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_BOTS,
+    },
+    {
+        "slug": "bots-realism",
+        "title": "Realism and design decisions",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_BOTS,
+    },
+    {
+        "slug": "bots-config",
+        "title": "Configuration and operations",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_BOTS,
+    },
+    # Services - the background service framework and every service in detail (admins only)
+    {
+        "slug": "services-overview",
+        "title": "Services overview",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_SERVICES,
+    },
+    {
+        "slug": "services-framework",
+        "title": "Service framework",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_SERVICES,
+    },
+    {
+        "slug": "services-data",
+        "title": "Reading service data",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_SERVICES,
+    },
+    {
+        "slug": "services-gateway",
+        "title": "GatewayService (AI)",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_SERVICES,
+    },
+    {
+        "slug": "services-devii",
+        "title": "DeviiService",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_SERVICES,
+    },
+    {
+        "slug": "services-news",
+        "title": "NewsService",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_SERVICES,
+    },
+    {
+        "slug": "services-bots",
+        "title": "BotsService",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_SERVICES,
+    },
+    {
+        "slug": "services-zip",
+        "title": "ZipService",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_SERVICES,
+    },
+    {
+        "slug": "services-containers",
+        "title": "Container Manager",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_SERVICES,
+    },
+    # Architecture - platform design, structure, and development process (admins only)
+    {
+        "slug": "architecture",
+        "title": "Architecture overview",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_ARCH,
+    },
+    {
+        "slug": "architecture-backend",
+        "title": "Backend and request pipeline",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_ARCH,
+    },
+    {
+        "slug": "architecture-frontend",
+        "title": "Frontend and components",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_ARCH,
+    },
+    {
+        "slug": "architecture-styling",
+        "title": "Styling and templates",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_ARCH,
+    },
+    {
+        "slug": "architecture-conventions",
+        "title": "Conventions and rules",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_ARCH,
+    },
+    {
+        "slug": "architecture-workflow",
+        "title": "Development workflow",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_ARCH,
+    },
+    {
+        "slug": "architecture-jobs",
+        "title": "Async job services",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_ARCH,
+    },
+    # Testing - test framework, load testing, and make targets (admins only)
+    {
+        "slug": "testing",
+        "title": "Testing overview",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_TESTING,
+    },
+    {
+        "slug": "testing-framework",
+        "title": "Test framework and rules",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_TESTING,
+    },
+    {
+        "slug": "testing-locust",
+        "title": "Load testing with Locust",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_TESTING,
+    },
+    {
+        "slug": "testing-make",
+        "title": "Running tests via make",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_TESTING,
+    },
+    {
+        "slug": "testing-cicd",
+        "title": "Coverage and CI/CD",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_TESTING,
+    },
+    # Production - deployment and operations reference (admins only)
+    {
+        "slug": "production",
+        "title": "Production overview",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_PROD,
+    },
+    {
+        "slug": "production-deploy",
+        "title": "Deploy and update",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_PROD,
+    },
+    {
+        "slug": "production-nginx",
+        "title": "nginx and networking",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_PROD,
+    },
+    {
+        "slug": "production-concurrency",
+        "title": "Multi-worker and concurrency",
+        "kind": "prose",
+        "admin": True,
+        "section": SECTION_PROD,
+    },
+]
+_PAGES_BY_SLUG = {page["slug"]: page for page in DOCS_PAGES}
+
+
+@router.get("/docs")
+async def docs_root(request: Request):
+    return RedirectResponse(url="/docs/index.html", status_code=302)
+
+
+def _export_context(request: Request):
+    user = get_current_user(request)
+    is_admin = user_is_admin(user)
+    return (
+        is_admin,
+        site_url(request),
+        user.get("username") if user else "",
+        user.get("api_key") if user else "",
+    )
+
+
+@router.get("/docs/download.md")
+async def docs_download_md(request: Request):
+    is_admin, base, username, api_key = _export_context(request)
+    markdown = docs_export.build_markdown(is_admin, base, username, api_key)
+    return Response(
+        content=markdown,
+        media_type="text/markdown; charset=utf-8",
+        headers={"Content-Disposition": 'attachment; filename="devplace-docs.md"'},
+    )
+
+
+@router.get("/docs/download.html")
+async def docs_download_html(request: Request):
+    is_admin, base, username, api_key = _export_context(request)
+    markdown = docs_export.build_markdown(is_admin, base, username, api_key)
+    return HTMLResponse(
+        content=docs_export.build_html(markdown),
+        headers={"Content-Disposition": 'attachment; filename="devplace-docs.html"'},
+    )
+
+
+@router.get("/docs/{slug}.html", response_class=HTMLResponse)
+async def docs_page(request: Request, slug: str):
+    user = get_current_user(request)
+    is_admin = user_is_admin(user)
+    base = site_url(request)
+    visible_pages = [p for p in DOCS_PAGES if not p.get("admin") or is_admin]
+
+    if slug == "search":
+        query = request.query_params.get("q", "")
+        results = docs_search.search(query, user=user, is_admin=is_admin)
+        seo_ctx = base_seo_context(
+            request,
+            title="Search - Documentation",
+            description="Search the DevPlace developer documentation.",
+            robots="noindex",
+            breadcrumbs=[
+                {"name": "Home", "url": "/feed"},
+                {"name": "Docs", "url": "/docs/index.html"},
+                {"name": "Search", "url": "/docs/search.html"},
+            ],
+            schemas=[website_schema(base)],
+        )
+        return templates.TemplateResponse(
+            request,
+            "docs_base.html",
+            {
+                **seo_ctx,
+                "request": request,
+                "user": user,
+                "pages": visible_pages,
+                "current": "search",
+                "kind": "search",
+                "page_title_doc": "Search",
+                "search_query": query,
+                "search_results": results,
+                "base": base,
+            },
+        )
+
+    page = _PAGES_BY_SLUG.get(slug)
+    if not page:
+        raise not_found("Documentation page not found")
+    if page.get("admin") and not is_admin:
+        raise not_found("Documentation page not found")
+    if page["kind"] == "live":
+        seo_ctx = base_seo_context(
+            request,
+            title=f"{page['title']} - Documentation",
+            description="Your live DevPlace stats, activity, and site data.",
+            robots="noindex",
+            breadcrumbs=[
+                {"name": "Home", "url": "/feed"},
+                {"name": "Docs", "url": "/docs/index.html"},
+                {"name": page["title"], "url": f"/docs/{slug}.html"},
+            ],
+            schemas=[website_schema(base)],
+        )
+        return templates.TemplateResponse(
+            request,
+            "docs_base.html",
+            {
+                **seo_ctx,
+                "request": request,
+                "user": user,
+                "pages": visible_pages,
+                "current": slug,
+                "kind": "live",
+                "facts": docs_live.build_live_facts(user),
+                "page_title_doc": page["title"],
+                "base": base,
+            },
+        )
+    prose_html = None
+    group = None
+    if page["kind"] == "prose":
+        prose_html = docs_prose.render_prose(
+            slug,
+            {
+                "base": base,
+                "user": user,
+                "username": user.get("username") if user else "",
+                "api_key": user.get("api_key") if user else "",
+            },
+        )
+    elif page.get("dynamic"):
+        group = build_services_group(service_manager.describe_all(), base)
+    else:
+        group = render_group(
+            slug,
+            base,
+            user.get("username") if user else "",
+            user.get("api_key") if user else "",
+        )
+    seo_ctx = base_seo_context(
+        request,
+        title=f"{page['title']} - Documentation",
+        description="DevPlace developer documentation.",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Docs", "url": "/docs/index.html"},
+            {"name": page["title"], "url": f"/docs/{slug}.html"},
+        ],
+        schemas=[website_schema(base)],
+    )
+    return templates.TemplateResponse(
+        request,
+        "docs_base.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "pages": visible_pages,
+            "current": slug,
+            "kind": page["kind"],
+            "group": group,
+            "prose_html": prose_html,
+            "page_title_doc": page["title"],
+            "base": base,
+        },
+    )

devplacepy/routers/feed.py

@@ -1,109 +1,121 @@
+# retoor <retoor@molodetz.nl>
+
 import logging
-from datetime import datetime
 from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse
-from devplacepy.database import get_table, get_daily_topic, get_users_by_uids, get_comment_counts_by_post_uids
+from devplacepy.database import (
+    get_table,
+    get_daily_topic,
+    get_users_by_uids,
+    get_comment_counts_by_post_uids,
+    get_recent_comments_by_post_uids,
+    get_site_stats,
+    get_top_authors,
+    get_reactions_by_targets,
+    get_user_bookmarks,
+    get_polls_by_post_uids,
+    paginate,
+)
 from devplacepy.attachments import get_attachments_batch
-from devplacepy.templating import templates
-from devplacepy.utils import get_current_user, time_ago
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, combine
+from devplacepy.content import enrich_items
+from devplacepy.utils import get_current_user
+from devplacepy.seo import list_page_seo, next_page_url
+from devplacepy.responses import respond
+from devplacepy.schemas import FeedOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
-PAGE_SIZE = 25
-
 
 def get_feed_posts(user, tab: str = "all", topic: str = None, before: str = None):
     posts_table = get_table("posts")
-
-    filters = {}
-    if topic:
-        filters["topic"] = topic
+    order = ["-stars", "-created_at"] if tab == "trending" else ["-created_at"]
 
     if tab == "following":
         if not user:
             return [], None
         follows = get_table("follows")
-        following = [f["following_uid"] for f in follows.find(follower_uid=user["uid"])]
+        following = [
+            f["following_uid"]
+            for f in follows.find(follower_uid=user["uid"], deleted_at=None)
+        ]
         if not following:
             return [], None
-        posts = list(posts_table.find(posts_table.table.columns.user_uid.in_(following), order_by=["-created_at"], _limit=PAGE_SIZE))
+        posts, next_cursor = paginate(
+            posts_table,
+            posts_table.table.columns.user_uid.in_(following),
+            before=before,
+            order=order,
+        )
     else:
-        order = ["-created_at"]
-        if tab == "trending":
-            order = ["-stars", "-created_at"]
-        if before:
-            posts = list(posts_table.find(**filters, order_by=order, _limit=PAGE_SIZE + 1))
-            posts = [p for p in posts if p["created_at"] < before][:PAGE_SIZE]
-        else:
-            posts = list(posts_table.find(**filters, order_by=order, _limit=PAGE_SIZE + 1))
-
-    has_more = len(posts) > PAGE_SIZE
-    posts = posts[:PAGE_SIZE]
-
-    next_cursor = None
-    if has_more and posts:
-        next_cursor = posts[-1]["created_at"]
+        filters = {"topic": topic} if topic else {}
+        posts, next_cursor = paginate(
+            posts_table, before=before, order=order, **filters
+        )
 
     if not posts:
         return [], next_cursor
 
-    uids = [p["user_uid"] for p in posts]
-    post_uids = [p["uid"] for p in posts]
-    authors = get_users_by_uids(uids)
-    counts = get_comment_counts_by_post_uids(post_uids)
+    authors = get_users_by_uids([p["user_uid"] for p in posts])
+    counts = get_comment_counts_by_post_uids([p["uid"] for p in posts])
 
-    result = []
-    for post in posts:
-        result.append({
-            "post": post,
-            "author": authors.get(post["user_uid"]),
-            "time_ago": time_ago(post["created_at"]),
-            "comment_count": counts.get(post["uid"], 0),
-        })
+    result = enrich_items(posts, "post", authors, {"comment_count": counts}, user=user)
     return result, next_cursor
 
 
 @router.get("", response_class=HTMLResponse)
-async def feed_page(request: Request, tab: str = "all", topic: str = None, before: str = None):
+async def feed_page(
+    request: Request, tab: str = "all", topic: str = None, before: str = None
+):
     user = get_current_user(request)
     posts, next_cursor = get_feed_posts(user, tab, topic, before)
-    users_table = get_table("users")
-    total_members = len(list(users_table.all()))
-    posts_table = get_table("posts")
-    today_start = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0).isoformat()
-    posts_today = len(list(posts_table.find(created_at={">=": today_start})))
-    total_projects = len(list(get_table("projects").all()))
-    total_gists = len(list(get_table("gists").all()))
-    top_authors = list(users_table.find(stars={">": 0}, order_by=["-stars"], _limit=5))
+    stats = get_site_stats()
+    top_authors = get_top_authors(5)
     daily_topic = get_daily_topic()
 
     post_uids_list = [item["post"]["uid"] for item in posts]
     attachments_map = get_attachments_batch("post", post_uids_list)
+    recent_comments = get_recent_comments_by_post_uids(post_uids_list, 3, user)
+    reactions_map = get_reactions_by_targets("post", post_uids_list, user)
+    bookmark_set = (
+        get_user_bookmarks(user["uid"], "post", post_uids_list) if user else set()
+    )
+    polls_map = get_polls_by_post_uids(post_uids_list, user)
     for item in posts:
-        item["attachments"] = attachments_map.get(item["post"]["uid"], [])
+        uid = item["post"]["uid"]
+        item["attachments"] = attachments_map.get(uid, [])
+        item["recent_comments"] = recent_comments.get(uid, [])
+        item["reactions"] = reactions_map.get(uid, {"counts": {}, "mine": []})
+        item["bookmarked"] = uid in bookmark_set
+        item["poll"] = polls_map.get(uid)
 
-    base = site_url(request)
-    seo_ctx = base_seo_context(
+    seo_ctx = list_page_seo(
         request,
         title="Feed",
         description="Discover the latest developer discussions, projects, and community activity on DevPlace.",
-        breadcrumbs=[{"name": "Home", "url": "/feed"}, {"name": "Feed", "url": "/feed"}],
-        schemas=[website_schema(base)],
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Feed", "url": "/feed"},
+        ],
+        next_url=next_page_url(request, next_cursor),
+    )
+    return respond(
+        request,
+        "feed.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "posts": posts,
+            "current_tab": tab,
+            "current_topic": topic,
+            "total_members": stats["total_members"],
+            "posts_today": stats["posts_today"],
+            "total_projects": stats["total_projects"],
+            "total_gists": stats["total_gists"],
+            "top_authors": top_authors,
+            "daily_topic": daily_topic,
+            "next_cursor": next_cursor,
+        },
+        model=FeedOut,
     )
-    return templates.TemplateResponse("feed.html", {
-        **seo_ctx,
-        "request": request,
-        "user": user,
-        "posts": posts,
-        "current_tab": tab,
-        "current_topic": topic,
-        "total_members": total_members,
-        "posts_today": posts_today,
-        "total_projects": total_projects,
-        "total_gists": total_gists,
-        "top_authors": top_authors,
-        "daily_topic": daily_topic,
-        "next_cursor": next_cursor,
-    })

devplacepy/routers/follow.py

@@ -1,9 +1,18 @@
+# retoor <retoor@molodetz.nl>
+
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from fastapi import APIRouter, Request
-from fastapi.responses import RedirectResponse
-from devplacepy.database import get_table
-from devplacepy.utils import generate_uid, require_user
+from devplacepy.database import get_table, _now_iso
+from devplacepy.utils import (
+    generate_uid,
+    require_user,
+    create_notification,
+    award_rewards,
+    XP_FOLLOW,
+)
+from devplacepy.responses import action_result
+from devplacepy.services.audit import record as audit
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -15,33 +24,49 @@ async def follow_user(request: Request, username: str):
     users = get_table("users")
     target = users.find_one(username=username)
     if not target or target["uid"] == user["uid"]:
-        return RedirectResponse(url=f"/profile/{username}", status_code=302)
+        return action_result(request, f"/profile/{username}")
 
     follows = get_table("follows")
     existing = follows.find_one(follower_uid=user["uid"], following_uid=target["uid"])
+    if existing and not existing.get("deleted_at"):
+        return action_result(request, f"/profile/{username}")
     if existing:
-        return RedirectResponse(url=f"/profile/{username}", status_code=302)
+        follows.update(
+            {"id": existing["id"], "deleted_at": None, "deleted_by": None}, ["id"]
+        )
+    else:
+        follows.insert(
+            {
+                "uid": generate_uid(),
+                "follower_uid": user["uid"],
+                "following_uid": target["uid"],
+                "created_at": datetime.now(timezone.utc).isoformat(),
+                "deleted_at": None,
+                "deleted_by": None,
+            }
+        )
 
-    follows.insert({
-        "uid": generate_uid(),
-        "follower_uid": user["uid"],
-        "following_uid": target["uid"],
-        "created_at": datetime.utcnow().isoformat(),
-    })
-
-    notifications = get_table("notifications")
-    notifications.insert({
-        "uid": generate_uid(),
-        "user_uid": target["uid"],
-        "type": "follow",
-        "message": f"{user['username']} started following you",
-        "related_uid": user["uid"],
-        "read": False,
-        "created_at": datetime.utcnow().isoformat(),
-    })
+    create_notification(
+        target["uid"],
+        "follow",
+        f"{user['username']} started following you",
+        user["uid"],
+        f"/profile/{user['username']}",
+    )
+    award_rewards(target["uid"], XP_FOLLOW)
 
     logger.info(f"{user['username']} followed {username}")
-    return RedirectResponse(url=f"/profile/{username}", status_code=302)
+    audit.record(
+        request,
+        "follow.follow",
+        user=user,
+        target_type="user",
+        target_uid=target["uid"],
+        target_label=username,
+        summary=f"{user['username']} followed {username}",
+        links=[audit.target("user", target["uid"], username)],
+    )
+    return action_result(request, f"/profile/{username}")
 
 
 @router.post("/unfollow/{username}")
@@ -50,12 +75,27 @@ async def unfollow_user(request: Request, username: str):
     users = get_table("users")
     target = users.find_one(username=username)
     if not target:
-        return RedirectResponse(url=f"/profile/{username}", status_code=302)
+        return action_result(request, f"/profile/{username}")
 
     follows = get_table("follows")
-    existing = follows.find_one(follower_uid=user["uid"], following_uid=target["uid"])
+    existing = follows.find_one(
+        follower_uid=user["uid"], following_uid=target["uid"], deleted_at=None
+    )
     if existing:
-        follows.delete(id=existing["id"])
+        follows.update(
+            {"id": existing["id"], "deleted_at": _now_iso(), "deleted_by": user["uid"]},
+            ["id"],
+        )
         logger.info(f"{user['username']} unfollowed {username}")
+        audit.record(
+            request,
+            "follow.unfollow",
+            user=user,
+            target_type="user",
+            target_uid=target["uid"],
+            target_label=username,
+            summary=f"{user['username']} unfollowed {username}",
+            links=[audit.target("user", target["uid"], username)],
+        )
 
-    return RedirectResponse(url=f"/profile/{username}", status_code=302)
+    return action_result(request, f"/profile/{username}")

devplacepy/routers/forks.py

@@ -0,0 +1,41 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+
+from fastapi import APIRouter, Request
+from fastapi.responses import JSONResponse
+
+from devplacepy.services.jobs import queue
+from devplacepy.schemas import ForkJobOut
+from devplacepy.utils import not_found
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+def _status_payload(job: dict) -> dict:
+    result = job.get("result", {})
+    done = job.get("status") == queue.DONE
+    return {
+        "uid": job.get("uid", ""),
+        "kind": job.get("kind", ""),
+        "status": job.get("status", ""),
+        "preferred_name": job.get("preferred_name"),
+        "project_uid": result.get("project_uid") if done else None,
+        "project_url": result.get("project_url") if done else None,
+        "source_project_uid": result.get("source_project_uid") if done else None,
+        "error": job.get("error") or None,
+        "item_count": int(job.get("item_count") or 0),
+        "created_at": job.get("created_at"),
+        "completed_at": job.get("completed_at") or None,
+    }
+
+
+@router.get("/{uid}")
+async def fork_status(request: Request, uid: str):
+    job = queue.get_job(uid)
+    if not job or job.get("kind") != "fork":
+        raise not_found("Job not found")
+    return JSONResponse(
+        ForkJobOut.model_validate(_status_payload(job)).model_dump(mode="json")
+    )

devplacepy/routers/gists.py

@@ -1,27 +1,73 @@
+# retoor <retoor@molodetz.nl>
+
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request, HTTPException
-from fastapi.responses import HTMLResponse, RedirectResponse
-from devplacepy.database import get_table, load_comments, get_vote_counts, get_attachments, get_attachments_by_type, delete_attachments, resolve_by_slug
-from devplacepy.templating import templates
-from devplacepy.utils import generate_uid, get_current_user, require_user, time_ago, make_combined_slug, create_mention_notifications
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, combine
+from typing import Annotated
+from fastapi import APIRouter, Request, Form
+from devplacepy.models import GistForm, GistEditForm
+from fastapi.responses import HTMLResponse
+from devplacepy.database import (
+    get_table,
+    get_users_by_uids,
+    get_gist_languages,
+    paginate,
+)
+from devplacepy.content import (
+    load_detail,
+    edit_content_item,
+    delete_content_item,
+    enrich_items,
+    create_content_item,
+    detail_context,
+    canonical_redirect,
+    first_image_url,
+)
+from devplacepy.utils import get_current_user, require_user, not_found, XP_GIST
+from devplacepy.seo import (
+    base_seo_context,
+    site_url,
+    website_schema,
+    software_source_code_schema,
+    list_page_seo,
+    next_page_url,
+)
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import GistsOut, GistDetailOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 LANGUAGES = [
-    ("python", "Python"), ("javascript", "JavaScript"), ("typescript", "TypeScript"),
-    ("html", "HTML"), ("css", "CSS"), ("c", "C"), ("cpp", "C++"), ("java", "Java"),
-    ("go", "Go"), ("rust", "Rust"), ("sql", "SQL"), ("bash", "Bash"),
-    ("yaml", "YAML"), ("json", "JSON"), ("markdown", "Markdown"),
-    ("swift", "Swift"), ("php", "PHP"), ("ruby", "Ruby"), ("kotlin", "Kotlin"),
-    ("lua", "Lua"), ("perl", "Perl"), ("haskell", "Haskell"), ("elixir", "Elixir"),
-    ("r", "R"), ("dart", "Dart"), ("scala", "Scala"), ("plaintext", "Plain Text"),
+    ("python", "Python"),
+    ("javascript", "JavaScript"),
+    ("typescript", "TypeScript"),
+    ("html", "HTML"),
+    ("css", "CSS"),
+    ("c", "C"),
+    ("cpp", "C++"),
+    ("java", "Java"),
+    ("go", "Go"),
+    ("rust", "Rust"),
+    ("sql", "SQL"),
+    ("bash", "Bash"),
+    ("yaml", "YAML"),
+    ("json", "JSON"),
+    ("markdown", "Markdown"),
+    ("swift", "Swift"),
+    ("php", "PHP"),
+    ("ruby", "Ruby"),
+    ("kotlin", "Kotlin"),
+    ("lua", "Lua"),
+    ("perl", "Perl"),
+    ("haskell", "Haskell"),
+    ("elixir", "Elixir"),
+    ("r", "R"),
+    ("dart", "Dart"),
+    ("scala", "Scala"),
+    ("plaintext", "Plain Text"),
 ]
 
 
-def get_gists_list(user_uid=None, language=None):
+def get_gists_list(user_uid=None, language=None, before=None, viewer=None):
     gists_table = get_table("gists")
     filters = {}
     if user_uid:
@@ -29,111 +75,155 @@ def get_gists_list(user_uid=None, language=None):
     if language:
         filters["language"] = language
 
-    all_gists = list(gists_table.find(**filters, order_by=["-created_at"]))
+    total = gists_table.count(deleted_at=None, **filters)
+    gists, next_cursor = paginate(gists_table, before=before, **filters)
 
-    if not all_gists:
-        return []
+    if not gists:
+        return [], next_cursor, total
 
-    from devplacepy.database import get_users_by_uids
-    uids = [g["user_uid"] for g in all_gists]
-    users_map = get_users_by_uids(uids)
+    users_map = get_users_by_uids([g["user_uid"] for g in gists])
+    return enrich_items(gists, "gist", users_map, user=viewer), next_cursor, total
 
-    result = []
-    for g in all_gists:
-        author = users_map.get(g["user_uid"])
-        result.append({
-            "gist": g,
-            "author": author,
-            "time_ago": time_ago(g["created_at"]),
-        })
-    return result
+
+@router.get("", response_class=HTMLResponse)
+async def gists_page(
+    request: Request, language: str = None, user_uid: str = None, before: str = None
+):
+    user = get_current_user(request)
+    gists_list, next_cursor, total_count = get_gists_list(
+        user_uid, language, before, viewer=user
+    )
+    seo_ctx = list_page_seo(
+        request,
+        title="Gists",
+        description=f"Browse {total_count} code snippets on DevPlace.",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Gists", "url": "/gists"},
+        ],
+        next_url=next_page_url(request, next_cursor),
+    )
+    return respond(
+        request,
+        "gists.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "gists": gists_list,
+            "total_count": total_count,
+            "next_cursor": next_cursor,
+            "current_language": language,
+            "languages": LANGUAGES,
+            "gist_language_codes": get_gist_languages(),
+        },
+        model=GistsOut,
+    )
 
 
 @router.get("/{gist_slug}", response_class=HTMLResponse)
 async def gist_detail(request: Request, gist_slug: str):
     user = get_current_user(request)
-    gists = get_table("gists")
-    gist = resolve_by_slug(gists, gist_slug)
-    if not gist:
-        raise HTTPException(status_code=404, detail="Gist not found")
-
-    from devplacepy.database import get_users_by_uids
-    users_map = get_users_by_uids([gist["user_uid"]])
-    author = users_map.get(gist["user_uid"])
-
-    is_owner = user and user["uid"] == gist["user_uid"]
-
-    ups, downs = get_vote_counts([gist["uid"]])
-    star_count = ups.get(gist["uid"], 0) - downs.get(gist["uid"], 0)
-
-    comments = load_comments("gist", gist["uid"])
-
-    gist_attachments = get_attachments("gist", gist["uid"])
+    detail = load_detail("gists", "gist", gist_slug, user)
+    if not detail:
+        raise not_found("Gist not found")
+    gist = detail["item"]
+    redirect = canonical_redirect("gists", gist, gist_slug)
+    if redirect:
+        return redirect
 
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
         title=gist.get("title", "Gist"),
         description=gist.get("description", "")[:160],
+        og_type="article",
+        og_image=first_image_url(gist, detail["attachments"]),
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
             {"name": "Gists", "url": "/gists"},
-            {"name": gist.get("title", "Gist"), "url": f"/gists/{gist['slug'] or gist['uid']}"},
+            {
+                "name": gist.get("title", "Gist"),
+                "url": f"/gists/{gist['slug'] or gist['uid']}",
+            },
         ],
-        schemas=[website_schema(base)],
+        schemas=[website_schema(base), software_source_code_schema(gist, base)],
+    )
+    return respond(
+        request,
+        "gist_detail.html",
+        detail_context(
+            request,
+            user,
+            detail,
+            "gist",
+            seo_ctx,
+            {
+                "languages": LANGUAGES,
+            },
+        ),
+        model=GistDetailOut,
     )
-    return templates.TemplateResponse("gist_detail.html", {
-        **seo_ctx,
-        "request": request,
-        "user": user,
-        "gist": gist,
-        "author": author,
-        "is_owner": is_owner,
-        "star_count": star_count,
-        "time_ago": time_ago(gist["created_at"]),
-        "comments": comments,
-        "languages": LANGUAGES,
-        "attachments": gist_attachments,
-    })
 
 
 @router.post("/create")
-async def create_gist(request: Request):
+async def create_gist(request: Request, data: Annotated[GistForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    title = form.get("title", "").strip()
-    description = form.get("description", "").strip()
-    source_code = form.get("source_code", "").strip()
-    language = form.get("language", "plaintext")
+    title = data.title.strip()
+    description = data.description.strip()
+    source_code = data.source_code.strip()
+    language = data.language
 
-    if not title:
-        return RedirectResponse(url="/gists", status_code=302)
-    if len(title) > 200:
-        return RedirectResponse(url="/gists", status_code=302)
-    if not source_code:
-        return RedirectResponse(url="/gists", status_code=302)
-    if len(source_code) > 50000:
-        return RedirectResponse(url="/gists", status_code=302)
-    if len(description) > 5000:
-        return RedirectResponse(url="/gists", status_code=302)
-
-    valid_languages = {l[0] for l in LANGUAGES}
+    valid_languages = {lang[0] for lang in LANGUAGES}
     if language not in valid_languages:
         language = "plaintext"
 
-    gists = get_table("gists")
-    uid = generate_uid()
-    gist_slug = make_combined_slug(title, uid)
-    gists.insert({
-        "uid": uid,
-        "user_uid": user["uid"],
-        "title": title,
-        "slug": gist_slug,
-        "description": description or None,
-        "source_code": source_code,
-        "language": language,
-        "stars": 0,
-        "created_at": datetime.utcnow().isoformat(),
-    })
+    uid, gist_slug = create_content_item(
+        "gists",
+        "gist",
+        user,
+        {
+            "title": title,
+            "description": description or None,
+            "source_code": source_code,
+            "language": language,
+        },
+        title,
+        XP_GIST,
+        "First Gist",
+        description or "",
+        data.attachment_uids,
+        request,
+    )
+    url = f"/gists/{gist_slug}"
+    return action_result(request, url, data={"uid": uid, "slug": gist_slug, "url": url})
 
-    attachment_uids = form.getlist("attachment_uids") if hasattr(form, "getlist") else []
+
+@router.post("/edit/{gist_slug}")
+async def edit_gist(
+    request: Request, gist_slug: str, data: Annotated[GistEditForm, Form()]
+):
+    user = require_user(request)
+    language = data.language
+    if language not in {lang[0] for lang in LANGUAGES}:
+        language = "plaintext"
+    return edit_content_item(
+        request,
+        "gists",
+        user,
+        gist_slug,
+        {
+            "title": data.title.strip(),
+            "description": data.description.strip() or None,
+            "source_code": data.source_code.strip(),
+            "language": language,
+        },
+        "/gists",
+        target_type="gist",
+    )
+
+
+@router.post("/delete/{gist_slug}")
+async def delete_gist(request: Request, gist_slug: str):
+    user = require_user(request)
+    return delete_content_item(request, "gists", "gist", user, gist_slug, "/gists")

devplacepy/routers/leaderboard.py

@@ -0,0 +1,61 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+from fastapi import APIRouter, Request
+from fastapi.responses import HTMLResponse
+from devplacepy.database import (
+    get_leaderboard,
+    get_user_rank,
+    get_site_stats,
+    get_top_authors,
+    get_featured_news,
+)
+from devplacepy.utils import get_current_user
+from devplacepy.seo import list_page_seo
+from devplacepy.responses import respond
+from devplacepy.schemas import LeaderboardOut
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+TOP_LIMIT = 50
+
+
+@router.get("", response_class=HTMLResponse)
+async def leaderboard_page(request: Request):
+    entries = get_leaderboard(TOP_LIMIT, 0)
+
+    user = get_current_user(request)
+    user_rank = get_user_rank(user["uid"]) if user else None
+
+    stats = get_site_stats()
+    top_authors = get_top_authors(5)
+    featured_news = get_featured_news(5)
+
+    seo_ctx = list_page_seo(
+        request,
+        title="Leaderboard",
+        description="Top contributors on DevPlace ranked by the stars their posts, projects, and gists have earned.",
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "Leaderboard", "url": "/leaderboard"},
+        ],
+    )
+    return respond(
+        request,
+        "leaderboard.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "entries": entries,
+            "user_rank": user_rank,
+            "total_members": stats["total_members"],
+            "posts_today": stats["posts_today"],
+            "total_projects": stats["total_projects"],
+            "total_gists": stats["total_gists"],
+            "top_authors": top_authors,
+            "featured_news": featured_news,
+        },
+        model=LeaderboardOut,
+    )

devplacepy/routers/media.py

@@ -0,0 +1,76 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+from fastapi import APIRouter, Request
+from devplacepy.database import get_table
+from devplacepy.attachments import soft_delete_attachment, restore_attachment
+from devplacepy.utils import require_user, require_admin, is_admin, not_found
+from devplacepy.responses import action_result, wants_json, json_error
+from devplacepy.services.audit import record as audit
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+def _media_redirect(request: Request, attachment: dict) -> str:
+    referer = request.headers.get("referer", "")
+    if referer:
+        return referer
+    uploader = get_table("users").find_one(uid=attachment.get("user_uid"))
+    if uploader:
+        return f"/profile/{uploader['username']}?tab=media"
+    return "/feed"
+
+
+@router.post("/{uid}/delete")
+async def delete_media(request: Request, uid: str):
+    user = require_user(request)
+    attachment = get_table("attachments").find_one(uid=uid)
+    if not attachment:
+        raise not_found("Media not found")
+    if attachment.get("user_uid") != user["uid"] and not is_admin(user):
+        audit.record(
+            request,
+            "attachment.delete",
+            user=user,
+            result="denied",
+            target_type="attachment",
+            target_uid=uid,
+            target_label=attachment.get("filename"),
+            summary=f"user {user['username']} denied delete of attachment {uid}",
+            links=[audit.attachment_link(uid, attachment.get("filename"))],
+        )
+        if wants_json(request):
+            return json_error(403, "Not allowed")
+        return action_result(request, _media_redirect(request, attachment))
+    soft_delete_attachment(uid, deleted_by=user["uid"])
+    logger.info(f"Media {uid} soft-deleted by {user['username']}")
+    audit.record(
+        request,
+        "attachment.delete",
+        user=user,
+        target_type="attachment",
+        target_uid=uid,
+        target_label=attachment.get("filename"),
+        summary=f"user {user['username']} deleted attachment {attachment.get('filename') or uid}",
+        links=[audit.attachment_link(uid, attachment.get("filename"))],
+    )
+    return action_result(request, _media_redirect(request, attachment))
+
+
+@router.post("/{uid}/restore")
+async def restore_media(request: Request, uid: str):
+    admin = require_admin(request)
+    restore_attachment(uid)
+    logger.info(f"Media {uid} restored by {admin['username']}")
+    audit.record(
+        request,
+        "attachment.restore",
+        user=admin,
+        target_type="attachment",
+        target_uid=uid,
+        target_label=uid,
+        summary=f"admin {admin['username']} restored attachment {uid}",
+        links=[audit.attachment_link(uid)],
+    )
+    return action_result(request, "/admin/media")

devplacepy/routers/messages.py

@@ -1,11 +1,24 @@
+# retoor <retoor@molodetz.nl>
+
 import logging
-from datetime import datetime
-from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
-from devplacepy.database import get_table, db, get_attachments_by_type
-from devplacepy.templating import templates
-from devplacepy.utils import generate_uid, require_user, time_ago, create_mention_notifications
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
+from devplacepy.models import MessageForm
+from fastapi.responses import HTMLResponse, JSONResponse
+from devplacepy.database import get_table, db
+from devplacepy.attachments import get_attachments_batch
+from devplacepy.templating import clear_unread_cache, clear_messages_cache
+from devplacepy.utils import (
+    generate_uid,
+    require_user,
+    time_ago,
+    create_mention_notifications,
+)
 from devplacepy.seo import base_seo_context
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import MessagesOut
+from devplacepy.services.audit import record as audit
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -13,9 +26,8 @@ router = APIRouter()
 
 def get_conversations(user_uid: str):
     messages_table = get_table("messages")
-    raw = (
-        list(messages_table.find(sender_uid=user_uid)) +
-        list(messages_table.find(receiver_uid=user_uid))
+    raw = list(messages_table.find(sender_uid=user_uid)) + list(
+        messages_table.find(receiver_uid=user_uid)
     )
     seen = set()
     all_messages = []
@@ -27,9 +39,14 @@ def get_conversations(user_uid: str):
     conversation_map = {}
     other_uids = set()
     for msg in all_messages:
-        other_uid = msg["receiver_uid"] if msg["sender_uid"] == user_uid else msg["sender_uid"]
+        other_uid = (
+            msg["receiver_uid"] if msg["sender_uid"] == user_uid else msg["sender_uid"]
+        )
         other_uids.add(other_uid)
-        if other_uid not in conversation_map or msg["created_at"] > conversation_map[other_uid]["last_message_at"]:
+        if (
+            other_uid not in conversation_map
+            or msg["created_at"] > conversation_map[other_uid]["last_message_at"]
+        ):
             conversation_map[other_uid] = {
                 "other_user": None,
                 "last_message": msg["content"],
@@ -39,6 +56,7 @@ def get_conversations(user_uid: str):
 
     if other_uids:
         from devplacepy.database import get_users_by_uids
+
         users_map = get_users_by_uids(list(other_uids))
         for uid, conv in conversation_map.items():
             conv["other_user"] = users_map.get(uid)
@@ -53,9 +71,8 @@ def get_conversations(user_uid: str):
 
 def get_conversation_messages(user_uid: str, other_uid: str):
     messages_table = get_table("messages")
-    raw = (
-        list(messages_table.find(sender_uid=user_uid, receiver_uid=other_uid)) +
-        list(messages_table.find(sender_uid=other_uid, receiver_uid=user_uid))
+    raw = list(messages_table.find(sender_uid=user_uid, receiver_uid=other_uid)) + list(
+        messages_table.find(sender_uid=other_uid, receiver_uid=user_uid)
     )
     seen = set()
     msgs = []
@@ -65,26 +82,34 @@ def get_conversation_messages(user_uid: str, other_uid: str):
             msgs.append(m)
     msgs.sort(key=lambda m: m["created_at"])
 
-    for msg in msgs:
-        if msg["receiver_uid"] == user_uid and not msg["read"]:
-            messages_table.update({"id": msg["id"], "read": True}, ["id"])
+    if "messages" in db.tables:
+        with db:
+            db.query(
+                "UPDATE messages SET read = 1 WHERE receiver_uid = :me AND sender_uid = :other AND read = 0",
+                me=user_uid,
+                other=other_uid,
+            )
+        clear_messages_cache(user_uid)
 
     from devplacepy.database import get_users_by_uids
+
     user_ids = list({m["sender_uid"] for m in msgs} | {other_uid})
     users_map = get_users_by_uids(user_ids)
     other_user = users_map.get(other_uid)
 
     result = []
     msg_uids = [m["uid"] for m in msgs]
-    attachments_map = get_attachments_by_type("message", msg_uids) if msg_uids else {}
+    attachments_map = get_attachments_batch("message", msg_uids) if msg_uids else {}
     for m in msgs:
-        result.append({
-            "message": m,
-            "sender": users_map.get(m["sender_uid"]),
-            "is_mine": m["sender_uid"] == user_uid,
-            "time_ago": time_ago(m["created_at"]),
-            "attachments": attachments_map.get(m["uid"], []),
-        })
+        result.append(
+            {
+                "message": m,
+                "sender": users_map.get(m["sender_uid"]),
+                "is_mine": m["sender_uid"] == user_uid,
+                "time_ago": time_ago(m["created_at"]),
+                "attachments": attachments_map.get(m["uid"], []),
+            }
+        )
     return result, other_user
 
 
@@ -105,6 +130,17 @@ async def messages_page(request: Request, with_uid: str = None, search: str = ""
     if with_uid:
         messages, other_user = get_conversation_messages(user["uid"], with_uid)
         current_conversation = with_uid
+        audit.record(
+            request,
+            "message.read_on_view",
+            user=user,
+            target_type="user",
+            target_uid=with_uid,
+            target_label=other_user.get("username") if other_user else None,
+            metadata={"message_count": len(messages)},
+            summary=f"{user['username']} read messages from {other_user.get('username') if other_user else with_uid}",
+            links=[audit.target("user", with_uid, other_user.get("username") if other_user else None)],
+        )
 
     seo_ctx = base_seo_context(
         request,
@@ -115,16 +151,21 @@ async def messages_page(request: Request, with_uid: str = None, search: str = ""
             {"name": "Messages", "url": "/messages"},
         ],
     )
-    return templates.TemplateResponse("messages.html", {
-        **seo_ctx,
-        "request": request,
-        "user": user,
-        "conversations": conversations,
-        "messages": messages,
-        "other_user": other_user,
-        "current_conversation": current_conversation,
-        "search": search,
-    })
+    return respond(
+        request,
+        "messages.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "conversations": conversations,
+            "messages": messages,
+            "other_user": other_user,
+            "current_conversation": current_conversation,
+            "search": search,
+        },
+        model=MessagesOut,
+    )
 
 
 @router.get("/search")
@@ -135,7 +176,8 @@ async def search_users(request: Request, q: str = ""):
     if "users" in db.tables:
         rows = db.query(
             "SELECT uid, username FROM users WHERE username LIKE :q AND uid != :me LIMIT 10",
-            q=f"%{q}%", me=user["uid"],
+            q=f"%{q}%",
+            me=user["uid"],
         )
         results = [{"uid": r["uid"], "username": r["username"]} for r in rows]
     else:
@@ -144,22 +186,66 @@ async def search_users(request: Request, q: str = ""):
 
 
 @router.post("/send")
-async def send_message(request: Request):
+async def send_message(request: Request, data: Annotated[MessageForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    content = form.get("content", "").strip()
-    receiver_uid = form.get("receiver_uid", "")
+    content = data.content.strip()
+    receiver_uid = data.receiver_uid
 
-    if content and receiver_uid:
-        messages_table = get_table("messages")
-        msg_uid = generate_uid()
-        messages_table.insert({
+    if not get_table("users").find_one(uid=receiver_uid):
+        return action_result(request, "/messages")
+
+    messages_table = get_table("messages")
+    msg_uid = generate_uid()
+    messages_table.insert(
+        {
             "uid": msg_uid,
             "sender_uid": user["uid"],
             "receiver_uid": receiver_uid,
             "content": content,
             "read": False,
-            "created_at": datetime.utcnow().isoformat(),
-        })
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+    )
 
-        attachment_uids = form.getlist("attachment_uids") if hasattr(form, "getlist") else []
+    from devplacepy.attachments import link_attachments
+
+    link_attachments(data.attachment_uids, "message", msg_uid)
+
+    if user["uid"] != receiver_uid:
+        notifications = get_table("notifications")
+        notifications.insert(
+            {
+                "uid": generate_uid(),
+                "user_uid": receiver_uid,
+                "type": "message",
+                "message": f"{user['username']} sent you a message",
+                "related_uid": user["uid"],
+                "target_url": f"/messages?with_uid={user['uid']}",
+                "read": False,
+                "created_at": datetime.now(timezone.utc).isoformat(),
+            }
+        )
+        clear_unread_cache(receiver_uid)
+        clear_messages_cache(receiver_uid)
+
+    create_mention_notifications(
+        content, user["uid"], f"/messages?with_uid={receiver_uid}"
+    )
+
+    logger.info(f"Message {msg_uid} sent from {user['username']} to {receiver_uid}")
+    receiver = get_table("users").find_one(uid=receiver_uid)
+    audit.record(
+        request,
+        "message.send",
+        user=user,
+        target_type="message",
+        target_uid=msg_uid,
+        summary=f"{user['username']} sent a message to {receiver.get('username') if receiver else receiver_uid}: {content}",
+        links=[
+            audit.target("message", msg_uid),
+            audit.recipient(receiver_uid, receiver.get("username") if receiver else None),
+        ],
+    )
+    return action_result(
+        request, f"/messages?with_uid={receiver_uid}", data={"uid": msg_uid}
+    )

devplacepy/routers/news.py

@@ -1,11 +1,30 @@
+# retoor <retoor@molodetz.nl>
+
 import logging
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse
-from devplacepy.database import get_table, db, load_comments, resolve_by_slug
-from devplacepy.templating import templates
-from devplacepy.utils import get_current_user, time_ago
-from devplacepy.seo import base_seo_context, website_schema, site_url, discussion_forum_posting, combine
+from devplacepy.database import (
+    get_table,
+    db,
+    load_comments,
+    resolve_by_slug,
+    get_news_images_by_uids,
+    get_user_bookmarks,
+    paginate,
+)
+from devplacepy.utils import get_current_user, time_ago, not_found
+from devplacepy.content import canonical_redirect
+from devplacepy.seo import (
+    base_seo_context,
+    website_schema,
+    site_url,
+    news_article_schema,
+    list_page_seo,
+    next_page_url,
+)
+from devplacepy.responses import respond
+from devplacepy.schemas import NewsListOut, NewsDetailOut
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -14,51 +33,59 @@ NEWS_MAX_AGE_DAYS = 4
 
 
 @router.get("", response_class=HTMLResponse)
-async def news_page(request: Request):
+async def news_page(request: Request, before: str = None):
     user = get_current_user(request)
-    cutoff = (datetime.utcnow() - timedelta(days=NEWS_MAX_AGE_DAYS)).isoformat()
+    cutoff = (
+        datetime.now(timezone.utc) - timedelta(days=NEWS_MAX_AGE_DAYS)
+    ).isoformat()
 
     news_table = get_table("news")
-    articles = list(news_table.find(
+    articles, next_cursor = paginate(
+        news_table,
+        before=before,
+        order=["-grade", "-synced_at"],
+        cursor_field="synced_at",
         status="published",
         synced_at={">=": cutoff},
-        order_by=["-grade", "-synced_at"],
-    ))
+    )
 
     article_uids = [a["uid"] for a in articles]
-
-    images_by_news = {}
-    if article_uids and "news_images" in db.tables:
-        images_table = get_table("news_images")
-        for uid in article_uids:
-            img = images_table.find_one(news_uid=uid, order_by=["uid"])
-            if img:
-                images_by_news[uid] = img["url"]
+    images_by_news = get_news_images_by_uids(article_uids)
 
     enriched = []
     for a in articles:
-        enriched.append({
-            "article": a,
-            "time_ago": time_ago(a["synced_at"]),
-            "image_url": images_by_news.get(a["uid"]),
-            "grade": a.get("grade", 0),
-        })
+        enriched.append(
+            {
+                "article": a,
+                "time_ago": time_ago(a["synced_at"]),
+                "image_url": images_by_news.get(a["uid"]),
+                "grade": a.get("grade", 0),
+            }
+        )
 
-    base = site_url(request)
-    seo_ctx = base_seo_context(
+    seo_ctx = list_page_seo(
         request,
         title="Developer News",
         description="Curated developer news and industry signals. Stay ahead with hand-picked articles.",
-        breadcrumbs=[{"name": "Home", "url": "/feed"}, {"name": "News", "url": "/news"}],
-        schemas=[website_schema(base)],
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "News", "url": "/news"},
+        ],
+        next_url=next_page_url(request, next_cursor),
     )
 
-    return templates.TemplateResponse("news.html", {
-        **seo_ctx,
-        "request": request,
-        "user": user,
-        "articles": enriched,
-    })
+    return respond(
+        request,
+        "news.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "articles": enriched,
+            "next_cursor": next_cursor,
+        },
+        model=NewsListOut,
+    )
 
 
 @router.get("/{news_slug}", response_class=HTMLResponse)
@@ -67,16 +94,24 @@ async def news_detail_page(request: Request, news_slug: str):
     news_table = get_table("news")
     article = resolve_by_slug(news_table, news_slug)
     if not article:
-        return HTMLResponse("News article not found", status_code=404)
+        raise not_found("News article not found")
+    redirect = canonical_redirect("news", article, news_slug)
+    if redirect:
+        return redirect
 
     image_url = ""
     if "news_images" in db.tables:
-        img = get_table("news_images").find_one(news_uid=article["uid"])
+        img = get_table("news_images").find_one(
+            news_uid=article["uid"], deleted_at=None
+        )
         if img:
             image_url = img["url"]
 
     canonical_slug = article.get("slug", "") or article["uid"]
-    comments = load_comments("news", article["uid"])
+    comments = load_comments("news", article["uid"], user)
+    bookmarked = bool(user) and article["uid"] in get_user_bookmarks(
+        user["uid"], "news", [article["uid"]]
+    )
 
     base = site_url(request)
     page_url = f"{base}/news/{canonical_slug}"
@@ -84,18 +119,30 @@ async def news_detail_page(request: Request, news_slug: str):
         request,
         title=article.get("title", "News Article"),
         description=(article.get("description", "") or "")[:200],
-        breadcrumbs=[{"name": "Home", "url": "/feed"}, {"name": "News", "url": "/news"}, {"name": article.get("title", "")[:60], "url": page_url}],
-        schemas=[website_schema(base)],
+        og_type="article",
+        og_image=image_url or None,
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {"name": "News", "url": "/news"},
+            {"name": article.get("title", "")[:60], "url": page_url},
+        ],
+        schemas=[website_schema(base), news_article_schema(article, base, image_url)],
     )
 
-    return templates.TemplateResponse("news_detail.html", {
-        **seo_ctx,
-        "request": request,
-        "user": user,
-        "article": article,
-        "canonical_slug": canonical_slug,
-        "image_url": image_url,
-        "grade": article.get("grade", 0),
-        "time_ago": time_ago(article["synced_at"]),
-        "comments": comments,
-    })
+    return respond(
+        request,
+        "news_detail.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "article": article,
+            "canonical_slug": canonical_slug,
+            "image_url": image_url,
+            "grade": article.get("grade", 0),
+            "time_ago": time_ago(article["synced_at"]),
+            "comments": comments,
+            "bookmarked": bookmarked,
+        },
+        model=NewsDetailOut,
+    )

devplacepy/routers/notifications.py

@@ -1,20 +1,31 @@
+# retoor <retoor@molodetz.nl>
+
 import logging
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
-from devplacepy.database import get_table
-from devplacepy.templating import templates
-from devplacepy.utils import require_user, time_ago
+from fastapi.responses import HTMLResponse, JSONResponse
+from devplacepy.database import get_table, db
+from devplacepy.templating import (
+    clear_unread_cache,
+    jinja_unread_count,
+    jinja_unread_messages,
+)
+from devplacepy.utils import require_user, get_current_user, time_ago
 from devplacepy.seo import base_seo_context
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import NotificationsOut
+from devplacepy.services.audit import record as audit
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
+PAGE_SIZE = 25
+
 
 def _group_label(created_at: str) -> str:
     try:
         dt = datetime.fromisoformat(created_at)
-        now = datetime.utcnow()
+        now = datetime.now(timezone.utc)
         today = now.date()
         date = dt.date()
         if date == today:
@@ -29,28 +40,44 @@ def _group_label(created_at: str) -> str:
 
 
 @router.get("", response_class=HTMLResponse)
-async def notifications_page(request: Request):
+async def notifications_page(request: Request, before: str = None):
     user = require_user(request)
 
+    next_cursor = None
     try:
         notifications_table = get_table("notifications")
+        filters = {"user_uid": user["uid"]}
+        if before:
+            filters["created_at"] = {"<": before}
         raw_notifications = list(
-            notifications_table.find(user_uid=user["uid"], order_by=["-created_at"])
+            notifications_table.find(
+                **filters, order_by=["-created_at"], _limit=PAGE_SIZE + 1
+            )
         )
 
+        has_more = len(raw_notifications) > PAGE_SIZE
+        raw_notifications = raw_notifications[:PAGE_SIZE]
+        if has_more and raw_notifications:
+            next_cursor = raw_notifications[-1]["created_at"]
+
         enriched = []
         if raw_notifications:
             from devplacepy.database import get_users_by_uids
-            actor_uids = [n.get("related_uid") for n in raw_notifications if n.get("related_uid")]
+
+            actor_uids = [
+                n.get("related_uid") for n in raw_notifications if n.get("related_uid")
+            ]
             actors = get_users_by_uids(actor_uids)
         else:
             actors = {}
         for n in raw_notifications:
-            enriched.append({
-                "notification": n,
-                "actor": actors.get(n.get("related_uid")),
-                "time_ago": time_ago(n["created_at"]),
-            })
+            enriched.append(
+                {
+                    "notification": n,
+                    "actor": actors.get(n.get("related_uid")),
+                    "time_ago": time_ago(n["created_at"]),
+                }
+            )
 
         groups = []
         current_label = None
@@ -78,12 +105,53 @@ async def notifications_page(request: Request):
             {"name": "Notifications", "url": "/notifications"},
         ],
     )
-    return templates.TemplateResponse("notifications.html", {
-        **seo_ctx,
-        "request": request,
-        "user": user,
-        "notification_groups": groups,
-    })
+    return respond(
+        request,
+        "notifications.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": user,
+            "notification_groups": groups,
+            "next_cursor": next_cursor,
+        },
+        model=NotificationsOut,
+    )
+
+
+@router.get("/counts")
+async def unread_counts(request: Request):
+    user = get_current_user(request)
+    if not user:
+        return JSONResponse({"notifications": 0, "messages": 0})
+    return JSONResponse(
+        {
+            "notifications": jinja_unread_count(user["uid"]),
+            "messages": jinja_unread_messages(user["uid"]),
+        }
+    )
+
+
+@router.get("/open/{notification_uid}")
+async def open_notification(request: Request, notification_uid: str):
+    user = require_user(request)
+    notifications_table = get_table("notifications")
+    n = notifications_table.find_one(uid=notification_uid)
+    if not n or n["user_uid"] != user["uid"]:
+        return action_result(request, "/notifications")
+    if not n["read"]:
+        notifications_table.update({"id": n["id"], "read": True}, ["id"])
+        clear_unread_cache(user["uid"])
+    audit.record(
+        request,
+        "notification.open",
+        user=user,
+        target_type="notification",
+        target_uid=notification_uid,
+        summary=f"{user['username']} opened a notification",
+        links=[audit.target("notification", notification_uid)],
+    )
+    return action_result(request, n.get("target_url") or "/notifications")
 
 
 @router.post("/mark-read/{notification_uid}")
@@ -93,13 +161,32 @@ async def mark_read(request: Request, notification_uid: str):
     n = notifications_table.find_one(uid=notification_uid)
     if n and n["user_uid"] == user["uid"]:
         notifications_table.update({"id": n["id"], "read": True}, ["id"])
-    return RedirectResponse(url="/notifications", status_code=302)
+        clear_unread_cache(user["uid"])
+        audit.record(
+            request,
+            "notification.read.one",
+            user=user,
+            target_type="notification",
+            target_uid=notification_uid,
+            summary=f"{user['username']} marked a notification read",
+            links=[audit.target("notification", notification_uid)],
+        )
+    return action_result(request, "/notifications")
 
 
 @router.post("/mark-all-read")
 async def mark_all_read(request: Request):
     user = require_user(request)
-    notifications_table = get_table("notifications")
-    for n in notifications_table.find(user_uid=user["uid"], read=False):
-        notifications_table.update({"id": n["id"], "read": True}, ["id"])
-    return RedirectResponse(url="/notifications", status_code=302)
+    with db:
+        db.query(
+            "UPDATE notifications SET read = 1 WHERE user_uid = :u AND read = 0",
+            u=user["uid"],
+        )
+    clear_unread_cache(user["uid"])
+    audit.record(
+        request,
+        "notification.read.all",
+        user=user,
+        summary=f"{user['username']} marked all notifications read",
+    )
+    return action_result(request, "/notifications")

devplacepy/routers/openai_gateway.py

@@ -0,0 +1,25 @@
+# retoor <retoor@molodetz.nl>
+
+import logging
+from fastapi import APIRouter, Request, HTTPException
+from devplacepy.services.manager import service_manager
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+def _service():
+    svc = service_manager.get_service("openai")
+    if svc is None:
+        raise HTTPException(status_code=503, detail="Gateway is not available")
+    return svc
+
+
+@router.post("/v1/chat/completions")
+async def chat_completions(request: Request):
+    return await _service().handle(request, "chat/completions")
+
+
+@router.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"])
+async def passthrough(request: Request, path: str):
+    return await _service().handle(request, path)

devplacepy/routers/polls.py

@@ -0,0 +1,87 @@
+# retoor <retoor@molodetz.nl>
+import logging
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
+from fastapi.responses import RedirectResponse, JSONResponse
+from devplacepy.database import get_table, get_poll_for_post, _now_iso
+from devplacepy.utils import generate_uid, require_user
+from devplacepy.models import PollVoteForm
+from devplacepy.services.audit import record as audit
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+@router.post("/{poll_uid}/vote")
+async def vote_poll(
+    request: Request, poll_uid: str, data: Annotated[PollVoteForm, Form()]
+):
+    user = require_user(request)
+    poll = get_table("polls").find_one(uid=poll_uid, deleted_at=None)
+    if not poll:
+        return JSONResponse({"error": "Poll not found"}, status_code=404)
+    option = get_table("poll_options").find_one(
+        uid=data.option_uid, poll_uid=poll_uid, deleted_at=None
+    )
+    if not option:
+        return JSONResponse({"error": "Invalid option"}, status_code=400)
+
+    poll_votes = get_table("poll_votes")
+    existing = poll_votes.find_one(poll_uid=poll_uid, user_uid=user["uid"])
+    if existing:
+        if existing.get("deleted_at"):
+            poll_votes.update(
+                {
+                    "id": existing["id"],
+                    "option_uid": data.option_uid,
+                    "deleted_at": None,
+                    "deleted_by": None,
+                },
+                ["id"],
+            )
+            poll_event = "poll.vote.cast"
+        elif existing["option_uid"] == data.option_uid:
+            poll_votes.update(
+                {"id": existing["id"], "deleted_at": _now_iso(), "deleted_by": user["uid"]},
+                ["id"],
+            )
+            poll_event = "poll.vote.clear"
+        else:
+            poll_votes.update(
+                {"id": existing["id"], "option_uid": data.option_uid}, ["id"]
+            )
+            poll_event = "poll.vote.change"
+    else:
+        poll_votes.insert(
+            {
+                "uid": generate_uid(),
+                "poll_uid": poll_uid,
+                "option_uid": data.option_uid,
+                "user_uid": user["uid"],
+                "created_at": datetime.now(timezone.utc).isoformat(),
+                "deleted_at": None,
+                "deleted_by": None,
+            }
+        )
+        poll_event = "poll.vote.cast"
+
+    audit.record(
+        request,
+        poll_event,
+        user=user,
+        target_type="poll",
+        target_uid=poll_uid,
+        old_value=existing["option_uid"] if existing else None,
+        new_value=None if poll_event == "poll.vote.clear" else data.option_uid,
+        metadata={"question": poll.get("question")},
+        summary=f"{user['username']} {poll_event.split('.')[-1]} vote in poll {poll.get('question')}",
+        links=[audit.poll(poll_uid, poll.get("question")), audit.option(data.option_uid, option.get("text"))],
+    )
+
+    result = get_poll_for_post(poll["post_uid"], user)
+    if request.headers.get("x-requested-with") == "fetch":
+        return JSONResponse(result)
+    return RedirectResponse(
+        url=request.headers.get("Referer", "/feed"), status_code=302
+    )

devplacepy/routers/posts.py

@@ -1,124 +1,156 @@
+# retoor <retoor@molodetz.nl>
+
 import logging
-import aiofiles
-from datetime import datetime
-from pathlib import Path
-from fastapi import APIRouter, Request, HTTPException
-from fastapi.responses import RedirectResponse, HTMLResponse
-from devplacepy.database import get_table, get_comment_counts_by_post_uids, load_comments, db
-from devplacepy.templating import templates
-from devplacepy.config import STATIC_DIR
-from devplacepy.utils import generate_uid, get_current_user, require_user, time_ago, make_combined_slug, create_mention_notifications
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, discussion_forum_posting, combine, truncate
+from typing import Annotated
+from datetime import datetime, timezone
+from fastapi import APIRouter, Request, Form
+from fastapi.responses import HTMLResponse
+from devplacepy.constants import TOPICS
+from devplacepy.database import db, get_table, resolve_by_slug
+from devplacepy.utils import (
+    get_current_user,
+    require_user,
+    time_ago,
+    not_found,
+    generate_uid,
+    XP_POST,
+)
+from devplacepy.content import (
+    load_detail,
+    edit_content_item,
+    delete_content_item,
+    create_content_item,
+    detail_context,
+    canonical_redirect,
+    first_image_url,
+)
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import PostDetailOut
+from devplacepy.seo import (
+    base_seo_context,
+    site_url,
+    website_schema,
+    discussion_forum_posting,
+    truncate,
+)
+from devplacepy.attachments import save_inline_image
+from devplacepy.models import PostForm, PostEditForm
+from devplacepy.services.audit import record as audit
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
 @router.post("/create")
-async def create_post(request: Request):
+async def create_post(request: Request, data: Annotated[PostForm, Form()]):
     user = require_user(request)
-    form = await request.form()
-    content = form.get("content", "").strip()
-    title = form.get("title", "").strip()
-    topic = form.get("topic", "random")
-    project_uid = form.get("project_uid", "")
-
-    errors = []
-    if not content:
-        errors.append("Content is required")
-    if content and len(content) < 10:
-        errors.append("Content must be at least 10 characters")
-    if len(content) > 2000:
-        errors.append("Content too long (max 2000 characters)")
-    if topic not in TOPICS:
-        topic = "random"
-
-    if errors:
-        return RedirectResponse(url="/feed", status_code=302)
+    content = data.content.strip()
+    title = data.title.strip()
+    topic = data.topic
+    project_uid = data.project_uid
 
     image_filename = None
+    form = await request.form()
     image_file = form.get("image")
-    if image_file and hasattr(image_file, "filename") and image_file.filename:
-        try:
-            content_bytes = await image_file.read()
-            if len(content_bytes) > 5 * 1024 * 1024:
-                logger.warning(f"Image too large: {image_file.filename}")
-            else:
-                import imghdr
-                ext = Path(image_file.filename).suffix.lower()
-                allowed = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg"}
-                if ext not in allowed:
-                    logger.warning(f"Unsupported image type: {ext}")
-                else:
-                    upload_dir = STATIC_DIR / "uploads"
-                    upload_dir.mkdir(parents=True, exist_ok=True)
-                    image_filename = f"{generate_uid()}{ext}"
-                    file_path = upload_dir / image_filename
-                    async with aiofiles.open(str(file_path), "wb") as f:
-                        await f.write(content_bytes)
-                    logger.info(f"Image saved: {image_filename}")
-                    content += f"\n\n![](/static/uploads/{image_filename})"
-        except Exception as e:
-            logger.warning(f"Image upload failed: {e}")
+    if (
+        image_file is not None
+        and hasattr(image_file, "filename")
+        and image_file.filename
+    ):
+        image_filename = save_inline_image(await image_file.read(), image_file.filename)
+        if image_filename:
+            content += f"\n\n![](/static/uploads/{image_filename})"
 
-    posts = get_table("posts")
-    uid = generate_uid()
     slug_text = title if title else content[:50]
-    post_slug = make_combined_slug(slug_text, uid)
-    posts.insert({
-        "uid": uid,
-        "user_uid": user["uid"],
-        "title": title or None,
-        "slug": post_slug,
-        "content": content,
-        "topic": topic,
-        "project_uid": project_uid or None,
-        "image": image_filename,
-        "stars": 0,
-        "created_at": datetime.utcnow().isoformat(),
-    })
+    uid, post_slug = create_content_item(
+        "posts",
+        "post",
+        user,
+        {
+            "title": title or None,
+            "content": content,
+            "topic": topic,
+            "project_uid": project_uid or None,
+            "image": image_filename,
+        },
+        slug_text,
+        XP_POST,
+        "First Post",
+        content,
+        data.attachment_uids,
+        request,
+    )
 
-    badges = get_table("badges")
-    existing = badges.find_one(user_uid=user["uid"], badge_name="First Post")
-    if not existing:
-        badges.insert({
-            "uid": generate_uid(),
+    create_poll(uid, user, data.poll_question, data.poll_options, request)
+    url = f"/posts/{post_slug}"
+    return action_result(request, url, data={"uid": uid, "slug": post_slug, "url": url})
+
+
+def create_poll(
+    post_uid: str, user: dict, question: str, options: list[str], request=None
+) -> None:
+    question = question.strip()
+    labels = [option.strip() for option in options if option.strip()][:6]
+    if not question or len(labels) < 2:
+        return
+    poll_uid = generate_uid()
+    get_table("polls").insert(
+        {
+            "uid": poll_uid,
+            "post_uid": post_uid,
             "user_uid": user["uid"],
-            "badge_name": "First Post",
-            "created_at": datetime.utcnow().isoformat(),
-        })
-
-    attachment_uids = form.getlist("attachment_uids")
-    if attachment_uids:
-        link_attachments(attachment_uids, "post", uid)
-    if attachment_uids:
-        link_attachments(attachment_uids, "post", uid)
-
-    create_mention_notifications(content, user["uid"], f"/posts/{post_slug}")
-    logger.info(f"Post {uid} created by {user['username']}")
-    return RedirectResponse(url=f"/posts/{post_slug}", status_code=302)
+            "question": question,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+            "deleted_at": None,
+            "deleted_by": None,
+        }
+    )
+    get_table("poll_options").insert_many(
+        [
+            {
+                "uid": generate_uid(),
+                "poll_uid": poll_uid,
+                "label": label,
+                "position": index,
+                "deleted_at": None,
+                "deleted_by": None,
+            }
+            for index, label in enumerate(labels)
+        ]
+    )
+    audit.record(
+        request,
+        "poll.create",
+        user=user,
+        target_type="poll",
+        target_uid=poll_uid,
+        target_label=question,
+        metadata={"option_count": len(labels), "question": question},
+        summary=f"{user['username']} added a poll to post {post_uid}",
+        links=[audit.poll(poll_uid, question), audit.parent("post", post_uid)],
+    )
 
 
 @router.get("/{post_slug}", response_class=HTMLResponse)
 async def view_post(request: Request, post_slug: str):
     user = get_current_user(request)
-    posts = get_table("posts")
-    post = resolve_by_slug(posts, post_slug)
-    if not post:
-        raise HTTPException(status_code=404, detail="Post not found")
-
-    users_table = get_table("users")
-    author = users_table.find_one(uid=post["user_uid"])
-
-    top_level = load_comments("post", post["uid"])
+    detail = load_detail("posts", "post", post_slug, user)
+    if not detail:
+        raise not_found("Post not found")
+    post = detail["item"]
+    redirect = canonical_redirect("posts", post, post_slug)
+    if redirect:
+        return redirect
+    author = detail["author"]
+    top_level = detail["comments"]
 
     def count_all(items):
         total = len(items)
         for item in items:
             total += count_all(item.get("children", []))
         return total
+
     comment_count = count_all(top_level)
-    star_count = post.get("stars", 0)
     base = site_url(request)
     seo_ctx = base_seo_context(
         request,
@@ -127,98 +159,92 @@ async def view_post(request: Request, post_slug: str):
         breadcrumbs=[
             {"name": "Home", "url": "/feed"},
             {"name": "Feed", "url": "/feed"},
-            {"name": post.get("title") or "Post", "url": f"/posts/{post['slug'] or post['uid']}"},
+            {
+                "name": post.get("title") or "Post",
+                "url": f"/posts/{post['slug'] or post['uid']}",
+            },
         ],
         og_type="article",
+        og_image=first_image_url(post, detail["attachments"]),
         schemas=[
             website_schema(base),
-            discussion_forum_posting(post, author, comment_count, star_count, base),
+            discussion_forum_posting(
+                post, author, comment_count, detail["star_count"], base
+            ),
         ],
     )
 
     related_posts = []
     if "posts" in db.tables:
-        rows = list(db.query(
-            "SELECT p.*, u.username FROM posts p JOIN users u ON p.user_uid = u.uid WHERE p.topic = :t AND p.uid != :uid ORDER BY p.created_at DESC LIMIT 5",
-            t=post.get("topic", ""), uid=post["uid"],
-        ))
+        rows = list(
+            db.query(
+                "SELECT p.*, u.username FROM posts p JOIN users u ON p.user_uid = u.uid WHERE p.topic = :t AND p.uid != :uid AND p.deleted_at IS NULL ORDER BY p.created_at DESC LIMIT 5",
+                t=post.get("topic", ""),
+                uid=post["uid"],
+            )
+        )
         for r in rows:
-            related_posts.append({
-                "post": r,
-                "author": {"username": r["username"]},
-                "time_ago": time_ago(r["created_at"]),
-            })
+            related_posts.append(
+                {
+                    "post": r,
+                    "author": {"username": r["username"]},
+                    "time_ago": time_ago(r["created_at"]),
+                }
+            )
 
-    post_attachments = get_attachments("post", post["uid"])
-
-    return templates.TemplateResponse("post.html", {
-        **seo_ctx,
-        "request": request,
-        "user": user,
-        "post": post,
-        "author": author,
-        "comments": top_level,
-        "time_ago": time_ago(post["created_at"]),
-        "comment_count": comment_count,
-        "related_posts": related_posts,
-        "topics": list(TOPICS),
-        "attachments": post_attachments,
-    })
+    return respond(
+        request,
+        "post.html",
+        detail_context(
+            request,
+            user,
+            detail,
+            "post",
+            seo_ctx,
+            {
+                "comment_count": comment_count,
+                "related_posts": related_posts,
+                "topics": list(TOPICS),
+            },
+        ),
+        model=PostDetailOut,
+    )
 
 
 @router.post("/edit/{post_slug}")
-async def edit_post(request: Request, post_slug: str):
+async def edit_post(
+    request: Request, post_slug: str, data: Annotated[PostEditForm, Form()]
+):
     user = require_user(request)
-    form = await request.form()
-    content = form.get("content", "").strip()
-    title = form.get("title", "").strip()
-    topic = form.get("topic", "random")
-
-    errors = []
-    if not content:
-        errors.append("Content is required")
-    if content and len(content) < 10:
-        errors.append("Content must be at least 10 characters")
-    if len(content) > 2000:
-        errors.append("Content too long (max 2000 characters)")
-    if topic not in TOPICS:
-        topic = "random"
-
-    posts = get_table("posts")
-    post = resolve_by_slug(posts, post_slug)
-    if not post or post["user_uid"] != user["uid"]:
-        return RedirectResponse(url="/feed", status_code=302)
-
-    if errors:
-        return RedirectResponse(url=f"/posts/{post['slug'] or post['uid']}", status_code=302)
-
-    posts.update({
-        "uid": post["uid"],
-        "content": content,
-        "title": title or None,
-        "topic": topic,
-    }, ["uid"])
-
-    logger.info(f"Post {post['uid']} edited by {user['username']}")
-    return RedirectResponse(url=f"/posts/{post['slug'] or post['uid']}", status_code=302)
+    result = edit_content_item(
+        request,
+        "posts",
+        user,
+        post_slug,
+        {
+            "content": data.content.strip(),
+            "title": data.title.strip() or None,
+            "topic": data.topic,
+        },
+        "/feed",
+        target_type="post",
+    )
+    if data.poll_question.strip():
+        post = resolve_by_slug(get_table("posts"), post_slug)
+        if (
+            post
+            and post["user_uid"] == user["uid"]
+            and not get_table("polls").find_one(post_uid=post["uid"], deleted_at=None)
+        ):
+            create_poll(
+                post["uid"], user, data.poll_question, data.poll_options, request
+            )
+    return result
 
 
 @router.post("/delete/{post_slug}")
 async def delete_post(request: Request, post_slug: str):
     user = require_user(request)
-    posts = get_table("posts")
-    post = resolve_by_slug(posts, post_slug)
-    if post and post["user_uid"] == user["uid"]:
-        delete_target_attachments("post", post["uid"])
-        get_table("comments").delete(post_uid=post["uid"])
-        get_table("votes").delete(target_uid=post["uid"])
-        image = post.get("image")
-        if image:
-            try:
-                (STATIC_DIR / "uploads" / image).unlink(missing_ok=True)
-            except Exception as e:
-                logger.warning(f"Failed to delete image {image}: {e}")
-        posts.delete(id=post["id"])
-        logger.info(f"Post {post['uid']} deleted by {user['username']}")
-    return RedirectResponse(url="/feed", status_code=302)
-from devplacepy.attachments import get_attachments, link_attachments, delete_target_attachments
+    return delete_content_item(
+        request, "posts", "post", user, post_slug, "/feed", inline_image_field="image"
+    )

devplacepy/routers/profile.py

@@ -1,97 +1,79 @@
+# retoor <retoor@molodetz.nl>
+
 import logging
-from fastapi import APIRouter, Request
-from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
-from devplacepy.database import get_table, db
-from devplacepy.templating import templates
-from devplacepy.utils import get_current_user, require_user, time_ago
-from devplacepy.seo import base_seo_context, site_url, website_schema, breadcrumb_schema, profile_page_schema, combine
+from typing import Annotated
+from fastapi import APIRouter, Request, Form
+from devplacepy.models import ProfileForm, CustomizationToggleForm
+from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
+from devplacepy.database import (
+    get_table,
+    db,
+    get_customization_prefs,
+    set_customization_pref,
+    get_user_stars,
+    get_user_rank,
+    get_comment_counts_by_post_uids,
+    get_reactions_by_targets,
+    get_user_bookmarks,
+    get_polls_by_post_uids,
+    get_activity_heatmap,
+    get_activity_months,
+    get_streaks,
+    get_follow_counts,
+    get_follow_list,
+    get_following_among,
+    get_user_media,
+)
+from devplacepy.content import can_view_project, enrich_items
+from devplacepy.utils import (
+    get_current_user,
+    require_user,
+    require_user_api,
+    time_ago,
+    clear_user_cache,
+    not_found,
+    generate_uid,
+)
+from devplacepy.responses import respond, action_result
+from devplacepy.schemas import ProfileOut
+from devplacepy.avatar import avatar_url
+from devplacepy.seo import (
+    base_seo_context,
+    site_url,
+    website_schema,
+    profile_page_schema,
+)
+from devplacepy.services.manager import service_manager
+from devplacepy.services.audit import record as audit
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-@router.get("/{username}", response_class=HTMLResponse)
-async def profile_page(request: Request, username: str, tab: str = "posts"):
-    current_user = get_current_user(request)
-    users = get_table("users")
-    profile_user = users.find_one(username=username)
-    if not profile_user:
-        return RedirectResponse(url="/feed", status_code=302)
-
-    posts = []
-    if tab == "posts":
-        posts_table = get_table("posts")
-        raw_posts = list(posts_table.find(user_uid=profile_user["uid"], order_by=["-created_at"]))
-        if raw_posts:
-            from devplacepy.database import get_comment_counts_by_post_uids
-            counts = get_comment_counts_by_post_uids([p["uid"] for p in raw_posts])
-        else:
-            counts = {}
-        for p in raw_posts:
-            posts.append({
-                "post": p,
-                "time_ago": time_ago(p["created_at"]),
-                "comment_count": counts.get(p["uid"], 0),
-            })
-
-    badges = list(get_table("badges").find(user_uid=profile_user["uid"]))
-    projects = list(get_table("projects").find(user_uid=profile_user["uid"]))
-    gists = list(get_table("gists").find(user_uid=profile_user["uid"]))
-    posts_count = len(posts) or len(list(get_table("posts").find(user_uid=profile_user["uid"])))
-
-    activities = []
-    if tab == "activity":
-        posts_table = get_table("posts")
-        for p in posts_table.find(user_uid=profile_user["uid"], order_by=["-created_at"], _limit=10):
-            activities.append({"type": "post", "content": p.get("title") or p["content"][:80], "time_ago": time_ago(p["created_at"]), "created_at": p["created_at"], "uid": p["uid"]})
-        comments_table = get_table("comments")
-        for c in comments_table.find(user_uid=profile_user["uid"], order_by=["-created_at"], _limit=10):
-            activities.append({"type": "comment", "content": c["content"][:80], "time_ago": time_ago(c["created_at"]), "created_at": c["created_at"], "uid": c["post_uid"]})
-        activities.sort(key=lambda a: a["created_at"], reverse=True)
-
-    is_following = False
-    if current_user:
-        follows = get_table("follows")
-        is_following = bool(follows.find_one(follower_uid=current_user["uid"], following_uid=profile_user["uid"]))
-
-    base = site_url(request)
-    robots = "noindex,follow" if posts_count < 2 else "index,follow"
-    bio = profile_user.get("bio", "")
-    desc = bio or f"View {profile_user['username']}'s profile on DevPlace. {posts_count} posts."
-    seo_ctx = base_seo_context(
-        request,
-        title=f"{profile_user['username']} (@{profile_user['username']})",
-        description=desc,
-        robots=robots,
-        breadcrumbs=[
-            {"name": "Home", "url": "/feed"},
-            {"name": profile_user['username'], "url": f"/profile/{profile_user['username']}"},
-        ],
-        schemas=[
-            website_schema(base),
-            profile_page_schema(profile_user, posts_count, base),
-        ],
-    )
-
-    return templates.TemplateResponse("profile.html", {
-        **seo_ctx,
-        "request": request,
-        "user": current_user,
-        "profile_user": profile_user,
-        "posts": posts,
-        "badges": badges,
-        "projects": projects,
-        "gists": gists,
-        "current_tab": tab,
-        "posts_count": posts_count,
-        "is_following": is_following,
-        "activities": activities,
-    })
+def _ai_quota(
+    user_uid: str, include_cost: bool = False, is_admin: bool = False
+) -> dict | None:
+    devii = service_manager.get_service("devii")
+    if devii is None:
+        return None
+    try:
+        limit = devii.daily_limit_for("user", is_admin)
+        spent = devii.spent_24h("user", user_uid)
+        turns = devii.hub().ledger.turns_24h("user", user_uid)
+    except Exception:
+        logger.exception("Failed to compute AI quota for %s", user_uid)
+        return None
+    used_pct = round(min(100.0, spent / limit * 100), 1) if limit > 0 else 0.0
+    quota = {"used_pct": used_pct, "turns": turns}
+    if include_cost:
+        quota["spent_usd"] = round(spent, 4)
+        quota["limit_usd"] = round(limit, 2)
+    return quota
 
 
 @router.get("/search")
 async def search_users(request: Request, q: str = ""):
-    require_user(request)
+    require_user_api(request)
     if not q or len(q) < 1:
         return JSONResponse({"results": []})
     if "users" in db.tables:
@@ -105,22 +87,366 @@ async def search_users(request: Request, q: str = ""):
     return JSONResponse({"results": results})
 
 
-@router.post("/update")
-async def update_profile(request: Request):
-    user = require_user(request)
-    form = await request.form()
-    bio = form.get("bio", "").strip()
-    location = form.get("location", "").strip()
-    git_link = form.get("git_link", "").strip()
-    website = form.get("website", "").strip()
+def _follow_people(profile_uid: str, mode: str, current_user, page: int):
+    people, pagination = get_follow_list(profile_uid, mode, page)
+    mine = (
+        get_following_among(current_user["uid"], [p["uid"] for p in people])
+        if current_user
+        else set()
+    )
+    for person in people:
+        person["is_following"] = person["uid"] in mine
+        person["is_self"] = bool(current_user and current_user["uid"] == person["uid"])
+    return people, pagination
+
+
+@router.get("/{username}/followers")
+async def profile_followers(request: Request, username: str, page: int = 1):
+    return _follow_list_json(request, username, "followers", page)
+
+
+@router.get("/{username}/following")
+async def profile_following(request: Request, username: str, page: int = 1):
+    return _follow_list_json(request, username, "following", page)
+
+
+def _follow_list_json(request: Request, username: str, mode: str, page: int):
+    profile_user = get_table("users").find_one(username=username)
+    if not profile_user:
+        raise not_found("Profile not found")
+    current_user = get_current_user(request)
+    people, pagination = _follow_people(profile_user["uid"], mode, current_user, page)
+    return JSONResponse(
+        {
+            "username": profile_user["username"],
+            "mode": mode,
+            "count": pagination["total"],
+            "page": pagination["page"],
+            "total_pages": pagination["total_pages"],
+            mode: [
+                {
+                    "uid": p["uid"],
+                    "username": p["username"],
+                    "bio": p["bio"],
+                    "is_following": p["is_following"],
+                }
+                for p in people
+            ],
+        }
+    )
+
+
+@router.get("/{username}", response_class=HTMLResponse)
+async def profile_page(
+    request: Request, username: str, tab: str = "posts", page: int = 1
+):
+    current_user = get_current_user(request)
     users = get_table("users")
-    users.update({
-        "uid": user["uid"],
-        "bio": bio,
-        "location": location,
-        "git_link": git_link,
-        "website": website,
-    }, ["uid"])
+    profile_user = users.find_one(username=username)
+    if not profile_user:
+        raise not_found("Profile not found")
+    profile_user["stars"] = get_user_stars(profile_user["uid"])
+    rank = get_user_rank(profile_user["uid"])
+    follow_counts = get_follow_counts(profile_user["uid"])
+
+    people = []
+    follow_pagination = None
+    if tab in ("followers", "following"):
+        people, follow_pagination = _follow_people(
+            profile_user["uid"], tab, current_user, page
+        )
+
+    media, media_pagination = [], None
+    if tab == "media":
+        media, media_pagination = get_user_media(profile_user["uid"], page)
+
+    posts = []
+    if tab == "posts":
+        posts_table = get_table("posts")
+        raw_posts = list(
+            posts_table.find(
+                user_uid=profile_user["uid"], deleted_at=None, order_by=["-created_at"]
+            )
+        )
+        post_uids = [p["uid"] for p in raw_posts]
+        counts = get_comment_counts_by_post_uids(post_uids) if raw_posts else {}
+        authors = {profile_user["uid"]: profile_user}
+        posts = enrich_items(
+            raw_posts, "post", authors, {"comment_count": counts}, user=current_user
+        )
+        reactions_map = get_reactions_by_targets("post", post_uids, current_user)
+        bookmark_set = (
+            get_user_bookmarks(current_user["uid"], "post", post_uids)
+            if current_user
+            else set()
+        )
+        polls_map = get_polls_by_post_uids(post_uids, current_user)
+        for item in posts:
+            uid = item["post"]["uid"]
+            item["reactions"] = reactions_map.get(uid, {"counts": {}, "mine": []})
+            item["bookmarked"] = uid in bookmark_set
+            item["poll"] = polls_map.get(uid)
+
+    badges = list(get_table("badges").find(user_uid=profile_user["uid"]))
+    projects = list(
+        get_table("projects").find(user_uid=profile_user["uid"], deleted_at=None)
+    )
+    projects = [p for p in projects if can_view_project(p, current_user)]
+    gists_raw = list(
+        get_table("gists").find(user_uid=profile_user["uid"], deleted_at=None)
+    )
+    gists = []
+    for g in gists_raw:
+        gists.append({"gist": g, "time_ago": time_ago(g["created_at"])})
+    posts_count = len(posts) or get_table("posts").count(
+        user_uid=profile_user["uid"], deleted_at=None
+    )
+
+    activities = []
+    if tab == "activity":
+        posts_table = get_table("posts")
+        for p in posts_table.find(
+            user_uid=profile_user["uid"],
+            deleted_at=None,
+            order_by=["-created_at"],
+            _limit=10,
+        ):
+            activities.append(
+                {
+                    "type": "post",
+                    "content": p.get("title") or p["content"][:80],
+                    "time_ago": time_ago(p["created_at"]),
+                    "created_at": p["created_at"],
+                    "uid": p["uid"],
+                }
+            )
+        comments_table = get_table("comments")
+        for c in comments_table.find(
+            user_uid=profile_user["uid"],
+            deleted_at=None,
+            order_by=["-created_at"],
+            _limit=10,
+        ):
+            target_uid = c.get("target_uid", c.get("post_uid", ""))
+            target_type = c.get("target_type", "post")
+            activities.append(
+                {
+                    "type": "comment",
+                    "content": c["content"][:80],
+                    "time_ago": time_ago(c["created_at"]),
+                    "created_at": c["created_at"],
+                    "uid": target_uid,
+                    "target_type": target_type,
+                }
+            )
+        activities.sort(key=lambda a: a["created_at"], reverse=True)
+
+    heatmap = get_activity_heatmap(profile_user["uid"])
+    heatmap_months = get_activity_months(heatmap)
+    streak = get_streaks(profile_user["uid"])
+
+    is_following = False
+    if current_user:
+        follows = get_table("follows")
+        is_following = bool(
+            follows.find_one(
+                follower_uid=current_user["uid"],
+                following_uid=profile_user["uid"],
+                deleted_at=None,
+            )
+        )
+
+    is_owner = bool(current_user and current_user["uid"] == profile_user["uid"])
+    viewer_is_admin = bool(current_user and current_user.get("role") == "Admin")
+    can_view_api_key = bool(current_user and (is_owner or viewer_is_admin))
+    api_key = profile_user.get("api_key", "") if can_view_api_key else ""
+    profile_is_admin = profile_user.get("role") == "Admin"
+    ai_quota = (
+        _ai_quota(
+            profile_user["uid"], include_cost=viewer_is_admin, is_admin=profile_is_admin
+        )
+        if (is_owner or viewer_is_admin)
+        else None
+    )
+    can_manage_customization = is_owner or viewer_is_admin
+    customization_prefs = (
+        get_customization_prefs("user", profile_user["uid"])
+        if can_manage_customization
+        else {"disable_global": False, "disable_pagetype": False}
+    )
+
+    base = site_url(request)
+    robots = "noindex,follow" if posts_count < 2 else "index,follow"
+    bio = profile_user.get("bio", "")
+    desc = (
+        bio
+        or f"View {profile_user['username']}'s profile on DevPlace. {posts_count} posts."
+    )
+    seo_ctx = base_seo_context(
+        request,
+        title=f"{profile_user['username']} (@{profile_user['username']})",
+        description=desc,
+        robots=robots,
+        og_type="profile",
+        og_image=avatar_url("multiavatar", profile_user["username"], 256),
+        breadcrumbs=[
+            {"name": "Home", "url": "/feed"},
+            {
+                "name": profile_user["username"],
+                "url": f"/profile/{profile_user['username']}",
+            },
+        ],
+        schemas=[
+            website_schema(base),
+            profile_page_schema(profile_user, posts_count, base),
+        ],
+    )
+
+    return respond(
+        request,
+        "profile.html",
+        {
+            **seo_ctx,
+            "request": request,
+            "user": current_user,
+            "profile_user": profile_user,
+            "posts": posts,
+            "badges": badges,
+            "projects": projects,
+            "gists": gists,
+            "current_tab": tab,
+            "posts_count": posts_count,
+            "is_following": is_following,
+            "is_owner": is_owner,
+            "viewer_is_admin": viewer_is_admin,
+            "media": media,
+            "media_pagination": media_pagination,
+            "can_view_api_key": can_view_api_key,
+            "api_key": api_key,
+            "can_manage_customization": can_manage_customization,
+            "cust_disable_global": customization_prefs["disable_global"],
+            "cust_disable_pagetype": customization_prefs["disable_pagetype"],
+            "ai_quota": ai_quota,
+            "activities": activities,
+            "rank": rank,
+            "heatmap": heatmap,
+            "heatmap_months": heatmap_months,
+            "streak": streak,
+            "people": people,
+            "follow_pagination": follow_pagination,
+            "followers_count": follow_counts["followers"],
+            "following_count": follow_counts["following"],
+        },
+        model=ProfileOut,
+    )
+
+
+@router.post("/update")
+async def update_profile(request: Request, data: Annotated[ProfileForm, Form()]):
+    user = require_user(request)
+    users = get_table("users")
+    users.update(
+        {
+            "uid": user["uid"],
+            "bio": data.bio.strip(),
+            "location": data.location.strip(),
+            "git_link": data.git_link.strip(),
+            "website": data.website.strip(),
+        },
+        ["uid"],
+    )
+    clear_user_cache(user["uid"])
 
     logger.info(f"Profile updated for {user['username']}")
-    return RedirectResponse(url=f"/profile/{user['username']}", status_code=302)
+    audit.record(
+        request,
+        "profile.update",
+        user=user,
+        target_type="user",
+        target_uid=user["uid"],
+        target_label=user["username"],
+        metadata={"fields": ["bio", "location", "git_link", "website"]},
+        summary=f"{user['username']} updated their profile",
+        links=[audit.target("user", user["uid"], user["username"])],
+    )
+    return action_result(request, f"/profile/{user['username']}")
+
+
+@router.post("/regenerate-api-key")
+async def regenerate_api_key(request: Request):
+    user = require_user(request)
+    new_key = generate_uid()
+    get_table("users").update({"uid": user["uid"], "api_key": new_key}, ["uid"])
+    clear_user_cache(user["uid"])
+    logger.info(f"API key regenerated for {user['username']}")
+    audit.record(
+        request,
+        "profile.api_key.regenerate",
+        user=user,
+        target_type="user",
+        target_uid=user["uid"],
+        target_label=user["username"],
+        summary=f"{user['username']} regenerated their API key",
+        links=[audit.target("user", user["uid"], user["username"])],
+    )
+    return JSONResponse({"api_key": new_key})
+
+
+def _resolve_customization_target(request: Request, username: str):
+    from devplacepy.responses import wants_json, json_error
+
+    current_user = require_user(request)
+    target = get_table("users").find_one(username=username)
+    if not target:
+        raise not_found("User not found")
+    is_owner = current_user["uid"] == target["uid"]
+    is_admin = current_user.get("role") == "Admin"
+    if not (is_owner or is_admin):
+        if wants_json(request):
+            return None, json_error(403, "Not allowed")
+        return None, RedirectResponse(url=f"/profile/{username}", status_code=302)
+    return target, None
+
+
+def _toggle_customization(
+    request: Request, username: str, category: str, value: bool
+):
+    target, denied = _resolve_customization_target(request, username)
+    if denied is not None:
+        return denied
+    set_customization_pref(target["uid"], category, disabled=not value)
+    column = "cust_disable_global" if category == "global" else "cust_disable_pagetype"
+    logger.info(
+        f"Customization {category} {'enabled' if value else 'disabled'} for {target['username']}"
+    )
+    audit.record(
+        request,
+        f"profile.customization.{category}.toggle",
+        target_type="user",
+        target_uid=target["uid"],
+        target_label=target["username"],
+        old_value=0 if value else 1,
+        new_value=1 if value else 0,
+        summary=f"{'enabled' if value else 'disabled'} {category} customizations for {target['username']}",
+        links=[audit.target("user", target["uid"], target["username"])],
+    )
+    url = f"/profile/{target['username']}"
+    return action_result(request, url, data={"url": url, column: not value})
+
+
+@router.post("/{username}/customization/global")
+async def set_customization_global(
+    request: Request,
+    username: str,
+    data: Annotated[CustomizationToggleForm, Form()],
+):
+    return _toggle_customization(request, username, "global", data.value)
+
+
+@router.post("/{username}/customization/pagetype")
+async def set_customization_pagetype(
+    request: Request,
+    username: str,
+    data: Annotated[CustomizationToggleForm, Form()],
+):
+    return _toggle_customization(request, username, "pagetype", data.value)