Docker dev and prod, staging refresh, multi-env galleries and operator docs

2026-10-05 14:32:21 +02:00
parent 94ae750e56
commit 4268b1ec9e
12 changed files with 768 additions and 7 deletions
@@ -1,6 +1,11 @@
 # retoor <retoor@molodetz.nl>
-.venv
-data
-out
+.venv/
+data/
+out/
+.git/
+__pycache__/
+*.pyc
 .env
-.git
+.pytest_cache/
+.coverage*
+*.egg-info/
@@ -7,14 +7,23 @@ MOLODETZ_ADMIN_PASSWORD=
 MOLODETZ_PORT=8088
 MOLODETZ_TEST_PORT=8099
 MOLODETZ_SITE_URL=
-MOLODETZ_DATA_DIR=
+MOLODETZ_DATA_DIR=./data
+MOLODETZ_DATABASE_URL=sqlite:///./data/molodetz.db
+MOLODETZ_INTERNAL_BASE_URL=http://localhost:8088
 MOLODETZ_DISABLE_SERVICES=
 MOLODETZ_DISABLE_RATE_LIMIT=
 MOLODETZ_RATE_LIMIT=60
+MOLODETZ_WEB_WORKERS=
 MOLODETZ_TEMPLATE_AUTO_RELOAD=1
 MOLODETZ_STATIC_VERSION=
 MOLODETZ_LOG_LEVEL=INFO
+MOLODETZ_PRESENCE_TIMEOUT_SECONDS=60
+MOLODETZ_PRESENCE_ONLINE_LIMIT=30
+MOLODETZ_PRESENCE_TRACK_LIMIT=500
+MOLODETZ_PRESENCE_ONLINE_MARGIN_SECONDS=20
 MOLODETZ_OUTBOUND_PROXY_URL=
+MOLODETZ_RCLONE_BIN=rclone
+MOLODETZ_RCLONE_CONFIG=/home/retoor/.config/rclone/rclone.conf
 MOLODETZ_BACKUP_OFFLOAD_REMOTE=
 MOLODETZ_SITEMAP_TTL=3600
 MOLODETZ_LANDING_TTL=30
@@ -9,3 +9,5 @@ __pycache__/
 .pytest_cache/
 .coverage
 .coverage.*
+build/
+dist/
@@ -0,0 +1,9 @@
+# retoor <retoor@molodetz.nl>
+FROM python:3.12-slim
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
+WORKDIR /app
+COPY pyproject.toml README.md ./
+COPY molodetz ./molodetz
+RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -e .
+EXPOSE 8088
+CMD ["python", "-m", "uvicorn", "molodetz.main:app", "--host", "0.0.0.0", "--port", "8088", "--reload", "--reload-dir", "molodetz"]
@@ -11,11 +11,17 @@ PORT ?= 8088
 WORKERS ?= 2
 LOCUST_PORT ?= 8097
 LOCUST_DATA := /tmp/molodetz-locust-data
+SCREENSHOTS_DIR ?= $(CURDIR)/../static/pizdetz
+SCREENSHOTS_PORT ?= 8094
+SCREENSHOTS_STAGING_URL ?= https://staging.app.molodetz.nl
+SCREENSHOTS_PRODUCTION_URL ?= https://molodetz.nl
+STAGING_ADMIN_PASSWORD ?= staging-admin-pass-1
 PYTEST := $(PY) -m pytest

 .PHONY: install dev prod test test-headed test-unit test-api test-e2e test-fast test-failed \
 	test-first-failure test-slowest test-cache-clean check-ui preflight coverage coverage-headed \
-	coverage-html locust locust-headless prune prune-dry-run tree tree-loc zip delete-pyc clean
+	coverage-html locust locust-headless screenshots screenshots-staging screenshots-production \
+	staging-refresh prune prune-dry-run tree tree-loc zip delete-pyc clean

 $(STAMP): pyproject.toml
 	python3 -m venv .venv
@@ -103,6 +109,20 @@ locust-run:
 	-for i in $$(seq 1 40); do kill -0 $$(cat $(LOCUST_DATA)/server.pid) 2>/dev/null || break; sleep 0.25; done
 	rm -rf $(LOCUST_DATA)

+screenshots: $(STAMP)
+	SCREENSHOTS_DIR=$(SCREENSHOTS_DIR) SCREENSHOTS_PORT=$(SCREENSHOTS_PORT) SCREENSHOTS_ENV=test $(PY) screenshots.py
+
+screenshots-staging: $(STAMP)
+	SCREENSHOTS_DIR=$(SCREENSHOTS_DIR) SCREENSHOTS_ENV=staging SCREENSHOTS_BASE_URL=$(SCREENSHOTS_STAGING_URL) \
+		SCREENSHOTS_ADMIN_PASSWORD=$(STAGING_ADMIN_PASSWORD) $(PY) screenshots.py
+
+screenshots-production: $(STAMP)
+	SCREENSHOTS_DIR=$(SCREENSHOTS_DIR) SCREENSHOTS_ENV=production SCREENSHOTS_BASE_URL=$(SCREENSHOTS_PRODUCTION_URL) \
+		SCREENSHOTS_ADMIN_PASSWORD=$(SCREENSHOTS_ADMIN_PASSWORD) $(PY) screenshots.py
+
+staging-refresh:
+	bin/staging-refresh.sh $(SNAPSHOT)
+
 prune: $(STAMP)
 	bin/maintenance.sh

@@ -14,6 +14,8 @@ make dev              # http://127.0.0.1:8088, reload
 make prod             # workers, pinned static version
 ```

+`make install` needs a `python3` with the `venv` module (Debian/Ubuntu ship it as `python3-venv`).
+
 Without make:

 ```
@@ -33,6 +35,36 @@ make coverage
 make locust-headless
 ```

+Tests boot their own server on port 8099; set `MOLODETZ_TEST_PORT` to a free port when 8099 is taken. Browser tests wait up to 60 seconds per step and retry a stalled page load once via `goto_ready` in `tests/conftest.py`.
+
+## Docker
+
+```
+docker compose up -d --build  # dev server on http://127.0.0.1:19847, reload on change
+docker compose logs -f        # follow logs
+docker compose down           # stop
+```
+
+The container runs the dev server (`uvicorn --reload`) with `./molodetz` mounted, so every saved change is live within seconds. Data lives in the file mount `/var/lib/molodetz-staging` (plain files, no named volumes anywhere). The admin account comes from `MOLODETZ_ADMIN_USERNAME` / `MOLODETZ_ADMIN_EMAIL` / `MOLODETZ_ADMIN_PASSWORD` (default password `staging-admin-pass-1`, override per shell); an empty password skips admin bootstrap. The compose port is fixed at `127.0.0.1:19847` and is what `staging.app.molodetz.nl` proxies to, so keep it stable.
+
+```
+make staging-refresh  # copy the holy prod database over staging, then verify
+```
+
+Refresh snapshots `/var/lib/molodetz` (online backup API, production keeps running), integrity-checks the copy, stops staging, swaps the staging file mount content, restarts, and verifies health. After a refresh the staging admin password is the production one, so pass it to `make screenshots-staging STAGING_ADMIN_PASSWORD=...`.
+
+## Screenshots
+
+```
+make screenshots                                                         # test gallery into ../static/pizdetz/test
+make screenshots-staging                                                 # staging gallery into ../static/pizdetz/staging
+make screenshots-production SCREENSHOTS_ADMIN_PASSWORD=...               # production gallery into ../static/pizdetz/production
+```
+
+Each run captures 32 full-page desktop shots (public, docs, admin) plus a 25-cell responsive matrix (home, roll, flyers, join, docs at 1440x900, 768x1024, 390x844, 360x740, 320x568) with a horizontal-overflow check per cell, then rebuilds the hub page at `../static/pizdetz/index.html` from whichever environments exist on disk.
+
+Rules, exactly: `test` always boots its own throwaway server and seeds one join request; `staging` and `production` shoot a live base URL and never seed. Staging logs in with `STAGING_ADMIN_PASSWORD` (default `staging-admin-pass-1`, must match the compose admin password); production has no default and fails loudly without `SCREENSHOTS_ADMIN_PASSWORD`. Any overflow is reported and exits nonzero. Override the output root with `SCREENSHOTS_DIR=...`.
+
 ## CLI

 `molodetz --help`: roles, API keys, posts list/import (.md and .pdf via pypdf), join requests, backups, prunes, sql-lint, emoji regenerate.
@@ -40,3 +72,7 @@ make locust-headless
 ## Data

 Everything lives under `data/` (database `molodetz.db`, uploads, backups, keys, locks). `.env` and `data/` are in `.gitignore`.
+
+## Operations
+
+Production serving runs through molohttp on this box. `molohttp_deploy.md` is the operator guide: access, file owners, the backup specification, site changes, and static publishing.
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# retoor <retoor@molodetz.nl>
+set -euo pipefail
+SRC="${1:-/var/lib/molodetz}"
+DEST=/var/lib/molodetz-staging
+WORK="$(mktemp -d)"
+trap 'sudo -n rm -rf "$WORK"' EXIT
+test -f "$SRC/molodetz.db" || { echo "no database at $SRC/molodetz.db"; exit 1; }
+test -d "$DEST" || { echo "no staging data dir at $DEST"; exit 1; }
+sudo -n python3 -c "import sqlite3; s = sqlite3.connect('$SRC/molodetz.db'); d = sqlite3.connect('$WORK/molodetz.db'); s.backup(d); s.close(); d.close()"
+for sub in uploads keys seo_reports; do
+  if [ -d "$SRC/$sub" ]; then sudo -n cp -a "$SRC/$sub" "$WORK/"; fi
+done
+CHECK="$(sudo -n python3 -c "import sqlite3; c = sqlite3.connect('file:$WORK/molodetz.db?mode=ro', uri=True); print(c.execute('pragma integrity_check').fetchone()[0])")"
+test "$CHECK" = "ok" || { echo "snapshot integrity check failed: $CHECK"; exit 1; }
+cd "$(dirname "$0")/.."
+docker compose stop web
+sudo -n rm -rf "$DEST/molodetz.db-wal" "$DEST/molodetz.db-shm" "$DEST/molodetz.db" "$DEST/uploads" "$DEST/keys" "$DEST/seo_reports"
+sudo -n cp -a "$WORK/." "$DEST/"
+docker compose up -d web
+for i in $(seq 1 60); do curl -fs http://127.0.0.1:19847/health >/dev/null && break; sleep 1; done
+curl -fs http://127.0.0.1:19847/health
+echo "staging refreshed from $SRC"
@@ -0,0 +1,20 @@
+# retoor <retoor@molodetz.nl>
+services:
+  web:
+    build: .
+    image: molodetz-prod
+    container_name: molodetz-prod
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:19848:8088"
+    volumes:
+      - /var/lib/molodetz:/app/data
+    environment:
+      MOLODETZ_DATA_DIR: /app/data
+      MOLODETZ_ADMIN_USERNAME: ${MOLODETZ_ADMIN_USERNAME:-retoor}
+      MOLODETZ_ADMIN_EMAIL: ${MOLODETZ_ADMIN_EMAIL:-retoor@molodetz.nl}
+      MOLODETZ_ADMIN_PASSWORD: ${PROD_ADMIN_PASSWORD:?set PROD_ADMIN_PASSWORD for production}
+      SECRET_KEY: ${PROD_SECRET_KEY:?set PROD_SECRET_KEY for production}
+      MOLODETZ_STATIC_VERSION: ${MOLODETZ_STATIC_VERSION:-prod}
+      MOLODETZ_TEMPLATE_AUTO_RELOAD: "0"
+    command: ["python", "-m", "uvicorn", "molodetz.main:app", "--host", "0.0.0.0", "--port", "8088", "--workers", "2", "--proxy-headers", "--forwarded-allow-ips", "*"]
@@ -0,0 +1,19 @@
+# retoor <retoor@molodetz.nl>
+services:
+  web:
+    build: .
+    image: molodetz-dev
+    container_name: molodetz-dev
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:19847:8088"
+    volumes:
+      - ./molodetz:/app/molodetz
+      - /var/lib/molodetz-staging:/app/data
+    environment:
+      MOLODETZ_DATA_DIR: /app/data
+      MOLODETZ_ADMIN_USERNAME: ${MOLODETZ_ADMIN_USERNAME:-retoor}
+      MOLODETZ_ADMIN_EMAIL: ${MOLODETZ_ADMIN_EMAIL:-retoor@molodetz.nl}
+      MOLODETZ_ADMIN_PASSWORD: ${MOLODETZ_ADMIN_PASSWORD:-staging-admin-pass-1}
+      SECRET_KEY: ${SECRET_KEY:-molodetz-docker-dev-secret}
+      MOLODETZ_STATIC_VERSION: ${MOLODETZ_STATIC_VERSION:-docker}
@@ -0,0 +1,142 @@
+<!-- retoor <retoor@molodetz.nl> -->
+# molohttp deploy and operations
+
+Production server `molodetz`. molohttp 0.9.1. Facts below were read from the live system on 2026-10-05. This is a production box: follow the backup specification before every change.
+
+## Access
+
+Operator account is `retoor`. Root login is never needed.
+
+- `retoor` is in the `sudo` group with `(ALL) NOPASSWD: ALL` (verified with `sudo -n true` and `sudo -n -l`).
+- Every privileged command in this guide runs under `sudo` and also works non-interactively in scripts.
+- `/etc/molohttp` is `molohttp:molohttp 750`: reads and edits both require `sudo`.
+
+## How it works
+
+molohttp is the single front door on ports 80 and 443. One systemd unit runs one process as user `molohttp` (24+ days uptime at time of writing).
+
+- Binary: `/usr/local/bin/molohttp` (`root:root 755`).
+- Unit: `/etc/systemd/system/molohttp.service` (`root:root 644`), enabled and active.
+- Start: `molohttp run --config /etc/molohttp/config.json --env /etc/molohttp/.env.json`, working directory `/etc/molohttp`.
+- Reload: `ExecReload=/bin/kill -HUP $MAINPID`, so `sudo systemctl reload molohttp` applies config without dropping connections. There is no config-validate subcommand; validate JSON by hand (see change procedure).
+- Privileged ports are bound via `CAP_NET_BIND_SERVICE`; the process itself is unprivileged with `ProtectSystem=strict` and `ProtectHome=true`.
+- `config.json` holds 94 sites, all `enabled`, all `reverse_proxy` to `http://127.0.0.1:<port>` with one exception: a single upstream on `http://pravda.education:10500`. One site carries a basic-auth middleware. `site-094` is `staging.app.molodetz.nl`, the Molodetz docker dev server (see Staging below).
+- TLS: minimum TLS 1.2, HSTS one year with subdomains. Certificates live per hostname as `/etc/molohttp/certs/<host>.pem` plus `<host>.key`.
+- ACME: production Let's Encrypt, account `retoor@molodetz.nl`, renewal check every 12 hours starting 30 days before expiry, state in `/var/lib/molohttp/acme`.
+- Logging: JSON to stdout, collected by journald (`SyslogIdentifier=molohttp`). The `access_log_path`/`error_log_path` settings are not materialized as live files.
+- Admin: the admin UI is deliberately not exposed. `admin.molodetz.nl` is not hosted, routed, or resolvable: no site entry, no DNS record. Exposing it would be a security flaw. The only admin path is the local CLI: `molohttp admin user-list`, `molohttp admin hash`. `molohttp install`/`uninstall` must run as root and are out of scope for routine work.
+
+## Files and owners
+
+Canonical state. Verify with `sudo find /etc/molohttp /var/lib/molohttp /var/log/molohttp -maxdepth 2 -exec stat -c '%U:%G %a %n' {} +`.
+
+| Path | Owner | Mode | Notes |
+|---|---|---|---|
+| `/etc/molohttp` | `molohttp:molohttp` | 750 | config root, sudo to enter |
+| `/etc/molohttp/config.json` | `molohttp:molohttp` | 640 | site table, exact path read |
+| `/etc/molohttp/.env.json` | `molohttp:molohttp` | 640 | global settings, holds password hash |
+| `/etc/molohttp/certs` | `molohttp:molohttp` | 700 | per-host pairs |
+| `/etc/molohttp/certs/*.key` | `molohttp:molohttp` | 600 | private keys |
+| `/etc/molohttp/certs/*.pem` | `molohttp:molohttp` | 644 | certificates |
+| `/etc/molohttp/*.bak-*` | `root:root` | 640 | operator backups, this spec only |
+| `/var/lib/molohttp` | `molohttp:molohttp` | 750 | state root |
+| `/var/lib/molohttp/acme` | `molohttp:molohttp` | 750 | live ACME state |
+| `/var/log/molohttp` | `molohttp:molohttp` | 750 | empty dir, daemon logs to journal |
+| `/var/www/.well-known` | `molohttp:molohttp` | 750 | ACME challenges |
+| `/etc/systemd/system/molohttp.service` | `root:root` | 644 | unit file |
+| `/usr/local/bin/molohttp` | `root:root` | 755 | binary |
+
+Known anomalies, observed 2026-10-05, left untouched pending a decision:
+
+- Six key pairs under `certs/` are owned `caddy:caddy` (the `pravda.education` hosts plus `mail.molodetz.nl`). The `.key` files at mode 600 are unreadable by the `molohttp` user; confirm those hosts serve TLS correctly, then normalize with `sudo chown molohttp:molohttp` on the pair.
+- `/var/lib/molohttp/analytics` plus `/var/lib/molohttp/stats` (`caddy:caddy`, stale since Jul 27) are leftovers from a previous server. Nothing reads them. (A stale 2.5 GB `caddy`-owned `access.log` in `/var/log/molohttp` was already removed on 2026-10-05 after confirming nothing held it open.)
+- Older backups in `/etc/molohttp` use four different naming schemes and mixed owners. They are harmless history; all new backups follow the one format below.
+
+## Backup specification
+
+Always back up before editing. One format, no variants:
+
+`<file>.bak-<reason>-<YYYYMMDDHHMMSS>`, owned `root:root`, mode 640, next to the original. Siblings are safe: the daemon reads `config.json` and `.env.json` by exact path only, never by glob.
+
+```
+FILE=/etc/molohttp/config.json
+REASON=site-add-example
+STAMP=$(date +%Y%m%d%H%M%S)
+sudo cp -p "$FILE" "$FILE.bak-$REASON-$STAMP"
+sudo chown root:root "$FILE.bak-$REASON-$STAMP"
+sudo chmod 640 "$FILE.bak-$REASON-$STAMP"
+ls -la "$FILE"*
+```
+
+Rules: reason is a required lowercase slug (`site-add-<host>`, `site-remove-<host>`, `upstream-<host>`, `tls-<host>`, `env-<what>`). Never delete old backups; at about 52 KB each they are the audit trail. Never invent a second format.
+
+## Change procedure
+
+1. Back up per the specification above.
+2. Edit with `sudo` (for example `sudo -e /etc/molohttp/config.json`).
+3. Restore ownership, since root-written files lock the daemon out: `sudo chown molohttp:molohttp /etc/molohttp/config.json`, `sudo chmod 640 /etc/molohttp/config.json`.
+4. Validate: `sudo -n cat /etc/molohttp/config.json | python3 -m json.tool > /dev/null`.
+5. Apply: `sudo systemctl reload molohttp`.
+6. Verify: `systemctl is-active molohttp`, `curl -s -o /dev/null -w "%{http_code}\n" https://<host>/`, and `sudo journalctl -u molohttp --no-pager -n 20`.
+7. Roll back on any failure: `sudo cp -p <backup> /etc/molohttp/config.json`, `sudo chown molohttp:molohttp /etc/molohttp/config.json`, `sudo chmod 640 /etc/molohttp/config.json`, then reload again and re-verify.
+
+Use `restart` instead of `reload` only when the process itself must be replaced. Never `stop` without `start` on this box.
+
+## Add or remove a site
+
+Append one object to the `sites` array. Keep `id` unique (`site-094`, `site-095`, ...); `name` equals the primary hostname.
+
+```
+{
+  "id": "site-094",
+  "name": "new.molodetz.nl",
+  "enabled": true,
+  "matchers": {"hosts": ["new.molodetz.nl"], "headers": {}, "paths": []},
+  "handler": {
+    "type": "reverse_proxy",
+    "upstreams": [{"url": "http://127.0.0.1:8XXX", "weight": 1}],
+    "load_balancing": "round_robin",
+    "websocket": false,
+    "connection_timeout_ms": 5000,
+    "read_timeout_ms": 30000
+  }
+}
+```
+
+Pick a free localhost port for the upstream and keep the app itself bound to 127.0.0.1. Certificates are provisioned automatically through ACME within seconds of the first HTTPS traffic; verify with `curl` and `openssl s_client -connect <host>:443 -servername <host>`, and check the journal on any TLS error. Per-host files under `certs/` may materialize later on molohttp's own schedule (observed with staging: valid certificate served immediately, files not yet present), so their absence right after creation is not a failure. Removal is the reverse: back up, delete the object, validate, reload, verify the hostname no longer routes.
+
+## Staging
+
+`staging.app.molodetz.nl` (`site-094`) proxies to `http://127.0.0.1:19847`, the Molodetz docker dev server from this repo (see Docker in `README.md`). It carries a normal Let's Encrypt certificate. Every saved change under `./molodetz` is live on staging within seconds through uvicorn reload; app edits never need a molohttp change. Staging data is the file mount `/var/lib/molodetz-staging`, refreshed from holy production with `make staging-refresh` (see `bin/staging-refresh.sh`); production data is the file mount `/var/lib/molodetz`. Named volumes are never used.
+
+## Static publishing
+
+`static.molodetz.nl` proxies to `http://127.0.0.1:8117`, served by `rserver` running as `retoor` with working directory `/home/retoor/projects/static`. Publishing static content needs no molohttp change: write files under that directory and they are live (verified end to end with `/pizdetz/`, which redirects to `/pizdetz/index.html`). The Molodetz screenshot gallery is produced this way; see the Screenshots section in `README.md`.
+
+## Ownership ensure
+
+Run after any manual intervention to restore the canonical state. It does not touch the documented anomalies.
+
+```
+sudo chown molohttp:molohttp /etc/molohttp /etc/molohttp/config.json /etc/molohttp/.env.json
+sudo chmod 750 /etc/molohttp
+sudo chmod 640 /etc/molohttp/config.json /etc/molohttp/.env.json
+sudo chown molohttp:molohttp /etc/molohttp/certs /var/lib/molohttp /var/log/molohttp /var/www/.well-known
+sudo chmod 700 /etc/molohttp/certs
+sudo chmod 750 /var/lib/molohttp /var/log/molohttp /var/www/.well-known
+sudo find /etc/molohttp/certs -name '*.key' -user molohttp -exec chmod 600 {} +
+sudo find /etc/molohttp/certs -name '*.pem' -user molohttp -exec chmod 644 {} +
+sudo find /etc/molohttp -maxdepth 1 -name '*.bak-*' -exec chown root:root {} + -exec chmod 640 {} +
+```
+
+## Quick commands
+
+| Task | Command |
+|---|---|
+| Status | `systemctl is-active molohttp`, `molohttp service-status` |
+| Logs | `sudo journalctl -u molohttp -f`, `molohttp service-logs` |
+| Reload config | `sudo systemctl reload molohttp` |
+| Restart | `sudo systemctl restart molohttp` |
+| Site table | `sudo -n cat /etc/molohttp/config.json \| python3 -m json.tool \| less` |
+| Port owners | `sudo -n ss -tlnp` |
+| Version | `molohttp --version` |
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"

 [project]
 name = "molodetz"
-version = "1.0.10"
+version = "1.0.11"
 description = "Molodetz, a calm community blog roll."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -0,0 +1,476 @@
+# retoor <retoor@molodetz.nl>
+import html
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+import tomllib
+from datetime import datetime, timezone
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent
+OUTPUT_DIR = Path(os.environ.get("SCREENSHOTS_DIR", str(REPO_ROOT.parent / "static" / "pizdetz")))
+ENV = os.environ.get("SCREENSHOTS_ENV", "test")
+BASE_URL = os.environ.get("SCREENSHOTS_BASE_URL", "").rstrip("/")
+PORT = os.environ.get("SCREENSHOTS_PORT", "8094")
+LOCAL_BASE = f"http://127.0.0.1:{PORT}"
+ADMIN_USERNAME = os.environ.get("SCREENSHOTS_ADMIN_USERNAME", "retoor")
+ADMIN_PASSWORD = os.environ.get("SCREENSHOTS_ADMIN_PASSWORD", "")
+BOOT_ADMIN_PASSWORD = "shots-admin-pass-9d3c"
+FULL_DESKTOP = {"width": 1440, "height": 900}
+KNOWN_ENVS = ["test", "staging", "production"]
+
+PUBLIC_SHOTS = [
+    ("home", "/", "Home"),
+    ("roll", "/roll", "Roll"),
+    ("standard", "/standard", "Standard"),
+    ("flyers", "/flyers", "Flyers"),
+    ("memes", "/memes", "Memes"),
+    ("people", "/people", "People"),
+    ("person", "/people/retoor", "Person"),
+    ("join", "/join", "Join"),
+    ("terms", "/terms", "Terms"),
+    ("privacy", "/privacy", "Privacy"),
+    ("login", "/auth/login", "Log in"),
+]
+
+DOCS_SHOTS = [
+    ("docs", "/docs", "Docs"),
+    ("docs-welcome", "/docs/welcome", "Docs welcome"),
+    ("docs-api", "/docs/api", "Docs API index"),
+    ("docs-api-content", "/docs/api/content", "Docs API content"),
+    ("docs-component-code", "/docs/component-code", "Docs component code"),
+]
+
+ADMIN_SHOTS = [
+    ("admin", "/admin", "Admin"),
+    ("admin-posts", "/admin/posts", "Admin posts"),
+    ("admin-post-new", "/admin/posts/new", "Admin editor"),
+    ("admin-joins", "/admin/joins", "Admin joins"),
+    ("admin-users", "/admin/users", "Admin users"),
+    ("admin-settings", "/admin/settings", "Admin settings"),
+    ("admin-services", "/admin/services", "Admin services"),
+    ("admin-services-backup", "/admin/services/backup", "Admin service backup"),
+    ("admin-backups", "/admin/backups", "Admin backups"),
+    ("admin-audit", "/admin/audit", "Admin audit"),
+    ("admin-stats", "/admin/stats", "Admin stats"),
+    ("admin-trash", "/admin/trash", "Admin trash"),
+    ("notifications", "/notifications", "Notifications"),
+    ("profile-api-key", "/profile/api-key", "Profile API key"),
+    ("docs-api-admin", "/docs/api/admin", "Docs API admin"),
+]
+
+RESPONSIVE_VIEWPORTS = [
+    ("1440x900", {"width": 1440, "height": 900}),
+    ("768x1024", {"width": 768, "height": 1024}),
+    ("390x844", {"width": 390, "height": 844}),
+    ("360x740", {"width": 360, "height": 740}),
+    ("320x568", {"width": 320, "height": 568}),
+]
+
+RESPONSIVE_PAGES = [
+    ("home", "/", "Home"),
+    ("roll", "/roll", "Roll"),
+    ("flyers", "/flyers", "Flyers"),
+    ("join", "/join", "Join"),
+    ("docs", "/docs", "Docs"),
+]
+
+
+def server_env(data_dir):
+    return {
+        "MOLODETZ_DATA_DIR": str(data_dir),
+        "MOLODETZ_DATABASE_URL": f"sqlite:///{data_dir / 'molodetz.db'}",
+        "MOLODETZ_ADMIN_USERNAME": ADMIN_USERNAME,
+        "MOLODETZ_ADMIN_EMAIL": "retoor@molodetz.nl",
+        "MOLODETZ_ADMIN_PASSWORD": BOOT_ADMIN_PASSWORD,
+        "MOLODETZ_DISABLE_SERVICES": "1",
+        "MOLODETZ_DISABLE_RATE_LIMIT": "1",
+        "MOLODETZ_PORT": PORT,
+        "MOLODETZ_INTERNAL_BASE_URL": LOCAL_BASE,
+        "MOLODETZ_SITEMAP_TTL": "0",
+        "MOLODETZ_LANDING_TTL": "0",
+        "MOLODETZ_UNREAD_TTL": "0",
+        "MOLODETZ_STATIC_VERSION": "shots",
+        "SECRET_KEY": "shots-secret-key-not-for-production",
+    }
+
+
+def wait_ready(process, base):
+    import httpx
+
+    deadline = time.monotonic() + 60.0
+    while time.monotonic() < deadline:
+        if process.poll() is not None:
+            raise RuntimeError("shots server exited during startup")
+        try:
+            if httpx.get(f"{base}/health", timeout=1.0).status_code == 200:
+                return
+        except httpx.HTTPError:
+            pass
+        time.sleep(0.2)
+    raise RuntimeError("shots server did not become ready")
+
+
+def start_server(data_dir):
+    log = open(data_dir / "server.log", "w")
+    process = subprocess.Popen(
+        [sys.executable, "-m", "uvicorn", "molodetz.main:app", "--host", "127.0.0.1", "--port", PORT],
+        cwd=REPO_ROOT,
+        env={**os.environ, **server_env(data_dir)},
+        stdout=log,
+        stderr=subprocess.STDOUT,
+    )
+    try:
+        wait_ready(process, LOCAL_BASE)
+    except Exception:
+        process.terminate()
+        log.close()
+        raise
+    return process, log
+
+
+def stop_server(process, log):
+    process.terminate()
+    try:
+        process.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        process.kill()
+    log.close()
+
+
+def goto_ready(page, url):
+    from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+
+    page.goto(url, wait_until="domcontentloaded")
+    try:
+        page.wait_for_selector("html[data-app=ready]")
+    except PlaywrightTimeoutError:
+        page.reload(wait_until="domcontentloaded")
+        page.wait_for_selector("html[data-app=ready]")
+
+
+def login_admin(page, base, password):
+    page.goto(base + "/auth/login", wait_until="domcontentloaded")
+    page.fill("input[name=username]", ADMIN_USERNAME)
+    page.fill("input[name=password]", password)
+    page.click("button[type=submit]")
+    page.wait_for_url("**/admin", wait_until="domcontentloaded")
+
+
+def seed_join(base):
+    import httpx
+
+    response = httpx.post(
+        base + "/join",
+        data={
+            "name": "Gallery Writer",
+            "contact": "gallery@example.invalid",
+            "repo_url": "https://example.com/gallery",
+            "message": "I want to write along.",
+            "website": "",
+        },
+        timeout=15.0,
+    )
+    if response.status_code not in (302, 303):
+        raise RuntimeError(f"seed join failed: {response.status_code}")
+
+
+def first_post_slug(base):
+    import httpx
+
+    response = httpx.get(base + "/roll", headers={"Accept": "application/json"}, timeout=15.0)
+    posts = response.json().get("posts", [])
+    if not posts:
+        raise RuntimeError("roll has no posts")
+    return posts[0]["slug"]
+
+
+def capture(page, out_dir, base, counter, name, path, title, section, viewport, full_page):
+    goto_ready(page, base + path)
+    filename = f"{counter:02d}-{name}-{viewport}.png"
+    page.screenshot(path=str(out_dir / filename), full_page=full_page)
+    return {"section": section, "title": title, "path": path, "viewport": viewport, "file": filename}
+
+
+def check_overflow(page, path, viewport):
+    width, inner = page.evaluate("[document.documentElement.scrollWidth, window.innerWidth]")
+    if width > inner:
+        return f"{path} at {viewport}: scroll {width} wider than viewport {inner}"
+    return ""
+
+
+def capture_full(browser, out_dir, base, password, records, counter):
+    anon = browser.new_context(viewport=FULL_DESKTOP)
+    page = anon.new_page()
+    page.set_default_timeout(60000)
+    for name, path, title in PUBLIC_SHOTS:
+        records.append(capture(page, out_dir, base, counter, name, path, title, "Public", "full", True))
+        counter += 1
+    slug = first_post_slug(base)
+    records.append(capture(page, out_dir, base, counter, "post", f"/posts/{slug}", "Post", "Public", "full", True))
+    counter += 1
+    for name, path, title in DOCS_SHOTS:
+        records.append(capture(page, out_dir, base, counter, name, path, title, "Docs", "full", True))
+        counter += 1
+    anon.close()
+    admin = browser.new_context(viewport=FULL_DESKTOP)
+    admin_page = admin.new_page()
+    admin_page.set_default_timeout(60000)
+    login_admin(admin_page, base, password)
+    for name, path, title in ADMIN_SHOTS:
+        records.append(capture(admin_page, out_dir, base, counter, name, path, title, "Administration", "full", True))
+        counter += 1
+    admin.close()
+    return counter
+
+
+def capture_responsive(browser, out_dir, base, records, failures, counter):
+    for viewport, size in RESPONSIVE_VIEWPORTS:
+        context = browser.new_context(viewport=size)
+        page = context.new_page()
+        page.set_default_timeout(60000)
+        for name, path, title in RESPONSIVE_PAGES:
+            goto_ready(page, base + path)
+            failure = check_overflow(page, path, viewport)
+            if failure:
+                failures.append(failure)
+            filename = f"{counter:02d}-{name}-{viewport}.png"
+            page.screenshot(path=str(out_dir / filename), full_page=False)
+            records.append({"section": "Responsive", "title": title, "path": path, "viewport": viewport, "file": filename})
+            counter += 1
+        context.close()
+    context = browser.new_context(viewport={"width": 390, "height": 844})
+    page = context.new_page()
+    page.set_default_timeout(60000)
+    goto_ready(page, base + "/")
+    page.click("[data-nav-toggle]")
+    filename = f"{counter:02d}-home-nav-open-390x844.png"
+    page.screenshot(path=str(out_dir / filename), full_page=False)
+    records.append({"section": "Responsive", "title": "Home navigation open", "path": "/", "viewport": "390x844", "file": filename})
+    context.close()
+    return counter + 1
+
+
+def app_version():
+    with (REPO_ROOT / "pyproject.toml").open("rb") as handle:
+        return tomllib.load(handle)["project"]["version"]
+
+
+def render_index(records, failures, checked, env, version, stamped):
+    sections = []
+    for section in ["Public", "Docs", "Administration", "Responsive"]:
+        figures = []
+        for record in records:
+            if record["section"] != section:
+                continue
+            figures.append(
+                '<figure><a href="{f}"><img src="{f}" alt="{t}" loading="lazy"></a>'
+                "<figcaption>{t} <code>{p}</code> ({v})</figcaption></figure>".format(
+                    f=html.escape(record["file"]),
+                    t=html.escape(record["title"]),
+                    p=html.escape(record["path"]),
+                    v=html.escape(record["viewport"]),
+                )
+            )
+        note = ""
+        if section == "Responsive":
+            passed = checked - len(failures)
+            items = "".join(f"<li>{html.escape(failure)}</li>" for failure in failures)
+            failures_html = f"<ul>{items}</ul>" if items else ""
+            note = f"<p class=\"checks\">{passed} of {checked} responsive checks passed, no horizontal overflow.</p>{failures_html}"
+        sections.append("<h2>{s}</h2>{n}<div class=\"shots\">{f}</div>".format(s=section, n=note, f="".join(figures)))
+    return """<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Molodetz in pictures ({e})</title>
+<meta name="description" content="Screenshot gallery of the Molodetz site.">
+<style>
+body{{margin:0 auto;max-width:1120px;padding:48px 24px 96px;font-family:Georgia,'Times New Roman',serif;color:#1a1815;background:#fff}}
+header{{border-bottom:1px solid #e2ddd4;padding-bottom:24px}}
+header nav{{font-size:14px;margin-bottom:16px}}
+header nav a{{color:#555}}
+h1{{font-size:34px;margin:0 0 8px;font-weight:600}}
+header p{{margin:4px 0;font-size:15px;color:#555}}
+h2{{font-size:22px;margin:44px 0 4px}}
+.checks{{font-size:14px;color:#555;margin:8px 0 0}}
+ul{{font-size:14px;color:#8a1f1f}}
+.shots{{display:grid;grid-template-columns:repeat(auto-fill,minmax(300px,1fr));gap:28px 24px;margin-top:16px}}
+figure{{margin:0}}
+figure a{{display:block;border:1px solid #e2ddd4;border-radius:4px;overflow:hidden}}
+figure img{{display:block;width:100%;height:220px;object-fit:cover;object-position:top}}
+figcaption{{font-size:13px;color:#555;margin-top:8px}}
+figcaption code{{font-family:ui-monospace,Menlo,monospace;font-size:12px}}
+footer{{margin-top:56px;border-top:1px solid #e2ddd4;padding-top:16px;font-size:13px;color:#555}}
+</style>
+</head>
+<body>
+<header>
+<nav><a href="../index.html">All environments</a></nav>
+<h1>Molodetz in pictures ({e})</h1>
+<p>{n} screenshots of version {v}, taken {d}.</p>
+<p>Every thumbnail links to the full image.</p>
+</header>
+{s}
+<footer>
+<p>Molodetz screenshot gallery. Generated from a seeded review build.</p>
+</footer>
+</body>
+</html>
+""".format(n=len(records), e=html.escape(env), v=html.escape(version), d=html.escape(stamped), s="".join(sections))
+
+
+def env_previews(out_dir):
+    preferred = ["01-home-full.png", "18-admin-full.png"]
+    found = [name for name in preferred if (out_dir / name).exists()]
+    for shot in sorted(out_dir.glob("*-home-390x844.png")):
+        if shot.name not in found:
+            found.append(shot.name)
+        if len(found) == 3:
+            break
+    for shot in sorted(out_dir.glob("*.png")):
+        if shot.name not in found:
+            found.append(shot.name)
+        if len(found) == 3:
+            break
+    return found
+
+
+def render_hub(entries):
+    cards = []
+    for env, count, previews in entries:
+        thumbs = "".join(
+            '<a href="{e}/index.html"><img src="{e}/{f}" alt="{e}" loading="lazy"></a>'.format(e=env, f=html.escape(name))
+            for name in previews
+        )
+        cards.append(
+            '<section><h2><a href="{e}/index.html">{e}</a></h2><p>{n} screenshots.</p><div class="previews">{t}</div></section>'.format(
+                e=html.escape(env), n=count, t=thumbs
+            )
+        )
+    return """<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Molodetz screenshot environments</title>
+<meta name="description" content="Screenshot galleries of the Molodetz site per environment.">
+<style>
+body{{margin:0 auto;max-width:1120px;padding:48px 24px 96px;font-family:Georgia,'Times New Roman',serif;color:#1a1815;background:#fff}}
+header{{border-bottom:1px solid #e2ddd4;padding-bottom:24px}}
+h1{{font-size:34px;margin:0 0 8px;font-weight:600}}
+header p{{margin:4px 0;font-size:15px;color:#555}}
+section{{margin-top:40px}}
+section h2{{font-size:24px;margin:0 0 4px;text-transform:capitalize}}
+section h2 a{{color:#1a1815}}
+section p{{font-size:14px;color:#555;margin:4px 0 12px}}
+.previews{{display:grid;grid-template-columns:repeat(3,1fr);gap:16px}}
+.previews a{{display:block;border:1px solid #e2ddd4;border-radius:4px;overflow:hidden}}
+.previews img{{display:block;width:100%;height:200px;object-fit:cover;object-position:top}}
+footer{{margin-top:56px;border-top:1px solid #e2ddd4;padding-top:16px;font-size:13px;color:#555}}
+footer code{{font-family:ui-monospace,Menlo,monospace;font-size:12px}}
+</style>
+</head>
+<body>
+<header>
+<h1>Molodetz screenshot environments</h1>
+<p>One gallery per environment. Regenerate with <code>make screenshots</code>, <code>make screenshots-staging</code>, <code>make screenshots-production</code>.</p>
+</header>
+{c}
+<footer>
+<p>Molodetz screenshot hub. Rebuilt on every gallery run from the environments present.</p>
+</footer>
+</body>
+</html>
+""".format(c="".join(cards))
+
+
+def rebuild_hub():
+    entries = []
+    for env in KNOWN_ENVS:
+        out_dir = OUTPUT_DIR / env
+        if not (out_dir / "index.html").exists():
+            continue
+        count = len(list(out_dir.glob("*.png")))
+        entries.append((env, count, env_previews(out_dir)))
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+    (OUTPUT_DIR / "index.html").write_text(render_hub(entries), encoding="utf-8")
+    return [env for env, _, _ in entries]
+
+
+def clear_output(out_dir):
+    out_dir.mkdir(parents=True, exist_ok=True)
+    for stale in out_dir.glob("*.png"):
+        stale.unlink()
+    index = out_dir / "index.html"
+    if index.exists():
+        index.unlink()
+
+
+def wait_live(base):
+    import httpx
+
+    try:
+        response = httpx.get(f"{base}/health", timeout=15.0)
+    except httpx.HTTPError as exc:
+        raise RuntimeError(f"base url {base} is not reachable: {exc}") from exc
+    if response.status_code != 200:
+        raise RuntimeError(f"base url {base} health is {response.status_code}")
+
+
+def main():
+    from playwright.sync_api import sync_playwright
+
+    if ENV not in KNOWN_ENVS:
+        raise RuntimeError(f"unknown SCREENSHOTS_ENV {ENV!r}, want one of {', '.join(KNOWN_ENVS)}")
+    boot = not BASE_URL
+    if boot and ENV != "test":
+        raise RuntimeError("booted runs always target the test gallery, set SCREENSHOTS_ENV=test")
+    if not boot and ENV == "test":
+        raise RuntimeError("base-url runs target staging or production, set SCREENSHOTS_ENV accordingly")
+    if not boot and not ADMIN_PASSWORD:
+        raise RuntimeError("base-url runs need SCREENSHOTS_ADMIN_PASSWORD for the admin shots")
+    base = LOCAL_BASE if boot else BASE_URL
+    password = BOOT_ADMIN_PASSWORD if boot else ADMIN_PASSWORD
+    out_dir = OUTPUT_DIR / ENV
+    data_dir = None
+    process = None
+    log = None
+    records = []
+    failures = []
+    try:
+        if boot:
+            data_dir = Path(tempfile.mkdtemp(prefix="molodetz-shots-"))
+            process, log = start_server(data_dir)
+            seed_join(base)
+        else:
+            wait_live(base)
+        headless = os.environ.get("PLAYWRIGHT_HEADLESS", "1") == "1"
+        slow_mo = int(os.environ.get("PLAYWRIGHT_SLOW_MO", "0") or 0)
+        clear_output(out_dir)
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(headless=headless, slow_mo=slow_mo)
+            counter = capture_full(browser, out_dir, base, password, records, 1)
+            capture_responsive(browser, out_dir, base, records, failures, counter)
+            browser.close()
+        checked = len(RESPONSIVE_PAGES) * len(RESPONSIVE_VIEWPORTS)
+        stamped = datetime.now(timezone.utc).date().isoformat()
+        (out_dir / "index.html").write_text(render_index(records, failures, checked, ENV, app_version(), stamped), encoding="utf-8")
+        present = rebuild_hub()
+    finally:
+        if process is not None:
+            stop_server(process, log)
+        if data_dir is not None:
+            shutil.rmtree(data_dir, ignore_errors=True)
+    print(f"wrote {len(records)} screenshots for {ENV} to {out_dir}, hub covers {', '.join(present)}")
+    for failure in failures:
+        print(f"OVERFLOW {failure}")
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())