diff --git a/.dockerignore b/.dockerignore index 401f4a8..c28ec57 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,11 @@ # retoor -.venv -data -out +.venv/ +data/ +out/ +.git/ +__pycache__/ +*.pyc .env -.git +.pytest_cache/ +.coverage* +*.egg-info/ diff --git a/.env.example b/.env.example index 73f1398..9b2b64e 100644 --- a/.env.example +++ b/.env.example @@ -7,14 +7,23 @@ MOLODETZ_ADMIN_PASSWORD= MOLODETZ_PORT=8088 MOLODETZ_TEST_PORT=8099 MOLODETZ_SITE_URL= -MOLODETZ_DATA_DIR= +MOLODETZ_DATA_DIR=./data +MOLODETZ_DATABASE_URL=sqlite:///./data/molodetz.db +MOLODETZ_INTERNAL_BASE_URL=http://localhost:8088 MOLODETZ_DISABLE_SERVICES= MOLODETZ_DISABLE_RATE_LIMIT= MOLODETZ_RATE_LIMIT=60 +MOLODETZ_WEB_WORKERS= MOLODETZ_TEMPLATE_AUTO_RELOAD=1 MOLODETZ_STATIC_VERSION= MOLODETZ_LOG_LEVEL=INFO +MOLODETZ_PRESENCE_TIMEOUT_SECONDS=60 +MOLODETZ_PRESENCE_ONLINE_LIMIT=30 +MOLODETZ_PRESENCE_TRACK_LIMIT=500 +MOLODETZ_PRESENCE_ONLINE_MARGIN_SECONDS=20 MOLODETZ_OUTBOUND_PROXY_URL= +MOLODETZ_RCLONE_BIN=rclone +MOLODETZ_RCLONE_CONFIG=/home/retoor/.config/rclone/rclone.conf MOLODETZ_BACKUP_OFFLOAD_REMOTE= MOLODETZ_SITEMAP_TTL=3600 MOLODETZ_LANDING_TTL=30 diff --git a/.gitignore b/.gitignore index 7812384..a04a241 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ __pycache__/ .pytest_cache/ .coverage .coverage.* +build/ +dist/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e0a519f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +# retoor +FROM python:3.12-slim +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /app +COPY pyproject.toml README.md ./ +COPY molodetz ./molodetz +RUN pip install --no-cache-dir --upgrade pip && pip install --no-cache-dir -e . +EXPOSE 8088 +CMD ["python", "-m", "uvicorn", "molodetz.main:app", "--host", "0.0.0.0", "--port", "8088", "--reload", "--reload-dir", "molodetz"] diff --git a/Makefile b/Makefile index 6410d71..1d3464a 100644 --- a/Makefile +++ b/Makefile @@ -11,11 +11,17 @@ PORT ?= 8088 WORKERS ?= 2 LOCUST_PORT ?= 8097 LOCUST_DATA := /tmp/molodetz-locust-data +SCREENSHOTS_DIR ?= $(CURDIR)/../static/pizdetz +SCREENSHOTS_PORT ?= 8094 +SCREENSHOTS_STAGING_URL ?= https://staging.app.molodetz.nl +SCREENSHOTS_PRODUCTION_URL ?= https://molodetz.nl +STAGING_ADMIN_PASSWORD ?= staging-admin-pass-1 PYTEST := $(PY) -m pytest .PHONY: install dev prod test test-headed test-unit test-api test-e2e test-fast test-failed \ test-first-failure test-slowest test-cache-clean check-ui preflight coverage coverage-headed \ - coverage-html locust locust-headless prune prune-dry-run tree tree-loc zip delete-pyc clean + coverage-html locust locust-headless screenshots screenshots-staging screenshots-production \ + staging-refresh prune prune-dry-run tree tree-loc zip delete-pyc clean $(STAMP): pyproject.toml python3 -m venv .venv @@ -103,6 +109,20 @@ locust-run: -for i in $$(seq 1 40); do kill -0 $$(cat $(LOCUST_DATA)/server.pid) 2>/dev/null || break; sleep 0.25; done rm -rf $(LOCUST_DATA) +screenshots: $(STAMP) + SCREENSHOTS_DIR=$(SCREENSHOTS_DIR) SCREENSHOTS_PORT=$(SCREENSHOTS_PORT) SCREENSHOTS_ENV=test $(PY) screenshots.py + +screenshots-staging: $(STAMP) + SCREENSHOTS_DIR=$(SCREENSHOTS_DIR) SCREENSHOTS_ENV=staging SCREENSHOTS_BASE_URL=$(SCREENSHOTS_STAGING_URL) \ + SCREENSHOTS_ADMIN_PASSWORD=$(STAGING_ADMIN_PASSWORD) $(PY) screenshots.py + +screenshots-production: $(STAMP) + SCREENSHOTS_DIR=$(SCREENSHOTS_DIR) SCREENSHOTS_ENV=production SCREENSHOTS_BASE_URL=$(SCREENSHOTS_PRODUCTION_URL) \ + SCREENSHOTS_ADMIN_PASSWORD=$(SCREENSHOTS_ADMIN_PASSWORD) $(PY) screenshots.py + +staging-refresh: + bin/staging-refresh.sh $(SNAPSHOT) + prune: $(STAMP) bin/maintenance.sh diff --git a/README.md b/README.md index 75fe1e2..84ed872 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ make dev # http://127.0.0.1:8088, reload make prod # workers, pinned static version ``` +`make install` needs a `python3` with the `venv` module (Debian/Ubuntu ship it as `python3-venv`). + Without make: ``` @@ -33,6 +35,36 @@ make coverage make locust-headless ``` +Tests boot their own server on port 8099; set `MOLODETZ_TEST_PORT` to a free port when 8099 is taken. Browser tests wait up to 60 seconds per step and retry a stalled page load once via `goto_ready` in `tests/conftest.py`. + +## Docker + +``` +docker compose up -d --build # dev server on http://127.0.0.1:19847, reload on change +docker compose logs -f # follow logs +docker compose down # stop +``` + +The container runs the dev server (`uvicorn --reload`) with `./molodetz` mounted, so every saved change is live within seconds. Data lives in the file mount `/var/lib/molodetz-staging` (plain files, no named volumes anywhere). The admin account comes from `MOLODETZ_ADMIN_USERNAME` / `MOLODETZ_ADMIN_EMAIL` / `MOLODETZ_ADMIN_PASSWORD` (default password `staging-admin-pass-1`, override per shell); an empty password skips admin bootstrap. The compose port is fixed at `127.0.0.1:19847` and is what `staging.app.molodetz.nl` proxies to, so keep it stable. + +``` +make staging-refresh # copy the holy prod database over staging, then verify +``` + +Refresh snapshots `/var/lib/molodetz` (online backup API, production keeps running), integrity-checks the copy, stops staging, swaps the staging file mount content, restarts, and verifies health. After a refresh the staging admin password is the production one, so pass it to `make screenshots-staging STAGING_ADMIN_PASSWORD=...`. + +## Screenshots + +``` +make screenshots # test gallery into ../static/pizdetz/test +make screenshots-staging # staging gallery into ../static/pizdetz/staging +make screenshots-production SCREENSHOTS_ADMIN_PASSWORD=... # production gallery into ../static/pizdetz/production +``` + +Each run captures 32 full-page desktop shots (public, docs, admin) plus a 25-cell responsive matrix (home, roll, flyers, join, docs at 1440x900, 768x1024, 390x844, 360x740, 320x568) with a horizontal-overflow check per cell, then rebuilds the hub page at `../static/pizdetz/index.html` from whichever environments exist on disk. + +Rules, exactly: `test` always boots its own throwaway server and seeds one join request; `staging` and `production` shoot a live base URL and never seed. Staging logs in with `STAGING_ADMIN_PASSWORD` (default `staging-admin-pass-1`, must match the compose admin password); production has no default and fails loudly without `SCREENSHOTS_ADMIN_PASSWORD`. Any overflow is reported and exits nonzero. Override the output root with `SCREENSHOTS_DIR=...`. + ## CLI `molodetz --help`: roles, API keys, posts list/import (.md and .pdf via pypdf), join requests, backups, prunes, sql-lint, emoji regenerate. @@ -40,3 +72,7 @@ make locust-headless ## Data Everything lives under `data/` (database `molodetz.db`, uploads, backups, keys, locks). `.env` and `data/` are in `.gitignore`. + +## Operations + +Production serving runs through molohttp on this box. `molohttp_deploy.md` is the operator guide: access, file owners, the backup specification, site changes, and static publishing. diff --git a/bin/staging-refresh.sh b/bin/staging-refresh.sh new file mode 100755 index 0000000..03f2b0a --- /dev/null +++ b/bin/staging-refresh.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# retoor +set -euo pipefail +SRC="${1:-/var/lib/molodetz}" +DEST=/var/lib/molodetz-staging +WORK="$(mktemp -d)" +trap 'sudo -n rm -rf "$WORK"' EXIT +test -f "$SRC/molodetz.db" || { echo "no database at $SRC/molodetz.db"; exit 1; } +test -d "$DEST" || { echo "no staging data dir at $DEST"; exit 1; } +sudo -n python3 -c "import sqlite3; s = sqlite3.connect('$SRC/molodetz.db'); d = sqlite3.connect('$WORK/molodetz.db'); s.backup(d); s.close(); d.close()" +for sub in uploads keys seo_reports; do + if [ -d "$SRC/$sub" ]; then sudo -n cp -a "$SRC/$sub" "$WORK/"; fi +done +CHECK="$(sudo -n python3 -c "import sqlite3; c = sqlite3.connect('file:$WORK/molodetz.db?mode=ro', uri=True); print(c.execute('pragma integrity_check').fetchone()[0])")" +test "$CHECK" = "ok" || { echo "snapshot integrity check failed: $CHECK"; exit 1; } +cd "$(dirname "$0")/.." +docker compose stop web +sudo -n rm -rf "$DEST/molodetz.db-wal" "$DEST/molodetz.db-shm" "$DEST/molodetz.db" "$DEST/uploads" "$DEST/keys" "$DEST/seo_reports" +sudo -n cp -a "$WORK/." "$DEST/" +docker compose up -d web +for i in $(seq 1 60); do curl -fs http://127.0.0.1:19847/health >/dev/null && break; sleep 1; done +curl -fs http://127.0.0.1:19847/health +echo "staging refreshed from $SRC" diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..6040b6b --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,20 @@ +# retoor +services: + web: + build: . + image: molodetz-prod + container_name: molodetz-prod + restart: unless-stopped + ports: + - "127.0.0.1:19848:8088" + volumes: + - /var/lib/molodetz:/app/data + environment: + MOLODETZ_DATA_DIR: /app/data + MOLODETZ_ADMIN_USERNAME: ${MOLODETZ_ADMIN_USERNAME:-retoor} + MOLODETZ_ADMIN_EMAIL: ${MOLODETZ_ADMIN_EMAIL:-retoor@molodetz.nl} + MOLODETZ_ADMIN_PASSWORD: ${PROD_ADMIN_PASSWORD:?set PROD_ADMIN_PASSWORD for production} + SECRET_KEY: ${PROD_SECRET_KEY:?set PROD_SECRET_KEY for production} + MOLODETZ_STATIC_VERSION: ${MOLODETZ_STATIC_VERSION:-prod} + MOLODETZ_TEMPLATE_AUTO_RELOAD: "0" + command: ["python", "-m", "uvicorn", "molodetz.main:app", "--host", "0.0.0.0", "--port", "8088", "--workers", "2", "--proxy-headers", "--forwarded-allow-ips", "*"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..efb1384 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +# retoor +services: + web: + build: . + image: molodetz-dev + container_name: molodetz-dev + restart: unless-stopped + ports: + - "127.0.0.1:19847:8088" + volumes: + - ./molodetz:/app/molodetz + - /var/lib/molodetz-staging:/app/data + environment: + MOLODETZ_DATA_DIR: /app/data + MOLODETZ_ADMIN_USERNAME: ${MOLODETZ_ADMIN_USERNAME:-retoor} + MOLODETZ_ADMIN_EMAIL: ${MOLODETZ_ADMIN_EMAIL:-retoor@molodetz.nl} + MOLODETZ_ADMIN_PASSWORD: ${MOLODETZ_ADMIN_PASSWORD:-staging-admin-pass-1} + SECRET_KEY: ${SECRET_KEY:-molodetz-docker-dev-secret} + MOLODETZ_STATIC_VERSION: ${MOLODETZ_STATIC_VERSION:-docker} diff --git a/molohttp_deploy.md b/molohttp_deploy.md new file mode 100644 index 0000000..a62ccf2 --- /dev/null +++ b/molohttp_deploy.md @@ -0,0 +1,142 @@ + +# molohttp deploy and operations + +Production server `molodetz`. molohttp 0.9.1. Facts below were read from the live system on 2026-10-05. This is a production box: follow the backup specification before every change. + +## Access + +Operator account is `retoor`. Root login is never needed. + +- `retoor` is in the `sudo` group with `(ALL) NOPASSWD: ALL` (verified with `sudo -n true` and `sudo -n -l`). +- Every privileged command in this guide runs under `sudo` and also works non-interactively in scripts. +- `/etc/molohttp` is `molohttp:molohttp 750`: reads and edits both require `sudo`. + +## How it works + +molohttp is the single front door on ports 80 and 443. One systemd unit runs one process as user `molohttp` (24+ days uptime at time of writing). + +- Binary: `/usr/local/bin/molohttp` (`root:root 755`). +- Unit: `/etc/systemd/system/molohttp.service` (`root:root 644`), enabled and active. +- Start: `molohttp run --config /etc/molohttp/config.json --env /etc/molohttp/.env.json`, working directory `/etc/molohttp`. +- Reload: `ExecReload=/bin/kill -HUP $MAINPID`, so `sudo systemctl reload molohttp` applies config without dropping connections. There is no config-validate subcommand; validate JSON by hand (see change procedure). +- Privileged ports are bound via `CAP_NET_BIND_SERVICE`; the process itself is unprivileged with `ProtectSystem=strict` and `ProtectHome=true`. +- `config.json` holds 94 sites, all `enabled`, all `reverse_proxy` to `http://127.0.0.1:` with one exception: a single upstream on `http://pravda.education:10500`. One site carries a basic-auth middleware. `site-094` is `staging.app.molodetz.nl`, the Molodetz docker dev server (see Staging below). +- TLS: minimum TLS 1.2, HSTS one year with subdomains. Certificates live per hostname as `/etc/molohttp/certs/.pem` plus `.key`. +- ACME: production Let's Encrypt, account `retoor@molodetz.nl`, renewal check every 12 hours starting 30 days before expiry, state in `/var/lib/molohttp/acme`. +- Logging: JSON to stdout, collected by journald (`SyslogIdentifier=molohttp`). The `access_log_path`/`error_log_path` settings are not materialized as live files. +- Admin: the admin UI is deliberately not exposed. `admin.molodetz.nl` is not hosted, routed, or resolvable: no site entry, no DNS record. Exposing it would be a security flaw. The only admin path is the local CLI: `molohttp admin user-list`, `molohttp admin hash`. `molohttp install`/`uninstall` must run as root and are out of scope for routine work. + +## Files and owners + +Canonical state. Verify with `sudo find /etc/molohttp /var/lib/molohttp /var/log/molohttp -maxdepth 2 -exec stat -c '%U:%G %a %n' {} +`. + +| Path | Owner | Mode | Notes | +|---|---|---|---| +| `/etc/molohttp` | `molohttp:molohttp` | 750 | config root, sudo to enter | +| `/etc/molohttp/config.json` | `molohttp:molohttp` | 640 | site table, exact path read | +| `/etc/molohttp/.env.json` | `molohttp:molohttp` | 640 | global settings, holds password hash | +| `/etc/molohttp/certs` | `molohttp:molohttp` | 700 | per-host pairs | +| `/etc/molohttp/certs/*.key` | `molohttp:molohttp` | 600 | private keys | +| `/etc/molohttp/certs/*.pem` | `molohttp:molohttp` | 644 | certificates | +| `/etc/molohttp/*.bak-*` | `root:root` | 640 | operator backups, this spec only | +| `/var/lib/molohttp` | `molohttp:molohttp` | 750 | state root | +| `/var/lib/molohttp/acme` | `molohttp:molohttp` | 750 | live ACME state | +| `/var/log/molohttp` | `molohttp:molohttp` | 750 | empty dir, daemon logs to journal | +| `/var/www/.well-known` | `molohttp:molohttp` | 750 | ACME challenges | +| `/etc/systemd/system/molohttp.service` | `root:root` | 644 | unit file | +| `/usr/local/bin/molohttp` | `root:root` | 755 | binary | + +Known anomalies, observed 2026-10-05, left untouched pending a decision: + +- Six key pairs under `certs/` are owned `caddy:caddy` (the `pravda.education` hosts plus `mail.molodetz.nl`). The `.key` files at mode 600 are unreadable by the `molohttp` user; confirm those hosts serve TLS correctly, then normalize with `sudo chown molohttp:molohttp` on the pair. +- `/var/lib/molohttp/analytics` plus `/var/lib/molohttp/stats` (`caddy:caddy`, stale since Jul 27) are leftovers from a previous server. Nothing reads them. (A stale 2.5 GB `caddy`-owned `access.log` in `/var/log/molohttp` was already removed on 2026-10-05 after confirming nothing held it open.) +- Older backups in `/etc/molohttp` use four different naming schemes and mixed owners. They are harmless history; all new backups follow the one format below. + +## Backup specification + +Always back up before editing. One format, no variants: + +`.bak--`, owned `root:root`, mode 640, next to the original. Siblings are safe: the daemon reads `config.json` and `.env.json` by exact path only, never by glob. + +``` +FILE=/etc/molohttp/config.json +REASON=site-add-example +STAMP=$(date +%Y%m%d%H%M%S) +sudo cp -p "$FILE" "$FILE.bak-$REASON-$STAMP" +sudo chown root:root "$FILE.bak-$REASON-$STAMP" +sudo chmod 640 "$FILE.bak-$REASON-$STAMP" +ls -la "$FILE"* +``` + +Rules: reason is a required lowercase slug (`site-add-`, `site-remove-`, `upstream-`, `tls-`, `env-`). Never delete old backups; at about 52 KB each they are the audit trail. Never invent a second format. + +## Change procedure + +1. Back up per the specification above. +2. Edit with `sudo` (for example `sudo -e /etc/molohttp/config.json`). +3. Restore ownership, since root-written files lock the daemon out: `sudo chown molohttp:molohttp /etc/molohttp/config.json`, `sudo chmod 640 /etc/molohttp/config.json`. +4. Validate: `sudo -n cat /etc/molohttp/config.json | python3 -m json.tool > /dev/null`. +5. Apply: `sudo systemctl reload molohttp`. +6. Verify: `systemctl is-active molohttp`, `curl -s -o /dev/null -w "%{http_code}\n" https:///`, and `sudo journalctl -u molohttp --no-pager -n 20`. +7. Roll back on any failure: `sudo cp -p /etc/molohttp/config.json`, `sudo chown molohttp:molohttp /etc/molohttp/config.json`, `sudo chmod 640 /etc/molohttp/config.json`, then reload again and re-verify. + +Use `restart` instead of `reload` only when the process itself must be replaced. Never `stop` without `start` on this box. + +## Add or remove a site + +Append one object to the `sites` array. Keep `id` unique (`site-094`, `site-095`, ...); `name` equals the primary hostname. + +``` +{ + "id": "site-094", + "name": "new.molodetz.nl", + "enabled": true, + "matchers": {"hosts": ["new.molodetz.nl"], "headers": {}, "paths": []}, + "handler": { + "type": "reverse_proxy", + "upstreams": [{"url": "http://127.0.0.1:8XXX", "weight": 1}], + "load_balancing": "round_robin", + "websocket": false, + "connection_timeout_ms": 5000, + "read_timeout_ms": 30000 + } +} +``` + +Pick a free localhost port for the upstream and keep the app itself bound to 127.0.0.1. Certificates are provisioned automatically through ACME within seconds of the first HTTPS traffic; verify with `curl` and `openssl s_client -connect :443 -servername `, and check the journal on any TLS error. Per-host files under `certs/` may materialize later on molohttp's own schedule (observed with staging: valid certificate served immediately, files not yet present), so their absence right after creation is not a failure. Removal is the reverse: back up, delete the object, validate, reload, verify the hostname no longer routes. + +## Staging + +`staging.app.molodetz.nl` (`site-094`) proxies to `http://127.0.0.1:19847`, the Molodetz docker dev server from this repo (see Docker in `README.md`). It carries a normal Let's Encrypt certificate. Every saved change under `./molodetz` is live on staging within seconds through uvicorn reload; app edits never need a molohttp change. Staging data is the file mount `/var/lib/molodetz-staging`, refreshed from holy production with `make staging-refresh` (see `bin/staging-refresh.sh`); production data is the file mount `/var/lib/molodetz`. Named volumes are never used. + +## Static publishing + +`static.molodetz.nl` proxies to `http://127.0.0.1:8117`, served by `rserver` running as `retoor` with working directory `/home/retoor/projects/static`. Publishing static content needs no molohttp change: write files under that directory and they are live (verified end to end with `/pizdetz/`, which redirects to `/pizdetz/index.html`). The Molodetz screenshot gallery is produced this way; see the Screenshots section in `README.md`. + +## Ownership ensure + +Run after any manual intervention to restore the canonical state. It does not touch the documented anomalies. + +``` +sudo chown molohttp:molohttp /etc/molohttp /etc/molohttp/config.json /etc/molohttp/.env.json +sudo chmod 750 /etc/molohttp +sudo chmod 640 /etc/molohttp/config.json /etc/molohttp/.env.json +sudo chown molohttp:molohttp /etc/molohttp/certs /var/lib/molohttp /var/log/molohttp /var/www/.well-known +sudo chmod 700 /etc/molohttp/certs +sudo chmod 750 /var/lib/molohttp /var/log/molohttp /var/www/.well-known +sudo find /etc/molohttp/certs -name '*.key' -user molohttp -exec chmod 600 {} + +sudo find /etc/molohttp/certs -name '*.pem' -user molohttp -exec chmod 644 {} + +sudo find /etc/molohttp -maxdepth 1 -name '*.bak-*' -exec chown root:root {} + -exec chmod 640 {} + +``` + +## Quick commands + +| Task | Command | +|---|---| +| Status | `systemctl is-active molohttp`, `molohttp service-status` | +| Logs | `sudo journalctl -u molohttp -f`, `molohttp service-logs` | +| Reload config | `sudo systemctl reload molohttp` | +| Restart | `sudo systemctl restart molohttp` | +| Site table | `sudo -n cat /etc/molohttp/config.json \| python3 -m json.tool \| less` | +| Port owners | `sudo -n ss -tlnp` | +| Version | `molohttp --version` | diff --git a/pyproject.toml b/pyproject.toml index b8313a3..8299e86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "molodetz" -version = "1.0.10" +version = "1.0.11" description = "Molodetz, a calm community blog roll." readme = "README.md" requires-python = ">=3.12" diff --git a/screenshots.py b/screenshots.py new file mode 100644 index 0000000..3f9d7c3 --- /dev/null +++ b/screenshots.py @@ -0,0 +1,476 @@ +# retoor +import html +import os +import shutil +import subprocess +import sys +import tempfile +import time +import tomllib +from datetime import datetime, timezone +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent +OUTPUT_DIR = Path(os.environ.get("SCREENSHOTS_DIR", str(REPO_ROOT.parent / "static" / "pizdetz"))) +ENV = os.environ.get("SCREENSHOTS_ENV", "test") +BASE_URL = os.environ.get("SCREENSHOTS_BASE_URL", "").rstrip("/") +PORT = os.environ.get("SCREENSHOTS_PORT", "8094") +LOCAL_BASE = f"http://127.0.0.1:{PORT}" +ADMIN_USERNAME = os.environ.get("SCREENSHOTS_ADMIN_USERNAME", "retoor") +ADMIN_PASSWORD = os.environ.get("SCREENSHOTS_ADMIN_PASSWORD", "") +BOOT_ADMIN_PASSWORD = "shots-admin-pass-9d3c" +FULL_DESKTOP = {"width": 1440, "height": 900} +KNOWN_ENVS = ["test", "staging", "production"] + +PUBLIC_SHOTS = [ + ("home", "/", "Home"), + ("roll", "/roll", "Roll"), + ("standard", "/standard", "Standard"), + ("flyers", "/flyers", "Flyers"), + ("memes", "/memes", "Memes"), + ("people", "/people", "People"), + ("person", "/people/retoor", "Person"), + ("join", "/join", "Join"), + ("terms", "/terms", "Terms"), + ("privacy", "/privacy", "Privacy"), + ("login", "/auth/login", "Log in"), +] + +DOCS_SHOTS = [ + ("docs", "/docs", "Docs"), + ("docs-welcome", "/docs/welcome", "Docs welcome"), + ("docs-api", "/docs/api", "Docs API index"), + ("docs-api-content", "/docs/api/content", "Docs API content"), + ("docs-component-code", "/docs/component-code", "Docs component code"), +] + +ADMIN_SHOTS = [ + ("admin", "/admin", "Admin"), + ("admin-posts", "/admin/posts", "Admin posts"), + ("admin-post-new", "/admin/posts/new", "Admin editor"), + ("admin-joins", "/admin/joins", "Admin joins"), + ("admin-users", "/admin/users", "Admin users"), + ("admin-settings", "/admin/settings", "Admin settings"), + ("admin-services", "/admin/services", "Admin services"), + ("admin-services-backup", "/admin/services/backup", "Admin service backup"), + ("admin-backups", "/admin/backups", "Admin backups"), + ("admin-audit", "/admin/audit", "Admin audit"), + ("admin-stats", "/admin/stats", "Admin stats"), + ("admin-trash", "/admin/trash", "Admin trash"), + ("notifications", "/notifications", "Notifications"), + ("profile-api-key", "/profile/api-key", "Profile API key"), + ("docs-api-admin", "/docs/api/admin", "Docs API admin"), +] + +RESPONSIVE_VIEWPORTS = [ + ("1440x900", {"width": 1440, "height": 900}), + ("768x1024", {"width": 768, "height": 1024}), + ("390x844", {"width": 390, "height": 844}), + ("360x740", {"width": 360, "height": 740}), + ("320x568", {"width": 320, "height": 568}), +] + +RESPONSIVE_PAGES = [ + ("home", "/", "Home"), + ("roll", "/roll", "Roll"), + ("flyers", "/flyers", "Flyers"), + ("join", "/join", "Join"), + ("docs", "/docs", "Docs"), +] + + +def server_env(data_dir): + return { + "MOLODETZ_DATA_DIR": str(data_dir), + "MOLODETZ_DATABASE_URL": f"sqlite:///{data_dir / 'molodetz.db'}", + "MOLODETZ_ADMIN_USERNAME": ADMIN_USERNAME, + "MOLODETZ_ADMIN_EMAIL": "retoor@molodetz.nl", + "MOLODETZ_ADMIN_PASSWORD": BOOT_ADMIN_PASSWORD, + "MOLODETZ_DISABLE_SERVICES": "1", + "MOLODETZ_DISABLE_RATE_LIMIT": "1", + "MOLODETZ_PORT": PORT, + "MOLODETZ_INTERNAL_BASE_URL": LOCAL_BASE, + "MOLODETZ_SITEMAP_TTL": "0", + "MOLODETZ_LANDING_TTL": "0", + "MOLODETZ_UNREAD_TTL": "0", + "MOLODETZ_STATIC_VERSION": "shots", + "SECRET_KEY": "shots-secret-key-not-for-production", + } + + +def wait_ready(process, base): + import httpx + + deadline = time.monotonic() + 60.0 + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("shots server exited during startup") + try: + if httpx.get(f"{base}/health", timeout=1.0).status_code == 200: + return + except httpx.HTTPError: + pass + time.sleep(0.2) + raise RuntimeError("shots server did not become ready") + + +def start_server(data_dir): + log = open(data_dir / "server.log", "w") + process = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "molodetz.main:app", "--host", "127.0.0.1", "--port", PORT], + cwd=REPO_ROOT, + env={**os.environ, **server_env(data_dir)}, + stdout=log, + stderr=subprocess.STDOUT, + ) + try: + wait_ready(process, LOCAL_BASE) + except Exception: + process.terminate() + log.close() + raise + return process, log + + +def stop_server(process, log): + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + log.close() + + +def goto_ready(page, url): + from playwright.sync_api import TimeoutError as PlaywrightTimeoutError + + page.goto(url, wait_until="domcontentloaded") + try: + page.wait_for_selector("html[data-app=ready]") + except PlaywrightTimeoutError: + page.reload(wait_until="domcontentloaded") + page.wait_for_selector("html[data-app=ready]") + + +def login_admin(page, base, password): + page.goto(base + "/auth/login", wait_until="domcontentloaded") + page.fill("input[name=username]", ADMIN_USERNAME) + page.fill("input[name=password]", password) + page.click("button[type=submit]") + page.wait_for_url("**/admin", wait_until="domcontentloaded") + + +def seed_join(base): + import httpx + + response = httpx.post( + base + "/join", + data={ + "name": "Gallery Writer", + "contact": "gallery@example.invalid", + "repo_url": "https://example.com/gallery", + "message": "I want to write along.", + "website": "", + }, + timeout=15.0, + ) + if response.status_code not in (302, 303): + raise RuntimeError(f"seed join failed: {response.status_code}") + + +def first_post_slug(base): + import httpx + + response = httpx.get(base + "/roll", headers={"Accept": "application/json"}, timeout=15.0) + posts = response.json().get("posts", []) + if not posts: + raise RuntimeError("roll has no posts") + return posts[0]["slug"] + + +def capture(page, out_dir, base, counter, name, path, title, section, viewport, full_page): + goto_ready(page, base + path) + filename = f"{counter:02d}-{name}-{viewport}.png" + page.screenshot(path=str(out_dir / filename), full_page=full_page) + return {"section": section, "title": title, "path": path, "viewport": viewport, "file": filename} + + +def check_overflow(page, path, viewport): + width, inner = page.evaluate("[document.documentElement.scrollWidth, window.innerWidth]") + if width > inner: + return f"{path} at {viewport}: scroll {width} wider than viewport {inner}" + return "" + + +def capture_full(browser, out_dir, base, password, records, counter): + anon = browser.new_context(viewport=FULL_DESKTOP) + page = anon.new_page() + page.set_default_timeout(60000) + for name, path, title in PUBLIC_SHOTS: + records.append(capture(page, out_dir, base, counter, name, path, title, "Public", "full", True)) + counter += 1 + slug = first_post_slug(base) + records.append(capture(page, out_dir, base, counter, "post", f"/posts/{slug}", "Post", "Public", "full", True)) + counter += 1 + for name, path, title in DOCS_SHOTS: + records.append(capture(page, out_dir, base, counter, name, path, title, "Docs", "full", True)) + counter += 1 + anon.close() + admin = browser.new_context(viewport=FULL_DESKTOP) + admin_page = admin.new_page() + admin_page.set_default_timeout(60000) + login_admin(admin_page, base, password) + for name, path, title in ADMIN_SHOTS: + records.append(capture(admin_page, out_dir, base, counter, name, path, title, "Administration", "full", True)) + counter += 1 + admin.close() + return counter + + +def capture_responsive(browser, out_dir, base, records, failures, counter): + for viewport, size in RESPONSIVE_VIEWPORTS: + context = browser.new_context(viewport=size) + page = context.new_page() + page.set_default_timeout(60000) + for name, path, title in RESPONSIVE_PAGES: + goto_ready(page, base + path) + failure = check_overflow(page, path, viewport) + if failure: + failures.append(failure) + filename = f"{counter:02d}-{name}-{viewport}.png" + page.screenshot(path=str(out_dir / filename), full_page=False) + records.append({"section": "Responsive", "title": title, "path": path, "viewport": viewport, "file": filename}) + counter += 1 + context.close() + context = browser.new_context(viewport={"width": 390, "height": 844}) + page = context.new_page() + page.set_default_timeout(60000) + goto_ready(page, base + "/") + page.click("[data-nav-toggle]") + filename = f"{counter:02d}-home-nav-open-390x844.png" + page.screenshot(path=str(out_dir / filename), full_page=False) + records.append({"section": "Responsive", "title": "Home navigation open", "path": "/", "viewport": "390x844", "file": filename}) + context.close() + return counter + 1 + + +def app_version(): + with (REPO_ROOT / "pyproject.toml").open("rb") as handle: + return tomllib.load(handle)["project"]["version"] + + +def render_index(records, failures, checked, env, version, stamped): + sections = [] + for section in ["Public", "Docs", "Administration", "Responsive"]: + figures = [] + for record in records: + if record["section"] != section: + continue + figures.append( + '
{t}' + "
{t} {p} ({v})
".format( + f=html.escape(record["file"]), + t=html.escape(record["title"]), + p=html.escape(record["path"]), + v=html.escape(record["viewport"]), + ) + ) + note = "" + if section == "Responsive": + passed = checked - len(failures) + items = "".join(f"
  • {html.escape(failure)}
    • " for failure in failures) + failures_html = f"
      {items}
    " if items else "" + note = f"

    {passed} of {checked} responsive checks passed, no horizontal overflow.

    {failures_html}" + sections.append("

    {s}

    {n}
    {f}
    ".format(s=section, n=note, f="".join(figures))) + return """ + + + + +Molodetz in pictures ({e}) + + + + +
    + +

    Molodetz in pictures ({e})

    +

    {n} screenshots of version {v}, taken {d}.

    +

    Every thumbnail links to the full image.

    +
    +{s} +
    +

    Molodetz screenshot gallery. Generated from a seeded review build.

    +
    + + +""".format(n=len(records), e=html.escape(env), v=html.escape(version), d=html.escape(stamped), s="".join(sections)) + + +def env_previews(out_dir): + preferred = ["01-home-full.png", "18-admin-full.png"] + found = [name for name in preferred if (out_dir / name).exists()] + for shot in sorted(out_dir.glob("*-home-390x844.png")): + if shot.name not in found: + found.append(shot.name) + if len(found) == 3: + break + for shot in sorted(out_dir.glob("*.png")): + if shot.name not in found: + found.append(shot.name) + if len(found) == 3: + break + return found + + +def render_hub(entries): + cards = [] + for env, count, previews in entries: + thumbs = "".join( + '{e}'.format(e=env, f=html.escape(name)) + for name in previews + ) + cards.append( + '

    {e}

    {n} screenshots.

    {t}
    '.format( + e=html.escape(env), n=count, t=thumbs + ) + ) + return """ + + + + +Molodetz screenshot environments + + + + +
    +

    Molodetz screenshot environments

    +

    One gallery per environment. Regenerate with make screenshots, make screenshots-staging, make screenshots-production.

    +
    +{c} + + + +""".format(c="".join(cards)) + + +def rebuild_hub(): + entries = [] + for env in KNOWN_ENVS: + out_dir = OUTPUT_DIR / env + if not (out_dir / "index.html").exists(): + continue + count = len(list(out_dir.glob("*.png"))) + entries.append((env, count, env_previews(out_dir))) + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + (OUTPUT_DIR / "index.html").write_text(render_hub(entries), encoding="utf-8") + return [env for env, _, _ in entries] + + +def clear_output(out_dir): + out_dir.mkdir(parents=True, exist_ok=True) + for stale in out_dir.glob("*.png"): + stale.unlink() + index = out_dir / "index.html" + if index.exists(): + index.unlink() + + +def wait_live(base): + import httpx + + try: + response = httpx.get(f"{base}/health", timeout=15.0) + except httpx.HTTPError as exc: + raise RuntimeError(f"base url {base} is not reachable: {exc}") from exc + if response.status_code != 200: + raise RuntimeError(f"base url {base} health is {response.status_code}") + + +def main(): + from playwright.sync_api import sync_playwright + + if ENV not in KNOWN_ENVS: + raise RuntimeError(f"unknown SCREENSHOTS_ENV {ENV!r}, want one of {', '.join(KNOWN_ENVS)}") + boot = not BASE_URL + if boot and ENV != "test": + raise RuntimeError("booted runs always target the test gallery, set SCREENSHOTS_ENV=test") + if not boot and ENV == "test": + raise RuntimeError("base-url runs target staging or production, set SCREENSHOTS_ENV accordingly") + if not boot and not ADMIN_PASSWORD: + raise RuntimeError("base-url runs need SCREENSHOTS_ADMIN_PASSWORD for the admin shots") + base = LOCAL_BASE if boot else BASE_URL + password = BOOT_ADMIN_PASSWORD if boot else ADMIN_PASSWORD + out_dir = OUTPUT_DIR / ENV + data_dir = None + process = None + log = None + records = [] + failures = [] + try: + if boot: + data_dir = Path(tempfile.mkdtemp(prefix="molodetz-shots-")) + process, log = start_server(data_dir) + seed_join(base) + else: + wait_live(base) + headless = os.environ.get("PLAYWRIGHT_HEADLESS", "1") == "1" + slow_mo = int(os.environ.get("PLAYWRIGHT_SLOW_MO", "0") or 0) + clear_output(out_dir) + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=headless, slow_mo=slow_mo) + counter = capture_full(browser, out_dir, base, password, records, 1) + capture_responsive(browser, out_dir, base, records, failures, counter) + browser.close() + checked = len(RESPONSIVE_PAGES) * len(RESPONSIVE_VIEWPORTS) + stamped = datetime.now(timezone.utc).date().isoformat() + (out_dir / "index.html").write_text(render_index(records, failures, checked, ENV, app_version(), stamped), encoding="utf-8") + present = rebuild_hub() + finally: + if process is not None: + stop_server(process, log) + if data_dir is not None: + shutil.rmtree(data_dir, ignore_errors=True) + print(f"wrote {len(records)} screenshots for {ENV} to {out_dir}, hub covers {', '.join(present)}") + for failure in failures: + print(f"OVERFLOW {failure}") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main())