From 60dd5976609e99a82c23cdc7e45be0ff0cf6b71a Mon Sep 17 00:00:00 2001 From: retoor Date: Mon, 5 Oct 2026 15:12:43 +0200 Subject: [PATCH] Production cutover, public test, tmp-shots galleries, development flow --- .gitignore | 4 ++ DEVELOPMENT_FLOW.md | 84 +++++++++++++++++++++++++++++++++++++++++ README.md | 10 +++-- docker-compose.prod.yml | 1 + docker-compose.test.yml | 22 +++++++++++ docker-compose.yml | 1 + molohttp_deploy.md | 13 ++++++- pyproject.toml | 2 +- screenshots.py | 11 +++--- 9 files changed, 138 insertions(+), 10 deletions(-) create mode 100644 DEVELOPMENT_FLOW.md create mode 100644 docker-compose.test.yml diff --git a/.gitignore b/.gitignore index a04a241..a0d569a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ __pycache__/ .coverage.* build/ dist/ +temp_* +*.swp +*~ +.DS_Store diff --git a/DEVELOPMENT_FLOW.md b/DEVELOPMENT_FLOW.md new file mode 100644 index 0000000..a2cbca6 --- /dev/null +++ b/DEVELOPMENT_FLOW.md @@ -0,0 +1,84 @@ + +# Development flow + +The complete blueprint for generating and running projects. This is the evolution of the frozen DPP template Grok originally followed (not present on this machine): everything below was learned by shipping Molodetz to production and must be applied at once, from the first commit, on every future project of any type. + +## Storage law + +All application data lives under `./data/` inside the project. No exceptions, no second locations. + +- `development` for local runs, `staging` and `production` as container file mounts, `test` for persistent test services. +- Ephemeral runs use dotted subdirectories that clean themselves: `./data/.tmp-pytest-`, `./data/.tmp-locust`, `./data/.tmp-shots-`. +- Each env dir holds the database file plus one subdirectory per kind (`uploads`, `backups`, `keys`, `locks`, `seo_reports`, ...). Database and files are separated by directory, never by system location. +- Enforcement beats convention: the application refuses to start when its data dir resolves outside `./data` or its sqlite file outside the data dir. Prove it with a maintained test that runs the real import in a subprocess against inside and outside paths. +- Named docker volumes are forbidden. Every mount is a file mount. Containers run as uid/gid 1000 so files stay human-owned; bind host directories must pre-exist with correct ownership because docker creates missing ones as root. + +## Environments + +Four environments, one shape each, fixed localhost ports (example offsets: staging 19847, production 19848, test 19849). + +- Production owns the holy database. Nothing writes to it except production itself. +- Staging is refreshed from production snapshots only: online sqlite backup API while prod keeps running, integrity check, stop, swap, start, verify health. Never the reverse direction. +- Test is hermetic: persistent service data for the public test site, wiped or temp dirs for gallery and suite runs. +- Snapshots flow one way: production to staging. Test never touches production data. + +## Docker law + +One compose file per environment, each with its own top-level `name:` so projects can never steal each other's containers. + +- Dev and test run the dev server (`uvicorn --reload`) with the package, the manifest (`pyproject.toml`), and `./data/` mounted: every saved change is live within seconds. +- Production runs workers, no reload, no source mount except the data dir. +- Production secrets use dedicated `PROD_`-prefixed variables with `:?` guards, so the dev `.env` can never silently satisfy them. Missing secrets fail loudly at startup, never with defaults. + +## Environment files law + +`.env.example` lists every variable the code reads, no more, no less. A maintained contract test asserts the exact key set plus non-empty values for every path-like key, and fails on the old file before passing on the new one. + +- Never leave a key empty when empty overrides a good default (paths, binaries, URLs): use safe literals (`./data/development`, `rclone`). +- The private `.env` is generated from the example with fresh random secrets, mode 600, never committed. The admin password applies at first bootstrap only and is reported to the owner once. + +## Serving law (molohttp) + +One front door on 80/443, all local reverse-proxy upstreams. + +- Back up before every edit, one format only: `.bak--`, `root:root 640`, reason slug required, never deleted. +- Assert unique site ids AND unique hostnames before appending: on reload molohttp keeps the first site per hostname and silently drops later duplicates. Re-read the file after reload and assert the new site survived. +- Change order: backup, edit, restore `molohttp:molohttp 640` ownership, validate JSON, reload, verify the site over HTTPS. Rollback is the same steps with the previous values; the old backend stays running until rollback is no longer wanted. +- The admin surface is never exposed publicly. Local CLI only. + +## Gallery law + +Every environment gets a screenshot gallery, generated the same way every time. + +- Boot mode builds a throwaway server on a fresh data dir and seeds one fixed record; base-url mode shoots a live environment and never writes seed data. +- Full-page desktop shots plus a responsive matrix (down to 320px) with a horizontal-overflow check per cell; overflows are reported and exit nonzero. +- One gallery per environment plus a hub page that rebuilds from whichever environments exist. Same script, same selectors, same waits for all. + +## Test law + +Mirror layout (`tests/unit`, `tests/api`, `tests/e2e`), full suite green before done. + +- Browser tests wait up to 60 seconds per step and retry a stalled page load once through one shared helper. Filter only proven environmental console noise, never application errors. +- Static gates run on every change: imports, linter, syntax, schema lint, UI invariants. Text gates (for example the em-dash ban) apply to docs too. +- Every behavior change ships the smallest maintained test exercising it, proven red before the fix and green after. Never weaken a failing test to get green. + +## Git law + +Stage named files only, never broad-add a dirty tree. Commit and push only on explicit ask, never as a side effect of finishing. + +- `.gitignore` contains exactly: `.venv/`, `__pycache__/`, `*.pyc`, `.pytest_cache/`, `.coverage*`, `data/`, `out/`, `.env`, `*.egg-info/`, `build/`, `dist/`, `temp_*`, `*.swp`, `*~`, `.DS_Store`. Never ignore source, tests, docs, compose files, Dockerfiles, `.env.example`, Makefiles, or CI configs. +- Any scratch, probe, or temporary file must be prefixed `temp_` and is never committed. `temp_*` in `.gitignore` enforces it. +- Markdown files are never committed without explicit user confirmation. Docs are reviewed prose, not byproducts. +- A pre-commit hook owns the patch version bump; nothing else edits the version. + +## Code law + +Explicit over implicit, flat over nested, sparse over dense, readable above all. No comments anywhere except the author line `retoor ` at the top of every file; code that needs a comment is rewritten until it does not. Defensive everywhere, errors never pass silently, no invented names with suffixes. JavaScript is ES6 modules, one class per file, one `Application` instantiated as `app` on `window`. Storage is dataset plus SQLite. The frontend is served through the backend. No emojis, no hype language, professional and scientific. + +## Docs law + +`README.md` is the living entry point (running, testing, docker, screenshots, data, operations) and stays true to the repo. Each deployment gets an operator guide with access facts, file owners, backup specification, and toggle procedures. Document only inspected facts, mark dated observations, and record anomalies instead of hiding them. + +## Handover law + +Secrets are generated with a cryptographic RNG, stored only in their target (`.env` 600, container env), and reported to the owner once in chat, never written to the repo, docs, or logs beyond that handoff. diff --git a/README.md b/README.md index cba06e4..52d863e 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,15 @@ Tests boot their own server on port 8099; set `MOLODETZ_TEST_PORT` to a free por ## Docker ``` -docker compose up -d --build # dev server on http://127.0.0.1:19847, reload on change -docker compose logs -f # follow logs -docker compose down # stop +docker compose up -d --build # staging dev server on http://127.0.0.1:19847, reload on change +docker compose -f docker-compose.test.yml up -d # public test server on http://127.0.0.1:19849 +PROD_SECRET_KEY=... PROD_ADMIN_PASSWORD=... docker compose -f docker-compose.prod.yml up -d --build # production on :19848 +docker compose logs -f # follow logs +docker compose down # stop ``` +Each compose file is its own project (`molodetz-dev`, `molodetz-test`, `molodetz-prod`) with its own `./data/` file mount, so the three never interfere. Production secrets come only from `PROD_SECRET_KEY` / `PROD_ADMIN_PASSWORD` and refuse to boot without them; `staging.app.molodetz.nl`, `molodetz-test.app.molodetz.nl`, and `molodetz.nl` proxy to `:19847`, `:19849`, and `:19848` (see `molohttp_deploy.md`). + The container runs the dev server (`uvicorn --reload`) with `./molodetz` mounted, so every saved change is live within seconds. Data lives in the file mount `./data/staging` (plain files, no named volumes anywhere). The admin account comes from `MOLODETZ_ADMIN_USERNAME` / `MOLODETZ_ADMIN_EMAIL` / `MOLODETZ_ADMIN_PASSWORD` (default password `staging-admin-pass-1`, override per shell); an empty password skips admin bootstrap. The compose port is fixed at `127.0.0.1:19847` and is what `staging.app.molodetz.nl` proxies to, so keep it stable. ``` diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 698fb29..da87c01 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,4 +1,5 @@ # retoor +name: molodetz-prod services: web: build: . diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..c3a35a5 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,22 @@ +# retoor +name: molodetz-test +services: + web: + build: . + image: molodetz-dev + container_name: molodetz-test + restart: unless-stopped + user: "1000:1000" + ports: + - "127.0.0.1:19849:8088" + volumes: + - ./molodetz:/app/molodetz + - ./pyproject.toml:/app/pyproject.toml + - ./data/test:/app/data + environment: + MOLODETZ_DATA_DIR: /app/data + MOLODETZ_ADMIN_USERNAME: ${MOLODETZ_ADMIN_USERNAME:-retoor} + MOLODETZ_ADMIN_EMAIL: ${MOLODETZ_ADMIN_EMAIL:-retoor@molodetz.nl} + MOLODETZ_ADMIN_PASSWORD: ${TEST_ADMIN_PASSWORD:-test-admin-pass-1} + SECRET_KEY: ${SECRET_KEY:-molodetz-docker-dev-secret} + MOLODETZ_STATIC_VERSION: ${MOLODETZ_STATIC_VERSION:-test} diff --git a/docker-compose.yml b/docker-compose.yml index 59f11dc..cab536a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,5 @@ # retoor +name: molodetz-dev services: web: build: . diff --git a/molohttp_deploy.md b/molohttp_deploy.md index bc84eb9..4c01ff0 100644 --- a/molohttp_deploy.md +++ b/molohttp_deploy.md @@ -20,7 +20,8 @@ molohttp is the single front door on ports 80 and 443. One systemd unit runs one - Start: `molohttp run --config /etc/molohttp/config.json --env /etc/molohttp/.env.json`, working directory `/etc/molohttp`. - Reload: `ExecReload=/bin/kill -HUP $MAINPID`, so `sudo systemctl reload molohttp` applies config without dropping connections. There is no config-validate subcommand; validate JSON by hand (see change procedure). - Privileged ports are bound via `CAP_NET_BIND_SERVICE`; the process itself is unprivileged with `ProtectSystem=strict` and `ProtectHome=true`. -- `config.json` holds 94 sites, all `enabled`, all `reverse_proxy` to `http://127.0.0.1:` with one exception: a single upstream on `http://pravda.education:10500`. One site carries a basic-auth middleware. `site-094` is `staging.app.molodetz.nl`, the Molodetz docker dev server (see Staging below). +- `config.json` holds 95 sites, all `enabled`, all `reverse_proxy` to `http://127.0.0.1:` with one exception: a single upstream on `http://pravda.education:10500`. One site carries a basic-auth middleware. Molodetz owns three: `site-022` `molodetz.nl` (production, see Cutover), `site-094` `staging.app.molodetz.nl` (see Staging), `site-095` `molodetz-test.app.molodetz.nl` (see Test). +- Hostnames must be unique across sites. On reload molohttp keeps the first site per hostname and silently drops later duplicates, persisting the result. Always assert the hostname is free before appending (observed 2026-10-05: a duplicate `test.app.molodetz.nl` entry vanished on reload because `site-085` already claimed it). - TLS: minimum TLS 1.2, HSTS one year with subdomains. Certificates live per hostname as `/etc/molohttp/certs/.pem` plus `.key`. - ACME: production Let's Encrypt, account `retoor@molodetz.nl`, renewal check every 12 hours starting 30 days before expiry, state in `/var/lib/molohttp/acme`. - Logging: JSON to stdout, collected by journald (`SyslogIdentifier=molohttp`). The `access_log_path`/`error_log_path` settings are not materialized as live files. @@ -109,6 +110,16 @@ Pick a free localhost port for the upstream and keep the app itself bound to 127 `staging.app.molodetz.nl` (`site-094`) proxies to `http://127.0.0.1:19847`, the Molodetz docker dev server from this repo (see Docker in `README.md`). It carries a normal Let's Encrypt certificate. Every saved change under `./molodetz` is live on staging within seconds through uvicorn reload; app edits never need a molohttp change. Staging data is the file mount `./data/staging`, refreshed from holy production (`./data/production`) with `make staging-refresh` (see `bin/staging-refresh.sh`). All environments live under `./data/`; named volumes are never used. +## Test + +`molodetz-test.app.molodetz.nl` (`site-095`) proxies to `http://127.0.0.1:19849`, the Molodetz docker test server (`docker-compose.test.yml`, dev server with reload, data in `./data/test`, default admin password `test-admin-pass-1`). `test.app.molodetz.nl` belongs to another app (`site-085`) and must not be touched. + +## Production cutover + +`molodetz.nl` (`site-022`) was cut over from molodev (`127.0.0.1:8084`) to the Molodetz prod container (`127.0.0.1:19848`) on 2026-10-05, backup `config.json.bak-site-cutover-prod-20261005145603`. Production data is holy in `./data/production`; the container runs `docker-compose.prod.yml` (2 workers, no reload) with secrets from `PROD_SECRET_KEY` and `PROD_ADMIN_PASSWORD`, which have no defaults and fail loudly when unset. + +Toggle back to the old app (molodev is still running on `:8084` for exactly this): back up per the specification with reason `site-cutover-rollback`, set `site-022` upstreams back to `[{"url": "http://127.0.0.1:8084", "weight": 1}]`, restore ownership, validate JSON, reload, verify `https://molodetz.nl/` answers the old site. Toggle forward again with the same steps in reverse (`:19848`). + ## Static publishing `static.molodetz.nl` proxies to `http://127.0.0.1:8117`, served by `rserver` running as `retoor` with working directory `/home/retoor/projects/static`. Publishing static content needs no molohttp change: write files under that directory and they are live (verified end to end with `/pizdetz/`, which redirects to `/pizdetz/index.html`). The Molodetz screenshot gallery is produced this way; see the Screenshots section in `README.md`. diff --git a/pyproject.toml b/pyproject.toml index 3ffb88b..24c98ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "molodetz" -version = "1.0.12" +version = "1.0.13" description = "Molodetz, a calm community blog roll." readme = "README.md" requires-python = ">=3.12" diff --git a/screenshots.py b/screenshots.py index b50e2da..fe43d16 100644 --- a/screenshots.py +++ b/screenshots.py @@ -10,7 +10,7 @@ from datetime import datetime, timezone from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent -TEST_DATA_DIR = REPO_ROOT / "data" / "test" +SHOTS_DATA_DIR = REPO_ROOT / "data" / f".tmp-shots-{os.getpid()}" OUTPUT_DIR = Path(os.environ.get("SCREENSHOTS_DIR", str(REPO_ROOT.parent / "static" / "pizdetz"))) ENV = os.environ.get("SCREENSHOTS_ENV", "test") BASE_URL = os.environ.get("SCREENSHOTS_BASE_URL", "").rstrip("/") @@ -443,11 +443,11 @@ def main(): failures = [] try: if boot: - if REPO_ROOT / "data" not in TEST_DATA_DIR.parents: + if REPO_ROOT / "data" not in SHOTS_DATA_DIR.parents: raise RuntimeError(f"refusing test data outside {REPO_ROOT / 'data'}") - shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) - TEST_DATA_DIR.mkdir(parents=True, exist_ok=True) - process, log = start_server(TEST_DATA_DIR) + shutil.rmtree(SHOTS_DATA_DIR, ignore_errors=True) + SHOTS_DATA_DIR.mkdir(parents=True, exist_ok=True) + process, log = start_server(SHOTS_DATA_DIR) seed_join(base) else: wait_live(base) @@ -466,6 +466,7 @@ def main(): finally: if process is not None: stop_server(process, log) + shutil.rmtree(SHOTS_DATA_DIR, ignore_errors=True) print(f"wrote {len(records)} screenshots for {ENV} to {out_dir}, hub covers {', '.join(present)}") for failure in failures: print(f"OVERFLOW {failure}")