diff --git a/.env.example b/.env.example index 9b2b64e..798bad6 100644 --- a/.env.example +++ b/.env.example @@ -7,8 +7,8 @@ MOLODETZ_ADMIN_PASSWORD= MOLODETZ_PORT=8088 MOLODETZ_TEST_PORT=8099 MOLODETZ_SITE_URL= -MOLODETZ_DATA_DIR=./data -MOLODETZ_DATABASE_URL=sqlite:///./data/molodetz.db +MOLODETZ_DATA_DIR=./data/development +MOLODETZ_DATABASE_URL=sqlite:///./data/development/molodetz.db MOLODETZ_INTERNAL_BASE_URL=http://localhost:8088 MOLODETZ_DISABLE_SERVICES= MOLODETZ_DISABLE_RATE_LIMIT= diff --git a/Makefile b/Makefile index 1d3464a..bf6d058 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ HOST ?= 127.0.0.1 PORT ?= 8088 WORKERS ?= 2 LOCUST_PORT ?= 8097 -LOCUST_DATA := /tmp/molodetz-locust-data +LOCUST_DATA := $(CURDIR)/data/.tmp-locust SCREENSHOTS_DIR ?= $(CURDIR)/../static/pizdetz SCREENSHOTS_PORT ?= 8094 SCREENSHOTS_STAGING_URL ?= https://staging.app.molodetz.nl diff --git a/README.md b/README.md index 84ed872..cba06e4 100644 --- a/README.md +++ b/README.md @@ -45,13 +45,13 @@ docker compose logs -f # follow logs docker compose down # stop ``` -The container runs the dev server (`uvicorn --reload`) with `./molodetz` mounted, so every saved change is live within seconds. Data lives in the file mount `/var/lib/molodetz-staging` (plain files, no named volumes anywhere). The admin account comes from `MOLODETZ_ADMIN_USERNAME` / `MOLODETZ_ADMIN_EMAIL` / `MOLODETZ_ADMIN_PASSWORD` (default password `staging-admin-pass-1`, override per shell); an empty password skips admin bootstrap. The compose port is fixed at `127.0.0.1:19847` and is what `staging.app.molodetz.nl` proxies to, so keep it stable. +The container runs the dev server (`uvicorn --reload`) with `./molodetz` mounted, so every saved change is live within seconds. Data lives in the file mount `./data/staging` (plain files, no named volumes anywhere). The admin account comes from `MOLODETZ_ADMIN_USERNAME` / `MOLODETZ_ADMIN_EMAIL` / `MOLODETZ_ADMIN_PASSWORD` (default password `staging-admin-pass-1`, override per shell); an empty password skips admin bootstrap. The compose port is fixed at `127.0.0.1:19847` and is what `staging.app.molodetz.nl` proxies to, so keep it stable. ``` make staging-refresh # copy the holy prod database over staging, then verify ``` -Refresh snapshots `/var/lib/molodetz` (online backup API, production keeps running), integrity-checks the copy, stops staging, swaps the staging file mount content, restarts, and verifies health. After a refresh the staging admin password is the production one, so pass it to `make screenshots-staging STAGING_ADMIN_PASSWORD=...`. +Refresh snapshots `./data/production` (online backup API, production keeps running), integrity-checks the copy, stops staging, swaps the staging file mount content, restarts, and verifies health. After a refresh the staging admin password is the production one, so pass it to `make screenshots-staging STAGING_ADMIN_PASSWORD=...`. ## Screenshots @@ -63,7 +63,7 @@ make screenshots-production SCREENSHOTS_ADMIN_PASSWORD=... # produ Each run captures 32 full-page desktop shots (public, docs, admin) plus a 25-cell responsive matrix (home, roll, flyers, join, docs at 1440x900, 768x1024, 390x844, 360x740, 320x568) with a horizontal-overflow check per cell, then rebuilds the hub page at `../static/pizdetz/index.html` from whichever environments exist on disk. -Rules, exactly: `test` always boots its own throwaway server and seeds one join request; `staging` and `production` shoot a live base URL and never seed. Staging logs in with `STAGING_ADMIN_PASSWORD` (default `staging-admin-pass-1`, must match the compose admin password); production has no default and fails loudly without `SCREENSHOTS_ADMIN_PASSWORD`. Any overflow is reported and exits nonzero. Override the output root with `SCREENSHOTS_DIR=...`. +Rules, exactly: `test` always wipes `./data/test`, boots its own server there, and seeds one join request; `staging` and `production` shoot a live base URL and never seed. Staging logs in with `STAGING_ADMIN_PASSWORD` (default `staging-admin-pass-1`, must match the compose admin password); production has no default and fails loudly without `SCREENSHOTS_ADMIN_PASSWORD`. Any overflow is reported and exits nonzero. Override the output root with `SCREENSHOTS_DIR=...`. ## CLI @@ -71,7 +71,15 @@ Rules, exactly: `test` always boots its own throwaway server and seeds one join ## Data -Everything lives under `data/` (database `molodetz.db`, uploads, backups, keys, locks). `.env` and `data/` are in `.gitignore`. +All application data lives under `./data/`, enforced in code: `molodetz/config.py` refuses to start with a `MOLODETZ_DATA_DIR` outside `./data` or a sqlite file outside the data dir. + +- `development` - local `make dev` runs (see `.env`) +- `staging` - docker dev server on `:19847` (file mount) +- `production` - holy production database (file mount, docker prod on `:19848`) +- `test` - screenshot runs, wiped at the start of every run +- `.tmp-pytest-`, `.tmp-locust` - single test runs, removed afterwards + +Each env dir holds `molodetz.db` (plus WAL files while running) and one subdirectory per kind: `uploads` (blobs, attachments), `backups`, `keys`, `locks`, `seo_reports`. The whole tree is git-ignored. Nothing else on the system holds application data; `molodetz/static/js/generated/EmojiMap.js` is a vendored build artifact the app only rewrites if missing. ## Operations diff --git a/bin/staging-refresh.sh b/bin/staging-refresh.sh index 03f2b0a..bd7cb70 100755 --- a/bin/staging-refresh.sh +++ b/bin/staging-refresh.sh @@ -1,22 +1,23 @@ #!/usr/bin/env bash # retoor set -euo pipefail -SRC="${1:-/var/lib/molodetz}" -DEST=/var/lib/molodetz-staging +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SRC="${1:-$ROOT/data/production}" +DEST="$ROOT/data/staging" WORK="$(mktemp -d)" -trap 'sudo -n rm -rf "$WORK"' EXIT +trap 'rm -rf "$WORK"' EXIT test -f "$SRC/molodetz.db" || { echo "no database at $SRC/molodetz.db"; exit 1; } test -d "$DEST" || { echo "no staging data dir at $DEST"; exit 1; } -sudo -n python3 -c "import sqlite3; s = sqlite3.connect('$SRC/molodetz.db'); d = sqlite3.connect('$WORK/molodetz.db'); s.backup(d); s.close(); d.close()" +python3 -c "import sqlite3; s = sqlite3.connect('$SRC/molodetz.db'); d = sqlite3.connect('$WORK/molodetz.db'); s.backup(d); s.close(); d.close()" for sub in uploads keys seo_reports; do - if [ -d "$SRC/$sub" ]; then sudo -n cp -a "$SRC/$sub" "$WORK/"; fi + if [ -d "$SRC/$sub" ]; then cp -a "$SRC/$sub" "$WORK/"; fi done -CHECK="$(sudo -n python3 -c "import sqlite3; c = sqlite3.connect('file:$WORK/molodetz.db?mode=ro', uri=True); print(c.execute('pragma integrity_check').fetchone()[0])")" +CHECK="$(python3 -c "import sqlite3; c = sqlite3.connect('file:$WORK/molodetz.db?mode=ro', uri=True); print(c.execute('pragma integrity_check').fetchone()[0])")" test "$CHECK" = "ok" || { echo "snapshot integrity check failed: $CHECK"; exit 1; } -cd "$(dirname "$0")/.." +cd "$ROOT" docker compose stop web -sudo -n rm -rf "$DEST/molodetz.db-wal" "$DEST/molodetz.db-shm" "$DEST/molodetz.db" "$DEST/uploads" "$DEST/keys" "$DEST/seo_reports" -sudo -n cp -a "$WORK/." "$DEST/" +rm -rf "$DEST/molodetz.db-wal" "$DEST/molodetz.db-shm" "$DEST/molodetz.db" "$DEST/uploads" "$DEST/keys" "$DEST/seo_reports" +cp -a "$WORK/." "$DEST/" docker compose up -d web for i in $(seq 1 60); do curl -fs http://127.0.0.1:19847/health >/dev/null && break; sleep 1; done curl -fs http://127.0.0.1:19847/health diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 6040b6b..698fb29 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -5,10 +5,11 @@ services: image: molodetz-prod container_name: molodetz-prod restart: unless-stopped + user: "1000:1000" ports: - "127.0.0.1:19848:8088" volumes: - - /var/lib/molodetz:/app/data + - ./data/production:/app/data environment: MOLODETZ_DATA_DIR: /app/data MOLODETZ_ADMIN_USERNAME: ${MOLODETZ_ADMIN_USERNAME:-retoor} diff --git a/docker-compose.yml b/docker-compose.yml index efb1384..59f11dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,11 +5,13 @@ services: image: molodetz-dev container_name: molodetz-dev restart: unless-stopped + user: "1000:1000" ports: - "127.0.0.1:19847:8088" volumes: - ./molodetz:/app/molodetz - - /var/lib/molodetz-staging:/app/data + - ./pyproject.toml:/app/pyproject.toml + - ./data/staging:/app/data environment: MOLODETZ_DATA_DIR: /app/data MOLODETZ_ADMIN_USERNAME: ${MOLODETZ_ADMIN_USERNAME:-retoor} diff --git a/molodetz/config.py b/molodetz/config.py index fd230e6..d74b014 100644 --- a/molodetz/config.py +++ b/molodetz/config.py @@ -32,6 +32,15 @@ def _env_int(name, default): DATA_DIR = Path(_env("DATA_DIR", str(ROOT / "data"))).resolve() DATABASE_URL = _env("DATABASE_URL", f"sqlite:///{DATA_DIR / 'molodetz.db'}") +DATA_ROOT = ROOT / "data" +if DATA_DIR != DATA_ROOT and DATA_ROOT not in DATA_DIR.parents: + raise RuntimeError(f"refusing MOLODETZ_DATA_DIR outside {DATA_ROOT}: {DATA_DIR}") +if DATABASE_URL.startswith("sqlite:///"): + DB_PATH = Path(DATABASE_URL.removeprefix("sqlite:///")) + if not DB_PATH.is_absolute(): + DB_PATH = Path.cwd() / DB_PATH + if DATA_DIR not in DB_PATH.resolve().parents: + raise RuntimeError(f"refusing sqlite database file outside {DATA_DIR}: {DB_PATH}") PORT = _env_int("PORT", APP_PORT) TEST_PORT = _env_int("TEST_PORT", DEFAULT_TEST_PORT) SECRET_KEY = os.environ.get("SECRET_KEY", "molodetz-development-secret-change-in-production") diff --git a/molohttp_deploy.md b/molohttp_deploy.md index a62ccf2..bc84eb9 100644 --- a/molohttp_deploy.md +++ b/molohttp_deploy.md @@ -107,7 +107,7 @@ Pick a free localhost port for the upstream and keep the app itself bound to 127 ## Staging -`staging.app.molodetz.nl` (`site-094`) proxies to `http://127.0.0.1:19847`, the Molodetz docker dev server from this repo (see Docker in `README.md`). It carries a normal Let's Encrypt certificate. Every saved change under `./molodetz` is live on staging within seconds through uvicorn reload; app edits never need a molohttp change. Staging data is the file mount `/var/lib/molodetz-staging`, refreshed from holy production with `make staging-refresh` (see `bin/staging-refresh.sh`); production data is the file mount `/var/lib/molodetz`. Named volumes are never used. +`staging.app.molodetz.nl` (`site-094`) proxies to `http://127.0.0.1:19847`, the Molodetz docker dev server from this repo (see Docker in `README.md`). It carries a normal Let's Encrypt certificate. Every saved change under `./molodetz` is live on staging within seconds through uvicorn reload; app edits never need a molohttp change. Staging data is the file mount `./data/staging`, refreshed from holy production (`./data/production`) with `make staging-refresh` (see `bin/staging-refresh.sh`). All environments live under `./data/`; named volumes are never used. ## Static publishing diff --git a/pyproject.toml b/pyproject.toml index 8299e86..3ffb88b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "molodetz" -version = "1.0.11" +version = "1.0.12" description = "Molodetz, a calm community blog roll." readme = "README.md" requires-python = ">=3.12" diff --git a/screenshots.py b/screenshots.py index 3f9d7c3..b50e2da 100644 --- a/screenshots.py +++ b/screenshots.py @@ -4,13 +4,13 @@ import os import shutil import subprocess import sys -import tempfile import time import tomllib from datetime import datetime, timezone from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent +TEST_DATA_DIR = REPO_ROOT / "data" / "test" OUTPUT_DIR = Path(os.environ.get("SCREENSHOTS_DIR", str(REPO_ROOT.parent / "static" / "pizdetz"))) ENV = os.environ.get("SCREENSHOTS_ENV", "test") BASE_URL = os.environ.get("SCREENSHOTS_BASE_URL", "").rstrip("/") @@ -437,15 +437,17 @@ def main(): base = LOCAL_BASE if boot else BASE_URL password = BOOT_ADMIN_PASSWORD if boot else ADMIN_PASSWORD out_dir = OUTPUT_DIR / ENV - data_dir = None process = None log = None records = [] failures = [] try: if boot: - data_dir = Path(tempfile.mkdtemp(prefix="molodetz-shots-")) - process, log = start_server(data_dir) + if REPO_ROOT / "data" not in TEST_DATA_DIR.parents: + raise RuntimeError(f"refusing test data outside {REPO_ROOT / 'data'}") + shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) + TEST_DATA_DIR.mkdir(parents=True, exist_ok=True) + process, log = start_server(TEST_DATA_DIR) seed_join(base) else: wait_live(base) @@ -464,8 +466,6 @@ def main(): finally: if process is not None: stop_server(process, log) - if data_dir is not None: - shutil.rmtree(data_dir, ignore_errors=True) print(f"wrote {len(records)} screenshots for {ENV} to {out_dir}, hub covers {', '.join(present)}") for failure in failures: print(f"OVERFLOW {failure}") diff --git a/tests/conftest.py b/tests/conftest.py index 8098aa4..e504aad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,14 +3,15 @@ import os import shutil import subprocess import sys -import tempfile import time from pathlib import Path import pytest REPO_ROOT = Path(__file__).resolve().parent.parent -TEST_DATA_DIR = Path(tempfile.mkdtemp(prefix="molodetz-test-")) +TEST_DATA_DIR = REPO_ROOT / "data" / f".tmp-pytest-{os.getpid()}" +shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) +TEST_DATA_DIR.mkdir(parents=True, exist_ok=True) TEST_PORT = int(os.environ.get("MOLODETZ_TEST_PORT", "8099")) ADMIN_USERNAME = "retoor" ADMIN_PASSWORD = "test-admin-pass-4f9c2a" diff --git a/tests/unit/config.py b/tests/unit/config.py new file mode 100644 index 0000000..2d54805 --- /dev/null +++ b/tests/unit/config.py @@ -0,0 +1,106 @@ +# retoor +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + +EXPECTED_KEYS = [ + "SECRET_KEY", + "PLAYWRIGHT_HEADLESS", + "PLAYWRIGHT_SLOW_MO", + "MOLODETZ_ADMIN_USERNAME", + "MOLODETZ_ADMIN_EMAIL", + "MOLODETZ_ADMIN_PASSWORD", + "MOLODETZ_PORT", + "MOLODETZ_TEST_PORT", + "MOLODETZ_SITE_URL", + "MOLODETZ_DATA_DIR", + "MOLODETZ_DATABASE_URL", + "MOLODETZ_INTERNAL_BASE_URL", + "MOLODETZ_DISABLE_SERVICES", + "MOLODETZ_DISABLE_RATE_LIMIT", + "MOLODETZ_RATE_LIMIT", + "MOLODETZ_WEB_WORKERS", + "MOLODETZ_TEMPLATE_AUTO_RELOAD", + "MOLODETZ_STATIC_VERSION", + "MOLODETZ_LOG_LEVEL", + "MOLODETZ_PRESENCE_TIMEOUT_SECONDS", + "MOLODETZ_PRESENCE_ONLINE_LIMIT", + "MOLODETZ_PRESENCE_TRACK_LIMIT", + "MOLODETZ_PRESENCE_ONLINE_MARGIN_SECONDS", + "MOLODETZ_OUTBOUND_PROXY_URL", + "MOLODETZ_RCLONE_BIN", + "MOLODETZ_RCLONE_CONFIG", + "MOLODETZ_BACKUP_OFFLOAD_REMOTE", + "MOLODETZ_SITEMAP_TTL", + "MOLODETZ_LANDING_TTL", + "MOLODETZ_UNREAD_TTL", +] + +NONEMPTY_PATH_KEYS = [ + "MOLODETZ_DATA_DIR", + "MOLODETZ_DATABASE_URL", + "MOLODETZ_INTERNAL_BASE_URL", + "MOLODETZ_RCLONE_BIN", + "MOLODETZ_RCLONE_CONFIG", +] + + +def read_example(): + values = {} + for line in (ROOT / ".env.example").read_text().splitlines(): + match = re.match(r"^([A-Z][A-Z0-9_]*)=(.*)$", line) + if match: + values[match.group(1)] = match.group(2) + return values + + +def test_env_example_lists_every_key(): + values = read_example() + assert sorted(values) == sorted(EXPECTED_KEYS) + + +def test_env_example_keeps_path_values_nonempty(): + values = read_example() + for key in NONEMPTY_PATH_KEYS: + assert values[key].strip() != "" + + +def test_config_reads_no_unknown_keys(): + source = (ROOT / "molodetz" / "config.py").read_text() + prefixed = set(re.findall(r'_env(?:_int)?\("([A-Z_]+)"', source)) + assert set(re.findall(r'os\.environ\.get\("([A-Z_]+)"', source)) <= {"SECRET_KEY", "PLAYWRIGHT_HEADLESS", "PLAYWRIGHT_SLOW_MO"} + assert {f"MOLODETZ_{name}" for name in prefixed} <= set(EXPECTED_KEYS) + + +def run_config_import(data_dir, database_url): + env = {**os.environ, "MOLODETZ_DATA_DIR": data_dir, "MOLODETZ_DATABASE_URL": database_url} + return subprocess.run( + [sys.executable, "-c", "from molodetz import config"], + cwd=ROOT, + env=env, + capture_output=True, + text=True, + ) + + +def test_data_dir_outside_root_refused(): + result = run_config_import("/tmp/evil", "sqlite:////tmp/evil/molodetz.db") + assert result.returncode != 0 + assert "outside" in result.stderr + + +def test_data_dir_inside_root_allowed(): + target = ROOT / "data" / ".tmp-guard-probe" + result = run_config_import(str(target), f"sqlite:///{target / 'molodetz.db'}") + assert result.returncode == 0 + + +def test_database_url_outside_data_dir_refused(): + target = ROOT / "data" / ".tmp-guard-probe" + result = run_config_import(str(target), "sqlite:////tmp/evil/molodetz.db") + assert result.returncode != 0 + assert "outside" in result.stderr