Zo doen wijdat in oost

tests/CLAUDE.md

@@ -24,7 +24,7 @@ The suite does NOT stop at the first failure. `-rf` lives in `pyproject.toml` `a
 | `make test-slowest` | all three tiers + the 40 slowest tests | full |
 | `make test-cache-clean` | drops `.pytest_cache/` when `--last-failed` misbehaves | instant |
 
-**Server-side display caches are disabled for the suite, never waited out.** A test that changes state the server serves from a TTL cache must not poll until the TTL expires - that burns the TTL in wall-clock on every run (the leaderboard ranking test cost 60s and the Code Farm saturation test 30s exactly this way). Each such cache reads its TTL from an env var with the production value as the default, and `conftest.py` sets it to `0` before the server subprocess starts: `DEVPLACE_SITEMAP_TTL` (`seo.SITEMAP_TTL`), `DEVPLACE_HOME_CACHE_TTL` (`main._home_cache`), `DEVPLACE_RANKING_TTL` (`database/ranking.py` `_authors_cache`), `DEVPLACE_MARKET_SATURATION_TTL` (`services/game/store/market.py` `_saturation_cache`). Add a new display cache to that list rather than sleeping in the test. **Correctness caches are different**: the cross-worker `cache_state` version read (`_cache_version_cache`, 1s) is load-bearing and stays on, so a test that flips the admin set from the test process must wait for the server to agree before asserting (see `tests/e2e/admin/containers/manage.py` `_await_instance_visible`), never assume the change is instant.
+**Server-side display caches are disabled for the suite, never waited out.** A test that changes state the server serves from a TTL cache must not poll until the TTL expires - that burns the TTL in wall-clock on every run (the leaderboard ranking test cost 60s and the Code Farm saturation test 30s exactly this way). Each such cache reads its TTL from an env var with the production value as the default, and `conftest.py` sets it to `0` before the server subprocess starts: `DEVPLACE_SITEMAP_TTL` (`seo.SITEMAP_TTL`), `DEVPLACE_HOME_CACHE_TTL` (`main._home_cache`), `DEVPLACE_RANKING_TTL` (`database/ranking.py` `_authors_cache`), `DEVPLACE_NODES_STATS_TTL` (`routers/nodes/me.py` `_public_cache`), `DEVPLACE_MARKET_SATURATION_TTL` (`services/game/store/market.py` `_saturation_cache`). Add a new display cache to that list rather than sleeping in the test. **Correctness caches are different**: the cross-worker `cache_state` version read (`_cache_version_cache`, 1s) is load-bearing and stays on, so a test that flips the admin set from the test process must wait for the server to agree before asserting (see `tests/e2e/admin/containers/manage.py` `_await_instance_visible`), never assume the change is instant.
 
 The loop: `make test-fast` -> fix -> `make test-failed` until it is empty -> `make test` as the final gate. `make test-first-failure` is only for the rare case where one early failure poisons every test after it (a corrupted session fixture, a stopped service). Pytest's cache lives in `.pytest_cache/` (git-ignored), so `--last-failed` works with no extra setup; do not pass `-p no:cacheprovider` in a run whose failures you intend to re-run. **If `make test-failed` runs the whole suite instead of just the failures, the cache is polluted**: pytest keeps `lastfailed` entries for tests it did not collect, so node ids from deleted files (or from a stray `pytest` invocation that walked `data/`/`var/`) linger forever and make `--last-failed` give up and select everything. `make test-cache-clean` drops `.pytest_cache/` and fixes it; `make clean` does it too.
 

tests/api/admin/communities/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/api/admin/communities/index.py

@@ -0,0 +1,247 @@
+# retoor <retoor@molodetz.nl>
+
+import time
+
+import pytest
+from fastapi.testclient import TestClient
+
+from devplacepy.database import get_setting, get_table, set_setting
+from devplacepy.main import app
+from devplacepy.routers.admin import communities as router_module
+from devplacepy.services.community.runner import CommunityRunner
+from devplacepy.services.community.store import store
+from devplacepy.utils import register_account
+from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep
+
+_counter = [0]
+
+
+def unique(prefix: str) -> str:
+    _counter[0] += 1
+    return f"{prefix}{int(time.time() * 1000)}{_counter[0]}"
+
+
+def make_account(role: str = "Member") -> dict:
+    name = unique("cmty")
+    uid, _, _ = register_account(name, f"{name}@t.dev", "secret123", age_band="adult", accepted_terms=True)
+    if role != "Member":
+        get_table("users").update({"uid": uid, "role": role}, ["uid"])
+    row = get_table("users").find_one(uid=uid)
+    return {"username": name, "headers": {"X-API-KEY": row["api_key"], "Accept": "application/json"}}
+
+
+@pytest.fixture(scope="module")
+def client():
+    previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins")}
+    set_setting("sync_enabled", "1")
+    set_setting("sync_origins", "")
+    with TestClient(app) as test_client:
+        yield test_client
+    for key, value in previous.items():
+        set_setting(key, value)
+
+
+@pytest.fixture
+def fake_world(monkeypatch):
+    routes = json_community()
+    monkeypatch.setattr(
+        router_module,
+        "CommunityRunner",
+        lambda: CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep),
+    )
+    get_table("communities").delete()
+    get_table("community_runs").delete()
+    get_table("sync_links").delete(origin="forum.test")
+    return routes
+
+
+@pytest.fixture
+def admin(client):
+    return make_account("Admin")
+
+
+def wait_for(predicate, seconds: float = 10.0):
+    deadline = time.time() + seconds
+    while time.time() < deadline:
+        value = predicate()
+        if value:
+            return value
+        time.sleep(0.1)
+    raise AssertionError("condition never became true")
+
+
+def add_ready(client, admin) -> dict:
+    response = client.post("/admin/communities", data={"query": "https://forum.test/api/posts"}, headers=admin["headers"])
+    assert response.status_code == 200, response.text
+    return wait_for(lambda: (row := store.get("forum-test")) and row["status"] == "ready" and row)
+
+
+def test_member_is_refused(client, admin, fake_world):
+    member = make_account()
+    assert client.get("/admin/communities", headers=member["headers"], follow_redirects=False).status_code in (303, 403)
+    assert client.post("/admin/communities", data={"query": "x.test"}, headers=member["headers"], follow_redirects=False).status_code in (303, 403)
+
+
+def test_add_learns_and_shows_detail(client, admin, fake_world):
+    row = add_ready(client, admin)
+    assert row["origin"] == "forum.test"
+    assert row["schema"]["feeds"][0]["type"] == "json_api"
+    assert row["description"] == "A friendly forum about code."
+    assert row["user_count"] == 1234
+    listing = client.get("/admin/communities", headers=admin["headers"])
+    assert listing.status_code == 200
+    assert listing.json()["items"][0]["slug"] == "forum-test"
+    detail = client.get("/admin/communities/forum-test", headers=admin["headers"])
+    assert detail.status_code == 200
+    body = detail.json()
+    assert body["feeds"][0]["pagination"] == "page"
+    assert body["collectors_lines"] > 100
+    html = client.get("/admin/communities/forum-test", headers={"X-API-KEY": admin["headers"]["X-API-KEY"]})
+    assert html.status_code == 200 and "Learned feeds" in html.text
+
+
+def test_add_is_idempotent(client, admin, fake_world):
+    add_ready(client, admin)
+    again = client.post("/admin/communities", data={"query": "https://forum.test/api/posts"}, headers=admin["headers"])
+    assert again.status_code == 200
+    assert len(store.list()) == 1
+
+
+def test_dry_run_writes_nothing_and_keeps_watermark(client, admin, fake_world):
+    row = add_ready(client, admin)
+    users_before = get_table("users").count()
+    response = client.post("/admin/communities/forum-test/run", data={"dry_run": "true"}, headers=admin["headers"])
+    assert response.status_code == 200
+    run = wait_for(lambda: next((item for item in store.runs(row["uid"]) if item["status"] == "dry_run"), None))
+    assert run["dry_run"] == 1
+    assert run["pushed"]["posts"]["created"] == 10
+    assert "DRY RUN" in run["log"]
+    assert get_table("users").count() == users_before
+    after = store.get("forum-test")
+    assert not after["watermark"]
+    assert after["run_count"] == 0
+
+
+def test_live_run_creates_then_is_unchanged(client, admin, fake_world):
+    row = add_ready(client, admin)
+    client.post("/admin/communities/forum-test/run", data={}, headers=admin["headers"])
+    first = wait_for(lambda: next((item for item in store.runs(row["uid"]) if item["status"] in ("ok", "partial", "failed")), None))
+    assert first["status"] == "ok", first["errors"]
+    assert first["pushed"]["posts"]["created"] == 10
+    assert get_table("sync_links").count(origin="forum.test", kind="posts") == 10
+    done = store.get("forum-test")
+    assert done["run_count"] == 1 and done["watermark"]["feeds"]["feed1"]["since"]
+    assert done["next_run_at"]
+    client.post("/admin/communities/forum-test/run", data={}, headers=admin["headers"])
+    second = wait_for(lambda: len([item for item in store.runs(row["uid"]) if item["status"] != "running"]) >= 2 and store.runs(row["uid"])[0])
+    assert second["pushed"].get("posts", {}).get("created", 0) == 0
+
+
+def test_live_run_refused_when_sync_disabled(client, admin, fake_world):
+    row = add_ready(client, admin)
+    set_setting("sync_enabled", "0")
+    try:
+        client.post("/admin/communities/forum-test/run", data={}, headers=admin["headers"])
+        run = wait_for(lambda: next((item for item in store.runs(row["uid"]) if item["status"] != "running"), None))
+        assert run["status"] == "failed"
+        assert any("disabled" in error for error in run["errors"])
+    finally:
+        set_setting("sync_enabled", "1")
+
+
+def test_update_and_delete(client, admin, fake_world):
+    add_ready(client, admin)
+    client.post("/admin/communities/forum-test/update", data={"interval_seconds": "7200", "enabled": "on"}, headers=admin["headers"])
+    assert store.get("forum-test")["interval_seconds"] == 7200
+    bad = client.post("/admin/communities/forum-test/update", data={"interval_seconds": "10"}, headers=admin["headers"])
+    assert bad.status_code in (400, 422)
+    client.post("/admin/communities/forum-test/delete", headers=admin["headers"])
+    assert store.get("forum-test") is None
+    assert get_table("communities").find_one(slug="forum-test")["deleted_at"]
+
+
+def test_unknown_community_is_404(client, admin, fake_world):
+    assert client.get("/admin/communities/nope", headers=admin["headers"]).status_code == 404
+
+
+GATED = [
+    ("get", "/admin/communities", {}),
+    ("post", "/admin/communities", {"query": "https://forum.test/api/posts"}),
+    ("get", "/admin/communities/forum-test", {}),
+    ("post", "/admin/communities/forum-test/run", {}),
+    ("post", "/admin/communities/forum-test/relearn", {}),
+    ("post", "/admin/communities/forum-test/update", {"interval_seconds": "7200"}),
+    ("post", "/admin/communities/forum-test/delete", {}),
+]
+
+
+@pytest.mark.parametrize("method,path,data", GATED)
+def test_every_endpoint_refuses_guests_and_members_and_changes_nothing(client, admin, fake_world, method, path, data):
+    add_ready(client, admin)
+    before = store.get("forum-test")
+    member = make_account()
+    for headers in ({"Accept": "application/json"}, member["headers"]):
+        response = client.request(method.upper(), path, data=data or None, headers=headers, follow_redirects=False)
+        assert response.status_code in (303, 401, 403), (headers, response.status_code)
+    after = store.get("forum-test")
+    assert after is not None
+    assert after["interval_seconds"] == before["interval_seconds"] and len(store.list()) == 1
+
+
+def test_password_is_stored_but_never_returned(client, admin, fake_world):
+    client.post(
+        "/admin/communities",
+        data={"query": "https://forum.test/api/posts", "login_username": "bot", "login_password": "s3cret-pw"},
+        headers=admin["headers"],
+    )
+    row = wait_for(lambda: (found := store.get("forum-test")) and found["status"] == "ready" and found)
+    assert row["login_password"] == "s3cret-pw"
+    for url in ("/admin/communities", "/admin/communities/forum-test"):
+        body = client.get(url, headers=admin["headers"])
+        assert "s3cret-pw" not in body.text and body.json()
+    page = client.get("/admin/communities/forum-test", headers={"X-API-KEY": admin["headers"]["X-API-KEY"]})
+    assert "s3cret-pw" not in page.text and "(stored)" in page.text
+    client.post("/admin/communities/forum-test/update", data={"interval_seconds": "7200", "login_username": "other"}, headers=admin["headers"])
+    assert store.get("forum-test")["login_password"] == "s3cret-pw"
+    client.post("/admin/communities/forum-test/update", data={"interval_seconds": "7200", "clear_login": "on"}, headers=admin["headers"])
+    assert store.get("forum-test")["login_password"] == ""
+
+
+def test_devii_tools_are_admin_only_and_delete_needs_confirmation():
+    from devplacepy.services.devii.actions.catalog.community import COMMUNITY_ACTIONS
+    from devplacepy.services.devii.actions.dispatcher import CONFIRM_REQUIRED
+    from devplacepy.docs_api.groups import community
+
+    assert {action.name for action in COMMUNITY_ACTIONS} >= {"community_add", "community_run", "community_delete"}
+    assert all(action.requires_admin for action in COMMUNITY_ACTIONS)
+    assert "community_delete" in CONFIRM_REQUIRED
+    assert next(action for action in COMMUNITY_ACTIONS if action.name == "community_delete").params[-1].name == "confirm"
+    assert community.GROUP["admin"] is True
+
+
+def test_update_saves_sync_options_and_keeps_unsent_ones(client, admin, fake_world):
+    add_ready(client, admin)
+    body = {"interval_seconds": 86400, "enabled": "true", "alias": "true", "sample_percent": 40, "topics": "devlog, rant"}
+    assert client.post("/admin/communities/forum-test/update", data=body, headers=admin["headers"]).status_code == 200
+    options = store.get("forum-test")["sync_options"]
+    assert options["alias"] is True and options["sample_percent"] == 40 and options["topics"] == ["devlog", "rant"]
+    client.post("/admin/communities/forum-test/update", data={"interval_seconds": 86400, "jitter_percent": 10}, headers=admin["headers"])
+    kept = store.get("forum-test")["sync_options"]
+    assert kept["alias"] is True and kept["jitter_percent"] == 10
+
+
+def test_update_rejects_bad_options_and_late_aliasing(client, admin, fake_world):
+    add_ready(client, admin)
+    bad = client.post("/admin/communities/forum-test/update", data={"interval_seconds": 86400, "topics": "nonsense"}, headers=admin["headers"])
+    assert bad.status_code == 400
+    get_table("sync_links").insert({"uid": "late1", "origin": "forum.test", "kind": "users", "origin_uid": "1", "local_uid": "x", "fingerprint": "f", "origin_updated_at": "", "first_synced_at": "", "synced_at": ""})
+    late = client.post("/admin/communities/forum-test/update", data={"interval_seconds": 86400, "alias": "true"}, headers=admin["headers"])
+    assert late.status_code == 400
+    assert not store.get("forum-test")["sync_options"].get("alias")
+
+
+def test_sync_options_are_admin_only(client, fake_world):
+    member = make_account("Member")
+    add_ready(client, make_account("Admin"))
+    assert client.get("/admin/communities/forum-test", headers=member["headers"]).status_code in (401, 403)
+    assert client.post("/admin/communities/forum-test/update", data={"interval_seconds": 86400, "alias": "true"}, headers=member["headers"]).status_code in (401, 403)

tests/api/admin/nodes/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/api/admin/nodes/conftest.py

@@ -0,0 +1,36 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+from fastapi.testclient import TestClient
+
+from devplacepy.database import get_setting, set_setting
+from devplacepy.database.nodes import create_device
+from devplacepy.main import app
+from tests.api.nodes.conftest import make_member
+
+JSON = {"Accept": "application/json"}
+
+
+def admin_headers(who: dict) -> dict:
+    return {**who["headers"], **JSON}
+
+
+def make_admin() -> dict:
+    return make_member(consent=False, enabled=False, role="Admin")
+
+
+def make_device(owner_uid: str, label: str = "Home laptop") -> dict:
+    return create_device(owner_uid, label=label, kind="native")[0]
+
+
+@pytest.fixture(scope="module")
+def client():
+    previous = get_setting("nodes_enabled", "0")
+    with TestClient(app) as test_client:
+        yield test_client
+    set_setting("nodes_enabled", previous)
+
+
+@pytest.fixture
+def admin():
+    return make_admin()

tests/api/admin/nodes/flags.py

@@ -0,0 +1,34 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database.nodes import list_flags, raise_flag
+from tests.api.admin.nodes.conftest import admin_headers, make_admin, make_device
+from tests.api.nodes.conftest import make_member
+
+
+def test_member_cannot_resolve(client):
+    member = make_member()
+    assert client.post("/admin/nodes/flags/x/resolve", headers=admin_headers(member)).status_code == 403
+
+
+def test_resolve_closes_the_flag_once(client, admin):
+    owner = make_member()
+    device = make_device(owner["uid"])
+    flag = raise_flag("refusal_mismatch", device_uid=device["uid"], user_uid=owner["uid"], detail="test")
+    listed = client.get("/admin/nodes", headers=admin_headers(admin)).json()["flags"]
+    assert flag["uid"] in {item["uid"] for item in listed}
+    assert client.post(f"/admin/nodes/flags/{flag['uid']}/resolve", headers=admin_headers(admin)).status_code == 200
+    assert flag["uid"] not in {item["uid"] for item in list_flags()}
+    assert client.post(f"/admin/nodes/flags/{flag['uid']}/resolve", headers=admin_headers(admin)).status_code == 409
+
+
+def test_unknown_flag_is_404(client, admin):
+    assert client.post("/admin/nodes/flags/nope/resolve", headers=admin_headers(admin)).status_code == 404
+
+
+def test_junior_admin_cannot_resolve_a_senior_admins_flag(client):
+    senior = make_admin()
+    junior = make_admin()
+    device = make_device(senior["uid"])
+    flag = raise_flag("integrity_failure", device_uid=device["uid"], user_uid=senior["uid"], detail="test")
+    client.post(f"/admin/nodes/flags/{flag['uid']}/resolve", headers=admin_headers(junior))
+    assert flag["uid"] in {item["uid"] for item in list_flags()}

tests/api/admin/nodes/index.py

@@ -0,0 +1,42 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database.nodes import LIMITS
+from tests.api.admin.nodes.conftest import admin_headers, make_device
+from tests.api.nodes.conftest import make_member
+
+
+def test_guest_is_refused(client):
+    assert client.get("/admin/nodes", headers={"Accept": "application/json"}).status_code == 401
+
+
+def test_member_is_refused(client):
+    member = make_member()
+    assert client.get("/admin/nodes", headers=admin_headers(member)).status_code == 403
+
+
+def test_admin_overview_shape(client, admin):
+    owner = make_member()
+    device = make_device(owner["uid"], "Overview laptop")
+    body = client.get("/admin/nodes", headers=admin_headers(admin)).json()
+    assert set(body["pool"]) >= {"online", "connected", "capacity", "open_circuits"}
+    assert body["queue"]["max_depth"] == LIMITS.max_queue_depth
+    assert body["limits"]["max_connected_nodes"] == LIMITS.max_connected_nodes
+    assert {item["key"] for item in body["settings"]} == {
+        "nodes_max_total",
+        "nodes_daily_quota_user",
+        "nodes_daily_quota_guest",
+        "nodes_cache_ttl_seconds",
+        "nodes_daily_social_user",
+        "nodes_social_via_nodes",
+    }
+    listed = {row["uid"]: row for row in body["devices"]}
+    assert listed[device["uid"]]["owner_username"] == owner["username"]
+    assert len(body["devices"]) <= 100
+    assert "secret_hash" not in listed[device["uid"]]
+
+
+def test_html_page_renders(client, admin):
+    response = client.get("/admin/nodes", headers=admin["headers"])
+    assert response.status_code == 200
+    assert "Community nodes" in response.text
+    assert "data-live=\"summary\"" in response.text

tests/api/admin/nodes/killswitch.py

@@ -0,0 +1,24 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_setting, set_setting
+from tests.api.admin.nodes.conftest import admin_headers
+from tests.api.nodes.conftest import make_member
+
+
+def test_member_cannot_flip_the_switch(client):
+    member = make_member()
+    assert client.post("/admin/nodes/killswitch", json={"enabled": True}, headers=admin_headers(member)).status_code == 403
+
+
+def test_admin_sets_and_flips_the_switch(client, admin):
+    previous = get_setting("nodes_enabled", "0")
+    try:
+        set_setting("nodes_enabled", "0")
+        on = client.post("/admin/nodes/killswitch", json={"enabled": True}, headers=admin_headers(admin))
+        assert on.json()["data"] == {"enabled": True}
+        assert get_setting("nodes_enabled", "0") == "1"
+        flipped = client.post("/admin/nodes/killswitch", json={}, headers=admin_headers(admin))
+        assert flipped.json()["data"] == {"enabled": False}
+        assert get_setting("nodes_enabled", "1") == "0"
+    finally:
+        set_setting("nodes_enabled", previous)

tests/api/admin/nodes/settings.py

@@ -0,0 +1,36 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_setting, set_setting
+from devplacepy.database.nodes import LIMITS
+from tests.api.admin.nodes.conftest import admin_headers
+from tests.api.nodes.conftest import make_member
+
+KEYS = ("nodes_max_total", "nodes_daily_quota_user", "nodes_social_via_nodes")
+
+
+def test_member_cannot_change_settings(client):
+    member = make_member()
+    assert client.post("/admin/nodes/settings", json={"nodes_max_total": 1}, headers=admin_headers(member)).status_code == 403
+
+
+def test_only_submitted_fields_change_and_ceilings_clamp(client, admin):
+    previous = {key: get_setting(key, "") for key in KEYS}
+    try:
+        response = client.post(
+            "/admin/nodes/settings",
+            json={"nodes_max_total": LIMITS.max_connected_nodes + 500, "nodes_social_via_nodes": "1"},
+            headers=admin_headers(admin),
+        )
+        data = response.json()["data"]
+        assert "nodes_max_total" in data["clamped"]
+        assert get_setting("nodes_max_total", "") == str(LIMITS.max_connected_nodes)
+        assert get_setting("nodes_social_via_nodes", "") == "1"
+        assert get_setting("nodes_daily_quota_user", "") == previous["nodes_daily_quota_user"]
+    finally:
+        for key, value in previous.items():
+            set_setting(key, value)
+
+
+def test_negative_and_unknown_values_are_rejected(client, admin):
+    assert client.post("/admin/nodes/settings", json={"nodes_max_total": -1}, headers=admin_headers(admin)).status_code in (400, 422)
+    assert client.post("/admin/nodes/settings", json={"nodes_social_via_nodes": "2"}, headers=admin_headers(admin)).status_code in (400, 422)

tests/api/admin/nodes/suspend.py

@@ -0,0 +1,47 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database.nodes import get_device
+from tests.api.admin.nodes.conftest import admin_headers, make_admin, make_device
+from tests.api.nodes.conftest import make_member
+
+
+def test_member_cannot_suspend(client):
+    member = make_member()
+    device = make_device(member["uid"])
+    response = client.post(f"/admin/nodes/devices/{device['uid']}/suspend", json={"minutes": 5}, headers=admin_headers(member))
+    assert response.status_code == 403
+
+
+def test_suspend_and_unsuspend_round_trip(client, admin):
+    owner = make_member()
+    device = make_device(owner["uid"])
+    url = f"/admin/nodes/devices/{device['uid']}"
+    response = client.post(f"{url}/suspend", json={"minutes": 30, "reason": "noisy"}, headers=admin_headers(admin))
+    assert response.json()["data"] == {"uid": device["uid"], "minutes": 30}
+    row = get_device(device["uid"])
+    assert row["status"] == "suspended"
+    assert row["suspend_reason"] == "noisy"
+    assert client.post(f"{url}/unsuspend", headers=admin_headers(admin)).status_code == 200
+    assert get_device(device["uid"])["status"] == "active"
+    assert client.post(f"{url}/unsuspend", headers=admin_headers(admin)).status_code == 409
+
+
+def test_minutes_are_bounded(client, admin):
+    owner = make_member()
+    device = make_device(owner["uid"])
+    url = f"/admin/nodes/devices/{device['uid']}/suspend"
+    for minutes in (0, 10081):
+        assert client.post(url, json={"minutes": minutes}, headers=admin_headers(admin)).status_code in (400, 422)
+
+
+def test_unknown_device_is_404(client, admin):
+    response = client.post("/admin/nodes/devices/nope/suspend", json={"minutes": 5}, headers=admin_headers(admin))
+    assert response.status_code == 404
+
+
+def test_junior_admin_cannot_suspend_a_senior_admins_device(client):
+    senior = make_admin()
+    junior = make_admin()
+    device = make_device(senior["uid"])
+    client.post(f"/admin/nodes/devices/{device['uid']}/suspend", json={"minutes": 5}, headers=admin_headers(junior))
+    assert get_device(device["uid"])["status"] == "active"

tests/api/nodes/clients.py

@@ -0,0 +1,47 @@
+# retoor <retoor@molodetz.nl>
+
+import hashlib
+
+import pytest
+
+from devplacepy import config
+
+
+@pytest.fixture
+def artifacts(tmp_path, monkeypatch):
+    monkeypatch.setattr(config, "NODE_CLIENTS_DIR", tmp_path)
+    (tmp_path / "devplace-node-linux-amd64").write_bytes(b"binary-bytes")
+    (tmp_path / "manifest.json").write_text('[{"version": "0.3.0"}]')
+    (tmp_path / "bad name.txt").write_text("x")
+    return tmp_path
+
+
+def test_client_version_lists_artifacts_with_checksums(client, artifacts):
+    body = client.get("/nodes/client-version").json()
+    assert body["version"] == "0.3.0" and body["protocol"] >= 1
+    names = [item["name"] for item in body["artifacts"]]
+    assert names == ["devplace-node-linux-amd64"]
+    assert body["artifacts"][0]["sha256"] == hashlib.sha256(b"binary-bytes").hexdigest()
+
+
+def test_download_serves_the_file_with_its_checksum(client, artifacts):
+    response = client.get("/nodes/download/devplace-node-linux-amd64")
+    assert response.status_code == 200 and response.content == b"binary-bytes"
+    assert response.headers["x-content-sha256"] == hashlib.sha256(b"binary-bytes").hexdigest()
+
+
+@pytest.mark.parametrize("name", ["manifest.json", "missing", "..%2Fsecret", "bad%20name.txt", ".hidden"])
+def test_download_refuses_everything_else(client, artifacts, name):
+    assert client.get(f"/nodes/download/{name}").status_code in (404, 422)
+
+
+def test_download_refuses_a_symlink(client, artifacts):
+    outside = artifacts.parent / "outside.bin"
+    outside.write_bytes(b"secret")
+    (artifacts / "link").symlink_to(outside)
+    assert client.get("/nodes/download/link").status_code == 404
+
+
+def test_empty_directory_is_an_empty_list(client, tmp_path, monkeypatch):
+    monkeypatch.setattr(config, "NODE_CLIENTS_DIR", tmp_path / "absent")
+    assert client.get("/nodes/client-version").json()["artifacts"] == []

tests/api/nodes/jobs.py

@@ -0,0 +1,187 @@
+# retoor <retoor@molodetz.nl>
+
+import base64
+import json
+import time
+
+import pytest
+from starlette.websockets import WebSocketDisconnect
+
+from devplacepy.database import get_table
+from devplacepy.database import nodes as store
+from devplacepy.services.manager import service_manager
+from devplacepy.services.nodes.broker import JobBroker
+from devplacepy.services.nodes.policy import DestinationPolicy
+from devplacepy.services.nodes.spec import CLOSE_PROTOCOL, SPEC
+from tests.api.nodes.conftest import make_member
+from tests.api.nodes.ws import hello, ticket_for
+
+URL = "https://www.reddit.com/r/python.json"
+BROKER = JobBroker()
+
+
+@pytest.fixture
+def relaxed(client):
+    service = service_manager.get_service("nodes")
+    original = service.scheduler.destination_ready
+    service.scheduler.destination_ready = lambda target: True
+    get_table("node_requests").delete()
+    yield service
+    service.scheduler.destination_ready = original
+    get_table("node_requests").delete()
+
+
+def enqueue(caller="caller-1", kind="user", url=URL):
+    target = DestinationPolicy().check(url)
+    return BROKER.enqueue(kind, caller, target, "any")
+
+
+def connect(client, member):
+    device_uid, _, ticket = ticket_for(client, member)
+    return device_uid, client.websocket_connect(f"/nodes/ws?ticket={ticket}")
+
+
+def next_job(ws, timeout=5.0):
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        frame = json.loads(ws.receive_text())
+        if frame["type"] == "job":
+            return frame
+        if frame["type"] == "ping":
+            ws.send_text(json.dumps({"type": "pong"}))
+    raise AssertionError("no job arrived")
+
+
+def wait_for(uid, status, timeout=5.0):
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        row = store.get_request(uid)
+        if row["status"] == status:
+            return row
+        time.sleep(0.05)
+    raise AssertionError(f"request stayed {store.get_request(uid)['status']}")
+
+
+def test_a_queued_request_reaches_the_node_and_the_result_comes_back(client, relaxed, member):
+    device_uid, connection = connect(client, member)
+    with connection as ws:
+        ws.send_text(hello(modes=["fetch"]))
+        assert json.loads(ws.receive_text())["type"] == "welcome"
+        uid = enqueue()
+        job = next_job(ws)
+        assert job["job_id"] == uid and job["url"] == URL and job["method"] == "GET"
+        assert job["timeout"] == SPEC.limits.job_timeout_seconds
+        ws.send_text(
+            json.dumps(
+                {
+                    "type": "job_result",
+                    "job_id": uid,
+                    "status": 200,
+                    "headers": {"Content-Type": "application/json"},
+                    "body_b64": base64.b64encode(b'{"ok": true}').decode(),
+                }
+            )
+        )
+        wait_for(uid, "done")
+    result = BROKER.read(uid)
+    assert result.http_status == 200 and result.body == b'{"ok": true}'
+    assert result.headers == {"content-type": "application/json"}
+    today = store.usage_today(device_uid)
+    assert today["requests_ok"] == 1 and today["bytes_down"] == 12
+    assert store.get_donor(member["uid"])["requests_ok"] == 1
+
+
+def test_a_node_that_speaks_no_known_transport_gets_nothing(client, relaxed, member):
+    _, connection = connect(client, member)
+    with connection as ws:
+        ws.send_text(hello(modes=["any"]))
+        ws.receive_text()
+        uid = enqueue()
+        time.sleep(0.7)
+        assert store.get_request(uid)["status"] == "queued"
+
+
+def test_a_donor_never_receives_its_own_request(client, relaxed, member):
+    _, connection = connect(client, member)
+    with connection as ws:
+        ws.send_text(hello(modes=["fetch"]))
+        ws.receive_text()
+        uid = enqueue(caller=member["uid"])
+        time.sleep(0.7)
+        assert store.get_request(uid)["status"] == "queued"
+
+
+def test_a_donor_opt_out_of_the_platform_is_respected_live(client, relaxed, member):
+    store.save_node_settings(member["uid"], {"allowed_platforms": []})
+    _, connection = connect(client, member)
+    with connection as ws:
+        ws.send_text(hello(modes=["fetch"]))
+        assert json.loads(ws.receive_text())["settings"]["allowed_platforms"] == []
+        uid = enqueue()
+        time.sleep(0.7)
+        assert store.get_request(uid)["status"] == "queued"
+
+
+def test_a_job_error_marks_the_request_failed(client, relaxed, member):
+    _, connection = connect(client, member)
+    with connection as ws:
+        ws.send_text(hello(modes=["fetch"]))
+        ws.receive_text()
+        uid = enqueue()
+        job = next_job(ws)
+        ws.send_text(json.dumps({"type": "job_error", "job_id": job["job_id"], "code": "timeout"}))
+        row = wait_for(uid, "failed")
+    assert row["error"] == "timeout"
+
+
+def test_disconnecting_with_a_job_in_flight_fails_it(client, relaxed, member):
+    _, connection = connect(client, member)
+    with connection as ws:
+        ws.send_text(hello(modes=["fetch"]))
+        ws.receive_text()
+        uid = enqueue()
+        next_job(ws)
+    row = wait_for(uid, "failed")
+    assert row["error"] == "node_disconnected"
+
+
+def test_a_forged_result_is_an_integrity_failure_and_a_flag(client, relaxed, member):
+    _, connection = connect(client, member)
+    with connection as ws:
+        ws.send_text(hello(modes=["fetch"]))
+        ws.receive_text()
+        uid = enqueue()
+        next_job(ws)
+        ws.send_text(
+            json.dumps({"type": "job_result", "job_id": uid, "status": 200, "body_b64": "***"})
+        )
+        row = wait_for(uid, "failed")
+    assert row["error"] == "bad_base64"
+    assert any(flag["kind"] == "integrity" for flag in store.list_flags())
+
+
+def test_answering_a_job_that_was_never_assigned_closes_the_socket(client, relaxed, member):
+    _, connection = connect(client, member)
+    with connection as ws:
+        ws.send_text(hello(modes=["fetch"]))
+        ws.receive_text()
+        ws.send_text(
+            json.dumps({"type": "job_result", "job_id": "made-up", "status": 200, "body_b64": ""})
+        )
+        with pytest.raises(WebSocketDisconnect) as info:
+            ws.receive_text()
+        assert info.value.code == CLOSE_PROTOCOL
+
+
+def test_two_donors_split_two_requests(client, relaxed):
+    first, second = make_member(), make_member()
+    _, one = connect(client, first)
+    _, two = connect(client, second)
+    with one as ws_one, two as ws_two:
+        for ws in (ws_one, ws_two):
+            ws.send_text(hello(modes=["fetch"]))
+            ws.receive_text()
+        a = enqueue(caller="x")
+        b = enqueue(caller="y", url="https://old.reddit.com/r/rust.json")
+        jobs = {next_job(ws_one)["job_id"], next_job(ws_two)["job_id"]}
+        assert jobs == {a, b}

tests/api/nodes/me.py

@@ -0,0 +1,84 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import set_consent
+from devplacepy.database.nodes import record_job, save_node_settings
+from tests.api.nodes.conftest import make_member
+
+
+def test_guests_get_401(client):
+    assert client.get("/nodes/me").status_code == 401
+    assert client.post("/nodes/settings", json={"enabled": False}).status_code == 401
+
+
+def test_me_reports_status_and_never_caches(client, member):
+    response = client.get("/nodes/me", headers=member["headers"])
+    assert response.status_code == 200
+    assert response.headers["cache-control"] == "no-store"
+    body = response.json()
+    assert body["consent"] is True and body["can_donate"] is True
+    assert body["settings"]["enabled"] is True
+    assert body["devices"] == [] and body["max_devices"] > 0
+    assert body["donated"]["lifetime_requests"] == 0
+    assert "reddit" in body["platforms"]
+
+
+def test_me_explains_why_a_member_cannot_donate(client):
+    who = make_member(consent=False, enabled=False)
+    body = client.get("/nodes/me", headers=who["headers"]).json()
+    assert body["can_donate"] is False
+    assert body["reason"] and body["message"]
+
+
+def test_settings_are_clamped_and_unsent_fields_keep_their_value(client, member):
+    response = client.post(
+        "/nodes/settings",
+        json={"bandwidth_mb_day": 100, "active_from": "08:00", "active_to": "22:00", "timezone": "Europe/Amsterdam"},
+        headers=member["headers"],
+    )
+    assert response.status_code == 200
+    settings = response.json()["settings"]
+    assert settings["bandwidth_mb_day"] == 100 and settings["active_from"] == "08:00"
+    assert settings["enabled"] is True
+    again = client.post("/nodes/settings", json={"max_concurrent": 1}, headers=member["headers"]).json()
+    assert again["settings"]["bandwidth_mb_day"] == 100 and again["settings"]["max_concurrent"] == 1
+    too_big = client.post("/nodes/settings", json={"bandwidth_mb_day": 10**9}, headers=member["headers"])
+    assert too_big.status_code in (400, 422)
+
+
+def test_switching_on_needs_the_consent(client):
+    who = make_member(consent=False, enabled=False)
+    refused = client.post("/nodes/settings", json={"enabled": True}, headers=who["headers"])
+    assert refused.status_code == 403
+    assert refused.json()["error"]["reason"] == "no_consent"
+    set_consent("user", who["uid"], "bandwidth_sharing", True)
+    assert client.post("/nodes/settings", json={"enabled": True}, headers=who["headers"]).status_code == 200
+
+
+def test_the_assistant_can_switch_off_but_never_on(client):
+    who = make_member(enabled=False)
+    headers = {**who["headers"], "X-Devii-Agent": "1"}
+    refused = client.post("/nodes/settings", json={"enabled": True}, headers=headers)
+    assert refused.status_code == 403
+    save_node_settings(who["uid"], {"enabled": True})
+    off = client.post("/nodes/settings", json={"enabled": False}, headers=headers)
+    assert off.status_code == 200 and off.json()["settings"]["enabled"] is False
+
+
+def test_stats_are_aggregate_only(client, member):
+    device = client.post("/nodes/devices", json={"label": "x"}, headers=member["headers"]).json()["device"]
+    record_job(device["uid"], member["uid"], ok=True, bytes_up=10, bytes_down=90)
+    body = client.get("/nodes/stats").json()
+    assert set(body) == {"requests", "bytes", "donors"}
+    assert body["requests"] >= 1 and body["bytes"] >= 100 and body["donors"] >= 1
+    assert member["username"] not in str(body)
+
+
+def test_leaderboard_lists_only_members_who_opted_in(client):
+    hidden = make_member()
+    shown = make_member()
+    for who, public in ((hidden, False), (shown, True)):
+        save_node_settings(who["uid"], {"public_display": public})
+        device = client.post("/nodes/devices", json={"label": "x"}, headers=who["headers"]).json()["device"]
+        record_job(device["uid"], who["uid"], ok=True, bytes_up=1, bytes_down=5)
+    names = [item["username"] for item in client.get("/nodes/leaderboard").json()["donors"]]
+    assert shown["username"] in names and hidden["username"] not in names

tests/api/nodes/social.py

@@ -0,0 +1,65 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+from devplacepy.database import get_setting, set_setting
+from devplacepy.services.nodes import social as social_module
+
+HN = json.dumps({"hits": [{"objectID": "9", "title": "A story", "points": 4}]}).encode()
+
+
+def test_guests_get_401(client):
+    assert client.get("/nodes/social/search?q=x").status_code == 401
+    assert client.post("/nodes/internal/fetch", json={"url": "https://www.reddit.com/"}).status_code == 401
+
+
+def test_query_is_required(client, member):
+    assert client.get("/nodes/social/search", headers=member["headers"]).status_code == 400
+
+
+def test_search_returns_hits_and_reasons(client, member, monkeypatch):
+    async def getter(url):
+        return HN
+
+    monkeypatch.setattr(social_module, "direct_get", getter)
+    body = client.get(
+        "/nodes/social/search?q=sqlite&platforms=hackernews,reddit", headers=member["headers"]
+    ).json()
+    assert body["hits"][0]["platform"] == "hackernews"
+    assert body["unavailable"] == {"reddit": "nodes_not_used"}
+    assert body["platforms"] == ["hackernews", "reddit"]
+
+
+def test_search_has_a_daily_quota(client, member, monkeypatch):
+    async def getter(url):
+        return HN
+
+    monkeypatch.setattr(social_module, "direct_get", getter)
+    previous = get_setting("nodes_daily_social_user", "300")
+    set_setting("nodes_daily_social_user", "2")
+    try:
+        codes = [
+            client.get("/nodes/social/search?q=a&platforms=hackernews", headers=member["headers"]).status_code
+            for _ in range(3)
+        ]
+    finally:
+        set_setting("nodes_daily_social_user", previous)
+    assert codes == [200, 200, 429]
+
+
+def test_internal_fetch_refuses_hosts_off_the_allowlist(client, member):
+    response = client.post(
+        "/nodes/internal/fetch", json={"url": "https://example.com/"}, headers=member["headers"]
+    )
+    assert response.status_code == 403
+    assert response.json()["error"]["reason"] == "host_not_allowed"
+
+
+def test_internal_fetch_without_donors_is_503(client, member):
+    response = client.post(
+        "/nodes/internal/fetch",
+        json={"url": "https://www.reddit.com/r/python.json", "cache": False},
+        headers=member["headers"],
+    )
+    assert response.status_code == 503
+    assert response.json()["error"]["reason"] in ("no_nodes", "kill_switch")

tests/api/sync/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/api/sync/comments.py

@@ -0,0 +1,70 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_table
+from tests.api.sync.conftest import comment_record, post_record, sync_ok, unique, user_record
+
+
+def seed_thread(client, admin, origin):
+    sync_ok(client, admin, "users", origin, [user_record("u1", unique("cu1")), user_record("u2", unique("cu2"))])
+    sync_ok(client, admin, "posts", origin, [post_record("p1", "u1"), post_record("p2", "u1")])
+
+
+def test_unknown_references_are_rejected_with_specific_reasons(client, admin, origin):
+    seed_thread(client, admin, origin)
+    body = sync_ok(
+        client, admin, "comments", origin,
+        [
+            comment_record("c1", author="nobody"),
+            comment_record("c2", post="nopost"),
+            comment_record("c3", parent_uid="noparent"),
+        ],
+    )
+    assert [r["reason"] for r in body["results"]] == ["unknown_author", "unknown_post", "unknown_parent"]
+
+
+def test_create_with_nested_reply_in_one_batch(client, admin, origin):
+    seed_thread(client, admin, origin)
+    body = sync_ok(
+        client, admin, "comments", origin,
+        [comment_record("c1", author="u2"), comment_record("c2", author="u1", parent_uid="c1")],
+    )
+    assert [r["status"] for r in body["results"]] == ["created", "created"]
+    parent, reply = (get_table("comments").find_one(uid=r["local_uid"]) for r in body["results"])
+    post = get_table("posts").find_one(uid=reply["post_uid"])
+    assert reply["parent_uid"] == parent["uid"]
+    assert reply["target_type"] == "post" and reply["target_uid"] == post["uid"]
+    assert parent["deleted_at"] is None
+
+
+def test_parent_on_a_different_post_is_rejected(client, admin, origin):
+    seed_thread(client, admin, origin)
+    sync_ok(client, admin, "comments", origin, [comment_record("c1", post="p1")])
+    body = sync_ok(client, admin, "comments", origin, [comment_record("c2", post="p2", parent_uid="c1")])
+    assert body["results"][0]["reason"] == "parent_mismatch"
+
+
+def test_edit_replay_and_tombstone(client, admin, origin):
+    seed_thread(client, admin, origin)
+    created = sync_ok(client, admin, "comments", origin, [comment_record("c1")])["results"][0]
+    assert sync_ok(client, admin, "comments", origin, [comment_record("c1")])["results"][0]["status"] == "unchanged"
+    edited = sync_ok(
+        client, admin, "comments", origin,
+        [comment_record("c1", content="edited comment", updated_at="2025-02-01T00:00:00Z")],
+    )["results"][0]
+    assert edited["status"] == "updated"
+    assert get_table("comments").find_one(uid=created["local_uid"])["content"] == "edited comment"
+    sync_ok(
+        client, admin, "comments", origin,
+        [comment_record("c1", content="edited comment", updated_at="2025-03-01T00:00:00Z", deleted=True)],
+    )
+    row = get_table("comments").find_one(uid=created["local_uid"])
+    assert row["deleted_at"] and row["deleted_by"] == "sync"
+
+
+def test_synced_comment_renders_on_the_post_page(client, admin, origin):
+    seed_thread(client, admin, origin)
+    sync_ok(client, admin, "comments", origin, [comment_record("c1", content="comment visible on the page")])
+    link = get_table("sync_links").find_one(origin=origin, kind="posts", origin_uid="p1")
+    slug = get_table("posts").find_one(uid=link["local_uid"])["slug"]
+    page = client.get(f"/posts/{slug}")
+    assert page.status_code == 200 and "comment visible on the page" in page.text

tests/api/sync/conftest.py

@@ -0,0 +1,98 @@
+# retoor <retoor@molodetz.nl>
+
+import time
+
+import pytest
+from fastapi.testclient import TestClient
+
+from devplacepy.database import get_setting, get_table, set_setting
+from devplacepy.main import app
+from devplacepy.utils import register_account
+
+_counter = [0]
+
+
+def unique(prefix: str) -> str:
+    _counter[0] += 1
+    return f"{prefix}{int(time.time() * 1000)}{_counter[0]}"
+
+
+def make_account(role: str = "Member") -> dict:
+    name = unique("syncmb")
+    uid, _, _ = register_account(
+        name, f"{name}@t.dev", "secret123", age_band="adult", accepted_terms=True
+    )
+    if role != "Member":
+        get_table("users").update({"uid": uid, "role": role}, ["uid"])
+    row = get_table("users").find_one(uid=uid)
+    return {
+        "uid": uid,
+        "username": name,
+        "headers": {"X-API-KEY": row["api_key"], "Accept": "application/json"},
+    }
+
+
+@pytest.fixture(scope="module")
+def client():
+    previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins")}
+    set_setting("sync_enabled", "1")
+    set_setting("sync_origins", "")
+    with TestClient(app) as test_client:
+        yield test_client
+    for key, value in previous.items():
+        set_setting(key, value)
+
+
+@pytest.fixture
+def admin(client):
+    return make_account("Admin")
+
+
+@pytest.fixture
+def origin():
+    return f"{unique('o')}.example.org"
+
+
+def sync(client, admin, kind, origin, records, **extra):
+    response = client.post(
+        f"/sync/{kind}",
+        json={"origin": origin, "records": records, **extra},
+        headers=admin["headers"],
+    )
+    return response
+
+
+def sync_ok(client, admin, kind, origin, records, **extra) -> dict:
+    response = sync(client, admin, kind, origin, records, **extra)
+    assert response.status_code == 200, response.text
+    return response.json()
+
+
+def user_record(uid="u1", username="syncuser", **extra) -> dict:
+    return {
+        "origin_uid": uid,
+        "username": username,
+        "created_at": "2025-01-01T00:00:00Z",
+        **extra,
+    }
+
+
+def post_record(uid="p1", author="u1", **extra) -> dict:
+    return {
+        "origin_uid": uid,
+        "author_uid": author,
+        "content": "a synced post with enough content",
+        "created_at": "2025-01-02T00:00:00Z",
+        **extra,
+    }
+
+
+def comment_record(uid="c1", post="p1", author="u1", **extra) -> dict:
+    return {
+        "origin_uid": uid,
+        "post_uid": post,
+        "author_uid": author,
+        "content": "a synced comment",
+        "created_at": "2025-01-03T00:00:00Z",
+        **extra,
+    }

tests/api/sync/posts.py

@@ -0,0 +1,118 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_table
+from tests.api.sync.conftest import post_record, sync_ok, unique, user_record
+
+
+def seed_author(client, admin, origin, uid="u1"):
+    body = sync_ok(client, admin, "users", origin, [user_record(uid, unique("author"))])
+    return body["results"][0]["local_uid"]
+
+
+def test_unknown_author_is_rejected_with_reason(client, admin, origin):
+    body = sync_ok(client, admin, "posts", origin, [post_record(author="missing")])
+    result = body["results"][0]
+    assert result["status"] == "rejected" and result["reason"] == "unknown_author"
+    assert get_table("sync_links").count(origin=origin) == 0
+
+
+def test_create_normalizes_and_has_no_side_effects(client, admin, origin):
+    author = seed_author(client, admin, origin)
+    notifications_before = get_table("notifications").count()
+    body = sync_ok(
+        client, admin, "posts", origin,
+        [post_record("p1", title="  Hello   World  ", content="line one\r\nline two ok", topic="nonsense", stars=50)],
+    )
+    result = body["results"][0]
+    assert result["status"] == "created" and result["ignored"] == ["stars"]
+    row = get_table("posts").find_one(uid=result["local_uid"])
+    assert row["user_uid"] == author
+    assert row["title"] == "Hello World"
+    assert row["content"] == "line one\nline two ok"
+    assert row["topic"] == "random"
+    assert row["stars"] == 0
+    assert row["created_at"] == "2025-01-02T00:00:00+00:00"
+    assert row["slug"].endswith("hello-world")
+    assert row["deleted_at"] is None
+    assert get_table("notifications").count() == notifications_before
+    assert get_table("users").find_one(uid=author)["xp"] == 0
+
+
+def test_replay_update_stale_and_tombstone(client, admin, origin):
+    seed_author(client, admin, origin)
+    created = sync_ok(client, admin, "posts", origin, [post_record("p1")])["results"][0]
+    assert sync_ok(client, admin, "posts", origin, [post_record("p1")])["results"][0]["status"] == "unchanged"
+    edited = sync_ok(
+        client, admin, "posts", origin,
+        [post_record("p1", content="edited synced content", updated_at="2025-03-01T00:00:00Z")],
+    )["results"][0]
+    assert edited["status"] == "updated" and edited["local_uid"] == created["local_uid"]
+    assert get_table("posts").find_one(uid=created["local_uid"])["content"] == "edited synced content"
+    stale = sync_ok(
+        client, admin, "posts", origin,
+        [post_record("p1", content="older content here", updated_at="2025-01-05T00:00:00Z")],
+    )["results"][0]
+    assert stale["status"] == "stale"
+    deleted = sync_ok(
+        client, admin, "posts", origin,
+        [post_record("p1", content="edited synced content", updated_at="2025-04-01T00:00:00Z", deleted=True)],
+    )["results"][0]
+    assert deleted["status"] == "updated"
+    row = get_table("posts").find_one(uid=created["local_uid"])
+    assert row["deleted_at"] and row["deleted_by"] == "sync"
+    revived = sync_ok(
+        client, admin, "posts", origin,
+        [post_record("p1", content="edited synced content", updated_at="2025-05-01T00:00:00Z")],
+    )["results"][0]
+    assert revived["status"] == "updated"
+    assert get_table("posts").find_one(uid=created["local_uid"])["deleted_at"] is None
+
+
+def test_locally_moderated_post_is_not_resurrected(client, admin, origin):
+    seed_author(client, admin, origin)
+    created = sync_ok(client, admin, "posts", origin, [post_record("p1")])["results"][0]
+    get_table("posts").update(
+        {"uid": created["local_uid"], "deleted_at": "2025-02-01T00:00:00+00:00", "deleted_by": admin["uid"]},
+        ["uid"],
+    )
+    again = sync_ok(
+        client, admin, "posts", origin,
+        [post_record("p1", content="remote edit arrives", updated_at="2025-03-01T00:00:00Z")],
+    )["results"][0]
+    assert again["status"] == "unchanged" and again["reason"] == "locally_deleted"
+    assert get_table("posts").find_one(uid=created["local_uid"])["content"] != "remote edit arrives"
+
+
+def test_purged_local_copy_is_not_recreated(client, admin, origin):
+    seed_author(client, admin, origin)
+    created = sync_ok(client, admin, "posts", origin, [post_record("p1")])["results"][0]
+    get_table("posts").delete(uid=created["local_uid"])
+    again = sync_ok(
+        client, admin, "posts", origin,
+        [post_record("p1", content="changed after purge", updated_at="2025-03-01T00:00:00Z")],
+    )["results"][0]
+    assert again["status"] == "rejected" and again["reason"] == "purged_locally"
+    assert get_table("posts").count(uid=created["local_uid"]) == 0
+
+
+def test_invalid_post_reports_field_errors(client, admin, origin):
+    seed_author(client, admin, origin)
+    body = sync_ok(client, admin, "posts", origin, [post_record("p1", content="short")])
+    result = body["results"][0]
+    assert result["status"] == "rejected" and result["reason"] == "invalid"
+    assert any(error.startswith("content:") for error in result["errors"])
+
+
+def test_future_timestamps_are_clamped(client, admin, origin):
+    seed_author(client, admin, origin)
+    body = sync_ok(client, admin, "posts", origin, [post_record("p1", created_at="2999-01-01T00:00:00Z")])
+    row = get_table("posts").find_one(uid=body["results"][0]["local_uid"])
+    assert row["created_at"] < "2100"
+
+
+def test_post_shows_up_on_its_detail_page(client, admin, origin):
+    seed_author(client, admin, origin)
+    body = sync_ok(client, admin, "posts", origin, [post_record("p1", title="Visible synced post")])
+    slug = get_table("posts").find_one(uid=body["results"][0]["local_uid"])["slug"]
+    page = client.get(f"/posts/{slug}")
+    assert page.status_code == 200 and "Visible synced post" in page.text

tests/api/sync/settings.py

@@ -0,0 +1,39 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_setting, set_setting
+from tests.api.sync.conftest import sync_ok, unique, user_record
+
+
+def test_admin_settings_save_normalizes_sync_origins(client, admin):
+    previous = get_setting("sync_origins", "")
+    try:
+        response = client.post(
+            "/admin/settings",
+            json={"sync_origins": "A.example.org, b.example.org\nA.example.org"},
+            headers=admin["headers"],
+        )
+        assert response.status_code == 200, response.text
+        set_setting_cache_free = get_setting("sync_origins", "")
+        assert set_setting_cache_free == "a.example.org,b.example.org"
+        refused = client.post(
+            "/admin/settings", json={"sync_origins": "bad origin!"}, headers=admin["headers"]
+        )
+        assert refused.status_code in (400, 422)
+    finally:
+        set_setting("sync_origins", previous)
+
+
+def test_admin_settings_page_shows_sync_controls(client, admin):
+    page = client.get("/admin/settings", headers={"X-API-KEY": admin["headers"]["X-API-KEY"]})
+    assert page.status_code == 200
+    assert 'name="sync_enabled"' in page.text and 'name="sync_origins"' in page.text
+
+
+def test_clearing_sync_origins_removes_the_allowlist(client, admin, origin):
+    set_setting("sync_origins", "only.example.org")
+    try:
+        client.post("/admin/settings", json={"sync_origins": ""}, headers=admin["headers"])
+        assert get_setting("sync_origins", "x") == ""
+        sync_ok(client, admin, "users", origin, [user_record("u1", unique("any"))])
+    finally:
+        set_setting("sync_origins", "")

tests/api/sync/spec.py

@@ -0,0 +1,15 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.sync.spec import SPEC
+
+
+def test_spec_is_public_and_matches_code(client):
+    response = client.get("/sync/spec.json")
+    assert response.status_code == 200
+    body = response.json()
+    assert body["kinds"] == list(SPEC.kinds)
+    assert body["statuses"] == list(SPEC.statuses)
+    assert body["limits"]["max_batch_records"] == SPEC.limits.max_batch_records
+    assert set(body["records"]) == set(SPEC.kinds)
+    assert "username" in body["records"]["users"]["properties"]
+    assert "stars" not in body["records"]["posts"]["properties"]

tests/api/sync/users.py

@@ -0,0 +1,173 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_table, set_setting
+from tests.api.sync.conftest import make_account, sync, sync_ok, unique, user_record
+
+
+def test_requires_authentication(client, origin):
+    response = client.post(
+        "/sync/users", json={"origin": origin, "records": [user_record()]}
+    )
+    assert response.status_code == 401
+
+
+def test_member_is_refused(client, origin):
+    member = make_account()
+    response = sync(client, member, "users", origin, [user_record()])
+    assert response.status_code == 403
+
+
+def test_kill_switch_refuses_when_disabled(client, admin, origin):
+    set_setting("sync_enabled", "0")
+    try:
+        response = sync(client, admin, "users", origin, [user_record()])
+        assert response.status_code == 503
+    finally:
+        set_setting("sync_enabled", "1")
+
+
+def test_origin_allowlist(client, admin, origin):
+    set_setting("sync_origins", "trusted.example.org")
+    try:
+        assert sync(client, admin, "users", origin, [user_record()]).status_code == 403
+        allowed = sync(client, admin, "users", "trusted.example.org", [user_record(unique("a"), unique("allow"))])
+        assert allowed.status_code == 200
+    finally:
+        set_setting("sync_origins", "")
+
+
+def test_invalid_envelope_is_422(client, admin, origin):
+    assert sync(client, admin, "users", origin, []).status_code == 422
+    assert sync(client, admin, "users", "bad origin!", [user_record()]).status_code == 422
+
+
+def test_create_then_replay_is_idempotent(client, admin, origin):
+    name = unique("bob")
+    first = sync_ok(client, admin, "users", origin, [user_record("u1", name, bio="hi")])
+    result = first["results"][0]
+    assert result["status"] == "created" and result["local_uid"]
+    again = sync_ok(client, admin, "users", origin, [user_record("u1", name, bio="hi")])
+    assert again["results"][0]["status"] == "unchanged"
+    assert again["results"][0]["local_uid"] == result["local_uid"]
+    assert get_table("users").count(username=name) == 1
+
+
+def test_update_changes_profile_and_keeps_local_uid(client, admin, origin):
+    name = unique("upd")
+    created = sync_ok(client, admin, "users", origin, [user_record("u1", name)])
+    uid = created["results"][0]["local_uid"]
+    updated = sync_ok(
+        client,
+        admin,
+        "users",
+        origin,
+        [user_record("u1", name, bio="new bio", updated_at="2025-06-01T00:00:00Z")],
+    )
+    assert updated["results"][0]["status"] == "updated"
+    assert get_table("users").find_one(uid=uid)["bio"] == "new bio"
+
+
+def test_older_update_is_stale(client, admin, origin):
+    name = unique("stale")
+    sync_ok(client, admin, "users", origin, [user_record("u1", name, updated_at="2025-06-01T00:00:00Z")])
+    older = sync_ok(
+        client, admin, "users", origin,
+        [user_record("u1", name, bio="old", updated_at="2025-02-01T00:00:00Z")],
+    )
+    assert older["results"][0]["status"] == "stale"
+
+
+def test_username_is_normalized_and_collisions_get_a_suffix(client, admin, origin):
+    taken = unique("taken")
+    make_account_row = get_table("users")
+    sync_ok(client, admin, "users", origin, [user_record("u0", taken)])
+    other = sync_ok(client, admin, "users", origin, [user_record("u1", taken)])
+    local = make_account_row.find_one(uid=other["results"][0]["local_uid"])
+    assert local["username"] != taken and local["username"].startswith(taken[:10])
+    odd = sync_ok(client, admin, "users", origin, [user_record("u2", "Ünï cøde!")])
+    created = make_account_row.find_one(uid=odd["results"][0]["local_uid"])
+    assert created["username"].replace("_", "").replace("-", "").isalnum()
+
+
+def test_counters_credentials_and_role_are_ignored_and_reported(client, admin, origin):
+    body = sync_ok(
+        client, admin, "users", origin,
+        [user_record("u1", unique("priv"), xp=999, level=50, stars=7, role="Admin", email="x@y.z", password_hash="p")],
+    )
+    result = body["results"][0]
+    assert set(result["ignored"]) == {"xp", "level", "stars", "role", "email", "password_hash"}
+    row = get_table("users").find_one(uid=result["local_uid"])
+    assert row["role"] == "Member" and row["xp"] == 0 and row["level"] == 1
+    assert row["email"].endswith("@sync.invalid")
+
+
+def test_synced_account_cannot_log_in(client, admin, origin):
+    body = sync_ok(client, admin, "users", origin, [user_record("u1", unique("nologin"))])
+    row = get_table("users").find_one(uid=body["results"][0]["local_uid"])
+    response = client.post(
+        "/auth/login",
+        json={"email": row["email"], "password": "anything1"},
+        headers={"Accept": "application/json"},
+    )
+    assert response.status_code == 401
+
+
+def test_dry_run_writes_nothing(client, admin, origin):
+    name = unique("dry")
+    body = sync_ok(client, admin, "users", origin, [user_record("u1", name)], dry_run=True)
+    assert body["dry_run"] is True and body["results"][0]["status"] == "created"
+    assert get_table("users").count(username=name) == 0
+    assert get_table("sync_links").count(origin=origin) == 0
+
+
+def test_one_bad_record_does_not_fail_the_batch(client, admin, origin):
+    body = sync_ok(
+        client, admin, "users", origin,
+        [user_record("u1", unique("good")), {"origin_uid": "", "username": "x"}, user_record("u3", unique("good"), created_at="nonsense")],
+    )
+    assert [r["status"] for r in body["results"]] == ["created", "rejected", "rejected"]
+    assert body["counts"]["created"] == 1 and body["counts"]["rejected"] == 2
+    assert all(r["errors"] for r in body["results"][1:])
+
+
+def test_deleted_flag_deactivates_account(client, admin, origin):
+    name = unique("gone")
+    created = sync_ok(client, admin, "users", origin, [user_record("u1", name)])
+    sync_ok(client, admin, "users", origin, [user_record("u1", name, deleted=True, updated_at="2025-05-01T00:00:00Z")])
+    assert not get_table("users").find_one(uid=created["results"][0]["local_uid"])["is_active"]
+
+
+def test_tombstone_for_unknown_record_creates_nothing(client, admin, origin):
+    body = sync_ok(client, admin, "users", origin, [user_record("u1", unique("ghost"), deleted=True)])
+    assert body["results"][0]["status"] == "unchanged"
+    assert body["results"][0]["reason"] == "tombstone_for_unknown"
+    assert get_table("sync_links").count(origin=origin) == 0
+
+
+def test_same_origin_uid_in_different_origins_are_distinct(client, admin, origin):
+    first = sync_ok(client, admin, "users", origin, [user_record("u1", unique("one"))])
+    second = sync_ok(client, admin, "users", f"b.{origin}", [user_record("u1", unique("two"))])
+    assert first["results"][0]["local_uid"] != second["results"][0]["local_uid"]
+
+
+def test_form_encoded_body_with_json_records_is_accepted(client, admin, origin):
+    import json
+
+    name = unique("form")
+    response = client.post(
+        "/sync/users",
+        data={"origin": origin, "records": json.dumps([user_record("u1", name)]), "dry_run": "true"},
+        headers=admin["headers"],
+    )
+    assert response.status_code == 200, response.text
+    assert response.json()["results"][0]["status"] == "created"
+    assert get_table("users").count(username=name) == 0
+
+
+def test_malformed_records_string_is_422(client, admin, origin):
+    response = client.post(
+        "/sync/users",
+        data={"origin": origin, "records": "not json"},
+        headers=admin["headers"],
+    )
+    assert response.status_code in (400, 422)

tests/conftest.py

@@ -38,6 +38,7 @@ os.environ["DEVPLACE_DISABLE_RATE_LIMIT"] = "1"
 os.environ["DEVPLACE_SITEMAP_TTL"] = "0"
 os.environ["DEVPLACE_HOME_CACHE_TTL"] = "0"
 os.environ["DEVPLACE_RANKING_TTL"] = "0"
+os.environ["DEVPLACE_NODES_STATS_TTL"] = "0"
 os.environ["DEVPLACE_MARKET_SATURATION_TTL"] = "0"
 os.environ["DEVPLACE_HAPPY_404_POOL_TTL"] = "0"
 os.environ["DEVPLACE_FEATURED_TOPICS_POOL_TTL"] = "0"

tests/unit/cli/nodes.py

@@ -0,0 +1,44 @@
+# retoor <retoor@molodetz.nl>
+
+import argparse
+
+from devplacepy.cli import nodes as cli
+from devplacepy.database import get_table
+from devplacepy.database import nodes as store
+from devplacepy.utils import generate_uid
+
+
+def parser():
+    root = argparse.ArgumentParser()
+    cli.register_nodes(root.add_subparsers(dest="command"))
+    return root
+
+
+def device(local_db):
+    row, _ = store.create_device(generate_uid(), "cli", "native")
+    return row["uid"]
+
+
+def test_suspend_and_unsuspend(local_db, capsys):
+    uid = device(local_db)
+    args = parser().parse_args(["nodes", "suspend", uid, "--minutes", "5"])
+    args.func(args)
+    assert store.effective_status(get_table("node_devices").find_one(uid=uid)) == "suspended"
+    args = parser().parse_args(["nodes", "unsuspend", uid])
+    args.func(args)
+    assert store.effective_status(get_table("node_devices").find_one(uid=uid)) == "active"
+
+
+def test_clear_is_a_dry_run_without_force(local_db, capsys):
+    get_table("node_events").insert({"uid": generate_uid(), "kind": "x", "detail": "", "created_at": "2026-01-01T00:00:00+00:00"})
+    before = store.clear_ephemeral(dry_run=True)["node_events"]
+    args = parser().parse_args(["nodes", "clear"])
+    args.func(args)
+    assert "pass --force" in capsys.readouterr().out
+    assert store.clear_ephemeral(dry_run=True)["node_events"] == before
+
+
+def test_prune_dry_run_reports_counts(local_db, capsys):
+    args = parser().parse_args(["nodes", "prune", "--dry-run"])
+    args.func(args)
+    assert "would remove" in capsys.readouterr().out

tests/unit/database/nodes.py

@@ -225,3 +225,170 @@ def test_ip_is_only_ever_stored_hashed():
     assert "203" not in digest and len(digest) == 32
     assert store.hash_ip("203.0.113.9") == digest
     assert store.hash_ip("") == ""
+
+
+def queue_one(url="https://reddit.com/r/x", caller="c1"):
+    return store.enqueue_request("user", caller, url, "GET", "reddit", "any")
+
+
+def test_the_queue_is_bounded(local_db):
+    get_table("node_requests").delete()
+    for _ in range(LIMITS.max_queue_depth):
+        queue_one()
+    with pytest.raises(store.QueueFull):
+        queue_one()
+    assert store.count_open_requests() == LIMITS.max_queue_depth
+    first = store.list_queued(1)[0]
+    store.fail_request(first["uid"], "x")
+    queue_one()
+    get_table("node_requests").delete()
+
+
+def test_a_request_is_claimed_by_exactly_one_assignment(local_db):
+    get_table("node_requests").delete()
+    uid = queue_one()
+    assert store.claim_request(uid, "device-a") is True
+    assert store.claim_request(uid, "device-b") is False
+    assert store.get_request(uid)["device_uid"] == "device-a"
+    get_table("node_requests").delete()
+
+
+def test_a_result_can_only_land_on_an_assigned_request(local_db):
+    get_table("node_requests").delete()
+    uid = queue_one()
+    done = dict(result_status=200, result_headers="{}", result_body="eA==", result_bytes=1, truncated=False)
+    assert store.finish_request(uid, **done) is False
+    store.claim_request(uid, "d")
+    assert store.finish_request(uid, **done) is True
+    assert store.finish_request(uid, **done) is False
+    assert store.fail_request(uid, "late") is False
+    assert store.get_request(uid)["status"] == "done"
+    get_table("node_requests").delete()
+
+
+def test_expiry_separates_waiting_from_stuck_requests(local_db):
+    get_table("node_requests").delete()
+    waiting, stuck, fresh = queue_one(), queue_one(), queue_one()
+    store.claim_request(stuck, "d")
+    later = datetime.now(timezone.utc) + timedelta(
+        seconds=LIMITS.job_timeout_seconds + LIMITS.job_grace_seconds + LIMITS.queue_max_wait_seconds + 1
+    )
+    counts = store.expire_requests(later)
+    assert counts == {"queued": 2, "assigned": 1}
+    assert store.get_request(waiting)["status"] == "expired"
+    assert store.get_request(stuck)["error"] == "timeout"
+    assert store.get_request(fresh)["status"] == "expired"
+    assert store.expire_requests(later) == {"queued": 0, "assigned": 0}
+    get_table("node_requests").delete()
+
+
+def test_requests_are_listed_oldest_first_and_limited(local_db):
+    get_table("node_requests").delete()
+    uids = [queue_one() for _ in range(5)]
+    assert [row["uid"] for row in store.list_queued(3)] == uids[:3]
+    get_table("node_requests").delete()
+
+
+def test_request_garbage_collection_leaves_recent_rows(local_db):
+    get_table("node_requests").delete()
+    queue_one()
+    assert store.prune_requests() == 0
+    far = datetime.now(timezone.utc) + timedelta(seconds=LIMITS.request_retention_seconds + 5)
+    assert store.prune_requests(dry_run=True, moment=far) == 1
+    assert store.prune_requests(moment=far) == 1
+
+
+def test_caller_quota_counts_exactly_to_the_limit(local_db):
+    caller = generate_uid()
+    results = [store.reserve_caller_request("user", caller, 5) for _ in range(9)]
+    assert results.count(True) == 5
+    assert store.caller_requests_today("user", caller) == 5
+
+
+def test_a_zero_quota_or_anonymous_caller_never_reserves(local_db):
+    assert not store.reserve_caller_request("guest", "someone", 0)
+    assert not store.reserve_caller_request("user", "", 10)
+    assert store.caller_requests_today("guest", "someone") == 0
+
+
+def test_caller_quota_is_per_caller_and_per_day(local_db):
+    a, b = generate_uid(), generate_uid()
+    for _ in range(3):
+        assert store.reserve_caller_request("user", a, 3)
+    assert not store.reserve_caller_request("user", a, 3)
+    assert store.reserve_caller_request("user", b, 3)
+    tomorrow = datetime.now(timezone.utc) + timedelta(days=1)
+    assert store.reserve_caller_request("user", a, 3, moment=tomorrow)
+    assert store.reserve_caller_request("guest", a, 3)
+
+
+def test_caller_counters_are_pruned_with_the_usage_buckets(local_db):
+    caller = generate_uid()
+    store.reserve_caller_request("user", caller, 5)
+    far = datetime.now(timezone.utc) + timedelta(days=LIMITS.usage_daily_retention_days + 2)
+    assert store.prune_usage_daily(dry_run=True, moment=far) >= 1
+    assert store.caller_requests_today("user", caller) == 1
+    store.prune_usage_daily(moment=far)
+    assert store.caller_requests_today("user", caller) == 0
+
+
+def test_batch_helpers_match_the_single_lookups(local_db):
+    owner = generate_uid()
+    row, _ = store.create_device(owner, "a", "native")
+    store.save_node_settings(owner, {"enabled": True, "bandwidth_mb_day": 7})
+    assert store.get_devices_by_uids([row["uid"], "missing", ""])[row["uid"]]["user_uid"] == owner
+    assert store.get_enabled_by_users([owner, "nobody"]) == {owner: True}
+    assert store.get_settings_by_users([owner])[owner]["bandwidth_mb_day"] == 7
+    assert store.get_devices_by_uids([]) == {}
+
+
+def test_batch_helpers_ignore_revoked_devices(local_db):
+    owner = generate_uid()
+    row, _ = store.create_device(owner, "a", "native")
+    store.revoke_device(row["uid"], owner)
+    assert store.get_devices_by_uids([row["uid"]]) == {}
+
+
+def test_device_breakdown_buckets_blank_values_as_unknown(local_db):
+    owner = generate_uid()
+    store.create_device(owner, "a", "native")
+    store.create_device(owner, "b", "extension")
+    kinds = store.device_breakdown("kind")
+    assert kinds["native"] >= 1 and kinds["extension"] >= 1
+    assert "unknown" in store.device_breakdown("country")
+    with pytest.raises(ValueError):
+        store.device_breakdown("secret_hash")
+
+
+def test_device_breakdown_ignores_revoked_devices(local_db):
+    owner = generate_uid()
+    row, _ = store.create_device(owner, "a", "native")
+    before = store.device_breakdown("status").get("active", 0)
+    store.revoke_device(row["uid"], owner)
+    assert store.device_breakdown("status").get("active", 0) == before - 1
+
+
+def test_request_status_counts_always_lists_every_status(local_db):
+    counts = store.request_status_counts()
+    assert set(store.REQUEST_STATUSES) <= set(counts)
+    assert all(value >= 0 for value in counts.values())
+
+
+def test_usage_day_totals_sums_the_day(local_db):
+    owner = generate_uid()
+    row, _ = store.create_device(owner, "a", "native")
+    before = store.usage_day_totals()
+    store.record_job(row["uid"], owner, ok=True, bytes_up=10, bytes_down=90)
+    after = store.usage_day_totals()
+    assert after["requests_ok"] == before["requests_ok"] + 1
+    assert after["bytes_down"] == before["bytes_down"] + 90
+
+
+def test_recent_devices_is_capped_and_excludes_deleted(local_db):
+    owner = generate_uid()
+    row, _ = store.create_device(owner, "a", "native")
+    assert row["uid"] in {item["uid"] for item in store.recent_devices(store.ADMIN_LIST_LIMIT)}
+    assert len(store.recent_devices(1)) == 1
+    assert len(store.recent_devices(10_000)) <= store.ADMIN_LIST_LIMIT
+    store.revoke_device(row["uid"], owner)
+    assert row["uid"] not in {item["uid"] for item in store.recent_devices(store.ADMIN_LIST_LIMIT)}

tests/unit/services/community/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/unit/services/community/ai.py

@@ -0,0 +1,67 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+from devplacepy.services.community.ai import AiAssistant, outline, parse_json, to_html
+from devplacepy.services.community.extract import parse_html, select_nodes
+from tests.unit.services.community.fakes import HTML_ITEM
+
+
+class Canned(AiAssistant):
+    def __init__(self, answer):
+        self.answer = answer
+        self.asked = []
+
+    def enabled(self):
+        return True
+
+    async def ask(self, system, user):
+        self.asked.append((system, user))
+        return self.answer
+
+
+def nodes():
+    root = parse_html("<main>" + "".join(HTML_ITEM.format(n=n) for n in range(1, 5)) + "</main>")
+    return root, select_nodes(root, "article.post")
+
+
+def test_json_is_extracted_from_fenced_or_chatty_replies():
+    assert parse_json('```json\n{"a": 1}\n```') == {"a": 1}
+    assert parse_json('Sure! Here you go: {"a": {"b": 2}} hope it helps') == {"a": {"b": 2}}
+    assert parse_json("no json") is None
+
+
+def test_only_proposals_that_match_the_sample_are_accepted():
+    _, items = nodes()
+    assistant = Canned({"title": "h2", "link": "h2 a::attr(href)", "author": ".nonexistent", "created_at": "time::attr(datetime)"})
+    accepted = run_async(assistant.post_fields(items))
+    assert accepted == {"title": "h2", "link": "h2 a::attr(href)", "created_at": "time::attr(datetime)"}
+
+
+def test_item_selector_must_match_at_least_three_elements():
+    root, _ = nodes()
+    assert run_async(Canned({"item_selector": "article.post"}).item_selector(root)) == "article.post"
+    assert run_async(Canned({"item_selector": "article.missing"}).item_selector(root)) is None
+    assert run_async(Canned({"item_selector": "!!"}).item_selector(root)) is None
+
+
+def test_comment_structure_needs_content():
+    root = parse_html('<div><div class="c" id="a1"><p>hello there</p><b>ann</b></div><div class="c" id="a2"><p>second text</p><b>bob</b></div></div>')
+    good = run_async(Canned({"item_selector": "div.c", "id": "::attr(id)", "content": "p", "author": "b"}).comment_structure(root))
+    assert good == {"item_selector": "div.c", "id": "::attr(id)", "content": "p", "author": "b"}
+    assert run_async(Canned({"item_selector": "div.c", "content": "span"}).comment_structure(root)) is None
+
+
+def test_disabled_assistant_never_asks():
+    class Off(AiAssistant):
+        def enabled(self):
+            return False
+
+    assert run_async(Off().ask("s", "u")) is None
+
+
+def test_outline_and_html_are_bounded():
+    root, items = nodes()
+    assert "article.post" in outline(root)
+    assert len(outline(root, 3).splitlines()) <= 3
+    assert len(to_html(items[0], 100)) <= 100

tests/unit/services/community/alias.py

@@ -0,0 +1,133 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.database import get_setting, get_table, init_db, set_setting
+from devplacepy.services.community.runner import TRIGGER_CLI, CommunityRunner
+from devplacepy.services.community.store import store
+from devplacepy.services.sync import ai
+from devplacepy.services.sync.options import ALIAS_KEY, SyncOptions
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep
+
+ORIGINALS = ("writer0", "writer1", "replier0", "replier1")
+
+
+@pytest.fixture
+def world(monkeypatch):
+    init_db()
+    keys = ("sync_enabled", "sync_origins", ALIAS_KEY)
+    previous = {key: get_setting(key, "") for key in keys}
+    set_setting("sync_enabled", "1")
+    set_setting("sync_origins", "")
+    set_setting(ALIAS_KEY, "1")
+    for table in ("communities", "community_runs"):
+        get_table(table).delete()
+    get_table("sync_links").delete(origin="forum.test")
+    get_table("sync_identities").delete(origin="forum.test")
+
+    async def unreachable(system, text, timeout):
+        return None
+
+    monkeypatch.setattr(ai, "complete", unreachable)
+    runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(json_community()), sleep=no_sleep)
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test"))
+    yield runner
+    for key, value in previous.items():
+        set_setting(key, value)
+    get_table("sync_identities").delete(origin="forum.test")
+
+
+def go(runner, dry_run=False):
+    return run_async(runner.run(store.get("forum-test"), dry_run=dry_run, trigger=TRIGGER_CLI))
+
+
+def synced(kind: str) -> list[dict]:
+    links = get_table("sync_links").find(origin="forum.test", kind=kind)
+    return [row for row in (get_table(kind).find_one(uid=link["local_uid"]) for link in links) if row]
+
+
+def local_text() -> str:
+    names = [row["username"] for row in synced("users")]
+    posts = [f"{row['title']} {row['content']}" for row in synced("posts")]
+    comments = [row["content"] for row in synced("comments")]
+    return " ".join(names + posts + comments).lower()
+
+
+def enable(**changes):
+    row = store.get("forum-test")
+    store.update(row["uid"], sync_options=SyncOptions.from_row(row).merged(**changes).to_dict())
+
+
+def test_aliasing_off_by_default_keeps_original_names(world):
+    report = go(world)
+    assert report.status == "ok", report.errors
+    assert "writer0" in local_text()
+    assert get_table("sync_identities").count(origin="forum.test") == 0
+
+
+def test_alias_run_never_stores_an_original_username(world):
+    enable(alias=True)
+    report = go(world)
+    assert report.status == "ok", report.errors
+    text = local_text()
+    assert not [name for name in ORIGINALS if name in text]
+    assert get_table("sync_identities").count(origin="forum.test") == 4
+    again = go(world)
+    assert again.status == "ok"
+    assert again.pushed["users"].get("created", 0) == 0
+    assert get_table("sync_identities").count(origin="forum.test") == 4
+
+
+def test_kill_switch_stops_aliased_runs_and_creates_nothing(world):
+    enable(alias=True)
+    set_setting(ALIAS_KEY, "0")
+    report = go(world)
+    assert any("switched off" in error for error in report.errors)
+    assert not synced("posts") and not synced("users")
+    assert get_table("sync_identities").count(origin="forum.test") == 0
+
+
+def test_dry_run_creates_no_identities_and_no_rows(world):
+    enable(alias=True)
+    users = get_table("users").count()
+    report = go(world, dry_run=True)
+    assert report.status == "dry_run", report.errors
+    assert get_table("sync_identities").count(origin="forum.test") == 0
+    assert get_table("users").count() == users
+
+
+def test_enabling_alias_after_unaliased_sync_is_refused_at_run(world):
+    assert go(world).status == "ok"
+    enable(alias=True)
+    report = go(world)
+    assert any("already has synced records" in error for error in report.errors)
+    assert get_table("sync_identities").count(origin="forum.test") == 0
+
+
+def test_selection_skips_are_reported_and_sampling_is_stable(world):
+    enable(sample_percent=30)
+    first = go(world)
+    count = len(synced("posts"))
+    assert first.pushed["skipped"]["sampled_out"] + count >= 10
+    assert 0 < count < 10
+    go(world)
+    assert len(synced("posts")) == count
+
+
+def test_translation_failure_holds_watermark_and_stores_nothing(world):
+    enable(alias=True, translate_to="nl")
+    row = store.get("forum-test")
+    store.update(row["uid"], language="en")
+    report = go(world)
+    assert not synced("posts")
+    assert report.status == "partial"
+    assert report.pushed["skipped"]["translation_failed"] >= 1
+    assert not store.get("forum-test")["watermark"]
+
+
+def test_jitter_moves_next_run_within_bounds(world):
+    row = store.get("forum-test")
+    moments = {store.schedule_next(row["uid"], 86400, 40) for _ in range(5)}
+    assert len(moments) > 1

tests/unit/services/community/conftest.py

@@ -0,0 +1,13 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.community.ai import AiAssistant
+
+
+@pytest.fixture(autouse=True)
+def silent_ai(monkeypatch):
+    async def never(self, system, user):
+        return None
+
+    monkeypatch.setattr(AiAssistant, "ask", never)

tests/unit/services/community/discoverer.py

@@ -0,0 +1,133 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+from devplacepy.services.community.discoverer import CommunityDiscoverer
+from devplacepy.services.community.spec import STATUS_FAILED, STATUS_LOGIN, STATUS_READY
+from tests.unit.services.community.fakes import (
+    FakeFetcher,
+    club_community,
+    html_community,
+    json_community,
+    rss_community,
+    spa_community,
+)
+
+
+def discover(routes: dict, query: str, credentials: dict | None = None):
+    fetcher = FakeFetcher(routes)
+    return run_async(CommunityDiscoverer(fetcher, credentials=credentials).discover(query)), fetcher
+
+
+def test_json_api_community_is_learned():
+    result, _ = discover(json_community(), "https://forum.test/api/posts")
+    assert result.status == STATUS_READY, result.error
+    feed = result.schema["feeds"][0]
+    assert feed["type"] == "json_api"
+    assert feed["list_path"] == "$.items"
+    assert feed["post"]["fields"]["id"] == ["$.id"]
+    assert feed["post"]["fields"]["content"][0] == "$.content_html"
+    assert feed["post"]["fields"]["created_at"] == ["$.created_at"]
+    assert feed["author"]["fields"]["username"] == ["$.author.username"]
+    assert feed["author"]["fields"]["id"] == ["$.author.id"]
+    assert feed["pagination"] == {"style": "page", "param": "page", "start": 1}
+    assert feed["comments"]["mode"] == "inline"
+    assert feed["comments"]["fields"]["content"] == ["$.body"]
+    assert feed["comments"]["author"]["fields"]["username"] == ["$.user.username"]
+    records = result.sample["records"]
+    assert len(records["posts"]) == 8
+    assert len(records["comments"]) == 6
+    assert {user["username"] for user in records["users"]} >= {"writer0", "writer1", "replier0"}
+
+
+def test_rss_community_is_learned_from_advertised_feed():
+    result, _ = discover(rss_community(), "https://blog.test/")
+    assert result.status == STATUS_READY, result.error
+    feed = result.schema["feeds"][0]
+    assert feed["type"] == "rss"
+    assert feed["post"]["fields"]["created_at"] == ["$.pubDate"]
+    assert result.sample["records"]["users"][0]["username"] == "Alice"
+    assert result.sample["records"]["posts"][0]["content"].startswith("Full body of entry")
+    assert feed["comments"]["mode"] == "page"
+    assert feed["comments"]["item_selector"] == "div.comment"
+    assert feed["post"]["detail"]["content"] == ["article"]
+    comments = result.sample["records"]["comments"]
+    assert len(comments) == 9
+    assert sum(1 for comment in comments if comment.get("parent_uid")) == 3
+
+
+def test_html_list_community_is_learned():
+    result, _ = discover(html_community(), "https://board.test/")
+    assert result.status == STATUS_READY, result.error
+    feed = result.schema["feeds"][0]
+    assert feed["type"] == "html_list"
+    assert feed["item_selector"] == "article.post"
+    assert len(result.sample["records"]["posts"]) == 6
+    assert result.sample["records"]["posts"][0]["title"].startswith("Thread 1")
+    assert feed["comments"]["mode"] == "page"
+    assert feed["comments"]["fields"]["id"] == ["::attr(id)"]
+    assert feed["comments"]["author"]["fields"]["username"] == [".author"]
+    assert len(result.sample["records"]["comments"]) == 9
+
+
+def test_unreachable_community_fails_with_reason():
+    result, _ = discover({}, "https://nothing.test/")
+    assert result.status == STATUS_FAILED
+    assert result.error
+
+
+def test_javascript_only_site_is_rendered():
+    result, _ = discover(spa_community(), "https://spa.test/")
+    assert result.status == STATUS_READY, result.error
+    feed = result.schema["feeds"][0]
+    assert feed["render"] is True
+    assert len(result.sample["records"]["posts"]) == 6
+
+
+def test_login_wall_without_credentials_is_flagged_not_failed():
+    state = {}
+    result, _ = discover(club_community(state), "https://club.test/")
+    assert result.status == STATUS_LOGIN
+    assert result.schema["login"]["mode"] == "form"
+    assert result.schema["login"]["username_field"] == "user"
+    assert result.schema["login_required"] is True
+
+
+def test_login_wall_with_credentials_is_learned_behind_the_wall():
+    state = {}
+    result, fetcher = discover(club_community(state), "https://club.test/", {"username": "bob", "password": "pw"})
+    assert result.status == STATUS_READY, result.error
+    assert result.schema["login_required"] is True
+    assert result.schema["feeds"][0]["type"] == "rss"
+    assert fetcher.posts[0] == ("https://club.test/login", {"csrf": "t1", "user": "bob", "pass": "pw"})
+
+
+def test_wrong_credentials_fail_loudly():
+    result, _ = discover(club_community({}), "https://club.test/", {"username": "bob", "password": "nope"})
+    assert result.status == STATUS_FAILED
+    assert "rejected the stored credentials" in result.error
+
+
+def test_feeds_listing_the_same_posts_are_learned_once_and_comment_feeds_are_skipped():
+    routes = rss_community()
+    routes["https://blog.test/"] = (
+        '<html><head><link rel="alternate" type="application/rss+xml" href="/feed.xml">'
+        '<link rel="alternate" type="application/rss+xml" href="/comments.xml"></head>'
+        '<body><a href="/latest">Latest posts</a></body></html>'
+    )
+    routes["https://blog.test/comments.xml"] = routes["https://blog.test/feed.xml"]
+    items = "".join(
+        f'<article class="post"><h2><a href="/p/{n}">Entry {n} on testing</a></h2><span class="author">Alice</span>'
+        f"<p>Body of entry {n} with enough text to count as content.</p></article>"
+        for n in range(1, 7)
+    )
+    routes["https://blog.test/latest"] = f"<html><body><main>{items}</main></body></html>"
+    result, fetcher = discover(routes, "https://blog.test/")
+    assert result.status == STATUS_READY, result.error
+    assert [feed["type"] for feed in result.schema["feeds"]] == ["rss"]
+    assert not any("comments.xml" in request for request in fetcher.requests)
+
+
+def test_generic_probe_paths_are_skipped_when_a_feed_is_advertised():
+    _, fetcher = discover(rss_community(), "https://blog.test/")
+    assert not any(request.endswith("/api/posts") or request.endswith("/rss.xml") for request in fetcher.requests)

tests/unit/services/community/extract.py

@@ -0,0 +1,60 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.community import extract
+
+
+def test_select_walks_keys_indexes_and_wildcards():
+    data = {"a": [{"b": 1}, {"b": 2}], "c": {"d": "x"}}
+    assert extract.select(data, "$.a[*].b") == [1, 2]
+    assert extract.select(data, "$.a[1].b") == [2]
+    assert extract.select(data, "$.c.d") == ["x"]
+    assert extract.select(data, "$.missing") == []
+
+
+@pytest.mark.parametrize("path", ["a.b", "$.a[", "$..", "$" + ".x" * 40])
+def test_bad_paths_raise(path):
+    with pytest.raises(ValueError):
+        extract.parse_path(path)
+
+
+def test_instants_accept_iso_rfc822_and_epochs():
+    assert extract.normalize_instant("2026-01-01T00:00:00Z") == "2026-01-01T00:00:00+00:00"
+    assert extract.normalize_instant("Tue, 14 Oct 2026 10:00:00 GMT") == "2026-10-14T10:00:00+00:00"
+    assert extract.normalize_instant(1700000000) == extract.normalize_instant(1700000000000)
+    assert extract.normalize_instant("garbage") is None
+    assert extract.normalize_instant("") is None
+
+
+def test_atom_feed_is_parsed_with_links_and_authors():
+    atom = (
+        '<feed xmlns="http://www.w3.org/2005/Atom"><title>T</title><entry><title>E</title>'
+        '<link href="http://x/1" rel="alternate"/><author><name>Ann</name></author>'
+        "<updated>2026-01-02T00:00:00Z</updated></entry></feed>"
+    )
+    kind, _, items = extract.parse_feed(atom)
+    assert kind == "atom"
+    assert extract.text_at(items[0], "$.link.@href") == "http://x/1"
+    assert extract.text_at(items[0], "$.author.name") == "Ann"
+
+
+def test_css_selectors_support_classes_ids_attributes_and_combinators():
+    root = extract.parse_html(
+        '<div id="m"><ul><li class="a b"><a href="/x" rel="next">One</a></li><li class="a">Two</li></ul></div>'
+    )
+    assert len(extract.select_nodes(root, "li.a")) == 2
+    assert len(extract.select_nodes(root, "li.a.b")) == 1
+    assert len(extract.select_nodes(root, "#m > ul > li")) == 2
+    assert len(extract.select_nodes(root, "div li")) == 2
+    assert extract.html_field(extract.select_nodes(root, "li.b")[0], "a[rel=next]::attr(href)") == "/x"
+
+
+def test_strip_tags_keeps_paragraph_breaks_and_drops_scripts():
+    assert extract.strip_tags("<p>a</p><script>x()</script><p>b<br>c</p>") == "a\n\nb\n\nc"
+    assert extract.strip_tags("plain text") == "plain text"
+
+
+def test_field_text_falls_through_alternates_and_now_token():
+    assert extract.field_text({"a": "", "b": "yes"}, ["$.a", "$.b"]) == "yes"
+    assert extract.field_text({}, [extract.NOW_TOKEN]).startswith("20")

tests/unit/services/community/fakes.py

@@ -0,0 +1,228 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+
+class FakeResponse:
+    def __init__(self, status: int, body):
+        self.status = status
+        self.body = body if isinstance(body, str) else json.dumps(body)
+        self.headers = {}
+
+    async def text(self):
+        return self.body
+
+    async def json(self):
+        return json.loads(self.body)
+
+
+class FakeFetcher:
+    def __init__(self, routes: dict):
+        self.routes = routes
+        self.requests: list[str] = []
+        self.session = None
+        self.posts: list[tuple] = []
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *exc):
+        return None
+
+    @staticmethod
+    def key(url: str, params: dict | None) -> str:
+        parts = urlsplit(url)
+        query = dict(parse_qsl(parts.query))
+        query.update({name: str(value) for name, value in (params or {}).items()})
+        return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(sorted(query.items())), ""))
+
+    async def get(self, url, params=None, timeout=None, **kwargs):
+        key = self.key(url, params)
+        self.requests.append(key)
+        route = self.routes.get(key)
+        if route is None:
+            return FakeResponse(404, "not found")
+        if callable(route):
+            route = route()
+        status, body = route if isinstance(route, tuple) else (200, route)
+        return FakeResponse(status, body)
+
+    async def post(self, url, data=None, timeout=None, **kwargs):
+        self.posts.append((url, dict(data or {})))
+        route = self.routes.get("post:" + url)
+        if callable(route):
+            route = route(dict(data or {}))
+        status, body = route if isinstance(route, tuple) else (200, route or "")
+        return FakeResponse(status, body)
+
+    async def render_html(self, url, session=None):
+        route = self.routes.get("render:" + url)
+        if route is None:
+            response = await self.get(url)
+            return response.status, response.body
+        return 200, route
+
+    def page(self, session=None):
+        return FakePage(self)
+
+    async def render(self, url, collect_links=False):
+        response = await self.get(url)
+        return {"status": response.status, "text": response.body, "links": [], "rendered": False}
+
+
+def json_post(number: int) -> dict:
+    return {
+        "id": 1000 + number,
+        "title": f"Post number {number} about asyncio",
+        "content_html": f"<p>This is the body of post {number}, long enough to sync.</p>",
+        "created_at": f"2026-10-{number:02d}T10:00:00Z",
+        "author": {"id": 7 + number % 2, "username": f"writer{number % 2}", "url": f"https://forum.test/u/{number % 2}"},
+        "url": f"https://forum.test/posts/{1000 + number}",
+        "comments": [
+            {
+                "id": 5000 + number * 10 + index,
+                "body": f"Comment {index} on post {number}",
+                "created_at": f"2026-10-{number:02d}T11:0{index}:00Z",
+                "user": {"id": 20 + index, "username": f"replier{index}"},
+            }
+            for index in range(2)
+        ],
+    }
+
+
+def json_community() -> dict:
+    posts = [json_post(number) for number in range(1, 9)]
+    return {
+        "https://forum.test/": '<html lang="en"><head><title>Forum Test</title>'
+        '<meta property="og:description" content="A friendly forum about code."></head>'
+        "<body><p>Join 1,234 members discussing code every single day on this community site.</p></body></html>",
+        "https://forum.test/api/posts": {"items": posts, "total": 8},
+        "https://forum.test/api/posts?page=2": {"items": [json_post(9), json_post(10)], "total": 8},
+        "https://forum.test/about": "<html><body><p>About: 2,500 members strong.</p></body></html>",
+    }
+
+
+RSS = """<?xml version="1.0"?>
+<rss version="2.0" xmlns:dc="http://purl.org/dc/elements/1.1/"><channel>
+<title>Blog Test</title><description>A blog</description>
+{items}
+</channel></rss>"""
+
+RSS_ITEM = """<item><title>Entry {n} on testing</title><link>https://blog.test/p/{n}</link>
+<guid>https://blog.test/p/{n}</guid><pubDate>Tue, {n:02d} Oct 2026 10:00:00 GMT</pubDate>
+<dc:creator>Alice</dc:creator><description>&lt;p&gt;Body of entry {n} with enough text.&lt;/p&gt;</description></item>"""
+
+
+def detail_page(kind: str, n: int, comments: int = 2) -> str:
+    rows = "".join(
+        f'<div class="comment" id="c{n}-{index}"><span class="author">replier{index}</span>'
+        f'<time datetime="2026-10-{n:02d}T12:0{index}:00Z">t</time>'
+        f'<div class="comment-body">Comment {index} on {kind} {n} with words</div>'
+        + (
+            f'<div class="comment" id="c{n}-{index}-r"><span class="author">deep{index}</span>'
+            f'<div class="comment-body">Nested reply to comment {index} on {n}</div></div>'
+            if index == 0
+            else ""
+        )
+        + "</div>"
+        for index in range(comments)
+    )
+    body = f"Full body of {kind} {n}. " + "It goes on at length about the topic. " * 6
+    return f"<html><body><article>{body}</article><section>{rows}</section></body></html>"
+
+
+def rss_community() -> dict:
+    items = "".join(RSS_ITEM.format(n=n) for n in range(1, 7))
+    pages = {f"https://blog.test/p/{n}": detail_page("entry", n) for n in range(1, 7)}
+    return {
+        **pages,
+        "https://blog.test/": '<html><head><link rel="alternate" type="application/rss+xml" href="/feed.xml">'
+        "<title>Blog Test</title></head><body>hi</body></html>",
+        "https://blog.test/feed.xml": RSS.format(items=items),
+    }
+
+
+HTML_ITEM = """<article class="post"><h2><a href="/t/{n}">Thread {n} on shells and terminals</a></h2>
+<span class="author">user{n}</span><time datetime="2026-10-{n:02d}T09:00:00Z">date</time>
+<p>Excerpt of thread {n} which is certainly long enough to count as content.</p></article>"""
+
+
+def html_community() -> dict:
+    items = "".join(HTML_ITEM.format(n=n) for n in range(1, 7))
+    pages = {f"https://board.test/t/{n}": detail_page("thread", n) for n in range(1, 7)}
+    return {
+        **pages,
+        "https://board.test/": f"<html><head><title>Board</title></head><body><main>{items}</main></body></html>",
+    }
+
+
+async def no_sleep(_seconds):
+    return None
+
+
+class FakePage:
+    def __init__(self, fetcher):
+        self.fetcher = fetcher
+        self.filled: dict[str, str] = {}
+        self.url = ""
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *exc):
+        return None
+
+    async def goto(self, url, **kwargs):
+        self.url = url
+        status, self.html = await self.fetcher.render_html(url)
+        return type("Resp", (), {"status": status})()
+
+    async def wait_for_timeout(self, _ms):
+        return None
+
+    async def content(self):
+        return self.html
+
+    async def fill(self, selector, value):
+        self.filled[selector] = value
+
+    async def click(self, selector):
+        self.fetcher.posts.append(("click", dict(self.filled)))
+
+
+SPA_SHELL = "<html><head><title>Spa</title></head><body><div id=app></div></body></html>"
+
+
+def spa_community() -> dict:
+    items = "".join(HTML_ITEM.format(n=n) for n in range(1, 7))
+    return {
+        "https://spa.test/": SPA_SHELL,
+        "render:https://spa.test/": f"<html><body><main>{items}</main></body></html>",
+    }
+
+
+WALL = (
+    '<html><body><form action="/login" method="post"><input type="hidden" name="csrf" value="t1">'
+    '<input type="text" name="user"><input type="password" name="pass"></form></body></html>'
+)
+CLUB_HOME = (
+    '<html><head><title>Club</title><link rel="alternate" type="application/rss+xml" href="/feed.xml"></head>'
+    "<body><p>Members only area with plenty of text on the page for the club.</p></body></html>"
+)
+
+
+def club_community(state: dict) -> dict:
+    items = "".join(RSS_ITEM.format(n=n).replace("blog.test", "club.test") for n in range(1, 7))
+    feed = RSS.format(items=items)
+
+    def login(data):
+        state["in"] = data.get("user") == "bob" and data.get("pass") == "pw" and data.get("csrf") == "t1"
+        return (200, "welcome" if state["in"] else WALL)
+
+    return {
+        "https://club.test/": lambda: CLUB_HOME if state.get("in") else WALL,
+        "https://club.test/login": WALL,
+        "https://club.test/feed.xml": lambda: feed if state.get("in") else (401, "login"),
+        "post:https://club.test/login": login,
+    }

tests/unit/services/community/generator.py

@@ -0,0 +1,168 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+import importlib.util
+import json
+import py_compile
+
+import pytest
+
+from devplacepy.services.community.discoverer import CommunityDiscoverer
+from devplacepy.services.community.generator import CodeGenerator
+from devplacepy.services.community.runtime import Collector, LoginRequired
+from devplacepy.services.sync.records import RECORD_MODELS
+from tests.unit.services.community.fakes import (
+    FakeFetcher,
+    FakeResponse,
+    html_community,
+    json_community,
+    rss_community,
+)
+
+
+async def no_sleep(_seconds):
+    return None
+
+
+def learn(routes: dict, query: str) -> dict:
+    result = run_async(CommunityDiscoverer(FakeFetcher(routes)).discover(query))
+    assert result.schema, result.error
+    return result.schema
+
+
+def load(path):
+    spec = importlib.util.spec_from_file_location("generated_collectors", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+@pytest.mark.parametrize(
+    "routes,query",
+    [
+        (json_community(), "https://forum.test/api/posts"),
+        (rss_community(), "https://blog.test/"),
+        (html_community(), "https://board.test/"),
+    ],
+)
+def test_generated_file_compiles_and_collects_valid_sync_records(tmp_path, routes, query):
+    schema = learn(routes, query)
+    target = CodeGenerator(schema).write(tmp_path)
+    py_compile.compile(str(target), doraise=True)
+    assert json.loads((tmp_path / "schema.json").read_text()) == schema
+    module = load(target)
+    result = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    assert result["ok"], result["errors"]
+    assert result["records"]["posts"]
+    for kind, rows in result["records"].items():
+        for row in rows:
+            RECORD_MODELS[kind].model_validate(row)
+    uids = {user["origin_uid"] for user in result["records"]["users"]}
+    assert {post["author_uid"] for post in result["records"]["posts"]} <= uids
+
+
+def test_generated_wrappers_exist_per_feed(tmp_path):
+    schema = learn(json_community(), "https://forum.test/api/posts")
+    module = load(CodeGenerator(schema).write(tmp_path))
+    assert callable(module.collect_feed1)
+    assert module.SCHEMA == schema
+
+
+def test_generated_output_matches_in_process_collector(tmp_path):
+    routes = json_community()
+    schema = learn(routes, "https://forum.test/api/posts")
+    module = load(CodeGenerator(schema).write(tmp_path))
+    generated = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    direct = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    assert generated["records"] == direct["records"]
+
+
+def test_incremental_run_collects_only_new_posts(tmp_path):
+    routes = json_community()
+    schema = learn(routes, "https://forum.test/api/posts")
+    first = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    again = run_async(Collector(FakeFetcher(routes), schema, first["state"], sleep=no_sleep).run())
+    assert len(first["records"]["posts"]) == 10
+    assert again["records"]["posts"] == []
+    assert again["state"]["feeds"]["feed1"]["since"] == first["state"]["feeds"]["feed1"]["since"]
+
+
+def test_incremental_run_picks_up_newer_post():
+    routes = json_community()
+    schema = learn(routes, "https://forum.test/api/posts")
+    first = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    newer = {
+        "id": 2000,
+        "title": "A brand new post",
+        "content_html": "<p>Fresh content that arrived later.</p>",
+        "created_at": "2026-11-01T10:00:00Z",
+        "author": {"id": 7, "username": "writer1"},
+        "comments": [],
+    }
+    routes["https://forum.test/api/posts"]["items"].insert(0, newer)
+    result = run_async(Collector(FakeFetcher(routes), schema, first["state"], sleep=no_sleep).run())
+    assert [post["origin_uid"] for post in result["records"]["posts"]] == ["2000"]
+
+
+def test_failed_requests_retry_with_backoff():
+    calls = {"count": 0}
+    delays = []
+
+    class Flaky:
+        async def get(self, url, params=None, timeout=None, **kwargs):
+            calls["count"] += 1
+            return FakeResponse(503 if calls["count"] < 3 else 200, "[]")
+
+    async def record(seconds):
+        delays.append(seconds)
+
+    collector = Collector(Flaky(), {"feeds": [], "limits": {"delay_seconds": 0, "backoff_seconds": 2, "retries": 3}}, sleep=record)
+    assert run_async(collector.fetch("https://x.test/")) == "[]"
+    assert calls["count"] == 3
+    assert [delay for delay in delays if delay] == [2.0, 4.0]
+
+
+def test_login_required_schema_raises_clear_error():
+    schema = {"community": "closed", "origin": "closed.test", "login_required": True, "feeds": []}
+    with pytest.raises(LoginRequired, match="requires a login"):
+        run_async(Collector(FakeFetcher({}), schema).run())
+
+
+def test_unauthorized_response_is_login_required():
+    schema = {
+        "community": "x",
+        "origin": "x.test",
+        "feeds": [{"id": "feed1", "url": "https://x.test/f", "type": "json_api", "post": {"fields": {"id": ["$.id"]}}, "author": {}}],
+        "limits": {"delay_seconds": 0, "retries": 1},
+    }
+    with pytest.raises(LoginRequired):
+        run_async(Collector(FakeFetcher({"https://x.test/f": (401, "no")}), schema, sleep=no_sleep).run())
+
+
+def test_generated_collectors_log_in_with_credentials(tmp_path):
+    from tests.unit.services.community.fakes import club_community
+
+    state = {}
+    routes = club_community(state)
+    result = run_async(CommunityDiscoverer(FakeFetcher(routes), credentials={"username": "bob", "password": "pw"}).discover("https://club.test/"))
+    module = load(CodeGenerator(result.schema).write(tmp_path))
+    state.clear()
+    with pytest.raises(module.LoginRequired, match="requires a login"):
+        run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    collected = run_async(
+        module.collect_all(FakeFetcher(routes), sleep=no_sleep, credentials={"username": "bob", "password": "pw"})
+    )
+    assert collected["ok"] and collected["records"]["posts"]
+    state.clear()
+    with pytest.raises(module.LoginRequired, match="rejected the stored credentials"):
+        run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep, credentials={"username": "bob", "password": "bad"}))
+
+
+def test_javascript_feed_is_collected_through_the_browser(tmp_path):
+    from tests.unit.services.community.fakes import spa_community
+
+    routes = spa_community()
+    schema = learn(routes, "https://spa.test/")
+    module = load(CodeGenerator(schema).write(tmp_path))
+    collected = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    assert len(collected["records"]["posts"]) == 6

tests/unit/services/community/search.py

@@ -0,0 +1,39 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+import pytest
+
+from devplacepy.services.community.search import SearchPhase, direct_url, rank
+from tests.unit.services.community.fakes import FakeFetcher
+
+
+def test_direct_addresses_skip_search():
+    assert direct_url("https://x.test/forum") == "https://x.test/forum"
+    assert direct_url("x.test") == "https://x.test"
+    assert direct_url("devplace") is None
+
+
+def test_rank_prefers_domain_match_and_drops_social_noise():
+    results = [
+        {"title": "DevPlace on LinkedIn", "url": "https://www.linkedin.com/company/devplace", "description": ""},
+        {"title": "Other", "url": "https://other.test/page", "description": "mentions devplace"},
+        {"title": "DevPlace", "url": "https://devplace.example/", "description": ""},
+    ]
+    ranked = rank("devplace", results)
+    assert [candidate.host for candidate in ranked] == ["devplace.example", "other.test"]
+
+
+def test_search_uses_rsearch_and_caps_candidates():
+    results = [{"title": f"t{n}", "url": f"https://site{n}.test/", "description": ""} for n in range(9)]
+    key = "https://rsearch.app.molodetz.nl/search?content=false&count=15&query=forum"
+    fetcher = FakeFetcher({key: {"success": True, "results": results}})
+    found = run_async(SearchPhase(fetcher).run("forum"))
+    assert len(found) == 5
+
+
+def test_search_failure_raises():
+    key = "https://rsearch.app.molodetz.nl/search?content=false&count=15&query=forum"
+    fetcher = FakeFetcher({key: (500, "boom")})
+    with pytest.raises(RuntimeError):
+        run_async(SearchPhase(fetcher).run("forum"))

tests/unit/services/community/service.py

@@ -0,0 +1,81 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+import pytest
+
+from devplacepy.database import get_setting, get_table, init_db, set_setting
+from devplacepy.services.community.runner import CommunityRunner
+from devplacepy.services.community.service import DRY_RUN_KEY, CommunityService
+from devplacepy.services.community.spec import STATUS_FAILING, STATUS_READY
+from devplacepy.services.community.store import store
+from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep
+
+
+@pytest.fixture
+def world():
+    init_db()
+    previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins", DRY_RUN_KEY)}
+    set_setting("sync_enabled", "1")
+    set_setting("sync_origins", "")
+    get_table("communities").delete()
+    get_table("community_runs").delete()
+    get_table("sync_links").delete(origin="forum.test")
+    routes = json_community()
+    runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep)
+    yield routes, runner
+    for key, value in previous.items():
+        set_setting(key, value)
+
+
+def seed(runner) -> dict:
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test"))
+    return store.get("forum-test")
+
+
+def test_dry_run_service_tick_learns_nothing_and_runs_nothing(world):
+    _, runner = world
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    set_setting(DRY_RUN_KEY, "1")
+    service = CommunityService(runner)
+    run_async(service.run_once())
+    assert store.get("forum-test")["status"] == "pending"
+    assert any("DRY RUN" in line for line in service.log_buffer)
+
+
+def test_live_tick_learns_pending_then_runs_due(world):
+    _, runner = world
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    set_setting(DRY_RUN_KEY, "0")
+    service = CommunityService(runner)
+    run_async(service.run_once())
+    assert store.get("forum-test")["status"] == STATUS_READY
+    run_async(service.run_once())
+    row = store.get("forum-test")
+    assert row["run_count"] == 1
+    assert get_table("sync_links").count(origin="forum.test", kind="posts") == 10
+    run_async(service.run_once())
+    assert store.get("forum-test")["run_count"] == 1
+
+
+def test_repeated_failures_disable_and_flag_the_community(world):
+    routes, runner = world
+    row = seed(runner)
+    routes["https://forum.test/api/posts"] = (500, "down")
+    failing = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep)
+    for _ in range(6):
+        current = store.get("forum-test")
+        report = run_async(failing.run(current, dry_run=False, trigger="test"))
+        assert report.status == "failed"
+    final = store.get("forum-test")
+    assert final["status"] == STATUS_FAILING and not final["enabled"]
+    assert "relearn" in "".join(store.runs(row["uid"])[0]["log"]).lower() or final["failure_count"] >= 5
+
+
+def test_run_history_is_bounded(world):
+    _, runner = world
+    row = seed(runner)
+    for _ in range(35):
+        run_async(runner.run(store.get("forum-test"), dry_run=True, trigger="test"))
+    assert len(store.runs(row["uid"], limit=100)) == 30

tests/unit/services/devii/actions/catalog.py

@@ -147,3 +147,17 @@ def test_scheduling_tools_are_hidden_from_guests():
     }
     for name in ("create_task", "update_task", "run_task_now"):
         assert name not in guest
+
+
+def test_sync_tools_are_admin_only_and_match_the_routes():
+    from devplacepy.main import app
+    from devplacepy.services.sync.spec import KINDS
+
+    paths = set(app.openapi()["paths"])
+    for kind in KINDS:
+        action = BY_NAME[f"sync_{kind}"]
+        assert action.method == "POST" and action.path == f"/sync/{kind}"
+        assert action.requires_admin is True
+        assert action.path in paths
+        assert {param.name for param in action.params} == {"origin", "records", "dry_run"}
+        assert _records_mutation(action.name) is True

tests/unit/services/nodes/adapters.py

@@ -0,0 +1,91 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+import pytest
+
+from devplacepy.services.nodes.adapters.base import AdapterError, clean, tag_for
+from devplacepy.services.nodes.adapters.direct import DevTo, GitHub, HackerNews, Mastodon, StackExchange
+from devplacepy.services.nodes.adapters.reddit import Reddit
+from devplacepy.services.nodes.adapters.registry import ADAPTERS
+from devplacepy.services.nodes.spec import SPEC
+
+
+def body(value) -> bytes:
+    return json.dumps(value).encode()
+
+
+def test_clean_strips_tags_entities_and_caps():
+    assert clean("<p>a &amp; b</p>  c", 50) == "a & b c"
+    assert len(clean("x" * 900, 100)) == 100
+    assert clean(None, 10) == ""
+
+
+def test_tag_for_picks_longest_word():
+    assert tag_for("sqlite wal mode") == "sqlite"
+    assert tag_for("!!") == ""
+
+
+def test_hackernews_parse():
+    data = {
+        "hits": [
+            {"objectID": "7", "title": "T", "points": 5, "num_comments": 2, "author": "a", "created_at": "2026"},
+            {"objectID": "", "title": "no id"},
+            "junk",
+        ]
+    }
+    hits = HackerNews().parse(body(data), 8)
+    assert [item.url for item in hits] == ["https://news.ycombinator.com/item?id=7"]
+    assert hits[0].score == 5 and hits[0].via == "direct"
+
+
+def test_stackexchange_parse():
+    data = {"items": [{"title": "Q &amp; A", "link": "https://stackoverflow.com/q/1", "score": 3, "answer_count": 2, "tags": ["c"], "owner": {"display_name": "bob"}}]}
+    hit = StackExchange().parse(body(data), 8)[0]
+    assert hit.title == "Q & A" and hit.author == "bob" and hit.comments == 2
+
+
+def test_github_parse_requires_http_url():
+    data = {"items": [{"title": "I", "html_url": "javascript:alert(1)"}, {"title": "J", "html_url": "https://github.com/a/b/issues/1", "user": {"login": "u"}}]}
+    hits = GitHub().parse(body(data), 8)
+    assert [item.title for item in hits] == ["J"]
+
+
+def test_devto_and_mastodon_need_a_tag():
+    with pytest.raises(AdapterError) as error:
+        DevTo().request("!!", 5)
+    assert error.value.code == "no_tag"
+    with pytest.raises(AdapterError):
+        Mastodon().request("", 5)
+    assert "tag=sqlite" in DevTo().request("sqlite", 5)
+    assert "/tag/sqlite" in Mastodon().request("sqlite", 5)
+
+
+def test_mastodon_parse_strips_html():
+    data = [{"url": "https://m.example/1", "content": "<p>hello <b>world</b></p>", "account": {"acct": "x@y"}, "favourites_count": 2, "reblogs_count": 1}]
+    hit = Mastodon().parse(body(data), 5)[0]
+    assert hit.snippet == "hello world" and hit.score == 3
+
+
+def test_reddit_parse_and_via():
+    data = {"data": {"children": [{"data": {"title": "T", "permalink": "/r/x/comments/1/t/", "score": 9, "num_comments": 4, "author": "a", "selftext": "body"}}, {"data": {"title": "bad", "permalink": "http://evil"}}]}}
+    hits = Reddit().parse(body(data), 5)
+    assert len(hits) == 1 and hits[0].url == "https://www.reddit.com/r/x/comments/1/t/" and hits[0].via == "fetch"
+
+
+@pytest.mark.parametrize("adapter", list(ADAPTERS.values()))
+def test_forged_structure_is_rejected(adapter):
+    for forged in (b"not json", body("string"), body({"unexpected": 1}), body(42)):
+        with pytest.raises(AdapterError):
+            adapter.parse(forged, 5)
+
+
+def test_oversize_body_is_rejected():
+    with pytest.raises(AdapterError) as error:
+        HackerNews().parse(b" " * (2 * 1024 * 1024 + 1), 5)
+    assert error.value.code == "body_too_large"
+
+
+def test_node_adapters_are_exactly_the_spec_adapters():
+    node_backed = {name for name, adapter in ADAPTERS.items() if not adapter.direct}
+    assert node_backed == {item.adapter for item in SPEC.destinations}

tests/unit/services/nodes/cache.py

@@ -0,0 +1,109 @@
+# retoor <retoor@molodetz.nl>
+
+import random
+
+from devplacepy.services.nodes.cache import ResultCache, cache_key
+from devplacepy.services.nodes.spec import SPEC
+
+LIMITS = SPEC.limits
+
+
+class Clock:
+    def __init__(self):
+        self.now = 100.0
+
+    def __call__(self):
+        return self.now
+
+
+def make(**kwargs):
+    clock = Clock()
+    return ResultCache(clock=clock, **kwargs), clock
+
+
+def test_hit_and_miss_counters():
+    cache, _ = make()
+    assert cache.get("GET", "u") is None
+    assert cache.put("GET", "u", 200, {"a": "b"}, b"xyz", False)
+    hit = cache.get("GET", "u")
+    assert hit.body == b"xyz" and hit.headers == {"a": "b"}
+    assert (cache.hits, cache.misses) == (1, 1)
+
+
+def test_method_is_part_of_the_key():
+    assert cache_key("GET", "u") != cache_key("HEAD", "u")
+    assert cache_key("get", "u") == cache_key("GET", "u")
+
+
+def test_entries_expire_after_the_ttl():
+    cache, clock = make(ttl_seconds=10)
+    cache.put("GET", "u", 200, {}, b"x", False)
+    clock.now += 9.9
+    assert cache.get("GET", "u") is not None
+    clock.now += 0.2
+    assert cache.get("GET", "u") is None
+    assert len(cache) == 0 and cache.size_bytes == 0
+
+
+def test_ttl_is_clamped_to_the_spec_ceiling_and_zero_disables():
+    assert ResultCache(ttl_seconds=10**9).ttl_seconds == LIMITS.cache_ttl_seconds
+    off, _ = make(ttl_seconds=0)
+    assert off.put("GET", "u", 200, {}, b"x", False) is False
+
+
+def test_errors_and_truncated_bodies_are_never_cached():
+    cache, _ = make()
+    assert not cache.put("GET", "u", 500, {}, b"x", False)
+    assert not cache.put("GET", "u", 503, {}, b"x", False)
+    assert not cache.put("GET", "u", 200, {}, b"x", True)
+    assert cache.put("GET", "u", 404, {}, b"x", False)
+
+
+def test_an_oversized_entry_is_refused():
+    cache, _ = make(entry_max_bytes=10, max_bytes=100)
+    assert not cache.put("GET", "u", 200, {}, b"x" * 11, False)
+    assert cache.put("GET", "u", 200, {}, b"x" * 10, False)
+
+
+def test_the_entry_cap_cannot_exceed_the_spec_ceiling():
+    assert ResultCache(entry_max_bytes=10**12).entry_max_bytes == LIMITS.cache_entry_max_bytes
+
+
+def test_least_recently_used_goes_first():
+    cache, _ = make(max_bytes=30, entry_max_bytes=30)
+    for name in "abc":
+        cache.put("GET", name, 200, {}, b"x" * 10, False)
+    cache.get("GET", "a")
+    cache.put("GET", "d", 200, {}, b"x" * 10, False)
+    assert cache.get("GET", "b") is None
+    assert cache.get("GET", "a") is not None
+    assert cache.get("GET", "c") is not None
+    assert cache.get("GET", "d") is not None
+
+
+def test_replacing_an_entry_does_not_leak_bytes():
+    cache, _ = make()
+    cache.put("GET", "u", 200, {}, b"x" * 100, False)
+    cache.put("GET", "u", 200, {}, b"y" * 40, False)
+    assert cache.size_bytes == 40 and len(cache) == 1
+
+
+def test_size_never_exceeds_the_cap_under_a_random_workload():
+    cache, clock = make(max_bytes=1000, entry_max_bytes=400, ttl_seconds=50)
+    rng = random.Random(7)
+    for step in range(5000):
+        clock.now += rng.random()
+        key = f"k{rng.randint(0, 60)}"
+        if rng.random() < 0.7:
+            cache.put("GET", key, rng.choice([200, 404, 500]), {}, b"x" * rng.randint(0, 500), False)
+        else:
+            cache.get("GET", key)
+        assert 0 <= cache.size_bytes <= 1000
+        assert cache.size_bytes == sum(len(e.body) for e in cache._entries.values())
+
+
+def test_clear_empties_everything():
+    cache, _ = make()
+    cache.put("GET", "u", 200, {}, b"x", False)
+    cache.clear()
+    assert len(cache) == 0 and cache.size_bytes == 0

tests/unit/services/nodes/circuit.py

@@ -0,0 +1,97 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.nodes.circuit import CLOSED, HALF_OPEN, OPEN, CircuitBreaker
+
+
+class Clock:
+    def __init__(self):
+        self.now = 0.0
+
+    def __call__(self):
+        return self.now
+
+
+def make(failures=3, open_seconds=60):
+    clock = Clock()
+    return CircuitBreaker(failures, open_seconds, clock), clock
+
+
+def test_unknown_keys_are_closed():
+    breaker, _ = make()
+    assert breaker.state("x") == CLOSED and breaker.allows("x")
+
+
+def test_it_opens_at_the_threshold_and_not_before():
+    breaker, _ = make(failures=3)
+    breaker.failure("x")
+    breaker.failure("x")
+    assert breaker.allows("x")
+    breaker.failure("x")
+    assert breaker.state("x") == OPEN and not breaker.allows("x")
+
+
+def test_a_success_resets_the_streak():
+    breaker, _ = make(failures=3)
+    breaker.failure("x")
+    breaker.failure("x")
+    breaker.success("x")
+    breaker.failure("x")
+    breaker.failure("x")
+    assert breaker.allows("x")
+
+
+def test_it_goes_half_open_after_the_cooldown_and_admits_one_probe():
+    breaker, clock = make(failures=1, open_seconds=60)
+    breaker.failure("x")
+    clock.now = 59
+    assert breaker.state("x") == OPEN
+    clock.now = 60
+    assert breaker.state("x") == HALF_OPEN and breaker.allows("x")
+    breaker.begin("x")
+    assert not breaker.allows("x")
+
+
+def test_a_failed_probe_reopens_for_a_full_cooldown():
+    breaker, clock = make(failures=1, open_seconds=60)
+    breaker.failure("x")
+    clock.now = 70
+    breaker.begin("x")
+    breaker.failure("x")
+    assert breaker.state("x") == OPEN
+    clock.now = 129
+    assert breaker.state("x") == OPEN
+    clock.now = 131
+    assert breaker.state("x") == HALF_OPEN
+
+
+def test_a_successful_probe_closes_it():
+    breaker, clock = make(failures=1)
+    breaker.failure("x")
+    clock.now = 100
+    breaker.begin("x")
+    breaker.success("x")
+    assert breaker.state("x") == CLOSED and breaker.allows("x")
+
+
+def test_keys_are_independent():
+    breaker, _ = make(failures=1)
+    breaker.failure(("node", "a"))
+    assert not breaker.allows(("node", "a"))
+    assert breaker.allows(("node", "b"))
+    assert breaker.allows(("destination", "a"))
+
+
+def test_open_count_and_forget():
+    breaker, _ = make(failures=1)
+    breaker.failure("a")
+    breaker.failure("b")
+    assert breaker.open_count() == 2
+    breaker.forget("a")
+    assert breaker.open_count() == 1
+
+
+def test_a_zero_threshold_still_needs_one_failure():
+    breaker = CircuitBreaker(failures=0)
+    assert breaker.allows("x")
+    breaker.failure("x")
+    assert not breaker.allows("x")

tests/unit/services/nodes/dispatcher.py

@@ -0,0 +1,415 @@
+# retoor <retoor@molodetz.nl>
+
+import base64
+import json
+import random
+import time
+
+import pytest
+
+from devplacepy.database import get_table
+from devplacepy.database import nodes as store
+from devplacepy.services.nodes.broker import BodyError, JobBroker, clean_headers, decode_body
+from devplacepy.services.nodes.circuit import CircuitBreaker
+from devplacepy.services.nodes.dispatcher import JOB_HEADERS, NodeDispatcher
+from devplacepy.services.nodes.ledger import UsageLedger
+from devplacepy.services.nodes.policy import DestinationPolicy
+from devplacepy.services.nodes.registry import NodeCounters, NodeRegistry
+from devplacepy.services.nodes.scheduler import MEGABYTE, NodeScheduler
+from devplacepy.services.nodes.session import NodeSession
+from devplacepy.services.nodes.spec import SPEC
+from devplacepy.utils import generate_uid
+from tests.conftest import run_async
+from tests.unit.services.nodes.scheduler import Clock, node, settings
+
+LIMITS = SPEC.limits
+TARGET_URL = "https://www.reddit.com/r/python.json"
+
+
+class Rig:
+    def __init__(self):
+        self.clock = Clock()
+        self.counters = NodeCounters()
+        self.registry = NodeRegistry(self.counters)
+        self.breaker = CircuitBreaker(clock=self.clock)
+        self.scheduler = NodeScheduler(self.registry, self.breaker, self.clock, rng=random.Random(2))
+        self.broker = JobBroker()
+        self.dispatcher = NodeDispatcher(
+            self.registry, self.scheduler, self.broker, self.breaker, UsageLedger(), self.counters,
+            clock=self.clock,
+        )
+
+    def connect(self, **overrides):
+        owner = generate_uid()
+        device, _ = store.create_device(owner, "d", "native")
+        session = NodeSession(
+            node().websocket, {"uid": device["uid"], "user_uid": owner, "kind": "native"}
+        )
+        session.hello = {"modes": ["fetch"]}
+        session.settings = settings(**overrides)
+        session.roll_day()
+        self.dispatcher.attach(session)
+        self.registry.admit(session)
+        return session
+
+    def enqueue(self, url=TARGET_URL, caller="c1", kind="user", mode="any"):
+        target = DestinationPolicy().check(url, "GET", mode)
+        return self.broker.enqueue(kind, caller, target, mode)
+
+    def tick(self):
+        return run_async(self.dispatcher.tick())
+
+    def answer(self, session, uid, **fields):
+        frame = {"type": "job_result", "job_id": uid, "status": 200, "body_b64": base64.b64encode(b"hello").decode()}
+        frame.update(fields)
+        run_async(session._handle(frame))
+
+
+@pytest.fixture
+def rig(local_db):
+    get_table("node_requests").delete()
+    yield Rig()
+    get_table("node_requests").delete()
+
+
+def sent_jobs(session):
+    return [frame for frame in session.websocket.sent if frame["type"] == "job"]
+
+
+def test_a_queued_request_becomes_a_job_frame(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    assert rig.tick() == 1
+    job = sent_jobs(session)[0]
+    assert job == {"type": "job", "job_id": uid, "method": "GET", "url": TARGET_URL, "headers": JOB_HEADERS, "timeout": LIMITS.job_timeout_seconds}
+    assert store.get_request(uid)["status"] == "assigned"
+    assert store.get_request(uid)["device_uid"] == session.device_uid
+    assert uid in session.inflight
+
+
+def test_the_result_completes_the_request_and_is_metered(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    rig.answer(session, uid, headers={"Content-Type": "application/json", "Set-Cookie": "x=1"})
+    result = rig.broker.read(uid)
+    assert result.ok and result.http_status == 200 and result.body == b"hello"
+    assert result.headers == {"content-type": "application/json"}
+    assert uid not in session.inflight
+    today = store.usage_today(session.device_uid)
+    assert today["requests_ok"] == 1 and today["bytes_down"] == 5 and today["bytes_up"] > len(TARGET_URL)
+    assert store.get_donor(session.user_uid)["requests_ok"] == 1
+    assert session.used_today == today["bytes_up"] + today["bytes_down"]
+    assert rig.counters.snapshot()["jobs_done"] == 1
+
+
+def test_no_eligible_node_leaves_the_request_queued(rig):
+    uid = rig.enqueue()
+    assert rig.tick() == 0
+    assert store.get_request(uid)["status"] == "queued"
+    rig.connect(allowed_platforms=[])
+    assert rig.tick() == 0
+    assert store.get_request(uid)["status"] == "queued"
+
+
+def test_one_request_goes_to_exactly_one_node(rig):
+    sessions = [rig.connect() for _ in range(4)]
+    rig.enqueue()
+    assert rig.tick() == 1
+    assert sum(len(sent_jobs(item)) for item in sessions) == 1
+
+
+def test_a_caller_never_gets_its_own_node(rig):
+    session = rig.connect()
+    uid = rig.enqueue(caller=session.user_uid)
+    assert rig.tick() == 0
+    assert store.get_request(uid)["status"] == "queued"
+    guest = rig.enqueue(caller=session.user_uid, kind="guest")
+    assert rig.tick() == 1 and guest in session.inflight
+
+
+def test_a_policy_failure_at_dispatch_time_fails_the_request(rig):
+    rig.connect()
+    uid = store.enqueue_request("user", "c", "https://evil.example/", "GET", "x", "any")
+    rig.tick()
+    row = store.get_request(uid)
+    assert row["status"] == "failed" and row["error"] == "host_not_allowed"
+    assert rig.counters.snapshot()["jobs_refused"] == 1
+
+
+def test_a_job_error_fails_the_request_and_counts_against_the_node(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    run_async(session._handle({"type": "job_error", "job_id": uid, "code": "timeout"}))
+    assert store.get_request(uid)["error"] == "timeout"
+    assert store.usage_today(session.device_uid)["requests_failed"] == 1
+    assert rig.counters.snapshot()["jobs_failed"] == 1
+
+
+def test_five_node_failures_open_that_nodes_circuit(rig):
+    session = rig.connect()
+    for _ in range(LIMITS.breaker_failures):
+        uid = rig.enqueue()
+        session.host_started.clear()
+        rig.scheduler._destination_started.clear()
+        rig.tick()
+        run_async(session._handle({"type": "job_error", "job_id": uid, "code": "network"}))
+    follow = rig.enqueue()
+    session.host_started.clear()
+    rig.scheduler._destination_started.clear()
+    assert rig.tick() == 0
+    assert store.get_request(follow)["status"] == "queued"
+    rig.clock.now += LIMITS.breaker_open_seconds
+    assert rig.tick() == 1
+
+
+def test_donor_limit_codes_pause_the_node_without_blaming_it(rig):
+    session = rig.connect(bandwidth_mb_day=50)
+    uid = rig.enqueue()
+    rig.tick()
+    run_async(session._handle({"type": "job_error", "job_id": uid, "code": "bandwidth_exhausted"}))
+    assert session.used_today == 50 * MEGABYTE
+    assert rig.breaker.allows(("node", session.device_uid))
+    other = rig.connect()
+    rig.clock.now += 10
+    uid = rig.enqueue()
+    assert rig.tick() == 1
+    run_async(other._handle({"type": "job_error", "job_id": uid, "code": "outside_hours"}))
+    assert other.paused_until > rig.clock.now
+    assert rig.breaker.allows(("node", other.device_uid))
+
+
+def test_a_donor_refusal_of_an_allowed_destination_raises_a_flag(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    run_async(session._handle({"type": "job_error", "job_id": uid, "code": "host_not_allowed"}))
+    kinds = [flag["kind"] for flag in store.list_flags()]
+    assert "refusal_mismatch" in kinds
+    assert rig.breaker.allows(("node", session.device_uid))
+
+
+@pytest.mark.parametrize(
+    "fields,code",
+    [
+        ({"status": 99}, "bad_status"),
+        ({"status": 600}, "bad_status"),
+        ({"body_b64": "***not base64***"}, "bad_base64"),
+        ({"body_b64": "abc"}, "bad_base64"),
+    ],
+)
+def test_forged_results_are_integrity_failures(rig, fields, code):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    rig.answer(session, uid, **fields)
+    assert store.get_request(uid)["error"] == code
+    assert store.usage_today(session.device_uid)["integrity_failures"] == 1
+    assert rig.counters.snapshot()["integrity_failures"] == 1
+    assert any(flag["kind"] == "integrity" for flag in store.list_flags())
+    assert store.get_donor(session.user_uid)["requests_ok"] == 0
+
+
+def test_an_oversized_body_is_truncated_and_flagged(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    big = base64.b64encode(b"x" * (LIMITS.max_body_bytes + 10)).decode()
+    rig.answer(session, uid, body_b64=big)
+    result = rig.broker.read(uid)
+    assert result.truncated and len(result.body) == LIMITS.max_body_bytes
+    assert rig.counters.snapshot()["results_truncated"] == 1
+
+
+def test_a_donor_declared_truncation_is_kept(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    rig.answer(session, uid, truncated=True)
+    assert rig.broker.read(uid).truncated
+
+
+def test_a_result_for_an_unknown_job_is_a_protocol_error(rig):
+    session = rig.connect()
+    from devplacepy.services.nodes.frames import FrameError
+
+    with pytest.raises(FrameError):
+        rig.answer(session, "never-assigned")
+
+
+def test_a_job_cannot_be_answered_twice(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    rig.answer(session, uid)
+    from devplacepy.services.nodes.frames import FrameError
+
+    with pytest.raises(FrameError):
+        rig.answer(session, uid)
+    assert store.get_donor(session.user_uid)["requests_ok"] == 1
+
+
+def test_rate_limited_and_server_error_results_open_the_destination_circuit(rig):
+    for index in range(LIMITS.breaker_failures):
+        session = rig.connect()
+        uid = rig.enqueue()
+        rig.scheduler._destination_started.clear()
+        rig.tick()
+        rig.answer(session, uid, status=429 if index % 2 else 503)
+        assert rig.broker.read(uid).ok
+        rig.registry.release(session)
+    assert not rig.breaker.allows(("destination", "reddit.com"))
+    rig.connect()
+    rig.enqueue()
+    rig.scheduler._destination_started.clear()
+    assert rig.tick() == 0
+
+
+def test_a_healthy_result_closes_the_destination_circuit(rig):
+    rig.breaker.failure(("destination", "reddit.com"))
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    rig.answer(session, uid, status=200)
+    assert rig.breaker.state(("destination", "reddit.com")) == "closed"
+
+
+def test_a_job_that_never_answers_times_out(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    rig.clock.now += LIMITS.job_timeout_seconds + LIMITS.job_grace_seconds + 1
+    assert run_async(rig.dispatcher.sweep_inflight()) == 1
+    assert store.get_request(uid)["error"] == "timeout"
+    assert session.inflight == {}
+
+
+def test_a_disconnect_fails_every_inflight_job(rig):
+    session = rig.connect(max_concurrent=2)
+    uids = [rig.enqueue(), rig.enqueue()]
+    for _ in uids:
+        session.host_started.clear()
+        rig.scheduler._destination_started.clear()
+        assert rig.tick() == 1
+    assert len(session.inflight) == 2
+    assert rig.dispatcher.release_session(session) == 2
+    for uid in uids:
+        row = store.get_request(uid)
+        assert row["status"] == "failed" and row["error"] == "node_disconnected"
+    assert session.inflight == {}
+
+
+def test_release_session_marks_assigned_jobs_as_node_disconnected(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    assert rig.dispatcher.release_session(session) == 1
+    assert store.get_request(uid)["error"] == "node_disconnected"
+    assert rig.dispatcher.release_session(session) == 0
+
+
+def test_a_failed_send_releases_the_job(rig):
+    session = rig.connect()
+
+    async def broken(*args, **kwargs):
+        raise ConnectionError("gone")
+
+    session.send = broken
+    uid = rig.enqueue()
+    assert rig.tick() == 0
+    assert store.get_request(uid)["error"] == "send_failed"
+    assert session.inflight == {}
+
+
+def test_an_idle_tick_does_no_writes_and_no_work(rig):
+    rig.connect()
+    started = time.perf_counter()
+    for _ in range(50):
+        assert rig.tick() == 0
+    assert (time.perf_counter() - started) / 50 < 0.02
+
+
+def test_the_cap_refusing_the_ledger_pauses_the_node(rig):
+    session = rig.connect(bandwidth_mb_day=1)
+    store.save_node_settings(session.user_uid, {"enabled": True, "bandwidth_mb_day": 1})
+    session.used_today = 0
+    uid = rig.enqueue()
+    rig.tick()
+    store.record_job(session.device_uid, session.user_uid, ok=True, bytes_down=MEGABYTE, cap_bytes=MEGABYTE)
+    rig.answer(session, uid)
+    assert rig.broker.read(uid).ok
+    assert session.used_today >= MEGABYTE
+
+
+def test_every_assigned_job_is_answered_at_most_once_under_a_random_workload(rig):
+    rng = random.Random(11)
+    sessions = [rig.connect(max_concurrent=2) for _ in range(4)]
+    seen = set()
+    for _ in range(150):
+        rig.enqueue(caller=generate_uid())
+        rig.clock.now += 10
+        rig.scheduler._destination_started.clear()
+        rig.tick()
+        for session in sessions:
+            for job_id in list(session.inflight):
+                assert job_id not in seen
+                action = rng.random()
+                if action < 0.6:
+                    rig.answer(session, job_id)
+                    seen.add(job_id)
+                elif action < 0.8:
+                    run_async(session._handle({"type": "job_error", "job_id": job_id, "code": "network"}))
+                    seen.add(job_id)
+            assert len(session.inflight) <= session.max_concurrent
+    open_now = {job for item in sessions for job in item.inflight}
+    assert not (open_now & seen)
+
+
+def test_body_helpers():
+    assert decode_body(base64.b64encode(b"abc").decode()) == (b"abc", False)
+    with pytest.raises(BodyError):
+        decode_body("!!!")
+    assert clean_headers({"A": "1", "set-cookie": "s", "Authorization": "x", 5: "x", "b": 5}) == {"a": "1"}
+    assert clean_headers("nope") == {}
+    many = {f"h{i}": "v" for i in range(500)}
+    assert len(clean_headers(many)) == 64
+    assert len(next(iter(clean_headers({"k": "v" * 5000}).values()))) == 1024
+    assert json.loads(json.dumps(clean_headers({"A": "1"}))) == {"a": "1"}
+
+
+def test_a_saturated_destination_is_scanned_once_per_tick_not_once_per_request(rig, monkeypatch):
+    rig.connect(allowed_platforms=[])
+    for index in range(30):
+        rig.enqueue(caller=f"c{index}")
+    scans = []
+    original = rig.scheduler.candidates
+    monkeypatch.setattr(rig.scheduler, "candidates", lambda *a: scans.append(a) or original(*a))
+    assert rig.tick() == 0
+    assert len(scans) == 1
+
+
+def test_saturation_does_not_hide_other_destinations(rig):
+    session = rig.connect()
+    for index in range(3):
+        rig.enqueue(caller=f"c{index}")
+    rig.scheduler._destination_started["reddit.com"] = rig.clock.now
+    assert rig.tick() == 0
+    rig.clock.now += 10
+    assert rig.tick() == 1
+    assert len(session.inflight) == 1
+
+
+def test_the_owner_exclusion_does_not_saturate_the_destination(rig):
+    mine = rig.connect()
+    other = rig.connect()
+    first = rig.enqueue(caller=mine.user_uid)
+    second = rig.enqueue(caller=other.user_uid)
+    rig.clock.now += 10
+    rig.tick()
+    assert first in other.inflight
+    rig.clock.now += 10
+    rig.scheduler._destination_started.clear()
+    other.host_started.clear()
+    rig.tick()
+    assert second in mine.inflight

tests/unit/services/nodes/fabric.py

@@ -0,0 +1,231 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+from datetime import datetime, timezone
+
+import pytest
+
+from devplacepy.database import get_table, set_setting
+from devplacepy.database import nodes as store
+from devplacepy.services.nodes.broker import BrokerResult, JobBroker
+from devplacepy.services.nodes.cache import ResultCache
+from devplacepy.services.nodes.fabric import (
+    FabricQuotaExceeded,
+    FabricRefused,
+    FabricUnavailable,
+    NodeFabric,
+)
+from devplacepy.services.nodes.spec import SPEC
+from devplacepy.utils import generate_uid
+from tests.conftest import run_async
+
+LIMITS = SPEC.limits
+URL = "https://www.reddit.com/r/python.json"
+
+
+class ScriptedBroker(JobBroker):
+    def __init__(self, outcomes):
+        self.outcomes = list(outcomes)
+        self.enqueued = 0
+
+    def enqueue(self, caller_kind, caller_id, target, mode):
+        self.enqueued += 1
+        return f"uid-{self.enqueued}"
+
+    async def wait(self, uid, timeout):
+        outcome = self.outcomes.pop(0)
+        if outcome == "ok":
+            return BrokerResult(uid, "done", 200, {"content-type": "text/plain"}, b"payload", False, "")
+        return BrokerResult(uid, "failed", 0, {}, b"", False, outcome)
+
+
+def heartbeat(connected):
+    table = get_table("service_state")
+    record = {
+        "name": "nodes",
+        "status": "running",
+        "heartbeat": datetime.now(timezone.utc).isoformat(),
+        "metrics": json.dumps({"stats": [{"label": "connected", "value": connected}]}),
+    }
+    existing = table.find_one(name="nodes")
+    if existing:
+        table.update({**record, "id": existing["id"]}, ["id"])
+    else:
+        table.insert(record)
+
+
+@pytest.fixture
+def online(local_db):
+    set_setting("nodes_enabled", "1")
+    set_setting("nodes_daily_quota_user", "1000")
+    set_setting("nodes_daily_quota_guest", "0")
+    heartbeat(3)
+    yield
+    set_setting("nodes_enabled", "0")
+    get_table("service_state").delete(name="nodes")
+
+
+def fetch(fabric, **kwargs):
+    kwargs.setdefault("caller_kind", "user")
+    kwargs.setdefault("caller_id", generate_uid())
+    return run_async(fabric.fetch(URL, **kwargs))
+
+
+def test_a_node_fetch_returns_the_body_and_caches_it(online):
+    broker = ScriptedBroker(["ok"])
+    fabric = NodeFabric(broker=broker)
+    first = fetch(fabric, caller_id="c")
+    assert (first.status, first.body, first.via) == (200, b"payload", "fetch")
+    second = fetch(fabric, caller_id="c")
+    assert second.via == "cache" and second.body == b"payload" and second.elapsed_ms == 0
+    assert broker.enqueued == 1
+
+
+def test_use_cache_false_always_asks_a_node(online):
+    broker = ScriptedBroker(["ok", "ok"])
+    fabric = NodeFabric(broker=broker)
+    fetch(fabric, caller_id="c", use_cache=False)
+    fetch(fabric, caller_id="c", use_cache=False)
+    assert broker.enqueued == 2
+
+
+def test_policy_refusals_surface_with_their_code(online):
+    fabric = NodeFabric(broker=ScriptedBroker([]))
+    for url, code in (("https://example.com/", "host_not_allowed"), ("https://reddit.com:22/", "port_not_allowed")):
+        with pytest.raises(FabricRefused) as info:
+            run_async(fabric.fetch(url, caller_kind="user", caller_id="c"))
+        assert info.value.code == code
+
+
+def test_the_kill_switch_and_missing_nodes_are_unavailable(online):
+    fabric = NodeFabric(broker=ScriptedBroker([]))
+    heartbeat(0)
+    with pytest.raises(FabricUnavailable) as info:
+        fetch(fabric)
+    assert info.value.code == "no_nodes"
+    heartbeat(2)
+    set_setting("nodes_enabled", "0")
+    with pytest.raises(FabricUnavailable) as info:
+        fetch(fabric)
+    assert info.value.code == "kill_switch"
+
+
+def test_a_stale_heartbeat_counts_as_no_nodes(online):
+    table = get_table("service_state")
+    row = table.find_one(name="nodes")
+    table.update({"id": row["id"], "heartbeat": "2000-01-01T00:00:00+00:00"}, ["id"])
+    assert NodeFabric(broker=ScriptedBroker([])).available().nodes_online == 0
+
+
+def test_no_service_row_means_no_nodes(online):
+    get_table("service_state").delete(name="nodes")
+    assert NodeFabric().available().nodes_online == 0
+
+
+def test_quota_is_enforced_before_any_node_is_touched(online):
+    set_setting("nodes_daily_quota_user", "2")
+    broker = ScriptedBroker(["ok"] * 5)
+    fabric = NodeFabric(broker=broker)
+    caller = generate_uid()
+    for _ in range(2):
+        fetch(fabric, caller_id=caller, use_cache=False)
+    with pytest.raises(FabricQuotaExceeded):
+        fetch(fabric, caller_id=caller, use_cache=False)
+    assert broker.enqueued == 2
+
+
+def test_guests_have_no_quota_by_default(online):
+    broker = ScriptedBroker(["ok"])
+    with pytest.raises(FabricQuotaExceeded):
+        fetch(NodeFabric(broker=broker), caller_kind="guest", caller_id="g")
+    assert broker.enqueued == 0
+
+
+def test_a_cache_hit_costs_no_quota(online):
+    set_setting("nodes_daily_quota_user", "1")
+    fabric = NodeFabric(broker=ScriptedBroker(["ok"]))
+    caller = generate_uid()
+    fetch(fabric, caller_id=caller)
+    for _ in range(5):
+        assert fetch(fabric, caller_id=caller).via == "cache"
+
+
+def test_a_retryable_failure_is_retried_once_on_another_node(online):
+    broker = ScriptedBroker(["node_disconnected", "ok"])
+    result = fetch(NodeFabric(broker=broker))
+    assert result.body == b"payload" and broker.enqueued == 2
+
+
+def test_the_retry_does_not_spend_a_second_quota_unit(online):
+    set_setting("nodes_daily_quota_user", "1")
+    caller = generate_uid()
+    fetch(NodeFabric(broker=ScriptedBroker(["timeout", "ok"])), caller_id=caller)
+    assert store.caller_requests_today("user", caller) == 1
+
+
+def test_retries_stop_after_the_attempt_cap(online):
+    broker = ScriptedBroker(["timeout"] * 5)
+    with pytest.raises(FabricUnavailable) as info:
+        fetch(NodeFabric(broker=broker))
+    assert info.value.code == "timeout" and broker.enqueued == 2
+
+
+def test_a_non_retryable_failure_is_not_retried(online):
+    broker = ScriptedBroker(["bad_status", "ok"])
+    with pytest.raises(FabricUnavailable):
+        fetch(NodeFabric(broker=broker))
+    assert broker.enqueued == 1
+
+
+def test_a_full_queue_is_a_clean_unavailable(online):
+    class Full(ScriptedBroker):
+        def enqueue(self, *args):
+            raise store.QueueFull("full")
+
+    with pytest.raises(FabricUnavailable) as info:
+        fetch(NodeFabric(broker=Full([])))
+    assert info.value.code == "queue_full"
+
+
+def test_the_wait_is_capped_at_the_spec_maximum(online):
+    seen = []
+
+    class Spy(ScriptedBroker):
+        async def wait(self, uid, timeout):
+            seen.append(timeout)
+            return await super().wait(uid, timeout)
+
+    fetch(NodeFabric(broker=Spy(["ok"])), timeout=10**6)
+    assert seen[0] <= LIMITS.max_wait_seconds
+
+
+def test_the_cache_ttl_follows_the_admin_setting_up_to_the_ceiling(online):
+    cache = ResultCache()
+    fabric = NodeFabric(broker=ScriptedBroker(["ok"]), cache=cache)
+    set_setting("nodes_cache_ttl_seconds", "0")
+    fetch(fabric, caller_id="c")
+    assert len(cache) == 0
+    set_setting("nodes_cache_ttl_seconds", str(10**9))
+    fabric._sync_cache_ttl()
+    assert cache.ttl_seconds == LIMITS.cache_ttl_seconds
+    set_setting("nodes_cache_ttl_seconds", str(LIMITS.cache_ttl_seconds))
+
+
+def test_can_serve_needs_policy_switch_and_a_node(online):
+    fabric = NodeFabric(broker=ScriptedBroker([]))
+    assert fabric.can_serve(URL)
+    assert not fabric.can_serve("https://example.com/")
+    assert not fabric.can_serve(URL, "POST")
+    heartbeat(0)
+    assert not fabric.can_serve(URL)
+
+
+def test_the_real_broker_times_out_a_request_nobody_picks_up(online):
+    get_table("node_requests").delete()
+    fabric = NodeFabric()
+    with pytest.raises(FabricUnavailable) as info:
+        fetch(fabric, timeout=0.3)
+    assert info.value.code in ("caller_timeout", "failed")
+    rows = list(get_table("node_requests").find())
+    assert rows and all(row["status"] == "failed" for row in rows)
+    get_table("node_requests").delete()

tests/unit/services/nodes/ledger.py

@@ -0,0 +1,55 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_table
+from devplacepy.database import nodes as store
+from devplacepy.services.nodes import ledger as ledger_module
+from devplacepy.services.nodes.ledger import FIRST_BADGE, GIGABYTE, UsageLedger
+from devplacepy.utils import generate_uid, register_account
+
+
+def member(local_db):
+    name = "ledger" + generate_uid()[-10:]
+    uid, _, _ = register_account(name, f"{name}@t.dev", "secret123", age_band="adult", accepted_terms=True)
+    return uid
+
+
+def badges(uid):
+    return {row["badge_name"] for row in get_table("badges").find(user_uid=uid)}
+
+
+def test_first_success_awards_xp_and_the_first_badge(local_db):
+    uid = member(local_db)
+    before = get_table("users").find_one(uid=uid).get("xp") or 0
+    store.record_job(generate_uid(), uid, ok=True, bytes_up=10, bytes_down=100)
+    UsageLedger().reward(uid)
+    assert FIRST_BADGE in badges(uid)
+    assert (get_table("users").find_one(uid=uid).get("xp") or 0) > before
+
+
+def test_xp_stops_at_the_daily_cap_but_not_the_counters(local_db):
+    uid = member(local_db)
+    ledger = UsageLedger()
+    store.record_job(generate_uid(), uid, ok=True, bytes_up=1, bytes_down=1)
+    granted = 0
+    for _ in range(store.LIMITS.xp_daily_cap + 10):
+        granted += int(ledger.claim_xp(uid, ledger_module.XP_PER_JOB))
+    assert granted == store.LIMITS.xp_daily_cap
+
+
+def test_byte_badges_follow_lifetime_totals(local_db):
+    uid = member(local_db)
+    store.record_job(
+        generate_uid(), uid, ok=True, bytes_up=0, bytes_down=GIGABYTE, cap_bytes=2 * GIGABYTE
+    )
+    UsageLedger().reward(uid)
+    assert {"Gigabyte Donor"} <= badges(uid)
+    assert "10 GB Donor" not in badges(uid)
+
+
+def test_a_second_reward_call_does_not_duplicate_badges(local_db):
+    uid = member(local_db)
+    store.record_job(generate_uid(), uid, ok=True, bytes_up=1, bytes_down=1)
+    ledger = UsageLedger()
+    ledger.reward(uid)
+    ledger.reward(uid)
+    assert len(list(get_table("badges").find(user_uid=uid, badge_name=FIRST_BADGE))) == 1

tests/unit/services/nodes/policy.py

@@ -0,0 +1,143 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.nodes.policy import DestinationPolicy, PolicyError, is_public_address
+from devplacepy.services.nodes.spec import SPEC
+
+POLICY = DestinationPolicy()
+
+
+def code(url, method="GET", mode="any"):
+    with pytest.raises(PolicyError) as info:
+        POLICY.check(url, method, mode)
+    return info.value.code
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "https://reddit.com/r/python.json",
+        "https://www.reddit.com/r/python/top.json?t=week",
+        "http://old.reddit.com/",
+        "https://REDDIT.COM/",
+        "https://reddit.com./r/x",
+        "https://redd.it/abc",
+        "https://reddit.com:443/x",
+        "http://reddit.com:80/x",
+    ],
+)
+def test_allowlisted_urls_pass(url):
+    target = POLICY.check(url)
+    assert target.method == "GET" and target.adapter == "reddit"
+    assert target.port in (80, 443)
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "https://evilreddit.com/",
+        "https://reddit.com.evil.net/",
+        "https://notreddit.com/",
+        "https://example.com/",
+        "https://localhost/",
+        "https://reddit.evil.com/",
+        "https://evil.com/reddit.com",
+    ],
+)
+def test_other_hosts_are_refused(url):
+    assert code(url) == "host_not_allowed"
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "https://127.0.0.1/",
+        "https://[::1]/",
+        "http://10.0.0.5/",
+        "https://169.254.169.254/latest/meta-data",
+        "https://0x7f.1/",
+        "https://2130706433/",
+    ],
+)
+def test_address_literals_are_refused(url):
+    assert code(url) in ("host_not_allowed", "bad_url")
+
+
+@pytest.mark.parametrize("port", [22, 25, 8080, 8443, 1, 65535])
+def test_only_web_ports(port):
+    assert code(f"https://reddit.com:{port}/") == "port_not_allowed"
+
+
+@pytest.mark.parametrize(
+    "url",
+    ["ftp://reddit.com/", "file:///etc/passwd", "javascript:alert(1)", "//reddit.com/x", "reddit.com/x", ""],
+)
+def test_other_schemes_and_shapes_are_bad_urls(url):
+    assert code(url) == "bad_url"
+
+
+def test_credentials_in_the_url_are_refused():
+    assert code("https://user:pw@reddit.com/") == "bad_url"
+    assert code("https://user@reddit.com/") == "bad_url"
+    assert code("https://evil.com@reddit.com.evil.com/") == "bad_url"
+    assert code("https://reddit.com@evil.com/") == "bad_url"
+
+
+def test_overlong_and_non_string_urls_are_refused():
+    assert code("https://reddit.com/" + "a" * SPEC.limits.url_max_chars) == "bad_url"
+    with pytest.raises(PolicyError):
+        POLICY.check(None)
+    with pytest.raises(PolicyError):
+        POLICY.check(["https://reddit.com/"])
+
+
+@pytest.mark.parametrize("method", ["POST", "PUT", "DELETE", "PATCH", "CONNECT", ""])
+def test_only_declared_methods(method):
+    assert code("https://reddit.com/", method) == "method_not_allowed"
+
+
+def test_method_is_case_insensitive():
+    assert POLICY.check("https://reddit.com/", "get").method == "GET"
+
+
+def test_unknown_mode_is_refused():
+    assert code("https://reddit.com/", mode="raw") == "bad_mode"
+
+
+def test_a_redirect_off_the_allowlist_is_a_redirect_refusal():
+    with pytest.raises(PolicyError) as info:
+        POLICY.check_redirect("https://evil.example/")
+    assert info.value.code == "redirect_not_allowed"
+    assert POLICY.check_redirect("https://www.reddit.com/new").host == "www.reddit.com"
+
+
+@pytest.mark.parametrize(
+    "address,expected",
+    [
+        ("8.8.8.8", True),
+        ("1.1.1.1", True),
+        ("2606:4700:4700::1111", True),
+        ("127.0.0.1", False),
+        ("10.1.2.3", False),
+        ("172.16.0.1", False),
+        ("192.168.1.1", False),
+        ("169.254.1.1", False),
+        ("100.64.0.1", False),
+        ("100.127.255.254", False),
+        ("224.0.0.1", False),
+        ("0.0.0.0", False),
+        ("::1", False),
+        ("fe80::1", False),
+        ("fc00::1", False),
+        ("not an ip", False),
+    ],
+)
+def test_public_address_classification(address, expected):
+    assert is_public_address(address) is expected
+
+
+def test_every_destination_in_the_spec_passes_its_own_policy():
+    for destination in SPEC.destinations:
+        target = POLICY.check(f"https://{destination.suffix}/x")
+        assert target.destination is destination

tests/unit/services/nodes/scheduler.py

@@ -0,0 +1,283 @@
+# retoor <retoor@molodetz.nl>
+
+import random
+from collections import Counter
+from datetime import datetime, timezone
+
+import pytest
+
+from devplacepy.services.nodes.circuit import CircuitBreaker
+from devplacepy.services.nodes.policy import DestinationPolicy
+from devplacepy.services.nodes.registry import NodeCounters, NodeRegistry
+from devplacepy.services.nodes.scheduler import MEGABYTE, NodeScheduler, in_active_hours
+from devplacepy.services.nodes.session import Inflight, NodeSession
+from devplacepy.services.nodes.spec import SPEC
+from tests.unit.services.nodes.session import FakeSocket
+
+LIMITS = SPEC.limits
+TARGET = DestinationPolicy().check("https://www.reddit.com/r/python.json")
+
+
+class Clock:
+    def __init__(self):
+        self.now = 1000.0
+
+    def __call__(self):
+        return self.now
+
+
+def settings(**overrides):
+    base = {
+        "enabled": True,
+        "bandwidth_mb_day": 100,
+        "max_concurrent": 2,
+        "active_from": "",
+        "active_to": "",
+        "timezone": "UTC",
+        "allowed_platforms": ["reddit"],
+        "pause_on_battery": True,
+        "public_display": False,
+    }
+    base.update(overrides)
+    return base
+
+
+def node(uid="d1", user="u1", modes=("fetch",), **overrides):
+    session = NodeSession(FakeSocket(), {"uid": uid, "user_uid": user, "kind": "native"})
+    session.hello = {"modes": list(modes)}
+    session.settings = settings(**overrides)
+    session.roll_day()
+    return session
+
+
+@pytest.fixture
+def world(local_db):
+    clock = Clock()
+    registry = NodeRegistry(NodeCounters())
+    breaker = CircuitBreaker(clock=clock)
+    wall = {"now": datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)}
+    scheduler = NodeScheduler(registry, breaker, clock, lambda: wall["now"], random.Random(1))
+    scheduler.wall = wall
+    scheduler.clock = clock
+    return scheduler
+
+
+def add(world, **kwargs):
+    session = node(**kwargs)
+    world.registry.admit(session)
+    return session
+
+
+def test_a_healthy_node_is_picked(world):
+    session = add(world)
+    assert world.pick(TARGET) is session
+
+
+def test_no_nodes_means_no_pick(world):
+    assert world.pick(TARGET) is None
+
+
+def test_tunnel_only_jobs_are_not_dispatchable_yet(world):
+    add(world)
+    assert world.pick(TARGET, "tunnel") is None
+    assert world.pick(TARGET, "fetch") is not None
+    assert world.pick(TARGET, "any") is not None
+
+
+def test_a_node_must_speak_the_transport(world):
+    add(world, modes=("other",))
+    assert world.pick(TARGET) is None
+
+
+def test_a_tunnel_node_serves_tunnel_and_any_but_not_fetch(world):
+    session = add(world, modes=("tunnel",))
+    assert world.pick(TARGET, "tunnel") is session
+    assert world.pick(TARGET, "any") is session
+    assert world.pick(TARGET, "fetch") is None
+    assert world.transport_of(session, TARGET, "any") == "tunnel"
+
+
+def test_any_prefers_nodes_that_can_tunnel(world):
+    fetcher = add(world, uid="f1", user="u1", modes=("fetch",))
+    tunneler = add(world, uid="t1", user="u2", modes=("fetch", "tunnel"))
+    assert world.candidates(TARGET, "any") == [tunneler]
+    assert world.transport_of(fetcher, TARGET, "any") == "fetch"
+
+
+def test_platform_opt_out_is_honoured(world):
+    add(world, allowed_platforms=[])
+    assert world.pick(TARGET) is None
+
+
+def test_a_node_never_serves_its_own_owner(world):
+    add(world, user="alice")
+    assert world.pick(TARGET, caller_owner="alice") is None
+    assert world.pick(TARGET, caller_owner="bob") is not None
+
+
+def test_concurrency_is_capped_by_the_lower_of_server_and_donor(world):
+    session = add(world, max_concurrent=1)
+    session.inflight["j"] = Inflight("j", "h", "reddit.com", "GET", "u", 0, 0)
+    assert world.pick(TARGET) is None
+    session.inflight.clear()
+    assert session.max_concurrent == 1
+    session.settings["max_concurrent"] = 99
+    assert session.max_concurrent == LIMITS.max_concurrent_per_node
+    session.hello["limits"] = {"max_concurrent": 1}
+    assert session.max_concurrent == 1
+
+
+def test_the_donor_bandwidth_cap_stops_assignment(world):
+    session = add(world, bandwidth_mb_day=1)
+    session.used_today = MEGABYTE - 1
+    assert world.pick(TARGET) is session
+    session.used_today = MEGABYTE
+    assert world.pick(TARGET) is None
+
+
+def test_a_new_day_resets_the_used_bytes(world):
+    session = add(world, bandwidth_mb_day=1)
+    session.used_today = MEGABYTE
+    session.used_day = "2000-01-01"
+    assert world.pick(TARGET) is session
+    assert session.used_today == 0
+
+
+def test_a_node_waits_between_requests_to_the_same_destination(world):
+    session = add(world)
+    world.mark_started(session, TARGET)
+    world.clock.now += LIMITS.min_seconds_per_node_domain - 0.1
+    assert world.eligible(TARGET) == []
+    world.clock.now += 0.2
+    world.clock.now += 1.0 / TARGET.destination.max_rps
+    assert world.eligible(TARGET) == [session]
+
+
+def test_the_destination_has_a_global_rate_ceiling(world):
+    first, second = add(world, uid="a", user="ua"), add(world, uid="b", user="ub")
+    world.mark_started(first, TARGET)
+    world.clock.now += 1.0 / TARGET.destination.max_rps - 0.01
+    assert world.pick(TARGET) is None
+    world.clock.now += 0.02
+    assert world.pick(TARGET) in (first, second)
+
+
+def test_a_paused_node_is_skipped_until_the_pause_ends(world):
+    session = add(world)
+    session.paused_until = world.clock.now + 5
+    assert world.pick(TARGET) is None
+    world.clock.now += 5
+    assert world.pick(TARGET) is session
+
+
+def test_an_open_node_circuit_removes_only_that_node(world):
+    bad, good = add(world, uid="bad", user="u1"), add(world, uid="good", user="u2")
+    for _ in range(LIMITS.breaker_failures):
+        world.breaker.failure(("node", "bad"))
+    assert world.eligible(TARGET) == [good]
+    world.clock.now += LIMITS.breaker_open_seconds
+    assert set(world.eligible(TARGET)) == {bad, good}
+
+
+def test_an_open_destination_circuit_stops_everything(world):
+    add(world)
+    for _ in range(LIMITS.breaker_failures):
+        world.breaker.failure(("destination", TARGET.destination.suffix))
+    assert world.pick(TARGET) is None
+
+
+def test_least_loaded_then_least_recently_used_wins(world):
+    busy, idle_old, idle_new = add(world, uid="a", user="1"), add(world, uid="b", user="2"), add(world, uid="c", user="3")
+    busy.inflight["j"] = Inflight("j", "h", "reddit.com", "GET", "u", 0, 0)
+    idle_old.last_assigned = 10
+    idle_new.last_assigned = 20
+    assert world.pick(TARGET) is idle_old
+
+
+def test_selection_is_fair_across_equal_nodes(world):
+    sessions = [add(world, uid=f"d{i}", user=f"u{i}") for i in range(5)]
+    picks = Counter()
+    for _ in range(500):
+        chosen = world.pick(TARGET)
+        picks[chosen.device_uid] += 1
+        chosen.last_assigned = world.clock.now
+        world.clock.now += 0.001
+    assert set(picks) == {item.device_uid for item in sessions}
+    assert max(picks.values()) - min(picks.values()) <= 1
+
+
+@pytest.mark.parametrize(
+    "start,end,hour,minute,expected",
+    [
+        ("", "", 3, 0, True),
+        ("09:00", "17:00", 9, 0, True),
+        ("09:00", "17:00", 16, 59, True),
+        ("09:00", "17:00", 17, 0, False),
+        ("09:00", "17:00", 8, 59, False),
+        ("22:00", "06:00", 23, 0, True),
+        ("22:00", "06:00", 5, 59, True),
+        ("22:00", "06:00", 6, 0, False),
+        ("22:00", "06:00", 12, 0, False),
+        ("09:00", "09:00", 3, 0, True),
+        ("bogus", "17:00", 3, 0, True),
+    ],
+)
+def test_active_hours_windows(start, end, hour, minute, expected):
+    moment = datetime(2026, 1, 1, hour, minute, tzinfo=timezone.utc)
+    assert in_active_hours({"active_from": start, "active_to": end, "timezone": "UTC"}, moment) is expected
+
+
+def test_active_hours_use_the_donors_timezone():
+    moment = datetime(2026, 1, 1, 22, 0, tzinfo=timezone.utc)
+    amsterdam = {"active_from": "09:00", "active_to": "17:00", "timezone": "Europe/Amsterdam"}
+    assert in_active_hours(amsterdam, moment) is False
+    morning = datetime(2026, 1, 1, 8, 30, tzinfo=timezone.utc)
+    assert in_active_hours(amsterdam, morning) is True
+    broken = {"active_from": "09:00", "active_to": "17:00", "timezone": "Mars/Base"}
+    assert in_active_hours(broken, datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) is True
+
+
+def test_a_node_outside_its_hours_is_skipped(world):
+    add(world, active_from="22:00", active_to="23:00")
+    assert world.pick(TARGET) is None
+    world.wall["now"] = datetime(2026, 1, 1, 22, 30, tzinfo=timezone.utc)
+    assert world.pick(TARGET) is not None
+
+
+def test_mark_started_records_both_clocks(world):
+    session = add(world)
+    world.mark_started(session, TARGET)
+    assert session.host_started[TARGET.destination.suffix] == world.clock.now
+    assert session.last_assigned == world.clock.now
+    assert world._destination_started[TARGET.destination.suffix] == world.clock.now
+
+
+def test_the_per_destination_maps_stay_bounded_by_the_allowlist(world):
+    session = add(world)
+    policy = DestinationPolicy()
+    for index in range(200):
+        world.mark_started(session, policy.check(f"https://sub{index}.reddit.com/"))
+    assert len(session.host_started) == 1
+    assert len(world._destination_started) == 1
+
+
+def test_window_checks_are_cached_per_minute_and_still_correct(world):
+    from devplacepy.services.nodes.scheduler import _window_open
+
+    _window_open.cache_clear()
+    window = {"active_from": "09:00", "active_to": "17:00", "timezone": "UTC"}
+    noon = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc)
+    for _ in range(100):
+        assert in_active_hours(window, noon) is True
+    assert _window_open.cache_info().misses == 1
+    night = datetime(2026, 1, 1, 3, 0, tzinfo=timezone.utc)
+    assert in_active_hours(window, night) is False
+
+
+def test_choose_applies_the_owner_exclusion_to_a_shared_candidate_list(world):
+    mine, theirs = add(world, uid="a", user="alice"), add(world, uid="b", user="bob")
+    found = world.candidates(TARGET)
+    assert world.choose(found, "alice") is theirs
+    assert world.choose(found, "bob") is mine
+    assert world.choose([mine], "alice") is None
+    assert world.choose([], "") is None

tests/unit/services/nodes/service.py

@@ -1,5 +1,6 @@
 # retoor <retoor@molodetz.nl>
 
+import asyncio
 import time
 
 import pytest
@@ -146,3 +147,32 @@ def test_metrics_report_connected_and_capacity(service):
 def test_the_service_has_the_kill_switch_as_a_config_field(service):
     keys = [field.key for field in service.all_fields()]
     assert "nodes_enabled" in keys and "nodes_max_total" in keys
+
+
+def test_changed_settings_are_pushed_to_the_node_once(service):
+    uid = member()
+    device, socket = connected(service, uid)
+    session = service.registry.get(device["uid"])
+    session.settings = {"stale": True}
+    assert reconcile(service) == 0
+    pushed = [frame for frame in socket.sent if frame["type"] == "settings_update"]
+    assert len(pushed) == 1 and pushed[0]["settings"]["enabled"] is True
+    assert session.settings == pushed[0]["settings"]
+    reconcile(service)
+    assert len([frame for frame in socket.sent if frame["type"] == "settings_update"]) == 1
+    save_node_settings(uid, {"bandwidth_mb_day": 9})
+    reconcile(service)
+    assert session.settings["bandwidth_mb_day"] == 9
+
+
+def test_the_dispatch_loop_starts_once_and_stops_with_the_service(service):
+    async def scenario():
+        service.ensure_dispatching()
+        first = service._dispatch_task
+        service.ensure_dispatching()
+        assert service._dispatch_task is first and not first.done()
+        await service.on_disable()
+        await asyncio.sleep(0)
+        assert service._dispatch_task is None and first.cancelled()
+
+    run_async(scenario())

tests/unit/services/nodes/session.py

@@ -93,15 +93,20 @@ def test_a_silent_client_times_out(local_db, monkeypatch):
 
 def test_usage_is_advisory_and_summed():
     session = session_for(FakeSocket())
-    session._handle({"type": "usage", "bytes_up": 5, "bytes_down": 7})
-    session._handle({"type": "usage", "bytes_up": -9, "bytes_down": 1})
+    run(session._handle({"type": "usage", "bytes_up": 5, "bytes_down": 7}))
+    run(session._handle({"type": "usage", "bytes_up": -9, "bytes_down": 1}))
     assert (session.reported_up, session.reported_down) == (5, 8)
 
 
-@pytest.mark.parametrize("kind", ["job_result", "job_error", "tunnel_data", "tunnel_close", "tunnel_opened"])
+@pytest.mark.parametrize("kind", ["job_result", "job_error"])
 def test_frames_without_an_assigned_job_are_protocol_errors(kind):
     with pytest.raises(FrameError):
-        session_for(FakeSocket())._handle({"type": kind})
+        run(session_for(FakeSocket())._handle({"type": kind, "job_id": "nope"}))
+
+
+@pytest.mark.parametrize("kind", ["tunnel_data", "tunnel_close", "tunnel_opened"])
+def test_late_tunnel_frames_for_a_closed_relay_are_ignored(kind):
+    run(session_for(FakeSocket())._handle({"type": kind, "conn_id": "gone"}))
 
 
 def test_missed_pings_evict_after_the_floor(monkeypatch):

tests/unit/services/nodes/social.py

@@ -0,0 +1,88 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+from devplacepy.database import set_setting
+from devplacepy.services.nodes import social as social_module
+from devplacepy.services.nodes.adapters.base import AdapterError
+from devplacepy.services.nodes.fabric import FabricUnavailable
+from devplacepy.services.nodes.social import SocialSearch
+from tests.conftest import run_async
+
+HN = json.dumps({"hits": [{"objectID": "1", "title": "HN story", "points": 1}]}).encode()
+SE = json.dumps({"items": [{"title": "SO question", "link": "https://stackoverflow.com/q/1"}]}).encode()
+
+
+class StubFabric:
+    def __init__(self, error=None):
+        self.error = error
+        self.calls = 0
+
+    async def fetch(self, url, **kwargs):
+        self.calls += 1
+        if self.error:
+            raise self.error
+        raise AssertionError("not expected")
+
+
+def search(monkeypatch, getter, platforms, fabric=None):
+    monkeypatch.setattr(social_module, "direct_get", getter)
+    return run_async(
+        SocialSearch(fabric or StubFabric()).search(
+            "sqlite", caller_kind="user", caller_id="u1", platforms=platforms
+        )
+    )
+
+
+def test_hits_are_merged_and_failures_reported(monkeypatch):
+    async def getter(url):
+        if "algolia" in url:
+            return HN
+        if "stackexchange" in url:
+            return SE
+        raise AdapterError("upstream", "HTTP 500")
+
+    outcome = search(monkeypatch, getter, ["hackernews", "stackexchange", "github"])
+    assert {item.platform for item in outcome.hits} == {"hackernews", "stackexchange"}
+    assert outcome.unavailable == {"github": "upstream"}
+
+
+def test_unknown_platform_is_dropped(monkeypatch):
+    async def getter(url):
+        return HN
+
+    outcome = search(monkeypatch, getter, ["hackernews", "x", "instagram"])
+    assert [item.platform for item in outcome.hits] == ["hackernews"]
+    assert outcome.unavailable == {}
+
+
+def test_reddit_is_off_unless_routing_through_nodes_is_enabled(monkeypatch):
+    async def getter(url):
+        return HN
+
+    fabric = StubFabric()
+    outcome = search(monkeypatch, getter, ["reddit"], fabric)
+    assert outcome.unavailable == {"reddit": "nodes_not_used"}
+    assert fabric.calls == 0
+
+
+def test_reddit_reports_the_fabric_reason(monkeypatch):
+    async def getter(url):
+        return HN
+
+    set_setting("nodes_social_via_nodes", "1")
+    try:
+        outcome = search(monkeypatch, getter, ["reddit"], StubFabric(FabricUnavailable("no_nodes")))
+    finally:
+        set_setting("nodes_social_via_nodes", "0")
+    assert outcome.unavailable == {"reddit": "no_nodes"}
+
+
+def test_blank_query_and_limit_cap(monkeypatch):
+    async def getter(url):
+        return HN
+
+    monkeypatch.setattr(social_module, "direct_get", getter)
+    empty = run_async(SocialSearch(StubFabric()).search("  ", caller_kind="user", caller_id="u"))
+    assert empty.hits == []
+    assert social_module.MAX_LIMIT == 20

tests/unit/services/nodes/tunnel.py

@@ -0,0 +1,112 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+import base64
+
+from devplacepy.services.nodes import tunnel as tunnel_module
+from devplacepy.services.nodes.policy import DestinationPolicy
+from devplacepy.services.nodes.tunnel import Relay, TunnelBridge
+from tests.conftest import run_async
+
+TARGET = DestinationPolicy().check("https://www.reddit.com/r/python.json")
+
+
+class Donor:
+    def __init__(self, bridge, accept=True, garbage=False):
+        self.bridge = bridge
+        self.accept = accept
+        self.garbage = garbage
+        self.tunnels = {}
+        self.sent = []
+
+    async def send(self, frame_type, **fields):
+        self.sent.append({"type": frame_type, **fields})
+        if frame_type == "tunnel_open":
+            self.bridge.on_frame(self, {"type": "tunnel_opened", "conn_id": fields["conn_id"], "ok": self.accept})
+        elif frame_type == "tunnel_data":
+            raw = base64.b64decode(fields["b64"])
+            reply = "***not base64***" if self.garbage else base64.b64encode(raw.upper()).decode()
+            self.bridge.on_frame(self, {"type": "tunnel_data", "conn_id": fields["conn_id"], "b64": reply})
+
+
+async def session_through(donor, connect_line, payload=b"hello"):
+    relay = Relay(donor, TARGET)
+    server = await asyncio.start_server(relay.handle, "127.0.0.1", 0)
+    port = server.sockets[0].getsockname()[1]
+    reader, writer = await asyncio.open_connection("127.0.0.1", port)
+    writer.write(connect_line.encode() + b"\r\n\r\n")
+    await writer.drain()
+    status = await asyncio.wait_for(reader.readline(), 5)
+    echoed = b""
+    if b" 200 " in status:
+        await reader.readuntil(b"\r\n")
+        writer.write(payload)
+        await writer.drain()
+        try:
+            echoed = await asyncio.wait_for(reader.read(len(payload)), 5)
+        except asyncio.TimeoutError:
+            echoed = b""
+    writer.close()
+    server.close()
+    await asyncio.sleep(0.05)
+    return status, echoed, relay
+
+
+def go(donor, line, payload=b"hello"):
+    return run_async(session_through(donor, line, payload))
+
+
+def test_connect_to_the_target_relays_both_ways():
+    donor = Donor(TunnelBridge())
+    status, echoed, relay = go(donor, "CONNECT www.reddit.com:443 HTTP/1.1")
+    assert b" 200 " in status and echoed == b"HELLO"
+    assert [item["type"] for item in donor.sent][:2] == ["tunnel_open", "tunnel_data"]
+    assert donor.sent[0]["host"] == "www.reddit.com" and donor.sent[0]["port"] == 443
+    assert relay.up == 5 and relay.down == 5
+
+
+def test_any_other_host_or_port_is_refused_without_bothering_the_donor():
+    for line in ("CONNECT evil.example:443 HTTP/1.1", "CONNECT www.reddit.com:22 HTTP/1.1", "CONNECT 10.0.0.1:443 HTTP/1.1"):
+        donor = Donor(TunnelBridge())
+        status, _, relay = go(donor, line)
+        assert b" 403 " in status and donor.sent == []
+        assert relay.error == "host_not_allowed"
+
+
+def test_only_connect_is_accepted():
+    donor = Donor(TunnelBridge())
+    status, _, _ = go(donor, "GET http://www.reddit.com/ HTTP/1.1")
+    assert b" 405 " in status and donor.sent == []
+
+
+def test_a_donor_that_refuses_is_a_bad_gateway():
+    donor = Donor(TunnelBridge(), accept=False)
+    status, _, relay = go(donor, "CONNECT www.reddit.com:443 HTTP/1.1")
+    assert b" 502 " in status and relay.error == "tunnel_refused"
+
+
+def test_the_relay_has_a_byte_cap(monkeypatch):
+    monkeypatch.setattr(tunnel_module, "RELAY_CAP_BYTES", 8)
+    donor = Donor(TunnelBridge())
+    _, echoed, relay = go(donor, "CONNECT www.reddit.com:443 HTTP/1.1", b"0123456789abcdef")
+    assert relay.error == "body_too_large" and len(echoed) < 16
+
+
+def test_invalid_base64_from_the_donor_ends_the_tunnel():
+    donor = Donor(TunnelBridge(), garbage=True)
+    _, echoed, relay = go(donor, "CONNECT www.reddit.com:443 HTTP/1.1")
+    assert relay.error == "bad_base64" and echoed == b""
+
+
+def test_releasing_a_session_closes_its_relays():
+    bridge = TunnelBridge()
+    donor = Donor(bridge)
+    relay = Relay(donor, TARGET)
+    donor.tunnels[relay.conn_id] = relay
+    bridge.release_session(donor)
+    assert donor.tunnels == {} and relay.frames.get_nowait() is tunnel_module.CLOSED
+
+
+def test_frames_for_unknown_connections_are_not_consumed():
+    bridge = TunnelBridge()
+    assert bridge.on_frame(Donor(bridge), {"type": "tunnel_data", "conn_id": "nope", "b64": ""}) is False

tests/unit/services/sync/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/unit/services/sync/alias.py

@@ -0,0 +1,273 @@
+# retoor <retoor@molodetz.nl>
+
+import random
+from collections import Counter
+
+import pytest
+
+from devplacepy.database import get_table, init_db
+from devplacepy.services.sync import ai
+from devplacepy.services.sync.aliaser import AliasRefused, Aliaser
+from devplacepy.services.sync.identity import IdentityResolver, NameGenerator, fallback_name, source_key
+from devplacepy.services.sync.options import SyncOptions
+from devplacepy.services.sync.rewriter import MentionRewriter
+from devplacepy.services.sync.selection import Selector, bucket, jittered
+from devplacepy.services.sync.style import Style, plausible
+from devplacepy.services.sync.translator import Translator
+from tests.conftest import run_async
+
+ORIGIN = "alias.test"
+
+
+@pytest.fixture(autouse=True)
+def clean(monkeypatch):
+    init_db()
+    for table in ("sync_identities", "sync_links"):
+        get_table(table).delete(origin=ORIGIN)
+
+    async def unreachable(system, text, timeout):
+        return None
+
+    monkeypatch.setattr(ai, "complete", unreachable)
+    yield
+    for table in ("sync_identities", "sync_links"):
+        get_table(table).delete(origin=ORIGIN)
+
+
+def test_rewriter_longest_first_and_single_pass():
+    rewriter = MentionRewriter({"alice": "bob", "alice_w": "carol", "bob": "dave"})
+    result = rewriter.rewrite("hi @alice_w and alice, also @bob")
+    assert result.text == "hi @carol and bob, also @dave"
+    assert len(result.substitutions) == 3
+
+
+def test_rewriter_protects_code_urls_and_inline_code():
+    rewriter = MentionRewriter({"alice": "zed"})
+    text = "see `alice` and https://x.test/~alice\n```\nalice = 1\n```\nthen alice"
+    assert rewriter.rewrite(text).text == text.rsplit("alice", 1)[0] + "zed"
+    assert rewriter.rewrite("```\nalice unterminated").text == "```\nalice unterminated"
+
+
+def test_rewriter_standalone_floor_and_case_and_boundaries():
+    rewriter = MentionRewriter({"ann": "x1", "tef": "y2", "markus": "zeta"})
+    assert rewriter.rewrite("ann do, @ann, @tef, tef, Markus, markusx, a-markus").text == "ann do, @x1, @y2, tef, zeta, markusx, a-markus"
+
+
+def test_rewriter_mentioned_skips_code():
+    found = MentionRewriter({}).mentioned("hi @Alice `@code` @ab ```\n@fenced\n``` @alice_2")
+    assert found == ["Alice", "alice_2"]
+
+
+def test_style_roundtrip_and_matching():
+    style = Style.of("Alice_99")
+    assert Style.load(style.dump()) == style
+    assert style.matches("Marta_12") and not style.matches("marta_12") and not style.matches("Marta12")
+    assert not plausible("alice_99x", "Alice_99", None)
+    assert not plausible("robotuser", "zzzz", None)
+    assert not plausible("a!", "zzzz", None)
+
+
+def test_fallback_name_is_deterministic_and_style_matched():
+    for source in ("alice", "Bob_Smith", "dev99", "UPPER", "kebab-case"):
+        style = Style.of(source)
+        assert fallback_name(ORIGIN, source, style, 0) == fallback_name(ORIGIN, source, style, 0)
+        assert any(plausible(fallback_name(ORIGIN, source, style, n), source, style) for n in range(60))
+
+
+def test_generator_rejects_bad_ai_then_falls_back(monkeypatch):
+    replies = iter(['{"username": "alice"}', "garbage", '{"username": "x"}'])
+
+    async def scripted(system, text, timeout):
+        return next(replies)
+
+    monkeypatch.setattr(ai, "complete", scripted)
+    name, by_ai = run_async(NameGenerator().generate(ORIGIN, "alice", Style.of("alice"), []))
+    assert not by_ai and plausible(name, "alice", Style.of("alice"))
+
+
+def test_generator_accepts_good_ai(monkeypatch):
+    async def good(system, text, timeout):
+        assert "alice" not in text
+        return '```json\n{"username": "marta"}\n```'
+
+    monkeypatch.setattr(ai, "complete", good)
+    assert run_async(NameGenerator().generate(ORIGIN, "alice", Style.of("alice"), [])) == ("marta", True)
+
+
+def test_identity_is_permanent_unique_and_bidirectional():
+    first = IdentityResolver(ORIGIN)
+    names = [run_async(first.ensure(n)) for n in ("alice", "Alice", "bobby", "carol")]
+    assert names[0] == names[1]
+    assert len({name.lower() for name in names[:1] + names[2:]}) == 3
+    second = IdentityResolver(ORIGIN)
+    assert run_async(second.ensure("ALICE")) == names[0]
+    assert second.created == 0
+    assert second.mapping()["alice"] == names[0]
+    rewriter = MentionRewriter({value: key for key, value in second.mapping().items()})
+    assert rewriter.rewrite(f"@{names[0]}").text == "@alice"
+
+
+def test_identity_cap_and_invalid_names_fail_closed():
+    resolver = IdentityResolver(ORIGIN, cap=1)
+    assert run_async(resolver.ensure("alice"))
+    assert run_async(resolver.ensure("bobby")) is None
+    assert run_async(resolver.ensure("not valid!")) is None
+    assert source_key("x" * 100) is None
+
+
+def test_dry_run_resolver_persists_nothing():
+    resolver = IdentityResolver(ORIGIN, persist=False)
+    assert run_async(resolver.ensure("alice"))
+    assert get_table("sync_identities").count(origin=ORIGIN) == 0
+
+
+def test_samples_are_capped():
+    resolver = IdentityResolver(ORIGIN)
+    run_async(resolver.ensure("alice"))
+    for n in range(6):
+        resolver.remember("alice", f"sample number {n}")
+    assert len(resolver.samples("alice")) == 3
+    assert len(IdentityResolver(ORIGIN).samples("alice")) == 3
+
+
+def records():
+    users = [
+        {"origin_uid": "1", "username": "alice", "bio": "secret bio", "website": "https://alice.test", "created_at": "2025-01-01T00:00:00Z"},
+        {"origin_uid": "2", "username": "bobby", "created_at": "2025-01-02T00:00:00Z"},
+    ]
+    posts = [
+        {"origin_uid": "p1", "author_uid": "1", "title": "Hello from alice", "content": "ping @bobby and @ghostly please", "created_at": "2025-02-01T00:00:00Z", "topic": "devlog"},
+        {"origin_uid": "p2", "author_uid": "9", "title": "orphan", "content": "no known author here", "created_at": "2025-02-02T00:00:00Z"},
+    ]
+    comments = [
+        {"origin_uid": "c1", "author_uid": "2", "post_uid": "p1", "content": "thanks alice", "created_at": "2025-02-03T00:00:00Z"},
+        {"origin_uid": "c2", "author_uid": "2", "post_uid": "p1", "parent_uid": "cX", "content": "reply to missing", "created_at": "2025-02-03T00:00:00Z"},
+    ]
+    return {"users": users, "posts": posts, "comments": comments}
+
+
+def leaked(outcome, names=("alice", "bobby")):
+    blob = " ".join(
+        str(item.get(name) or "")
+        for kind in ("users", "posts", "comments")
+        for item in outcome.records[kind]
+        for name in ("username", "title", "content", "bio", "website")
+    ).lower()
+    return [name for name in names if name in blob]
+
+
+def test_aliaser_rewrites_everything_and_fails_closed():
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records()))
+    assert leaked(outcome) == []
+    assert len(outcome.records["posts"]) == 1
+    assert outcome.skipped["no_author"] == 1
+    assert [c["origin_uid"] for c in outcome.records["comments"]] == ["c1"]
+    assert outcome.skipped["parent_skipped"] == 1
+    users = outcome.records["users"]
+    assert len(users) == 3
+    assert all(set(user) == {"origin_uid", "username", "created_at", "deleted"} for user in users)
+    assert {user["origin_uid"] for user in users} == {"alice", "bobby", "ghostly"}
+    assert outcome.records["posts"][0]["author_uid"] == "alice"
+    assert outcome.substitutions >= 4
+
+
+def test_aliaser_is_stable_across_runs():
+    first = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records()))
+    second = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records()))
+    assert first.records == second.records
+    assert second.identities_created == 0
+
+
+def test_aliaser_refuses_origin_with_original_links():
+    get_table("sync_links").insert({"uid": "l1", "origin": ORIGIN, "kind": "users", "origin_uid": "1", "local_uid": "x", "fingerprint": "f", "origin_updated_at": "", "first_synced_at": "", "synced_at": ""})
+    with pytest.raises(AliasRefused):
+        run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records()))
+
+
+def test_aliaser_cap_skips_item_as_transient(monkeypatch):
+    resolver = IdentityResolver(ORIGIN, cap=1)
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True), resolver=resolver).apply(records()))
+    assert outcome.transient() >= 1
+    assert leaked(outcome) == []
+
+
+class FakeTranslator(Translator):
+    def __init__(self, reply):
+        super().__init__("nl")
+        self.reply = reply
+
+    async def translate(self, text, samples=None):
+        return self.reply(text) if self.reply else None
+
+
+def test_translation_failure_skips_item_and_holds():
+    translator = FakeTranslator(None)
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True, translate_to="nl"), language="en", translator=translator).apply(records()))
+    assert outcome.records["posts"] == [] and outcome.skipped["translation_failed"] == 1
+    assert outcome.transient() >= 1
+
+
+def test_translation_output_is_rewritten_again():
+    translator = FakeTranslator(lambda text: text + " (met @alice)")
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True, translate_to="nl"), language="en", translator=translator).apply(records()))
+    assert leaked(outcome) == []
+    assert outcome.translated >= 1
+
+
+def test_translation_skipped_when_language_matches():
+    assert not Translator("en").needed("en-US")
+    assert Translator("en").needed("") and Translator("nl").needed("en")
+
+
+def test_options_validation_and_roundtrip():
+    options = SyncOptions.build(alias=True, translate_to="NL", sample_percent=40, min_comments=2, topics="devlog, rant", jitter_percent=10)
+    assert SyncOptions.from_row({"sync_options": options.to_dict()}) == options
+    assert not SyncOptions().active()
+    for bad in ({"sample_percent": 0}, {"jitter_percent": 99}, {"topics": "nonsense"}, {"translate_to": "x y"}, {"min_comments": -1}):
+        with pytest.raises(ValueError):
+            SyncOptions.build(**bad)
+    assert options.merged(alias=False).alias is False and options.merged(alias=None).alias is True
+
+
+def test_selector_deterministic_sampling_and_filters():
+    posts = [{"origin_uid": f"p{n}", "author_uid": "1", "topic": "devlog"} for n in range(400)]
+    comments = [{"origin_uid": "c", "author_uid": "1", "post_uid": "p0"}]
+    data = {"users": [], "posts": posts, "comments": comments}
+    options = SyncOptions(sample_percent=50)
+    kept, skipped = Selector(ORIGIN, options).select(data)
+    again, _ = Selector(ORIGIN, options).select(data)
+    assert kept["posts"] == again["posts"]
+    assert 150 < len(kept["posts"]) < 250
+    assert skipped["sampled_out"] == 400 - len(kept["posts"])
+    assert bucket(ORIGIN, "p0") == bucket(ORIGIN, "p0")
+    kept, skipped = Selector(ORIGIN, SyncOptions(topics=("rant",))).select(data)
+    assert kept["posts"] == [] and kept["comments"] == [] and skipped["topic_not_allowed"] == 400
+    kept, skipped = Selector(ORIGIN, SyncOptions(min_comments=1)).select(data)
+    assert [post["origin_uid"] for post in kept["posts"]] == ["p0"]
+    untouched, none = Selector(ORIGIN, SyncOptions()).select(data)
+    assert untouched is data and not none
+
+
+def test_selector_never_drops_already_synced_posts():
+    get_table("sync_links").insert({"uid": "l2", "origin": ORIGIN, "kind": "posts", "origin_uid": "p0", "local_uid": "x", "fingerprint": "f", "origin_updated_at": "", "first_synced_at": "", "synced_at": ""})
+    data = {"users": [], "posts": [{"origin_uid": "p0", "author_uid": "1", "topic": "devlog"}], "comments": []}
+    kept, _ = Selector(ORIGIN, SyncOptions(topics=("rant",))).select(data)
+    assert len(kept["posts"]) == 1
+
+
+def test_jitter_is_bounded():
+    rng = random.Random(1)
+    values = [jittered(1000, 20, rng) for _ in range(500)]
+    assert min(values) >= 800 and max(values) <= 1200 and len(set(values)) > 50
+    assert jittered(1000, 0) == 1000
+    assert Counter() == Counter()
+
+
+def test_mentions_are_case_insensitive_end_to_end():
+    rewriter = MentionRewriter({"Alice": "zed"})
+    assert rewriter.rewrite("@ALICE @alice @AlIcE and ALICE").text == "@zed @zed @zed and zed"
+    data = records()
+    data["posts"][0]["content"] = "ping @BOBBY and @Bobby and @bObBy"
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(data))
+    assert leaked(outcome) == []
+    assert len(outcome.records["users"]) == 2 + 0 or {u["origin_uid"] for u in outcome.records["users"]} >= {"alice", "bobby"}

tests/unit/services/sync/normalize.py

@@ -0,0 +1,97 @@
+# retoor <retoor@molodetz.nl>
+
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+from devplacepy.services.sync.normalize import (
+    clamp_instant,
+    clean_text,
+    fingerprint,
+    instant_iso,
+    normalize_origin,
+    normalize_origin_uid,
+    normalize_username,
+    parse_instant,
+    username_candidates,
+)
+from devplacepy.services.sync.spec import LIMITS
+
+
+@pytest.mark.parametrize(
+    "raw,expected",
+    [
+        ("Other.Example.ORG", "other.example.org"),
+        ("  a1  ", "a1"),
+        ("devplace-net", "devplace-net"),
+    ],
+)
+def test_origin_is_normalized(raw, expected):
+    assert normalize_origin(raw) == expected
+
+
+@pytest.mark.parametrize("raw", ["", "a", "-bad", "bad-", "a..b", "has space", "x" * 64, "ünï.org"])
+def test_origin_is_rejected(raw):
+    with pytest.raises(ValueError):
+        normalize_origin(raw)
+
+
+@pytest.mark.parametrize("raw", ["", "  ", "a b", "x" * 65, "a/b"])
+def test_origin_uid_is_rejected(raw):
+    with pytest.raises(ValueError):
+        normalize_origin_uid(raw)
+
+
+def test_origin_uid_accepts_integers_and_uuids():
+    assert normalize_origin_uid(42) == "42"
+    assert normalize_origin_uid("01a139e9-a2f0-7fa1-8635-1a98fbf95e3d")
+
+
+def test_clean_text_strips_control_characters_and_newlines():
+    assert clean_text("a\x00b\r\nc\rd\x07", multiline=True) == "ab\nc\nd"
+    assert clean_text("a \n  b\t c", multiline=False) == "a b c"
+
+
+def test_username_always_valid_across_inputs():
+    samples = ["", "a", "ab", "Ünï cøde!", "x" * 80, "--__--", "john.doe@mail", "日本語", "ok_name-1"]
+    for sample in samples:
+        name = normalize_username(sample, "uid-1")
+        assert LIMITS.min_username_length <= len(name) <= LIMITS.max_username_length
+        assert all(ch.isalnum() or ch in "_-" for ch in name) and name.isascii()
+
+
+def test_username_is_stable_for_same_input():
+    assert normalize_username("a", "u1") == normalize_username("a", "u1")
+    assert normalize_username("a", "u1") != normalize_username("a", "u2")
+
+
+def test_username_candidates_are_distinct_and_bounded():
+    seen = []
+    for candidate in username_candidates("x" * 32, "other.example.org"):
+        seen.append(candidate)
+        assert len(candidate) <= LIMITS.max_username_length
+        if len(seen) == 50:
+            break
+    assert len(set(seen)) == 50
+    assert seen[1].endswith("-other")
+
+
+def test_instants_are_utc_iso_and_clamped():
+    assert instant_iso("2025-01-01T01:00:00+01:00") == "2025-01-01T00:00:00+00:00"
+    assert instant_iso("2025-01-01").startswith("2025-01-01T00:00:00")
+    assert parse_instant(1700000000).year == 2023
+    now = datetime.now(timezone.utc)
+    future = now + timedelta(days=30)
+    assert clamp_instant(future, now) == now
+    assert instant_iso("1970-01-01").startswith("2000-01-01")
+
+
+@pytest.mark.parametrize("raw", ["", None, "nope", True, "2025-13-45"])
+def test_instant_rejects_garbage(raw):
+    with pytest.raises(ValueError):
+        parse_instant(raw)
+
+
+def test_fingerprint_ignores_key_order_and_detects_change():
+    assert fingerprint({"a": 1, "b": 2}) == fingerprint({"b": 2, "a": 1})
+    assert fingerprint({"a": 1}) != fingerprint({"a": 2})

tests/unit/services/sync/records.py

@@ -0,0 +1,85 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+from pydantic import ValidationError
+
+from devplacepy.services.sync.records import (
+    CommentRecord,
+    PostRecord,
+    SyncBatchForm,
+    UserRecord,
+)
+from devplacepy.services.sync.spec import LIMITS
+
+BASE = {"origin_uid": "r1", "created_at": "2025-01-01T00:00:00Z"}
+
+
+def test_updated_at_defaults_to_created_and_never_precedes_it():
+    record = PostRecord.model_validate(
+        {**BASE, "author_uid": "u", "content": "ten chars!!"}
+    )
+    assert record.updated_at == record.created_at
+    earlier = PostRecord.model_validate(
+        {**BASE, "author_uid": "u", "content": "ten chars!!", "updated_at": "2024-01-01"}
+    )
+    assert earlier.updated_at == earlier.created_at
+
+
+def test_unknown_topic_becomes_random_and_known_is_kept():
+    base = {**BASE, "author_uid": "u", "content": "ten chars!!"}
+    assert PostRecord.model_validate({**base, "topic": "zzz"}).topic == "random"
+    assert PostRecord.model_validate({**base, "topic": "DevLog"}).topic == "devlog"
+
+
+@pytest.mark.parametrize(
+    "payload",
+    [
+        {"author_uid": "u", "content": "short"},
+        {"author_uid": "u", "content": "x" * (LIMITS.max_post_length + 1)},
+        {"author_uid": "", "content": "ten chars!!"},
+        {"content": "ten chars!!"},
+        {"author_uid": "u", "content": "ten chars!!", "title": "t" * (LIMITS.max_title_length + 1)},
+    ],
+)
+def test_invalid_posts_are_rejected(payload):
+    with pytest.raises(ValidationError):
+        PostRecord.model_validate({**BASE, **payload})
+
+
+def test_comment_parent_is_optional_and_post_is_required():
+    base = {**BASE, "author_uid": "u", "content": "fine"}
+    assert CommentRecord.model_validate({**base, "post_uid": "p"}).parent_uid is None
+    assert CommentRecord.model_validate({**base, "post_uid": "p", "parent_uid": ""}).parent_uid is None
+    with pytest.raises(ValidationError):
+        CommentRecord.model_validate(base)
+
+
+def test_user_urls_are_fixed_or_dropped():
+    record = UserRecord.model_validate(
+        {**BASE, "username": "bob", "website": "example.com", "git_link": "not a url"}
+    )
+    assert record.website == "https://example.com"
+    assert record.git_link == ""
+
+
+def test_user_bio_over_limit_is_rejected():
+    with pytest.raises(ValidationError):
+        UserRecord.model_validate({**BASE, "username": "bob", "bio": "x" * (LIMITS.max_bio_length + 1)})
+
+
+def test_user_counters_and_credentials_are_not_fields():
+    known = UserRecord.known_fields()
+    for name in ("email", "password_hash", "role", "xp", "level", "stars", "api_key"):
+        assert name not in known
+    assert "stars" not in PostRecord.known_fields()
+
+
+def test_envelope_normalizes_origin_and_bounds_batch():
+    form = SyncBatchForm(origin="Other.ORG", records=[{}])
+    assert form.origin == "other.org"
+    with pytest.raises(ValidationError):
+        SyncBatchForm(origin="other.org", records=[])
+    with pytest.raises(ValidationError):
+        SyncBatchForm(origin="other.org", records=[{}] * (LIMITS.max_batch_records + 1))
+    with pytest.raises(ValidationError):
+        SyncBatchForm(origin="bad origin", records=[{}])