Zo doen wijdat in oost

tests/unit/cli/nodes.py

@@ -0,0 +1,44 @@
+# retoor <retoor@molodetz.nl>
+
+import argparse
+
+from devplacepy.cli import nodes as cli
+from devplacepy.database import get_table
+from devplacepy.database import nodes as store
+from devplacepy.utils import generate_uid
+
+
+def parser():
+    root = argparse.ArgumentParser()
+    cli.register_nodes(root.add_subparsers(dest="command"))
+    return root
+
+
+def device(local_db):
+    row, _ = store.create_device(generate_uid(), "cli", "native")
+    return row["uid"]
+
+
+def test_suspend_and_unsuspend(local_db, capsys):
+    uid = device(local_db)
+    args = parser().parse_args(["nodes", "suspend", uid, "--minutes", "5"])
+    args.func(args)
+    assert store.effective_status(get_table("node_devices").find_one(uid=uid)) == "suspended"
+    args = parser().parse_args(["nodes", "unsuspend", uid])
+    args.func(args)
+    assert store.effective_status(get_table("node_devices").find_one(uid=uid)) == "active"
+
+
+def test_clear_is_a_dry_run_without_force(local_db, capsys):
+    get_table("node_events").insert({"uid": generate_uid(), "kind": "x", "detail": "", "created_at": "2026-01-01T00:00:00+00:00"})
+    before = store.clear_ephemeral(dry_run=True)["node_events"]
+    args = parser().parse_args(["nodes", "clear"])
+    args.func(args)
+    assert "pass --force" in capsys.readouterr().out
+    assert store.clear_ephemeral(dry_run=True)["node_events"] == before
+
+
+def test_prune_dry_run_reports_counts(local_db, capsys):
+    args = parser().parse_args(["nodes", "prune", "--dry-run"])
+    args.func(args)
+    assert "would remove" in capsys.readouterr().out

tests/unit/database/nodes.py

@@ -225,3 +225,170 @@ def test_ip_is_only_ever_stored_hashed():
     assert "203" not in digest and len(digest) == 32
     assert store.hash_ip("203.0.113.9") == digest
     assert store.hash_ip("") == ""
+
+
+def queue_one(url="https://reddit.com/r/x", caller="c1"):
+    return store.enqueue_request("user", caller, url, "GET", "reddit", "any")
+
+
+def test_the_queue_is_bounded(local_db):
+    get_table("node_requests").delete()
+    for _ in range(LIMITS.max_queue_depth):
+        queue_one()
+    with pytest.raises(store.QueueFull):
+        queue_one()
+    assert store.count_open_requests() == LIMITS.max_queue_depth
+    first = store.list_queued(1)[0]
+    store.fail_request(first["uid"], "x")
+    queue_one()
+    get_table("node_requests").delete()
+
+
+def test_a_request_is_claimed_by_exactly_one_assignment(local_db):
+    get_table("node_requests").delete()
+    uid = queue_one()
+    assert store.claim_request(uid, "device-a") is True
+    assert store.claim_request(uid, "device-b") is False
+    assert store.get_request(uid)["device_uid"] == "device-a"
+    get_table("node_requests").delete()
+
+
+def test_a_result_can_only_land_on_an_assigned_request(local_db):
+    get_table("node_requests").delete()
+    uid = queue_one()
+    done = dict(result_status=200, result_headers="{}", result_body="eA==", result_bytes=1, truncated=False)
+    assert store.finish_request(uid, **done) is False
+    store.claim_request(uid, "d")
+    assert store.finish_request(uid, **done) is True
+    assert store.finish_request(uid, **done) is False
+    assert store.fail_request(uid, "late") is False
+    assert store.get_request(uid)["status"] == "done"
+    get_table("node_requests").delete()
+
+
+def test_expiry_separates_waiting_from_stuck_requests(local_db):
+    get_table("node_requests").delete()
+    waiting, stuck, fresh = queue_one(), queue_one(), queue_one()
+    store.claim_request(stuck, "d")
+    later = datetime.now(timezone.utc) + timedelta(
+        seconds=LIMITS.job_timeout_seconds + LIMITS.job_grace_seconds + LIMITS.queue_max_wait_seconds + 1
+    )
+    counts = store.expire_requests(later)
+    assert counts == {"queued": 2, "assigned": 1}
+    assert store.get_request(waiting)["status"] == "expired"
+    assert store.get_request(stuck)["error"] == "timeout"
+    assert store.get_request(fresh)["status"] == "expired"
+    assert store.expire_requests(later) == {"queued": 0, "assigned": 0}
+    get_table("node_requests").delete()
+
+
+def test_requests_are_listed_oldest_first_and_limited(local_db):
+    get_table("node_requests").delete()
+    uids = [queue_one() for _ in range(5)]
+    assert [row["uid"] for row in store.list_queued(3)] == uids[:3]
+    get_table("node_requests").delete()
+
+
+def test_request_garbage_collection_leaves_recent_rows(local_db):
+    get_table("node_requests").delete()
+    queue_one()
+    assert store.prune_requests() == 0
+    far = datetime.now(timezone.utc) + timedelta(seconds=LIMITS.request_retention_seconds + 5)
+    assert store.prune_requests(dry_run=True, moment=far) == 1
+    assert store.prune_requests(moment=far) == 1
+
+
+def test_caller_quota_counts_exactly_to_the_limit(local_db):
+    caller = generate_uid()
+    results = [store.reserve_caller_request("user", caller, 5) for _ in range(9)]
+    assert results.count(True) == 5
+    assert store.caller_requests_today("user", caller) == 5
+
+
+def test_a_zero_quota_or_anonymous_caller_never_reserves(local_db):
+    assert not store.reserve_caller_request("guest", "someone", 0)
+    assert not store.reserve_caller_request("user", "", 10)
+    assert store.caller_requests_today("guest", "someone") == 0
+
+
+def test_caller_quota_is_per_caller_and_per_day(local_db):
+    a, b = generate_uid(), generate_uid()
+    for _ in range(3):
+        assert store.reserve_caller_request("user", a, 3)
+    assert not store.reserve_caller_request("user", a, 3)
+    assert store.reserve_caller_request("user", b, 3)
+    tomorrow = datetime.now(timezone.utc) + timedelta(days=1)
+    assert store.reserve_caller_request("user", a, 3, moment=tomorrow)
+    assert store.reserve_caller_request("guest", a, 3)
+
+
+def test_caller_counters_are_pruned_with_the_usage_buckets(local_db):
+    caller = generate_uid()
+    store.reserve_caller_request("user", caller, 5)
+    far = datetime.now(timezone.utc) + timedelta(days=LIMITS.usage_daily_retention_days + 2)
+    assert store.prune_usage_daily(dry_run=True, moment=far) >= 1
+    assert store.caller_requests_today("user", caller) == 1
+    store.prune_usage_daily(moment=far)
+    assert store.caller_requests_today("user", caller) == 0
+
+
+def test_batch_helpers_match_the_single_lookups(local_db):
+    owner = generate_uid()
+    row, _ = store.create_device(owner, "a", "native")
+    store.save_node_settings(owner, {"enabled": True, "bandwidth_mb_day": 7})
+    assert store.get_devices_by_uids([row["uid"], "missing", ""])[row["uid"]]["user_uid"] == owner
+    assert store.get_enabled_by_users([owner, "nobody"]) == {owner: True}
+    assert store.get_settings_by_users([owner])[owner]["bandwidth_mb_day"] == 7
+    assert store.get_devices_by_uids([]) == {}
+
+
+def test_batch_helpers_ignore_revoked_devices(local_db):
+    owner = generate_uid()
+    row, _ = store.create_device(owner, "a", "native")
+    store.revoke_device(row["uid"], owner)
+    assert store.get_devices_by_uids([row["uid"]]) == {}
+
+
+def test_device_breakdown_buckets_blank_values_as_unknown(local_db):
+    owner = generate_uid()
+    store.create_device(owner, "a", "native")
+    store.create_device(owner, "b", "extension")
+    kinds = store.device_breakdown("kind")
+    assert kinds["native"] >= 1 and kinds["extension"] >= 1
+    assert "unknown" in store.device_breakdown("country")
+    with pytest.raises(ValueError):
+        store.device_breakdown("secret_hash")
+
+
+def test_device_breakdown_ignores_revoked_devices(local_db):
+    owner = generate_uid()
+    row, _ = store.create_device(owner, "a", "native")
+    before = store.device_breakdown("status").get("active", 0)
+    store.revoke_device(row["uid"], owner)
+    assert store.device_breakdown("status").get("active", 0) == before - 1
+
+
+def test_request_status_counts_always_lists_every_status(local_db):
+    counts = store.request_status_counts()
+    assert set(store.REQUEST_STATUSES) <= set(counts)
+    assert all(value >= 0 for value in counts.values())
+
+
+def test_usage_day_totals_sums_the_day(local_db):
+    owner = generate_uid()
+    row, _ = store.create_device(owner, "a", "native")
+    before = store.usage_day_totals()
+    store.record_job(row["uid"], owner, ok=True, bytes_up=10, bytes_down=90)
+    after = store.usage_day_totals()
+    assert after["requests_ok"] == before["requests_ok"] + 1
+    assert after["bytes_down"] == before["bytes_down"] + 90
+
+
+def test_recent_devices_is_capped_and_excludes_deleted(local_db):
+    owner = generate_uid()
+    row, _ = store.create_device(owner, "a", "native")
+    assert row["uid"] in {item["uid"] for item in store.recent_devices(store.ADMIN_LIST_LIMIT)}
+    assert len(store.recent_devices(1)) == 1
+    assert len(store.recent_devices(10_000)) <= store.ADMIN_LIST_LIMIT
+    store.revoke_device(row["uid"], owner)
+    assert row["uid"] not in {item["uid"] for item in store.recent_devices(store.ADMIN_LIST_LIMIT)}

tests/unit/services/community/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/unit/services/community/ai.py

@@ -0,0 +1,67 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+from devplacepy.services.community.ai import AiAssistant, outline, parse_json, to_html
+from devplacepy.services.community.extract import parse_html, select_nodes
+from tests.unit.services.community.fakes import HTML_ITEM
+
+
+class Canned(AiAssistant):
+    def __init__(self, answer):
+        self.answer = answer
+        self.asked = []
+
+    def enabled(self):
+        return True
+
+    async def ask(self, system, user):
+        self.asked.append((system, user))
+        return self.answer
+
+
+def nodes():
+    root = parse_html("<main>" + "".join(HTML_ITEM.format(n=n) for n in range(1, 5)) + "</main>")
+    return root, select_nodes(root, "article.post")
+
+
+def test_json_is_extracted_from_fenced_or_chatty_replies():
+    assert parse_json('```json\n{"a": 1}\n```') == {"a": 1}
+    assert parse_json('Sure! Here you go: {"a": {"b": 2}} hope it helps') == {"a": {"b": 2}}
+    assert parse_json("no json") is None
+
+
+def test_only_proposals_that_match_the_sample_are_accepted():
+    _, items = nodes()
+    assistant = Canned({"title": "h2", "link": "h2 a::attr(href)", "author": ".nonexistent", "created_at": "time::attr(datetime)"})
+    accepted = run_async(assistant.post_fields(items))
+    assert accepted == {"title": "h2", "link": "h2 a::attr(href)", "created_at": "time::attr(datetime)"}
+
+
+def test_item_selector_must_match_at_least_three_elements():
+    root, _ = nodes()
+    assert run_async(Canned({"item_selector": "article.post"}).item_selector(root)) == "article.post"
+    assert run_async(Canned({"item_selector": "article.missing"}).item_selector(root)) is None
+    assert run_async(Canned({"item_selector": "!!"}).item_selector(root)) is None
+
+
+def test_comment_structure_needs_content():
+    root = parse_html('<div><div class="c" id="a1"><p>hello there</p><b>ann</b></div><div class="c" id="a2"><p>second text</p><b>bob</b></div></div>')
+    good = run_async(Canned({"item_selector": "div.c", "id": "::attr(id)", "content": "p", "author": "b"}).comment_structure(root))
+    assert good == {"item_selector": "div.c", "id": "::attr(id)", "content": "p", "author": "b"}
+    assert run_async(Canned({"item_selector": "div.c", "content": "span"}).comment_structure(root)) is None
+
+
+def test_disabled_assistant_never_asks():
+    class Off(AiAssistant):
+        def enabled(self):
+            return False
+
+    assert run_async(Off().ask("s", "u")) is None
+
+
+def test_outline_and_html_are_bounded():
+    root, items = nodes()
+    assert "article.post" in outline(root)
+    assert len(outline(root, 3).splitlines()) <= 3
+    assert len(to_html(items[0], 100)) <= 100

tests/unit/services/community/alias.py

@@ -0,0 +1,133 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.database import get_setting, get_table, init_db, set_setting
+from devplacepy.services.community.runner import TRIGGER_CLI, CommunityRunner
+from devplacepy.services.community.store import store
+from devplacepy.services.sync import ai
+from devplacepy.services.sync.options import ALIAS_KEY, SyncOptions
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep
+
+ORIGINALS = ("writer0", "writer1", "replier0", "replier1")
+
+
+@pytest.fixture
+def world(monkeypatch):
+    init_db()
+    keys = ("sync_enabled", "sync_origins", ALIAS_KEY)
+    previous = {key: get_setting(key, "") for key in keys}
+    set_setting("sync_enabled", "1")
+    set_setting("sync_origins", "")
+    set_setting(ALIAS_KEY, "1")
+    for table in ("communities", "community_runs"):
+        get_table(table).delete()
+    get_table("sync_links").delete(origin="forum.test")
+    get_table("sync_identities").delete(origin="forum.test")
+
+    async def unreachable(system, text, timeout):
+        return None
+
+    monkeypatch.setattr(ai, "complete", unreachable)
+    runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(json_community()), sleep=no_sleep)
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test"))
+    yield runner
+    for key, value in previous.items():
+        set_setting(key, value)
+    get_table("sync_identities").delete(origin="forum.test")
+
+
+def go(runner, dry_run=False):
+    return run_async(runner.run(store.get("forum-test"), dry_run=dry_run, trigger=TRIGGER_CLI))
+
+
+def synced(kind: str) -> list[dict]:
+    links = get_table("sync_links").find(origin="forum.test", kind=kind)
+    return [row for row in (get_table(kind).find_one(uid=link["local_uid"]) for link in links) if row]
+
+
+def local_text() -> str:
+    names = [row["username"] for row in synced("users")]
+    posts = [f"{row['title']} {row['content']}" for row in synced("posts")]
+    comments = [row["content"] for row in synced("comments")]
+    return " ".join(names + posts + comments).lower()
+
+
+def enable(**changes):
+    row = store.get("forum-test")
+    store.update(row["uid"], sync_options=SyncOptions.from_row(row).merged(**changes).to_dict())
+
+
+def test_aliasing_off_by_default_keeps_original_names(world):
+    report = go(world)
+    assert report.status == "ok", report.errors
+    assert "writer0" in local_text()
+    assert get_table("sync_identities").count(origin="forum.test") == 0
+
+
+def test_alias_run_never_stores_an_original_username(world):
+    enable(alias=True)
+    report = go(world)
+    assert report.status == "ok", report.errors
+    text = local_text()
+    assert not [name for name in ORIGINALS if name in text]
+    assert get_table("sync_identities").count(origin="forum.test") == 4
+    again = go(world)
+    assert again.status == "ok"
+    assert again.pushed["users"].get("created", 0) == 0
+    assert get_table("sync_identities").count(origin="forum.test") == 4
+
+
+def test_kill_switch_stops_aliased_runs_and_creates_nothing(world):
+    enable(alias=True)
+    set_setting(ALIAS_KEY, "0")
+    report = go(world)
+    assert any("switched off" in error for error in report.errors)
+    assert not synced("posts") and not synced("users")
+    assert get_table("sync_identities").count(origin="forum.test") == 0
+
+
+def test_dry_run_creates_no_identities_and_no_rows(world):
+    enable(alias=True)
+    users = get_table("users").count()
+    report = go(world, dry_run=True)
+    assert report.status == "dry_run", report.errors
+    assert get_table("sync_identities").count(origin="forum.test") == 0
+    assert get_table("users").count() == users
+
+
+def test_enabling_alias_after_unaliased_sync_is_refused_at_run(world):
+    assert go(world).status == "ok"
+    enable(alias=True)
+    report = go(world)
+    assert any("already has synced records" in error for error in report.errors)
+    assert get_table("sync_identities").count(origin="forum.test") == 0
+
+
+def test_selection_skips_are_reported_and_sampling_is_stable(world):
+    enable(sample_percent=30)
+    first = go(world)
+    count = len(synced("posts"))
+    assert first.pushed["skipped"]["sampled_out"] + count >= 10
+    assert 0 < count < 10
+    go(world)
+    assert len(synced("posts")) == count
+
+
+def test_translation_failure_holds_watermark_and_stores_nothing(world):
+    enable(alias=True, translate_to="nl")
+    row = store.get("forum-test")
+    store.update(row["uid"], language="en")
+    report = go(world)
+    assert not synced("posts")
+    assert report.status == "partial"
+    assert report.pushed["skipped"]["translation_failed"] >= 1
+    assert not store.get("forum-test")["watermark"]
+
+
+def test_jitter_moves_next_run_within_bounds(world):
+    row = store.get("forum-test")
+    moments = {store.schedule_next(row["uid"], 86400, 40) for _ in range(5)}
+    assert len(moments) > 1

tests/unit/services/community/conftest.py

@@ -0,0 +1,13 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.community.ai import AiAssistant
+
+
+@pytest.fixture(autouse=True)
+def silent_ai(monkeypatch):
+    async def never(self, system, user):
+        return None
+
+    monkeypatch.setattr(AiAssistant, "ask", never)

tests/unit/services/community/discoverer.py

@@ -0,0 +1,133 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+from devplacepy.services.community.discoverer import CommunityDiscoverer
+from devplacepy.services.community.spec import STATUS_FAILED, STATUS_LOGIN, STATUS_READY
+from tests.unit.services.community.fakes import (
+    FakeFetcher,
+    club_community,
+    html_community,
+    json_community,
+    rss_community,
+    spa_community,
+)
+
+
+def discover(routes: dict, query: str, credentials: dict | None = None):
+    fetcher = FakeFetcher(routes)
+    return run_async(CommunityDiscoverer(fetcher, credentials=credentials).discover(query)), fetcher
+
+
+def test_json_api_community_is_learned():
+    result, _ = discover(json_community(), "https://forum.test/api/posts")
+    assert result.status == STATUS_READY, result.error
+    feed = result.schema["feeds"][0]
+    assert feed["type"] == "json_api"
+    assert feed["list_path"] == "$.items"
+    assert feed["post"]["fields"]["id"] == ["$.id"]
+    assert feed["post"]["fields"]["content"][0] == "$.content_html"
+    assert feed["post"]["fields"]["created_at"] == ["$.created_at"]
+    assert feed["author"]["fields"]["username"] == ["$.author.username"]
+    assert feed["author"]["fields"]["id"] == ["$.author.id"]
+    assert feed["pagination"] == {"style": "page", "param": "page", "start": 1}
+    assert feed["comments"]["mode"] == "inline"
+    assert feed["comments"]["fields"]["content"] == ["$.body"]
+    assert feed["comments"]["author"]["fields"]["username"] == ["$.user.username"]
+    records = result.sample["records"]
+    assert len(records["posts"]) == 8
+    assert len(records["comments"]) == 6
+    assert {user["username"] for user in records["users"]} >= {"writer0", "writer1", "replier0"}
+
+
+def test_rss_community_is_learned_from_advertised_feed():
+    result, _ = discover(rss_community(), "https://blog.test/")
+    assert result.status == STATUS_READY, result.error
+    feed = result.schema["feeds"][0]
+    assert feed["type"] == "rss"
+    assert feed["post"]["fields"]["created_at"] == ["$.pubDate"]
+    assert result.sample["records"]["users"][0]["username"] == "Alice"
+    assert result.sample["records"]["posts"][0]["content"].startswith("Full body of entry")
+    assert feed["comments"]["mode"] == "page"
+    assert feed["comments"]["item_selector"] == "div.comment"
+    assert feed["post"]["detail"]["content"] == ["article"]
+    comments = result.sample["records"]["comments"]
+    assert len(comments) == 9
+    assert sum(1 for comment in comments if comment.get("parent_uid")) == 3
+
+
+def test_html_list_community_is_learned():
+    result, _ = discover(html_community(), "https://board.test/")
+    assert result.status == STATUS_READY, result.error
+    feed = result.schema["feeds"][0]
+    assert feed["type"] == "html_list"
+    assert feed["item_selector"] == "article.post"
+    assert len(result.sample["records"]["posts"]) == 6
+    assert result.sample["records"]["posts"][0]["title"].startswith("Thread 1")
+    assert feed["comments"]["mode"] == "page"
+    assert feed["comments"]["fields"]["id"] == ["::attr(id)"]
+    assert feed["comments"]["author"]["fields"]["username"] == [".author"]
+    assert len(result.sample["records"]["comments"]) == 9
+
+
+def test_unreachable_community_fails_with_reason():
+    result, _ = discover({}, "https://nothing.test/")
+    assert result.status == STATUS_FAILED
+    assert result.error
+
+
+def test_javascript_only_site_is_rendered():
+    result, _ = discover(spa_community(), "https://spa.test/")
+    assert result.status == STATUS_READY, result.error
+    feed = result.schema["feeds"][0]
+    assert feed["render"] is True
+    assert len(result.sample["records"]["posts"]) == 6
+
+
+def test_login_wall_without_credentials_is_flagged_not_failed():
+    state = {}
+    result, _ = discover(club_community(state), "https://club.test/")
+    assert result.status == STATUS_LOGIN
+    assert result.schema["login"]["mode"] == "form"
+    assert result.schema["login"]["username_field"] == "user"
+    assert result.schema["login_required"] is True
+
+
+def test_login_wall_with_credentials_is_learned_behind_the_wall():
+    state = {}
+    result, fetcher = discover(club_community(state), "https://club.test/", {"username": "bob", "password": "pw"})
+    assert result.status == STATUS_READY, result.error
+    assert result.schema["login_required"] is True
+    assert result.schema["feeds"][0]["type"] == "rss"
+    assert fetcher.posts[0] == ("https://club.test/login", {"csrf": "t1", "user": "bob", "pass": "pw"})
+
+
+def test_wrong_credentials_fail_loudly():
+    result, _ = discover(club_community({}), "https://club.test/", {"username": "bob", "password": "nope"})
+    assert result.status == STATUS_FAILED
+    assert "rejected the stored credentials" in result.error
+
+
+def test_feeds_listing_the_same_posts_are_learned_once_and_comment_feeds_are_skipped():
+    routes = rss_community()
+    routes["https://blog.test/"] = (
+        '<html><head><link rel="alternate" type="application/rss+xml" href="/feed.xml">'
+        '<link rel="alternate" type="application/rss+xml" href="/comments.xml"></head>'
+        '<body><a href="/latest">Latest posts</a></body></html>'
+    )
+    routes["https://blog.test/comments.xml"] = routes["https://blog.test/feed.xml"]
+    items = "".join(
+        f'<article class="post"><h2><a href="/p/{n}">Entry {n} on testing</a></h2><span class="author">Alice</span>'
+        f"<p>Body of entry {n} with enough text to count as content.</p></article>"
+        for n in range(1, 7)
+    )
+    routes["https://blog.test/latest"] = f"<html><body><main>{items}</main></body></html>"
+    result, fetcher = discover(routes, "https://blog.test/")
+    assert result.status == STATUS_READY, result.error
+    assert [feed["type"] for feed in result.schema["feeds"]] == ["rss"]
+    assert not any("comments.xml" in request for request in fetcher.requests)
+
+
+def test_generic_probe_paths_are_skipped_when_a_feed_is_advertised():
+    _, fetcher = discover(rss_community(), "https://blog.test/")
+    assert not any(request.endswith("/api/posts") or request.endswith("/rss.xml") for request in fetcher.requests)

tests/unit/services/community/extract.py

@@ -0,0 +1,60 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.community import extract
+
+
+def test_select_walks_keys_indexes_and_wildcards():
+    data = {"a": [{"b": 1}, {"b": 2}], "c": {"d": "x"}}
+    assert extract.select(data, "$.a[*].b") == [1, 2]
+    assert extract.select(data, "$.a[1].b") == [2]
+    assert extract.select(data, "$.c.d") == ["x"]
+    assert extract.select(data, "$.missing") == []
+
+
+@pytest.mark.parametrize("path", ["a.b", "$.a[", "$..", "$" + ".x" * 40])
+def test_bad_paths_raise(path):
+    with pytest.raises(ValueError):
+        extract.parse_path(path)
+
+
+def test_instants_accept_iso_rfc822_and_epochs():
+    assert extract.normalize_instant("2026-01-01T00:00:00Z") == "2026-01-01T00:00:00+00:00"
+    assert extract.normalize_instant("Tue, 14 Oct 2026 10:00:00 GMT") == "2026-10-14T10:00:00+00:00"
+    assert extract.normalize_instant(1700000000) == extract.normalize_instant(1700000000000)
+    assert extract.normalize_instant("garbage") is None
+    assert extract.normalize_instant("") is None
+
+
+def test_atom_feed_is_parsed_with_links_and_authors():
+    atom = (
+        '<feed xmlns="http://www.w3.org/2005/Atom"><title>T</title><entry><title>E</title>'
+        '<link href="http://x/1" rel="alternate"/><author><name>Ann</name></author>'
+        "<updated>2026-01-02T00:00:00Z</updated></entry></feed>"
+    )
+    kind, _, items = extract.parse_feed(atom)
+    assert kind == "atom"
+    assert extract.text_at(items[0], "$.link.@href") == "http://x/1"
+    assert extract.text_at(items[0], "$.author.name") == "Ann"
+
+
+def test_css_selectors_support_classes_ids_attributes_and_combinators():
+    root = extract.parse_html(
+        '<div id="m"><ul><li class="a b"><a href="/x" rel="next">One</a></li><li class="a">Two</li></ul></div>'
+    )
+    assert len(extract.select_nodes(root, "li.a")) == 2
+    assert len(extract.select_nodes(root, "li.a.b")) == 1
+    assert len(extract.select_nodes(root, "#m > ul > li")) == 2
+    assert len(extract.select_nodes(root, "div li")) == 2
+    assert extract.html_field(extract.select_nodes(root, "li.b")[0], "a[rel=next]::attr(href)") == "/x"
+
+
+def test_strip_tags_keeps_paragraph_breaks_and_drops_scripts():
+    assert extract.strip_tags("<p>a</p><script>x()</script><p>b<br>c</p>") == "a\n\nb\n\nc"
+    assert extract.strip_tags("plain text") == "plain text"
+
+
+def test_field_text_falls_through_alternates_and_now_token():
+    assert extract.field_text({"a": "", "b": "yes"}, ["$.a", "$.b"]) == "yes"
+    assert extract.field_text({}, [extract.NOW_TOKEN]).startswith("20")

tests/unit/services/community/fakes.py

@@ -0,0 +1,228 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+
+class FakeResponse:
+    def __init__(self, status: int, body):
+        self.status = status
+        self.body = body if isinstance(body, str) else json.dumps(body)
+        self.headers = {}
+
+    async def text(self):
+        return self.body
+
+    async def json(self):
+        return json.loads(self.body)
+
+
+class FakeFetcher:
+    def __init__(self, routes: dict):
+        self.routes = routes
+        self.requests: list[str] = []
+        self.session = None
+        self.posts: list[tuple] = []
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *exc):
+        return None
+
+    @staticmethod
+    def key(url: str, params: dict | None) -> str:
+        parts = urlsplit(url)
+        query = dict(parse_qsl(parts.query))
+        query.update({name: str(value) for name, value in (params or {}).items()})
+        return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(sorted(query.items())), ""))
+
+    async def get(self, url, params=None, timeout=None, **kwargs):
+        key = self.key(url, params)
+        self.requests.append(key)
+        route = self.routes.get(key)
+        if route is None:
+            return FakeResponse(404, "not found")
+        if callable(route):
+            route = route()
+        status, body = route if isinstance(route, tuple) else (200, route)
+        return FakeResponse(status, body)
+
+    async def post(self, url, data=None, timeout=None, **kwargs):
+        self.posts.append((url, dict(data or {})))
+        route = self.routes.get("post:" + url)
+        if callable(route):
+            route = route(dict(data or {}))
+        status, body = route if isinstance(route, tuple) else (200, route or "")
+        return FakeResponse(status, body)
+
+    async def render_html(self, url, session=None):
+        route = self.routes.get("render:" + url)
+        if route is None:
+            response = await self.get(url)
+            return response.status, response.body
+        return 200, route
+
+    def page(self, session=None):
+        return FakePage(self)
+
+    async def render(self, url, collect_links=False):
+        response = await self.get(url)
+        return {"status": response.status, "text": response.body, "links": [], "rendered": False}
+
+
+def json_post(number: int) -> dict:
+    return {
+        "id": 1000 + number,
+        "title": f"Post number {number} about asyncio",
+        "content_html": f"<p>This is the body of post {number}, long enough to sync.</p>",
+        "created_at": f"2026-10-{number:02d}T10:00:00Z",
+        "author": {"id": 7 + number % 2, "username": f"writer{number % 2}", "url": f"https://forum.test/u/{number % 2}"},
+        "url": f"https://forum.test/posts/{1000 + number}",
+        "comments": [
+            {
+                "id": 5000 + number * 10 + index,
+                "body": f"Comment {index} on post {number}",
+                "created_at": f"2026-10-{number:02d}T11:0{index}:00Z",
+                "user": {"id": 20 + index, "username": f"replier{index}"},
+            }
+            for index in range(2)
+        ],
+    }
+
+
+def json_community() -> dict:
+    posts = [json_post(number) for number in range(1, 9)]
+    return {
+        "https://forum.test/": '<html lang="en"><head><title>Forum Test</title>'
+        '<meta property="og:description" content="A friendly forum about code."></head>'
+        "<body><p>Join 1,234 members discussing code every single day on this community site.</p></body></html>",
+        "https://forum.test/api/posts": {"items": posts, "total": 8},
+        "https://forum.test/api/posts?page=2": {"items": [json_post(9), json_post(10)], "total": 8},
+        "https://forum.test/about": "<html><body><p>About: 2,500 members strong.</p></body></html>",
+    }
+
+
+RSS = """<?xml version="1.0"?>
+<rss version="2.0" xmlns:dc="http://purl.org/dc/elements/1.1/"><channel>
+<title>Blog Test</title><description>A blog</description>
+{items}
+</channel></rss>"""
+
+RSS_ITEM = """<item><title>Entry {n} on testing</title><link>https://blog.test/p/{n}</link>
+<guid>https://blog.test/p/{n}</guid><pubDate>Tue, {n:02d} Oct 2026 10:00:00 GMT</pubDate>
+<dc:creator>Alice</dc:creator><description>&lt;p&gt;Body of entry {n} with enough text.&lt;/p&gt;</description></item>"""
+
+
+def detail_page(kind: str, n: int, comments: int = 2) -> str:
+    rows = "".join(
+        f'<div class="comment" id="c{n}-{index}"><span class="author">replier{index}</span>'
+        f'<time datetime="2026-10-{n:02d}T12:0{index}:00Z">t</time>'
+        f'<div class="comment-body">Comment {index} on {kind} {n} with words</div>'
+        + (
+            f'<div class="comment" id="c{n}-{index}-r"><span class="author">deep{index}</span>'
+            f'<div class="comment-body">Nested reply to comment {index} on {n}</div></div>'
+            if index == 0
+            else ""
+        )
+        + "</div>"
+        for index in range(comments)
+    )
+    body = f"Full body of {kind} {n}. " + "It goes on at length about the topic. " * 6
+    return f"<html><body><article>{body}</article><section>{rows}</section></body></html>"
+
+
+def rss_community() -> dict:
+    items = "".join(RSS_ITEM.format(n=n) for n in range(1, 7))
+    pages = {f"https://blog.test/p/{n}": detail_page("entry", n) for n in range(1, 7)}
+    return {
+        **pages,
+        "https://blog.test/": '<html><head><link rel="alternate" type="application/rss+xml" href="/feed.xml">'
+        "<title>Blog Test</title></head><body>hi</body></html>",
+        "https://blog.test/feed.xml": RSS.format(items=items),
+    }
+
+
+HTML_ITEM = """<article class="post"><h2><a href="/t/{n}">Thread {n} on shells and terminals</a></h2>
+<span class="author">user{n}</span><time datetime="2026-10-{n:02d}T09:00:00Z">date</time>
+<p>Excerpt of thread {n} which is certainly long enough to count as content.</p></article>"""
+
+
+def html_community() -> dict:
+    items = "".join(HTML_ITEM.format(n=n) for n in range(1, 7))
+    pages = {f"https://board.test/t/{n}": detail_page("thread", n) for n in range(1, 7)}
+    return {
+        **pages,
+        "https://board.test/": f"<html><head><title>Board</title></head><body><main>{items}</main></body></html>",
+    }
+
+
+async def no_sleep(_seconds):
+    return None
+
+
+class FakePage:
+    def __init__(self, fetcher):
+        self.fetcher = fetcher
+        self.filled: dict[str, str] = {}
+        self.url = ""
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *exc):
+        return None
+
+    async def goto(self, url, **kwargs):
+        self.url = url
+        status, self.html = await self.fetcher.render_html(url)
+        return type("Resp", (), {"status": status})()
+
+    async def wait_for_timeout(self, _ms):
+        return None
+
+    async def content(self):
+        return self.html
+
+    async def fill(self, selector, value):
+        self.filled[selector] = value
+
+    async def click(self, selector):
+        self.fetcher.posts.append(("click", dict(self.filled)))
+
+
+SPA_SHELL = "<html><head><title>Spa</title></head><body><div id=app></div></body></html>"
+
+
+def spa_community() -> dict:
+    items = "".join(HTML_ITEM.format(n=n) for n in range(1, 7))
+    return {
+        "https://spa.test/": SPA_SHELL,
+        "render:https://spa.test/": f"<html><body><main>{items}</main></body></html>",
+    }
+
+
+WALL = (
+    '<html><body><form action="/login" method="post"><input type="hidden" name="csrf" value="t1">'
+    '<input type="text" name="user"><input type="password" name="pass"></form></body></html>'
+)
+CLUB_HOME = (
+    '<html><head><title>Club</title><link rel="alternate" type="application/rss+xml" href="/feed.xml"></head>'
+    "<body><p>Members only area with plenty of text on the page for the club.</p></body></html>"
+)
+
+
+def club_community(state: dict) -> dict:
+    items = "".join(RSS_ITEM.format(n=n).replace("blog.test", "club.test") for n in range(1, 7))
+    feed = RSS.format(items=items)
+
+    def login(data):
+        state["in"] = data.get("user") == "bob" and data.get("pass") == "pw" and data.get("csrf") == "t1"
+        return (200, "welcome" if state["in"] else WALL)
+
+    return {
+        "https://club.test/": lambda: CLUB_HOME if state.get("in") else WALL,
+        "https://club.test/login": WALL,
+        "https://club.test/feed.xml": lambda: feed if state.get("in") else (401, "login"),
+        "post:https://club.test/login": login,
+    }

tests/unit/services/community/generator.py

@@ -0,0 +1,168 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+import importlib.util
+import json
+import py_compile
+
+import pytest
+
+from devplacepy.services.community.discoverer import CommunityDiscoverer
+from devplacepy.services.community.generator import CodeGenerator
+from devplacepy.services.community.runtime import Collector, LoginRequired
+from devplacepy.services.sync.records import RECORD_MODELS
+from tests.unit.services.community.fakes import (
+    FakeFetcher,
+    FakeResponse,
+    html_community,
+    json_community,
+    rss_community,
+)
+
+
+async def no_sleep(_seconds):
+    return None
+
+
+def learn(routes: dict, query: str) -> dict:
+    result = run_async(CommunityDiscoverer(FakeFetcher(routes)).discover(query))
+    assert result.schema, result.error
+    return result.schema
+
+
+def load(path):
+    spec = importlib.util.spec_from_file_location("generated_collectors", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+@pytest.mark.parametrize(
+    "routes,query",
+    [
+        (json_community(), "https://forum.test/api/posts"),
+        (rss_community(), "https://blog.test/"),
+        (html_community(), "https://board.test/"),
+    ],
+)
+def test_generated_file_compiles_and_collects_valid_sync_records(tmp_path, routes, query):
+    schema = learn(routes, query)
+    target = CodeGenerator(schema).write(tmp_path)
+    py_compile.compile(str(target), doraise=True)
+    assert json.loads((tmp_path / "schema.json").read_text()) == schema
+    module = load(target)
+    result = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    assert result["ok"], result["errors"]
+    assert result["records"]["posts"]
+    for kind, rows in result["records"].items():
+        for row in rows:
+            RECORD_MODELS[kind].model_validate(row)
+    uids = {user["origin_uid"] for user in result["records"]["users"]}
+    assert {post["author_uid"] for post in result["records"]["posts"]} <= uids
+
+
+def test_generated_wrappers_exist_per_feed(tmp_path):
+    schema = learn(json_community(), "https://forum.test/api/posts")
+    module = load(CodeGenerator(schema).write(tmp_path))
+    assert callable(module.collect_feed1)
+    assert module.SCHEMA == schema
+
+
+def test_generated_output_matches_in_process_collector(tmp_path):
+    routes = json_community()
+    schema = learn(routes, "https://forum.test/api/posts")
+    module = load(CodeGenerator(schema).write(tmp_path))
+    generated = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    direct = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    assert generated["records"] == direct["records"]
+
+
+def test_incremental_run_collects_only_new_posts(tmp_path):
+    routes = json_community()
+    schema = learn(routes, "https://forum.test/api/posts")
+    first = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    again = run_async(Collector(FakeFetcher(routes), schema, first["state"], sleep=no_sleep).run())
+    assert len(first["records"]["posts"]) == 10
+    assert again["records"]["posts"] == []
+    assert again["state"]["feeds"]["feed1"]["since"] == first["state"]["feeds"]["feed1"]["since"]
+
+
+def test_incremental_run_picks_up_newer_post():
+    routes = json_community()
+    schema = learn(routes, "https://forum.test/api/posts")
+    first = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    newer = {
+        "id": 2000,
+        "title": "A brand new post",
+        "content_html": "<p>Fresh content that arrived later.</p>",
+        "created_at": "2026-11-01T10:00:00Z",
+        "author": {"id": 7, "username": "writer1"},
+        "comments": [],
+    }
+    routes["https://forum.test/api/posts"]["items"].insert(0, newer)
+    result = run_async(Collector(FakeFetcher(routes), schema, first["state"], sleep=no_sleep).run())
+    assert [post["origin_uid"] for post in result["records"]["posts"]] == ["2000"]
+
+
+def test_failed_requests_retry_with_backoff():
+    calls = {"count": 0}
+    delays = []
+
+    class Flaky:
+        async def get(self, url, params=None, timeout=None, **kwargs):
+            calls["count"] += 1
+            return FakeResponse(503 if calls["count"] < 3 else 200, "[]")
+
+    async def record(seconds):
+        delays.append(seconds)
+
+    collector = Collector(Flaky(), {"feeds": [], "limits": {"delay_seconds": 0, "backoff_seconds": 2, "retries": 3}}, sleep=record)
+    assert run_async(collector.fetch("https://x.test/")) == "[]"
+    assert calls["count"] == 3
+    assert [delay for delay in delays if delay] == [2.0, 4.0]
+
+
+def test_login_required_schema_raises_clear_error():
+    schema = {"community": "closed", "origin": "closed.test", "login_required": True, "feeds": []}
+    with pytest.raises(LoginRequired, match="requires a login"):
+        run_async(Collector(FakeFetcher({}), schema).run())
+
+
+def test_unauthorized_response_is_login_required():
+    schema = {
+        "community": "x",
+        "origin": "x.test",
+        "feeds": [{"id": "feed1", "url": "https://x.test/f", "type": "json_api", "post": {"fields": {"id": ["$.id"]}}, "author": {}}],
+        "limits": {"delay_seconds": 0, "retries": 1},
+    }
+    with pytest.raises(LoginRequired):
+        run_async(Collector(FakeFetcher({"https://x.test/f": (401, "no")}), schema, sleep=no_sleep).run())
+
+
+def test_generated_collectors_log_in_with_credentials(tmp_path):
+    from tests.unit.services.community.fakes import club_community
+
+    state = {}
+    routes = club_community(state)
+    result = run_async(CommunityDiscoverer(FakeFetcher(routes), credentials={"username": "bob", "password": "pw"}).discover("https://club.test/"))
+    module = load(CodeGenerator(result.schema).write(tmp_path))
+    state.clear()
+    with pytest.raises(module.LoginRequired, match="requires a login"):
+        run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    collected = run_async(
+        module.collect_all(FakeFetcher(routes), sleep=no_sleep, credentials={"username": "bob", "password": "pw"})
+    )
+    assert collected["ok"] and collected["records"]["posts"]
+    state.clear()
+    with pytest.raises(module.LoginRequired, match="rejected the stored credentials"):
+        run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep, credentials={"username": "bob", "password": "bad"}))
+
+
+def test_javascript_feed_is_collected_through_the_browser(tmp_path):
+    from tests.unit.services.community.fakes import spa_community
+
+    routes = spa_community()
+    schema = learn(routes, "https://spa.test/")
+    module = load(CodeGenerator(schema).write(tmp_path))
+    collected = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    assert len(collected["records"]["posts"]) == 6

tests/unit/services/community/search.py

@@ -0,0 +1,39 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+import pytest
+
+from devplacepy.services.community.search import SearchPhase, direct_url, rank
+from tests.unit.services.community.fakes import FakeFetcher
+
+
+def test_direct_addresses_skip_search():
+    assert direct_url("https://x.test/forum") == "https://x.test/forum"
+    assert direct_url("x.test") == "https://x.test"
+    assert direct_url("devplace") is None
+
+
+def test_rank_prefers_domain_match_and_drops_social_noise():
+    results = [
+        {"title": "DevPlace on LinkedIn", "url": "https://www.linkedin.com/company/devplace", "description": ""},
+        {"title": "Other", "url": "https://other.test/page", "description": "mentions devplace"},
+        {"title": "DevPlace", "url": "https://devplace.example/", "description": ""},
+    ]
+    ranked = rank("devplace", results)
+    assert [candidate.host for candidate in ranked] == ["devplace.example", "other.test"]
+
+
+def test_search_uses_rsearch_and_caps_candidates():
+    results = [{"title": f"t{n}", "url": f"https://site{n}.test/", "description": ""} for n in range(9)]
+    key = "https://rsearch.app.molodetz.nl/search?content=false&count=15&query=forum"
+    fetcher = FakeFetcher({key: {"success": True, "results": results}})
+    found = run_async(SearchPhase(fetcher).run("forum"))
+    assert len(found) == 5
+
+
+def test_search_failure_raises():
+    key = "https://rsearch.app.molodetz.nl/search?content=false&count=15&query=forum"
+    fetcher = FakeFetcher({key: (500, "boom")})
+    with pytest.raises(RuntimeError):
+        run_async(SearchPhase(fetcher).run("forum"))

tests/unit/services/community/service.py

@@ -0,0 +1,81 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+import pytest
+
+from devplacepy.database import get_setting, get_table, init_db, set_setting
+from devplacepy.services.community.runner import CommunityRunner
+from devplacepy.services.community.service import DRY_RUN_KEY, CommunityService
+from devplacepy.services.community.spec import STATUS_FAILING, STATUS_READY
+from devplacepy.services.community.store import store
+from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep
+
+
+@pytest.fixture
+def world():
+    init_db()
+    previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins", DRY_RUN_KEY)}
+    set_setting("sync_enabled", "1")
+    set_setting("sync_origins", "")
+    get_table("communities").delete()
+    get_table("community_runs").delete()
+    get_table("sync_links").delete(origin="forum.test")
+    routes = json_community()
+    runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep)
+    yield routes, runner
+    for key, value in previous.items():
+        set_setting(key, value)
+
+
+def seed(runner) -> dict:
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test"))
+    return store.get("forum-test")
+
+
+def test_dry_run_service_tick_learns_nothing_and_runs_nothing(world):
+    _, runner = world
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    set_setting(DRY_RUN_KEY, "1")
+    service = CommunityService(runner)
+    run_async(service.run_once())
+    assert store.get("forum-test")["status"] == "pending"
+    assert any("DRY RUN" in line for line in service.log_buffer)
+
+
+def test_live_tick_learns_pending_then_runs_due(world):
+    _, runner = world
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    set_setting(DRY_RUN_KEY, "0")
+    service = CommunityService(runner)
+    run_async(service.run_once())
+    assert store.get("forum-test")["status"] == STATUS_READY
+    run_async(service.run_once())
+    row = store.get("forum-test")
+    assert row["run_count"] == 1
+    assert get_table("sync_links").count(origin="forum.test", kind="posts") == 10
+    run_async(service.run_once())
+    assert store.get("forum-test")["run_count"] == 1
+
+
+def test_repeated_failures_disable_and_flag_the_community(world):
+    routes, runner = world
+    row = seed(runner)
+    routes["https://forum.test/api/posts"] = (500, "down")
+    failing = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep)
+    for _ in range(6):
+        current = store.get("forum-test")
+        report = run_async(failing.run(current, dry_run=False, trigger="test"))
+        assert report.status == "failed"
+    final = store.get("forum-test")
+    assert final["status"] == STATUS_FAILING and not final["enabled"]
+    assert "relearn" in "".join(store.runs(row["uid"])[0]["log"]).lower() or final["failure_count"] >= 5
+
+
+def test_run_history_is_bounded(world):
+    _, runner = world
+    row = seed(runner)
+    for _ in range(35):
+        run_async(runner.run(store.get("forum-test"), dry_run=True, trigger="test"))
+    assert len(store.runs(row["uid"], limit=100)) == 30

tests/unit/services/devii/actions/catalog.py

@@ -147,3 +147,17 @@ def test_scheduling_tools_are_hidden_from_guests():
     }
     for name in ("create_task", "update_task", "run_task_now"):
         assert name not in guest
+
+
+def test_sync_tools_are_admin_only_and_match_the_routes():
+    from devplacepy.main import app
+    from devplacepy.services.sync.spec import KINDS
+
+    paths = set(app.openapi()["paths"])
+    for kind in KINDS:
+        action = BY_NAME[f"sync_{kind}"]
+        assert action.method == "POST" and action.path == f"/sync/{kind}"
+        assert action.requires_admin is True
+        assert action.path in paths
+        assert {param.name for param in action.params} == {"origin", "records", "dry_run"}
+        assert _records_mutation(action.name) is True

tests/unit/services/nodes/adapters.py

@@ -0,0 +1,91 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+import pytest
+
+from devplacepy.services.nodes.adapters.base import AdapterError, clean, tag_for
+from devplacepy.services.nodes.adapters.direct import DevTo, GitHub, HackerNews, Mastodon, StackExchange
+from devplacepy.services.nodes.adapters.reddit import Reddit
+from devplacepy.services.nodes.adapters.registry import ADAPTERS
+from devplacepy.services.nodes.spec import SPEC
+
+
+def body(value) -> bytes:
+    return json.dumps(value).encode()
+
+
+def test_clean_strips_tags_entities_and_caps():
+    assert clean("<p>a &amp; b</p>  c", 50) == "a & b c"
+    assert len(clean("x" * 900, 100)) == 100
+    assert clean(None, 10) == ""
+
+
+def test_tag_for_picks_longest_word():
+    assert tag_for("sqlite wal mode") == "sqlite"
+    assert tag_for("!!") == ""
+
+
+def test_hackernews_parse():
+    data = {
+        "hits": [
+            {"objectID": "7", "title": "T", "points": 5, "num_comments": 2, "author": "a", "created_at": "2026"},
+            {"objectID": "", "title": "no id"},
+            "junk",
+        ]
+    }
+    hits = HackerNews().parse(body(data), 8)
+    assert [item.url for item in hits] == ["https://news.ycombinator.com/item?id=7"]
+    assert hits[0].score == 5 and hits[0].via == "direct"
+
+
+def test_stackexchange_parse():
+    data = {"items": [{"title": "Q &amp; A", "link": "https://stackoverflow.com/q/1", "score": 3, "answer_count": 2, "tags": ["c"], "owner": {"display_name": "bob"}}]}
+    hit = StackExchange().parse(body(data), 8)[0]
+    assert hit.title == "Q & A" and hit.author == "bob" and hit.comments == 2
+
+
+def test_github_parse_requires_http_url():
+    data = {"items": [{"title": "I", "html_url": "javascript:alert(1)"}, {"title": "J", "html_url": "https://github.com/a/b/issues/1", "user": {"login": "u"}}]}
+    hits = GitHub().parse(body(data), 8)
+    assert [item.title for item in hits] == ["J"]
+
+
+def test_devto_and_mastodon_need_a_tag():
+    with pytest.raises(AdapterError) as error:
+        DevTo().request("!!", 5)
+    assert error.value.code == "no_tag"
+    with pytest.raises(AdapterError):
+        Mastodon().request("", 5)
+    assert "tag=sqlite" in DevTo().request("sqlite", 5)
+    assert "/tag/sqlite" in Mastodon().request("sqlite", 5)
+
+
+def test_mastodon_parse_strips_html():
+    data = [{"url": "https://m.example/1", "content": "<p>hello <b>world</b></p>", "account": {"acct": "x@y"}, "favourites_count": 2, "reblogs_count": 1}]
+    hit = Mastodon().parse(body(data), 5)[0]
+    assert hit.snippet == "hello world" and hit.score == 3
+
+
+def test_reddit_parse_and_via():
+    data = {"data": {"children": [{"data": {"title": "T", "permalink": "/r/x/comments/1/t/", "score": 9, "num_comments": 4, "author": "a", "selftext": "body"}}, {"data": {"title": "bad", "permalink": "http://evil"}}]}}
+    hits = Reddit().parse(body(data), 5)
+    assert len(hits) == 1 and hits[0].url == "https://www.reddit.com/r/x/comments/1/t/" and hits[0].via == "fetch"
+
+
+@pytest.mark.parametrize("adapter", list(ADAPTERS.values()))
+def test_forged_structure_is_rejected(adapter):
+    for forged in (b"not json", body("string"), body({"unexpected": 1}), body(42)):
+        with pytest.raises(AdapterError):
+            adapter.parse(forged, 5)
+
+
+def test_oversize_body_is_rejected():
+    with pytest.raises(AdapterError) as error:
+        HackerNews().parse(b" " * (2 * 1024 * 1024 + 1), 5)
+    assert error.value.code == "body_too_large"
+
+
+def test_node_adapters_are_exactly_the_spec_adapters():
+    node_backed = {name for name, adapter in ADAPTERS.items() if not adapter.direct}
+    assert node_backed == {item.adapter for item in SPEC.destinations}

tests/unit/services/nodes/cache.py

@@ -0,0 +1,109 @@
+# retoor <retoor@molodetz.nl>
+
+import random
+
+from devplacepy.services.nodes.cache import ResultCache, cache_key
+from devplacepy.services.nodes.spec import SPEC
+
+LIMITS = SPEC.limits
+
+
+class Clock:
+    def __init__(self):
+        self.now = 100.0
+
+    def __call__(self):
+        return self.now
+
+
+def make(**kwargs):
+    clock = Clock()
+    return ResultCache(clock=clock, **kwargs), clock
+
+
+def test_hit_and_miss_counters():
+    cache, _ = make()
+    assert cache.get("GET", "u") is None
+    assert cache.put("GET", "u", 200, {"a": "b"}, b"xyz", False)
+    hit = cache.get("GET", "u")
+    assert hit.body == b"xyz" and hit.headers == {"a": "b"}
+    assert (cache.hits, cache.misses) == (1, 1)
+
+
+def test_method_is_part_of_the_key():
+    assert cache_key("GET", "u") != cache_key("HEAD", "u")
+    assert cache_key("get", "u") == cache_key("GET", "u")
+
+
+def test_entries_expire_after_the_ttl():
+    cache, clock = make(ttl_seconds=10)
+    cache.put("GET", "u", 200, {}, b"x", False)
+    clock.now += 9.9
+    assert cache.get("GET", "u") is not None
+    clock.now += 0.2
+    assert cache.get("GET", "u") is None
+    assert len(cache) == 0 and cache.size_bytes == 0
+
+
+def test_ttl_is_clamped_to_the_spec_ceiling_and_zero_disables():
+    assert ResultCache(ttl_seconds=10**9).ttl_seconds == LIMITS.cache_ttl_seconds
+    off, _ = make(ttl_seconds=0)
+    assert off.put("GET", "u", 200, {}, b"x", False) is False
+
+
+def test_errors_and_truncated_bodies_are_never_cached():
+    cache, _ = make()
+    assert not cache.put("GET", "u", 500, {}, b"x", False)
+    assert not cache.put("GET", "u", 503, {}, b"x", False)
+    assert not cache.put("GET", "u", 200, {}, b"x", True)
+    assert cache.put("GET", "u", 404, {}, b"x", False)
+
+
+def test_an_oversized_entry_is_refused():
+    cache, _ = make(entry_max_bytes=10, max_bytes=100)
+    assert not cache.put("GET", "u", 200, {}, b"x" * 11, False)
+    assert cache.put("GET", "u", 200, {}, b"x" * 10, False)
+
+
+def test_the_entry_cap_cannot_exceed_the_spec_ceiling():
+    assert ResultCache(entry_max_bytes=10**12).entry_max_bytes == LIMITS.cache_entry_max_bytes
+
+
+def test_least_recently_used_goes_first():
+    cache, _ = make(max_bytes=30, entry_max_bytes=30)
+    for name in "abc":
+        cache.put("GET", name, 200, {}, b"x" * 10, False)
+    cache.get("GET", "a")
+    cache.put("GET", "d", 200, {}, b"x" * 10, False)
+    assert cache.get("GET", "b") is None
+    assert cache.get("GET", "a") is not None
+    assert cache.get("GET", "c") is not None
+    assert cache.get("GET", "d") is not None
+
+
+def test_replacing_an_entry_does_not_leak_bytes():
+    cache, _ = make()
+    cache.put("GET", "u", 200, {}, b"x" * 100, False)
+    cache.put("GET", "u", 200, {}, b"y" * 40, False)
+    assert cache.size_bytes == 40 and len(cache) == 1
+
+
+def test_size_never_exceeds_the_cap_under_a_random_workload():
+    cache, clock = make(max_bytes=1000, entry_max_bytes=400, ttl_seconds=50)
+    rng = random.Random(7)
+    for step in range(5000):
+        clock.now += rng.random()
+        key = f"k{rng.randint(0, 60)}"
+        if rng.random() < 0.7:
+            cache.put("GET", key, rng.choice([200, 404, 500]), {}, b"x" * rng.randint(0, 500), False)
+        else:
+            cache.get("GET", key)
+        assert 0 <= cache.size_bytes <= 1000
+        assert cache.size_bytes == sum(len(e.body) for e in cache._entries.values())
+
+
+def test_clear_empties_everything():
+    cache, _ = make()
+    cache.put("GET", "u", 200, {}, b"x", False)
+    cache.clear()
+    assert len(cache) == 0 and cache.size_bytes == 0

tests/unit/services/nodes/circuit.py

@@ -0,0 +1,97 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.nodes.circuit import CLOSED, HALF_OPEN, OPEN, CircuitBreaker
+
+
+class Clock:
+    def __init__(self):
+        self.now = 0.0
+
+    def __call__(self):
+        return self.now
+
+
+def make(failures=3, open_seconds=60):
+    clock = Clock()
+    return CircuitBreaker(failures, open_seconds, clock), clock
+
+
+def test_unknown_keys_are_closed():
+    breaker, _ = make()
+    assert breaker.state("x") == CLOSED and breaker.allows("x")
+
+
+def test_it_opens_at_the_threshold_and_not_before():
+    breaker, _ = make(failures=3)
+    breaker.failure("x")
+    breaker.failure("x")
+    assert breaker.allows("x")
+    breaker.failure("x")
+    assert breaker.state("x") == OPEN and not breaker.allows("x")
+
+
+def test_a_success_resets_the_streak():
+    breaker, _ = make(failures=3)
+    breaker.failure("x")
+    breaker.failure("x")
+    breaker.success("x")
+    breaker.failure("x")
+    breaker.failure("x")
+    assert breaker.allows("x")
+
+
+def test_it_goes_half_open_after_the_cooldown_and_admits_one_probe():
+    breaker, clock = make(failures=1, open_seconds=60)
+    breaker.failure("x")
+    clock.now = 59
+    assert breaker.state("x") == OPEN
+    clock.now = 60
+    assert breaker.state("x") == HALF_OPEN and breaker.allows("x")
+    breaker.begin("x")
+    assert not breaker.allows("x")
+
+
+def test_a_failed_probe_reopens_for_a_full_cooldown():
+    breaker, clock = make(failures=1, open_seconds=60)
+    breaker.failure("x")
+    clock.now = 70
+    breaker.begin("x")
+    breaker.failure("x")
+    assert breaker.state("x") == OPEN
+    clock.now = 129
+    assert breaker.state("x") == OPEN
+    clock.now = 131
+    assert breaker.state("x") == HALF_OPEN
+
+
+def test_a_successful_probe_closes_it():
+    breaker, clock = make(failures=1)
+    breaker.failure("x")
+    clock.now = 100
+    breaker.begin("x")
+    breaker.success("x")
+    assert breaker.state("x") == CLOSED and breaker.allows("x")
+
+
+def test_keys_are_independent():
+    breaker, _ = make(failures=1)
+    breaker.failure(("node", "a"))
+    assert not breaker.allows(("node", "a"))
+    assert breaker.allows(("node", "b"))
+    assert breaker.allows(("destination", "a"))
+
+
+def test_open_count_and_forget():
+    breaker, _ = make(failures=1)
+    breaker.failure("a")
+    breaker.failure("b")
+    assert breaker.open_count() == 2
+    breaker.forget("a")
+    assert breaker.open_count() == 1
+
+
+def test_a_zero_threshold_still_needs_one_failure():
+    breaker = CircuitBreaker(failures=0)
+    assert breaker.allows("x")
+    breaker.failure("x")
+    assert not breaker.allows("x")

tests/unit/services/nodes/dispatcher.py

@@ -0,0 +1,415 @@
+# retoor <retoor@molodetz.nl>
+
+import base64
+import json
+import random
+import time
+
+import pytest
+
+from devplacepy.database import get_table
+from devplacepy.database import nodes as store
+from devplacepy.services.nodes.broker import BodyError, JobBroker, clean_headers, decode_body
+from devplacepy.services.nodes.circuit import CircuitBreaker
+from devplacepy.services.nodes.dispatcher import JOB_HEADERS, NodeDispatcher
+from devplacepy.services.nodes.ledger import UsageLedger
+from devplacepy.services.nodes.policy import DestinationPolicy
+from devplacepy.services.nodes.registry import NodeCounters, NodeRegistry
+from devplacepy.services.nodes.scheduler import MEGABYTE, NodeScheduler
+from devplacepy.services.nodes.session import NodeSession
+from devplacepy.services.nodes.spec import SPEC
+from devplacepy.utils import generate_uid
+from tests.conftest import run_async
+from tests.unit.services.nodes.scheduler import Clock, node, settings
+
+LIMITS = SPEC.limits
+TARGET_URL = "https://www.reddit.com/r/python.json"
+
+
+class Rig:
+    def __init__(self):
+        self.clock = Clock()
+        self.counters = NodeCounters()
+        self.registry = NodeRegistry(self.counters)
+        self.breaker = CircuitBreaker(clock=self.clock)
+        self.scheduler = NodeScheduler(self.registry, self.breaker, self.clock, rng=random.Random(2))
+        self.broker = JobBroker()
+        self.dispatcher = NodeDispatcher(
+            self.registry, self.scheduler, self.broker, self.breaker, UsageLedger(), self.counters,
+            clock=self.clock,
+        )
+
+    def connect(self, **overrides):
+        owner = generate_uid()
+        device, _ = store.create_device(owner, "d", "native")
+        session = NodeSession(
+            node().websocket, {"uid": device["uid"], "user_uid": owner, "kind": "native"}
+        )
+        session.hello = {"modes": ["fetch"]}
+        session.settings = settings(**overrides)
+        session.roll_day()
+        self.dispatcher.attach(session)
+        self.registry.admit(session)
+        return session
+
+    def enqueue(self, url=TARGET_URL, caller="c1", kind="user", mode="any"):
+        target = DestinationPolicy().check(url, "GET", mode)
+        return self.broker.enqueue(kind, caller, target, mode)
+
+    def tick(self):
+        return run_async(self.dispatcher.tick())
+
+    def answer(self, session, uid, **fields):
+        frame = {"type": "job_result", "job_id": uid, "status": 200, "body_b64": base64.b64encode(b"hello").decode()}
+        frame.update(fields)
+        run_async(session._handle(frame))
+
+
+@pytest.fixture
+def rig(local_db):
+    get_table("node_requests").delete()
+    yield Rig()
+    get_table("node_requests").delete()
+
+
+def sent_jobs(session):
+    return [frame for frame in session.websocket.sent if frame["type"] == "job"]
+
+
+def test_a_queued_request_becomes_a_job_frame(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    assert rig.tick() == 1
+    job = sent_jobs(session)[0]
+    assert job == {"type": "job", "job_id": uid, "method": "GET", "url": TARGET_URL, "headers": JOB_HEADERS, "timeout": LIMITS.job_timeout_seconds}
+    assert store.get_request(uid)["status"] == "assigned"
+    assert store.get_request(uid)["device_uid"] == session.device_uid
+    assert uid in session.inflight
+
+
+def test_the_result_completes_the_request_and_is_metered(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    rig.answer(session, uid, headers={"Content-Type": "application/json", "Set-Cookie": "x=1"})
+    result = rig.broker.read(uid)
+    assert result.ok and result.http_status == 200 and result.body == b"hello"
+    assert result.headers == {"content-type": "application/json"}
+    assert uid not in session.inflight
+    today = store.usage_today(session.device_uid)
+    assert today["requests_ok"] == 1 and today["bytes_down"] == 5 and today["bytes_up"] > len(TARGET_URL)
+    assert store.get_donor(session.user_uid)["requests_ok"] == 1
+    assert session.used_today == today["bytes_up"] + today["bytes_down"]
+    assert rig.counters.snapshot()["jobs_done"] == 1
+
+
+def test_no_eligible_node_leaves_the_request_queued(rig):
+    uid = rig.enqueue()
+    assert rig.tick() == 0
+    assert store.get_request(uid)["status"] == "queued"
+    rig.connect(allowed_platforms=[])
+    assert rig.tick() == 0
+    assert store.get_request(uid)["status"] == "queued"
+
+
+def test_one_request_goes_to_exactly_one_node(rig):
+    sessions = [rig.connect() for _ in range(4)]
+    rig.enqueue()
+    assert rig.tick() == 1
+    assert sum(len(sent_jobs(item)) for item in sessions) == 1
+
+
+def test_a_caller_never_gets_its_own_node(rig):
+    session = rig.connect()
+    uid = rig.enqueue(caller=session.user_uid)
+    assert rig.tick() == 0
+    assert store.get_request(uid)["status"] == "queued"
+    guest = rig.enqueue(caller=session.user_uid, kind="guest")
+    assert rig.tick() == 1 and guest in session.inflight
+
+
+def test_a_policy_failure_at_dispatch_time_fails_the_request(rig):
+    rig.connect()
+    uid = store.enqueue_request("user", "c", "https://evil.example/", "GET", "x", "any")
+    rig.tick()
+    row = store.get_request(uid)
+    assert row["status"] == "failed" and row["error"] == "host_not_allowed"
+    assert rig.counters.snapshot()["jobs_refused"] == 1
+
+
+def test_a_job_error_fails_the_request_and_counts_against_the_node(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    run_async(session._handle({"type": "job_error", "job_id": uid, "code": "timeout"}))
+    assert store.get_request(uid)["error"] == "timeout"
+    assert store.usage_today(session.device_uid)["requests_failed"] == 1
+    assert rig.counters.snapshot()["jobs_failed"] == 1
+
+
+def test_five_node_failures_open_that_nodes_circuit(rig):
+    session = rig.connect()
+    for _ in range(LIMITS.breaker_failures):
+        uid = rig.enqueue()
+        session.host_started.clear()
+        rig.scheduler._destination_started.clear()
+        rig.tick()
+        run_async(session._handle({"type": "job_error", "job_id": uid, "code": "network"}))
+    follow = rig.enqueue()
+    session.host_started.clear()
+    rig.scheduler._destination_started.clear()
+    assert rig.tick() == 0
+    assert store.get_request(follow)["status"] == "queued"
+    rig.clock.now += LIMITS.breaker_open_seconds
+    assert rig.tick() == 1
+
+
+def test_donor_limit_codes_pause_the_node_without_blaming_it(rig):
+    session = rig.connect(bandwidth_mb_day=50)
+    uid = rig.enqueue()
+    rig.tick()
+    run_async(session._handle({"type": "job_error", "job_id": uid, "code": "bandwidth_exhausted"}))
+    assert session.used_today == 50 * MEGABYTE
+    assert rig.breaker.allows(("node", session.device_uid))
+    other = rig.connect()
+    rig.clock.now += 10
+    uid = rig.enqueue()
+    assert rig.tick() == 1
+    run_async(other._handle({"type": "job_error", "job_id": uid, "code": "outside_hours"}))
+    assert other.paused_until > rig.clock.now
+    assert rig.breaker.allows(("node", other.device_uid))
+
+
+def test_a_donor_refusal_of_an_allowed_destination_raises_a_flag(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    run_async(session._handle({"type": "job_error", "job_id": uid, "code": "host_not_allowed"}))
+    kinds = [flag["kind"] for flag in store.list_flags()]
+    assert "refusal_mismatch" in kinds
+    assert rig.breaker.allows(("node", session.device_uid))
+
+
+@pytest.mark.parametrize(
+    "fields,code",
+    [
+        ({"status": 99}, "bad_status"),
+        ({"status": 600}, "bad_status"),
+        ({"body_b64": "***not base64***"}, "bad_base64"),
+        ({"body_b64": "abc"}, "bad_base64"),
+    ],
+)
+def test_forged_results_are_integrity_failures(rig, fields, code):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    rig.answer(session, uid, **fields)
+    assert store.get_request(uid)["error"] == code
+    assert store.usage_today(session.device_uid)["integrity_failures"] == 1
+    assert rig.counters.snapshot()["integrity_failures"] == 1
+    assert any(flag["kind"] == "integrity" for flag in store.list_flags())
+    assert store.get_donor(session.user_uid)["requests_ok"] == 0
+
+
+def test_an_oversized_body_is_truncated_and_flagged(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    big = base64.b64encode(b"x" * (LIMITS.max_body_bytes + 10)).decode()
+    rig.answer(session, uid, body_b64=big)
+    result = rig.broker.read(uid)
+    assert result.truncated and len(result.body) == LIMITS.max_body_bytes
+    assert rig.counters.snapshot()["results_truncated"] == 1
+
+
+def test_a_donor_declared_truncation_is_kept(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    rig.answer(session, uid, truncated=True)
+    assert rig.broker.read(uid).truncated
+
+
+def test_a_result_for_an_unknown_job_is_a_protocol_error(rig):
+    session = rig.connect()
+    from devplacepy.services.nodes.frames import FrameError
+
+    with pytest.raises(FrameError):
+        rig.answer(session, "never-assigned")
+
+
+def test_a_job_cannot_be_answered_twice(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    rig.answer(session, uid)
+    from devplacepy.services.nodes.frames import FrameError
+
+    with pytest.raises(FrameError):
+        rig.answer(session, uid)
+    assert store.get_donor(session.user_uid)["requests_ok"] == 1
+
+
+def test_rate_limited_and_server_error_results_open_the_destination_circuit(rig):
+    for index in range(LIMITS.breaker_failures):
+        session = rig.connect()
+        uid = rig.enqueue()
+        rig.scheduler._destination_started.clear()
+        rig.tick()
+        rig.answer(session, uid, status=429 if index % 2 else 503)
+        assert rig.broker.read(uid).ok
+        rig.registry.release(session)
+    assert not rig.breaker.allows(("destination", "reddit.com"))
+    rig.connect()
+    rig.enqueue()
+    rig.scheduler._destination_started.clear()
+    assert rig.tick() == 0
+
+
+def test_a_healthy_result_closes_the_destination_circuit(rig):
+    rig.breaker.failure(("destination", "reddit.com"))
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    rig.answer(session, uid, status=200)
+    assert rig.breaker.state(("destination", "reddit.com")) == "closed"
+
+
+def test_a_job_that_never_answers_times_out(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    rig.clock.now += LIMITS.job_timeout_seconds + LIMITS.job_grace_seconds + 1
+    assert run_async(rig.dispatcher.sweep_inflight()) == 1
+    assert store.get_request(uid)["error"] == "timeout"
+    assert session.inflight == {}
+
+
+def test_a_disconnect_fails_every_inflight_job(rig):
+    session = rig.connect(max_concurrent=2)
+    uids = [rig.enqueue(), rig.enqueue()]
+    for _ in uids:
+        session.host_started.clear()
+        rig.scheduler._destination_started.clear()
+        assert rig.tick() == 1
+    assert len(session.inflight) == 2
+    assert rig.dispatcher.release_session(session) == 2
+    for uid in uids:
+        row = store.get_request(uid)
+        assert row["status"] == "failed" and row["error"] == "node_disconnected"
+    assert session.inflight == {}
+
+
+def test_release_session_marks_assigned_jobs_as_node_disconnected(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    assert rig.dispatcher.release_session(session) == 1
+    assert store.get_request(uid)["error"] == "node_disconnected"
+    assert rig.dispatcher.release_session(session) == 0
+
+
+def test_a_failed_send_releases_the_job(rig):
+    session = rig.connect()
+
+    async def broken(*args, **kwargs):
+        raise ConnectionError("gone")
+
+    session.send = broken
+    uid = rig.enqueue()
+    assert rig.tick() == 0
+    assert store.get_request(uid)["error"] == "send_failed"
+    assert session.inflight == {}
+
+
+def test_an_idle_tick_does_no_writes_and_no_work(rig):
+    rig.connect()
+    started = time.perf_counter()
+    for _ in range(50):
+        assert rig.tick() == 0
+    assert (time.perf_counter() - started) / 50 < 0.02
+
+
+def test_the_cap_refusing_the_ledger_pauses_the_node(rig):
+    session = rig.connect(bandwidth_mb_day=1)
+    store.save_node_settings(session.user_uid, {"enabled": True, "bandwidth_mb_day": 1})
+    session.used_today = 0
+    uid = rig.enqueue()
+    rig.tick()
+    store.record_job(session.device_uid, session.user_uid, ok=True, bytes_down=MEGABYTE, cap_bytes=MEGABYTE)
+    rig.answer(session, uid)
+    assert rig.broker.read(uid).ok
+    assert session.used_today >= MEGABYTE
+
+
+def test_every_assigned_job_is_answered_at_most_once_under_a_random_workload(rig):
+    rng = random.Random(11)
+    sessions = [rig.connect(max_concurrent=2) for _ in range(4)]
+    seen = set()
+    for _ in range(150):
+        rig.enqueue(caller=generate_uid())
+        rig.clock.now += 10
+        rig.scheduler._destination_started.clear()
+        rig.tick()
+        for session in sessions:
+            for job_id in list(session.inflight):
+                assert job_id not in seen
+                action = rng.random()
+                if action < 0.6:
+                    rig.answer(session, job_id)
+                    seen.add(job_id)
+                elif action < 0.8:
+                    run_async(session._handle({"type": "job_error", "job_id": job_id, "code": "network"}))
+                    seen.add(job_id)
+            assert len(session.inflight) <= session.max_concurrent
+    open_now = {job for item in sessions for job in item.inflight}
+    assert not (open_now & seen)
+
+
+def test_body_helpers():
+    assert decode_body(base64.b64encode(b"abc").decode()) == (b"abc", False)
+    with pytest.raises(BodyError):
+        decode_body("!!!")
+    assert clean_headers({"A": "1", "set-cookie": "s", "Authorization": "x", 5: "x", "b": 5}) == {"a": "1"}
+    assert clean_headers("nope") == {}
+    many = {f"h{i}": "v" for i in range(500)}
+    assert len(clean_headers(many)) == 64
+    assert len(next(iter(clean_headers({"k": "v" * 5000}).values()))) == 1024
+    assert json.loads(json.dumps(clean_headers({"A": "1"}))) == {"a": "1"}
+
+
+def test_a_saturated_destination_is_scanned_once_per_tick_not_once_per_request(rig, monkeypatch):
+    rig.connect(allowed_platforms=[])
+    for index in range(30):
+        rig.enqueue(caller=f"c{index}")
+    scans = []
+    original = rig.scheduler.candidates
+    monkeypatch.setattr(rig.scheduler, "candidates", lambda *a: scans.append(a) or original(*a))
+    assert rig.tick() == 0
+    assert len(scans) == 1
+
+
+def test_saturation_does_not_hide_other_destinations(rig):
+    session = rig.connect()
+    for index in range(3):
+        rig.enqueue(caller=f"c{index}")
+    rig.scheduler._destination_started["reddit.com"] = rig.clock.now
+    assert rig.tick() == 0
+    rig.clock.now += 10
+    assert rig.tick() == 1
+    assert len(session.inflight) == 1
+
+
+def test_the_owner_exclusion_does_not_saturate_the_destination(rig):
+    mine = rig.connect()
+    other = rig.connect()
+    first = rig.enqueue(caller=mine.user_uid)
+    second = rig.enqueue(caller=other.user_uid)
+    rig.clock.now += 10
+    rig.tick()
+    assert first in other.inflight
+    rig.clock.now += 10
+    rig.scheduler._destination_started.clear()
+    other.host_started.clear()
+    rig.tick()
+    assert second in mine.inflight

tests/unit/services/nodes/fabric.py

@@ -0,0 +1,231 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+from datetime import datetime, timezone
+
+import pytest
+
+from devplacepy.database import get_table, set_setting
+from devplacepy.database import nodes as store
+from devplacepy.services.nodes.broker import BrokerResult, JobBroker
+from devplacepy.services.nodes.cache import ResultCache
+from devplacepy.services.nodes.fabric import (
+    FabricQuotaExceeded,
+    FabricRefused,
+    FabricUnavailable,
+    NodeFabric,
+)
+from devplacepy.services.nodes.spec import SPEC
+from devplacepy.utils import generate_uid
+from tests.conftest import run_async
+
+LIMITS = SPEC.limits
+URL = "https://www.reddit.com/r/python.json"
+
+
+class ScriptedBroker(JobBroker):
+    def __init__(self, outcomes):
+        self.outcomes = list(outcomes)
+        self.enqueued = 0
+
+    def enqueue(self, caller_kind, caller_id, target, mode):
+        self.enqueued += 1
+        return f"uid-{self.enqueued}"
+
+    async def wait(self, uid, timeout):
+        outcome = self.outcomes.pop(0)
+        if outcome == "ok":
+            return BrokerResult(uid, "done", 200, {"content-type": "text/plain"}, b"payload", False, "")
+        return BrokerResult(uid, "failed", 0, {}, b"", False, outcome)
+
+
+def heartbeat(connected):
+    table = get_table("service_state")
+    record = {
+        "name": "nodes",
+        "status": "running",
+        "heartbeat": datetime.now(timezone.utc).isoformat(),
+        "metrics": json.dumps({"stats": [{"label": "connected", "value": connected}]}),
+    }
+    existing = table.find_one(name="nodes")
+    if existing:
+        table.update({**record, "id": existing["id"]}, ["id"])
+    else:
+        table.insert(record)
+
+
+@pytest.fixture
+def online(local_db):
+    set_setting("nodes_enabled", "1")
+    set_setting("nodes_daily_quota_user", "1000")
+    set_setting("nodes_daily_quota_guest", "0")
+    heartbeat(3)
+    yield
+    set_setting("nodes_enabled", "0")
+    get_table("service_state").delete(name="nodes")
+
+
+def fetch(fabric, **kwargs):
+    kwargs.setdefault("caller_kind", "user")
+    kwargs.setdefault("caller_id", generate_uid())
+    return run_async(fabric.fetch(URL, **kwargs))
+
+
+def test_a_node_fetch_returns_the_body_and_caches_it(online):
+    broker = ScriptedBroker(["ok"])
+    fabric = NodeFabric(broker=broker)
+    first = fetch(fabric, caller_id="c")
+    assert (first.status, first.body, first.via) == (200, b"payload", "fetch")
+    second = fetch(fabric, caller_id="c")
+    assert second.via == "cache" and second.body == b"payload" and second.elapsed_ms == 0
+    assert broker.enqueued == 1
+
+
+def test_use_cache_false_always_asks_a_node(online):
+    broker = ScriptedBroker(["ok", "ok"])
+    fabric = NodeFabric(broker=broker)
+    fetch(fabric, caller_id="c", use_cache=False)
+    fetch(fabric, caller_id="c", use_cache=False)
+    assert broker.enqueued == 2
+
+
+def test_policy_refusals_surface_with_their_code(online):
+    fabric = NodeFabric(broker=ScriptedBroker([]))
+    for url, code in (("https://example.com/", "host_not_allowed"), ("https://reddit.com:22/", "port_not_allowed")):
+        with pytest.raises(FabricRefused) as info:
+            run_async(fabric.fetch(url, caller_kind="user", caller_id="c"))
+        assert info.value.code == code
+
+
+def test_the_kill_switch_and_missing_nodes_are_unavailable(online):
+    fabric = NodeFabric(broker=ScriptedBroker([]))
+    heartbeat(0)
+    with pytest.raises(FabricUnavailable) as info:
+        fetch(fabric)
+    assert info.value.code == "no_nodes"
+    heartbeat(2)
+    set_setting("nodes_enabled", "0")
+    with pytest.raises(FabricUnavailable) as info:
+        fetch(fabric)
+    assert info.value.code == "kill_switch"
+
+
+def test_a_stale_heartbeat_counts_as_no_nodes(online):
+    table = get_table("service_state")
+    row = table.find_one(name="nodes")
+    table.update({"id": row["id"], "heartbeat": "2000-01-01T00:00:00+00:00"}, ["id"])
+    assert NodeFabric(broker=ScriptedBroker([])).available().nodes_online == 0
+
+
+def test_no_service_row_means_no_nodes(online):
+    get_table("service_state").delete(name="nodes")
+    assert NodeFabric().available().nodes_online == 0
+
+
+def test_quota_is_enforced_before_any_node_is_touched(online):
+    set_setting("nodes_daily_quota_user", "2")
+    broker = ScriptedBroker(["ok"] * 5)
+    fabric = NodeFabric(broker=broker)
+    caller = generate_uid()
+    for _ in range(2):
+        fetch(fabric, caller_id=caller, use_cache=False)
+    with pytest.raises(FabricQuotaExceeded):
+        fetch(fabric, caller_id=caller, use_cache=False)
+    assert broker.enqueued == 2
+
+
+def test_guests_have_no_quota_by_default(online):
+    broker = ScriptedBroker(["ok"])
+    with pytest.raises(FabricQuotaExceeded):
+        fetch(NodeFabric(broker=broker), caller_kind="guest", caller_id="g")
+    assert broker.enqueued == 0
+
+
+def test_a_cache_hit_costs_no_quota(online):
+    set_setting("nodes_daily_quota_user", "1")
+    fabric = NodeFabric(broker=ScriptedBroker(["ok"]))
+    caller = generate_uid()
+    fetch(fabric, caller_id=caller)
+    for _ in range(5):
+        assert fetch(fabric, caller_id=caller).via == "cache"
+
+
+def test_a_retryable_failure_is_retried_once_on_another_node(online):
+    broker = ScriptedBroker(["node_disconnected", "ok"])
+    result = fetch(NodeFabric(broker=broker))
+    assert result.body == b"payload" and broker.enqueued == 2
+
+
+def test_the_retry_does_not_spend_a_second_quota_unit(online):
+    set_setting("nodes_daily_quota_user", "1")
+    caller = generate_uid()
+    fetch(NodeFabric(broker=ScriptedBroker(["timeout", "ok"])), caller_id=caller)
+    assert store.caller_requests_today("user", caller) == 1
+
+
+def test_retries_stop_after_the_attempt_cap(online):
+    broker = ScriptedBroker(["timeout"] * 5)
+    with pytest.raises(FabricUnavailable) as info:
+        fetch(NodeFabric(broker=broker))
+    assert info.value.code == "timeout" and broker.enqueued == 2
+
+
+def test_a_non_retryable_failure_is_not_retried(online):
+    broker = ScriptedBroker(["bad_status", "ok"])
+    with pytest.raises(FabricUnavailable):
+        fetch(NodeFabric(broker=broker))
+    assert broker.enqueued == 1
+
+
+def test_a_full_queue_is_a_clean_unavailable(online):
+    class Full(ScriptedBroker):
+        def enqueue(self, *args):
+            raise store.QueueFull("full")
+
+    with pytest.raises(FabricUnavailable) as info:
+        fetch(NodeFabric(broker=Full([])))
+    assert info.value.code == "queue_full"
+
+
+def test_the_wait_is_capped_at_the_spec_maximum(online):
+    seen = []
+
+    class Spy(ScriptedBroker):
+        async def wait(self, uid, timeout):
+            seen.append(timeout)
+            return await super().wait(uid, timeout)
+
+    fetch(NodeFabric(broker=Spy(["ok"])), timeout=10**6)
+    assert seen[0] <= LIMITS.max_wait_seconds
+
+
+def test_the_cache_ttl_follows_the_admin_setting_up_to_the_ceiling(online):
+    cache = ResultCache()
+    fabric = NodeFabric(broker=ScriptedBroker(["ok"]), cache=cache)
+    set_setting("nodes_cache_ttl_seconds", "0")
+    fetch(fabric, caller_id="c")
+    assert len(cache) == 0
+    set_setting("nodes_cache_ttl_seconds", str(10**9))
+    fabric._sync_cache_ttl()
+    assert cache.ttl_seconds == LIMITS.cache_ttl_seconds
+    set_setting("nodes_cache_ttl_seconds", str(LIMITS.cache_ttl_seconds))
+
+
+def test_can_serve_needs_policy_switch_and_a_node(online):
+    fabric = NodeFabric(broker=ScriptedBroker([]))
+    assert fabric.can_serve(URL)
+    assert not fabric.can_serve("https://example.com/")
+    assert not fabric.can_serve(URL, "POST")
+    heartbeat(0)
+    assert not fabric.can_serve(URL)
+
+
+def test_the_real_broker_times_out_a_request_nobody_picks_up(online):
+    get_table("node_requests").delete()
+    fabric = NodeFabric()
+    with pytest.raises(FabricUnavailable) as info:
+        fetch(fabric, timeout=0.3)
+    assert info.value.code in ("caller_timeout", "failed")
+    rows = list(get_table("node_requests").find())
+    assert rows and all(row["status"] == "failed" for row in rows)
+    get_table("node_requests").delete()

tests/unit/services/nodes/ledger.py

@@ -0,0 +1,55 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_table
+from devplacepy.database import nodes as store
+from devplacepy.services.nodes import ledger as ledger_module
+from devplacepy.services.nodes.ledger import FIRST_BADGE, GIGABYTE, UsageLedger
+from devplacepy.utils import generate_uid, register_account
+
+
+def member(local_db):
+    name = "ledger" + generate_uid()[-10:]
+    uid, _, _ = register_account(name, f"{name}@t.dev", "secret123", age_band="adult", accepted_terms=True)
+    return uid
+
+
+def badges(uid):
+    return {row["badge_name"] for row in get_table("badges").find(user_uid=uid)}
+
+
+def test_first_success_awards_xp_and_the_first_badge(local_db):
+    uid = member(local_db)
+    before = get_table("users").find_one(uid=uid).get("xp") or 0
+    store.record_job(generate_uid(), uid, ok=True, bytes_up=10, bytes_down=100)
+    UsageLedger().reward(uid)
+    assert FIRST_BADGE in badges(uid)
+    assert (get_table("users").find_one(uid=uid).get("xp") or 0) > before
+
+
+def test_xp_stops_at_the_daily_cap_but_not_the_counters(local_db):
+    uid = member(local_db)
+    ledger = UsageLedger()
+    store.record_job(generate_uid(), uid, ok=True, bytes_up=1, bytes_down=1)
+    granted = 0
+    for _ in range(store.LIMITS.xp_daily_cap + 10):
+        granted += int(ledger.claim_xp(uid, ledger_module.XP_PER_JOB))
+    assert granted == store.LIMITS.xp_daily_cap
+
+
+def test_byte_badges_follow_lifetime_totals(local_db):
+    uid = member(local_db)
+    store.record_job(
+        generate_uid(), uid, ok=True, bytes_up=0, bytes_down=GIGABYTE, cap_bytes=2 * GIGABYTE
+    )
+    UsageLedger().reward(uid)
+    assert {"Gigabyte Donor"} <= badges(uid)
+    assert "10 GB Donor" not in badges(uid)
+
+
+def test_a_second_reward_call_does_not_duplicate_badges(local_db):
+    uid = member(local_db)
+    store.record_job(generate_uid(), uid, ok=True, bytes_up=1, bytes_down=1)
+    ledger = UsageLedger()
+    ledger.reward(uid)
+    ledger.reward(uid)
+    assert len(list(get_table("badges").find(user_uid=uid, badge_name=FIRST_BADGE))) == 1

tests/unit/services/nodes/policy.py

@@ -0,0 +1,143 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.nodes.policy import DestinationPolicy, PolicyError, is_public_address
+from devplacepy.services.nodes.spec import SPEC
+
+POLICY = DestinationPolicy()
+
+
+def code(url, method="GET", mode="any"):
+    with pytest.raises(PolicyError) as info:
+        POLICY.check(url, method, mode)
+    return info.value.code
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "https://reddit.com/r/python.json",
+        "https://www.reddit.com/r/python/top.json?t=week",
+        "http://old.reddit.com/",
+        "https://REDDIT.COM/",
+        "https://reddit.com./r/x",
+        "https://redd.it/abc",
+        "https://reddit.com:443/x",
+        "http://reddit.com:80/x",
+    ],
+)
+def test_allowlisted_urls_pass(url):
+    target = POLICY.check(url)
+    assert target.method == "GET" and target.adapter == "reddit"
+    assert target.port in (80, 443)
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "https://evilreddit.com/",
+        "https://reddit.com.evil.net/",
+        "https://notreddit.com/",
+        "https://example.com/",
+        "https://localhost/",
+        "https://reddit.evil.com/",
+        "https://evil.com/reddit.com",
+    ],
+)
+def test_other_hosts_are_refused(url):
+    assert code(url) == "host_not_allowed"
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "https://127.0.0.1/",
+        "https://[::1]/",
+        "http://10.0.0.5/",
+        "https://169.254.169.254/latest/meta-data",
+        "https://0x7f.1/",
+        "https://2130706433/",
+    ],
+)
+def test_address_literals_are_refused(url):
+    assert code(url) in ("host_not_allowed", "bad_url")
+
+
+@pytest.mark.parametrize("port", [22, 25, 8080, 8443, 1, 65535])
+def test_only_web_ports(port):
+    assert code(f"https://reddit.com:{port}/") == "port_not_allowed"
+
+
+@pytest.mark.parametrize(
+    "url",
+    ["ftp://reddit.com/", "file:///etc/passwd", "javascript:alert(1)", "//reddit.com/x", "reddit.com/x", ""],
+)
+def test_other_schemes_and_shapes_are_bad_urls(url):
+    assert code(url) == "bad_url"
+
+
+def test_credentials_in_the_url_are_refused():
+    assert code("https://user:pw@reddit.com/") == "bad_url"
+    assert code("https://user@reddit.com/") == "bad_url"
+    assert code("https://evil.com@reddit.com.evil.com/") == "bad_url"
+    assert code("https://reddit.com@evil.com/") == "bad_url"
+
+
+def test_overlong_and_non_string_urls_are_refused():
+    assert code("https://reddit.com/" + "a" * SPEC.limits.url_max_chars) == "bad_url"
+    with pytest.raises(PolicyError):
+        POLICY.check(None)
+    with pytest.raises(PolicyError):
+        POLICY.check(["https://reddit.com/"])
+
+
+@pytest.mark.parametrize("method", ["POST", "PUT", "DELETE", "PATCH", "CONNECT", ""])
+def test_only_declared_methods(method):
+    assert code("https://reddit.com/", method) == "method_not_allowed"
+
+
+def test_method_is_case_insensitive():
+    assert POLICY.check("https://reddit.com/", "get").method == "GET"
+
+
+def test_unknown_mode_is_refused():
+    assert code("https://reddit.com/", mode="raw") == "bad_mode"
+
+
+def test_a_redirect_off_the_allowlist_is_a_redirect_refusal():
+    with pytest.raises(PolicyError) as info:
+        POLICY.check_redirect("https://evil.example/")
+    assert info.value.code == "redirect_not_allowed"
+    assert POLICY.check_redirect("https://www.reddit.com/new").host == "www.reddit.com"
+
+
+@pytest.mark.parametrize(
+    "address,expected",
+    [
+        ("8.8.8.8", True),
+        ("1.1.1.1", True),
+        ("2606:4700:4700::1111", True),
+        ("127.0.0.1", False),
+        ("10.1.2.3", False),
+        ("172.16.0.1", False),
+        ("192.168.1.1", False),
+        ("169.254.1.1", False),
+        ("100.64.0.1", False),
+        ("100.127.255.254", False),
+        ("224.0.0.1", False),
+        ("0.0.0.0", False),
+        ("::1", False),
+        ("fe80::1", False),
+        ("fc00::1", False),
+        ("not an ip", False),
+    ],
+)
+def test_public_address_classification(address, expected):
+    assert is_public_address(address) is expected
+
+
+def test_every_destination_in_the_spec_passes_its_own_policy():
+    for destination in SPEC.destinations:
+        target = POLICY.check(f"https://{destination.suffix}/x")
+        assert target.destination is destination

tests/unit/services/nodes/scheduler.py

@@ -0,0 +1,283 @@
+# retoor <retoor@molodetz.nl>
+
+import random
+from collections import Counter
+from datetime import datetime, timezone
+
+import pytest
+
+from devplacepy.services.nodes.circuit import CircuitBreaker
+from devplacepy.services.nodes.policy import DestinationPolicy
+from devplacepy.services.nodes.registry import NodeCounters, NodeRegistry
+from devplacepy.services.nodes.scheduler import MEGABYTE, NodeScheduler, in_active_hours
+from devplacepy.services.nodes.session import Inflight, NodeSession
+from devplacepy.services.nodes.spec import SPEC
+from tests.unit.services.nodes.session import FakeSocket
+
+LIMITS = SPEC.limits
+TARGET = DestinationPolicy().check("https://www.reddit.com/r/python.json")
+
+
+class Clock:
+    def __init__(self):
+        self.now = 1000.0
+
+    def __call__(self):
+        return self.now
+
+
+def settings(**overrides):
+    base = {
+        "enabled": True,
+        "bandwidth_mb_day": 100,
+        "max_concurrent": 2,
+        "active_from": "",
+        "active_to": "",
+        "timezone": "UTC",
+        "allowed_platforms": ["reddit"],
+        "pause_on_battery": True,
+        "public_display": False,
+    }
+    base.update(overrides)
+    return base
+
+
+def node(uid="d1", user="u1", modes=("fetch",), **overrides):
+    session = NodeSession(FakeSocket(), {"uid": uid, "user_uid": user, "kind": "native"})
+    session.hello = {"modes": list(modes)}
+    session.settings = settings(**overrides)
+    session.roll_day()
+    return session
+
+
+@pytest.fixture
+def world(local_db):
+    clock = Clock()
+    registry = NodeRegistry(NodeCounters())
+    breaker = CircuitBreaker(clock=clock)
+    wall = {"now": datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)}
+    scheduler = NodeScheduler(registry, breaker, clock, lambda: wall["now"], random.Random(1))
+    scheduler.wall = wall
+    scheduler.clock = clock
+    return scheduler
+
+
+def add(world, **kwargs):
+    session = node(**kwargs)
+    world.registry.admit(session)
+    return session
+
+
+def test_a_healthy_node_is_picked(world):
+    session = add(world)
+    assert world.pick(TARGET) is session
+
+
+def test_no_nodes_means_no_pick(world):
+    assert world.pick(TARGET) is None
+
+
+def test_tunnel_only_jobs_are_not_dispatchable_yet(world):
+    add(world)
+    assert world.pick(TARGET, "tunnel") is None
+    assert world.pick(TARGET, "fetch") is not None
+    assert world.pick(TARGET, "any") is not None
+
+
+def test_a_node_must_speak_the_transport(world):
+    add(world, modes=("other",))
+    assert world.pick(TARGET) is None
+
+
+def test_a_tunnel_node_serves_tunnel_and_any_but_not_fetch(world):
+    session = add(world, modes=("tunnel",))
+    assert world.pick(TARGET, "tunnel") is session
+    assert world.pick(TARGET, "any") is session
+    assert world.pick(TARGET, "fetch") is None
+    assert world.transport_of(session, TARGET, "any") == "tunnel"
+
+
+def test_any_prefers_nodes_that_can_tunnel(world):
+    fetcher = add(world, uid="f1", user="u1", modes=("fetch",))
+    tunneler = add(world, uid="t1", user="u2", modes=("fetch", "tunnel"))
+    assert world.candidates(TARGET, "any") == [tunneler]
+    assert world.transport_of(fetcher, TARGET, "any") == "fetch"
+
+
+def test_platform_opt_out_is_honoured(world):
+    add(world, allowed_platforms=[])
+    assert world.pick(TARGET) is None
+
+
+def test_a_node_never_serves_its_own_owner(world):
+    add(world, user="alice")
+    assert world.pick(TARGET, caller_owner="alice") is None
+    assert world.pick(TARGET, caller_owner="bob") is not None
+
+
+def test_concurrency_is_capped_by_the_lower_of_server_and_donor(world):
+    session = add(world, max_concurrent=1)
+    session.inflight["j"] = Inflight("j", "h", "reddit.com", "GET", "u", 0, 0)
+    assert world.pick(TARGET) is None
+    session.inflight.clear()
+    assert session.max_concurrent == 1
+    session.settings["max_concurrent"] = 99
+    assert session.max_concurrent == LIMITS.max_concurrent_per_node
+    session.hello["limits"] = {"max_concurrent": 1}
+    assert session.max_concurrent == 1
+
+
+def test_the_donor_bandwidth_cap_stops_assignment(world):
+    session = add(world, bandwidth_mb_day=1)
+    session.used_today = MEGABYTE - 1
+    assert world.pick(TARGET) is session
+    session.used_today = MEGABYTE
+    assert world.pick(TARGET) is None
+
+
+def test_a_new_day_resets_the_used_bytes(world):
+    session = add(world, bandwidth_mb_day=1)
+    session.used_today = MEGABYTE
+    session.used_day = "2000-01-01"
+    assert world.pick(TARGET) is session
+    assert session.used_today == 0
+
+
+def test_a_node_waits_between_requests_to_the_same_destination(world):
+    session = add(world)
+    world.mark_started(session, TARGET)
+    world.clock.now += LIMITS.min_seconds_per_node_domain - 0.1
+    assert world.eligible(TARGET) == []
+    world.clock.now += 0.2
+    world.clock.now += 1.0 / TARGET.destination.max_rps
+    assert world.eligible(TARGET) == [session]
+
+
+def test_the_destination_has_a_global_rate_ceiling(world):
+    first, second = add(world, uid="a", user="ua"), add(world, uid="b", user="ub")
+    world.mark_started(first, TARGET)
+    world.clock.now += 1.0 / TARGET.destination.max_rps - 0.01
+    assert world.pick(TARGET) is None
+    world.clock.now += 0.02
+    assert world.pick(TARGET) in (first, second)
+
+
+def test_a_paused_node_is_skipped_until_the_pause_ends(world):
+    session = add(world)
+    session.paused_until = world.clock.now + 5
+    assert world.pick(TARGET) is None
+    world.clock.now += 5
+    assert world.pick(TARGET) is session
+
+
+def test_an_open_node_circuit_removes_only_that_node(world):
+    bad, good = add(world, uid="bad", user="u1"), add(world, uid="good", user="u2")
+    for _ in range(LIMITS.breaker_failures):
+        world.breaker.failure(("node", "bad"))
+    assert world.eligible(TARGET) == [good]
+    world.clock.now += LIMITS.breaker_open_seconds
+    assert set(world.eligible(TARGET)) == {bad, good}
+
+
+def test_an_open_destination_circuit_stops_everything(world):
+    add(world)
+    for _ in range(LIMITS.breaker_failures):
+        world.breaker.failure(("destination", TARGET.destination.suffix))
+    assert world.pick(TARGET) is None
+
+
+def test_least_loaded_then_least_recently_used_wins(world):
+    busy, idle_old, idle_new = add(world, uid="a", user="1"), add(world, uid="b", user="2"), add(world, uid="c", user="3")
+    busy.inflight["j"] = Inflight("j", "h", "reddit.com", "GET", "u", 0, 0)
+    idle_old.last_assigned = 10
+    idle_new.last_assigned = 20
+    assert world.pick(TARGET) is idle_old
+
+
+def test_selection_is_fair_across_equal_nodes(world):
+    sessions = [add(world, uid=f"d{i}", user=f"u{i}") for i in range(5)]
+    picks = Counter()
+    for _ in range(500):
+        chosen = world.pick(TARGET)
+        picks[chosen.device_uid] += 1
+        chosen.last_assigned = world.clock.now
+        world.clock.now += 0.001
+    assert set(picks) == {item.device_uid for item in sessions}
+    assert max(picks.values()) - min(picks.values()) <= 1
+
+
+@pytest.mark.parametrize(
+    "start,end,hour,minute,expected",
+    [
+        ("", "", 3, 0, True),
+        ("09:00", "17:00", 9, 0, True),
+        ("09:00", "17:00", 16, 59, True),
+        ("09:00", "17:00", 17, 0, False),
+        ("09:00", "17:00", 8, 59, False),
+        ("22:00", "06:00", 23, 0, True),
+        ("22:00", "06:00", 5, 59, True),
+        ("22:00", "06:00", 6, 0, False),
+        ("22:00", "06:00", 12, 0, False),
+        ("09:00", "09:00", 3, 0, True),
+        ("bogus", "17:00", 3, 0, True),
+    ],
+)
+def test_active_hours_windows(start, end, hour, minute, expected):
+    moment = datetime(2026, 1, 1, hour, minute, tzinfo=timezone.utc)
+    assert in_active_hours({"active_from": start, "active_to": end, "timezone": "UTC"}, moment) is expected
+
+
+def test_active_hours_use_the_donors_timezone():
+    moment = datetime(2026, 1, 1, 22, 0, tzinfo=timezone.utc)
+    amsterdam = {"active_from": "09:00", "active_to": "17:00", "timezone": "Europe/Amsterdam"}
+    assert in_active_hours(amsterdam, moment) is False
+    morning = datetime(2026, 1, 1, 8, 30, tzinfo=timezone.utc)
+    assert in_active_hours(amsterdam, morning) is True
+    broken = {"active_from": "09:00", "active_to": "17:00", "timezone": "Mars/Base"}
+    assert in_active_hours(broken, datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) is True
+
+
+def test_a_node_outside_its_hours_is_skipped(world):
+    add(world, active_from="22:00", active_to="23:00")
+    assert world.pick(TARGET) is None
+    world.wall["now"] = datetime(2026, 1, 1, 22, 30, tzinfo=timezone.utc)
+    assert world.pick(TARGET) is not None
+
+
+def test_mark_started_records_both_clocks(world):
+    session = add(world)
+    world.mark_started(session, TARGET)
+    assert session.host_started[TARGET.destination.suffix] == world.clock.now
+    assert session.last_assigned == world.clock.now
+    assert world._destination_started[TARGET.destination.suffix] == world.clock.now
+
+
+def test_the_per_destination_maps_stay_bounded_by_the_allowlist(world):
+    session = add(world)
+    policy = DestinationPolicy()
+    for index in range(200):
+        world.mark_started(session, policy.check(f"https://sub{index}.reddit.com/"))
+    assert len(session.host_started) == 1
+    assert len(world._destination_started) == 1
+
+
+def test_window_checks_are_cached_per_minute_and_still_correct(world):
+    from devplacepy.services.nodes.scheduler import _window_open
+
+    _window_open.cache_clear()
+    window = {"active_from": "09:00", "active_to": "17:00", "timezone": "UTC"}
+    noon = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc)
+    for _ in range(100):
+        assert in_active_hours(window, noon) is True
+    assert _window_open.cache_info().misses == 1
+    night = datetime(2026, 1, 1, 3, 0, tzinfo=timezone.utc)
+    assert in_active_hours(window, night) is False
+
+
+def test_choose_applies_the_owner_exclusion_to_a_shared_candidate_list(world):
+    mine, theirs = add(world, uid="a", user="alice"), add(world, uid="b", user="bob")
+    found = world.candidates(TARGET)
+    assert world.choose(found, "alice") is theirs
+    assert world.choose(found, "bob") is mine
+    assert world.choose([mine], "alice") is None
+    assert world.choose([], "") is None

tests/unit/services/nodes/service.py

@@ -1,5 +1,6 @@
 # retoor <retoor@molodetz.nl>
 
+import asyncio
 import time
 
 import pytest
@@ -146,3 +147,32 @@ def test_metrics_report_connected_and_capacity(service):
 def test_the_service_has_the_kill_switch_as_a_config_field(service):
     keys = [field.key for field in service.all_fields()]
     assert "nodes_enabled" in keys and "nodes_max_total" in keys
+
+
+def test_changed_settings_are_pushed_to_the_node_once(service):
+    uid = member()
+    device, socket = connected(service, uid)
+    session = service.registry.get(device["uid"])
+    session.settings = {"stale": True}
+    assert reconcile(service) == 0
+    pushed = [frame for frame in socket.sent if frame["type"] == "settings_update"]
+    assert len(pushed) == 1 and pushed[0]["settings"]["enabled"] is True
+    assert session.settings == pushed[0]["settings"]
+    reconcile(service)
+    assert len([frame for frame in socket.sent if frame["type"] == "settings_update"]) == 1
+    save_node_settings(uid, {"bandwidth_mb_day": 9})
+    reconcile(service)
+    assert session.settings["bandwidth_mb_day"] == 9
+
+
+def test_the_dispatch_loop_starts_once_and_stops_with_the_service(service):
+    async def scenario():
+        service.ensure_dispatching()
+        first = service._dispatch_task
+        service.ensure_dispatching()
+        assert service._dispatch_task is first and not first.done()
+        await service.on_disable()
+        await asyncio.sleep(0)
+        assert service._dispatch_task is None and first.cancelled()
+
+    run_async(scenario())

tests/unit/services/nodes/session.py

@@ -93,15 +93,20 @@ def test_a_silent_client_times_out(local_db, monkeypatch):
 
 def test_usage_is_advisory_and_summed():
     session = session_for(FakeSocket())
-    session._handle({"type": "usage", "bytes_up": 5, "bytes_down": 7})
-    session._handle({"type": "usage", "bytes_up": -9, "bytes_down": 1})
+    run(session._handle({"type": "usage", "bytes_up": 5, "bytes_down": 7}))
+    run(session._handle({"type": "usage", "bytes_up": -9, "bytes_down": 1}))
     assert (session.reported_up, session.reported_down) == (5, 8)
 
 
-@pytest.mark.parametrize("kind", ["job_result", "job_error", "tunnel_data", "tunnel_close", "tunnel_opened"])
+@pytest.mark.parametrize("kind", ["job_result", "job_error"])
 def test_frames_without_an_assigned_job_are_protocol_errors(kind):
     with pytest.raises(FrameError):
-        session_for(FakeSocket())._handle({"type": kind})
+        run(session_for(FakeSocket())._handle({"type": kind, "job_id": "nope"}))
+
+
+@pytest.mark.parametrize("kind", ["tunnel_data", "tunnel_close", "tunnel_opened"])
+def test_late_tunnel_frames_for_a_closed_relay_are_ignored(kind):
+    run(session_for(FakeSocket())._handle({"type": kind, "conn_id": "gone"}))
 
 
 def test_missed_pings_evict_after_the_floor(monkeypatch):

tests/unit/services/nodes/social.py

@@ -0,0 +1,88 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+from devplacepy.database import set_setting
+from devplacepy.services.nodes import social as social_module
+from devplacepy.services.nodes.adapters.base import AdapterError
+from devplacepy.services.nodes.fabric import FabricUnavailable
+from devplacepy.services.nodes.social import SocialSearch
+from tests.conftest import run_async
+
+HN = json.dumps({"hits": [{"objectID": "1", "title": "HN story", "points": 1}]}).encode()
+SE = json.dumps({"items": [{"title": "SO question", "link": "https://stackoverflow.com/q/1"}]}).encode()
+
+
+class StubFabric:
+    def __init__(self, error=None):
+        self.error = error
+        self.calls = 0
+
+    async def fetch(self, url, **kwargs):
+        self.calls += 1
+        if self.error:
+            raise self.error
+        raise AssertionError("not expected")
+
+
+def search(monkeypatch, getter, platforms, fabric=None):
+    monkeypatch.setattr(social_module, "direct_get", getter)
+    return run_async(
+        SocialSearch(fabric or StubFabric()).search(
+            "sqlite", caller_kind="user", caller_id="u1", platforms=platforms
+        )
+    )
+
+
+def test_hits_are_merged_and_failures_reported(monkeypatch):
+    async def getter(url):
+        if "algolia" in url:
+            return HN
+        if "stackexchange" in url:
+            return SE
+        raise AdapterError("upstream", "HTTP 500")
+
+    outcome = search(monkeypatch, getter, ["hackernews", "stackexchange", "github"])
+    assert {item.platform for item in outcome.hits} == {"hackernews", "stackexchange"}
+    assert outcome.unavailable == {"github": "upstream"}
+
+
+def test_unknown_platform_is_dropped(monkeypatch):
+    async def getter(url):
+        return HN
+
+    outcome = search(monkeypatch, getter, ["hackernews", "x", "instagram"])
+    assert [item.platform for item in outcome.hits] == ["hackernews"]
+    assert outcome.unavailable == {}
+
+
+def test_reddit_is_off_unless_routing_through_nodes_is_enabled(monkeypatch):
+    async def getter(url):
+        return HN
+
+    fabric = StubFabric()
+    outcome = search(monkeypatch, getter, ["reddit"], fabric)
+    assert outcome.unavailable == {"reddit": "nodes_not_used"}
+    assert fabric.calls == 0
+
+
+def test_reddit_reports_the_fabric_reason(monkeypatch):
+    async def getter(url):
+        return HN
+
+    set_setting("nodes_social_via_nodes", "1")
+    try:
+        outcome = search(monkeypatch, getter, ["reddit"], StubFabric(FabricUnavailable("no_nodes")))
+    finally:
+        set_setting("nodes_social_via_nodes", "0")
+    assert outcome.unavailable == {"reddit": "no_nodes"}
+
+
+def test_blank_query_and_limit_cap(monkeypatch):
+    async def getter(url):
+        return HN
+
+    monkeypatch.setattr(social_module, "direct_get", getter)
+    empty = run_async(SocialSearch(StubFabric()).search("  ", caller_kind="user", caller_id="u"))
+    assert empty.hits == []
+    assert social_module.MAX_LIMIT == 20

tests/unit/services/nodes/tunnel.py

@@ -0,0 +1,112 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+import base64
+
+from devplacepy.services.nodes import tunnel as tunnel_module
+from devplacepy.services.nodes.policy import DestinationPolicy
+from devplacepy.services.nodes.tunnel import Relay, TunnelBridge
+from tests.conftest import run_async
+
+TARGET = DestinationPolicy().check("https://www.reddit.com/r/python.json")
+
+
+class Donor:
+    def __init__(self, bridge, accept=True, garbage=False):
+        self.bridge = bridge
+        self.accept = accept
+        self.garbage = garbage
+        self.tunnels = {}
+        self.sent = []
+
+    async def send(self, frame_type, **fields):
+        self.sent.append({"type": frame_type, **fields})
+        if frame_type == "tunnel_open":
+            self.bridge.on_frame(self, {"type": "tunnel_opened", "conn_id": fields["conn_id"], "ok": self.accept})
+        elif frame_type == "tunnel_data":
+            raw = base64.b64decode(fields["b64"])
+            reply = "***not base64***" if self.garbage else base64.b64encode(raw.upper()).decode()
+            self.bridge.on_frame(self, {"type": "tunnel_data", "conn_id": fields["conn_id"], "b64": reply})
+
+
+async def session_through(donor, connect_line, payload=b"hello"):
+    relay = Relay(donor, TARGET)
+    server = await asyncio.start_server(relay.handle, "127.0.0.1", 0)
+    port = server.sockets[0].getsockname()[1]
+    reader, writer = await asyncio.open_connection("127.0.0.1", port)
+    writer.write(connect_line.encode() + b"\r\n\r\n")
+    await writer.drain()
+    status = await asyncio.wait_for(reader.readline(), 5)
+    echoed = b""
+    if b" 200 " in status:
+        await reader.readuntil(b"\r\n")
+        writer.write(payload)
+        await writer.drain()
+        try:
+            echoed = await asyncio.wait_for(reader.read(len(payload)), 5)
+        except asyncio.TimeoutError:
+            echoed = b""
+    writer.close()
+    server.close()
+    await asyncio.sleep(0.05)
+    return status, echoed, relay
+
+
+def go(donor, line, payload=b"hello"):
+    return run_async(session_through(donor, line, payload))
+
+
+def test_connect_to_the_target_relays_both_ways():
+    donor = Donor(TunnelBridge())
+    status, echoed, relay = go(donor, "CONNECT www.reddit.com:443 HTTP/1.1")
+    assert b" 200 " in status and echoed == b"HELLO"
+    assert [item["type"] for item in donor.sent][:2] == ["tunnel_open", "tunnel_data"]
+    assert donor.sent[0]["host"] == "www.reddit.com" and donor.sent[0]["port"] == 443
+    assert relay.up == 5 and relay.down == 5
+
+
+def test_any_other_host_or_port_is_refused_without_bothering_the_donor():
+    for line in ("CONNECT evil.example:443 HTTP/1.1", "CONNECT www.reddit.com:22 HTTP/1.1", "CONNECT 10.0.0.1:443 HTTP/1.1"):
+        donor = Donor(TunnelBridge())
+        status, _, relay = go(donor, line)
+        assert b" 403 " in status and donor.sent == []
+        assert relay.error == "host_not_allowed"
+
+
+def test_only_connect_is_accepted():
+    donor = Donor(TunnelBridge())
+    status, _, _ = go(donor, "GET http://www.reddit.com/ HTTP/1.1")
+    assert b" 405 " in status and donor.sent == []
+
+
+def test_a_donor_that_refuses_is_a_bad_gateway():
+    donor = Donor(TunnelBridge(), accept=False)
+    status, _, relay = go(donor, "CONNECT www.reddit.com:443 HTTP/1.1")
+    assert b" 502 " in status and relay.error == "tunnel_refused"
+
+
+def test_the_relay_has_a_byte_cap(monkeypatch):
+    monkeypatch.setattr(tunnel_module, "RELAY_CAP_BYTES", 8)
+    donor = Donor(TunnelBridge())
+    _, echoed, relay = go(donor, "CONNECT www.reddit.com:443 HTTP/1.1", b"0123456789abcdef")
+    assert relay.error == "body_too_large" and len(echoed) < 16
+
+
+def test_invalid_base64_from_the_donor_ends_the_tunnel():
+    donor = Donor(TunnelBridge(), garbage=True)
+    _, echoed, relay = go(donor, "CONNECT www.reddit.com:443 HTTP/1.1")
+    assert relay.error == "bad_base64" and echoed == b""
+
+
+def test_releasing_a_session_closes_its_relays():
+    bridge = TunnelBridge()
+    donor = Donor(bridge)
+    relay = Relay(donor, TARGET)
+    donor.tunnels[relay.conn_id] = relay
+    bridge.release_session(donor)
+    assert donor.tunnels == {} and relay.frames.get_nowait() is tunnel_module.CLOSED
+
+
+def test_frames_for_unknown_connections_are_not_consumed():
+    bridge = TunnelBridge()
+    assert bridge.on_frame(Donor(bridge), {"type": "tunnel_data", "conn_id": "nope", "b64": ""}) is False

tests/unit/services/sync/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/unit/services/sync/alias.py

@@ -0,0 +1,273 @@
+# retoor <retoor@molodetz.nl>
+
+import random
+from collections import Counter
+
+import pytest
+
+from devplacepy.database import get_table, init_db
+from devplacepy.services.sync import ai
+from devplacepy.services.sync.aliaser import AliasRefused, Aliaser
+from devplacepy.services.sync.identity import IdentityResolver, NameGenerator, fallback_name, source_key
+from devplacepy.services.sync.options import SyncOptions
+from devplacepy.services.sync.rewriter import MentionRewriter
+from devplacepy.services.sync.selection import Selector, bucket, jittered
+from devplacepy.services.sync.style import Style, plausible
+from devplacepy.services.sync.translator import Translator
+from tests.conftest import run_async
+
+ORIGIN = "alias.test"
+
+
+@pytest.fixture(autouse=True)
+def clean(monkeypatch):
+    init_db()
+    for table in ("sync_identities", "sync_links"):
+        get_table(table).delete(origin=ORIGIN)
+
+    async def unreachable(system, text, timeout):
+        return None
+
+    monkeypatch.setattr(ai, "complete", unreachable)
+    yield
+    for table in ("sync_identities", "sync_links"):
+        get_table(table).delete(origin=ORIGIN)
+
+
+def test_rewriter_longest_first_and_single_pass():
+    rewriter = MentionRewriter({"alice": "bob", "alice_w": "carol", "bob": "dave"})
+    result = rewriter.rewrite("hi @alice_w and alice, also @bob")
+    assert result.text == "hi @carol and bob, also @dave"
+    assert len(result.substitutions) == 3
+
+
+def test_rewriter_protects_code_urls_and_inline_code():
+    rewriter = MentionRewriter({"alice": "zed"})
+    text = "see `alice` and https://x.test/~alice\n```\nalice = 1\n```\nthen alice"
+    assert rewriter.rewrite(text).text == text.rsplit("alice", 1)[0] + "zed"
+    assert rewriter.rewrite("```\nalice unterminated").text == "```\nalice unterminated"
+
+
+def test_rewriter_standalone_floor_and_case_and_boundaries():
+    rewriter = MentionRewriter({"ann": "x1", "tef": "y2", "markus": "zeta"})
+    assert rewriter.rewrite("ann do, @ann, @tef, tef, Markus, markusx, a-markus").text == "ann do, @x1, @y2, tef, zeta, markusx, a-markus"
+
+
+def test_rewriter_mentioned_skips_code():
+    found = MentionRewriter({}).mentioned("hi @Alice `@code` @ab ```\n@fenced\n``` @alice_2")
+    assert found == ["Alice", "alice_2"]
+
+
+def test_style_roundtrip_and_matching():
+    style = Style.of("Alice_99")
+    assert Style.load(style.dump()) == style
+    assert style.matches("Marta_12") and not style.matches("marta_12") and not style.matches("Marta12")
+    assert not plausible("alice_99x", "Alice_99", None)
+    assert not plausible("robotuser", "zzzz", None)
+    assert not plausible("a!", "zzzz", None)
+
+
+def test_fallback_name_is_deterministic_and_style_matched():
+    for source in ("alice", "Bob_Smith", "dev99", "UPPER", "kebab-case"):
+        style = Style.of(source)
+        assert fallback_name(ORIGIN, source, style, 0) == fallback_name(ORIGIN, source, style, 0)
+        assert any(plausible(fallback_name(ORIGIN, source, style, n), source, style) for n in range(60))
+
+
+def test_generator_rejects_bad_ai_then_falls_back(monkeypatch):
+    replies = iter(['{"username": "alice"}', "garbage", '{"username": "x"}'])
+
+    async def scripted(system, text, timeout):
+        return next(replies)
+
+    monkeypatch.setattr(ai, "complete", scripted)
+    name, by_ai = run_async(NameGenerator().generate(ORIGIN, "alice", Style.of("alice"), []))
+    assert not by_ai and plausible(name, "alice", Style.of("alice"))
+
+
+def test_generator_accepts_good_ai(monkeypatch):
+    async def good(system, text, timeout):
+        assert "alice" not in text
+        return '```json\n{"username": "marta"}\n```'
+
+    monkeypatch.setattr(ai, "complete", good)
+    assert run_async(NameGenerator().generate(ORIGIN, "alice", Style.of("alice"), [])) == ("marta", True)
+
+
+def test_identity_is_permanent_unique_and_bidirectional():
+    first = IdentityResolver(ORIGIN)
+    names = [run_async(first.ensure(n)) for n in ("alice", "Alice", "bobby", "carol")]
+    assert names[0] == names[1]
+    assert len({name.lower() for name in names[:1] + names[2:]}) == 3
+    second = IdentityResolver(ORIGIN)
+    assert run_async(second.ensure("ALICE")) == names[0]
+    assert second.created == 0
+    assert second.mapping()["alice"] == names[0]
+    rewriter = MentionRewriter({value: key for key, value in second.mapping().items()})
+    assert rewriter.rewrite(f"@{names[0]}").text == "@alice"
+
+
+def test_identity_cap_and_invalid_names_fail_closed():
+    resolver = IdentityResolver(ORIGIN, cap=1)
+    assert run_async(resolver.ensure("alice"))
+    assert run_async(resolver.ensure("bobby")) is None
+    assert run_async(resolver.ensure("not valid!")) is None
+    assert source_key("x" * 100) is None
+
+
+def test_dry_run_resolver_persists_nothing():
+    resolver = IdentityResolver(ORIGIN, persist=False)
+    assert run_async(resolver.ensure("alice"))
+    assert get_table("sync_identities").count(origin=ORIGIN) == 0
+
+
+def test_samples_are_capped():
+    resolver = IdentityResolver(ORIGIN)
+    run_async(resolver.ensure("alice"))
+    for n in range(6):
+        resolver.remember("alice", f"sample number {n}")
+    assert len(resolver.samples("alice")) == 3
+    assert len(IdentityResolver(ORIGIN).samples("alice")) == 3
+
+
+def records():
+    users = [
+        {"origin_uid": "1", "username": "alice", "bio": "secret bio", "website": "https://alice.test", "created_at": "2025-01-01T00:00:00Z"},
+        {"origin_uid": "2", "username": "bobby", "created_at": "2025-01-02T00:00:00Z"},
+    ]
+    posts = [
+        {"origin_uid": "p1", "author_uid": "1", "title": "Hello from alice", "content": "ping @bobby and @ghostly please", "created_at": "2025-02-01T00:00:00Z", "topic": "devlog"},
+        {"origin_uid": "p2", "author_uid": "9", "title": "orphan", "content": "no known author here", "created_at": "2025-02-02T00:00:00Z"},
+    ]
+    comments = [
+        {"origin_uid": "c1", "author_uid": "2", "post_uid": "p1", "content": "thanks alice", "created_at": "2025-02-03T00:00:00Z"},
+        {"origin_uid": "c2", "author_uid": "2", "post_uid": "p1", "parent_uid": "cX", "content": "reply to missing", "created_at": "2025-02-03T00:00:00Z"},
+    ]
+    return {"users": users, "posts": posts, "comments": comments}
+
+
+def leaked(outcome, names=("alice", "bobby")):
+    blob = " ".join(
+        str(item.get(name) or "")
+        for kind in ("users", "posts", "comments")
+        for item in outcome.records[kind]
+        for name in ("username", "title", "content", "bio", "website")
+    ).lower()
+    return [name for name in names if name in blob]
+
+
+def test_aliaser_rewrites_everything_and_fails_closed():
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records()))
+    assert leaked(outcome) == []
+    assert len(outcome.records["posts"]) == 1
+    assert outcome.skipped["no_author"] == 1
+    assert [c["origin_uid"] for c in outcome.records["comments"]] == ["c1"]
+    assert outcome.skipped["parent_skipped"] == 1
+    users = outcome.records["users"]
+    assert len(users) == 3
+    assert all(set(user) == {"origin_uid", "username", "created_at", "deleted"} for user in users)
+    assert {user["origin_uid"] for user in users} == {"alice", "bobby", "ghostly"}
+    assert outcome.records["posts"][0]["author_uid"] == "alice"
+    assert outcome.substitutions >= 4
+
+
+def test_aliaser_is_stable_across_runs():
+    first = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records()))
+    second = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records()))
+    assert first.records == second.records
+    assert second.identities_created == 0
+
+
+def test_aliaser_refuses_origin_with_original_links():
+    get_table("sync_links").insert({"uid": "l1", "origin": ORIGIN, "kind": "users", "origin_uid": "1", "local_uid": "x", "fingerprint": "f", "origin_updated_at": "", "first_synced_at": "", "synced_at": ""})
+    with pytest.raises(AliasRefused):
+        run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records()))
+
+
+def test_aliaser_cap_skips_item_as_transient(monkeypatch):
+    resolver = IdentityResolver(ORIGIN, cap=1)
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True), resolver=resolver).apply(records()))
+    assert outcome.transient() >= 1
+    assert leaked(outcome) == []
+
+
+class FakeTranslator(Translator):
+    def __init__(self, reply):
+        super().__init__("nl")
+        self.reply = reply
+
+    async def translate(self, text, samples=None):
+        return self.reply(text) if self.reply else None
+
+
+def test_translation_failure_skips_item_and_holds():
+    translator = FakeTranslator(None)
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True, translate_to="nl"), language="en", translator=translator).apply(records()))
+    assert outcome.records["posts"] == [] and outcome.skipped["translation_failed"] == 1
+    assert outcome.transient() >= 1
+
+
+def test_translation_output_is_rewritten_again():
+    translator = FakeTranslator(lambda text: text + " (met @alice)")
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True, translate_to="nl"), language="en", translator=translator).apply(records()))
+    assert leaked(outcome) == []
+    assert outcome.translated >= 1
+
+
+def test_translation_skipped_when_language_matches():
+    assert not Translator("en").needed("en-US")
+    assert Translator("en").needed("") and Translator("nl").needed("en")
+
+
+def test_options_validation_and_roundtrip():
+    options = SyncOptions.build(alias=True, translate_to="NL", sample_percent=40, min_comments=2, topics="devlog, rant", jitter_percent=10)
+    assert SyncOptions.from_row({"sync_options": options.to_dict()}) == options
+    assert not SyncOptions().active()
+    for bad in ({"sample_percent": 0}, {"jitter_percent": 99}, {"topics": "nonsense"}, {"translate_to": "x y"}, {"min_comments": -1}):
+        with pytest.raises(ValueError):
+            SyncOptions.build(**bad)
+    assert options.merged(alias=False).alias is False and options.merged(alias=None).alias is True
+
+
+def test_selector_deterministic_sampling_and_filters():
+    posts = [{"origin_uid": f"p{n}", "author_uid": "1", "topic": "devlog"} for n in range(400)]
+    comments = [{"origin_uid": "c", "author_uid": "1", "post_uid": "p0"}]
+    data = {"users": [], "posts": posts, "comments": comments}
+    options = SyncOptions(sample_percent=50)
+    kept, skipped = Selector(ORIGIN, options).select(data)
+    again, _ = Selector(ORIGIN, options).select(data)
+    assert kept["posts"] == again["posts"]
+    assert 150 < len(kept["posts"]) < 250
+    assert skipped["sampled_out"] == 400 - len(kept["posts"])
+    assert bucket(ORIGIN, "p0") == bucket(ORIGIN, "p0")
+    kept, skipped = Selector(ORIGIN, SyncOptions(topics=("rant",))).select(data)
+    assert kept["posts"] == [] and kept["comments"] == [] and skipped["topic_not_allowed"] == 400
+    kept, skipped = Selector(ORIGIN, SyncOptions(min_comments=1)).select(data)
+    assert [post["origin_uid"] for post in kept["posts"]] == ["p0"]
+    untouched, none = Selector(ORIGIN, SyncOptions()).select(data)
+    assert untouched is data and not none
+
+
+def test_selector_never_drops_already_synced_posts():
+    get_table("sync_links").insert({"uid": "l2", "origin": ORIGIN, "kind": "posts", "origin_uid": "p0", "local_uid": "x", "fingerprint": "f", "origin_updated_at": "", "first_synced_at": "", "synced_at": ""})
+    data = {"users": [], "posts": [{"origin_uid": "p0", "author_uid": "1", "topic": "devlog"}], "comments": []}
+    kept, _ = Selector(ORIGIN, SyncOptions(topics=("rant",))).select(data)
+    assert len(kept["posts"]) == 1
+
+
+def test_jitter_is_bounded():
+    rng = random.Random(1)
+    values = [jittered(1000, 20, rng) for _ in range(500)]
+    assert min(values) >= 800 and max(values) <= 1200 and len(set(values)) > 50
+    assert jittered(1000, 0) == 1000
+    assert Counter() == Counter()
+
+
+def test_mentions_are_case_insensitive_end_to_end():
+    rewriter = MentionRewriter({"Alice": "zed"})
+    assert rewriter.rewrite("@ALICE @alice @AlIcE and ALICE").text == "@zed @zed @zed and zed"
+    data = records()
+    data["posts"][0]["content"] = "ping @BOBBY and @Bobby and @bObBy"
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(data))
+    assert leaked(outcome) == []
+    assert len(outcome.records["users"]) == 2 + 0 or {u["origin_uid"] for u in outcome.records["users"]} >= {"alice", "bobby"}

tests/unit/services/sync/normalize.py

@@ -0,0 +1,97 @@
+# retoor <retoor@molodetz.nl>
+
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+from devplacepy.services.sync.normalize import (
+    clamp_instant,
+    clean_text,
+    fingerprint,
+    instant_iso,
+    normalize_origin,
+    normalize_origin_uid,
+    normalize_username,
+    parse_instant,
+    username_candidates,
+)
+from devplacepy.services.sync.spec import LIMITS
+
+
+@pytest.mark.parametrize(
+    "raw,expected",
+    [
+        ("Other.Example.ORG", "other.example.org"),
+        ("  a1  ", "a1"),
+        ("devplace-net", "devplace-net"),
+    ],
+)
+def test_origin_is_normalized(raw, expected):
+    assert normalize_origin(raw) == expected
+
+
+@pytest.mark.parametrize("raw", ["", "a", "-bad", "bad-", "a..b", "has space", "x" * 64, "ünï.org"])
+def test_origin_is_rejected(raw):
+    with pytest.raises(ValueError):
+        normalize_origin(raw)
+
+
+@pytest.mark.parametrize("raw", ["", "  ", "a b", "x" * 65, "a/b"])
+def test_origin_uid_is_rejected(raw):
+    with pytest.raises(ValueError):
+        normalize_origin_uid(raw)
+
+
+def test_origin_uid_accepts_integers_and_uuids():
+    assert normalize_origin_uid(42) == "42"
+    assert normalize_origin_uid("01a139e9-a2f0-7fa1-8635-1a98fbf95e3d")
+
+
+def test_clean_text_strips_control_characters_and_newlines():
+    assert clean_text("a\x00b\r\nc\rd\x07", multiline=True) == "ab\nc\nd"
+    assert clean_text("a \n  b\t c", multiline=False) == "a b c"
+
+
+def test_username_always_valid_across_inputs():
+    samples = ["", "a", "ab", "Ünï cøde!", "x" * 80, "--__--", "john.doe@mail", "日本語", "ok_name-1"]
+    for sample in samples:
+        name = normalize_username(sample, "uid-1")
+        assert LIMITS.min_username_length <= len(name) <= LIMITS.max_username_length
+        assert all(ch.isalnum() or ch in "_-" for ch in name) and name.isascii()
+
+
+def test_username_is_stable_for_same_input():
+    assert normalize_username("a", "u1") == normalize_username("a", "u1")
+    assert normalize_username("a", "u1") != normalize_username("a", "u2")
+
+
+def test_username_candidates_are_distinct_and_bounded():
+    seen = []
+    for candidate in username_candidates("x" * 32, "other.example.org"):
+        seen.append(candidate)
+        assert len(candidate) <= LIMITS.max_username_length
+        if len(seen) == 50:
+            break
+    assert len(set(seen)) == 50
+    assert seen[1].endswith("-other")
+
+
+def test_instants_are_utc_iso_and_clamped():
+    assert instant_iso("2025-01-01T01:00:00+01:00") == "2025-01-01T00:00:00+00:00"
+    assert instant_iso("2025-01-01").startswith("2025-01-01T00:00:00")
+    assert parse_instant(1700000000).year == 2023
+    now = datetime.now(timezone.utc)
+    future = now + timedelta(days=30)
+    assert clamp_instant(future, now) == now
+    assert instant_iso("1970-01-01").startswith("2000-01-01")
+
+
+@pytest.mark.parametrize("raw", ["", None, "nope", True, "2025-13-45"])
+def test_instant_rejects_garbage(raw):
+    with pytest.raises(ValueError):
+        parse_instant(raw)
+
+
+def test_fingerprint_ignores_key_order_and_detects_change():
+    assert fingerprint({"a": 1, "b": 2}) == fingerprint({"b": 2, "a": 1})
+    assert fingerprint({"a": 1}) != fingerprint({"a": 2})

tests/unit/services/sync/records.py

@@ -0,0 +1,85 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+from pydantic import ValidationError
+
+from devplacepy.services.sync.records import (
+    CommentRecord,
+    PostRecord,
+    SyncBatchForm,
+    UserRecord,
+)
+from devplacepy.services.sync.spec import LIMITS
+
+BASE = {"origin_uid": "r1", "created_at": "2025-01-01T00:00:00Z"}
+
+
+def test_updated_at_defaults_to_created_and_never_precedes_it():
+    record = PostRecord.model_validate(
+        {**BASE, "author_uid": "u", "content": "ten chars!!"}
+    )
+    assert record.updated_at == record.created_at
+    earlier = PostRecord.model_validate(
+        {**BASE, "author_uid": "u", "content": "ten chars!!", "updated_at": "2024-01-01"}
+    )
+    assert earlier.updated_at == earlier.created_at
+
+
+def test_unknown_topic_becomes_random_and_known_is_kept():
+    base = {**BASE, "author_uid": "u", "content": "ten chars!!"}
+    assert PostRecord.model_validate({**base, "topic": "zzz"}).topic == "random"
+    assert PostRecord.model_validate({**base, "topic": "DevLog"}).topic == "devlog"
+
+
+@pytest.mark.parametrize(
+    "payload",
+    [
+        {"author_uid": "u", "content": "short"},
+        {"author_uid": "u", "content": "x" * (LIMITS.max_post_length + 1)},
+        {"author_uid": "", "content": "ten chars!!"},
+        {"content": "ten chars!!"},
+        {"author_uid": "u", "content": "ten chars!!", "title": "t" * (LIMITS.max_title_length + 1)},
+    ],
+)
+def test_invalid_posts_are_rejected(payload):
+    with pytest.raises(ValidationError):
+        PostRecord.model_validate({**BASE, **payload})
+
+
+def test_comment_parent_is_optional_and_post_is_required():
+    base = {**BASE, "author_uid": "u", "content": "fine"}
+    assert CommentRecord.model_validate({**base, "post_uid": "p"}).parent_uid is None
+    assert CommentRecord.model_validate({**base, "post_uid": "p", "parent_uid": ""}).parent_uid is None
+    with pytest.raises(ValidationError):
+        CommentRecord.model_validate(base)
+
+
+def test_user_urls_are_fixed_or_dropped():
+    record = UserRecord.model_validate(
+        {**BASE, "username": "bob", "website": "example.com", "git_link": "not a url"}
+    )
+    assert record.website == "https://example.com"
+    assert record.git_link == ""
+
+
+def test_user_bio_over_limit_is_rejected():
+    with pytest.raises(ValidationError):
+        UserRecord.model_validate({**BASE, "username": "bob", "bio": "x" * (LIMITS.max_bio_length + 1)})
+
+
+def test_user_counters_and_credentials_are_not_fields():
+    known = UserRecord.known_fields()
+    for name in ("email", "password_hash", "role", "xp", "level", "stars", "api_key"):
+        assert name not in known
+    assert "stars" not in PostRecord.known_fields()
+
+
+def test_envelope_normalizes_origin_and_bounds_batch():
+    form = SyncBatchForm(origin="Other.ORG", records=[{}])
+    assert form.origin == "other.org"
+    with pytest.raises(ValidationError):
+        SyncBatchForm(origin="other.org", records=[])
+    with pytest.raises(ValidationError):
+        SyncBatchForm(origin="other.org", records=[{}] * (LIMITS.max_batch_records + 1))
+    with pytest.raises(ValidationError):
+        SyncBatchForm(origin="bad origin", records=[{}])