Code Farm: per-plot auto-replant pause and crop-lock check; community sync and node work

Continuous Delivery could never be turned off once a crop was planted. Add a
one-shot per-plot "stop after harvest" flag (POST /game/plot/stop, Devii tool
game_plot_stop, game_plots.stop_after_harvest), cleared by the next plant,
autoreplant claim or Refactor. Autoreplant now also respects crop locks through
the shared _crop_lock predicate. Plan and measurements in happyplanting.md.

Also includes the pending community sync, node network and related work in the
tree; the full suite passed (4999) with all of it.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

tests/unit/database/nodes.py

@@ -392,3 +392,65 @@ def test_recent_devices_is_capped_and_excludes_deleted(local_db):
     assert len(store.recent_devices(10_000)) <= store.ADMIN_LIST_LIMIT
     store.revoke_device(row["uid"], owner)
     assert row["uid"] not in {item["uid"] for item in store.recent_devices(store.ADMIN_LIST_LIMIT)}
+
+
+def test_touch_devices_updates_only_the_given_devices_in_one_statement(owner):
+    first, _ = store.create_device(owner, "a", "native")
+    second, _ = store.create_device(owner, "b", "native")
+    assert store.touch_devices([first["uid"]]) == 1
+    assert store.get_device(first["uid"])["last_seen_at"]
+    assert not store.get_device(second["uid"]).get("last_seen_at")
+    assert store.touch_devices([]) == 0
+    many = [generate_uid() for _ in range(store.BATCH_SIZE + 5)]
+    assert store.touch_devices(many) == 0
+
+
+def test_class_override_round_trip(owner):
+    device, _ = store.create_device(owner, "a", "native")
+    assert store.set_class_override(device["uid"], "allow")
+    assert store.get_device(device["uid"])["class_override"] == "allow"
+    assert store.set_class_override(device["uid"], "")
+    assert not store.get_device(device["uid"])["class_override"]
+    assert not store.set_class_override(generate_uid(), "allow")
+
+
+def test_connection_records_country_asn_and_class(owner):
+    device, _ = store.create_device(owner, "a", "native")
+    store.record_device_connection(device["uid"], platform="linux/amd64", country="nl", asn="3320", asn_class="residential")
+    row = store.get_device(device["uid"])
+    assert (row["country"], row["asn"], row["asn_class"]) == ("NL", "3320", "residential")
+
+
+def test_counters_accumulate_and_ignore_empty_or_non_positive(local_db):
+    name = f"test_counter_{generate_uid()[:8]}"
+    assert store.flush_counters({}) == 0
+    assert store.flush_counters({name: 0}) == 0
+    assert store.flush_counters({name: 3}) == 1
+    assert store.flush_counters({name: 4}) == 1
+    assert store.lifetime_counters(name)[name] == 7
+
+
+def test_lifetime_prefix_is_a_literal_prefix(local_db):
+    store.flush_counters({"outcome:reddit:residential:ok": 1, "outcomeXreddit:residential:ok": 1})
+    found = store.lifetime_counters("outcome:")
+    assert "outcome:reddit:residential:ok" in found and "outcomeXreddit:residential:ok" not in found
+    assert store.lifetime_counters("100%") == {}
+
+
+def test_circuit_snapshots_replace_each_other_and_are_capped(local_db):
+    rows = [{"key": f"destination:d{index}", "failures": 5, "state": "open", "opened_at": 1.5} for index in range(LIMITS.circuit_snapshot_max + 20)]
+    assert store.save_circuits(rows) == LIMITS.circuit_snapshot_max
+    assert len(store.load_circuits()) == LIMITS.circuit_snapshot_max
+    assert store.save_circuits([]) == 0
+    assert store.load_circuits() == []
+
+
+def test_the_request_queue_cap_counts_shadows_too(local_db):
+    get_table("node_requests").delete()
+    try:
+        for index in range(LIMITS.max_queue_depth):
+            store.enqueue_request("user", "c", f"https://reddit.com/{index}", "GET", "reddit", "any")
+        with pytest.raises(store.QueueFull):
+            store.enqueue_request("shadow", "x", "https://reddit.com/z", "GET", "reddit", "fetch", shadow_of="u")
+    finally:
+        get_table("node_requests").delete()

tests/unit/services/community/alias.py

@@ -60,11 +60,12 @@ def enable(**changes):
     store.update(row["uid"], sync_options=SyncOptions.from_row(row).merged(**changes).to_dict())
 
 
-def test_aliasing_off_by_default_keeps_original_names(world):
+def test_aliasing_off_by_default_creates_no_identities_and_derives_from_originals(world):
     report = go(world)
     assert report.status == "ok", report.errors
-    assert "writer0" in local_text()
+    assert report.pushed["users"].get("created", 0) == 4
     assert get_table("sync_identities").count(origin="forum.test") == 0
+    assert "writer" in local_text().replace("wr1ter", "writer")
 
 
 def test_alias_run_never_stores_an_original_username(world):

tests/unit/services/community/checkpoint.py

@@ -0,0 +1,152 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+from dataclasses import replace
+
+import pytest
+
+from devplacepy.database import get_setting, get_table, init_db, set_setting
+from devplacepy.services.community import checkpoint as checkpoint_module
+from devplacepy.services.community.ledger import Ledger
+from devplacepy.services.community.ledger_store import TABLE, ledger_store
+from devplacepy.services.community.runner import CommunityRunner
+from devplacepy.services.community.spec import LIMITS, RUN_INTERRUPTED
+from devplacepy.services.community.store import iso, now, store
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep
+
+
+@pytest.fixture
+def world():
+    init_db()
+    previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins")}
+    set_setting("sync_enabled", "1")
+    set_setting("sync_origins", "")
+    for table in ("communities", "community_runs", TABLE):
+        get_table(table).delete()
+    get_table("sync_links").delete(origin="forum.test")
+    yield
+    for key, value in previous.items():
+        set_setting(key, value)
+
+
+class SlowFetcher(FakeFetcher):
+    async def get(self, url, params=None, timeout=None, **kwargs):
+        await asyncio.sleep(0.12)
+        return await super().get(url, params, timeout, **kwargs)
+
+
+def learned(runner):
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test"))
+    return store.get("forum-test")
+
+
+def test_a_running_run_exposes_its_log_and_ledger_before_it_finishes(world, monkeypatch):
+    runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(json_community()), sleep=no_sleep)
+    row = learned(runner)
+    monkeypatch.setattr(checkpoint_module, "LIMITS", replace(LIMITS, checkpoint_seconds=0.05))
+    seen = []
+    original = store.update_run
+
+    def spy(uid, **fields):
+        original(uid, **fields)
+        seen.append((get_table("community_runs").find_one(uid=uid)["status"], fields.get("log", ""), get_table(TABLE).count(run_uid=uid)))
+
+    monkeypatch.setattr(store, "update_run", spy)
+    runner.fetcher_factory = lambda: SlowFetcher(json_community())
+    report = run_async(runner.run(row, dry_run=False, trigger="test"))
+    assert seen, "no checkpoint was written while the run was going"
+    assert all(status == "running" for status, _, _ in seen)
+    assert any("collect" in log or "run[" in log for _, log, _ in seen)
+    final = store.run(report.run_uid)
+    assert final["status"] in ("ok", "partial") and final["verdict"] != "running"
+    assert final["items_stored"] == get_table(TABLE).count(run_uid=report.run_uid) > 0
+
+
+def test_checkpointed_ledger_rows_are_never_written_twice(world):
+    community = store.create("keep-cp", "q", "tester", name="cp")
+    run_uid = store.start_run(community["uid"], "test", False)
+    ledger = Ledger()
+    from devplacepy.services.community.checkpoint import RunCheckpoint
+    from devplacepy.services.community.logbook import LogBook
+    from devplacepy.services.community.pusher import PushReport
+    from devplacepy.services.community.runner import RunReport
+
+    report = RunReport(status="running", dry_run=False, run_uid=run_uid, ledger=ledger)
+    live = PushReport(dry_run=False)
+    with LogBook() as book:
+        checkpoint = RunCheckpoint(store, ledger_store, run_uid, community["uid"], report, live, book)
+        for number in range(3):
+            ledger.add("posts", "created", origin_uid=f"a{number}")
+        checkpoint.flush()
+        checkpoint.flush()
+        assert get_table(TABLE).count(run_uid=run_uid) == 3
+        for number in range(2):
+            ledger.add("posts", "rejected", reason="invalid", origin_uid=f"b{number}")
+        checkpoint.flush()
+        assert get_table(TABLE).count(run_uid=run_uid) == 5
+        final = ledger_store.append(run_uid, community["uid"], ledger, checkpoint.saved)
+    assert final == 0
+    seqs = sorted(row["seq"] for row in get_table(TABLE).find(run_uid=run_uid))
+    assert seqs == [0, 1, 2, 3, 4]
+
+
+def test_a_killed_run_is_marked_interrupted_and_keeps_what_was_saved(world):
+    community = store.create("keep-kill", "q", "tester", name="kill")
+    run_uid = store.start_run(community["uid"], "test", False)
+    store.update_run(run_uid, log="line one\nline two", tally={"created": 4}, items_stored=4)
+    assert store.reap_interrupted() == 0
+    stale = iso(now().replace(year=2000))
+    get_table("community_runs").update({"uid": run_uid, "started_at": stale}, ["uid"])
+    assert store.reap_interrupted() == 1
+    found = store.run(run_uid)
+    assert found["status"] == RUN_INTERRUPTED and found["verdict"] == "bad" and "Interrupted" in found["headline"]
+    assert found["log"] == "line one\nline two" and found["tally"] == {"created": 4} and found["finished_at"]
+    assert store.reap_interrupted() == 0
+    assert not store.busy(community["uid"])
+
+
+def test_the_interrupted_verdict_is_bad():
+    from devplacepy.services.community.ledger import verdict
+
+    assert verdict("interrupted", False, {})["level"] == "bad"
+
+
+def test_the_search_index_follows_inserts_and_prunes(world):
+    ledger = Ledger()
+    ledger.add("posts", "created", origin_uid="needle-1", label="Alpha Beta Gamma", detail="")
+    ledger.add("posts", "created", origin_uid="hay-2", label="Delta", detail="contains the NEEDLE word")
+    ledger.add("posts", "created", origin_uid="hay-3", label="100% done", detail="")
+    ledger_store.save("search-run", "search-community", ledger)
+    assert ledger_store.search_ready()
+
+    def find(text):
+        rows, page = ledger_store.page({"q": text}, 1, community_uid="search-community")
+        scoped, _ = ledger_store.page({"q": text}, 1, run_uid="search-run")
+        assert sorted(row["origin_uid"] for row in scoped) == sorted(row["origin_uid"] for row in rows)
+        return sorted(row["origin_uid"] for row in rows)
+
+    assert find("needle") == ["hay-2", "needle-1"]
+    assert find("beta g") == ["needle-1"]
+    assert find("100%") == ["hay-3"]
+    assert find("a\"b") == []
+    assert find("ne") == ["hay-2", "hay-3", "needle-1"]
+    get_table(TABLE).delete(run_uid="search-run")
+    assert find("needle") == []
+
+
+def test_a_community_wide_text_search_goes_through_the_index(world):
+    from devplacepy.database import db
+
+    ledger = Ledger()
+    for number in range(50):
+        ledger.add("posts", "created", origin_uid=f"p{number}", label=f"label {number}")
+    ledger_store.save("plan-run-a", "plan-community-a", ledger)
+    where, params, order = ledger_store.compile({"q": "label 4"}, community_uid="plan-community-a")
+    plan = " | ".join(row["detail"] for row in db.query(f"EXPLAIN QUERY PLAN SELECT * FROM {TABLE} WHERE {where} ORDER BY {order} LIMIT 50", **params))
+    assert "community_run_items_fts" in plan
+    where, params, order = ledger_store.compile({"q": "la"}, community_uid="plan-community-a")
+    assert "community_run_items_fts" not in where and "LIKE" in where
+    where, params, order = ledger_store.compile({"q": "label 4"}, run_uid="plan-run-a")
+    assert "community_run_items_fts" not in where

tests/unit/services/community/dedupe.py

@@ -0,0 +1,92 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+from devplacepy.services.community.generator import CodeGenerator
+from devplacepy.services.community.runtime import Collector
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher
+from tests.unit.services.community.generator import load, no_sleep
+
+
+def feed(identifier, url, comments="none"):
+    return {
+        "id": identifier,
+        "url": url,
+        "type": "json_api",
+        "list_path": "$.items",
+        "post": {"fields": {"id": ["$.id"], "title": ["$.title"], "content": ["$.body"], "created_at": ["$.at"], "url": ["$.link"]}},
+        "author": {"fields": {"username": ["$.who"]}},
+        "comments": {"mode": comments},
+    }
+
+
+def schema(limits=None):
+    return {
+        "community": "twins",
+        "origin": "twins.test",
+        "limits": {"delay_seconds": 0, "retries": 1, **(limits or {})},
+        "feeds": [feed("latest", "https://twins.test/latest"), feed("top", "https://twins.test/top")],
+    }
+
+
+def entry(identifier, link, title="A shared story", at="2026-10-01T10:00:00Z"):
+    return {"id": identifier, "title": title, "body": "a body that is long enough to sync", "at": at, "who": "alice", "link": link}
+
+
+def routes(latest, top):
+    return {"https://twins.test/latest": {"items": latest}, "https://twins.test/top": {"items": top}}
+
+
+def collect(fetcher_routes, state=None, only=None, limits=None):
+    return run_async(Collector(FakeFetcher(fetcher_routes), schema(limits), state, sleep=no_sleep).run(only))
+
+
+def uids(result):
+    return {record["origin_uid"] for record in result["records"]["posts"]}
+
+
+def test_the_same_story_in_a_second_feed_in_a_later_run_is_not_a_second_post():
+    shared = "https://news.test/story/42?utm=a"
+    sources = routes([entry("L1", shared)], [entry("T9", "https://www.news.test/story/42/?utm=b")])
+    first = collect(sources, only="latest")
+    assert len(first["records"]["posts"]) == 1 and first["state"]["links"]
+    second = collect(sources, state=json.loads(json.dumps(first["state"])), only="top")
+    assert second["records"]["posts"] == []
+
+
+def test_without_the_remembered_map_the_twin_would_become_a_post():
+    shared = "https://news.test/story/42"
+    sources = routes([entry("L1", shared)], [entry("T9", shared)])
+    first = collect(sources, only="latest")
+    assert len(collect(sources, state={"feeds": first["state"]["feeds"]}, only="top")["records"]["posts"]) == 1
+
+
+def test_one_feed_never_merges_its_own_posts_that_share_a_link():
+    shared = "https://news.test/root"
+    result = collect(routes([entry("A", shared, "first"), entry("B", shared, "second")], []))
+    assert len(result["records"]["posts"]) == 2
+
+
+def test_the_same_feed_keeps_its_own_post_when_it_changes_in_a_later_run():
+    shared = "https://news.test/story/7"
+    first = collect(routes([entry("L1", shared)], []))
+    changed = routes([entry("L1", shared, "A shared story, edited", "2026-10-05T10:00:00Z")], [])
+    again = collect(changed, state=first["state"])
+    assert uids(again) == uids(first) and again["records"]["posts"][0]["title"].endswith("edited")
+
+
+def test_the_remembered_map_is_bounded_and_keeps_the_newest():
+    items = [entry(f"L{number}", f"https://news.test/story/{number}") for number in range(40)]
+    result = collect(routes(items, []), limits={"max_remembered_links": 10})
+    kept = list(result["state"]["links"])
+    assert len(kept) == 10 and kept[-1].endswith("/story/39") and not any(key.endswith("/story/0") for key in kept)
+
+
+def test_a_generated_collector_remembers_links_too(tmp_path):
+    module = load(CodeGenerator(schema()).write(tmp_path))
+    shared = "https://news.test/story/42"
+    sources = routes([entry("L1", shared)], [entry("T9", shared)])
+    first = run_async(module.collect_all(FakeFetcher(sources), only="latest", sleep=no_sleep))
+    second = run_async(module.collect_all(FakeFetcher(sources), state=json.loads(json.dumps(first["state"])), only="top", sleep=no_sleep))
+    assert len(first["records"]["posts"]) == 1 and second["records"]["posts"] == []

tests/unit/services/community/discoverer.py

@@ -131,3 +131,44 @@ def test_feeds_listing_the_same_posts_are_learned_once_and_comment_feeds_are_ski
 def test_generic_probe_paths_are_skipped_when_a_feed_is_advertised():
     _, fetcher = discover(rss_community(), "https://blog.test/")
     assert not any(request.endswith("/api/posts") or request.endswith("/rss.xml") for request in fetcher.requests)
+
+
+def test_a_captcha_on_the_login_wall_is_reported_and_never_attempted():
+    routes = club_community({})
+    captcha_wall = routes["https://club.test/"]().replace("<form", '<div class="g-recaptcha" data-sitekey="k"></div><form', 1)
+    routes["https://club.test/"] = captcha_wall
+    routes["https://club.test/login"] = captcha_wall
+    result, fetcher = discover(routes, "https://club.test/")
+    assert result.status == "login_required" and "captcha" in result.error
+    assert result.schema["login"]["captcha"] is True and fetcher.posts == []
+    result, fetcher = discover(routes, "https://club.test/", {"username": "bob", "password": "pw"})
+    assert result.status == "login_required" and "captcha" in result.error and fetcher.posts == []
+
+
+def test_a_second_factor_with_a_secret_logs_in_and_without_one_asks_a_human(monkeypatch):
+    from devplacepy.services.community import runtime
+    from tests.unit.services.community.login import OTP_PAGE
+
+    monkeypatch.setattr(runtime, "totp", lambda secret: "123456")
+    state = {}
+    routes = club_community(state)
+    password_step = routes["post:https://club.test/login"]
+
+    def first_step(data):
+        answer = password_step(data)
+        if state.get("in"):
+            state["in"] = False
+            return (200, OTP_PAGE)
+        return answer
+
+    def second_step(data):
+        state["in"] = data.get("otp_code") == "123456"
+        return (200, "welcome" if state["in"] else OTP_PAGE)
+
+    routes["post:https://club.test/login"] = first_step
+    routes["post:https://club.test/2fa"] = second_step
+    result, _ = discover(routes, "https://club.test/", {"username": "bob", "password": "pw", "totp": "JBSWY3DPEHPK3PXP"})
+    assert result.status == "ready", result.error
+    state.clear()
+    result, _ = discover(routes, "https://club.test/", {"username": "bob", "password": "pw"})
+    assert result.status == "login_required" and "authenticator secret" in result.error

tests/unit/services/community/engagement.py

@@ -0,0 +1,199 @@
+# retoor <retoor@molodetz.nl>
+
+import base64
+
+import pytest
+
+from devplacepy.database import get_setting, get_table, init_db, set_setting
+from devplacepy.services.community.discoverer import CommunityDiscoverer
+from devplacepy.services.community.generator import CodeGenerator
+from devplacepy.services.community.learner import learn_engagement
+from devplacepy.services.community.runner import CommunityRunner
+from devplacepy.services.community.runtime import Collector
+from devplacepy.services.community.store import store
+from devplacepy.services.sync.options import SyncOptions
+from devplacepy.services.sync.records import RECORD_MODELS
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeBinary, FakeFetcher
+from tests.unit.services.community.generator import load, no_sleep
+
+PNG = base64.b64decode(
+    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
+)
+
+
+def person(number):
+    return {"id": 500 + number, "username": f"fan{number}"}
+
+
+def post(number, voters=3, with_engagement=True):
+    item = {
+        "id": 9000 + number,
+        "title": f"Engaged post number {number}",
+        "content_html": f'<p>Body of engaged post {number}, long enough to sync.</p><img src="/img/p{number}.png"><a href="https://cdn.test/doc{number}.pdf">doc</a><a href="/page">page</a>',
+        "created_at": f"2026-10-{number:02d}T10:00:00Z",
+        "author": {"id": 77, "username": "writer"},
+        "comments": [
+            {"id": 7000 + number, "body": f"A comment on {number}", "created_at": f"2026-10-{number:02d}T11:00:00Z", "user": {"id": 88, "username": "replier"},
+             "votes": [{"user": person(1), "value": 1}, {"user": person(2), "value": -1}] if with_engagement else []},
+        ],
+    }
+    if with_engagement:
+        item["votes"] = [{"user": person(index), "value": -1 if index == 3 else 1, "created_at": f"2026-10-{number:02d}T12:00:00Z"} for index in range(1, voters + 1)]
+        item["reactions"] = [{"user": person(1), "emoji": "\U0001f525"}, {"user": person(2), "emoji": "\U0001f44d"}, {"user": person(3), "emoji": "nope"}]
+    return item
+
+
+def routes(count=4, **kwargs):
+    return {
+        "https://eng.test/": '<html lang="en"><head><title>Eng</title></head><body><p>Join 100 members discussing code every day on this site.</p></body></html>',
+        "https://eng.test/api/posts": {"items": [post(number, **kwargs) for number in range(1, count + 1)], "total": count},
+        "https://eng.test/about": "<html><body><p>About: 100 members.</p></body></html>",
+        **{f"https://eng.test/img/p{number}.png": FakeBinary(PNG) for number in range(1, count + 1)},
+        **{f"https://cdn.test/doc{number}.pdf": FakeBinary(b"%PDF-1.4 " + b"x" * 40, "application/pdf") for number in range(1, count + 1)},
+        "https://eng.test/page": "<html><body>not a file</body></html>",
+    }
+
+
+def learned(source=None):
+    result = run_async(CommunityDiscoverer(FakeFetcher(source or routes())).discover("https://eng.test/api/posts"))
+    assert result.status == "ready", result.error
+    return result.schema
+
+
+def test_the_learner_finds_per_voter_lists_and_names_the_emoji_field():
+    schema = learned()
+    post_spec = schema["feeds"][0]["post"]
+    assert post_spec["votes"]["list_path"] == "$.votes" and post_spec["votes"]["author"]["fields"]["username"] == ["$.user.username"]
+    assert post_spec["votes"]["fields"]["value"] == ["$.value"]
+    assert post_spec["reactions"]["fields"]["emoji"] == ["$.emoji"]
+    assert schema["feeds"][0]["comments"]["votes"]["list_path"] == "$.votes"
+
+
+def test_aggregate_counts_alone_learn_nothing():
+    samples = [{"id": number, "votes": number * 3, "likes": 5, "reactions": {"fire": 2}} for number in range(1, 5)]
+    assert learn_engagement(samples) == {}
+    assert learn_engagement([{"id": 1, "votes": [{"user": {"username": "a"}}]}]) == {}
+    assert learn_engagement([{"id": number, "votes": [{"score": 1}, {"score": 2}]} for number in range(3)]) == {}
+
+
+def test_the_collector_turns_voter_lists_into_records_with_users():
+    schema = learned()
+    result = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep).run())
+    records = result["records"]
+    assert len(records["votes"]) == 4 * 3 + 4 * 2 and len(records["reactions"]) == 4 * 2
+    assert {item["value"] for item in records["votes"]} == {1, -1}
+    assert {item["emoji"] for item in records["reactions"]} == {"\U0001f525", "\U0001f44d"}
+    voters = {item["voter_uid"] for item in records["votes"]}
+    assert voters <= {user["origin_uid"] for user in records["users"]}
+    for kind in ("votes", "reactions"):
+        for row in records[kind]:
+            RECORD_MODELS[kind].model_validate(row)
+    assert len({item["origin_uid"] for item in records["votes"]}) == len(records["votes"])
+    assert {item["target_kind"] for item in records["votes"]} == {"post", "comment"}
+
+
+def test_attachments_are_off_by_default_and_bounded_when_on():
+    schema = learned()
+    off = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep).run())
+    assert off["records"]["attachments"] == []
+    on = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True}).run())
+    files = on["records"]["attachments"]
+    assert len(files) == 8 and {item["filename"].rsplit(".", 1)[-1] for item in files} == {"png", "pdf"}
+    assert base64.b64decode(next(item for item in files if item["filename"] == "p1.png")["content_b64"]) == PNG
+    for row in files:
+        RECORD_MODELS["attachments"].model_validate(row)
+    capped = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True, "max_attachments": 3}).run())
+    assert len(capped["records"]["attachments"]) == 3
+    one_each = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True, "max_attachments_per_post": 1}).run())
+    assert len(one_each["records"]["attachments"]) == 4
+
+
+def test_missing_oversize_and_wrong_type_files_become_notices_not_failures():
+    source = routes(count=2)
+    source["https://eng.test/img/p1.png"] = (404, "gone")
+    source["https://cdn.test/doc1.pdf"] = FakeBinary(b"x" * 500, "application/pdf")
+    source["https://eng.test/img/p2.png"] = FakeBinary(b"<html>", "text/html")
+    schema = learned()
+    result = run_async(Collector(FakeFetcher(source), schema, sleep=no_sleep, options={"attachments": True, "max_attachment_bytes": 100}).run())
+    reasons = sorted(notice["reason"] for notice in result["notices"])
+    assert reasons.count("attachment_unreachable") == 2 and reasons.count("attachment_too_large") == 1
+    assert result["ok"] and [item["filename"] for item in result["records"]["attachments"]] == ["doc2.pdf"]
+
+
+def test_a_generated_collector_collects_the_same_engagement(tmp_path):
+    schema = learned()
+    module = load(CodeGenerator(schema).write(tmp_path))
+    generated = run_async(module.collect_all(FakeFetcher(routes()), sleep=no_sleep, options={"attachments": True}))
+    direct = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True}).run())
+    for kind in ("votes", "reactions", "attachments"):
+        assert len(generated["records"][kind]) == len(direct["records"][kind]) > 0
+
+
+def test_an_old_generated_file_is_regenerated_keeping_its_hand_edited_schema(tmp_path):
+    from devplacepy.services.community.files import CommunityFiles
+    from devplacepy.services.community.generator import CodeGenerator
+
+    schema = learned()
+    files = CommunityFiles(tmp_path)
+    path = files.write("old-site", schema)
+    edited = path.read_text().replace(f"# generator-version: {CodeGenerator.VERSION}", "# generator-version: 1").replace('"max_posts": 150', '"max_posts": 7')
+    path.write_text(edited)
+    module = files.load("old-site", schema)
+    assert module.SCHEMA["limits"]["max_posts"] == 7
+    assert f"# generator-version: {CodeGenerator.VERSION}" in path.read_text()
+
+
+@pytest.fixture
+def world():
+    init_db()
+    previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins", "community_alias_enabled")}
+    set_setting("sync_enabled", "1")
+    set_setting("sync_origins", "")
+    for table in ("communities", "community_runs"):
+        get_table(table).delete()
+    get_table("sync_links").delete(origin="eng.test")
+    get_table("sync_identities").delete(origin="eng.test")
+    yield
+    for key, value in previous.items():
+        set_setting(key, value)
+
+
+def make_row(**options):
+    runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes()), sleep=no_sleep)
+    store.create("eng-test", "https://eng.test/api/posts", "tester", name="eng")
+    run_async(runner.discover("https://eng.test/api/posts", "tester", relearn_slug="eng-test"))
+    row = store.get("eng-test")
+    if options:
+        store.update(row["uid"], sync_options=SyncOptions.build(**options).to_dict())
+    return runner, store.get("eng-test")
+
+
+def test_a_live_run_syncs_votes_reactions_and_attachments(world):
+    runner, row = make_row(collect_attachments=True)
+    report = run_async(runner.run(row, dry_run=False, trigger="test"))
+    assert report.status in ("ok", "partial"), report.errors
+    assert report.pushed["votes"].get("created") == 20 and report.pushed["reactions"].get("created") == 8
+    assert report.pushed["attachments"].get("created") == 8
+    post_link = get_table("sync_links").find_one(origin="eng.test", kind="posts")
+    stars = get_table("posts").find_one(uid=post_link["local_uid"])["stars"]
+    assert stars == 1
+    again = run_async(runner.run(store.get("eng-test"), dry_run=False, trigger="test"))
+    assert again.pushed.get("votes", {}).get("created", 0) == 0
+
+
+def test_without_the_option_no_file_is_downloaded_or_pushed(world):
+    runner, row = make_row()
+    report = run_async(runner.run(row, dry_run=False, trigger="test"))
+    assert report.pushed["attachments"] == {}
+
+
+def test_voters_are_never_stored_under_pseudonyms_so_votes_wait_while_aliasing_is_on(world):
+    set_setting("community_alias_enabled", "1")
+    runner, row = make_row(alias=True, collect_attachments=True)
+    report = run_async(runner.run(row, dry_run=False, trigger="test"))
+    assert report.pushed["votes"] == {} and report.pushed["reactions"] == {}
+    assert report.pushed["skipped"]["alias_engagement"] == 20 + 8
+    assert report.pushed["attachments"].get("created") == 8
+    owners = {get_table("attachments").find_one(uid=link["local_uid"])["user_uid"] for link in get_table("sync_links").find(origin="eng.test", kind="attachments")}
+    assert owners and all(get_table("users").find_one(uid=uid)["username"] != "writer" for uid in owners)

tests/unit/services/community/fakes.py

@@ -4,15 +4,29 @@ import json
 from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
 
 
+class FakeBinary:
+    def __init__(self, data: bytes, content_type: str = "image/png"):
+        self.data = data
+        self.content_type = content_type
+
+
 class FakeResponse:
     def __init__(self, status: int, body):
         self.status = status
-        self.body = body if isinstance(body, str) else json.dumps(body)
+        self.raw = b""
         self.headers = {}
+        if isinstance(body, FakeBinary):
+            self.raw, self.body = body.data, ""
+            self.headers = {"content-type": body.content_type}
+        else:
+            self.body = body if isinstance(body, str) else json.dumps(body)
 
     async def text(self):
         return self.body
 
+    async def read(self):
+        return self.raw or self.body.encode()
+
     async def json(self):
         return json.loads(self.body)
 
@@ -23,6 +37,8 @@ class FakeFetcher:
         self.requests: list[str] = []
         self.session = None
         self.posts: list[tuple] = []
+        self.last_headers = None
+        self.get_headers: list = []
 
     async def __aenter__(self):
         return self
@@ -40,6 +56,7 @@ class FakeFetcher:
     async def get(self, url, params=None, timeout=None, **kwargs):
         key = self.key(url, params)
         self.requests.append(key)
+        self.get_headers.append(kwargs.get("headers"))
         route = self.routes.get(key)
         if route is None:
             return FakeResponse(404, "not found")
@@ -49,10 +66,12 @@ class FakeFetcher:
         return FakeResponse(status, body)
 
     async def post(self, url, data=None, timeout=None, **kwargs):
-        self.posts.append((url, dict(data or {})))
+        body = dict(data or kwargs.get("json") or {})
+        self.posts.append((url, body))
+        self.last_headers = kwargs.get("headers")
         route = self.routes.get("post:" + url)
         if callable(route):
-            route = route(dict(data or {}))
+            route = route(body)
         status, body = route if isinstance(route, tuple) else (200, route or "")
         return FakeResponse(status, body)
 

tests/unit/services/community/ledger.py

@@ -63,9 +63,9 @@ def test_pusher_records_created_rejected_and_held_records_with_reasons(world):
     by_origin = {item.origin_uid: item for item in ledger.items}
     assert by_origin["u1"].outcome == "created" and by_origin["u1"].label == "ledgerone" and by_origin["u1"].local_uid
     assert by_origin["p1"].outcome == "created" and by_origin["p1"].label == "A good post"
-    assert by_origin["p2"].outcome == "rejected" and by_origin["p2"].reason == "unknown_author" and by_origin["p2"].label == "Orphan post"
+    assert by_origin["p2"].outcome == "deferred" and by_origin["p2"].reason == "unknown_author" and by_origin["p2"].label == "Orphan post"
     assert by_origin["p3"].outcome == "unchanged" and by_origin["p3"].reason == "tombstone_for_unknown"
-    assert tally_of(report.counts, ledger) == {"created": 2, "unchanged": 1, "rejected": 1}
+    assert tally_of(report.counts, ledger) == {"created": 2, "unchanged": 1, "deferred": 1}
 
 
 def test_repeating_the_same_batch_lists_nothing_that_is_merely_unchanged(world):

tests/unit/services/community/login.py

@@ -0,0 +1,238 @@
+# retoor <retoor@molodetz.nl>
+
+import base64
+
+import pytest
+
+from devplacepy.services.community import runtime
+from devplacepy.services.community.discoverer import LoginLearner
+from devplacepy.services.community.extract import detect_captcha, find_otp_form, parse_html, totp
+from devplacepy.services.community.runtime import (
+    CaptchaRequired,
+    Collector,
+    LoginFailed,
+    SecondFactorRequired,
+    code_challenge,
+)
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher
+
+
+async def no_sleep(_seconds):
+    return None
+
+
+RFC_SECRET = base64.b32encode(b"12345678901234567890").decode()
+
+
+@pytest.mark.parametrize(
+    "moment,expected",
+    [(59, "94287082"), (1111111109, "07081804"), (1111111111, "14050471"), (1234567890, "89005924"), (2000000000, "69279037"), (20000000000, "65353130")],
+)
+def test_totp_matches_the_rfc_6238_vectors(moment, expected):
+    assert totp(RFC_SECRET, moment, digits=8) == expected
+
+
+def test_totp_default_is_six_digits_zero_padded_and_tolerates_formatting():
+    code = totp(RFC_SECRET.lower(), 59)
+    assert len(code) == 6 and code.isdigit()
+    spaced = " ".join(RFC_SECRET[index : index + 4] for index in range(0, len(RFC_SECRET), 4))
+    assert totp(spaced, 59) == code
+    assert totp(RFC_SECRET.rstrip("="), 59) == code
+    assert totp(RFC_SECRET, 59 + 30) != code
+    assert totp(RFC_SECRET, 59) == totp(RFC_SECRET, 30)
+
+
+OTP_PAGE = """<html><body><form action="/2fa" method="post">
+<input type="hidden" name="csrf" value="t2"><input type="text" name="otp_code" autocomplete="one-time-code"><button>Verify</button></form></body></html>"""
+LOGIN_PAGE = """<html><body><form action="/login" method="post">
+<input type="hidden" name="csrf" value="t1"><input type="text" name="user"><input type="password" name="pass"><button>Sign in</button></form></body></html>"""
+HOME_OPEN = "<html><body><p>Welcome back to the forum, you are signed in</p></body></html>"
+
+
+def test_otp_forms_and_captcha_markers_are_recognised():
+    form = find_otp_form(parse_html(OTP_PAGE), "https://x.test/login")
+    assert form == {"action": "https://x.test/2fa", "method": "POST", "field": "otp_code", "fields": {"csrf": "t2"}}
+    assert find_otp_form(parse_html(LOGIN_PAGE), "https://x.test/") is None
+    named = parse_html('<form><input type="text" name="verification_code"></form>')
+    assert find_otp_form(named, "https://x.test/")["field"] == "verification_code"
+    hidden_token = parse_html('<form><input type="hidden" name="token"><input type="text" name="user"></form>')
+    assert find_otp_form(hidden_token, "https://x.test/") is None
+    assert detect_captcha('<div class="g-recaptcha" data-sitekey="x"></div>') and detect_captcha("<div class='cf-turnstile'></div>")
+    assert not detect_captcha("<p>no puzzle here</p>")
+
+
+def form_schema():
+    return {
+        "community": "club",
+        "origin": "club.test",
+        "base_url": "https://club.test/",
+        "feeds": [],
+        "limits": {"delay_seconds": 0, "retries": 1},
+        "login": {
+            "mode": "form", "url": "https://club.test/login", "action": "https://club.test/login", "method": "POST",
+            "fields": {"csrf": "t1"}, "username_field": "user", "password_field": "pass", "identity": "username",
+        },
+    }
+
+
+def club(state, challenge=True, good_code="123456"):
+    def login(data):
+        if data.get("user") == "bob" and data.get("pass") == "pw":
+            if challenge:
+                return (200, OTP_PAGE)
+            state["in"] = True
+            return (200, "ok")
+        return (200, LOGIN_PAGE)
+
+    def second(data):
+        state["codes"] = state.get("codes", []) + [data.get("otp_code")]
+        state["in"] = data.get("otp_code") == good_code and data.get("csrf") == "t2"
+        return (200, "ok" if state["in"] else OTP_PAGE)
+
+    return {
+        "https://club.test/login": LOGIN_PAGE,
+        "https://club.test/": lambda: HOME_OPEN if state.get("in") else LOGIN_PAGE,
+        "post:https://club.test/login": login,
+        "post:https://club.test/2fa": second,
+    }
+
+
+def test_a_form_login_with_a_second_factor_submits_the_current_code(monkeypatch):
+    monkeypatch.setattr(runtime, "totp", lambda secret: "123456")
+    state = {}
+    collector = Collector(FakeFetcher(club(state)), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "pw", "totp": RFC_SECRET})
+    assert run_async(collector.login()) is True
+    assert state["in"] and state["codes"] == ["123456"]
+
+
+def test_a_second_factor_without_a_secret_says_what_is_missing():
+    collector = Collector(FakeFetcher(club({})), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "pw"})
+    with pytest.raises(SecondFactorRequired, match="authenticator secret"):
+        run_async(collector.login())
+
+
+def test_a_rejected_code_is_reported_not_retried(monkeypatch):
+    monkeypatch.setattr(runtime, "totp", lambda secret: "000000")
+    state = {}
+    collector = Collector(FakeFetcher(club(state)), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "pw", "totp": RFC_SECRET})
+    with pytest.raises(SecondFactorRequired, match="rejected the one-time code"):
+        run_async(collector.login())
+    assert state["codes"] == ["000000"]
+
+
+def test_a_login_without_a_second_factor_is_unchanged():
+    state = {}
+    collector = Collector(FakeFetcher(club(state, challenge=False)), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "pw"})
+    assert run_async(collector.login()) is True and state["in"]
+
+
+def test_wrong_password_still_fails_as_before():
+    collector = Collector(FakeFetcher(club({})), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "nope"})
+    with pytest.raises(LoginFailed, match="rejected the stored credentials"):
+        run_async(collector.login())
+
+
+def test_a_captcha_on_the_login_page_is_never_attempted():
+    routes = club({})
+    routes["https://club.test/login"] = LOGIN_PAGE.replace("<form", '<div class="g-recaptcha" data-sitekey="k"></div><form', 1)
+    fetcher = FakeFetcher(routes)
+    collector = Collector(fetcher, form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "pw"})
+    with pytest.raises(CaptchaRequired, match="captcha"):
+        run_async(collector.login())
+    assert fetcher.posts == []
+
+
+def test_a_learned_captcha_flag_stops_before_any_request():
+    schema = form_schema()
+    schema["login"]["captcha"] = True
+    fetcher = FakeFetcher({})
+    with pytest.raises(CaptchaRequired):
+        run_async(Collector(fetcher, schema, sleep=no_sleep, credentials={"username": "bob", "password": "pw"}).login())
+    assert fetcher.requests == [] and fetcher.posts == []
+
+
+def json_schema(**login):
+    return {
+        "community": "api",
+        "origin": "api.test",
+        "base_url": "https://api.test/",
+        "feeds": [],
+        "limits": {"delay_seconds": 0, "retries": 1},
+        "login": {"mode": "json", "url": "https://api.test/api/login", "username_field": "email", "password_field": "password", "identity": "email", **login},
+    }
+
+
+def api(state, challenge=False):
+    def login(body):
+        state["bodies"] = state.get("bodies", []) + [dict(body)]
+        if body.get("email") != "bob@x.test" or body.get("password") != "pw":
+            return (401, {"error": "bad credentials"})
+        if challenge and body.get("otp") != "123456":
+            return (200, {"otp_required": True}) if "otp" not in body else (401, {"error": "invalid verification code"})
+        return (200, {"data": {"token": "tok-1"}})
+
+    return {"post:https://api.test/api/login": login, "https://api.test/api/me": {"ok": True}}
+
+
+def test_a_json_login_stores_the_token_and_sends_it_on_later_requests():
+    state = {}
+    fetcher = FakeFetcher(api(state))
+    collector = Collector(fetcher, json_schema(token_path="$.data.token"), sleep=no_sleep, credentials={"email": "bob@x.test", "password": "pw"})
+    assert run_async(collector.login()) is True
+    assert state["bodies"] == [{"email": "bob@x.test", "password": "pw"}]
+    run_async(collector.fetch("https://api.test/api/me"))
+    assert fetcher.get_headers[-1] == {"Authorization": "Bearer tok-1"}
+
+
+def test_a_json_login_without_a_token_relies_on_the_session_cookie():
+    collector = Collector(FakeFetcher(api({})), json_schema(), sleep=no_sleep, credentials={"email": "bob@x.test", "password": "pw"})
+    assert run_async(collector.login()) is True and collector.headers == {}
+
+
+def test_a_json_login_with_a_missing_token_or_bad_password_fails():
+    with pytest.raises(LoginFailed, match="without the expected token"):
+        run_async(Collector(FakeFetcher(api({})), json_schema(token_path="$.nope"), sleep=no_sleep, credentials={"email": "bob@x.test", "password": "pw"}).login())
+    with pytest.raises(LoginFailed, match="rejected the stored credentials"):
+        run_async(Collector(FakeFetcher(api({})), json_schema(), sleep=no_sleep, credentials={"email": "bob@x.test", "password": "bad"}).login())
+
+
+def test_a_json_login_answers_the_code_challenge(monkeypatch):
+    monkeypatch.setattr(runtime, "totp", lambda secret: "123456")
+    state = {}
+    collector = Collector(FakeFetcher(api(state, challenge=True)), json_schema(token_path="$.data.token"), sleep=no_sleep, credentials={"email": "bob@x.test", "password": "pw", "totp": RFC_SECRET})
+    assert run_async(collector.login()) is True
+    assert [("otp" in body) for body in state["bodies"]] == [False, True]
+    with pytest.raises(SecondFactorRequired, match="authenticator secret"):
+        run_async(Collector(FakeFetcher(api({}, challenge=True)), json_schema(), sleep=no_sleep, credentials={"email": "bob@x.test", "password": "pw"}).login())
+
+
+def test_code_challenge_detection():
+    assert code_challenge({"otp_required": True}, "", 200)
+    assert code_challenge({"Two_Factor": 1}, "", 200)
+    assert code_challenge(None, "Please enter your verification code", 401)
+    assert not code_challenge({"token": "x"}, "", 200)
+    assert not code_challenge(None, "Please enter your verification code", 200)
+
+
+def test_the_learner_finds_a_json_login_and_describes_it():
+    state = {}
+    routes = api(state)
+    routes["post:https://api.test/api/auth/login"] = (404, "nope")
+    learner = LoginLearner(FakeFetcher(routes))
+    spec = run_async(learner.find_json("https://api.test/", {"email": "bob@x.test", "password": "pw"}))
+    assert spec == {
+        "mode": "json", "url": "https://api.test/api/login", "username_field": "email", "password_field": "password",
+        "identity": "email", "token_path": "$.data.token", "token_scheme": "Bearer",
+    }
+    token_routes = {"post:https://api.test/api/login": lambda body: (200, {"access_token": "t"}) if body.get("username") == "bob" else (401, {"error": "x"})}
+    spec = run_async(LoginLearner(FakeFetcher(token_routes)).find_json("https://api.test/", {"username": "bob", "password": "pw"}))
+    assert spec["token_path"] == "$.access_token" and spec["username_field"] == "username"
+
+
+def test_the_learner_gives_up_quietly_and_within_its_attempt_cap():
+    fetcher = FakeFetcher({"post:https://api.test/api/login": (401, {"error": "no"})})
+    assert run_async(LoginLearner(fetcher).find_json("https://api.test/", {"username": "bob", "password": "x"})) is None
+    assert len(fetcher.posts) <= 12
+    html_only = FakeFetcher({"post:https://api.test/api/login": (200, "<html>not json</html>")})
+    assert run_async(LoginLearner(html_only).find_json("https://api.test/", {"username": "bob", "password": "x"})) is None

tests/unit/services/community/pagination.py

@@ -0,0 +1,176 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.community.extract import (
+    find_next_page,
+    page_number,
+    parse_html,
+    with_page_number,
+)
+from devplacepy.services.community.generator import CodeGenerator
+from devplacepy.services.community.runtime import Collector
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher, detail_page, html_community
+from tests.unit.services.community.generator import learn, load, no_sleep, post_comment_count
+
+BASE = "https://board.test/t/1"
+
+
+def page(markup: str):
+    return parse_html(f"<html><body><article>text</article>{markup}</body></html>")
+
+
+@pytest.mark.parametrize(
+    "markup,expected",
+    [
+        ('<a rel="next" href="/t/1?p=2">x</a>', f"{BASE}?p=2"),
+        ('<link rel="next" href="/t/1?p=2">', f"{BASE}?p=2"),
+        ('<ul class="pagination"><li class="next"><a href="/t/1?p=2">go</a></li></ul>', f"{BASE}?p=2"),
+        ('<nav><a href="/t/1?p=2">Next</a></nav>', f"{BASE}?p=2"),
+        ('<div class="pager"><a href="/t/1?p=2">›</a></div>', f"{BASE}?p=2"),
+        ('<div class="pagination"><a href="/t/1?p=2">Volgende pagina</a></div>', f"{BASE}?p=2"),
+        ('<div class="pagination"><a href="/t/1">1</a><a href="/t/1?page=2">2</a><a href="/t/1?page=3">3</a></div>', f"{BASE}?page=2"),
+        ('<a aria-label="Next page" href="/t/1?p=2">arrow</a>', f"{BASE}?p=2"),
+    ],
+)
+def test_next_page_is_found_in_every_common_markup(markup, expected):
+    assert find_next_page(page(markup), BASE) == expected
+
+
+@pytest.mark.parametrize(
+    "markup",
+    [
+        '<p><a href="/other">More</a> from the author</p>',
+        '<a href="javascript:void(0)" class="next">next</a>',
+        '<a href="#">Next</a>',
+        '<nav><a href="/t/1?p=1">1</a></nav>',
+        '<a href="/t/2?page=2">elsewhere 2</a>',
+        "",
+    ],
+)
+def test_no_next_page_is_invented(markup):
+    assert find_next_page(page(markup), BASE) == ""
+
+
+def test_hints_override_the_heuristics():
+    root = page('<a rel="next" href="/t/1?p=2">x</a><div class="x"><a class="more" href="/t/1?custom=2">c</a></div>')
+    assert find_next_page(root, BASE, {"style": "none"}) == ""
+    assert find_next_page(root, BASE, {"style": "next_link", "next": ".more::attr(href)"}) == f"{BASE}?custom=2"
+    assert find_next_page(root, BASE, {"style": "next_link", "next": ".absent::attr(href)"}) == ""
+    assert find_next_page(root, f"{BASE}?page=3", {"style": "page_param", "param": "page"}) == f"{BASE}?page=4"
+    assert find_next_page(root, BASE, {"style": "page_param"}) == f"{BASE}?page=2"
+
+
+def test_page_numbers_round_trip():
+    assert page_number("https://x.test/t?page=4") == 4
+    assert page_number("https://x.test/t/page/7") == 7
+    assert page_number("https://x.test/t?x=1&p=12") == 12
+    assert page_number("https://x.test/t") is None
+    assert with_page_number("https://x.test/t?a=1&page=2", 3) == "https://x.test/t?a=1&page=3"
+    assert with_page_number("https://x.test/t/page/2", 3) == "https://x.test/t/page/3"
+    assert with_page_number("https://x.test/t", 2, "p") == "https://x.test/t?p=2"
+
+
+def board(next_markup: str, second_next: str = "") -> dict:
+    routes = html_community()
+    routes[BASE] = detail_page("thread", 1).replace("</body>", f"{next_markup}</body>")
+    routes[f"{BASE}?cp=2"] = detail_page("thread", 1, comments=4).replace("</body>", f"{second_next}</body>")
+    return routes
+
+
+@pytest.mark.parametrize(
+    "markup",
+    [
+        '<nav class="pagination"><a href="/t/1?cp=2">Next</a></nav>',
+        '<div class="pager"><a href="/t/1?cp=2">»</a></div>',
+        '<div class="pagination"><a href="/t/1">1</a><a href="/t/1?cp=2">2</a></div>',
+    ],
+)
+def test_collector_and_generated_file_follow_every_style(markup, tmp_path):
+    routes = board(markup)
+    schema = learn(html_community(), "https://board.test/")
+    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    first = next(post["origin_uid"] for post in result["records"]["posts"] if post["title"].startswith("Thread 1 "))
+    assert post_comment_count(result, first) == 5
+    module = load(CodeGenerator(schema).write(tmp_path))
+    generated = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    assert len(generated["records"]["comments"]) == len(result["records"]["comments"])
+
+
+def test_a_page_that_adds_no_new_comment_ends_the_walk():
+    routes = board('<nav class="pagination"><a href="/t/1?cp=2">Next</a></nav>', '<nav class="pagination"><a href="/t/1?cp=3">Next</a></nav>')
+    routes[f"{BASE}?cp=3"] = routes[f"{BASE}?cp=2"].replace("cp=3", "cp=4")
+    routes[f"{BASE}?cp=4"] = routes[f"{BASE}?cp=2"]
+    schema = learn(html_community(), "https://board.test/")
+    fetcher = FakeFetcher(routes)
+    run_async(Collector(fetcher, schema, sleep=no_sleep).run())
+    assert f"{BASE}?cp=3" in fetcher.requests and f"{BASE}?cp=4" not in fetcher.requests
+
+
+def test_a_declared_no_pagination_hint_stops_after_the_first_page():
+    routes = board('<a rel="next" href="/t/1?cp=2">more</a>')
+    schema = learn(html_community(), "https://board.test/")
+    for feed in schema["feeds"]:
+        if (feed.get("comments") or {}).get("mode") == "page":
+            feed["comments"]["pagination"] = {"style": "none"}
+    fetcher = FakeFetcher(routes)
+    run_async(Collector(fetcher, schema, sleep=no_sleep).run())
+    assert f"{BASE}?cp=2" not in fetcher.requests
+
+
+def test_the_learner_records_a_selector_hint_when_one_matches():
+    schema = learn(board('<a rel="next" href="/t/1?cp=2">more</a>'), "https://board.test/")
+    modes = [feed["comments"] for feed in schema["feeds"] if (feed.get("comments") or {}).get("mode") == "page"]
+    assert modes and modes[0].get("pagination") == {"style": "next_link", "next": "a[rel=next]::attr(href)"}
+
+
+def json_comment_schema(pagination):
+    comments = {
+        "mode": "url",
+        "url_template": "https://api.test/posts/{id}/comments",
+        "list_path": "$.items",
+        "fields": {"id": ["$.id"], "content": ["$.body"], "created_at": ["$.at"]},
+        "author": {"fields": {"username": ["$.who"]}},
+    }
+    if pagination:
+        comments["pagination"] = pagination
+    return {
+        "community": "api",
+        "origin": "api.test",
+        "limits": {"delay_seconds": 0, "retries": 1, "max_comment_pages": 3},
+        "feeds": [
+            {
+                "id": "feed1",
+                "url": "https://api.test/posts",
+                "type": "json_api",
+                "list_path": "$.items",
+                "post": {"fields": {"id": ["$.id"], "title": ["$.title"], "content": ["$.body"], "created_at": ["$.at"]}},
+                "author": {"fields": {"username": ["$.who"]}},
+                "comments": comments,
+            }
+        ],
+    }
+
+
+def json_routes():
+    entry = lambda n: {"id": f"c{n}", "body": f"comment number {n}", "at": "2026-10-01T10:00:00Z", "who": "alice"}
+    return {
+        "https://api.test/posts": {"items": [{"id": "p1", "title": "A post", "body": "a post body long enough", "at": "2026-10-01T10:00:00Z", "who": "alice"}]},
+        "https://api.test/posts/p1/comments": {"items": [entry(1), entry(2)], "next": "/posts/p1/comments?page=2"},
+        "https://api.test/posts/p1/comments?page=2": {"items": [entry(3)], "next": "/posts/p1/comments?page=3"},
+        "https://api.test/posts/p1/comments?page=3": {"items": [entry(3)], "next": "/posts/p1/comments?page=4"},
+    }
+
+
+def test_json_comments_follow_a_declared_next_key_and_stop_without_news():
+    fetcher = FakeFetcher(json_routes())
+    result = run_async(Collector(fetcher, json_comment_schema({"style": "json_next", "path": "$.next"}), sleep=no_sleep).run())
+    assert sorted(comment["content"] for comment in result["records"]["comments"]) == ["comment number 1", "comment number 2", "comment number 3"]
+    assert "https://api.test/posts/p1/comments?page=4" not in fetcher.requests
+
+
+def test_json_comments_without_a_hint_read_only_the_first_page():
+    fetcher = FakeFetcher(json_routes())
+    result = run_async(Collector(fetcher, json_comment_schema(None), sleep=no_sleep).run())
+    assert len(result["records"]["comments"]) == 2

tests/unit/services/community/schemacheck.py

@@ -0,0 +1,89 @@
+# retoor <retoor@molodetz.nl>
+
+import copy
+
+import pytest
+
+from devplacepy.services.community.discoverer import CommunityDiscoverer
+from devplacepy.services.community.schemacheck import sample_limits, validate_schema
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher, html_community, json_community
+
+
+@pytest.fixture(scope="module")
+def json_schema():
+    return run_async(CommunityDiscoverer(FakeFetcher(json_community())).discover("https://forum.test/")).schema
+
+
+@pytest.fixture(scope="module")
+def html_schema():
+    return run_async(CommunityDiscoverer(FakeFetcher(html_community())).discover("https://board.test/")).schema
+
+
+def test_learned_schemas_are_valid_by_construction(json_schema, html_schema):
+    assert validate_schema(json_schema) == []
+    assert validate_schema(html_schema) == []
+
+
+@pytest.mark.parametrize("bad", [None, [], "text", 4])
+def test_the_top_level_must_be_an_object(bad):
+    assert validate_schema(bad) == ["the schema must be a JSON object"]
+
+
+def mutated(schema, change):
+    copied = copy.deepcopy(schema)
+    change(copied)
+    return validate_schema(copied)
+
+
+def test_every_structural_mistake_is_named_with_its_path(json_schema, html_schema):
+    assert any("origin" in item for item in mutated(json_schema, lambda s: s.update(origin="Not A Host!")))
+    assert any("community" in item for item in mutated(json_schema, lambda s: s.update(community="")))
+    assert any("feeds" in item for item in mutated(json_schema, lambda s: s.update(feeds=[])))
+    assert any("feeds: at most" in item for item in mutated(json_schema, lambda s: s.update(feeds=[copy.deepcopy(s["feeds"][0]) | {"id": f"f{n}"} for n in range(9)])))
+    assert any("used twice" in item for item in mutated(json_schema, lambda s: s.update(feeds=[s["feeds"][0], copy.deepcopy(s["feeds"][0])])))
+    assert any("feeds[0].url" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(url="ftp://x")))
+    assert any("feeds[0].type" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(type="soap")))
+    assert any("post.fields.id" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["post"]["fields"].pop("id")))
+    assert any("content or title" in item for item in mutated(json_schema, lambda s: [s["feeds"][0]["post"]["fields"].pop(k) for k in ("content", "title")]))
+    assert any("$" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["post"]["fields"].update(id=["id"])))
+    assert any("item_selector" in item for item in mutated(html_schema, lambda s: s["feeds"][0].pop("item_selector")))
+    assert any("unsupported" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["post"]["fields"].update(title=["h2::bogus"])))
+    assert any("selector" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["post"]["fields"].update(title=["h2[unclosed"])))
+
+
+def test_comment_and_pagination_specs_are_checked(json_schema, html_schema):
+    assert any("comments.mode" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["comments"].update(mode="telepathy")))
+    assert any("list_path" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["comments"].pop("list_path")))
+    assert any("url_template" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(comments={"mode": "url", "fields": {"id": ["$.id"], "content": ["$.b"]}, "url_template": "https://x/y"})))
+    assert any("item_selector" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["comments"].pop("item_selector")))
+    assert any("pagination.style" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "psychic"})))
+    assert any("param" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "page"})))
+    assert any("per_page" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "offset", "param": "o"})))
+    assert mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "none"})) == []
+    assert any("pagination.next" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["comments"].update(pagination={"style": "next_link", "next": "a[::attr(href)"})))
+    assert mutated(html_schema, lambda s: s["feeds"][0]["comments"].update(pagination={"style": "page_param", "param": "cp"})) == []
+
+
+def test_limits_have_ceilings_and_a_politeness_floor(json_schema):
+    assert any("max_posts" in item for item in mutated(json_schema, lambda s: s["limits"].update(max_posts=100000)))
+    assert any("delay_seconds" in item and "polite" in item for item in mutated(json_schema, lambda s: s["limits"].update(delay_seconds=0.01)))
+    assert any("unknown limit" in item for item in mutated(json_schema, lambda s: s["limits"].update(turbo=1)))
+    assert any("positive" in item for item in mutated(json_schema, lambda s: s["limits"].update(retries=-1)))
+    assert any("positive" in item for item in mutated(json_schema, lambda s: s["limits"].update(retries=True)))
+    assert mutated(json_schema, lambda s: s["limits"].update(delay_seconds=2.5, max_posts=10)) == []
+
+
+def test_login_specs(json_schema):
+    assert any("login.mode" in item for item in mutated(json_schema, lambda s: s.update(login={"mode": "magic"})))
+    assert any("username_field" in item for item in mutated(json_schema, lambda s: s.update(login={"mode": "json", "url": "https://x/l"})))
+    assert mutated(json_schema, lambda s: s.update(login={"mode": "json", "url": "https://x/l", "username_field": "u", "password_field": "p", "token_path": "$.t"})) == []
+
+
+def test_the_sample_run_is_small_and_polite(json_schema):
+    limits = sample_limits(json_schema)
+    assert limits["max_pages"] == 1 and limits["max_comment_pages"] == 1 and limits["max_posts"] <= 12
+    assert limits["delay_seconds"] >= 1.0
+    fast = copy.deepcopy(json_schema)
+    fast["limits"]["delay_seconds"] = 5.0
+    assert sample_limits(fast)["delay_seconds"] == 5.0

tests/unit/services/community/structured.py

@@ -0,0 +1,100 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+import pytest
+
+from devplacepy.services.community.discoverer import CommunityDiscoverer
+from devplacepy.services.community.extract import extract_json_ld_posts, parse_html
+from devplacepy.services.community.generator import CodeGenerator
+from devplacepy.services.community.runtime import Collector
+from devplacepy.services.community.schemacheck import validate_schema
+from devplacepy.services.sync.records import RECORD_MODELS
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher
+from tests.unit.services.community.generator import load, no_sleep
+
+
+def thread(number: int, comments: int = 2) -> dict:
+    return {
+        "@type": "DiscussionForumPosting",
+        "@id": f"https://ld.test/t/{number}",
+        "headline": f"Thread number {number} about parsing",
+        "text": f"This is the body of thread {number}, written long enough to sync as a post.",
+        "author": {"@type": "Person", "name": f"writer{number % 2}", "url": f"https://ld.test/u/{number % 2}"},
+        "datePublished": f"2026-10-{number:02d}T10:00:00Z",
+        "dateModified": f"2026-10-{number:02d}T12:00:00Z",
+        "url": f"https://ld.test/t/{number}",
+        "comment": [
+            {
+                "@type": "Comment",
+                "@id": f"https://ld.test/t/{number}#c{index}",
+                "text": f"Comment {index} on thread {number}",
+                "author": {"@type": "Person", "name": f"replier{index}"},
+                "dateCreated": f"2026-10-{number:02d}T11:0{index}:00Z",
+            }
+            for index in range(comments)
+        ],
+    }
+
+
+def page(blobs: list) -> str:
+    scripts = "".join(f'<script type="application/ld+json">{json.dumps(blob)}</script>' for blob in blobs)
+    return f"<html><head><title>LD Forum</title>{scripts}</head><body><div id=app>Loading</div></body></html>"
+
+
+def ld_site(blobs=None) -> dict:
+    blobs = blobs or [{"@context": "https://schema.org", "@graph": [thread(number) for number in range(1, 7)]}]
+    return {"https://ld.test/": page(blobs), "https://ld.test/about": "<html><body><p>About: 100 members.</p></body></html>"}
+
+
+def test_the_extractor_normalizes_graphs_lists_and_nested_comments():
+    root = parse_html(page([{"@graph": [thread(1), {"@type": "WebSite", "name": "x"}]}, thread(2), [thread(3)], {"@type": "ItemList", "itemListElement": [{"item": thread(4)}]}]))
+    posts = extract_json_ld_posts(root)
+    assert [item["id"] for item in posts] == [f"https://ld.test/t/{n}" for n in (1, 2, 3, 4)]
+    first = posts[0]
+    assert first["title"] == "Thread number 1 about parsing" and first["author"] == {"username": "writer1", "id": "https://ld.test/u/1"}
+    assert first["created_at"].startswith("2026-10-01") and len(first["comments"]) == 2
+    assert first["comments"][0]["content"] == "Comment 0 on thread 1" and first["comments"][0]["author"] == {"username": "replier0"}
+
+
+def test_garbage_and_non_posts_are_ignored():
+    root = parse_html('<script type="application/ld+json">{not json</script><script type="application/ld+json">{"@type":"Organization","name":"x"}</script><script>var x=1</script>')
+    assert extract_json_ld_posts(root) == []
+    nameless = parse_html(page([{"@type": "Article", "headline": "No identity at all"}]))
+    assert extract_json_ld_posts(nameless) == []
+
+
+def test_a_site_with_only_structured_data_is_learned_end_to_end():
+    result = run_async(CommunityDiscoverer(FakeFetcher(ld_site())).discover("https://ld.test/"))
+    assert result.status == "ready", result.error
+    feed = result.schema["feeds"][0]
+    assert feed["type"] == "json_ld" and feed["post"]["fields"]["id"] and feed["comments"]["mode"] == "inline"
+    assert validate_schema(result.schema) == []
+    records = result.sample["records"]
+    assert len(records["posts"]) == 6 and len(records["comments"]) == 6 and records["users"]
+    full = run_async(Collector(FakeFetcher(ld_site()), result.schema, sleep=no_sleep).run())
+    assert len(full["records"]["comments"]) == 12
+    for kind, rows in records.items():
+        for row in rows:
+            RECORD_MODELS[kind].model_validate(row)
+    assert {user["username"] for user in records["users"]} >= {"writer0", "writer1", "replier0"}
+
+
+def test_the_collector_and_the_generated_file_agree_on_structured_data(tmp_path):
+    schema = run_async(CommunityDiscoverer(FakeFetcher(ld_site())).discover("https://ld.test/")).schema
+    direct = run_async(Collector(FakeFetcher(ld_site()), schema, sleep=no_sleep).run())
+    module = load(CodeGenerator(schema).write(tmp_path))
+    generated = run_async(module.collect_all(FakeFetcher(ld_site()), sleep=no_sleep))
+    assert {kind: len(rows) for kind, rows in direct["records"].items()} == {kind: len(rows) for kind, rows in generated["records"].items()}
+
+
+def test_a_single_structured_post_is_not_a_feed():
+    result = run_async(CommunityDiscoverer(FakeFetcher(ld_site([thread(1)]))).discover("https://ld.test/"))
+    assert result.status != "ready"
+
+
+@pytest.mark.parametrize("count", [2, 5])
+def test_few_posts_still_make_a_feed(count):
+    result = run_async(CommunityDiscoverer(FakeFetcher(ld_site([{"@graph": [thread(number, 0) for number in range(1, count + 1)]}]))).discover("https://ld.test/"))
+    assert result.status == "ready", result.error

tests/unit/services/community/submitter.py

@@ -0,0 +1,60 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.community.extract import split_submitter
+from devplacepy.services.community.runtime import Collector
+from tests.conftest import run_async
+from tests.unit.services.community.dedupe import entry
+from tests.unit.services.community.fakes import FakeFetcher
+from tests.unit.services.community.generator import no_sleep
+
+
+@pytest.mark.parametrize(
+    "raw,expected",
+    [
+        ("lore.kernel.org via thang", "thang"),
+        ("jakearchibald.com by jaffathecake", "jaffathecake"),
+        ("decomp.dev via FedericoSchonborn", "FedericoSchonborn"),
+        ("via rberger 2026-10-15 12:36:38", "rberger"),
+        (" via spikewall", "spikewall"),
+        ("plainname", "plainname"),
+        ("two words", "two words"),
+        ("", ""),
+        ("site.org VIA Someone", "Someone"),
+    ],
+)
+def test_the_submitter_is_the_person_behind_by_or_via(raw, expected):
+    assert split_submitter(raw) == expected
+
+
+def schema():
+    return {
+        "community": "lobsters",
+        "origin": "lobsters.test",
+        "limits": {"delay_seconds": 0, "retries": 1},
+        "feeds": [
+            {
+                "id": "front",
+                "url": "https://lobsters.test/front",
+                "type": "json_api",
+                "list_path": "$.items",
+                "post": {"fields": {"id": ["$.id"], "title": ["$.title"], "content": ["$.body"], "created_at": ["$.at"], "url": ["$.link"]}},
+                "author": {"fields": {"username": ["$.who"]}},
+                "comments": {"mode": "none"},
+            }
+        ],
+    }
+
+
+def test_one_person_sharing_from_many_sites_is_one_user_with_one_identity():
+    items = [
+        {**entry("S1", "https://decomp.dev/a", "First"), "who": "decomp.dev via FedericoSchonborn"},
+        {**entry("S2", "https://costa.jacobpalm.dk/b", "Second"), "who": "costa.jacobpalm.dk via FedericoSchonborn"},
+        {**entry("S3", "https://abner.page/c", "Third"), "who": "abner.page by abnercoimbre"},
+    ]
+    result = run_async(Collector(FakeFetcher({"https://lobsters.test/front": {"items": items}}), schema(), None, sleep=no_sleep).run(None))
+    users = result["records"]["users"]
+    assert sorted(user["username"] for user in users) == ["FedericoSchonborn", "abnercoimbre"]
+    posts = {post["title"]: post["author_uid"] for post in result["records"]["posts"]}
+    assert posts["First"] == posts["Second"] != posts["Third"]

tests/unit/services/devii/actions/catalog.py

@@ -95,6 +95,7 @@ def test_game_actions_registered_with_correct_auth():
     auth = {
         "game_plant",
         "game_harvest",
+        "game_plot_stop",
         "game_buy_plot",
         "game_upgrade_ci",
         "game_water",

tests/unit/services/game/store.py

@@ -1178,3 +1178,111 @@ def test_claim_grant_partial_treasury(local_db):
     result = store.claim_grant(user)
     assert result["amount"] == 300
     assert store.treasury_balance() == 0
+
+
+# --- per-plot auto-replant pause ---------------------------------------------------------
+
+
+def _plot(user, slot=0):
+    return store.get_plots(store.get_farm(user["uid"])["uid"])[slot]
+
+
+def test_autoreplant_replants_by_default(local_db):
+    user = _reset("unit_a")
+    _set(user, mastery_autoreplant=1)
+    store.plant(user, 0, "shell")
+    _warp(user)
+    store.harvest(user, 0)
+    plot = _plot(user)
+    assert plot["crop_key"] == "shell"
+    assert not plot.get("stop_after_harvest")
+
+
+def test_plot_stop_skips_one_replant_then_plant_resets_flag(local_db):
+    user = _reset("unit_a")
+    _set(user, mastery_autoreplant=1)
+    store.plant(user, 0, "shell")
+    assert store.set_plot_stop(user, 0, True)["stop"] is True
+    _warp(user)
+    store.harvest(user, 0)
+    assert not _plot(user).get("crop_key")
+    store.plant(user, 0, "python")
+    plot = _plot(user)
+    assert plot["crop_key"] == "python"
+    assert not plot.get("stop_after_harvest")
+    _warp(user)
+    store.harvest(user, 0)
+    assert _plot(user)["crop_key"] == "python"
+
+
+def test_plot_stop_can_be_cancelled(local_db):
+    user = _reset("unit_a")
+    _set(user, mastery_autoreplant=1)
+    store.plant(user, 0, "shell")
+    store.set_plot_stop(user, 0, True)
+    store.set_plot_stop(user, 0, False)
+    _warp(user)
+    store.harvest(user, 0)
+    assert _plot(user)["crop_key"] == "shell"
+
+
+def test_plot_stop_requires_continuous_delivery(local_db):
+    user = _reset("unit_a")
+    store.plant(user, 0, "shell")
+    with pytest.raises(GameError):
+        store.set_plot_stop(user, 0, True)
+
+
+def test_plot_stop_rejects_empty_plot(local_db):
+    user = _reset("unit_a")
+    _set(user, mastery_autoreplant=1)
+    with pytest.raises(GameError):
+        store.set_plot_stop(user, 0, True)
+
+
+def test_plot_stop_honoured_by_auto_harvest(local_db):
+    user = _reset("unit_a")
+    _set(user, mastery_autoreplant=1, legacy_autoharvest=1)
+    store.plant(user, 0, "shell")
+    store.plant(user, 1, "shell")
+    store.set_plot_stop(user, 0, True)
+    _warp(user)
+    store.serialize_farm(store.get_farm(user["uid"]), viewer=user, owner=user)
+    assert not _plot(user, 0).get("crop_key")
+    assert _plot(user, 1)["crop_key"] == "shell"
+
+
+def test_autoreplant_skips_locked_crop(local_db):
+    user = _reset("unit_a")
+    _set(user, mastery_autoreplant=1)
+    store.plant(user, 0, "shell")
+    _warp(user)
+    original = economy.crop_lock_reason
+    economy.crop_lock_reason = lambda *args, **kwargs: ("level", "is locked")
+    try:
+        store.harvest(user, 0)
+    finally:
+        economy.crop_lock_reason = original
+    assert not _plot(user).get("crop_key")
+
+
+def test_plot_stop_serialized_for_owner_only(local_db):
+    owner = _reset("unit_a")
+    visitor = _reset("unit_b")
+    _set(owner, mastery_autoreplant=1)
+    store.plant(owner, 0, "shell")
+    store.set_plot_stop(owner, 0, True)
+    farm = store.get_farm(owner["uid"])
+    own_view = store.serialize_farm(farm, viewer=owner, owner=owner)["plots"][0]
+    assert own_view["can_stop_replant"] and own_view["stop_after_harvest"]
+    other_view = store.serialize_farm(farm, viewer=visitor, owner=owner)["plots"][0]
+    assert not other_view["can_stop_replant"] and not other_view["stop_after_harvest"]
+
+
+def test_prestige_resets_plot_stop(local_db):
+    user = _reset("unit_a", coins=10**9)
+    _set(user, mastery_autoreplant=1, xp=10**7, level=economy.PRESTIGE_MIN_LEVEL)
+    store.plant(user, 0, "shell")
+    store.set_plot_stop(user, 0, True)
+    store.prestige(user)
+    assert not _plot(user).get("stop_after_harvest")

tests/unit/services/jobs/deepsearch/crawl.py

@@ -135,3 +135,89 @@ def test_crawl_degrades_to_httpx_when_the_playwright_driver_cannot_start(monkeyp
     )
     assert len(outcome.pages) == 1
     assert outcome.pages[0].source == "httpx"
+
+
+REDDIT_URL = "https://www.reddit.com/r/python/comments/abc/post/"
+REDDIT_HTML = "<html><head><title>A thread</title></head><body><p>" + "Real thread text from a donor. " * 30 + "</p></body></html>"
+REDDIT_CANDIDATE = {"url": REDDIT_URL, "title": "t", "description": "", "content": "snippet " * 30}
+
+
+def _patch_node_client(monkeypatch, handler):
+    import httpx
+
+    real = crawl_module.stealth.stealth_async_client
+
+    def factory(**kwargs):
+        kwargs.pop("transport", None)
+        return real(transport=httpx.MockTransport(handler), **kwargs)
+
+    monkeypatch.setattr(crawl_module.stealth, "stealth_async_client", factory)
+
+
+def test_node_servable_is_exactly_the_node_allowlist():
+    assert crawl_module.node_servable(REDDIT_URL)
+    assert crawl_module.node_servable("https://redd.it/abc")
+    assert not crawl_module.node_servable("https://x.com/a/status/1")
+    assert not crawl_module.node_servable("https://evilreddit.com/")
+    assert not crawl_module.node_servable("https://reddit.com:22/")
+
+
+def test_a_hostile_reddit_page_is_fetched_through_the_nodes_when_a_key_is_present(monkeypatch):
+    seen = []
+
+    def handler(request):
+        import httpx
+
+        seen.append((request.url.path, request.headers.get("authorization"), request.content.decode()))
+        return httpx.Response(200, json={"url": REDDIT_URL, "status": 200, "via": "tunnel", "text": REDDIT_HTML})
+
+    _patch_node_client(monkeypatch, handler)
+    page = run_async(crawl_module._resolve_candidate(REDDIT_CANDIDATE, 1, None, "key123"))
+    assert page.source == "node-tunnel" and page.title == "A thread" and "Real thread text" in page.text
+    assert seen[0][0] == "/nodes/internal/fetch" and seen[0][1] == "Bearer key123" and "reddit.com" in seen[0][2]
+
+
+def test_without_a_key_or_when_the_nodes_refuse_the_snippet_is_used(monkeypatch):
+    def handler(request):
+        import httpx
+
+        return httpx.Response(503, json={"error": {"reason": "no_nodes"}})
+
+    _patch_node_client(monkeypatch, handler)
+    assert run_async(crawl_module._resolve_candidate(REDDIT_CANDIDATE, 1, None, "")).source == "search"
+    assert run_async(crawl_module._resolve_candidate(REDDIT_CANDIDATE, 1, None, "key")).source == "search"
+
+
+def test_a_blocked_donor_answer_falls_back_to_the_snippet(monkeypatch):
+    def handler(request):
+        import httpx
+
+        return httpx.Response(200, json={"url": REDDIT_URL, "status": 403, "via": "fetch", "text": "blocked " * 100})
+
+    _patch_node_client(monkeypatch, handler)
+    assert run_async(crawl_module._resolve_candidate(REDDIT_CANDIDATE, 1, None, "key")).source == "search"
+
+
+def test_other_hostile_hosts_never_reach_the_nodes(monkeypatch):
+    calls = []
+
+    def handler(request):
+        import httpx
+
+        calls.append(request.url)
+        return httpx.Response(200, json={})
+
+    _patch_node_client(monkeypatch, handler)
+    candidate = {"url": "https://x.com/a/status/1", "title": "t", "description": "", "content": "snippet " * 30}
+    assert run_async(crawl_module._resolve_candidate(candidate, 1, None, "key")).source == "search"
+    assert calls == []
+
+
+def test_a_thin_node_page_is_rejected(monkeypatch):
+    def handler(request):
+        import httpx
+
+        return httpx.Response(200, json={"url": REDDIT_URL, "status": 200, "via": "fetch", "text": "<html><body>tiny</body></html>"})
+
+    _patch_node_client(monkeypatch, handler)
+    assert run_async(crawl_module.node_page(REDDIT_URL, 1, "key")) is None

tests/unit/services/jobs/deepsearch/worker.py

@@ -21,7 +21,7 @@ def _patch_pipeline(monkeypatch, pages):
         return [{"url": page.url, "title": page.title, "description": ""} for page in pages]
 
     async def fake_crawl(
-        candidates, max_pages, emit, is_cached, should_stop, query="", depth=1, seen_hashes=None
+        candidates, max_pages, emit, is_cached, should_stop, query="", depth=1, seen_hashes=None, api_key=""
     ):
         outcome = CrawlOutcome(seen_hashes=seen_hashes if seen_hashes is not None else set())
         for page in pages[:max_pages]:
@@ -176,7 +176,7 @@ def test_worker_run_performs_refinement_round_when_budget_remains(monkeypatch):
         return [{"url": refined_page.url, "title": refined_page.title, "description": ""}]
 
     async def fake_crawl_refinement(
-        candidates, max_pages, emit, is_cached, should_stop, query="", depth=1, seen_hashes=None
+        candidates, max_pages, emit, is_cached, should_stop, query="", depth=1, seen_hashes=None, api_key=""
     ):
         outcome = CrawlOutcome(seen_hashes=seen_hashes if seen_hashes is not None else set())
         outcome.pages.append(refined_page)

tests/unit/services/nodes/addressclass.py

@@ -0,0 +1,56 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.nodes.addressclass import (
+    CLASS_DATACENTER,
+    CLASS_MOBILE,
+    CLASS_RESIDENTIAL,
+    CLASS_UNKNOWN,
+    DATACENTER_ASNS,
+    ELIGIBLE_CLASSES,
+    classify,
+)
+from devplacepy.services.nodes.spec import ASN_CLASSES
+
+
+def test_the_classes_are_exactly_the_specified_ones():
+    assert set(ELIGIBLE_CLASSES) | {CLASS_DATACENTER, CLASS_UNKNOWN} == set(ASN_CLASSES)
+
+
+@pytest.mark.parametrize(
+    "asn,org,expected",
+    [
+        (24940, "Hetzner Online GmbH", CLASS_DATACENTER),
+        (16509, "", CLASS_DATACENTER),
+        (64500, "Acme Web Hosting Ltd", CLASS_DATACENTER),
+        (64500, "Fast Cloud Services", CLASS_DATACENTER),
+        (64500, "Some VPN Provider", CLASS_DATACENTER),
+        (64500, "Data Center Holdings", CLASS_DATACENTER),
+        (21928, "T-Mobile USA, Inc.", CLASS_MOBILE),
+        (64500, "Example Wireless", CLASS_MOBILE),
+        (3320, "Deutsche Telekom AG", CLASS_RESIDENTIAL),
+        (1136, "KPN B.V.", CLASS_RESIDENTIAL),
+        (0, "", CLASS_UNKNOWN),
+        (None, None, CLASS_UNKNOWN),
+        ("", "", CLASS_UNKNOWN),
+    ],
+)
+def test_classification_table(asn, org, expected):
+    assert classify(asn, org) == expected
+
+
+def test_keywords_match_whole_words_only():
+    assert classify(64500, "Hostingly Residential Broadband") == CLASS_RESIDENTIAL
+    assert classify(64500, "Cloudless Telecom") == CLASS_RESIDENTIAL
+    assert classify(64500, "Wirelesstown ISP") == CLASS_RESIDENTIAL
+
+
+def test_a_known_asn_wins_over_a_residential_name():
+    assert 14061 in DATACENTER_ASNS
+    assert classify(14061, "Friendly Home Internet") == CLASS_DATACENTER
+
+
+def test_garbage_asn_input_is_unknown_or_name_driven():
+    assert classify("not-a-number", "") == CLASS_UNKNOWN
+    assert classify("not-a-number", "Plain ISP") == CLASS_RESIDENTIAL

tests/unit/services/nodes/cache.py

@@ -107,3 +107,73 @@ def test_clear_empties_everything():
     cache.put("GET", "u", 200, {}, b"x", False)
     cache.clear()
     assert len(cache) == 0 and cache.size_bytes == 0
+
+
+def test_age_moves_the_expiry_forward():
+    cache, clock = make(ttl_seconds=100)
+    cache.put("GET", "https://r/x", 200, {}, b"x", False, age_seconds=90)
+    clock.now = 105
+    assert cache.get("GET", "https://r/x") is not None
+    clock.now = 111
+    assert cache.get("GET", "https://r/x") is None
+
+
+def shared(local_db, **kwargs):
+    from devplacepy.database import nodes as store
+    from devplacepy.services.nodes.cache import SharedResultCache
+
+    store.cache_clear()
+    clock = Clock()
+    return SharedResultCache(clock=clock, **kwargs), clock, store
+
+
+def test_a_second_process_sees_what_the_first_stored(local_db):
+    first, _, store = shared(local_db, ttl_seconds=60)
+    assert first.put("GET", "https://r/a", 200, {"content-type": "text/plain"}, b"payload", False)
+    from devplacepy.services.nodes.cache import SharedResultCache
+
+    second = SharedResultCache(ttl_seconds=60)
+    hit = second.get("GET", "https://r/a")
+    assert hit is not None and hit.body == b"payload" and hit.headers == {"content-type": "text/plain"}
+    assert (second.hits, second.misses) == (1, 0)
+    again = second.get("GET", "https://r/a")
+    assert again is not None and second.hits == 2
+    store.cache_clear()
+
+
+def test_the_shared_layer_honours_ttl_truncation_and_server_errors(local_db):
+    cache, _, store = shared(local_db, ttl_seconds=60)
+    assert not cache.put("GET", "https://r/b", 500, {}, b"x", False)
+    assert not cache.put("GET", "https://r/b", 200, {}, b"x", True)
+    assert store.cache_stats()["entries"] == 0
+    cache.put("GET", "https://r/c", 200, {}, b"x", False)
+    from datetime import datetime, timedelta, timezone
+
+    assert store.cache_get("k-not-there") is None
+    key = cache_key("GET", "https://r/c")
+    assert store.cache_get(key) is not None
+    assert store.cache_get(key, datetime.now(timezone.utc) + timedelta(seconds=120)) is None
+    store.cache_clear()
+
+
+def test_clear_empties_both_layers(local_db):
+    cache, _, store = shared(local_db, ttl_seconds=60)
+    cache.put("GET", "https://r/d", 200, {}, b"x", False)
+    cache.clear()
+    assert len(cache) == 0 and store.cache_stats()["entries"] == 0
+
+
+def test_the_shared_layer_prunes_expired_rows_and_evicts_oldest_over_the_cap(local_db):
+    from datetime import datetime, timedelta, timezone
+
+    cache, _, store = shared(local_db, ttl_seconds=60)
+    now = datetime.now(timezone.utc)
+    for index in range(4):
+        store.cache_put(f"k{index}", "GET", f"https://r/{index}", 200, "{}", "AAAA", 1000, 60, now + timedelta(seconds=index))
+    store.cache_put("old", "GET", "https://r/old", 200, "{}", "AAAA", 10, 60, now - timedelta(seconds=300))
+    removed = store.cache_prune(2500)
+    assert removed["expired"] == 1 and removed["evicted"] == 2 and removed["bytes"] == 2000
+    left = store.cache_stats()
+    assert left["entries"] == 2
+    assert store.cache_get("k0") is None and store.cache_get("k3") is not None
+    store.cache_clear()

tests/unit/services/nodes/circuit.py

@@ -95,3 +95,57 @@ def test_a_zero_threshold_still_needs_one_failure():
     assert breaker.allows("x")
     breaker.failure("x")
     assert not breaker.allows("x")
+
+
+def test_snapshot_lists_open_circuits_with_time_left():
+    breaker, clock = make(failures=2, open_seconds=60)
+    breaker.failure(("destination", "reddit.com"))
+    breaker.failure(("destination", "reddit.com"))
+    clock.now = 20
+    rows = breaker.snapshot()
+    assert rows[0]["key"] == "destination:reddit.com" and rows[0]["state"] == "open"
+    assert 39 <= rows[0]["seconds_left"] <= 40 and rows[0]["failures"] == 2
+
+
+def test_dirty_flag_follows_every_change():
+    breaker, _ = make(failures=2)
+    assert not breaker.dirty
+    breaker.failure("x")
+    assert breaker.dirty
+    breaker.dirty = False
+    breaker.success("x")
+    assert breaker.dirty
+    breaker.dirty = False
+    breaker.success("never-seen")
+    assert not breaker.dirty
+
+
+def test_a_restart_keeps_an_open_circuit_open_until_its_time_is_up():
+    wall = {"now": 1_000_000.0}
+    clock = Clock()
+    first = CircuitBreaker(2, 60, clock, wall=lambda: wall["now"])
+    first.failure(("destination", "reddit.com"))
+    first.failure(("destination", "reddit.com"))
+    clock.now = 10
+    wall["now"] += 10
+    saved = first.snapshot()
+    second_clock = Clock()
+    second_clock.now = 5000
+    second = CircuitBreaker(2, 60, second_clock, wall=lambda: wall["now"] + 5)
+    assert second.restore(saved) == 1
+    assert second.state(("destination", "reddit.com")) == OPEN
+    second_clock.now += 60
+    assert second.state(("destination", "reddit.com")) == HALF_OPEN
+
+
+def test_restore_ignores_closed_ancient_and_malformed_rows():
+    wall = {"now": 1_000_000.0}
+    breaker = CircuitBreaker(3, 60, Clock(), wall=lambda: wall["now"])
+    rows = [
+        {"key": "destination:a", "failures": 1, "opened_at": wall["now"]},
+        {"key": "destination:b", "failures": 3, "opened_at": wall["now"] - 10_000},
+        {"key": "nocolon", "failures": 9, "opened_at": wall["now"]},
+        {"key": "destination:c", "failures": 9, "opened_at": 0},
+    ]
+    assert breaker.restore(rows) == 0
+    assert breaker.open_count() == 0

tests/unit/services/nodes/dispatcher.py

@@ -19,6 +19,9 @@ from devplacepy.services.nodes.scheduler import MEGABYTE, NodeScheduler
 from devplacepy.services.nodes.session import NodeSession
 from devplacepy.services.nodes.spec import SPEC
 from devplacepy.utils import generate_uid
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+
 from tests.conftest import run_async
 from tests.unit.services.nodes.scheduler import Clock, node, settings
 
@@ -26,6 +29,11 @@ LIMITS = SPEC.limits
 TARGET_URL = "https://www.reddit.com/r/python.json"
 
 
+def run_in_thread(coro):
+    with ThreadPoolExecutor(max_workers=1) as pool:
+        return pool.submit(asyncio.run, coro).result()
+
+
 class Rig:
     def __init__(self):
         self.clock = Clock()
@@ -38,12 +46,13 @@ class Rig:
             self.registry, self.scheduler, self.broker, self.breaker, UsageLedger(), self.counters,
             clock=self.clock,
         )
+        self.dispatcher.redundancy_percent = staticmethod(lambda: 0)
 
     def connect(self, **overrides):
         owner = generate_uid()
         device, _ = store.create_device(owner, "d", "native")
         session = NodeSession(
-            node().websocket, {"uid": device["uid"], "user_uid": owner, "kind": "native"}
+            node().websocket, {"uid": device["uid"], "user_uid": owner, "kind": "native", "asn_class": "residential"}
         )
         session.hello = {"modes": ["fetch"]}
         session.settings = settings(**overrides)
@@ -413,3 +422,210 @@ def test_the_owner_exclusion_does_not_saturate_the_destination(rig):
     other.host_started.clear()
     rig.tick()
     assert second in mine.inflight
+
+
+def test_the_claim_records_the_donor_address_class(rig):
+    session = rig.connect()
+    session.asn_class = "mobile"
+    uid = rig.enqueue()
+    rig.tick()
+    assert store.get_request(uid)["device_asn_class"] == "mobile"
+
+
+def test_outcomes_are_observed_per_class(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    rig.answer(session, uid)
+    other = rig.enqueue(url="https://www.reddit.com/r/two.json")
+    rig.clock.now += 100
+    rig.tick()
+    rig.answer(session, other, status=403)
+    third = rig.enqueue(url="https://www.reddit.com/r/three.json")
+    rig.clock.now += 100
+    rig.tick()
+    rig.answer(session, third, status=404)
+    seen = rig.counters.outcomes()
+    assert seen["outcome:reddit:residential:ok"] == 2 and seen["outcome:reddit:residential:blocked"] == 1
+
+
+def test_a_node_failure_is_observed_but_a_donor_limit_is_not(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    run_async(session._handle({"type": "job_error", "job_id": uid, "code": "fetch_failed"}))
+    assert rig.counters.outcomes().get("outcome:reddit:residential:failed") == 1
+    rig.clock.now += 100
+    session.paused_until = 0
+    second = rig.enqueue(url="https://www.reddit.com/r/two.json")
+    rig.tick()
+    run_async(session._handle({"type": "job_error", "job_id": second, "code": "outside_hours"}))
+    assert rig.counters.outcomes().get("outcome:reddit:residential:failed") == 1
+
+
+def test_on_battery_pauses_the_donor_without_blaming_it(rig):
+    session = rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    run_async(session._handle({"type": "job_error", "job_id": uid, "code": "on_battery"}))
+    assert session.paused_until > rig.clock()
+    assert not rig.breaker.failures or rig.breaker.allows(("node", session.device_uid))
+    assert store.get_request(uid)["error"] == "on_battery"
+
+
+class SampleAll:
+    def __init__(self, rig):
+        rig.dispatcher.redundancy_percent = staticmethod(lambda: 100)
+
+
+def mismatch_flags(device_uid=None):
+    return [
+        flag
+        for flag in store.list_flags()
+        if flag["kind"] == "redundancy_mismatch" and (device_uid is None or flag["device_uid"] == device_uid)
+    ]
+
+
+def sent_to(session):
+    return [frame["job_id"] for frame in session.websocket.sent if frame["type"] == "job"]
+
+
+def test_a_sampled_fetch_result_is_cross_checked_on_a_different_donor(rig):
+    SampleAll(rig)
+    first, second = rig.connect(), rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    holder = first if uid in sent_to(first) else second
+    other = second if holder is first else first
+    rig.answer(holder, uid, headers={"content-type": "application/json"}, body_b64=base64.b64encode(b'{"kind":"Listing","data":{}}').decode())
+    shadows = list(get_table("node_requests").find(shadow_of=uid))
+    assert len(shadows) == 1 and shadows[0]["exclude_device"] == holder.device_uid
+    assert shadows[0]["caller_kind"] == "shadow"
+    rig.clock.now += 100
+    assert rig.tick() == 1
+    assert sent_to(other) and shadows[0]["uid"] in sent_to(other)
+    rig.answer(other, shadows[0]["uid"], headers={"content-type": "application/json"}, body_b64=base64.b64encode(b'{"kind":"Listing","data":{}}').decode())
+    counts = rig.counters.snapshot()
+    assert counts["redundancy_checked"] == 1 and counts["redundancy_matched"] == 1
+    assert not mismatch_flags(other.device_uid)
+
+
+def test_a_forged_second_opinion_raises_a_flag_and_no_ban(rig):
+    SampleAll(rig)
+    first, second = rig.connect(), rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    holder = first if uid in sent_to(first) else second
+    other = second if holder is first else first
+    rig.answer(holder, uid, headers={"content-type": "application/json"}, body_b64=base64.b64encode(b'{"kind":"Listing","data":{}}').decode())
+    shadow = list(get_table("node_requests").find(shadow_of=uid))[0]["uid"]
+    rig.clock.now += 100
+    rig.tick()
+    rig.answer(other, shadow, headers={"content-type": "application/json"}, body_b64=base64.b64encode(b'{"buy":"now"}').decode())
+    assert rig.counters.snapshot()["redundancy_mismatches"] == 1
+    flags = mismatch_flags(other.device_uid)
+    assert len(flags) == 1
+    assert effective_status_of(other) == "active"
+
+
+def effective_status_of(session):
+    return store.effective_status(store.get_device(session.device_uid))
+
+
+def test_a_shadow_never_spends_the_caller_quota_and_is_not_cross_checked_again(rig):
+    SampleAll(rig)
+    first, second = rig.connect(), rig.connect()
+    uid = rig.enqueue()
+    rig.tick()
+    holder = first if uid in sent_to(first) else second
+    other = second if holder is first else first
+    rig.answer(holder, uid)
+    shadow = list(get_table("node_requests").find(shadow_of=uid))[0]["uid"]
+    rig.clock.now += 100
+    rig.tick()
+    rig.answer(other, shadow)
+    assert get_table("node_requests").count(shadow_of={"!=": ""}) == 1
+
+
+def test_sampling_is_deterministic_and_close_to_the_requested_rate():
+    uids = [generate_uid() for _ in range(10000)]
+    chosen = [uid for uid in uids if NodeDispatcher.sampled(uid, 5)]
+    assert chosen == [uid for uid in uids if NodeDispatcher.sampled(uid, 5)]
+    assert 350 <= len(chosen) <= 650
+    assert not any(NodeDispatcher.sampled(uid, 0) for uid in uids[:500])
+    assert all(NodeDispatcher.sampled(uid, 100) for uid in uids[:500])
+
+
+def test_the_percent_is_clamped_to_the_spec_ceiling(local_db):
+    from devplacepy.database import set_setting
+
+    set_setting("nodes_redundancy_percent", "99")
+    try:
+        assert NodeDispatcher.redundancy_percent() == LIMITS.redundancy_max_percent
+        set_setting("nodes_redundancy_percent", "-4")
+        assert NodeDispatcher.redundancy_percent() == 0
+    finally:
+        set_setting("nodes_redundancy_percent", str(LIMITS.redundancy_percent))
+
+
+def test_a_missing_original_is_inconclusive_not_a_flag(rig):
+    session = rig.connect()
+    rig.dispatcher._compare_shadow(session, type("J", (), {"shadow_of": "gone", "destination": "reddit.com"})(), 200, {}, b"x")
+    assert rig.counters.snapshot()["redundancy_inconclusive"] == 1
+    assert not mismatch_flags(session.device_uid)
+
+
+def test_a_waiter_in_the_dispatcher_process_wakes_without_polling(rig):
+    import asyncio
+
+    from devplacepy.services.nodes.wake import HUB
+
+    async def scenario():
+        HUB.attach()
+        target = DestinationPolicy().check(TARGET_URL)
+        uid = rig.broker.enqueue("user", "c1", target, "any")
+        store.claim_request(uid, "d1")
+        loop = asyncio.get_running_loop()
+        loop.call_later(0.02, lambda: rig.broker.complete(uid, 200, {}, b"ok", False))
+        started = time.monotonic()
+        result = await rig.broker.wait(uid, 5)
+        return result, time.monotonic() - started
+
+    try:
+        result, elapsed = run_in_thread(scenario())
+    finally:
+        HUB.enabled = True
+    assert result.ok and result.body == b"ok"
+    assert elapsed < LIMITS.poll_interval_ms / 1000
+
+
+def test_a_waiter_without_the_hub_still_finishes_by_polling(rig):
+    import asyncio
+
+    from devplacepy.services.nodes.wake import HUB
+
+    async def scenario():
+        target = DestinationPolicy().check(TARGET_URL)
+        uid = rig.broker.enqueue("user", "c2", target, "any")
+        store.claim_request(uid, "d1")
+        asyncio.get_running_loop().call_later(0.02, lambda: rig.broker.complete(uid, 200, {}, b"ok", False))
+        return await rig.broker.wait(uid, 5)
+
+    HUB._loop = None
+    assert run_in_thread(scenario()).ok
+
+
+def test_enqueue_signals_the_dispatcher(rig):
+    import asyncio
+
+    from devplacepy.services.nodes.wake import HUB
+
+    async def scenario():
+        HUB.attach()
+        target = DestinationPolicy().check(TARGET_URL)
+        woke = asyncio.ensure_future(HUB.wait_queued(5))
+        await asyncio.sleep(0)
+        rig.broker.enqueue("user", "c3", target, "any")
+        return await asyncio.wait_for(woke, 1)
+
+    assert run_in_thread(scenario()) is True

tests/unit/services/nodes/fabric.py

@@ -60,7 +60,9 @@ def online(local_db):
     set_setting("nodes_daily_quota_user", "1000")
     set_setting("nodes_daily_quota_guest", "0")
     heartbeat(3)
+    store.cache_clear()
     yield
+    store.cache_clear()
     set_setting("nodes_enabled", "0")
     get_table("service_state").delete(name="nodes")
 

tests/unit/services/nodes/geo.py

@@ -0,0 +1,126 @@
+# retoor <retoor@molodetz.nl>
+
+import gzip
+from pathlib import Path
+
+import pytest
+
+from devplacepy.services.nodes import geo
+from devplacepy.services.nodes.geo import GeoDatabase, GeoError, build, pack
+
+ASN_ROWS = [
+    ("1.0.0.0", "1.0.0.255", 13335, "Cloudflare, Inc."),
+    ("8.8.8.0", "8.8.8.255", 15169, "Google LLC"),
+    ("20.0.0.0", "20.0.0.255", 64500, "Home Fiber BV"),
+    ("2001:db8::", "2001:db8::ffff", 64501, "Mobile Wireless NV"),
+]
+COUNTRY_ROWS = [
+    ("1.0.0.0", "1.0.0.255", "AU"),
+    ("8.8.8.0", "8.8.8.255", "US"),
+    ("20.0.0.0", "20.0.0.255", "NL"),
+    ("2001:db8::", "2001:db8::ffff", "ZZ"),
+]
+
+
+def lines(rows):
+    return [",".join(f'"{part}"' if "," in str(part) else str(part) for part in row) for row in rows]
+
+
+def make(tmp_path, monkeypatch, checks=()):
+    monkeypatch.setattr(geo, "SPEC", geo.GeoSpec(min_asn_rows=1, min_country_rows=1, checks=checks))
+    target = tmp_path / "geo.db"
+    built = build(lines(ASN_ROWS), lines(COUNTRY_ROWS), target, "2026-10")
+    geo.install(built, target)
+    return GeoDatabase(target)
+
+
+def test_pack_normalises_v4_mapped_and_rejects_garbage():
+    assert pack("::ffff:8.8.8.8") == pack("8.8.8.8")
+    assert pack("[2001:db8::1]")[0] == 6
+    with pytest.raises(ValueError):
+        pack("nope")
+
+
+def test_lookup_hits_first_last_and_middle_of_a_range(tmp_path, monkeypatch):
+    database = make(tmp_path, monkeypatch)
+    for address in ("1.0.0.0", "1.0.0.77", "1.0.0.255"):
+        found = database.lookup(address)
+        assert (found.asn, found.country, found.asn_class) == (13335, "AU", "datacenter")
+
+
+def test_lookup_between_and_outside_ranges_is_unknown(tmp_path, monkeypatch):
+    database = make(tmp_path, monkeypatch)
+    for address in ("1.0.1.0", "0.255.255.255", "9.9.9.9", "255.255.255.255"):
+        assert database.lookup(address).asn_class == "unknown"
+
+
+def test_ipv6_and_zz_country(tmp_path, monkeypatch):
+    database = make(tmp_path, monkeypatch)
+    found = database.lookup("2001:db8::1234")
+    assert found.asn == 64501 and found.country == "" and found.asn_class == "mobile"
+    assert database.lookup("2001:db9::1").asn == 0
+
+
+def test_classification_of_a_residential_range(tmp_path, monkeypatch):
+    assert make(tmp_path, monkeypatch).lookup("20.0.0.9").asn_class == "residential"
+
+
+def test_missing_file_and_bad_input_degrade_to_unknown(tmp_path):
+    database = GeoDatabase(tmp_path / "absent.db")
+    assert not database.available()
+    assert database.lookup("8.8.8.8").asn_class == "unknown"
+    assert database.status()["available"] is False
+    assert GeoDatabase(tmp_path / "absent.db").lookup("").asn_class == "unknown"
+
+
+def test_a_too_small_dataset_is_refused_and_leaves_no_file(tmp_path, monkeypatch):
+    monkeypatch.setattr(geo, "SPEC", geo.GeoSpec(min_asn_rows=10, min_country_rows=10, checks=()))
+    target = tmp_path / "geo.db"
+    with pytest.raises(GeoError):
+        build(lines(ASN_ROWS), lines(COUNTRY_ROWS), target, "2026-10")
+    assert not target.exists() and not target.with_suffix(".tmp").exists()
+
+
+def test_a_failed_sanity_check_refuses_and_keeps_the_current_database(tmp_path, monkeypatch):
+    current = make(tmp_path, monkeypatch)
+    before = current.path.read_bytes()
+    monkeypatch.setattr(geo, "SPEC", geo.GeoSpec(min_asn_rows=1, min_country_rows=1, checks=(("8.8.8.8", 99999),)))
+    with pytest.raises(GeoError):
+        build(lines(ASN_ROWS), lines(COUNTRY_ROWS), tmp_path / "geo.db", "2026-11")
+    assert current.path.read_bytes() == before
+
+
+def test_the_row_cap_is_enforced(tmp_path, monkeypatch):
+    monkeypatch.setattr(geo, "SPEC", geo.GeoSpec(max_rows=2, min_asn_rows=1, min_country_rows=1, checks=()))
+    with pytest.raises(GeoError):
+        build(lines(ASN_ROWS), lines(COUNTRY_ROWS), tmp_path / "geo.db", "2026-10")
+
+
+def test_malformed_rows_are_skipped_not_fatal(tmp_path, monkeypatch):
+    monkeypatch.setattr(geo, "SPEC", geo.GeoSpec(min_asn_rows=1, min_country_rows=1, checks=()))
+    asn = lines(ASN_ROWS) + ["garbage", "a,b,c,d", "1.2.3.4,1.2.3.5,notanumber,x"]
+    built = build(asn, lines(COUNTRY_ROWS), tmp_path / "geo.db", "2026-10")
+    assert built["asn_rows"] == len(ASN_ROWS)
+
+
+def test_status_reports_metadata_and_attribution(tmp_path, monkeypatch):
+    status = make(tmp_path, monkeypatch).status()
+    assert status["available"] and status["month"] == "2026-10" and "DB-IP" in status["attribution"]
+
+
+def test_months_walk_back_over_a_year_boundary():
+    from datetime import datetime, timezone
+
+    assert geo.months(datetime(2026, 1, 15, tzinfo=timezone.utc)) == ["2026-01", "2025-12", "2025-11"]
+
+
+def test_build_from_files_reads_gzip(tmp_path, monkeypatch):
+    monkeypatch.setattr(geo, "SPEC", geo.GeoSpec(min_asn_rows=1, min_country_rows=1, checks=()))
+    asn_file, country_file = tmp_path / "a.csv.gz", tmp_path / "c.csv.gz"
+    with gzip.open(asn_file, "wt") as handle:
+        handle.write("\n".join(lines(ASN_ROWS)))
+    with gzip.open(country_file, "wt") as handle:
+        handle.write("\n".join(lines(COUNTRY_ROWS)))
+    built = geo.build_from_files(asn_file, country_file, tmp_path / "geo.db", "2026-10")
+    geo.install(built, tmp_path / "geo.db")
+    assert GeoDatabase(tmp_path / "geo.db").lookup("8.8.8.8").asn == 15169

tests/unit/services/nodes/registry.py

@@ -0,0 +1,55 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.nodes.registry import (
+    COUNTER_NAMES,
+    NodeCounters,
+    outcome_name,
+)
+
+
+def test_inc_counts_both_the_live_view_and_the_pending_delta():
+    counters = NodeCounters()
+    counters.inc("kicked")
+    counters.inc("kicked")
+    assert counters.snapshot()["kicked"] == 2
+    assert counters.take_pending() == {"kicked": 2}
+    assert counters.take_pending() == {}
+    assert counters.snapshot()["kicked"] == 2
+
+
+def test_unknown_names_are_rejected():
+    counters = NodeCounters()
+    with pytest.raises(KeyError):
+        counters.inc("nope")
+
+
+def test_restore_pending_adds_back_what_a_failed_flush_took():
+    counters = NodeCounters()
+    counters.inc("jobs_done")
+    taken = counters.take_pending()
+    counters.inc("jobs_done")
+    counters.restore_pending(taken)
+    assert counters.take_pending() == {"jobs_done": 2}
+
+
+def test_outcomes_are_validated_and_bounded_in_shape():
+    counters = NodeCounters()
+    counters.observe("reddit", "residential", "ok")
+    counters.observe("reddit", "residential", "ok")
+    counters.observe("reddit", "", "blocked")
+    assert counters.outcomes() == {
+        "outcome:reddit:residential:ok": 2,
+        "outcome:reddit:unknown:blocked": 1,
+    }
+    for adapter, asn_class, kind in (("Re:dit", "x", "ok"), ("reddit", "a b", "ok"), ("reddit", "x", "weird"), ("x" * 40, "x", "ok")):
+        with pytest.raises(KeyError):
+            outcome_name(adapter, asn_class, kind)
+
+
+def test_every_declared_counter_can_be_incremented():
+    counters = NodeCounters()
+    for name in COUNTER_NAMES:
+        counters.inc(name)
+    assert all(value == 1 for value in counters.snapshot().values())

tests/unit/services/nodes/scheduler.py

@@ -43,7 +43,7 @@ def settings(**overrides):
 
 
 def node(uid="d1", user="u1", modes=("fetch",), **overrides):
-    session = NodeSession(FakeSocket(), {"uid": uid, "user_uid": user, "kind": "native"})
+    session = NodeSession(FakeSocket(), {"uid": uid, "user_uid": user, "kind": "native", "asn_class": "residential"})
     session.hello = {"modes": list(modes)}
     session.settings = settings(**overrides)
     session.roll_day()
@@ -281,3 +281,48 @@ def test_choose_applies_the_owner_exclusion_to_a_shared_candidate_list(world):
     assert world.choose(found, "bob") is mine
     assert world.choose([mine], "alice") is None
     assert world.choose([], "") is None
+
+
+def test_a_datacenter_donor_is_not_scheduled(world):
+    session = add(world)
+    session.asn_class = "datacenter"
+    assert world.pick(TARGET) is None
+
+
+def test_an_unknown_class_is_refused_unless_the_site_allows_it(world, local_db):
+    from devplacepy.database import set_setting
+
+    session = add(world)
+    session.asn_class = "unknown"
+    assert world.pick(TARGET) is None
+    set_setting("nodes_allow_unknown_class", "1")
+    try:
+        assert world.pick(TARGET) is session
+        session.asn_class = "datacenter"
+        assert world.pick(TARGET) is None
+    finally:
+        set_setting("nodes_allow_unknown_class", "0")
+
+
+def test_mobile_and_residential_are_eligible(world):
+    for index, asn_class in enumerate(("mobile", "residential")):
+        session = add(world, uid=f"c{index}", user=f"cu{index}")
+        session.asn_class = asn_class
+    assert len(world.candidates(TARGET)) == 2
+
+
+def test_an_admin_override_beats_the_class(world):
+    session = add(world)
+    session.asn_class = "datacenter"
+    session.class_override = "allow"
+    assert world.pick(TARGET) is session
+    session.class_override = ""
+    assert world.pick(TARGET) is None
+
+
+def test_class_refusals_are_counted(world):
+    session = add(world)
+    session.asn_class = "datacenter"
+    before = world.class_refusals
+    world.candidates(TARGET)
+    assert world.class_refusals > before

tests/unit/services/nodes/service.py

@@ -176,3 +176,78 @@ def test_the_dispatch_loop_starts_once_and_stops_with_the_service(service):
         assert service._dispatch_task is None and first.cancelled()
 
     run_async(scenario())
+
+
+def test_a_timed_suspension_tells_the_node_when_it_ends(service):
+    uid = member()
+    device, socket = connected(service, uid)
+    suspend_device(device["uid"], "test", 30)
+    reconcile(service)
+    frame = socket.sent[-1]
+    assert frame["reason"] == "suspended" and frame["until"].startswith("20")
+
+
+def test_other_kicks_carry_no_until(service):
+    _, socket = connected(service, member())
+    set_setting("nodes_enabled", "0")
+    reconcile(service)
+    assert "until" not in socket.sent[-1]
+
+
+def test_persist_state_touches_flushes_and_saves_circuits(service):
+    from devplacepy.database import nodes as store
+
+    uid = member()
+    device, _ = connected(service, uid)
+    get_table("node_devices").update({"uid": device["uid"], "last_seen_at": "2000-01-01T00:00:00+00:00"}, ["uid"])
+    store.flush_counters({})
+    service.counters.inc("kicked")
+    service.counters.observe("reddit", "residential", "ok")
+    service.breaker.failure(("destination", "reddit.com"))
+    service.persist_state(force=True)
+    assert store.get_device(device["uid"])["last_seen_at"] > "2020"
+    lifetime = store.lifetime_counters()
+    assert lifetime["kicked"] >= 1 and lifetime["outcome:reddit:residential:ok"] >= 1
+    assert service.counters.take_pending() == {}
+    assert not service.breaker.dirty
+
+
+def test_a_failed_flush_keeps_the_deltas(service, monkeypatch):
+    from devplacepy.services.nodes import service as module
+
+    service.counters.inc("kicked")
+
+    def boom(deltas):
+        raise RuntimeError("disk full")
+
+    monkeypatch.setattr(module, "flush_counters", boom)
+    with pytest.raises(RuntimeError):
+        service.persist_state(force=True)
+    assert service.counters.take_pending() == {"kicked": 1}
+
+
+def test_persist_state_respects_the_intervals(service, monkeypatch):
+    from devplacepy.services.nodes import service as module
+
+    connected(service, member())
+    calls = []
+    monkeypatch.setattr(module, "touch_devices", lambda uids: calls.append(list(uids)))
+    service.persist_state(force=True)
+    service.persist_state()
+    assert len(calls) == 1
+
+
+def test_open_circuits_survive_a_restart(service):
+    from devplacepy.database.nodes import load_circuits
+
+    for _ in range(service.breaker.failures):
+        service.breaker.failure(("destination", "reddit.com"))
+    service.persist_state()
+    assert [row["key"] for row in load_circuits()] == ["destination:reddit.com"]
+    fresh = NodesService()
+    run_async(fresh.on_enable())
+    assert fresh.breaker.open_count() == 1
+    fresh.breaker.success(("destination", "reddit.com"))
+    fresh.persist_state()
+    assert load_circuits() == []
+    run_async(fresh.on_disable())

tests/unit/services/nodes/shape.py

@@ -0,0 +1,70 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+from devplacepy.services.nodes.shape import INCONCLUSIVE, MATCH, MISMATCH, compare, shape_of
+
+JSON = {"content-type": "application/json; charset=utf-8"}
+HTML = {"content-type": "text/html"}
+
+
+def listing(children):
+    return json.dumps({"kind": "Listing", "data": {"after": None, "children": children}}).encode()
+
+
+def test_two_listings_with_different_content_match():
+    first = shape_of(200, JSON, listing([{"id": 1}]))
+    second = shape_of(200, JSON, listing([{"id": 2}, {"id": 3}]))
+    assert compare(first, second) == MATCH
+
+
+def test_a_forged_json_shape_mismatches():
+    first = shape_of(200, JSON, listing([]))
+    forged = shape_of(200, JSON, json.dumps({"message": "buy now"}).encode())
+    assert compare(first, forged) == MISMATCH
+
+
+def test_html_titles_decide():
+    page = b"<html><head><title> Python : r/python </title></head><body>" + b"x" * 600 + b"</body></html>"
+    same = b"<html><head><title>Python :   r/python</title></head><body>" + b"y" * 700 + b"</body></html>"
+    other = b"<html><head><title>Totally different</title></head><body>" + b"x" * 600 + b"</body></html>"
+    assert compare(shape_of(200, HTML, page), shape_of(200, HTML, same)) == MATCH
+    assert compare(shape_of(200, HTML, page), shape_of(200, HTML, other)) == MISMATCH
+
+
+def test_html_without_a_title_falls_back_to_the_first_heading():
+    first = shape_of(200, HTML, b"<h1>Hello <b>world</b></h1>")
+    second = shape_of(200, HTML, b"<h1>Hello world</h1>")
+    assert first.signature == "Hello world" and compare(first, second) == MATCH
+
+
+def test_a_kind_change_mismatches_even_with_the_same_status():
+    assert compare(shape_of(200, JSON, listing([])), shape_of(200, HTML, b"<title>x</title>")) == MISMATCH
+
+
+def test_blocked_or_error_responses_are_never_conclusive():
+    ok = shape_of(200, JSON, listing([]))
+    for status in (403, 429, 500, 404):
+        assert compare(ok, shape_of(status, HTML, b"<title>blocked</title>")) == INCONCLUSIVE
+        assert compare(shape_of(status, HTML, b""), ok) == INCONCLUSIVE
+
+
+def test_a_wild_size_difference_mismatches_only_for_real_bodies():
+    big = shape_of(200, HTML, b"<title>t</title>" + b"a" * 5000)
+    small = shape_of(200, HTML, b"<title>t</title>" + b"a" * 600)
+    tiny = shape_of(200, HTML, b"<title>t</title>")
+    assert compare(big, small) == MISMATCH
+    assert compare(big, tiny) == MATCH
+
+
+def test_invalid_json_is_its_own_kind():
+    broken = shape_of(200, JSON, b"{not json")
+    assert broken.kind == "json-invalid"
+    assert compare(broken, shape_of(200, JSON, listing([]))) == MISMATCH
+
+
+def test_nested_key_sets_are_bounded_and_unknown_types_compare_by_type():
+    deep = {"a": {"b": {"c": {"d": 1}}}}
+    assert "a.b" in shape_of(200, JSON, json.dumps(deep).encode()).signature
+    assert "a.b.c.d" not in shape_of(200, JSON, json.dumps(deep).encode()).signature
+    assert compare(shape_of(200, {"content-type": "image/png"}, b"1"), shape_of(200, {"content-type": "image/png"}, b"2")) == MATCH

tests/unit/services/nodes/wake.py

@@ -0,0 +1,90 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+import threading
+from concurrent.futures import ThreadPoolExecutor
+import time
+
+from devplacepy.services.nodes.wake import WakeHub
+
+
+def run(coro):
+    with ThreadPoolExecutor(max_workers=1) as pool:
+        return pool.submit(asyncio.run, coro).result()
+
+
+def test_a_disabled_or_detached_hub_just_sleeps():
+    async def scenario():
+        hub = WakeHub()
+        started = time.monotonic()
+        assert await hub.wait_queued(0.05) is False
+        assert time.monotonic() - started >= 0.04
+        hub.enabled = False
+        hub.notify_queued()
+        assert hub.job_event("x") is None
+
+    run(scenario())
+
+
+def test_notify_wakes_the_dispatcher_immediately():
+    async def scenario():
+        hub = WakeHub()
+        hub.attach()
+        asyncio.get_running_loop().call_later(0.02, hub.notify_queued)
+        started = time.monotonic()
+        assert await hub.wait_queued(5) is True
+        assert time.monotonic() - started < 1
+
+    run(scenario())
+
+
+def test_a_notification_before_the_wait_is_not_lost():
+    async def scenario():
+        hub = WakeHub()
+        hub.attach()
+        hub.notify_queued()
+        assert await hub.wait_queued(5) is True
+        assert await hub.wait_queued(0.02) is False
+
+    run(scenario())
+
+
+def test_job_events_are_per_job_and_only_for_the_attached_loop():
+    async def scenario():
+        hub = WakeHub()
+        assert hub.job_event("a") is None
+        hub.attach()
+        first, second = hub.job_event("a"), hub.job_event("b")
+        assert first is not second and hub.job_event("a") is first
+        hub.job_done("a")
+        assert first.is_set() and not second.is_set()
+        hub.forget("a")
+        assert hub.job_event("a") is not first
+
+    run(scenario())
+
+
+def test_another_loop_gets_no_event():
+    hub = WakeHub()
+    seen = []
+
+    async def attach():
+        hub.attach()
+
+    run(attach())
+
+    async def foreign():
+        seen.append(hub.job_event("z"))
+
+    run(foreign())
+    assert seen == [None]
+
+
+def test_notifying_from_another_thread_wakes_the_loop():
+    async def scenario():
+        hub = WakeHub()
+        hub.attach()
+        threading.Timer(0.02, hub.notify_queued).start()
+        assert await hub.wait_queued(5) is True
+
+    run(scenario())

tests/unit/services/skills/fetcher.py

@@ -92,3 +92,135 @@ def test_decode_text():
         decode_text(b"a\x00b")
     with pytest.raises(SkillFormatError):
         decode_text(b"\xff\xfe\xfa")
+
+
+import ipaddress
+import socket
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+from devplacepy.net_guard import is_blocked_address
+from devplacepy.services.skills.fetcher import fetch_skill
+from tests.conftest import run_async
+
+SKILL_TEXT = (
+    "---\nname: wire-skill\ndescription: Fetched over a real socket.\n---\n"
+    "# Wire skill\n\nRead [the guide](references/guide.md) and `references/missing.md`.\n"
+)
+
+
+class Handler(BaseHTTPRequestHandler):
+    routes: dict = {}
+
+    def do_GET(self):
+        entry = self.routes.get(self.path.split("?", 1)[0])
+        if entry is None:
+            self.send_response(404)
+            self.end_headers()
+            return
+        status, headers, body = entry
+        self.send_response(status)
+        for name, value in headers.items():
+            self.send_header(name, value)
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, *args):
+        return None
+
+
+def global_address() -> str | None:
+    probe = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    try:
+        probe.connect(("8.8.8.8", 80))
+        address = probe.getsockname()[0]
+    except OSError:
+        return None
+    finally:
+        probe.close()
+    return None if is_blocked_address(ipaddress.ip_address(address)) else address
+
+
+needs_global_address = pytest.mark.skipif(
+    global_address() is None,
+    reason="this host has no globally routable IPv4 address, so a local server cannot pass the SSRF guard without patching it",
+)
+
+
+@pytest.fixture
+def origin():
+    port_holder = {}
+
+    def text(body, kind="text/markdown"):
+        return 200, {"Content-Type": kind}, body.encode()
+
+    Handler.routes = {
+        "/ok/": text("<html><body>skill directory</body></html>", "text/html"),
+        "/ok/SKILL.md": text(SKILL_TEXT),
+        "/ok/references/guide.md": text("# Guide\n\nStep one.\n"),
+        "/big/SKILL.md": text("---\nname: big\ndescription: too big\n---\n" + "x" * (limits.SKILL_MD_MAX_BYTES + 10)),
+        "/html/SKILL.md": text("<html><body>not a skill</body></html>", "text/html"),
+        "/bounce/SKILL.md": (302, {"Location": "PLACEHOLDER"}, b""),
+        "/loop/SKILL.md": (302, {"Location": "/loop/SKILL.md"}, b""),
+    }
+    server = ThreadingHTTPServer(("0.0.0.0", 0), Handler)
+    port_holder["port"] = server.server_address[1]
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    yield port_holder["port"]
+    server.shutdown()
+    server.server_close()
+
+
+@needs_global_address
+def test_a_skill_is_fetched_over_a_real_socket_with_its_bundled_file(origin):
+    address = global_address()
+    fetched = run_async(fetch_skill(f"http://{address}:{origin}/ok/SKILL.md"))
+    assert fetched.parsed.name == "wire-skill" and fetched.source_url.endswith("/ok/SKILL.md")
+    assert fetched.files == {"references/guide.md": "# Guide\n\nStep one.\n"}
+    assert [(item.path, item.reason) for item in fetched.skipped] == [("references/missing.md", "the server returned 404")]
+
+
+@needs_global_address
+def test_the_directory_address_resolves_to_the_skill_file(origin):
+    address = global_address()
+    fetched = run_async(fetch_skill(f"http://{address}:{origin}/ok/"))
+    assert fetched.parsed.name == "wire-skill"
+
+
+@needs_global_address
+def test_loopback_is_refused_even_though_the_server_is_right_there(origin):
+    with pytest.raises(SkillFetchError, match="private or local"):
+        run_async(fetch_skill(f"http://127.0.0.1:{origin}/ok/SKILL.md"))
+
+
+@needs_global_address
+def test_a_redirect_to_loopback_is_refused_on_the_second_hop(origin):
+    address = global_address()
+    Handler.routes["/bounce/SKILL.md"] = (302, {"Location": f"http://127.0.0.1:{origin}/ok/SKILL.md"}, b"")
+    with pytest.raises(SkillFetchError, match="private or local"):
+        run_async(fetch_skill(f"http://{address}:{origin}/bounce/SKILL.md"))
+
+
+@needs_global_address
+def test_a_redirect_loop_ends_with_a_clean_error(origin):
+    address = global_address()
+    with pytest.raises(SkillFetchError):
+        run_async(fetch_skill(f"http://{address}:{origin}/loop/SKILL.md"))
+
+
+@needs_global_address
+def test_an_oversize_skill_is_cut_at_the_limit_and_html_is_rejected(origin):
+    address = global_address()
+    with pytest.raises(SkillFetchError, match="byte limit"):
+        run_async(fetch_skill(f"http://{address}:{origin}/big/SKILL.md"))
+    with pytest.raises(SkillFetchError, match="HTML page"):
+        run_async(fetch_skill(f"http://{address}:{origin}/html/SKILL.md"))
+
+
+@needs_global_address
+def test_a_missing_skill_reports_the_status(origin):
+    address = global_address()
+    with pytest.raises(SkillFetchError, match="404"):
+        run_async(fetch_skill(f"http://{address}:{origin}/nothing/SKILL.md"))

tests/unit/services/sync/handles.py

@@ -0,0 +1,134 @@
+# retoor <retoor@molodetz.nl>
+
+from collections import Counter
+from itertools import islice
+
+import pytest
+
+from devplacepy.database import get_table, init_db
+from devplacepy.database.sync import claim_identity
+from devplacepy.services.sync.batch import BatchRunner
+from devplacepy.services.sync.handles import HandleBuilder
+from devplacepy.services.sync.records import SyncBatchForm
+from devplacepy.services.sync.spec import HANDLES
+from devplacepy.utils import generate_uid
+
+ORIGIN = "handles.example.org"
+OWN_PERSON = {
+    "abner_page_by_abnercoimbre": "abnercoimbre",
+    "jakearchibald_com_by_jaffathecak": "jaffathecak",
+    "spritely_institute_via_untrusem": "untrusem",
+    "libreboot_org_via_runxiyu": "runxiyu",
+    "lumi_codeberg_page_via_stilic": "stilic",
+    "decomp_dev_via_FedericoSchonborn": "federicoschonborn",
+    "via_rberger_2026-10-15_12_36_38_": "rberger",
+    "via_veqq_2026-10-14_23_47_30_cac": "veqq",
+    "costa_jacobpalm_dk_via_FedericoS": "federicos",
+    "entropicthoughts_com_via_lcapald": "lcapald",
+    "github_com_via_gmem": "gmem",
+    "try_haxy_net_via_krig": "krig",
+    "youtu_be_via_jeffective": "jeffective",
+}
+DOMAIN_ONLY = {
+    "costa_jacobpalm_dk": "jacobpalm",
+    "techcommunity_microsoft_com": "microsoft",
+    "gtoal_com": "gtoal",
+    "javidjamae": "javidjamae",
+    "john_smith": "john_smith",
+}
+
+
+@pytest.mark.parametrize("raw,expected", sorted((OWN_PERSON | DOMAIN_ONLY).items()))
+def test_the_person_is_found_behind_the_site_and_the_stamp(raw, expected):
+    assert HandleBuilder.person(raw) == expected
+
+
+@pytest.mark.parametrize("raw", sorted(OWN_PERSON))
+def test_no_candidate_keeps_the_site_the_stamp_or_the_relation_word(raw):
+    for candidate in islice(HandleBuilder(ORIGIN, raw).candidates(raw), HANDLES.rerolls):
+        lowered = candidate.lower()
+        assert "_via_" not in lowered and "_by_" not in lowered
+        assert "2026" not in lowered and not lowered.startswith("via_")
+
+
+def test_every_candidate_is_a_valid_handle_across_the_input_domain():
+    raws = ["", "a", "ab", "x" * 80, "UPPER_case", "ünï-cödé", "__", "-a-", "via", "by_via", "9999", "john.smith", "a_b_c_d_e_f_g_h"]
+    raws += [f"user{index}_by_someone{index}" for index in range(300)]
+    for index, raw in enumerate(raws):
+        seen = list(islice(HandleBuilder(ORIGIN, f"u{index}").candidates(raw), 60))
+        assert seen
+        for candidate in seen:
+            assert HANDLES.min_length <= len(candidate) <= HANDLES.max_length, (raw, candidate)
+            assert HandleBuilder.valid(candidate), (raw, candidate)
+
+
+def test_the_same_record_always_gets_the_same_candidates():
+    first = list(islice(HandleBuilder(ORIGIN, "u1").candidates("someone_via_other"), 40))
+    second = list(islice(HandleBuilder(ORIGIN, "u1").candidates("someone_via_other"), 40))
+    assert first == second
+
+
+def test_the_mix_follows_the_spec_weights():
+    counts = Counter()
+    total = 4000
+    for index in range(total):
+        first = next(iter(HandleBuilder(ORIGIN, f"id{index}").candidates("kernelhacker")))
+        counts["plain" if first == "kernelhacker" else "digits" if any(char.isdigit() for char in first) else "word"] += 1
+    plain = counts["plain"] / total * 100
+    assert abs(plain - HANDLES.plain_weight) < 5
+    assert counts["digits"] / total * 100 > 25
+
+
+def test_candidates_differ_between_records_with_the_same_name():
+    firsts = {next(iter(HandleBuilder(ORIGIN, f"id{index}").candidates("same"))) for index in range(200)}
+    assert len(firsts) > 40
+
+
+@pytest.fixture
+def clean(local_db):
+    init_db()
+    yield
+    for table in ("sync_links", "sync_identities"):
+        get_table(table).delete(origin=ORIGIN)
+
+
+def push(records, humanize):
+    return BatchRunner("users", humanize=humanize).run(SyncBatchForm(origin=ORIGIN, records=records))
+
+
+def stored_name(origin_uid):
+    link = get_table("sync_links").find_one(origin=ORIGIN, kind="users", origin_uid=origin_uid)
+    return get_table("users").find_one(uid=link["local_uid"])["username"]
+
+
+def test_a_new_user_gets_a_realistic_handle_and_an_update_keeps_it(clean):
+    raw = f"abner_page_by_abnercoimbre{generate_uid()[-4:]}"
+    record = {"origin_uid": "h1", "username": raw, "created_at": "2025-01-01T00:00:00Z"}
+    push([record], humanize=True)
+    name = stored_name("h1")
+    assert name != raw and "_by_" not in name and name.startswith("abnercoimbre")
+    push([{**record, "bio": "changed bio", "updated_at": "2025-02-01T00:00:00Z"}], humanize=True)
+    assert stored_name("h1") == name
+
+
+def test_without_the_flag_the_original_behaviour_is_untouched(clean):
+    raw = f"plain_sender_{generate_uid()[-6:]}"
+    push([{"origin_uid": "h2", "username": raw, "created_at": "2025-01-01T00:00:00Z"}], humanize=False)
+    assert stored_name("h2") == raw
+
+
+def test_an_alias_clone_name_is_never_decorated(clean):
+    alias = f"mirakou{generate_uid()[-5:]}"
+    claim_identity(ORIGIN, "alice", "alice", alias, "{}", False)
+    push([{"origin_uid": "alice", "username": alias, "created_at": "2025-01-01T00:00:00Z"}], humanize=True)
+    assert stored_name("alice") == alias
+
+
+def test_two_senders_with_one_person_get_two_distinct_free_handles(clean):
+    base = f"samename{generate_uid()[-5:]}"
+    push(
+        [{"origin_uid": f"s{index}", "username": f"site_{index}_via_{base}", "created_at": "2025-01-01T00:00:00Z"} for index in range(12)],
+        humanize=True,
+    )
+    names = [stored_name(f"s{index}") for index in range(12)]
+    assert len({name.lower() for name in names}) == 12

tests/unit/services/sync/healing.py

@@ -0,0 +1,99 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.database import get_table, init_db
+from devplacepy.database.sync import claim_identity, claim_link
+from devplacepy.services.sync.healing import AliasHealer
+from devplacepy.services.sync.processors import UserProcessor
+from devplacepy.utils import generate_uid
+
+ORIGIN = "heal.example.org"
+
+
+@pytest.fixture(autouse=True)
+def clean(local_db):
+    init_db()
+    for table in ("sync_identities", "sync_links"):
+        get_table(table).delete(origin=ORIGIN)
+    yield
+    for table in ("sync_identities", "sync_links"):
+        get_table(table).delete(origin=ORIGIN)
+
+
+def clone(source="alice", alias="Mira"):
+    suffix = generate_uid()[-6:]
+    alias = f"{alias}{suffix}"
+    processor = UserProcessor(ORIGIN)
+    result = processor.process(0, {"origin_uid": source, "username": alias, "created_at": "2026-01-01T00:00:00Z"})
+    claim_identity(ORIGIN, source.capitalize(), source, alias, "style", False)
+    return result.local_uid, alias
+
+
+def synced_text(origin_uid, table, text, kind):
+    uid = generate_uid()
+    get_table(table).insert(
+        {"uid": uid, "user_uid": "u", "title": None, "content": text, "created_at": "2026-01-01T00:00:00+00:00", "target_uid": "x", "target_type": "post", "deleted_at": None, "deleted_by": None}
+    )
+    claim_link(ORIGIN, kind, origin_uid, uid, "fp", "2026-01-01T00:00:00+00:00")
+    return uid
+
+
+def test_nothing_changes_when_the_clone_still_has_the_alias():
+    clone()
+    report = AliasHealer(ORIGIN).heal()
+    assert not report.renamed and not report.conflicts and report.rewritten == 0
+
+
+def test_an_administrator_rename_moves_the_identity_and_the_stored_mentions():
+    uid, alias = clone()
+    post = synced_text("p1", "posts", f"thanks @{alias} and {alias} again, @{alias}x stays", "posts")
+    comment = synced_text("c1", "comments", f"@{alias.lower()} agreed", "comments")
+    other = synced_text("p2", "posts", "unrelated text", "posts")
+    renamed = f"Renamed{generate_uid()[-6:]}"
+    get_table("users").update({"uid": uid, "username": renamed}, ["uid"])
+    report = AliasHealer(ORIGIN).heal()
+    assert report.renamed == {alias: renamed} and report.rewritten == 2
+    row = get_table("sync_identities").find_one(origin=ORIGIN, source_key="alice")
+    assert row["our_name"] == renamed and row["our_key"] == renamed.lower()
+    assert get_table("posts").find_one(uid=post)["content"] == f"thanks @{renamed} and {renamed} again, @{alias}x stays"
+    assert get_table("comments").find_one(uid=comment)["content"] == f"@{renamed} agreed"
+    assert get_table("posts").find_one(uid=other)["content"] == "unrelated text"
+    assert not AliasHealer(ORIGIN).heal().renamed
+
+
+def test_a_suffixed_clone_name_is_adopted(local_db):
+    uid, alias = clone("bob", "Taken")
+    suffixed = f"{alias}-heal"
+    get_table("users").update({"uid": uid, "username": suffixed}, ["uid"])
+    assert AliasHealer(ORIGIN).heal().renamed == {alias: suffixed}
+    assert get_table("sync_identities").find_one(origin=ORIGIN, source_key="bob")["our_name"] == suffixed
+
+
+def test_a_name_that_belongs_to_another_identity_is_a_conflict_not_a_steal():
+    first_uid, first_alias = clone("alice", "Una")
+    _, second_alias = clone("carol", "Duo")
+    get_table("users").update({"uid": first_uid, "username": second_alias}, ["uid"])
+    report = AliasHealer(ORIGIN).heal()
+    assert report.conflicts == ["alice"] and not report.renamed
+    assert get_table("sync_identities").find_one(origin=ORIGIN, source_key="alice")["our_name"] == first_alias
+
+
+def test_a_purged_clone_is_ignored():
+    uid, _ = clone()
+    get_table("users").delete(uid=uid)
+    assert AliasHealer(ORIGIN).heal().renamed == {}
+
+
+def test_the_rewrite_work_per_run_is_bounded(monkeypatch):
+    from dataclasses import replace
+
+    from devplacepy.services.sync import healing
+    from devplacepy.services.sync.spec import ALIAS
+
+    uid, alias = clone()
+    for index in range(5):
+        synced_text(f"p{index}", "posts", f"hello @{alias}", "posts")
+    monkeypatch.setattr(healing, "ALIAS", replace(ALIAS, heal_rewrites_per_run=2))
+    get_table("users").update({"uid": uid, "username": f"Bounded{generate_uid()[-6:]}"}, ["uid"])
+    assert AliasHealer(ORIGIN).heal().rewritten == 2

tests/unit/services/sync/records.py

@@ -83,3 +83,67 @@ def test_envelope_normalizes_origin_and_bounds_batch():
         SyncBatchForm(origin="other.org", records=[{}] * (LIMITS.max_batch_records + 1))
     with pytest.raises(ValidationError):
         SyncBatchForm(origin="bad origin", records=[{}])
+
+
+def test_new_kinds_have_models_and_a_published_schema():
+    from devplacepy.services.sync.records import RECORD_MODELS
+    from devplacepy.services.sync.spec import KINDS
+
+    assert set(RECORD_MODELS) == set(KINDS)
+    for model in RECORD_MODELS.values():
+        assert model.model_json_schema()["title"]
+
+
+def test_vote_value_and_target_kind_are_closed_sets():
+    import pytest
+    from pydantic import ValidationError
+
+    from devplacepy.services.sync.records import VoteRecord
+
+    base = {"origin_uid": "v", "voter_uid": "u", "target_kind": "post", "target_uid": "p", "value": 1, "created_at": "2025-01-01T00:00:00Z"}
+    assert VoteRecord.model_validate(base).value == 1
+    for change in ({"value": 0}, {"value": 2}, {"target_kind": "gist"}, {"voter_uid": "bad id"}):
+        with pytest.raises(ValidationError):
+            VoteRecord.model_validate({**base, **change})
+
+
+def test_attachment_fingerprint_ignores_the_blob_but_not_its_hash():
+    import base64
+
+    from devplacepy.services.sync.records import AttachmentRecord
+
+    def make(data):
+        return AttachmentRecord.model_validate({
+            "origin_uid": "a", "author_uid": "u", "target_kind": "post", "target_uid": "p",
+            "filename": "dir/evil\\name.png", "content_b64": base64.b64encode(data).decode(), "created_at": "2025-01-01T00:00:00Z",
+        })
+
+    first, second = make(b"one"), make(b"two")
+    assert first.filename == "name.png"
+    assert "content_b64" not in first.content_fields() and first.content_fields()["sha256"] != second.content_fields()["sha256"]
+    assert first.content_bytes() == b"one"
+
+
+def test_batch_byte_cap(monkeypatch):
+    import pytest
+    from dataclasses import replace
+    from pydantic import ValidationError
+
+    from devplacepy.services.sync import records
+    from devplacepy.services.sync.spec import LIMITS
+
+    monkeypatch.setattr(records, "LIMITS", replace(LIMITS, max_batch_bytes=100))
+    with pytest.raises(ValidationError):
+        records.SyncBatchForm(origin="a.b", records=[{"x": "y" * 200}])
+
+
+def test_ack_form_validates_kinds_and_ids():
+    import pytest
+    from pydantic import ValidationError
+
+    from devplacepy.services.sync.records import SyncAckForm
+
+    assert SyncAckForm(origin="A.B", items='[{"kind": "posts", "origin_uid": "p1"}]').items == [{"kind": "posts", "origin_uid": "p1"}]
+    for items in ([], [{"kind": "gists", "origin_uid": "x"}], [{"kind": "posts", "origin_uid": ""}], "not json"):
+        with pytest.raises(ValidationError):
+            SyncAckForm(origin="a.b", items=items)

tests/unit/services/sync/selection.py

@@ -0,0 +1,125 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+import pytest
+
+from devplacepy.database import get_table, init_db
+from devplacepy.database.sync import claim_link, load_candidates
+from devplacepy.services.sync.options import SyncOptions
+from devplacepy.services.sync.selection import Selector
+from devplacepy.services.sync.spec import LIMITS, SKIP_MIN_COMMENTS
+
+ORIGIN = "selection.example.org"
+
+
+@pytest.fixture
+def world(local_db):
+    init_db()
+    get_table("sync_candidates").delete()
+    get_table("sync_links").delete(origin=ORIGIN)
+    yield
+    get_table("sync_candidates").delete()
+    get_table("sync_links").delete(origin=ORIGIN)
+
+
+def options(**extra):
+    return SyncOptions(min_comments=2, **extra)
+
+
+def user(uid):
+    return {"origin_uid": uid, "username": f"user{uid}", "created_at": "2026-01-01T00:00:00Z"}
+
+
+def post(uid, author="a"):
+    return {"origin_uid": uid, "author_uid": author, "content": f"post body {uid} long enough", "created_at": "2026-01-02T00:00:00Z"}
+
+
+def comment(uid, on, author="b", parent=None):
+    record = {"origin_uid": uid, "post_uid": on, "author_uid": author, "content": f"comment {uid}", "created_at": "2026-01-03T00:00:00Z"}
+    if parent:
+        record["parent_uid"] = parent
+    return record
+
+
+def run(records, persist=True, **extra):
+    selector = Selector(ORIGIN, options(**extra), persist=persist)
+    return selector, *selector.select(records)
+
+
+def test_a_post_below_the_minimum_is_parked_with_its_comments_and_authors(world):
+    selector, kept, skipped = run({"users": [user("a"), user("b")], "posts": [post("p1")], "comments": [comment("c1", "p1")]})
+    assert kept["posts"] == [] and kept["comments"] == [] and skipped[SKIP_MIN_COMMENTS] == 1
+    assert selector.parked == 1
+    stored = json.loads(load_candidates(ORIGIN)[0]["record"])
+    assert stored["post"]["origin_uid"] == "p1" and [item["origin_uid"] for item in stored["comments"]] == ["c1"]
+    assert {item["origin_uid"] for item in stored["users"]} == {"a", "b"}
+
+
+def test_comments_arriving_in_a_later_run_complete_the_post(world):
+    run({"users": [user("a"), user("b")], "posts": [post("p1")], "comments": [comment("c1", "p1")]})
+    selector, kept, skipped = run({"users": [user("b")], "posts": [], "comments": [comment("c2", "p1")]})
+    assert selector.revived == 1
+    assert [item["origin_uid"] for item in kept["posts"]] == ["p1"]
+    assert [item["origin_uid"] for item in kept["comments"]] == ["c1", "c2"]
+    assert {item["origin_uid"] for item in kept["users"]} == {"a", "b"}
+    assert not skipped and load_candidates(ORIGIN) == []
+
+
+def test_a_repeated_comment_is_counted_once(world):
+    run({"users": [user("a"), user("b")], "posts": [post("p1")], "comments": [comment("c1", "p1")]})
+    selector, kept, skipped = run({"users": [], "posts": [], "comments": [comment("c1", "p1")]})
+    assert kept["posts"] == [] and skipped[SKIP_MIN_COMMENTS] == 1
+    assert len(load_candidates(ORIGIN)) == 1
+
+
+def test_reply_order_is_kept_parent_before_reply(world):
+    run({"users": [user("a"), user("b")], "posts": [post("p1")], "comments": [comment("c1", "p1")]})
+    _, kept, _ = run({"users": [], "posts": [], "comments": [comment("c2", "p1", parent="c1")]})
+    assert [item["origin_uid"] for item in kept["comments"]] == ["c1", "c2"]
+
+
+def test_a_dry_run_neither_reads_nor_writes_candidates(world):
+    selector, _, skipped = run({"users": [user("a")], "posts": [post("p1")], "comments": []}, persist=False)
+    assert skipped[SKIP_MIN_COMMENTS] == 1 and selector.parked == 0 and load_candidates(ORIGIN) == []
+
+
+def test_a_known_post_is_never_parked_or_dropped(world):
+    claim_link(ORIGIN, "posts", "p9", "local-9", "x", "2026-01-01T00:00:00+00:00")
+    selector, kept, skipped = run({"users": [user("a")], "posts": [post("p9")], "comments": []})
+    assert [item["origin_uid"] for item in kept["posts"]] == ["p9"] and not skipped and selector.parked == 0
+
+
+def test_the_candidate_list_is_capped_per_origin(world, monkeypatch):
+    from dataclasses import replace
+
+    from devplacepy.services.sync import selection
+
+    monkeypatch.setattr(selection, "LIMITS", replace(LIMITS, candidate_max_per_origin=2))
+    selector, _, _ = run({"users": [user("a")], "posts": [post("p1"), post("p2"), post("p3")], "comments": []})
+    assert selector.parked == 2 and len(load_candidates(ORIGIN)) == 2
+
+
+def test_an_oversize_candidate_is_dropped_not_stored(world, monkeypatch):
+    from dataclasses import replace
+
+    from devplacepy.services.sync import selection
+
+    monkeypatch.setattr(selection, "LIMITS", replace(LIMITS, candidate_max_bytes=50))
+    selector, _, skipped = run({"users": [user("a")], "posts": [post("p1")], "comments": []})
+    assert skipped[SKIP_MIN_COMMENTS] == 1 and selector.parked == 0
+
+
+def test_expired_candidates_are_not_revived(world):
+    run({"users": [user("a")], "posts": [post("p1")], "comments": []})
+    get_table("sync_candidates").update({"uid": load_candidates(ORIGIN)[0]["uid"], "expires_at": "2000-01-01T00:00:00+00:00"}, ["uid"])
+    selector, _, _ = run({"users": [], "posts": [], "comments": []})
+    assert selector.revived == 0 and load_candidates(ORIGIN) == []
+
+
+def test_other_filters_stay_final_and_clear_the_candidate(world):
+    run({"users": [user("a")], "posts": [post("p1")], "comments": []})
+    selector, kept, skipped = run({"users": [user("a")], "posts": [], "comments": []}, topics=("devlog",))
+    from devplacepy.services.sync.spec import SKIP_TOPIC
+
+    assert kept["posts"] == [] and skipped[SKIP_TOPIC] == 1 and load_candidates(ORIGIN) == []