Code Farm: per-plot auto-replant pause and crop-lock check; community sync and node work

Continuous Delivery could never be turned off once a crop was planted. Add a
one-shot per-plot "stop after harvest" flag (POST /game/plot/stop, Devii tool
game_plot_stop, game_plots.stop_after_harvest), cleared by the next plant,
autoreplant claim or Refactor. Autoreplant now also respects crop locks through
the shared _crop_lock predicate. Plan and measurements in happyplanting.md.

Also includes the pending community sync, node network and related work in the
tree; the full suite passed (4999) with all of it.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

tests/unit/services/community/alias.py

@@ -60,11 +60,12 @@ def enable(**changes):
     store.update(row["uid"], sync_options=SyncOptions.from_row(row).merged(**changes).to_dict())
 
 
-def test_aliasing_off_by_default_keeps_original_names(world):
+def test_aliasing_off_by_default_creates_no_identities_and_derives_from_originals(world):
     report = go(world)
     assert report.status == "ok", report.errors
-    assert "writer0" in local_text()
+    assert report.pushed["users"].get("created", 0) == 4
     assert get_table("sync_identities").count(origin="forum.test") == 0
+    assert "writer" in local_text().replace("wr1ter", "writer")
 
 
 def test_alias_run_never_stores_an_original_username(world):

tests/unit/services/community/checkpoint.py

@@ -0,0 +1,152 @@
+# retoor <retoor@molodetz.nl>
+
+import asyncio
+from dataclasses import replace
+
+import pytest
+
+from devplacepy.database import get_setting, get_table, init_db, set_setting
+from devplacepy.services.community import checkpoint as checkpoint_module
+from devplacepy.services.community.ledger import Ledger
+from devplacepy.services.community.ledger_store import TABLE, ledger_store
+from devplacepy.services.community.runner import CommunityRunner
+from devplacepy.services.community.spec import LIMITS, RUN_INTERRUPTED
+from devplacepy.services.community.store import iso, now, store
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep
+
+
+@pytest.fixture
+def world():
+    init_db()
+    previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins")}
+    set_setting("sync_enabled", "1")
+    set_setting("sync_origins", "")
+    for table in ("communities", "community_runs", TABLE):
+        get_table(table).delete()
+    get_table("sync_links").delete(origin="forum.test")
+    yield
+    for key, value in previous.items():
+        set_setting(key, value)
+
+
+class SlowFetcher(FakeFetcher):
+    async def get(self, url, params=None, timeout=None, **kwargs):
+        await asyncio.sleep(0.12)
+        return await super().get(url, params, timeout, **kwargs)
+
+
+def learned(runner):
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test"))
+    return store.get("forum-test")
+
+
+def test_a_running_run_exposes_its_log_and_ledger_before_it_finishes(world, monkeypatch):
+    runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(json_community()), sleep=no_sleep)
+    row = learned(runner)
+    monkeypatch.setattr(checkpoint_module, "LIMITS", replace(LIMITS, checkpoint_seconds=0.05))
+    seen = []
+    original = store.update_run
+
+    def spy(uid, **fields):
+        original(uid, **fields)
+        seen.append((get_table("community_runs").find_one(uid=uid)["status"], fields.get("log", ""), get_table(TABLE).count(run_uid=uid)))
+
+    monkeypatch.setattr(store, "update_run", spy)
+    runner.fetcher_factory = lambda: SlowFetcher(json_community())
+    report = run_async(runner.run(row, dry_run=False, trigger="test"))
+    assert seen, "no checkpoint was written while the run was going"
+    assert all(status == "running" for status, _, _ in seen)
+    assert any("collect" in log or "run[" in log for _, log, _ in seen)
+    final = store.run(report.run_uid)
+    assert final["status"] in ("ok", "partial") and final["verdict"] != "running"
+    assert final["items_stored"] == get_table(TABLE).count(run_uid=report.run_uid) > 0
+
+
+def test_checkpointed_ledger_rows_are_never_written_twice(world):
+    community = store.create("keep-cp", "q", "tester", name="cp")
+    run_uid = store.start_run(community["uid"], "test", False)
+    ledger = Ledger()
+    from devplacepy.services.community.checkpoint import RunCheckpoint
+    from devplacepy.services.community.logbook import LogBook
+    from devplacepy.services.community.pusher import PushReport
+    from devplacepy.services.community.runner import RunReport
+
+    report = RunReport(status="running", dry_run=False, run_uid=run_uid, ledger=ledger)
+    live = PushReport(dry_run=False)
+    with LogBook() as book:
+        checkpoint = RunCheckpoint(store, ledger_store, run_uid, community["uid"], report, live, book)
+        for number in range(3):
+            ledger.add("posts", "created", origin_uid=f"a{number}")
+        checkpoint.flush()
+        checkpoint.flush()
+        assert get_table(TABLE).count(run_uid=run_uid) == 3
+        for number in range(2):
+            ledger.add("posts", "rejected", reason="invalid", origin_uid=f"b{number}")
+        checkpoint.flush()
+        assert get_table(TABLE).count(run_uid=run_uid) == 5
+        final = ledger_store.append(run_uid, community["uid"], ledger, checkpoint.saved)
+    assert final == 0
+    seqs = sorted(row["seq"] for row in get_table(TABLE).find(run_uid=run_uid))
+    assert seqs == [0, 1, 2, 3, 4]
+
+
+def test_a_killed_run_is_marked_interrupted_and_keeps_what_was_saved(world):
+    community = store.create("keep-kill", "q", "tester", name="kill")
+    run_uid = store.start_run(community["uid"], "test", False)
+    store.update_run(run_uid, log="line one\nline two", tally={"created": 4}, items_stored=4)
+    assert store.reap_interrupted() == 0
+    stale = iso(now().replace(year=2000))
+    get_table("community_runs").update({"uid": run_uid, "started_at": stale}, ["uid"])
+    assert store.reap_interrupted() == 1
+    found = store.run(run_uid)
+    assert found["status"] == RUN_INTERRUPTED and found["verdict"] == "bad" and "Interrupted" in found["headline"]
+    assert found["log"] == "line one\nline two" and found["tally"] == {"created": 4} and found["finished_at"]
+    assert store.reap_interrupted() == 0
+    assert not store.busy(community["uid"])
+
+
+def test_the_interrupted_verdict_is_bad():
+    from devplacepy.services.community.ledger import verdict
+
+    assert verdict("interrupted", False, {})["level"] == "bad"
+
+
+def test_the_search_index_follows_inserts_and_prunes(world):
+    ledger = Ledger()
+    ledger.add("posts", "created", origin_uid="needle-1", label="Alpha Beta Gamma", detail="")
+    ledger.add("posts", "created", origin_uid="hay-2", label="Delta", detail="contains the NEEDLE word")
+    ledger.add("posts", "created", origin_uid="hay-3", label="100% done", detail="")
+    ledger_store.save("search-run", "search-community", ledger)
+    assert ledger_store.search_ready()
+
+    def find(text):
+        rows, page = ledger_store.page({"q": text}, 1, community_uid="search-community")
+        scoped, _ = ledger_store.page({"q": text}, 1, run_uid="search-run")
+        assert sorted(row["origin_uid"] for row in scoped) == sorted(row["origin_uid"] for row in rows)
+        return sorted(row["origin_uid"] for row in rows)
+
+    assert find("needle") == ["hay-2", "needle-1"]
+    assert find("beta g") == ["needle-1"]
+    assert find("100%") == ["hay-3"]
+    assert find("a\"b") == []
+    assert find("ne") == ["hay-2", "hay-3", "needle-1"]
+    get_table(TABLE).delete(run_uid="search-run")
+    assert find("needle") == []
+
+
+def test_a_community_wide_text_search_goes_through_the_index(world):
+    from devplacepy.database import db
+
+    ledger = Ledger()
+    for number in range(50):
+        ledger.add("posts", "created", origin_uid=f"p{number}", label=f"label {number}")
+    ledger_store.save("plan-run-a", "plan-community-a", ledger)
+    where, params, order = ledger_store.compile({"q": "label 4"}, community_uid="plan-community-a")
+    plan = " | ".join(row["detail"] for row in db.query(f"EXPLAIN QUERY PLAN SELECT * FROM {TABLE} WHERE {where} ORDER BY {order} LIMIT 50", **params))
+    assert "community_run_items_fts" in plan
+    where, params, order = ledger_store.compile({"q": "la"}, community_uid="plan-community-a")
+    assert "community_run_items_fts" not in where and "LIKE" in where
+    where, params, order = ledger_store.compile({"q": "label 4"}, run_uid="plan-run-a")
+    assert "community_run_items_fts" not in where

tests/unit/services/community/dedupe.py

@@ -0,0 +1,92 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+from devplacepy.services.community.generator import CodeGenerator
+from devplacepy.services.community.runtime import Collector
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher
+from tests.unit.services.community.generator import load, no_sleep
+
+
+def feed(identifier, url, comments="none"):
+    return {
+        "id": identifier,
+        "url": url,
+        "type": "json_api",
+        "list_path": "$.items",
+        "post": {"fields": {"id": ["$.id"], "title": ["$.title"], "content": ["$.body"], "created_at": ["$.at"], "url": ["$.link"]}},
+        "author": {"fields": {"username": ["$.who"]}},
+        "comments": {"mode": comments},
+    }
+
+
+def schema(limits=None):
+    return {
+        "community": "twins",
+        "origin": "twins.test",
+        "limits": {"delay_seconds": 0, "retries": 1, **(limits or {})},
+        "feeds": [feed("latest", "https://twins.test/latest"), feed("top", "https://twins.test/top")],
+    }
+
+
+def entry(identifier, link, title="A shared story", at="2026-10-01T10:00:00Z"):
+    return {"id": identifier, "title": title, "body": "a body that is long enough to sync", "at": at, "who": "alice", "link": link}
+
+
+def routes(latest, top):
+    return {"https://twins.test/latest": {"items": latest}, "https://twins.test/top": {"items": top}}
+
+
+def collect(fetcher_routes, state=None, only=None, limits=None):
+    return run_async(Collector(FakeFetcher(fetcher_routes), schema(limits), state, sleep=no_sleep).run(only))
+
+
+def uids(result):
+    return {record["origin_uid"] for record in result["records"]["posts"]}
+
+
+def test_the_same_story_in_a_second_feed_in_a_later_run_is_not_a_second_post():
+    shared = "https://news.test/story/42?utm=a"
+    sources = routes([entry("L1", shared)], [entry("T9", "https://www.news.test/story/42/?utm=b")])
+    first = collect(sources, only="latest")
+    assert len(first["records"]["posts"]) == 1 and first["state"]["links"]
+    second = collect(sources, state=json.loads(json.dumps(first["state"])), only="top")
+    assert second["records"]["posts"] == []
+
+
+def test_without_the_remembered_map_the_twin_would_become_a_post():
+    shared = "https://news.test/story/42"
+    sources = routes([entry("L1", shared)], [entry("T9", shared)])
+    first = collect(sources, only="latest")
+    assert len(collect(sources, state={"feeds": first["state"]["feeds"]}, only="top")["records"]["posts"]) == 1
+
+
+def test_one_feed_never_merges_its_own_posts_that_share_a_link():
+    shared = "https://news.test/root"
+    result = collect(routes([entry("A", shared, "first"), entry("B", shared, "second")], []))
+    assert len(result["records"]["posts"]) == 2
+
+
+def test_the_same_feed_keeps_its_own_post_when_it_changes_in_a_later_run():
+    shared = "https://news.test/story/7"
+    first = collect(routes([entry("L1", shared)], []))
+    changed = routes([entry("L1", shared, "A shared story, edited", "2026-10-05T10:00:00Z")], [])
+    again = collect(changed, state=first["state"])
+    assert uids(again) == uids(first) and again["records"]["posts"][0]["title"].endswith("edited")
+
+
+def test_the_remembered_map_is_bounded_and_keeps_the_newest():
+    items = [entry(f"L{number}", f"https://news.test/story/{number}") for number in range(40)]
+    result = collect(routes(items, []), limits={"max_remembered_links": 10})
+    kept = list(result["state"]["links"])
+    assert len(kept) == 10 and kept[-1].endswith("/story/39") and not any(key.endswith("/story/0") for key in kept)
+
+
+def test_a_generated_collector_remembers_links_too(tmp_path):
+    module = load(CodeGenerator(schema()).write(tmp_path))
+    shared = "https://news.test/story/42"
+    sources = routes([entry("L1", shared)], [entry("T9", shared)])
+    first = run_async(module.collect_all(FakeFetcher(sources), only="latest", sleep=no_sleep))
+    second = run_async(module.collect_all(FakeFetcher(sources), state=json.loads(json.dumps(first["state"])), only="top", sleep=no_sleep))
+    assert len(first["records"]["posts"]) == 1 and second["records"]["posts"] == []

tests/unit/services/community/discoverer.py

@@ -131,3 +131,44 @@ def test_feeds_listing_the_same_posts_are_learned_once_and_comment_feeds_are_ski
 def test_generic_probe_paths_are_skipped_when_a_feed_is_advertised():
     _, fetcher = discover(rss_community(), "https://blog.test/")
     assert not any(request.endswith("/api/posts") or request.endswith("/rss.xml") for request in fetcher.requests)
+
+
+def test_a_captcha_on_the_login_wall_is_reported_and_never_attempted():
+    routes = club_community({})
+    captcha_wall = routes["https://club.test/"]().replace("<form", '<div class="g-recaptcha" data-sitekey="k"></div><form', 1)
+    routes["https://club.test/"] = captcha_wall
+    routes["https://club.test/login"] = captcha_wall
+    result, fetcher = discover(routes, "https://club.test/")
+    assert result.status == "login_required" and "captcha" in result.error
+    assert result.schema["login"]["captcha"] is True and fetcher.posts == []
+    result, fetcher = discover(routes, "https://club.test/", {"username": "bob", "password": "pw"})
+    assert result.status == "login_required" and "captcha" in result.error and fetcher.posts == []
+
+
+def test_a_second_factor_with_a_secret_logs_in_and_without_one_asks_a_human(monkeypatch):
+    from devplacepy.services.community import runtime
+    from tests.unit.services.community.login import OTP_PAGE
+
+    monkeypatch.setattr(runtime, "totp", lambda secret: "123456")
+    state = {}
+    routes = club_community(state)
+    password_step = routes["post:https://club.test/login"]
+
+    def first_step(data):
+        answer = password_step(data)
+        if state.get("in"):
+            state["in"] = False
+            return (200, OTP_PAGE)
+        return answer
+
+    def second_step(data):
+        state["in"] = data.get("otp_code") == "123456"
+        return (200, "welcome" if state["in"] else OTP_PAGE)
+
+    routes["post:https://club.test/login"] = first_step
+    routes["post:https://club.test/2fa"] = second_step
+    result, _ = discover(routes, "https://club.test/", {"username": "bob", "password": "pw", "totp": "JBSWY3DPEHPK3PXP"})
+    assert result.status == "ready", result.error
+    state.clear()
+    result, _ = discover(routes, "https://club.test/", {"username": "bob", "password": "pw"})
+    assert result.status == "login_required" and "authenticator secret" in result.error

tests/unit/services/community/engagement.py

@@ -0,0 +1,199 @@
+# retoor <retoor@molodetz.nl>
+
+import base64
+
+import pytest
+
+from devplacepy.database import get_setting, get_table, init_db, set_setting
+from devplacepy.services.community.discoverer import CommunityDiscoverer
+from devplacepy.services.community.generator import CodeGenerator
+from devplacepy.services.community.learner import learn_engagement
+from devplacepy.services.community.runner import CommunityRunner
+from devplacepy.services.community.runtime import Collector
+from devplacepy.services.community.store import store
+from devplacepy.services.sync.options import SyncOptions
+from devplacepy.services.sync.records import RECORD_MODELS
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeBinary, FakeFetcher
+from tests.unit.services.community.generator import load, no_sleep
+
+PNG = base64.b64decode(
+    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
+)
+
+
+def person(number):
+    return {"id": 500 + number, "username": f"fan{number}"}
+
+
+def post(number, voters=3, with_engagement=True):
+    item = {
+        "id": 9000 + number,
+        "title": f"Engaged post number {number}",
+        "content_html": f'<p>Body of engaged post {number}, long enough to sync.</p><img src="/img/p{number}.png"><a href="https://cdn.test/doc{number}.pdf">doc</a><a href="/page">page</a>',
+        "created_at": f"2026-10-{number:02d}T10:00:00Z",
+        "author": {"id": 77, "username": "writer"},
+        "comments": [
+            {"id": 7000 + number, "body": f"A comment on {number}", "created_at": f"2026-10-{number:02d}T11:00:00Z", "user": {"id": 88, "username": "replier"},
+             "votes": [{"user": person(1), "value": 1}, {"user": person(2), "value": -1}] if with_engagement else []},
+        ],
+    }
+    if with_engagement:
+        item["votes"] = [{"user": person(index), "value": -1 if index == 3 else 1, "created_at": f"2026-10-{number:02d}T12:00:00Z"} for index in range(1, voters + 1)]
+        item["reactions"] = [{"user": person(1), "emoji": "\U0001f525"}, {"user": person(2), "emoji": "\U0001f44d"}, {"user": person(3), "emoji": "nope"}]
+    return item
+
+
+def routes(count=4, **kwargs):
+    return {
+        "https://eng.test/": '<html lang="en"><head><title>Eng</title></head><body><p>Join 100 members discussing code every day on this site.</p></body></html>',
+        "https://eng.test/api/posts": {"items": [post(number, **kwargs) for number in range(1, count + 1)], "total": count},
+        "https://eng.test/about": "<html><body><p>About: 100 members.</p></body></html>",
+        **{f"https://eng.test/img/p{number}.png": FakeBinary(PNG) for number in range(1, count + 1)},
+        **{f"https://cdn.test/doc{number}.pdf": FakeBinary(b"%PDF-1.4 " + b"x" * 40, "application/pdf") for number in range(1, count + 1)},
+        "https://eng.test/page": "<html><body>not a file</body></html>",
+    }
+
+
+def learned(source=None):
+    result = run_async(CommunityDiscoverer(FakeFetcher(source or routes())).discover("https://eng.test/api/posts"))
+    assert result.status == "ready", result.error
+    return result.schema
+
+
+def test_the_learner_finds_per_voter_lists_and_names_the_emoji_field():
+    schema = learned()
+    post_spec = schema["feeds"][0]["post"]
+    assert post_spec["votes"]["list_path"] == "$.votes" and post_spec["votes"]["author"]["fields"]["username"] == ["$.user.username"]
+    assert post_spec["votes"]["fields"]["value"] == ["$.value"]
+    assert post_spec["reactions"]["fields"]["emoji"] == ["$.emoji"]
+    assert schema["feeds"][0]["comments"]["votes"]["list_path"] == "$.votes"
+
+
+def test_aggregate_counts_alone_learn_nothing():
+    samples = [{"id": number, "votes": number * 3, "likes": 5, "reactions": {"fire": 2}} for number in range(1, 5)]
+    assert learn_engagement(samples) == {}
+    assert learn_engagement([{"id": 1, "votes": [{"user": {"username": "a"}}]}]) == {}
+    assert learn_engagement([{"id": number, "votes": [{"score": 1}, {"score": 2}]} for number in range(3)]) == {}
+
+
+def test_the_collector_turns_voter_lists_into_records_with_users():
+    schema = learned()
+    result = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep).run())
+    records = result["records"]
+    assert len(records["votes"]) == 4 * 3 + 4 * 2 and len(records["reactions"]) == 4 * 2
+    assert {item["value"] for item in records["votes"]} == {1, -1}
+    assert {item["emoji"] for item in records["reactions"]} == {"\U0001f525", "\U0001f44d"}
+    voters = {item["voter_uid"] for item in records["votes"]}
+    assert voters <= {user["origin_uid"] for user in records["users"]}
+    for kind in ("votes", "reactions"):
+        for row in records[kind]:
+            RECORD_MODELS[kind].model_validate(row)
+    assert len({item["origin_uid"] for item in records["votes"]}) == len(records["votes"])
+    assert {item["target_kind"] for item in records["votes"]} == {"post", "comment"}
+
+
+def test_attachments_are_off_by_default_and_bounded_when_on():
+    schema = learned()
+    off = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep).run())
+    assert off["records"]["attachments"] == []
+    on = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True}).run())
+    files = on["records"]["attachments"]
+    assert len(files) == 8 and {item["filename"].rsplit(".", 1)[-1] for item in files} == {"png", "pdf"}
+    assert base64.b64decode(next(item for item in files if item["filename"] == "p1.png")["content_b64"]) == PNG
+    for row in files:
+        RECORD_MODELS["attachments"].model_validate(row)
+    capped = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True, "max_attachments": 3}).run())
+    assert len(capped["records"]["attachments"]) == 3
+    one_each = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True, "max_attachments_per_post": 1}).run())
+    assert len(one_each["records"]["attachments"]) == 4
+
+
+def test_missing_oversize_and_wrong_type_files_become_notices_not_failures():
+    source = routes(count=2)
+    source["https://eng.test/img/p1.png"] = (404, "gone")
+    source["https://cdn.test/doc1.pdf"] = FakeBinary(b"x" * 500, "application/pdf")
+    source["https://eng.test/img/p2.png"] = FakeBinary(b"<html>", "text/html")
+    schema = learned()
+    result = run_async(Collector(FakeFetcher(source), schema, sleep=no_sleep, options={"attachments": True, "max_attachment_bytes": 100}).run())
+    reasons = sorted(notice["reason"] for notice in result["notices"])
+    assert reasons.count("attachment_unreachable") == 2 and reasons.count("attachment_too_large") == 1
+    assert result["ok"] and [item["filename"] for item in result["records"]["attachments"]] == ["doc2.pdf"]
+
+
+def test_a_generated_collector_collects_the_same_engagement(tmp_path):
+    schema = learned()
+    module = load(CodeGenerator(schema).write(tmp_path))
+    generated = run_async(module.collect_all(FakeFetcher(routes()), sleep=no_sleep, options={"attachments": True}))
+    direct = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True}).run())
+    for kind in ("votes", "reactions", "attachments"):
+        assert len(generated["records"][kind]) == len(direct["records"][kind]) > 0
+
+
+def test_an_old_generated_file_is_regenerated_keeping_its_hand_edited_schema(tmp_path):
+    from devplacepy.services.community.files import CommunityFiles
+    from devplacepy.services.community.generator import CodeGenerator
+
+    schema = learned()
+    files = CommunityFiles(tmp_path)
+    path = files.write("old-site", schema)
+    edited = path.read_text().replace(f"# generator-version: {CodeGenerator.VERSION}", "# generator-version: 1").replace('"max_posts": 150', '"max_posts": 7')
+    path.write_text(edited)
+    module = files.load("old-site", schema)
+    assert module.SCHEMA["limits"]["max_posts"] == 7
+    assert f"# generator-version: {CodeGenerator.VERSION}" in path.read_text()
+
+
+@pytest.fixture
+def world():
+    init_db()
+    previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins", "community_alias_enabled")}
+    set_setting("sync_enabled", "1")
+    set_setting("sync_origins", "")
+    for table in ("communities", "community_runs"):
+        get_table(table).delete()
+    get_table("sync_links").delete(origin="eng.test")
+    get_table("sync_identities").delete(origin="eng.test")
+    yield
+    for key, value in previous.items():
+        set_setting(key, value)
+
+
+def make_row(**options):
+    runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes()), sleep=no_sleep)
+    store.create("eng-test", "https://eng.test/api/posts", "tester", name="eng")
+    run_async(runner.discover("https://eng.test/api/posts", "tester", relearn_slug="eng-test"))
+    row = store.get("eng-test")
+    if options:
+        store.update(row["uid"], sync_options=SyncOptions.build(**options).to_dict())
+    return runner, store.get("eng-test")
+
+
+def test_a_live_run_syncs_votes_reactions_and_attachments(world):
+    runner, row = make_row(collect_attachments=True)
+    report = run_async(runner.run(row, dry_run=False, trigger="test"))
+    assert report.status in ("ok", "partial"), report.errors
+    assert report.pushed["votes"].get("created") == 20 and report.pushed["reactions"].get("created") == 8
+    assert report.pushed["attachments"].get("created") == 8
+    post_link = get_table("sync_links").find_one(origin="eng.test", kind="posts")
+    stars = get_table("posts").find_one(uid=post_link["local_uid"])["stars"]
+    assert stars == 1
+    again = run_async(runner.run(store.get("eng-test"), dry_run=False, trigger="test"))
+    assert again.pushed.get("votes", {}).get("created", 0) == 0
+
+
+def test_without_the_option_no_file_is_downloaded_or_pushed(world):
+    runner, row = make_row()
+    report = run_async(runner.run(row, dry_run=False, trigger="test"))
+    assert report.pushed["attachments"] == {}
+
+
+def test_voters_are_never_stored_under_pseudonyms_so_votes_wait_while_aliasing_is_on(world):
+    set_setting("community_alias_enabled", "1")
+    runner, row = make_row(alias=True, collect_attachments=True)
+    report = run_async(runner.run(row, dry_run=False, trigger="test"))
+    assert report.pushed["votes"] == {} and report.pushed["reactions"] == {}
+    assert report.pushed["skipped"]["alias_engagement"] == 20 + 8
+    assert report.pushed["attachments"].get("created") == 8
+    owners = {get_table("attachments").find_one(uid=link["local_uid"])["user_uid"] for link in get_table("sync_links").find(origin="eng.test", kind="attachments")}
+    assert owners and all(get_table("users").find_one(uid=uid)["username"] != "writer" for uid in owners)

tests/unit/services/community/fakes.py

@@ -4,15 +4,29 @@ import json
 from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
 
 
+class FakeBinary:
+    def __init__(self, data: bytes, content_type: str = "image/png"):
+        self.data = data
+        self.content_type = content_type
+
+
 class FakeResponse:
     def __init__(self, status: int, body):
         self.status = status
-        self.body = body if isinstance(body, str) else json.dumps(body)
+        self.raw = b""
         self.headers = {}
+        if isinstance(body, FakeBinary):
+            self.raw, self.body = body.data, ""
+            self.headers = {"content-type": body.content_type}
+        else:
+            self.body = body if isinstance(body, str) else json.dumps(body)
 
     async def text(self):
         return self.body
 
+    async def read(self):
+        return self.raw or self.body.encode()
+
     async def json(self):
         return json.loads(self.body)
 
@@ -23,6 +37,8 @@ class FakeFetcher:
         self.requests: list[str] = []
         self.session = None
         self.posts: list[tuple] = []
+        self.last_headers = None
+        self.get_headers: list = []
 
     async def __aenter__(self):
         return self
@@ -40,6 +56,7 @@ class FakeFetcher:
     async def get(self, url, params=None, timeout=None, **kwargs):
         key = self.key(url, params)
         self.requests.append(key)
+        self.get_headers.append(kwargs.get("headers"))
         route = self.routes.get(key)
         if route is None:
             return FakeResponse(404, "not found")
@@ -49,10 +66,12 @@ class FakeFetcher:
         return FakeResponse(status, body)
 
     async def post(self, url, data=None, timeout=None, **kwargs):
-        self.posts.append((url, dict(data or {})))
+        body = dict(data or kwargs.get("json") or {})
+        self.posts.append((url, body))
+        self.last_headers = kwargs.get("headers")
         route = self.routes.get("post:" + url)
         if callable(route):
-            route = route(dict(data or {}))
+            route = route(body)
         status, body = route if isinstance(route, tuple) else (200, route or "")
         return FakeResponse(status, body)
 

tests/unit/services/community/ledger.py

@@ -63,9 +63,9 @@ def test_pusher_records_created_rejected_and_held_records_with_reasons(world):
     by_origin = {item.origin_uid: item for item in ledger.items}
     assert by_origin["u1"].outcome == "created" and by_origin["u1"].label == "ledgerone" and by_origin["u1"].local_uid
     assert by_origin["p1"].outcome == "created" and by_origin["p1"].label == "A good post"
-    assert by_origin["p2"].outcome == "rejected" and by_origin["p2"].reason == "unknown_author" and by_origin["p2"].label == "Orphan post"
+    assert by_origin["p2"].outcome == "deferred" and by_origin["p2"].reason == "unknown_author" and by_origin["p2"].label == "Orphan post"
     assert by_origin["p3"].outcome == "unchanged" and by_origin["p3"].reason == "tombstone_for_unknown"
-    assert tally_of(report.counts, ledger) == {"created": 2, "unchanged": 1, "rejected": 1}
+    assert tally_of(report.counts, ledger) == {"created": 2, "unchanged": 1, "deferred": 1}
 
 
 def test_repeating_the_same_batch_lists_nothing_that_is_merely_unchanged(world):

tests/unit/services/community/login.py

@@ -0,0 +1,238 @@
+# retoor <retoor@molodetz.nl>
+
+import base64
+
+import pytest
+
+from devplacepy.services.community import runtime
+from devplacepy.services.community.discoverer import LoginLearner
+from devplacepy.services.community.extract import detect_captcha, find_otp_form, parse_html, totp
+from devplacepy.services.community.runtime import (
+    CaptchaRequired,
+    Collector,
+    LoginFailed,
+    SecondFactorRequired,
+    code_challenge,
+)
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher
+
+
+async def no_sleep(_seconds):
+    return None
+
+
+RFC_SECRET = base64.b32encode(b"12345678901234567890").decode()
+
+
+@pytest.mark.parametrize(
+    "moment,expected",
+    [(59, "94287082"), (1111111109, "07081804"), (1111111111, "14050471"), (1234567890, "89005924"), (2000000000, "69279037"), (20000000000, "65353130")],
+)
+def test_totp_matches_the_rfc_6238_vectors(moment, expected):
+    assert totp(RFC_SECRET, moment, digits=8) == expected
+
+
+def test_totp_default_is_six_digits_zero_padded_and_tolerates_formatting():
+    code = totp(RFC_SECRET.lower(), 59)
+    assert len(code) == 6 and code.isdigit()
+    spaced = " ".join(RFC_SECRET[index : index + 4] for index in range(0, len(RFC_SECRET), 4))
+    assert totp(spaced, 59) == code
+    assert totp(RFC_SECRET.rstrip("="), 59) == code
+    assert totp(RFC_SECRET, 59 + 30) != code
+    assert totp(RFC_SECRET, 59) == totp(RFC_SECRET, 30)
+
+
+OTP_PAGE = """<html><body><form action="/2fa" method="post">
+<input type="hidden" name="csrf" value="t2"><input type="text" name="otp_code" autocomplete="one-time-code"><button>Verify</button></form></body></html>"""
+LOGIN_PAGE = """<html><body><form action="/login" method="post">
+<input type="hidden" name="csrf" value="t1"><input type="text" name="user"><input type="password" name="pass"><button>Sign in</button></form></body></html>"""
+HOME_OPEN = "<html><body><p>Welcome back to the forum, you are signed in</p></body></html>"
+
+
+def test_otp_forms_and_captcha_markers_are_recognised():
+    form = find_otp_form(parse_html(OTP_PAGE), "https://x.test/login")
+    assert form == {"action": "https://x.test/2fa", "method": "POST", "field": "otp_code", "fields": {"csrf": "t2"}}
+    assert find_otp_form(parse_html(LOGIN_PAGE), "https://x.test/") is None
+    named = parse_html('<form><input type="text" name="verification_code"></form>')
+    assert find_otp_form(named, "https://x.test/")["field"] == "verification_code"
+    hidden_token = parse_html('<form><input type="hidden" name="token"><input type="text" name="user"></form>')
+    assert find_otp_form(hidden_token, "https://x.test/") is None
+    assert detect_captcha('<div class="g-recaptcha" data-sitekey="x"></div>') and detect_captcha("<div class='cf-turnstile'></div>")
+    assert not detect_captcha("<p>no puzzle here</p>")
+
+
+def form_schema():
+    return {
+        "community": "club",
+        "origin": "club.test",
+        "base_url": "https://club.test/",
+        "feeds": [],
+        "limits": {"delay_seconds": 0, "retries": 1},
+        "login": {
+            "mode": "form", "url": "https://club.test/login", "action": "https://club.test/login", "method": "POST",
+            "fields": {"csrf": "t1"}, "username_field": "user", "password_field": "pass", "identity": "username",
+        },
+    }
+
+
+def club(state, challenge=True, good_code="123456"):
+    def login(data):
+        if data.get("user") == "bob" and data.get("pass") == "pw":
+            if challenge:
+                return (200, OTP_PAGE)
+            state["in"] = True
+            return (200, "ok")
+        return (200, LOGIN_PAGE)
+
+    def second(data):
+        state["codes"] = state.get("codes", []) + [data.get("otp_code")]
+        state["in"] = data.get("otp_code") == good_code and data.get("csrf") == "t2"
+        return (200, "ok" if state["in"] else OTP_PAGE)
+
+    return {
+        "https://club.test/login": LOGIN_PAGE,
+        "https://club.test/": lambda: HOME_OPEN if state.get("in") else LOGIN_PAGE,
+        "post:https://club.test/login": login,
+        "post:https://club.test/2fa": second,
+    }
+
+
+def test_a_form_login_with_a_second_factor_submits_the_current_code(monkeypatch):
+    monkeypatch.setattr(runtime, "totp", lambda secret: "123456")
+    state = {}
+    collector = Collector(FakeFetcher(club(state)), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "pw", "totp": RFC_SECRET})
+    assert run_async(collector.login()) is True
+    assert state["in"] and state["codes"] == ["123456"]
+
+
+def test_a_second_factor_without_a_secret_says_what_is_missing():
+    collector = Collector(FakeFetcher(club({})), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "pw"})
+    with pytest.raises(SecondFactorRequired, match="authenticator secret"):
+        run_async(collector.login())
+
+
+def test_a_rejected_code_is_reported_not_retried(monkeypatch):
+    monkeypatch.setattr(runtime, "totp", lambda secret: "000000")
+    state = {}
+    collector = Collector(FakeFetcher(club(state)), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "pw", "totp": RFC_SECRET})
+    with pytest.raises(SecondFactorRequired, match="rejected the one-time code"):
+        run_async(collector.login())
+    assert state["codes"] == ["000000"]
+
+
+def test_a_login_without_a_second_factor_is_unchanged():
+    state = {}
+    collector = Collector(FakeFetcher(club(state, challenge=False)), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "pw"})
+    assert run_async(collector.login()) is True and state["in"]
+
+
+def test_wrong_password_still_fails_as_before():
+    collector = Collector(FakeFetcher(club({})), form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "nope"})
+    with pytest.raises(LoginFailed, match="rejected the stored credentials"):
+        run_async(collector.login())
+
+
+def test_a_captcha_on_the_login_page_is_never_attempted():
+    routes = club({})
+    routes["https://club.test/login"] = LOGIN_PAGE.replace("<form", '<div class="g-recaptcha" data-sitekey="k"></div><form', 1)
+    fetcher = FakeFetcher(routes)
+    collector = Collector(fetcher, form_schema(), sleep=no_sleep, credentials={"username": "bob", "password": "pw"})
+    with pytest.raises(CaptchaRequired, match="captcha"):
+        run_async(collector.login())
+    assert fetcher.posts == []
+
+
+def test_a_learned_captcha_flag_stops_before_any_request():
+    schema = form_schema()
+    schema["login"]["captcha"] = True
+    fetcher = FakeFetcher({})
+    with pytest.raises(CaptchaRequired):
+        run_async(Collector(fetcher, schema, sleep=no_sleep, credentials={"username": "bob", "password": "pw"}).login())
+    assert fetcher.requests == [] and fetcher.posts == []
+
+
+def json_schema(**login):
+    return {
+        "community": "api",
+        "origin": "api.test",
+        "base_url": "https://api.test/",
+        "feeds": [],
+        "limits": {"delay_seconds": 0, "retries": 1},
+        "login": {"mode": "json", "url": "https://api.test/api/login", "username_field": "email", "password_field": "password", "identity": "email", **login},
+    }
+
+
+def api(state, challenge=False):
+    def login(body):
+        state["bodies"] = state.get("bodies", []) + [dict(body)]
+        if body.get("email") != "bob@x.test" or body.get("password") != "pw":
+            return (401, {"error": "bad credentials"})
+        if challenge and body.get("otp") != "123456":
+            return (200, {"otp_required": True}) if "otp" not in body else (401, {"error": "invalid verification code"})
+        return (200, {"data": {"token": "tok-1"}})
+
+    return {"post:https://api.test/api/login": login, "https://api.test/api/me": {"ok": True}}
+
+
+def test_a_json_login_stores_the_token_and_sends_it_on_later_requests():
+    state = {}
+    fetcher = FakeFetcher(api(state))
+    collector = Collector(fetcher, json_schema(token_path="$.data.token"), sleep=no_sleep, credentials={"email": "bob@x.test", "password": "pw"})
+    assert run_async(collector.login()) is True
+    assert state["bodies"] == [{"email": "bob@x.test", "password": "pw"}]
+    run_async(collector.fetch("https://api.test/api/me"))
+    assert fetcher.get_headers[-1] == {"Authorization": "Bearer tok-1"}
+
+
+def test_a_json_login_without_a_token_relies_on_the_session_cookie():
+    collector = Collector(FakeFetcher(api({})), json_schema(), sleep=no_sleep, credentials={"email": "bob@x.test", "password": "pw"})
+    assert run_async(collector.login()) is True and collector.headers == {}
+
+
+def test_a_json_login_with_a_missing_token_or_bad_password_fails():
+    with pytest.raises(LoginFailed, match="without the expected token"):
+        run_async(Collector(FakeFetcher(api({})), json_schema(token_path="$.nope"), sleep=no_sleep, credentials={"email": "bob@x.test", "password": "pw"}).login())
+    with pytest.raises(LoginFailed, match="rejected the stored credentials"):
+        run_async(Collector(FakeFetcher(api({})), json_schema(), sleep=no_sleep, credentials={"email": "bob@x.test", "password": "bad"}).login())
+
+
+def test_a_json_login_answers_the_code_challenge(monkeypatch):
+    monkeypatch.setattr(runtime, "totp", lambda secret: "123456")
+    state = {}
+    collector = Collector(FakeFetcher(api(state, challenge=True)), json_schema(token_path="$.data.token"), sleep=no_sleep, credentials={"email": "bob@x.test", "password": "pw", "totp": RFC_SECRET})
+    assert run_async(collector.login()) is True
+    assert [("otp" in body) for body in state["bodies"]] == [False, True]
+    with pytest.raises(SecondFactorRequired, match="authenticator secret"):
+        run_async(Collector(FakeFetcher(api({}, challenge=True)), json_schema(), sleep=no_sleep, credentials={"email": "bob@x.test", "password": "pw"}).login())
+
+
+def test_code_challenge_detection():
+    assert code_challenge({"otp_required": True}, "", 200)
+    assert code_challenge({"Two_Factor": 1}, "", 200)
+    assert code_challenge(None, "Please enter your verification code", 401)
+    assert not code_challenge({"token": "x"}, "", 200)
+    assert not code_challenge(None, "Please enter your verification code", 200)
+
+
+def test_the_learner_finds_a_json_login_and_describes_it():
+    state = {}
+    routes = api(state)
+    routes["post:https://api.test/api/auth/login"] = (404, "nope")
+    learner = LoginLearner(FakeFetcher(routes))
+    spec = run_async(learner.find_json("https://api.test/", {"email": "bob@x.test", "password": "pw"}))
+    assert spec == {
+        "mode": "json", "url": "https://api.test/api/login", "username_field": "email", "password_field": "password",
+        "identity": "email", "token_path": "$.data.token", "token_scheme": "Bearer",
+    }
+    token_routes = {"post:https://api.test/api/login": lambda body: (200, {"access_token": "t"}) if body.get("username") == "bob" else (401, {"error": "x"})}
+    spec = run_async(LoginLearner(FakeFetcher(token_routes)).find_json("https://api.test/", {"username": "bob", "password": "pw"}))
+    assert spec["token_path"] == "$.access_token" and spec["username_field"] == "username"
+
+
+def test_the_learner_gives_up_quietly_and_within_its_attempt_cap():
+    fetcher = FakeFetcher({"post:https://api.test/api/login": (401, {"error": "no"})})
+    assert run_async(LoginLearner(fetcher).find_json("https://api.test/", {"username": "bob", "password": "x"})) is None
+    assert len(fetcher.posts) <= 12
+    html_only = FakeFetcher({"post:https://api.test/api/login": (200, "<html>not json</html>")})
+    assert run_async(LoginLearner(html_only).find_json("https://api.test/", {"username": "bob", "password": "x"})) is None

tests/unit/services/community/pagination.py

@@ -0,0 +1,176 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.community.extract import (
+    find_next_page,
+    page_number,
+    parse_html,
+    with_page_number,
+)
+from devplacepy.services.community.generator import CodeGenerator
+from devplacepy.services.community.runtime import Collector
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher, detail_page, html_community
+from tests.unit.services.community.generator import learn, load, no_sleep, post_comment_count
+
+BASE = "https://board.test/t/1"
+
+
+def page(markup: str):
+    return parse_html(f"<html><body><article>text</article>{markup}</body></html>")
+
+
+@pytest.mark.parametrize(
+    "markup,expected",
+    [
+        ('<a rel="next" href="/t/1?p=2">x</a>', f"{BASE}?p=2"),
+        ('<link rel="next" href="/t/1?p=2">', f"{BASE}?p=2"),
+        ('<ul class="pagination"><li class="next"><a href="/t/1?p=2">go</a></li></ul>', f"{BASE}?p=2"),
+        ('<nav><a href="/t/1?p=2">Next</a></nav>', f"{BASE}?p=2"),
+        ('<div class="pager"><a href="/t/1?p=2">›</a></div>', f"{BASE}?p=2"),
+        ('<div class="pagination"><a href="/t/1?p=2">Volgende pagina</a></div>', f"{BASE}?p=2"),
+        ('<div class="pagination"><a href="/t/1">1</a><a href="/t/1?page=2">2</a><a href="/t/1?page=3">3</a></div>', f"{BASE}?page=2"),
+        ('<a aria-label="Next page" href="/t/1?p=2">arrow</a>', f"{BASE}?p=2"),
+    ],
+)
+def test_next_page_is_found_in_every_common_markup(markup, expected):
+    assert find_next_page(page(markup), BASE) == expected
+
+
+@pytest.mark.parametrize(
+    "markup",
+    [
+        '<p><a href="/other">More</a> from the author</p>',
+        '<a href="javascript:void(0)" class="next">next</a>',
+        '<a href="#">Next</a>',
+        '<nav><a href="/t/1?p=1">1</a></nav>',
+        '<a href="/t/2?page=2">elsewhere 2</a>',
+        "",
+    ],
+)
+def test_no_next_page_is_invented(markup):
+    assert find_next_page(page(markup), BASE) == ""
+
+
+def test_hints_override_the_heuristics():
+    root = page('<a rel="next" href="/t/1?p=2">x</a><div class="x"><a class="more" href="/t/1?custom=2">c</a></div>')
+    assert find_next_page(root, BASE, {"style": "none"}) == ""
+    assert find_next_page(root, BASE, {"style": "next_link", "next": ".more::attr(href)"}) == f"{BASE}?custom=2"
+    assert find_next_page(root, BASE, {"style": "next_link", "next": ".absent::attr(href)"}) == ""
+    assert find_next_page(root, f"{BASE}?page=3", {"style": "page_param", "param": "page"}) == f"{BASE}?page=4"
+    assert find_next_page(root, BASE, {"style": "page_param"}) == f"{BASE}?page=2"
+
+
+def test_page_numbers_round_trip():
+    assert page_number("https://x.test/t?page=4") == 4
+    assert page_number("https://x.test/t/page/7") == 7
+    assert page_number("https://x.test/t?x=1&p=12") == 12
+    assert page_number("https://x.test/t") is None
+    assert with_page_number("https://x.test/t?a=1&page=2", 3) == "https://x.test/t?a=1&page=3"
+    assert with_page_number("https://x.test/t/page/2", 3) == "https://x.test/t/page/3"
+    assert with_page_number("https://x.test/t", 2, "p") == "https://x.test/t?p=2"
+
+
+def board(next_markup: str, second_next: str = "") -> dict:
+    routes = html_community()
+    routes[BASE] = detail_page("thread", 1).replace("</body>", f"{next_markup}</body>")
+    routes[f"{BASE}?cp=2"] = detail_page("thread", 1, comments=4).replace("</body>", f"{second_next}</body>")
+    return routes
+
+
+@pytest.mark.parametrize(
+    "markup",
+    [
+        '<nav class="pagination"><a href="/t/1?cp=2">Next</a></nav>',
+        '<div class="pager"><a href="/t/1?cp=2">»</a></div>',
+        '<div class="pagination"><a href="/t/1">1</a><a href="/t/1?cp=2">2</a></div>',
+    ],
+)
+def test_collector_and_generated_file_follow_every_style(markup, tmp_path):
+    routes = board(markup)
+    schema = learn(html_community(), "https://board.test/")
+    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    first = next(post["origin_uid"] for post in result["records"]["posts"] if post["title"].startswith("Thread 1 "))
+    assert post_comment_count(result, first) == 5
+    module = load(CodeGenerator(schema).write(tmp_path))
+    generated = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    assert len(generated["records"]["comments"]) == len(result["records"]["comments"])
+
+
+def test_a_page_that_adds_no_new_comment_ends_the_walk():
+    routes = board('<nav class="pagination"><a href="/t/1?cp=2">Next</a></nav>', '<nav class="pagination"><a href="/t/1?cp=3">Next</a></nav>')
+    routes[f"{BASE}?cp=3"] = routes[f"{BASE}?cp=2"].replace("cp=3", "cp=4")
+    routes[f"{BASE}?cp=4"] = routes[f"{BASE}?cp=2"]
+    schema = learn(html_community(), "https://board.test/")
+    fetcher = FakeFetcher(routes)
+    run_async(Collector(fetcher, schema, sleep=no_sleep).run())
+    assert f"{BASE}?cp=3" in fetcher.requests and f"{BASE}?cp=4" not in fetcher.requests
+
+
+def test_a_declared_no_pagination_hint_stops_after_the_first_page():
+    routes = board('<a rel="next" href="/t/1?cp=2">more</a>')
+    schema = learn(html_community(), "https://board.test/")
+    for feed in schema["feeds"]:
+        if (feed.get("comments") or {}).get("mode") == "page":
+            feed["comments"]["pagination"] = {"style": "none"}
+    fetcher = FakeFetcher(routes)
+    run_async(Collector(fetcher, schema, sleep=no_sleep).run())
+    assert f"{BASE}?cp=2" not in fetcher.requests
+
+
+def test_the_learner_records_a_selector_hint_when_one_matches():
+    schema = learn(board('<a rel="next" href="/t/1?cp=2">more</a>'), "https://board.test/")
+    modes = [feed["comments"] for feed in schema["feeds"] if (feed.get("comments") or {}).get("mode") == "page"]
+    assert modes and modes[0].get("pagination") == {"style": "next_link", "next": "a[rel=next]::attr(href)"}
+
+
+def json_comment_schema(pagination):
+    comments = {
+        "mode": "url",
+        "url_template": "https://api.test/posts/{id}/comments",
+        "list_path": "$.items",
+        "fields": {"id": ["$.id"], "content": ["$.body"], "created_at": ["$.at"]},
+        "author": {"fields": {"username": ["$.who"]}},
+    }
+    if pagination:
+        comments["pagination"] = pagination
+    return {
+        "community": "api",
+        "origin": "api.test",
+        "limits": {"delay_seconds": 0, "retries": 1, "max_comment_pages": 3},
+        "feeds": [
+            {
+                "id": "feed1",
+                "url": "https://api.test/posts",
+                "type": "json_api",
+                "list_path": "$.items",
+                "post": {"fields": {"id": ["$.id"], "title": ["$.title"], "content": ["$.body"], "created_at": ["$.at"]}},
+                "author": {"fields": {"username": ["$.who"]}},
+                "comments": comments,
+            }
+        ],
+    }
+
+
+def json_routes():
+    entry = lambda n: {"id": f"c{n}", "body": f"comment number {n}", "at": "2026-10-01T10:00:00Z", "who": "alice"}
+    return {
+        "https://api.test/posts": {"items": [{"id": "p1", "title": "A post", "body": "a post body long enough", "at": "2026-10-01T10:00:00Z", "who": "alice"}]},
+        "https://api.test/posts/p1/comments": {"items": [entry(1), entry(2)], "next": "/posts/p1/comments?page=2"},
+        "https://api.test/posts/p1/comments?page=2": {"items": [entry(3)], "next": "/posts/p1/comments?page=3"},
+        "https://api.test/posts/p1/comments?page=3": {"items": [entry(3)], "next": "/posts/p1/comments?page=4"},
+    }
+
+
+def test_json_comments_follow_a_declared_next_key_and_stop_without_news():
+    fetcher = FakeFetcher(json_routes())
+    result = run_async(Collector(fetcher, json_comment_schema({"style": "json_next", "path": "$.next"}), sleep=no_sleep).run())
+    assert sorted(comment["content"] for comment in result["records"]["comments"]) == ["comment number 1", "comment number 2", "comment number 3"]
+    assert "https://api.test/posts/p1/comments?page=4" not in fetcher.requests
+
+
+def test_json_comments_without_a_hint_read_only_the_first_page():
+    fetcher = FakeFetcher(json_routes())
+    result = run_async(Collector(fetcher, json_comment_schema(None), sleep=no_sleep).run())
+    assert len(result["records"]["comments"]) == 2

tests/unit/services/community/schemacheck.py

@@ -0,0 +1,89 @@
+# retoor <retoor@molodetz.nl>
+
+import copy
+
+import pytest
+
+from devplacepy.services.community.discoverer import CommunityDiscoverer
+from devplacepy.services.community.schemacheck import sample_limits, validate_schema
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher, html_community, json_community
+
+
+@pytest.fixture(scope="module")
+def json_schema():
+    return run_async(CommunityDiscoverer(FakeFetcher(json_community())).discover("https://forum.test/")).schema
+
+
+@pytest.fixture(scope="module")
+def html_schema():
+    return run_async(CommunityDiscoverer(FakeFetcher(html_community())).discover("https://board.test/")).schema
+
+
+def test_learned_schemas_are_valid_by_construction(json_schema, html_schema):
+    assert validate_schema(json_schema) == []
+    assert validate_schema(html_schema) == []
+
+
+@pytest.mark.parametrize("bad", [None, [], "text", 4])
+def test_the_top_level_must_be_an_object(bad):
+    assert validate_schema(bad) == ["the schema must be a JSON object"]
+
+
+def mutated(schema, change):
+    copied = copy.deepcopy(schema)
+    change(copied)
+    return validate_schema(copied)
+
+
+def test_every_structural_mistake_is_named_with_its_path(json_schema, html_schema):
+    assert any("origin" in item for item in mutated(json_schema, lambda s: s.update(origin="Not A Host!")))
+    assert any("community" in item for item in mutated(json_schema, lambda s: s.update(community="")))
+    assert any("feeds" in item for item in mutated(json_schema, lambda s: s.update(feeds=[])))
+    assert any("feeds: at most" in item for item in mutated(json_schema, lambda s: s.update(feeds=[copy.deepcopy(s["feeds"][0]) | {"id": f"f{n}"} for n in range(9)])))
+    assert any("used twice" in item for item in mutated(json_schema, lambda s: s.update(feeds=[s["feeds"][0], copy.deepcopy(s["feeds"][0])])))
+    assert any("feeds[0].url" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(url="ftp://x")))
+    assert any("feeds[0].type" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(type="soap")))
+    assert any("post.fields.id" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["post"]["fields"].pop("id")))
+    assert any("content or title" in item for item in mutated(json_schema, lambda s: [s["feeds"][0]["post"]["fields"].pop(k) for k in ("content", "title")]))
+    assert any("$" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["post"]["fields"].update(id=["id"])))
+    assert any("item_selector" in item for item in mutated(html_schema, lambda s: s["feeds"][0].pop("item_selector")))
+    assert any("unsupported" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["post"]["fields"].update(title=["h2::bogus"])))
+    assert any("selector" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["post"]["fields"].update(title=["h2[unclosed"])))
+
+
+def test_comment_and_pagination_specs_are_checked(json_schema, html_schema):
+    assert any("comments.mode" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["comments"].update(mode="telepathy")))
+    assert any("list_path" in item for item in mutated(json_schema, lambda s: s["feeds"][0]["comments"].pop("list_path")))
+    assert any("url_template" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(comments={"mode": "url", "fields": {"id": ["$.id"], "content": ["$.b"]}, "url_template": "https://x/y"})))
+    assert any("item_selector" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["comments"].pop("item_selector")))
+    assert any("pagination.style" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "psychic"})))
+    assert any("param" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "page"})))
+    assert any("per_page" in item for item in mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "offset", "param": "o"})))
+    assert mutated(json_schema, lambda s: s["feeds"][0].update(pagination={"style": "none"})) == []
+    assert any("pagination.next" in item for item in mutated(html_schema, lambda s: s["feeds"][0]["comments"].update(pagination={"style": "next_link", "next": "a[::attr(href)"})))
+    assert mutated(html_schema, lambda s: s["feeds"][0]["comments"].update(pagination={"style": "page_param", "param": "cp"})) == []
+
+
+def test_limits_have_ceilings_and_a_politeness_floor(json_schema):
+    assert any("max_posts" in item for item in mutated(json_schema, lambda s: s["limits"].update(max_posts=100000)))
+    assert any("delay_seconds" in item and "polite" in item for item in mutated(json_schema, lambda s: s["limits"].update(delay_seconds=0.01)))
+    assert any("unknown limit" in item for item in mutated(json_schema, lambda s: s["limits"].update(turbo=1)))
+    assert any("positive" in item for item in mutated(json_schema, lambda s: s["limits"].update(retries=-1)))
+    assert any("positive" in item for item in mutated(json_schema, lambda s: s["limits"].update(retries=True)))
+    assert mutated(json_schema, lambda s: s["limits"].update(delay_seconds=2.5, max_posts=10)) == []
+
+
+def test_login_specs(json_schema):
+    assert any("login.mode" in item for item in mutated(json_schema, lambda s: s.update(login={"mode": "magic"})))
+    assert any("username_field" in item for item in mutated(json_schema, lambda s: s.update(login={"mode": "json", "url": "https://x/l"})))
+    assert mutated(json_schema, lambda s: s.update(login={"mode": "json", "url": "https://x/l", "username_field": "u", "password_field": "p", "token_path": "$.t"})) == []
+
+
+def test_the_sample_run_is_small_and_polite(json_schema):
+    limits = sample_limits(json_schema)
+    assert limits["max_pages"] == 1 and limits["max_comment_pages"] == 1 and limits["max_posts"] <= 12
+    assert limits["delay_seconds"] >= 1.0
+    fast = copy.deepcopy(json_schema)
+    fast["limits"]["delay_seconds"] = 5.0
+    assert sample_limits(fast)["delay_seconds"] == 5.0

tests/unit/services/community/structured.py

@@ -0,0 +1,100 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+import pytest
+
+from devplacepy.services.community.discoverer import CommunityDiscoverer
+from devplacepy.services.community.extract import extract_json_ld_posts, parse_html
+from devplacepy.services.community.generator import CodeGenerator
+from devplacepy.services.community.runtime import Collector
+from devplacepy.services.community.schemacheck import validate_schema
+from devplacepy.services.sync.records import RECORD_MODELS
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher
+from tests.unit.services.community.generator import load, no_sleep
+
+
+def thread(number: int, comments: int = 2) -> dict:
+    return {
+        "@type": "DiscussionForumPosting",
+        "@id": f"https://ld.test/t/{number}",
+        "headline": f"Thread number {number} about parsing",
+        "text": f"This is the body of thread {number}, written long enough to sync as a post.",
+        "author": {"@type": "Person", "name": f"writer{number % 2}", "url": f"https://ld.test/u/{number % 2}"},
+        "datePublished": f"2026-10-{number:02d}T10:00:00Z",
+        "dateModified": f"2026-10-{number:02d}T12:00:00Z",
+        "url": f"https://ld.test/t/{number}",
+        "comment": [
+            {
+                "@type": "Comment",
+                "@id": f"https://ld.test/t/{number}#c{index}",
+                "text": f"Comment {index} on thread {number}",
+                "author": {"@type": "Person", "name": f"replier{index}"},
+                "dateCreated": f"2026-10-{number:02d}T11:0{index}:00Z",
+            }
+            for index in range(comments)
+        ],
+    }
+
+
+def page(blobs: list) -> str:
+    scripts = "".join(f'<script type="application/ld+json">{json.dumps(blob)}</script>' for blob in blobs)
+    return f"<html><head><title>LD Forum</title>{scripts}</head><body><div id=app>Loading</div></body></html>"
+
+
+def ld_site(blobs=None) -> dict:
+    blobs = blobs or [{"@context": "https://schema.org", "@graph": [thread(number) for number in range(1, 7)]}]
+    return {"https://ld.test/": page(blobs), "https://ld.test/about": "<html><body><p>About: 100 members.</p></body></html>"}
+
+
+def test_the_extractor_normalizes_graphs_lists_and_nested_comments():
+    root = parse_html(page([{"@graph": [thread(1), {"@type": "WebSite", "name": "x"}]}, thread(2), [thread(3)], {"@type": "ItemList", "itemListElement": [{"item": thread(4)}]}]))
+    posts = extract_json_ld_posts(root)
+    assert [item["id"] for item in posts] == [f"https://ld.test/t/{n}" for n in (1, 2, 3, 4)]
+    first = posts[0]
+    assert first["title"] == "Thread number 1 about parsing" and first["author"] == {"username": "writer1", "id": "https://ld.test/u/1"}
+    assert first["created_at"].startswith("2026-10-01") and len(first["comments"]) == 2
+    assert first["comments"][0]["content"] == "Comment 0 on thread 1" and first["comments"][0]["author"] == {"username": "replier0"}
+
+
+def test_garbage_and_non_posts_are_ignored():
+    root = parse_html('<script type="application/ld+json">{not json</script><script type="application/ld+json">{"@type":"Organization","name":"x"}</script><script>var x=1</script>')
+    assert extract_json_ld_posts(root) == []
+    nameless = parse_html(page([{"@type": "Article", "headline": "No identity at all"}]))
+    assert extract_json_ld_posts(nameless) == []
+
+
+def test_a_site_with_only_structured_data_is_learned_end_to_end():
+    result = run_async(CommunityDiscoverer(FakeFetcher(ld_site())).discover("https://ld.test/"))
+    assert result.status == "ready", result.error
+    feed = result.schema["feeds"][0]
+    assert feed["type"] == "json_ld" and feed["post"]["fields"]["id"] and feed["comments"]["mode"] == "inline"
+    assert validate_schema(result.schema) == []
+    records = result.sample["records"]
+    assert len(records["posts"]) == 6 and len(records["comments"]) == 6 and records["users"]
+    full = run_async(Collector(FakeFetcher(ld_site()), result.schema, sleep=no_sleep).run())
+    assert len(full["records"]["comments"]) == 12
+    for kind, rows in records.items():
+        for row in rows:
+            RECORD_MODELS[kind].model_validate(row)
+    assert {user["username"] for user in records["users"]} >= {"writer0", "writer1", "replier0"}
+
+
+def test_the_collector_and_the_generated_file_agree_on_structured_data(tmp_path):
+    schema = run_async(CommunityDiscoverer(FakeFetcher(ld_site())).discover("https://ld.test/")).schema
+    direct = run_async(Collector(FakeFetcher(ld_site()), schema, sleep=no_sleep).run())
+    module = load(CodeGenerator(schema).write(tmp_path))
+    generated = run_async(module.collect_all(FakeFetcher(ld_site()), sleep=no_sleep))
+    assert {kind: len(rows) for kind, rows in direct["records"].items()} == {kind: len(rows) for kind, rows in generated["records"].items()}
+
+
+def test_a_single_structured_post_is_not_a_feed():
+    result = run_async(CommunityDiscoverer(FakeFetcher(ld_site([thread(1)]))).discover("https://ld.test/"))
+    assert result.status != "ready"
+
+
+@pytest.mark.parametrize("count", [2, 5])
+def test_few_posts_still_make_a_feed(count):
+    result = run_async(CommunityDiscoverer(FakeFetcher(ld_site([{"@graph": [thread(number, 0) for number in range(1, count + 1)]}]))).discover("https://ld.test/"))
+    assert result.status == "ready", result.error

tests/unit/services/community/submitter.py

@@ -0,0 +1,60 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.community.extract import split_submitter
+from devplacepy.services.community.runtime import Collector
+from tests.conftest import run_async
+from tests.unit.services.community.dedupe import entry
+from tests.unit.services.community.fakes import FakeFetcher
+from tests.unit.services.community.generator import no_sleep
+
+
+@pytest.mark.parametrize(
+    "raw,expected",
+    [
+        ("lore.kernel.org via thang", "thang"),
+        ("jakearchibald.com by jaffathecake", "jaffathecake"),
+        ("decomp.dev via FedericoSchonborn", "FedericoSchonborn"),
+        ("via rberger 2026-10-15 12:36:38", "rberger"),
+        (" via spikewall", "spikewall"),
+        ("plainname", "plainname"),
+        ("two words", "two words"),
+        ("", ""),
+        ("site.org VIA Someone", "Someone"),
+    ],
+)
+def test_the_submitter_is_the_person_behind_by_or_via(raw, expected):
+    assert split_submitter(raw) == expected
+
+
+def schema():
+    return {
+        "community": "lobsters",
+        "origin": "lobsters.test",
+        "limits": {"delay_seconds": 0, "retries": 1},
+        "feeds": [
+            {
+                "id": "front",
+                "url": "https://lobsters.test/front",
+                "type": "json_api",
+                "list_path": "$.items",
+                "post": {"fields": {"id": ["$.id"], "title": ["$.title"], "content": ["$.body"], "created_at": ["$.at"], "url": ["$.link"]}},
+                "author": {"fields": {"username": ["$.who"]}},
+                "comments": {"mode": "none"},
+            }
+        ],
+    }
+
+
+def test_one_person_sharing_from_many_sites_is_one_user_with_one_identity():
+    items = [
+        {**entry("S1", "https://decomp.dev/a", "First"), "who": "decomp.dev via FedericoSchonborn"},
+        {**entry("S2", "https://costa.jacobpalm.dk/b", "Second"), "who": "costa.jacobpalm.dk via FedericoSchonborn"},
+        {**entry("S3", "https://abner.page/c", "Third"), "who": "abner.page by abnercoimbre"},
+    ]
+    result = run_async(Collector(FakeFetcher({"https://lobsters.test/front": {"items": items}}), schema(), None, sleep=no_sleep).run(None))
+    users = result["records"]["users"]
+    assert sorted(user["username"] for user in users) == ["FedericoSchonborn", "abnercoimbre"]
+    posts = {post["title"]: post["author_uid"] for post in result["records"]["posts"]}
+    assert posts["First"] == posts["Second"] != posts["Third"]