Community sync: fix comment, identity and refusal bugs; add per-record sync ledger

Fixes found by investigating the live lobste.rs run:
- Comments came back 0: the learner probed the external article, not the
  discussion page. A comments_url field (RSS <comments>, JSON key, or an HTML
  "comments" anchor) now drives comment learning; the article is still used
  for the post body.
- HTML id selectors collapsed every story into one identity (constant /login
  vote link). Id/url selectors must now be distinct; no distinct selector
  rejects the feed.
- Over-long posts were rejected; title, post and comment are truncated to the
  sync limits and reported as a warning.
- A failed article fetch is a warning, not a run error.
- A gate refusal (sync disabled, origin not allowed, aliasing off) ends the run
  as skipped and no longer counts toward the failure floor.
- The same story from two feeds becomes one post, comments attached to it.
- HTML comments follow rel=next up to max_comment_pages.
- A dry run now predicts an unknown-record tombstone as unchanged, like a live
  run.
- /nodes is exempt from the happy-404 decoy so a missing download is a real 404.

Sync log (admin): every run stores a verdict, exact per-outcome counts and one
row per record needing an explanation, with the reason in plain words, at
/admin/communities/{slug}/ledger (HTML, JSON, Devii tool, docs entry).
Bounded: 2000 routine + 2000 problem rows per run, 50 per page, detail kept for
the 10 newest runs, four indexes matching every ORDER BY (asserted by an
EXPLAIN QUERY PLAN test). Measured worst case 37 ms at the 40000-row ceiling.

Also: privacy disclosure for imported community content, status colour channel
tokens, and tests/unit/cli/nodes.py renamed to fix a duplicate-basename
collection error.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

tests/unit/services/community/discussion.py

@@ -0,0 +1,121 @@
+# retoor <retoor@molodetz.nl>
+
+import copy
+
+from tests.conftest import run_async
+
+from devplacepy.services.community.discoverer import CommunityDiscoverer
+from devplacepy.services.community.learner import HtmlInferrer
+from devplacepy.services.community.runtime import MAX_POST_LENGTH, Collector
+from devplacepy.services.community.extract import parse_html, select_nodes
+from devplacepy.services.sync.records import RECORD_MODELS
+from tests.unit.services.community.fakes import (
+    FakeFetcher,
+    RSS,
+    detail_page,
+    json_community,
+    no_sleep,
+)
+
+LINKED_ITEM = """<item><title>Linked story {n} on testing</title><link>https://elsewhere.test/a/{n}</link>
+<guid>https://blog.test/s/{n}</guid><pubDate>Tue, {n:02d} Oct 2026 10:00:00 GMT</pubDate>
+<comments>https://blog.test/s/{n}</comments>
+<dc:creator>Alice</dc:creator><description>&lt;p&gt;Body of story {n} with enough text.&lt;/p&gt;</description></item>"""
+
+
+def discussion_community() -> dict:
+    items = "".join(LINKED_ITEM.format(n=n) for n in range(1, 7))
+    routes = {
+        "https://blog.test/": '<html><head><link rel="alternate" type="application/rss+xml" href="/feed.xml">'
+        "<title>Blog Test</title></head><body>hi</body></html>",
+        "https://blog.test/feed.xml": RSS.format(items=items),
+    }
+    for n in range(1, 7):
+        routes[f"https://elsewhere.test/a/{n}"] = f"<html><body><article>{'External article text. ' * 20}</article></body></html>"
+        routes[f"https://blog.test/s/{n}"] = detail_page("story", n)
+    return routes
+
+
+def learn(routes: dict, query: str) -> dict:
+    result = run_async(CommunityDiscoverer(FakeFetcher(routes)).discover(query))
+    assert result.schema, result.error
+    return result.schema
+
+
+def test_rss_comments_are_learned_from_the_discussion_link_not_the_article():
+    routes = discussion_community()
+    schema = learn(routes, "https://blog.test/")
+    feed = schema["feeds"][0]
+    assert feed["post"]["fields"]["comments_url"] == ["$.comments"]
+    assert feed["comments"]["mode"] == "page"
+    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    assert result["records"]["comments"]
+    posts = {post["origin_uid"] for post in result["records"]["posts"]}
+    assert {comment["post_uid"] for comment in result["records"]["comments"]} <= posts
+
+
+def test_post_body_still_comes_from_the_article_page():
+    routes = discussion_community()
+    schema = learn(routes, "https://blog.test/")
+    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    assert all("External article text." in post["content"] for post in result["records"]["posts"])
+
+
+LISTING = """<div class="story"><a class="vote" href="/login">1</a>
+<a class="u-url" href="https://far.test/{n}">Story {n} about shells</a>
+<a class="discuss" href="/s/{n}">{n} comments</a><span class="byline">via user{n}</span>
+<time datetime="2026-10-{n:02d}T09:00:00Z">d</time><p>Excerpt of story {n} that is long enough to count.</p></div>"""
+
+
+def listing_nodes() -> list:
+    root = parse_html("<html><body>" + "".join(LISTING.format(n=n) for n in range(1, 9)) + "</body></html>")
+    return select_nodes(root, "div.story")
+
+
+def test_constant_first_anchor_is_never_chosen_as_identifier():
+    fields = HtmlInferrer(listing_nodes(), "https://board.test/").post_fields()
+    assert fields["id"] == ["a.u-url::attr(href)"]
+    assert fields["title"] == ["a.u-url"]
+    assert fields["comments_url"] == ["a.discuss::attr(href)"]
+
+
+def test_no_distinct_anchor_means_no_identifier():
+    page = "".join('<div class="story"><a href="/login">x</a><p>Some text that is long enough here.</p></div>' for _ in range(6))
+    nodes = select_nodes(parse_html(f"<html><body>{page}</body></html>"), "div.story")
+    assert "id" not in HtmlInferrer(nodes, "https://board.test/").post_fields()
+
+
+def test_oversized_posts_and_titles_are_truncated_to_valid_sync_records():
+    routes = json_community()
+    schema = learn(routes, "https://forum.test/api/posts")
+    routes["https://forum.test/api/posts"]["items"][0]["content_html"] = "<p>" + "x" * (MAX_POST_LENGTH * 2) + "</p>"
+    routes["https://forum.test/api/posts"]["items"][0]["title"] = "t" * 900
+    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    for post in result["records"]["posts"]:
+        assert len(post["content"]) <= MAX_POST_LENGTH
+        RECORD_MODELS["posts"].model_validate(post)
+    assert any(len(post["content"]) == MAX_POST_LENGTH for post in result["records"]["posts"])
+
+
+def test_failed_article_fetch_is_a_warning_and_the_post_is_still_imported():
+    routes = discussion_community()
+    schema = learn(routes, "https://blog.test/")
+    assert schema["feeds"][0]["post"].get("detail")
+    routes["https://elsewhere.test/a/1"] = (403, "forbidden")
+    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    assert result["errors"] == []
+    assert any("elsewhere.test/a/1" in line for line in result["warnings"])
+    assert len(result["records"]["posts"]) == 6
+
+
+def test_same_story_from_two_feeds_is_one_post_and_comments_follow_it():
+    routes = discussion_community()
+    schema = learn(routes, "https://blog.test/")
+    second = copy.deepcopy(schema["feeds"][0])
+    second["id"] = "feed2"
+    second["post"]["fields"]["id"] = ["$.link"]
+    schema["feeds"].append(second)
+    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    assert len(result["records"]["posts"]) == 6
+    posts = {post["origin_uid"] for post in result["records"]["posts"]}
+    assert {comment["post_uid"] for comment in result["records"]["comments"]} <= posts

tests/unit/services/community/generator.py

@@ -166,3 +166,41 @@ def test_javascript_feed_is_collected_through_the_browser(tmp_path):
     module = load(CodeGenerator(schema).write(tmp_path))
     collected = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
     assert len(collected["records"]["posts"]) == 6
+
+
+def post_comment_count(result: dict, post_origin_uid: str) -> int:
+    return sum(1 for comment in result["records"]["comments"] if comment["post_uid"] == post_origin_uid)
+
+
+def paginated_board(next_href: str) -> dict:
+    from tests.unit.services.community.fakes import detail_page
+
+    routes = html_community()
+    first = detail_page("thread", 1).replace("</body>", f'<a rel="next" href="{next_href}">more</a></body>')
+    routes["https://board.test/t/1"] = first
+    routes["https://board.test/t/1?cp=2"] = detail_page("thread", 1, comments=4)
+    return routes
+
+
+def test_html_comments_follow_the_next_page_link():
+    routes = paginated_board("/t/1?cp=2")
+    schema = learn(html_community(), "https://board.test/")
+    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    first_post = next(post["origin_uid"] for post in result["records"]["posts"] if post["title"].startswith("Thread 1 "))
+    assert post_comment_count(result, first_post) == 5
+
+
+def test_html_comment_pagination_stops_on_a_cycle_and_at_the_page_cap():
+    routes = paginated_board("/t/1")
+    schema = learn(html_community(), "https://board.test/")
+    fetcher = FakeFetcher(routes)
+    run_async(Collector(fetcher, schema, sleep=no_sleep).run())
+    assert fetcher.requests.count("https://board.test/t/1") == 1
+    routes = paginated_board("/t/1?cp=2")
+    routes["https://board.test/t/1?cp=2"] = routes["https://board.test/t/1?cp=2"].replace("</body>", '<a rel="next" href="/t/1?cp=3">more</a></body>')
+    routes["https://board.test/t/1?cp=3"] = routes["https://board.test/t/1?cp=2"].replace("cp=3", "cp=4")
+    routes["https://board.test/t/1?cp=4"] = routes["https://board.test/t/1?cp=2"]
+    fetcher = FakeFetcher(routes)
+    schema["limits"] = {**schema.get("limits", {}), "max_comment_pages": 3}
+    run_async(Collector(fetcher, schema, sleep=no_sleep).run())
+    assert "https://board.test/t/1?cp=4" not in fetcher.requests

tests/unit/services/community/ledger.py

@@ -0,0 +1,226 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+import pytest
+
+from tests.conftest import run_async
+
+from devplacepy.database import db, get_setting, get_table, init_db, set_setting
+from devplacepy.services.community.ledger import (
+    LEDGER,
+    Ledger,
+    OUTCOME_KEYS,
+    REASON_TEXT,
+    explain,
+    label_for,
+    tally_of,
+    verdict,
+)
+from devplacepy.services.community.ledger_store import TABLE, ledger_store
+from devplacepy.services.community.pusher import SyncPusher
+from devplacepy.services.community.runner import CommunityRunner
+from devplacepy.services.community.store import store
+from devplacepy.services.sync.options import SyncOptions
+from devplacepy.services.sync.selection import Selector
+from devplacepy.services.sync.spec import REASONS, SKIP_REASONS
+from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep
+
+ORIGIN = "ledger-unit.test"
+
+
+@pytest.fixture
+def world():
+    init_db()
+    previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins")}
+    set_setting("sync_enabled", "1")
+    set_setting("sync_origins", "")
+    get_table("communities").delete()
+    get_table("community_runs").delete()
+    get_table(TABLE).delete()
+    get_table("sync_links").delete(origin=ORIGIN)
+    get_table("sync_links").delete(origin="forum.test")
+    yield
+    for key, value in previous.items():
+        set_setting(key, value)
+
+
+def records() -> dict:
+    return {
+        "users": [{"origin_uid": "u1", "username": "ledgerone", "created_at": "2026-10-01T10:00:00Z"}],
+        "posts": [
+            {"origin_uid": "p1", "author_uid": "u1", "title": "A good post", "content": "Body of a good post, long enough.", "created_at": "2026-10-01T10:00:00Z"},
+            {"origin_uid": "p2", "author_uid": "ghost", "title": "Orphan post", "content": "Written by someone who was never sent.", "created_at": "2026-10-01T10:00:00Z"},
+            {"origin_uid": "p3", "title": "Gone", "deleted": True, "author_uid": "u1", "content": "Deleted at the origin before it ever arrived.", "created_at": "2026-10-01T10:00:00Z"},
+        ],
+        "comments": [],
+    }
+
+
+def test_pusher_records_created_rejected_and_held_records_with_reasons(world):
+    ledger = Ledger()
+    report = SyncPusher(ORIGIN, ledger=ledger).push(records())
+    by_origin = {item.origin_uid: item for item in ledger.items}
+    assert by_origin["u1"].outcome == "created" and by_origin["u1"].label == "ledgerone" and by_origin["u1"].local_uid
+    assert by_origin["p1"].outcome == "created" and by_origin["p1"].label == "A good post"
+    assert by_origin["p2"].outcome == "rejected" and by_origin["p2"].reason == "unknown_author" and by_origin["p2"].label == "Orphan post"
+    assert by_origin["p3"].outcome == "unchanged" and by_origin["p3"].reason == "tombstone_for_unknown"
+    assert tally_of(report.counts, ledger) == {"created": 2, "unchanged": 1, "rejected": 1}
+
+
+def test_repeating_the_same_batch_lists_nothing_that_is_merely_unchanged(world):
+    SyncPusher(ORIGIN).push(records())
+    ledger = Ledger()
+    report = SyncPusher(ORIGIN, ledger=ledger).push(records())
+    assert {item.origin_uid for item in ledger.items} == {"p2", "p3"}
+    assert report.counts["users"] == {"unchanged": 1}
+
+
+def test_a_dry_run_lists_what_it_would_create(world):
+    ledger = Ledger()
+    SyncPusher(ORIGIN, dry_run=True, ledger=ledger).push(records())
+    created = [item for item in ledger.items if item.outcome == "created"]
+    assert {item.origin_uid for item in created} == {"u1", "p1"}
+    assert get_table("sync_links").count(origin=ORIGIN) == 0
+
+
+def test_caps_never_let_routine_rows_push_out_problems_and_counts_stay_exact():
+    ledger = Ledger(routine_limit=3, problem_limit=2)
+    for number in range(10):
+        ledger.add("posts", "created", origin_uid=f"c{number}")
+    for number in range(5):
+        ledger.add("posts", "rejected", reason="invalid", origin_uid=f"r{number}")
+    kept = [item.outcome for item in ledger.items]
+    assert kept.count("created") == 3 and kept.count("rejected") == 2
+    assert ledger.dropped == 7 + 3
+    assert ledger.counts["created"] == 10 and ledger.counts["rejected"] == 5
+
+
+def test_every_reason_the_system_can_emit_has_a_plain_explanation():
+    for reason in (*REASONS, *SKIP_REASONS, "article_unreachable", "truncated", "collect_error"):
+        assert reason in REASON_TEXT, reason
+        assert explain(reason)["title"] and explain(reason)["text"]
+    assert explain("never_heard_of_it")["title"] == "Never heard of it"
+    assert explain("identity_failed")["retried"] and not explain("sampled_out")["retried"]
+    assert explain("") == {"title": "", "text": "", "retried": False}
+
+
+@pytest.mark.parametrize(
+    "status,dry,tally,level",
+    [
+        ("ok", False, {"created": 5}, "ok"),
+        ("ok", False, {"unchanged": 40}, "idle"),
+        ("ok", False, {}, "idle"),
+        ("partial", False, {"created": 5, "rejected": 1}, "attention"),
+        ("partial", False, {"created": 5, "skipped": 3}, "attention"),
+        ("partial", False, {"created": 5, "warning": 1}, "attention"),
+        ("partial", False, {"created": 5, "failed": 1}, "bad"),
+        ("partial", False, {"error": 2}, "bad"),
+        ("failed", False, {}, "bad"),
+        ("skipped", False, {}, "attention"),
+        ("running", False, {}, "running"),
+        ("dry_run", True, {"created": 5}, "ok"),
+    ],
+)
+def test_verdict_levels(status, dry, tally, level):
+    outcome = verdict(status, dry, tally)
+    assert outcome["level"] == level and outcome["headline"]
+    assert outcome["headline"].endswith("(dry run).") == (dry and status != "running")
+
+
+def test_tally_merges_sync_statuses_skips_and_collector_findings():
+    ledger = Ledger()
+    ledger.add("posts", "warning", reason="truncated")
+    ledger.add("feed", "error", reason="collect_error")
+    pushed = {"users": {"created": 2}, "posts": {"created": 1, "updated": 1, "rejected": 1}, "skipped": {"sampled_out": 4, "topic_not_allowed": 1}}
+    assert tally_of(pushed, ledger) == {"created": 3, "updated": 1, "rejected": 1, "skipped": 5, "warning": 1, "error": 1}
+    assert set(tally_of(pushed, ledger)) <= set(OUTCOME_KEYS)
+
+
+def test_labels_are_clipped_single_lines():
+    assert label_for("users", {"username": "bob"}) == "bob"
+    assert label_for("posts", {"title": "", "content": "line one\n\nline   two"}) == "line one line two"
+    assert len(label_for("comments", {"content": "x" * 900})) == LEDGER.label_chars
+
+
+def test_selector_reports_every_record_it_drops():
+    selector = Selector(ORIGIN, SyncOptions.build(sample_percent=1))
+    batch = {
+        "posts": [{"origin_uid": f"s{number}", "title": "t", "content": "c" * 20} for number in range(60)],
+        "comments": [{"origin_uid": f"c{number}", "post_uid": f"s{number}", "content": "hey"} for number in range(60)],
+    }
+    kept, skipped = selector.select(batch)
+    assert len(selector.dropped) == sum(skipped.values()) > 0
+    assert {reason for _, _, reason in selector.dropped} <= {"sampled_out", "parent_skipped"}
+    ledger = Ledger()
+    for kind, record, reason in selector.dropped:
+        ledger.add_dropped_record(kind, record, reason)
+    assert ledger.counts["skipped"] == len(selector.dropped) and all(item.reason for item in ledger.items)
+
+
+def test_a_run_stores_its_ledger_and_a_repeat_stores_nothing(world):
+    routes = json_community()
+    runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep)
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test"))
+    first = run_async(runner.run(store.get("forum-test"), dry_run=False, trigger="test"))
+    saved = store.run(first.run_uid)
+    assert saved["items_stored"] == get_table(TABLE).count(run_uid=first.run_uid) > 0
+    assert saved["tally"]["created"] == ledger_store.counts(run_uid=first.run_uid)["created"]
+    assert saved["verdict"] == "ok" and saved["items_dropped"] == 0
+    second = run_async(runner.run(store.get("forum-test"), dry_run=False, trigger="test"))
+    again = store.run(second.run_uid)
+    assert again["items_stored"] == 0 and again["verdict"] == "idle" and again["tally"]["unchanged"] > 0
+
+
+def test_query_plans_use_an_index_and_never_sort_or_scan(world):
+    ledger = Ledger(routine_limit=4000, problem_limit=4000)
+    for number in range(3000):
+        ledger.add("posts", ("created", "rejected", "skipped", "updated")[number % 4], reason="invalid", origin_uid=f"o{number}", label=f"title {number}")
+    ledger_store.save("plan-run", "plan-community", ledger)
+    db.query("ANALYZE")
+    cases = [
+        ({}, {"run_uid": "plan-run"}),
+        ({"outcome": "rejected"}, {"run_uid": "plan-run"}),
+        ({"outcome": "rejected", "kind": "posts"}, {"run_uid": "plan-run"}),
+        ({}, {"community_uid": "plan-community"}),
+        ({"outcome": "skipped"}, {"community_uid": "plan-community"}),
+    ]
+    for filters, scope in cases:
+        where, params, order = ledger_store.compile(filters, **scope)
+        for sql in (
+            f"SELECT * FROM {TABLE} WHERE {where} ORDER BY {order} LIMIT 50 OFFSET 0",
+            f"SELECT COUNT(*) FROM (SELECT 1 FROM {TABLE} WHERE {where} LIMIT 10001)",
+        ):
+            plan = " | ".join(row["detail"] for row in db.query("EXPLAIN QUERY PLAN " + sql, **params))
+            assert "USING INDEX" in plan or "USING COVERING INDEX" in plan, (filters, scope, plan)
+            assert "SCAN community_run_items" not in plan, plan
+            if not filters.get("outcome") or not scope.get("run_uid"):
+                assert "TEMP B-TREE" not in plan, (filters, scope, plan)
+    assert ledger_store.counts(run_uid="plan-run") == {"created": 750, "rejected": 750, "skipped": 750, "updated": 750}
+    rows, pagination = ledger_store.page({"q": "title 7%"}, 1, run_uid="plan-run")
+    assert rows == [] and pagination["total"] == 0
+    rows, pagination = ledger_store.page({"q": "title 77"}, 1, run_uid="plan-run")
+    assert pagination["total"] == 11 and len(rows) == 11
+    rows, pagination = ledger_store.page({}, 10**6, run_uid="plan-run")
+    assert pagination["page"] == pagination["total_pages"] and len(rows) == 50 and pagination["capped"] is False
+    json.dumps(rows)
+
+
+def test_record_detail_is_kept_for_the_newest_runs_only_but_their_counts_stay(world):
+    from devplacepy.services.community.spec import LIMITS
+
+    community = store.create("keep-test", "q", "tester", name="keep")
+    uids = []
+    for number in range(LIMITS.ledger_runs_kept + 3):
+        run_uid = store.start_run(community["uid"], "test", False)
+        ledger = Ledger()
+        ledger.add("posts", "created", origin_uid=f"k{number}")
+        ledger_store.save(run_uid, community["uid"], ledger)
+        store.finish_run(run_uid, "ok", tally={"created": 1}, items_stored=1)
+        uids.append(run_uid)
+    store.prune_runs(community["uid"])
+    kept = [get_table(TABLE).count(run_uid=uid) for uid in uids]
+    assert sum(kept) == LIMITS.ledger_runs_kept and kept[0] == 0 and kept[-1] == 1
+    assert len(store.run_summaries(community["uid"])) == LIMITS.ledger_runs_kept + 3
+    assert store.run(uids[0])["tally"] == {"created": 1}

tests/unit/services/community/service.py

@@ -7,7 +7,7 @@ import pytest
 from devplacepy.database import get_setting, get_table, init_db, set_setting
 from devplacepy.services.community.runner import CommunityRunner
 from devplacepy.services.community.service import DRY_RUN_KEY, CommunityService
-from devplacepy.services.community.spec import STATUS_FAILING, STATUS_READY
+from devplacepy.services.community.spec import LIMITS, RUN_SKIPPED, STATUS_FAILING, STATUS_READY
 from devplacepy.services.community.store import store
 from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep
 
@@ -79,3 +79,17 @@ def test_run_history_is_bounded(world):
     for _ in range(35):
         run_async(runner.run(store.get("forum-test"), dry_run=True, trigger="test"))
     assert len(store.runs(row["uid"], limit=100)) == 30
+
+
+def test_gate_refusal_skips_the_run_without_counting_a_failure(world):
+    _, runner = world
+    row = seed(runner)
+    set_setting("sync_enabled", "0")
+    for _ in range(LIMITS.max_failures + 1):
+        report = run_async(runner.run(store.get("forum-test"), dry_run=False, trigger="test"))
+        assert report.status == RUN_SKIPPED
+    final = store.get("forum-test")
+    assert final["status"] == STATUS_READY and final["enabled"]
+    assert final["failure_count"] == 0 and final["last_status"] == RUN_SKIPPED
+    assert final["next_run_at"] and get_table("sync_links").count(origin="forum.test") == 0
+    assert store.runs(row["uid"])[0]["status"] == RUN_SKIPPED