Zo doen wijdat in oost

tests/unit/services/community/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/unit/services/community/ai.py

@@ -0,0 +1,67 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+from devplacepy.services.community.ai import AiAssistant, outline, parse_json, to_html
+from devplacepy.services.community.extract import parse_html, select_nodes
+from tests.unit.services.community.fakes import HTML_ITEM
+
+
+class Canned(AiAssistant):
+    def __init__(self, answer):
+        self.answer = answer
+        self.asked = []
+
+    def enabled(self):
+        return True
+
+    async def ask(self, system, user):
+        self.asked.append((system, user))
+        return self.answer
+
+
+def nodes():
+    root = parse_html("<main>" + "".join(HTML_ITEM.format(n=n) for n in range(1, 5)) + "</main>")
+    return root, select_nodes(root, "article.post")
+
+
+def test_json_is_extracted_from_fenced_or_chatty_replies():
+    assert parse_json('```json\n{"a": 1}\n```') == {"a": 1}
+    assert parse_json('Sure! Here you go: {"a": {"b": 2}} hope it helps') == {"a": {"b": 2}}
+    assert parse_json("no json") is None
+
+
+def test_only_proposals_that_match_the_sample_are_accepted():
+    _, items = nodes()
+    assistant = Canned({"title": "h2", "link": "h2 a::attr(href)", "author": ".nonexistent", "created_at": "time::attr(datetime)"})
+    accepted = run_async(assistant.post_fields(items))
+    assert accepted == {"title": "h2", "link": "h2 a::attr(href)", "created_at": "time::attr(datetime)"}
+
+
+def test_item_selector_must_match_at_least_three_elements():
+    root, _ = nodes()
+    assert run_async(Canned({"item_selector": "article.post"}).item_selector(root)) == "article.post"
+    assert run_async(Canned({"item_selector": "article.missing"}).item_selector(root)) is None
+    assert run_async(Canned({"item_selector": "!!"}).item_selector(root)) is None
+
+
+def test_comment_structure_needs_content():
+    root = parse_html('<div><div class="c" id="a1"><p>hello there</p><b>ann</b></div><div class="c" id="a2"><p>second text</p><b>bob</b></div></div>')
+    good = run_async(Canned({"item_selector": "div.c", "id": "::attr(id)", "content": "p", "author": "b"}).comment_structure(root))
+    assert good == {"item_selector": "div.c", "id": "::attr(id)", "content": "p", "author": "b"}
+    assert run_async(Canned({"item_selector": "div.c", "content": "span"}).comment_structure(root)) is None
+
+
+def test_disabled_assistant_never_asks():
+    class Off(AiAssistant):
+        def enabled(self):
+            return False
+
+    assert run_async(Off().ask("s", "u")) is None
+
+
+def test_outline_and_html_are_bounded():
+    root, items = nodes()
+    assert "article.post" in outline(root)
+    assert len(outline(root, 3).splitlines()) <= 3
+    assert len(to_html(items[0], 100)) <= 100

tests/unit/services/community/alias.py

@@ -0,0 +1,133 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.database import get_setting, get_table, init_db, set_setting
+from devplacepy.services.community.runner import TRIGGER_CLI, CommunityRunner
+from devplacepy.services.community.store import store
+from devplacepy.services.sync import ai
+from devplacepy.services.sync.options import ALIAS_KEY, SyncOptions
+from tests.conftest import run_async
+from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep
+
+ORIGINALS = ("writer0", "writer1", "replier0", "replier1")
+
+
+@pytest.fixture
+def world(monkeypatch):
+    init_db()
+    keys = ("sync_enabled", "sync_origins", ALIAS_KEY)
+    previous = {key: get_setting(key, "") for key in keys}
+    set_setting("sync_enabled", "1")
+    set_setting("sync_origins", "")
+    set_setting(ALIAS_KEY, "1")
+    for table in ("communities", "community_runs"):
+        get_table(table).delete()
+    get_table("sync_links").delete(origin="forum.test")
+    get_table("sync_identities").delete(origin="forum.test")
+
+    async def unreachable(system, text, timeout):
+        return None
+
+    monkeypatch.setattr(ai, "complete", unreachable)
+    runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(json_community()), sleep=no_sleep)
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test"))
+    yield runner
+    for key, value in previous.items():
+        set_setting(key, value)
+    get_table("sync_identities").delete(origin="forum.test")
+
+
+def go(runner, dry_run=False):
+    return run_async(runner.run(store.get("forum-test"), dry_run=dry_run, trigger=TRIGGER_CLI))
+
+
+def synced(kind: str) -> list[dict]:
+    links = get_table("sync_links").find(origin="forum.test", kind=kind)
+    return [row for row in (get_table(kind).find_one(uid=link["local_uid"]) for link in links) if row]
+
+
+def local_text() -> str:
+    names = [row["username"] for row in synced("users")]
+    posts = [f"{row['title']} {row['content']}" for row in synced("posts")]
+    comments = [row["content"] for row in synced("comments")]
+    return " ".join(names + posts + comments).lower()
+
+
+def enable(**changes):
+    row = store.get("forum-test")
+    store.update(row["uid"], sync_options=SyncOptions.from_row(row).merged(**changes).to_dict())
+
+
+def test_aliasing_off_by_default_keeps_original_names(world):
+    report = go(world)
+    assert report.status == "ok", report.errors
+    assert "writer0" in local_text()
+    assert get_table("sync_identities").count(origin="forum.test") == 0
+
+
+def test_alias_run_never_stores_an_original_username(world):
+    enable(alias=True)
+    report = go(world)
+    assert report.status == "ok", report.errors
+    text = local_text()
+    assert not [name for name in ORIGINALS if name in text]
+    assert get_table("sync_identities").count(origin="forum.test") == 4
+    again = go(world)
+    assert again.status == "ok"
+    assert again.pushed["users"].get("created", 0) == 0
+    assert get_table("sync_identities").count(origin="forum.test") == 4
+
+
+def test_kill_switch_stops_aliased_runs_and_creates_nothing(world):
+    enable(alias=True)
+    set_setting(ALIAS_KEY, "0")
+    report = go(world)
+    assert any("switched off" in error for error in report.errors)
+    assert not synced("posts") and not synced("users")
+    assert get_table("sync_identities").count(origin="forum.test") == 0
+
+
+def test_dry_run_creates_no_identities_and_no_rows(world):
+    enable(alias=True)
+    users = get_table("users").count()
+    report = go(world, dry_run=True)
+    assert report.status == "dry_run", report.errors
+    assert get_table("sync_identities").count(origin="forum.test") == 0
+    assert get_table("users").count() == users
+
+
+def test_enabling_alias_after_unaliased_sync_is_refused_at_run(world):
+    assert go(world).status == "ok"
+    enable(alias=True)
+    report = go(world)
+    assert any("already has synced records" in error for error in report.errors)
+    assert get_table("sync_identities").count(origin="forum.test") == 0
+
+
+def test_selection_skips_are_reported_and_sampling_is_stable(world):
+    enable(sample_percent=30)
+    first = go(world)
+    count = len(synced("posts"))
+    assert first.pushed["skipped"]["sampled_out"] + count >= 10
+    assert 0 < count < 10
+    go(world)
+    assert len(synced("posts")) == count
+
+
+def test_translation_failure_holds_watermark_and_stores_nothing(world):
+    enable(alias=True, translate_to="nl")
+    row = store.get("forum-test")
+    store.update(row["uid"], language="en")
+    report = go(world)
+    assert not synced("posts")
+    assert report.status == "partial"
+    assert report.pushed["skipped"]["translation_failed"] >= 1
+    assert not store.get("forum-test")["watermark"]
+
+
+def test_jitter_moves_next_run_within_bounds(world):
+    row = store.get("forum-test")
+    moments = {store.schedule_next(row["uid"], 86400, 40) for _ in range(5)}
+    assert len(moments) > 1

tests/unit/services/community/conftest.py

@@ -0,0 +1,13 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.community.ai import AiAssistant
+
+
+@pytest.fixture(autouse=True)
+def silent_ai(monkeypatch):
+    async def never(self, system, user):
+        return None
+
+    monkeypatch.setattr(AiAssistant, "ask", never)

tests/unit/services/community/discoverer.py

@@ -0,0 +1,133 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+from devplacepy.services.community.discoverer import CommunityDiscoverer
+from devplacepy.services.community.spec import STATUS_FAILED, STATUS_LOGIN, STATUS_READY
+from tests.unit.services.community.fakes import (
+    FakeFetcher,
+    club_community,
+    html_community,
+    json_community,
+    rss_community,
+    spa_community,
+)
+
+
+def discover(routes: dict, query: str, credentials: dict | None = None):
+    fetcher = FakeFetcher(routes)
+    return run_async(CommunityDiscoverer(fetcher, credentials=credentials).discover(query)), fetcher
+
+
+def test_json_api_community_is_learned():
+    result, _ = discover(json_community(), "https://forum.test/api/posts")
+    assert result.status == STATUS_READY, result.error
+    feed = result.schema["feeds"][0]
+    assert feed["type"] == "json_api"
+    assert feed["list_path"] == "$.items"
+    assert feed["post"]["fields"]["id"] == ["$.id"]
+    assert feed["post"]["fields"]["content"][0] == "$.content_html"
+    assert feed["post"]["fields"]["created_at"] == ["$.created_at"]
+    assert feed["author"]["fields"]["username"] == ["$.author.username"]
+    assert feed["author"]["fields"]["id"] == ["$.author.id"]
+    assert feed["pagination"] == {"style": "page", "param": "page", "start": 1}
+    assert feed["comments"]["mode"] == "inline"
+    assert feed["comments"]["fields"]["content"] == ["$.body"]
+    assert feed["comments"]["author"]["fields"]["username"] == ["$.user.username"]
+    records = result.sample["records"]
+    assert len(records["posts"]) == 8
+    assert len(records["comments"]) == 6
+    assert {user["username"] for user in records["users"]} >= {"writer0", "writer1", "replier0"}
+
+
+def test_rss_community_is_learned_from_advertised_feed():
+    result, _ = discover(rss_community(), "https://blog.test/")
+    assert result.status == STATUS_READY, result.error
+    feed = result.schema["feeds"][0]
+    assert feed["type"] == "rss"
+    assert feed["post"]["fields"]["created_at"] == ["$.pubDate"]
+    assert result.sample["records"]["users"][0]["username"] == "Alice"
+    assert result.sample["records"]["posts"][0]["content"].startswith("Full body of entry")
+    assert feed["comments"]["mode"] == "page"
+    assert feed["comments"]["item_selector"] == "div.comment"
+    assert feed["post"]["detail"]["content"] == ["article"]
+    comments = result.sample["records"]["comments"]
+    assert len(comments) == 9
+    assert sum(1 for comment in comments if comment.get("parent_uid")) == 3
+
+
+def test_html_list_community_is_learned():
+    result, _ = discover(html_community(), "https://board.test/")
+    assert result.status == STATUS_READY, result.error
+    feed = result.schema["feeds"][0]
+    assert feed["type"] == "html_list"
+    assert feed["item_selector"] == "article.post"
+    assert len(result.sample["records"]["posts"]) == 6
+    assert result.sample["records"]["posts"][0]["title"].startswith("Thread 1")
+    assert feed["comments"]["mode"] == "page"
+    assert feed["comments"]["fields"]["id"] == ["::attr(id)"]
+    assert feed["comments"]["author"]["fields"]["username"] == [".author"]
+    assert len(result.sample["records"]["comments"]) == 9
+
+
+def test_unreachable_community_fails_with_reason():
+    result, _ = discover({}, "https://nothing.test/")
+    assert result.status == STATUS_FAILED
+    assert result.error
+
+
+def test_javascript_only_site_is_rendered():
+    result, _ = discover(spa_community(), "https://spa.test/")
+    assert result.status == STATUS_READY, result.error
+    feed = result.schema["feeds"][0]
+    assert feed["render"] is True
+    assert len(result.sample["records"]["posts"]) == 6
+
+
+def test_login_wall_without_credentials_is_flagged_not_failed():
+    state = {}
+    result, _ = discover(club_community(state), "https://club.test/")
+    assert result.status == STATUS_LOGIN
+    assert result.schema["login"]["mode"] == "form"
+    assert result.schema["login"]["username_field"] == "user"
+    assert result.schema["login_required"] is True
+
+
+def test_login_wall_with_credentials_is_learned_behind_the_wall():
+    state = {}
+    result, fetcher = discover(club_community(state), "https://club.test/", {"username": "bob", "password": "pw"})
+    assert result.status == STATUS_READY, result.error
+    assert result.schema["login_required"] is True
+    assert result.schema["feeds"][0]["type"] == "rss"
+    assert fetcher.posts[0] == ("https://club.test/login", {"csrf": "t1", "user": "bob", "pass": "pw"})
+
+
+def test_wrong_credentials_fail_loudly():
+    result, _ = discover(club_community({}), "https://club.test/", {"username": "bob", "password": "nope"})
+    assert result.status == STATUS_FAILED
+    assert "rejected the stored credentials" in result.error
+
+
+def test_feeds_listing_the_same_posts_are_learned_once_and_comment_feeds_are_skipped():
+    routes = rss_community()
+    routes["https://blog.test/"] = (
+        '<html><head><link rel="alternate" type="application/rss+xml" href="/feed.xml">'
+        '<link rel="alternate" type="application/rss+xml" href="/comments.xml"></head>'
+        '<body><a href="/latest">Latest posts</a></body></html>'
+    )
+    routes["https://blog.test/comments.xml"] = routes["https://blog.test/feed.xml"]
+    items = "".join(
+        f'<article class="post"><h2><a href="/p/{n}">Entry {n} on testing</a></h2><span class="author">Alice</span>'
+        f"<p>Body of entry {n} with enough text to count as content.</p></article>"
+        for n in range(1, 7)
+    )
+    routes["https://blog.test/latest"] = f"<html><body><main>{items}</main></body></html>"
+    result, fetcher = discover(routes, "https://blog.test/")
+    assert result.status == STATUS_READY, result.error
+    assert [feed["type"] for feed in result.schema["feeds"]] == ["rss"]
+    assert not any("comments.xml" in request for request in fetcher.requests)
+
+
+def test_generic_probe_paths_are_skipped_when_a_feed_is_advertised():
+    _, fetcher = discover(rss_community(), "https://blog.test/")
+    assert not any(request.endswith("/api/posts") or request.endswith("/rss.xml") for request in fetcher.requests)

tests/unit/services/community/extract.py

@@ -0,0 +1,60 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+
+from devplacepy.services.community import extract
+
+
+def test_select_walks_keys_indexes_and_wildcards():
+    data = {"a": [{"b": 1}, {"b": 2}], "c": {"d": "x"}}
+    assert extract.select(data, "$.a[*].b") == [1, 2]
+    assert extract.select(data, "$.a[1].b") == [2]
+    assert extract.select(data, "$.c.d") == ["x"]
+    assert extract.select(data, "$.missing") == []
+
+
+@pytest.mark.parametrize("path", ["a.b", "$.a[", "$..", "$" + ".x" * 40])
+def test_bad_paths_raise(path):
+    with pytest.raises(ValueError):
+        extract.parse_path(path)
+
+
+def test_instants_accept_iso_rfc822_and_epochs():
+    assert extract.normalize_instant("2026-01-01T00:00:00Z") == "2026-01-01T00:00:00+00:00"
+    assert extract.normalize_instant("Tue, 14 Oct 2026 10:00:00 GMT") == "2026-10-14T10:00:00+00:00"
+    assert extract.normalize_instant(1700000000) == extract.normalize_instant(1700000000000)
+    assert extract.normalize_instant("garbage") is None
+    assert extract.normalize_instant("") is None
+
+
+def test_atom_feed_is_parsed_with_links_and_authors():
+    atom = (
+        '<feed xmlns="http://www.w3.org/2005/Atom"><title>T</title><entry><title>E</title>'
+        '<link href="http://x/1" rel="alternate"/><author><name>Ann</name></author>'
+        "<updated>2026-01-02T00:00:00Z</updated></entry></feed>"
+    )
+    kind, _, items = extract.parse_feed(atom)
+    assert kind == "atom"
+    assert extract.text_at(items[0], "$.link.@href") == "http://x/1"
+    assert extract.text_at(items[0], "$.author.name") == "Ann"
+
+
+def test_css_selectors_support_classes_ids_attributes_and_combinators():
+    root = extract.parse_html(
+        '<div id="m"><ul><li class="a b"><a href="/x" rel="next">One</a></li><li class="a">Two</li></ul></div>'
+    )
+    assert len(extract.select_nodes(root, "li.a")) == 2
+    assert len(extract.select_nodes(root, "li.a.b")) == 1
+    assert len(extract.select_nodes(root, "#m > ul > li")) == 2
+    assert len(extract.select_nodes(root, "div li")) == 2
+    assert extract.html_field(extract.select_nodes(root, "li.b")[0], "a[rel=next]::attr(href)") == "/x"
+
+
+def test_strip_tags_keeps_paragraph_breaks_and_drops_scripts():
+    assert extract.strip_tags("<p>a</p><script>x()</script><p>b<br>c</p>") == "a\n\nb\n\nc"
+    assert extract.strip_tags("plain text") == "plain text"
+
+
+def test_field_text_falls_through_alternates_and_now_token():
+    assert extract.field_text({"a": "", "b": "yes"}, ["$.a", "$.b"]) == "yes"
+    assert extract.field_text({}, [extract.NOW_TOKEN]).startswith("20")

tests/unit/services/community/fakes.py

@@ -0,0 +1,228 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+
+class FakeResponse:
+    def __init__(self, status: int, body):
+        self.status = status
+        self.body = body if isinstance(body, str) else json.dumps(body)
+        self.headers = {}
+
+    async def text(self):
+        return self.body
+
+    async def json(self):
+        return json.loads(self.body)
+
+
+class FakeFetcher:
+    def __init__(self, routes: dict):
+        self.routes = routes
+        self.requests: list[str] = []
+        self.session = None
+        self.posts: list[tuple] = []
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *exc):
+        return None
+
+    @staticmethod
+    def key(url: str, params: dict | None) -> str:
+        parts = urlsplit(url)
+        query = dict(parse_qsl(parts.query))
+        query.update({name: str(value) for name, value in (params or {}).items()})
+        return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(sorted(query.items())), ""))
+
+    async def get(self, url, params=None, timeout=None, **kwargs):
+        key = self.key(url, params)
+        self.requests.append(key)
+        route = self.routes.get(key)
+        if route is None:
+            return FakeResponse(404, "not found")
+        if callable(route):
+            route = route()
+        status, body = route if isinstance(route, tuple) else (200, route)
+        return FakeResponse(status, body)
+
+    async def post(self, url, data=None, timeout=None, **kwargs):
+        self.posts.append((url, dict(data or {})))
+        route = self.routes.get("post:" + url)
+        if callable(route):
+            route = route(dict(data or {}))
+        status, body = route if isinstance(route, tuple) else (200, route or "")
+        return FakeResponse(status, body)
+
+    async def render_html(self, url, session=None):
+        route = self.routes.get("render:" + url)
+        if route is None:
+            response = await self.get(url)
+            return response.status, response.body
+        return 200, route
+
+    def page(self, session=None):
+        return FakePage(self)
+
+    async def render(self, url, collect_links=False):
+        response = await self.get(url)
+        return {"status": response.status, "text": response.body, "links": [], "rendered": False}
+
+
+def json_post(number: int) -> dict:
+    return {
+        "id": 1000 + number,
+        "title": f"Post number {number} about asyncio",
+        "content_html": f"<p>This is the body of post {number}, long enough to sync.</p>",
+        "created_at": f"2026-10-{number:02d}T10:00:00Z",
+        "author": {"id": 7 + number % 2, "username": f"writer{number % 2}", "url": f"https://forum.test/u/{number % 2}"},
+        "url": f"https://forum.test/posts/{1000 + number}",
+        "comments": [
+            {
+                "id": 5000 + number * 10 + index,
+                "body": f"Comment {index} on post {number}",
+                "created_at": f"2026-10-{number:02d}T11:0{index}:00Z",
+                "user": {"id": 20 + index, "username": f"replier{index}"},
+            }
+            for index in range(2)
+        ],
+    }
+
+
+def json_community() -> dict:
+    posts = [json_post(number) for number in range(1, 9)]
+    return {
+        "https://forum.test/": '<html lang="en"><head><title>Forum Test</title>'
+        '<meta property="og:description" content="A friendly forum about code."></head>'
+        "<body><p>Join 1,234 members discussing code every single day on this community site.</p></body></html>",
+        "https://forum.test/api/posts": {"items": posts, "total": 8},
+        "https://forum.test/api/posts?page=2": {"items": [json_post(9), json_post(10)], "total": 8},
+        "https://forum.test/about": "<html><body><p>About: 2,500 members strong.</p></body></html>",
+    }
+
+
+RSS = """<?xml version="1.0"?>
+<rss version="2.0" xmlns:dc="http://purl.org/dc/elements/1.1/"><channel>
+<title>Blog Test</title><description>A blog</description>
+{items}
+</channel></rss>"""
+
+RSS_ITEM = """<item><title>Entry {n} on testing</title><link>https://blog.test/p/{n}</link>
+<guid>https://blog.test/p/{n}</guid><pubDate>Tue, {n:02d} Oct 2026 10:00:00 GMT</pubDate>
+<dc:creator>Alice</dc:creator><description>&lt;p&gt;Body of entry {n} with enough text.&lt;/p&gt;</description></item>"""
+
+
+def detail_page(kind: str, n: int, comments: int = 2) -> str:
+    rows = "".join(
+        f'<div class="comment" id="c{n}-{index}"><span class="author">replier{index}</span>'
+        f'<time datetime="2026-10-{n:02d}T12:0{index}:00Z">t</time>'
+        f'<div class="comment-body">Comment {index} on {kind} {n} with words</div>'
+        + (
+            f'<div class="comment" id="c{n}-{index}-r"><span class="author">deep{index}</span>'
+            f'<div class="comment-body">Nested reply to comment {index} on {n}</div></div>'
+            if index == 0
+            else ""
+        )
+        + "</div>"
+        for index in range(comments)
+    )
+    body = f"Full body of {kind} {n}. " + "It goes on at length about the topic. " * 6
+    return f"<html><body><article>{body}</article><section>{rows}</section></body></html>"
+
+
+def rss_community() -> dict:
+    items = "".join(RSS_ITEM.format(n=n) for n in range(1, 7))
+    pages = {f"https://blog.test/p/{n}": detail_page("entry", n) for n in range(1, 7)}
+    return {
+        **pages,
+        "https://blog.test/": '<html><head><link rel="alternate" type="application/rss+xml" href="/feed.xml">'
+        "<title>Blog Test</title></head><body>hi</body></html>",
+        "https://blog.test/feed.xml": RSS.format(items=items),
+    }
+
+
+HTML_ITEM = """<article class="post"><h2><a href="/t/{n}">Thread {n} on shells and terminals</a></h2>
+<span class="author">user{n}</span><time datetime="2026-10-{n:02d}T09:00:00Z">date</time>
+<p>Excerpt of thread {n} which is certainly long enough to count as content.</p></article>"""
+
+
+def html_community() -> dict:
+    items = "".join(HTML_ITEM.format(n=n) for n in range(1, 7))
+    pages = {f"https://board.test/t/{n}": detail_page("thread", n) for n in range(1, 7)}
+    return {
+        **pages,
+        "https://board.test/": f"<html><head><title>Board</title></head><body><main>{items}</main></body></html>",
+    }
+
+
+async def no_sleep(_seconds):
+    return None
+
+
+class FakePage:
+    def __init__(self, fetcher):
+        self.fetcher = fetcher
+        self.filled: dict[str, str] = {}
+        self.url = ""
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *exc):
+        return None
+
+    async def goto(self, url, **kwargs):
+        self.url = url
+        status, self.html = await self.fetcher.render_html(url)
+        return type("Resp", (), {"status": status})()
+
+    async def wait_for_timeout(self, _ms):
+        return None
+
+    async def content(self):
+        return self.html
+
+    async def fill(self, selector, value):
+        self.filled[selector] = value
+
+    async def click(self, selector):
+        self.fetcher.posts.append(("click", dict(self.filled)))
+
+
+SPA_SHELL = "<html><head><title>Spa</title></head><body><div id=app></div></body></html>"
+
+
+def spa_community() -> dict:
+    items = "".join(HTML_ITEM.format(n=n) for n in range(1, 7))
+    return {
+        "https://spa.test/": SPA_SHELL,
+        "render:https://spa.test/": f"<html><body><main>{items}</main></body></html>",
+    }
+
+
+WALL = (
+    '<html><body><form action="/login" method="post"><input type="hidden" name="csrf" value="t1">'
+    '<input type="text" name="user"><input type="password" name="pass"></form></body></html>'
+)
+CLUB_HOME = (
+    '<html><head><title>Club</title><link rel="alternate" type="application/rss+xml" href="/feed.xml"></head>'
+    "<body><p>Members only area with plenty of text on the page for the club.</p></body></html>"
+)
+
+
+def club_community(state: dict) -> dict:
+    items = "".join(RSS_ITEM.format(n=n).replace("blog.test", "club.test") for n in range(1, 7))
+    feed = RSS.format(items=items)
+
+    def login(data):
+        state["in"] = data.get("user") == "bob" and data.get("pass") == "pw" and data.get("csrf") == "t1"
+        return (200, "welcome" if state["in"] else WALL)
+
+    return {
+        "https://club.test/": lambda: CLUB_HOME if state.get("in") else WALL,
+        "https://club.test/login": WALL,
+        "https://club.test/feed.xml": lambda: feed if state.get("in") else (401, "login"),
+        "post:https://club.test/login": login,
+    }

tests/unit/services/community/generator.py

@@ -0,0 +1,168 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+import importlib.util
+import json
+import py_compile
+
+import pytest
+
+from devplacepy.services.community.discoverer import CommunityDiscoverer
+from devplacepy.services.community.generator import CodeGenerator
+from devplacepy.services.community.runtime import Collector, LoginRequired
+from devplacepy.services.sync.records import RECORD_MODELS
+from tests.unit.services.community.fakes import (
+    FakeFetcher,
+    FakeResponse,
+    html_community,
+    json_community,
+    rss_community,
+)
+
+
+async def no_sleep(_seconds):
+    return None
+
+
+def learn(routes: dict, query: str) -> dict:
+    result = run_async(CommunityDiscoverer(FakeFetcher(routes)).discover(query))
+    assert result.schema, result.error
+    return result.schema
+
+
+def load(path):
+    spec = importlib.util.spec_from_file_location("generated_collectors", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+@pytest.mark.parametrize(
+    "routes,query",
+    [
+        (json_community(), "https://forum.test/api/posts"),
+        (rss_community(), "https://blog.test/"),
+        (html_community(), "https://board.test/"),
+    ],
+)
+def test_generated_file_compiles_and_collects_valid_sync_records(tmp_path, routes, query):
+    schema = learn(routes, query)
+    target = CodeGenerator(schema).write(tmp_path)
+    py_compile.compile(str(target), doraise=True)
+    assert json.loads((tmp_path / "schema.json").read_text()) == schema
+    module = load(target)
+    result = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    assert result["ok"], result["errors"]
+    assert result["records"]["posts"]
+    for kind, rows in result["records"].items():
+        for row in rows:
+            RECORD_MODELS[kind].model_validate(row)
+    uids = {user["origin_uid"] for user in result["records"]["users"]}
+    assert {post["author_uid"] for post in result["records"]["posts"]} <= uids
+
+
+def test_generated_wrappers_exist_per_feed(tmp_path):
+    schema = learn(json_community(), "https://forum.test/api/posts")
+    module = load(CodeGenerator(schema).write(tmp_path))
+    assert callable(module.collect_feed1)
+    assert module.SCHEMA == schema
+
+
+def test_generated_output_matches_in_process_collector(tmp_path):
+    routes = json_community()
+    schema = learn(routes, "https://forum.test/api/posts")
+    module = load(CodeGenerator(schema).write(tmp_path))
+    generated = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    direct = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    assert generated["records"] == direct["records"]
+
+
+def test_incremental_run_collects_only_new_posts(tmp_path):
+    routes = json_community()
+    schema = learn(routes, "https://forum.test/api/posts")
+    first = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    again = run_async(Collector(FakeFetcher(routes), schema, first["state"], sleep=no_sleep).run())
+    assert len(first["records"]["posts"]) == 10
+    assert again["records"]["posts"] == []
+    assert again["state"]["feeds"]["feed1"]["since"] == first["state"]["feeds"]["feed1"]["since"]
+
+
+def test_incremental_run_picks_up_newer_post():
+    routes = json_community()
+    schema = learn(routes, "https://forum.test/api/posts")
+    first = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    newer = {
+        "id": 2000,
+        "title": "A brand new post",
+        "content_html": "<p>Fresh content that arrived later.</p>",
+        "created_at": "2026-11-01T10:00:00Z",
+        "author": {"id": 7, "username": "writer1"},
+        "comments": [],
+    }
+    routes["https://forum.test/api/posts"]["items"].insert(0, newer)
+    result = run_async(Collector(FakeFetcher(routes), schema, first["state"], sleep=no_sleep).run())
+    assert [post["origin_uid"] for post in result["records"]["posts"]] == ["2000"]
+
+
+def test_failed_requests_retry_with_backoff():
+    calls = {"count": 0}
+    delays = []
+
+    class Flaky:
+        async def get(self, url, params=None, timeout=None, **kwargs):
+            calls["count"] += 1
+            return FakeResponse(503 if calls["count"] < 3 else 200, "[]")
+
+    async def record(seconds):
+        delays.append(seconds)
+
+    collector = Collector(Flaky(), {"feeds": [], "limits": {"delay_seconds": 0, "backoff_seconds": 2, "retries": 3}}, sleep=record)
+    assert run_async(collector.fetch("https://x.test/")) == "[]"
+    assert calls["count"] == 3
+    assert [delay for delay in delays if delay] == [2.0, 4.0]
+
+
+def test_login_required_schema_raises_clear_error():
+    schema = {"community": "closed", "origin": "closed.test", "login_required": True, "feeds": []}
+    with pytest.raises(LoginRequired, match="requires a login"):
+        run_async(Collector(FakeFetcher({}), schema).run())
+
+
+def test_unauthorized_response_is_login_required():
+    schema = {
+        "community": "x",
+        "origin": "x.test",
+        "feeds": [{"id": "feed1", "url": "https://x.test/f", "type": "json_api", "post": {"fields": {"id": ["$.id"]}}, "author": {}}],
+        "limits": {"delay_seconds": 0, "retries": 1},
+    }
+    with pytest.raises(LoginRequired):
+        run_async(Collector(FakeFetcher({"https://x.test/f": (401, "no")}), schema, sleep=no_sleep).run())
+
+
+def test_generated_collectors_log_in_with_credentials(tmp_path):
+    from tests.unit.services.community.fakes import club_community
+
+    state = {}
+    routes = club_community(state)
+    result = run_async(CommunityDiscoverer(FakeFetcher(routes), credentials={"username": "bob", "password": "pw"}).discover("https://club.test/"))
+    module = load(CodeGenerator(result.schema).write(tmp_path))
+    state.clear()
+    with pytest.raises(module.LoginRequired, match="requires a login"):
+        run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    collected = run_async(
+        module.collect_all(FakeFetcher(routes), sleep=no_sleep, credentials={"username": "bob", "password": "pw"})
+    )
+    assert collected["ok"] and collected["records"]["posts"]
+    state.clear()
+    with pytest.raises(module.LoginRequired, match="rejected the stored credentials"):
+        run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep, credentials={"username": "bob", "password": "bad"}))
+
+
+def test_javascript_feed_is_collected_through_the_browser(tmp_path):
+    from tests.unit.services.community.fakes import spa_community
+
+    routes = spa_community()
+    schema = learn(routes, "https://spa.test/")
+    module = load(CodeGenerator(schema).write(tmp_path))
+    collected = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    assert len(collected["records"]["posts"]) == 6

tests/unit/services/community/search.py

@@ -0,0 +1,39 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+import pytest
+
+from devplacepy.services.community.search import SearchPhase, direct_url, rank
+from tests.unit.services.community.fakes import FakeFetcher
+
+
+def test_direct_addresses_skip_search():
+    assert direct_url("https://x.test/forum") == "https://x.test/forum"
+    assert direct_url("x.test") == "https://x.test"
+    assert direct_url("devplace") is None
+
+
+def test_rank_prefers_domain_match_and_drops_social_noise():
+    results = [
+        {"title": "DevPlace on LinkedIn", "url": "https://www.linkedin.com/company/devplace", "description": ""},
+        {"title": "Other", "url": "https://other.test/page", "description": "mentions devplace"},
+        {"title": "DevPlace", "url": "https://devplace.example/", "description": ""},
+    ]
+    ranked = rank("devplace", results)
+    assert [candidate.host for candidate in ranked] == ["devplace.example", "other.test"]
+
+
+def test_search_uses_rsearch_and_caps_candidates():
+    results = [{"title": f"t{n}", "url": f"https://site{n}.test/", "description": ""} for n in range(9)]
+    key = "https://rsearch.app.molodetz.nl/search?content=false&count=15&query=forum"
+    fetcher = FakeFetcher({key: {"success": True, "results": results}})
+    found = run_async(SearchPhase(fetcher).run("forum"))
+    assert len(found) == 5
+
+
+def test_search_failure_raises():
+    key = "https://rsearch.app.molodetz.nl/search?content=false&count=15&query=forum"
+    fetcher = FakeFetcher({key: (500, "boom")})
+    with pytest.raises(RuntimeError):
+        run_async(SearchPhase(fetcher).run("forum"))

tests/unit/services/community/service.py

@@ -0,0 +1,81 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+
+import pytest
+
+from devplacepy.database import get_setting, get_table, init_db, set_setting
+from devplacepy.services.community.runner import CommunityRunner
+from devplacepy.services.community.service import DRY_RUN_KEY, CommunityService
+from devplacepy.services.community.spec import STATUS_FAILING, STATUS_READY
+from devplacepy.services.community.store import store
+from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep
+
+
+@pytest.fixture
+def world():
+    init_db()
+    previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins", DRY_RUN_KEY)}
+    set_setting("sync_enabled", "1")
+    set_setting("sync_origins", "")
+    get_table("communities").delete()
+    get_table("community_runs").delete()
+    get_table("sync_links").delete(origin="forum.test")
+    routes = json_community()
+    runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep)
+    yield routes, runner
+    for key, value in previous.items():
+        set_setting(key, value)
+
+
+def seed(runner) -> dict:
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test"))
+    return store.get("forum-test")
+
+
+def test_dry_run_service_tick_learns_nothing_and_runs_nothing(world):
+    _, runner = world
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    set_setting(DRY_RUN_KEY, "1")
+    service = CommunityService(runner)
+    run_async(service.run_once())
+    assert store.get("forum-test")["status"] == "pending"
+    assert any("DRY RUN" in line for line in service.log_buffer)
+
+
+def test_live_tick_learns_pending_then_runs_due(world):
+    _, runner = world
+    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
+    set_setting(DRY_RUN_KEY, "0")
+    service = CommunityService(runner)
+    run_async(service.run_once())
+    assert store.get("forum-test")["status"] == STATUS_READY
+    run_async(service.run_once())
+    row = store.get("forum-test")
+    assert row["run_count"] == 1
+    assert get_table("sync_links").count(origin="forum.test", kind="posts") == 10
+    run_async(service.run_once())
+    assert store.get("forum-test")["run_count"] == 1
+
+
+def test_repeated_failures_disable_and_flag_the_community(world):
+    routes, runner = world
+    row = seed(runner)
+    routes["https://forum.test/api/posts"] = (500, "down")
+    failing = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep)
+    for _ in range(6):
+        current = store.get("forum-test")
+        report = run_async(failing.run(current, dry_run=False, trigger="test"))
+        assert report.status == "failed"
+    final = store.get("forum-test")
+    assert final["status"] == STATUS_FAILING and not final["enabled"]
+    assert "relearn" in "".join(store.runs(row["uid"])[0]["log"]).lower() or final["failure_count"] >= 5
+
+
+def test_run_history_is_bounded(world):
+    _, runner = world
+    row = seed(runner)
+    for _ in range(35):
+        run_async(runner.run(store.get("forum-test"), dry_run=True, trigger="test"))
+    assert len(store.runs(row["uid"], limit=100)) == 30