Zo doen wijdat in oost

tests/unit/services/community/generator.py

@@ -0,0 +1,168 @@
+# retoor <retoor@molodetz.nl>
+
+from tests.conftest import run_async
+import importlib.util
+import json
+import py_compile
+
+import pytest
+
+from devplacepy.services.community.discoverer import CommunityDiscoverer
+from devplacepy.services.community.generator import CodeGenerator
+from devplacepy.services.community.runtime import Collector, LoginRequired
+from devplacepy.services.sync.records import RECORD_MODELS
+from tests.unit.services.community.fakes import (
+    FakeFetcher,
+    FakeResponse,
+    html_community,
+    json_community,
+    rss_community,
+)
+
+
+async def no_sleep(_seconds):
+    return None
+
+
+def learn(routes: dict, query: str) -> dict:
+    result = run_async(CommunityDiscoverer(FakeFetcher(routes)).discover(query))
+    assert result.schema, result.error
+    return result.schema
+
+
+def load(path):
+    spec = importlib.util.spec_from_file_location("generated_collectors", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+@pytest.mark.parametrize(
+    "routes,query",
+    [
+        (json_community(), "https://forum.test/api/posts"),
+        (rss_community(), "https://blog.test/"),
+        (html_community(), "https://board.test/"),
+    ],
+)
+def test_generated_file_compiles_and_collects_valid_sync_records(tmp_path, routes, query):
+    schema = learn(routes, query)
+    target = CodeGenerator(schema).write(tmp_path)
+    py_compile.compile(str(target), doraise=True)
+    assert json.loads((tmp_path / "schema.json").read_text()) == schema
+    module = load(target)
+    result = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    assert result["ok"], result["errors"]
+    assert result["records"]["posts"]
+    for kind, rows in result["records"].items():
+        for row in rows:
+            RECORD_MODELS[kind].model_validate(row)
+    uids = {user["origin_uid"] for user in result["records"]["users"]}
+    assert {post["author_uid"] for post in result["records"]["posts"]} <= uids
+
+
+def test_generated_wrappers_exist_per_feed(tmp_path):
+    schema = learn(json_community(), "https://forum.test/api/posts")
+    module = load(CodeGenerator(schema).write(tmp_path))
+    assert callable(module.collect_feed1)
+    assert module.SCHEMA == schema
+
+
+def test_generated_output_matches_in_process_collector(tmp_path):
+    routes = json_community()
+    schema = learn(routes, "https://forum.test/api/posts")
+    module = load(CodeGenerator(schema).write(tmp_path))
+    generated = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    direct = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    assert generated["records"] == direct["records"]
+
+
+def test_incremental_run_collects_only_new_posts(tmp_path):
+    routes = json_community()
+    schema = learn(routes, "https://forum.test/api/posts")
+    first = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    again = run_async(Collector(FakeFetcher(routes), schema, first["state"], sleep=no_sleep).run())
+    assert len(first["records"]["posts"]) == 10
+    assert again["records"]["posts"] == []
+    assert again["state"]["feeds"]["feed1"]["since"] == first["state"]["feeds"]["feed1"]["since"]
+
+
+def test_incremental_run_picks_up_newer_post():
+    routes = json_community()
+    schema = learn(routes, "https://forum.test/api/posts")
+    first = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
+    newer = {
+        "id": 2000,
+        "title": "A brand new post",
+        "content_html": "<p>Fresh content that arrived later.</p>",
+        "created_at": "2026-11-01T10:00:00Z",
+        "author": {"id": 7, "username": "writer1"},
+        "comments": [],
+    }
+    routes["https://forum.test/api/posts"]["items"].insert(0, newer)
+    result = run_async(Collector(FakeFetcher(routes), schema, first["state"], sleep=no_sleep).run())
+    assert [post["origin_uid"] for post in result["records"]["posts"]] == ["2000"]
+
+
+def test_failed_requests_retry_with_backoff():
+    calls = {"count": 0}
+    delays = []
+
+    class Flaky:
+        async def get(self, url, params=None, timeout=None, **kwargs):
+            calls["count"] += 1
+            return FakeResponse(503 if calls["count"] < 3 else 200, "[]")
+
+    async def record(seconds):
+        delays.append(seconds)
+
+    collector = Collector(Flaky(), {"feeds": [], "limits": {"delay_seconds": 0, "backoff_seconds": 2, "retries": 3}}, sleep=record)
+    assert run_async(collector.fetch("https://x.test/")) == "[]"
+    assert calls["count"] == 3
+    assert [delay for delay in delays if delay] == [2.0, 4.0]
+
+
+def test_login_required_schema_raises_clear_error():
+    schema = {"community": "closed", "origin": "closed.test", "login_required": True, "feeds": []}
+    with pytest.raises(LoginRequired, match="requires a login"):
+        run_async(Collector(FakeFetcher({}), schema).run())
+
+
+def test_unauthorized_response_is_login_required():
+    schema = {
+        "community": "x",
+        "origin": "x.test",
+        "feeds": [{"id": "feed1", "url": "https://x.test/f", "type": "json_api", "post": {"fields": {"id": ["$.id"]}}, "author": {}}],
+        "limits": {"delay_seconds": 0, "retries": 1},
+    }
+    with pytest.raises(LoginRequired):
+        run_async(Collector(FakeFetcher({"https://x.test/f": (401, "no")}), schema, sleep=no_sleep).run())
+
+
+def test_generated_collectors_log_in_with_credentials(tmp_path):
+    from tests.unit.services.community.fakes import club_community
+
+    state = {}
+    routes = club_community(state)
+    result = run_async(CommunityDiscoverer(FakeFetcher(routes), credentials={"username": "bob", "password": "pw"}).discover("https://club.test/"))
+    module = load(CodeGenerator(result.schema).write(tmp_path))
+    state.clear()
+    with pytest.raises(module.LoginRequired, match="requires a login"):
+        run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    collected = run_async(
+        module.collect_all(FakeFetcher(routes), sleep=no_sleep, credentials={"username": "bob", "password": "pw"})
+    )
+    assert collected["ok"] and collected["records"]["posts"]
+    state.clear()
+    with pytest.raises(module.LoginRequired, match="rejected the stored credentials"):
+        run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep, credentials={"username": "bob", "password": "bad"}))
+
+
+def test_javascript_feed_is_collected_through_the_browser(tmp_path):
+    from tests.unit.services.community.fakes import spa_community
+
+    routes = spa_community()
+    schema = learn(routes, "https://spa.test/")
+    module = load(CodeGenerator(schema).write(tmp_path))
+    collected = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
+    assert len(collected["records"]["posts"]) == 6