devplacepy/tests/unit/services/community/generator.py

# retoor <retoor@molodetz.nl>

from tests.conftest import run_async
import importlib.util
import json
import py_compile

import pytest

from devplacepy.services.community.discoverer import CommunityDiscoverer
from devplacepy.services.community.generator import CodeGenerator
from devplacepy.services.community.runtime import Collector, LoginRequired
from devplacepy.services.sync.records import RECORD_MODELS
from tests.unit.services.community.fakes import (
    FakeFetcher,
    FakeResponse,
    html_community,
    json_community,
    rss_community,
)


async def no_sleep(_seconds):
    return None


def learn(routes: dict, query: str) -> dict:
    result = run_async(CommunityDiscoverer(FakeFetcher(routes)).discover(query))
    assert result.schema, result.error
    return result.schema


def load(path):
    spec = importlib.util.spec_from_file_location("generated_collectors", path)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


@pytest.mark.parametrize(
    "routes,query",
    [
        (json_community(), "https://forum.test/api/posts"),
        (rss_community(), "https://blog.test/"),
        (html_community(), "https://board.test/"),
    ],
)
def test_generated_file_compiles_and_collects_valid_sync_records(tmp_path, routes, query):
    schema = learn(routes, query)
    target = CodeGenerator(schema).write(tmp_path)
    py_compile.compile(str(target), doraise=True)
    assert json.loads((tmp_path / "schema.json").read_text()) == schema
    module = load(target)
    result = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
    assert result["ok"], result["errors"]
    assert result["records"]["posts"]
    for kind, rows in result["records"].items():
        for row in rows:
            RECORD_MODELS[kind].model_validate(row)
    uids = {user["origin_uid"] for user in result["records"]["users"]}
    assert {post["author_uid"] for post in result["records"]["posts"]} <= uids


def test_generated_wrappers_exist_per_feed(tmp_path):
    schema = learn(json_community(), "https://forum.test/api/posts")
    module = load(CodeGenerator(schema).write(tmp_path))
    assert callable(module.collect_feed1)
    assert module.SCHEMA == schema


def test_generated_output_matches_in_process_collector(tmp_path):
    routes = json_community()
    schema = learn(routes, "https://forum.test/api/posts")
    module = load(CodeGenerator(schema).write(tmp_path))
    generated = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
    direct = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
    assert generated["records"] == direct["records"]


def test_incremental_run_collects_only_new_posts(tmp_path):
    routes = json_community()
    schema = learn(routes, "https://forum.test/api/posts")
    first = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
    again = run_async(Collector(FakeFetcher(routes), schema, first["state"], sleep=no_sleep).run())
    assert len(first["records"]["posts"]) == 10
    assert again["records"]["posts"] == []
    assert again["state"]["feeds"]["feed1"]["since"] == first["state"]["feeds"]["feed1"]["since"]


def test_incremental_run_picks_up_newer_post():
    routes = json_community()
    schema = learn(routes, "https://forum.test/api/posts")
    first = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
    newer = {
        "id": 2000,
        "title": "A brand new post",
        "content_html": "<p>Fresh content that arrived later.</p>",
        "created_at": "2026-11-01T10:00:00Z",
        "author": {"id": 7, "username": "writer1"},
        "comments": [],
    }
    routes["https://forum.test/api/posts"]["items"].insert(0, newer)
    result = run_async(Collector(FakeFetcher(routes), schema, first["state"], sleep=no_sleep).run())
    assert [post["origin_uid"] for post in result["records"]["posts"]] == ["2000"]


def test_failed_requests_retry_with_backoff():
    calls = {"count": 0}
    delays = []

    class Flaky:
        async def get(self, url, params=None, timeout=None, **kwargs):
            calls["count"] += 1
            return FakeResponse(503 if calls["count"] < 3 else 200, "[]")

    async def record(seconds):
        delays.append(seconds)

    collector = Collector(Flaky(), {"feeds": [], "limits": {"delay_seconds": 0, "backoff_seconds": 2, "retries": 3}}, sleep=record)
    assert run_async(collector.fetch("https://x.test/")) == "[]"
    assert calls["count"] == 3
    assert [delay for delay in delays if delay] == [2.0, 4.0]


def test_login_required_schema_raises_clear_error():
    schema = {"community": "closed", "origin": "closed.test", "login_required": True, "feeds": []}
    with pytest.raises(LoginRequired, match="requires a login"):
        run_async(Collector(FakeFetcher({}), schema).run())


def test_unauthorized_response_is_login_required():
    schema = {
        "community": "x",
        "origin": "x.test",
        "feeds": [{"id": "feed1", "url": "https://x.test/f", "type": "json_api", "post": {"fields": {"id": ["$.id"]}}, "author": {}}],
        "limits": {"delay_seconds": 0, "retries": 1},
    }
    with pytest.raises(LoginRequired):
        run_async(Collector(FakeFetcher({"https://x.test/f": (401, "no")}), schema, sleep=no_sleep).run())


def test_generated_collectors_log_in_with_credentials(tmp_path):
    from tests.unit.services.community.fakes import club_community

    state = {}
    routes = club_community(state)
    result = run_async(CommunityDiscoverer(FakeFetcher(routes), credentials={"username": "bob", "password": "pw"}).discover("https://club.test/"))
    module = load(CodeGenerator(result.schema).write(tmp_path))
    state.clear()
    with pytest.raises(module.LoginRequired, match="requires a login"):
        run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
    collected = run_async(
        module.collect_all(FakeFetcher(routes), sleep=no_sleep, credentials={"username": "bob", "password": "pw"})
    )
    assert collected["ok"] and collected["records"]["posts"]
    state.clear()
    with pytest.raises(module.LoginRequired, match="rejected the stored credentials"):
        run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep, credentials={"username": "bob", "password": "bad"}))


def test_javascript_feed_is_collected_through_the_browser(tmp_path):
    from tests.unit.services.community.fakes import spa_community

    routes = spa_community()
    schema = learn(routes, "https://spa.test/")
    module = load(CodeGenerator(schema).write(tmp_path))
    collected = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
    assert len(collected["records"]["posts"]) == 6


def post_comment_count(result: dict, post_origin_uid: str) -> int:
    return sum(1 for comment in result["records"]["comments"] if comment["post_uid"] == post_origin_uid)


def paginated_board(next_href: str) -> dict:
    from tests.unit.services.community.fakes import detail_page

    routes = html_community()
    first = detail_page("thread", 1).replace("</body>", f'<a rel="next" href="{next_href}">more</a></body>')
    routes["https://board.test/t/1"] = first
    routes["https://board.test/t/1?cp=2"] = detail_page("thread", 1, comments=4)
    return routes


def test_html_comments_follow_the_next_page_link():
    routes = paginated_board("/t/1?cp=2")
    schema = learn(html_community(), "https://board.test/")
    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
    first_post = next(post["origin_uid"] for post in result["records"]["posts"] if post["title"].startswith("Thread 1 "))
    assert post_comment_count(result, first_post) == 5


def test_html_comment_pagination_stops_on_a_cycle_and_at_the_page_cap():
    routes = paginated_board("/t/1")
    schema = learn(html_community(), "https://board.test/")
    fetcher = FakeFetcher(routes)
    run_async(Collector(fetcher, schema, sleep=no_sleep).run())
    assert fetcher.requests.count("https://board.test/t/1") == 1
    routes = paginated_board("/t/1?cp=2")
    routes["https://board.test/t/1?cp=2"] = routes["https://board.test/t/1?cp=2"].replace("</body>", '<a rel="next" href="/t/1?cp=3">more</a></body>')
    routes["https://board.test/t/1?cp=3"] = routes["https://board.test/t/1?cp=2"].replace("cp=3", "cp=4")
    routes["https://board.test/t/1?cp=4"] = routes["https://board.test/t/1?cp=2"]
    fetcher = FakeFetcher(routes)
    schema["limits"] = {**schema.get("limits", {}), "max_comment_pages": 3}
    run_async(Collector(fetcher, schema, sleep=no_sleep).run())
    assert "https://board.test/t/1?cp=4" not in fetcher.requests