devplacepy/tests/unit/services/community/pagination.py

# retoor <retoor@molodetz.nl>

import pytest

from devplacepy.services.community.extract import (
    find_next_page,
    page_number,
    parse_html,
    with_page_number,
)
from devplacepy.services.community.generator import CodeGenerator
from devplacepy.services.community.runtime import Collector
from tests.conftest import run_async
from tests.unit.services.community.fakes import FakeFetcher, detail_page, html_community
from tests.unit.services.community.generator import learn, load, no_sleep, post_comment_count

BASE = "https://board.test/t/1"


def page(markup: str):
    return parse_html(f"<html><body><article>text</article>{markup}</body></html>")


@pytest.mark.parametrize(
    "markup,expected",
    [
        ('<a rel="next" href="/t/1?p=2">x</a>', f"{BASE}?p=2"),
        ('<link rel="next" href="/t/1?p=2">', f"{BASE}?p=2"),
        ('<ul class="pagination"><li class="next"><a href="/t/1?p=2">go</a></li></ul>', f"{BASE}?p=2"),
        ('<nav><a href="/t/1?p=2">Next</a></nav>', f"{BASE}?p=2"),
        ('<div class="pager"><a href="/t/1?p=2">›</a></div>', f"{BASE}?p=2"),
        ('<div class="pagination"><a href="/t/1?p=2">Volgende pagina</a></div>', f"{BASE}?p=2"),
        ('<div class="pagination"><a href="/t/1">1</a><a href="/t/1?page=2">2</a><a href="/t/1?page=3">3</a></div>', f"{BASE}?page=2"),
        ('<a aria-label="Next page" href="/t/1?p=2">arrow</a>', f"{BASE}?p=2"),
    ],
)
def test_next_page_is_found_in_every_common_markup(markup, expected):
    assert find_next_page(page(markup), BASE) == expected


@pytest.mark.parametrize(
    "markup",
    [
        '<p><a href="/other">More</a> from the author</p>',
        '<a href="javascript:void(0)" class="next">next</a>',
        '<a href="#">Next</a>',
        '<nav><a href="/t/1?p=1">1</a></nav>',
        '<a href="/t/2?page=2">elsewhere 2</a>',
        "",
    ],
)
def test_no_next_page_is_invented(markup):
    assert find_next_page(page(markup), BASE) == ""


def test_hints_override_the_heuristics():
    root = page('<a rel="next" href="/t/1?p=2">x</a><div class="x"><a class="more" href="/t/1?custom=2">c</a></div>')
    assert find_next_page(root, BASE, {"style": "none"}) == ""
    assert find_next_page(root, BASE, {"style": "next_link", "next": ".more::attr(href)"}) == f"{BASE}?custom=2"
    assert find_next_page(root, BASE, {"style": "next_link", "next": ".absent::attr(href)"}) == ""
    assert find_next_page(root, f"{BASE}?page=3", {"style": "page_param", "param": "page"}) == f"{BASE}?page=4"
    assert find_next_page(root, BASE, {"style": "page_param"}) == f"{BASE}?page=2"


def test_page_numbers_round_trip():
    assert page_number("https://x.test/t?page=4") == 4
    assert page_number("https://x.test/t/page/7") == 7
    assert page_number("https://x.test/t?x=1&p=12") == 12
    assert page_number("https://x.test/t") is None
    assert with_page_number("https://x.test/t?a=1&page=2", 3) == "https://x.test/t?a=1&page=3"
    assert with_page_number("https://x.test/t/page/2", 3) == "https://x.test/t/page/3"
    assert with_page_number("https://x.test/t", 2, "p") == "https://x.test/t?p=2"


def board(next_markup: str, second_next: str = "") -> dict:
    routes = html_community()
    routes[BASE] = detail_page("thread", 1).replace("</body>", f"{next_markup}</body>")
    routes[f"{BASE}?cp=2"] = detail_page("thread", 1, comments=4).replace("</body>", f"{second_next}</body>")
    return routes


@pytest.mark.parametrize(
    "markup",
    [
        '<nav class="pagination"><a href="/t/1?cp=2">Next</a></nav>',
        '<div class="pager"><a href="/t/1?cp=2">»</a></div>',
        '<div class="pagination"><a href="/t/1">1</a><a href="/t/1?cp=2">2</a></div>',
    ],
)
def test_collector_and_generated_file_follow_every_style(markup, tmp_path):
    routes = board(markup)
    schema = learn(html_community(), "https://board.test/")
    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
    first = next(post["origin_uid"] for post in result["records"]["posts"] if post["title"].startswith("Thread 1 "))
    assert post_comment_count(result, first) == 5
    module = load(CodeGenerator(schema).write(tmp_path))
    generated = run_async(module.collect_all(FakeFetcher(routes), sleep=no_sleep))
    assert len(generated["records"]["comments"]) == len(result["records"]["comments"])


def test_a_page_that_adds_no_new_comment_ends_the_walk():
    routes = board('<nav class="pagination"><a href="/t/1?cp=2">Next</a></nav>', '<nav class="pagination"><a href="/t/1?cp=3">Next</a></nav>')
    routes[f"{BASE}?cp=3"] = routes[f"{BASE}?cp=2"].replace("cp=3", "cp=4")
    routes[f"{BASE}?cp=4"] = routes[f"{BASE}?cp=2"]
    schema = learn(html_community(), "https://board.test/")
    fetcher = FakeFetcher(routes)
    run_async(Collector(fetcher, schema, sleep=no_sleep).run())
    assert f"{BASE}?cp=3" in fetcher.requests and f"{BASE}?cp=4" not in fetcher.requests


def test_a_declared_no_pagination_hint_stops_after_the_first_page():
    routes = board('<a rel="next" href="/t/1?cp=2">more</a>')
    schema = learn(html_community(), "https://board.test/")
    for feed in schema["feeds"]:
        if (feed.get("comments") or {}).get("mode") == "page":
            feed["comments"]["pagination"] = {"style": "none"}
    fetcher = FakeFetcher(routes)
    run_async(Collector(fetcher, schema, sleep=no_sleep).run())
    assert f"{BASE}?cp=2" not in fetcher.requests


def test_the_learner_records_a_selector_hint_when_one_matches():
    schema = learn(board('<a rel="next" href="/t/1?cp=2">more</a>'), "https://board.test/")
    modes = [feed["comments"] for feed in schema["feeds"] if (feed.get("comments") or {}).get("mode") == "page"]
    assert modes and modes[0].get("pagination") == {"style": "next_link", "next": "a[rel=next]::attr(href)"}


def json_comment_schema(pagination):
    comments = {
        "mode": "url",
        "url_template": "https://api.test/posts/{id}/comments",
        "list_path": "$.items",
        "fields": {"id": ["$.id"], "content": ["$.body"], "created_at": ["$.at"]},
        "author": {"fields": {"username": ["$.who"]}},
    }
    if pagination:
        comments["pagination"] = pagination
    return {
        "community": "api",
        "origin": "api.test",
        "limits": {"delay_seconds": 0, "retries": 1, "max_comment_pages": 3},
        "feeds": [
            {
                "id": "feed1",
                "url": "https://api.test/posts",
                "type": "json_api",
                "list_path": "$.items",
                "post": {"fields": {"id": ["$.id"], "title": ["$.title"], "content": ["$.body"], "created_at": ["$.at"]}},
                "author": {"fields": {"username": ["$.who"]}},
                "comments": comments,
            }
        ],
    }


def json_routes():
    entry = lambda n: {"id": f"c{n}", "body": f"comment number {n}", "at": "2026-10-01T10:00:00Z", "who": "alice"}
    return {
        "https://api.test/posts": {"items": [{"id": "p1", "title": "A post", "body": "a post body long enough", "at": "2026-10-01T10:00:00Z", "who": "alice"}]},
        "https://api.test/posts/p1/comments": {"items": [entry(1), entry(2)], "next": "/posts/p1/comments?page=2"},
        "https://api.test/posts/p1/comments?page=2": {"items": [entry(3)], "next": "/posts/p1/comments?page=3"},
        "https://api.test/posts/p1/comments?page=3": {"items": [entry(3)], "next": "/posts/p1/comments?page=4"},
    }


def test_json_comments_follow_a_declared_next_key_and_stop_without_news():
    fetcher = FakeFetcher(json_routes())
    result = run_async(Collector(fetcher, json_comment_schema({"style": "json_next", "path": "$.next"}), sleep=no_sleep).run())
    assert sorted(comment["content"] for comment in result["records"]["comments"]) == ["comment number 1", "comment number 2", "comment number 3"]
    assert "https://api.test/posts/p1/comments?page=4" not in fetcher.requests


def test_json_comments_without_a_hint_read_only_the_first_page():
    fetcher = FakeFetcher(json_routes())
    result = run_async(Collector(fetcher, json_comment_schema(None), sleep=no_sleep).run())
    assert len(result["records"]["comments"]) == 2