devplacepy/tests/unit/services/community/discussion.py

# retoor <retoor@molodetz.nl>

import copy

from tests.conftest import run_async

from devplacepy.services.community.discoverer import CommunityDiscoverer
from devplacepy.services.community.learner import HtmlInferrer
from devplacepy.services.community.runtime import MAX_POST_LENGTH, Collector
from devplacepy.services.community.extract import parse_html, select_nodes
from devplacepy.services.sync.records import RECORD_MODELS
from tests.unit.services.community.fakes import (
    FakeFetcher,
    RSS,
    detail_page,
    json_community,
    no_sleep,
)

LINKED_ITEM = """<item><title>Linked story {n} on testing</title><link>https://elsewhere.test/a/{n}</link>
<guid>https://blog.test/s/{n}</guid><pubDate>Tue, {n:02d} Oct 2026 10:00:00 GMT</pubDate>
<comments>https://blog.test/s/{n}</comments>
<dc:creator>Alice</dc:creator><description>&lt;p&gt;Body of story {n} with enough text.&lt;/p&gt;</description></item>"""


def discussion_community() -> dict:
    items = "".join(LINKED_ITEM.format(n=n) for n in range(1, 7))
    routes = {
        "https://blog.test/": '<html><head><link rel="alternate" type="application/rss+xml" href="/feed.xml">'
        "<title>Blog Test</title></head><body>hi</body></html>",
        "https://blog.test/feed.xml": RSS.format(items=items),
    }
    for n in range(1, 7):
        routes[f"https://elsewhere.test/a/{n}"] = f"<html><body><article>{'External article text. ' * 20}</article></body></html>"
        routes[f"https://blog.test/s/{n}"] = detail_page("story", n)
    return routes


def learn(routes: dict, query: str) -> dict:
    result = run_async(CommunityDiscoverer(FakeFetcher(routes)).discover(query))
    assert result.schema, result.error
    return result.schema


def test_rss_comments_are_learned_from_the_discussion_link_not_the_article():
    routes = discussion_community()
    schema = learn(routes, "https://blog.test/")
    feed = schema["feeds"][0]
    assert feed["post"]["fields"]["comments_url"] == ["$.comments"]
    assert feed["comments"]["mode"] == "page"
    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
    assert result["records"]["comments"]
    posts = {post["origin_uid"] for post in result["records"]["posts"]}
    assert {comment["post_uid"] for comment in result["records"]["comments"]} <= posts


def test_post_body_still_comes_from_the_article_page():
    routes = discussion_community()
    schema = learn(routes, "https://blog.test/")
    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
    assert all("External article text." in post["content"] for post in result["records"]["posts"])


LISTING = """<div class="story"><a class="vote" href="/login">1</a>
<a class="u-url" href="https://far.test/{n}">Story {n} about shells</a>
<a class="discuss" href="/s/{n}">{n} comments</a><span class="byline">via user{n}</span>
<time datetime="2026-10-{n:02d}T09:00:00Z">d</time><p>Excerpt of story {n} that is long enough to count.</p></div>"""


def listing_nodes() -> list:
    root = parse_html("<html><body>" + "".join(LISTING.format(n=n) for n in range(1, 9)) + "</body></html>")
    return select_nodes(root, "div.story")


def test_constant_first_anchor_is_never_chosen_as_identifier():
    fields = HtmlInferrer(listing_nodes(), "https://board.test/").post_fields()
    assert fields["id"] == ["a.u-url::attr(href)"]
    assert fields["title"] == ["a.u-url"]
    assert fields["comments_url"] == ["a.discuss::attr(href)"]


def test_no_distinct_anchor_means_no_identifier():
    page = "".join('<div class="story"><a href="/login">x</a><p>Some text that is long enough here.</p></div>' for _ in range(6))
    nodes = select_nodes(parse_html(f"<html><body>{page}</body></html>"), "div.story")
    assert "id" not in HtmlInferrer(nodes, "https://board.test/").post_fields()


def test_oversized_posts_and_titles_are_truncated_to_valid_sync_records():
    routes = json_community()
    schema = learn(routes, "https://forum.test/api/posts")
    routes["https://forum.test/api/posts"]["items"][0]["content_html"] = "<p>" + "x" * (MAX_POST_LENGTH * 2) + "</p>"
    routes["https://forum.test/api/posts"]["items"][0]["title"] = "t" * 900
    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
    for post in result["records"]["posts"]:
        assert len(post["content"]) <= MAX_POST_LENGTH
        RECORD_MODELS["posts"].model_validate(post)
    assert any(len(post["content"]) == MAX_POST_LENGTH for post in result["records"]["posts"])


def test_failed_article_fetch_is_a_warning_and_the_post_is_still_imported():
    routes = discussion_community()
    schema = learn(routes, "https://blog.test/")
    assert schema["feeds"][0]["post"].get("detail")
    routes["https://elsewhere.test/a/1"] = (403, "forbidden")
    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
    assert result["errors"] == []
    assert any("elsewhere.test/a/1" in line for line in result["warnings"])
    assert len(result["records"]["posts"]) == 6


def test_same_story_from_two_feeds_is_one_post_and_comments_follow_it():
    routes = discussion_community()
    schema = learn(routes, "https://blog.test/")
    second = copy.deepcopy(schema["feeds"][0])
    second["id"] = "feed2"
    second["post"]["fields"]["id"] = ["$.link"]
    schema["feeds"].append(second)
    result = run_async(Collector(FakeFetcher(routes), schema, sleep=no_sleep).run())
    assert len(result["records"]["posts"]) == 6
    posts = {post["origin_uid"] for post in result["records"]["posts"]}
    assert {comment["post_uid"] for comment in result["records"]["comments"]} <= posts