devplacepy/tests/unit/services/community/engagement.py

# retoor <retoor@molodetz.nl>

import base64

import pytest

from devplacepy.database import get_setting, get_table, init_db, set_setting
from devplacepy.services.community.discoverer import CommunityDiscoverer
from devplacepy.services.community.generator import CodeGenerator
from devplacepy.services.community.learner import learn_engagement
from devplacepy.services.community.runner import CommunityRunner
from devplacepy.services.community.runtime import Collector
from devplacepy.services.community.store import store
from devplacepy.services.sync.options import SyncOptions
from devplacepy.services.sync.records import RECORD_MODELS
from tests.conftest import run_async
from tests.unit.services.community.fakes import FakeBinary, FakeFetcher
from tests.unit.services.community.generator import load, no_sleep

PNG = base64.b64decode(
    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
)


def person(number):
    return {"id": 500 + number, "username": f"fan{number}"}


def post(number, voters=3, with_engagement=True):
    item = {
        "id": 9000 + number,
        "title": f"Engaged post number {number}",
        "content_html": f'<p>Body of engaged post {number}, long enough to sync.</p><img src="/img/p{number}.png"><a href="https://cdn.test/doc{number}.pdf">doc</a><a href="/page">page</a>',
        "created_at": f"2026-10-{number:02d}T10:00:00Z",
        "author": {"id": 77, "username": "writer"},
        "comments": [
            {"id": 7000 + number, "body": f"A comment on {number}", "created_at": f"2026-10-{number:02d}T11:00:00Z", "user": {"id": 88, "username": "replier"},
             "votes": [{"user": person(1), "value": 1}, {"user": person(2), "value": -1}] if with_engagement else []},
        ],
    }
    if with_engagement:
        item["votes"] = [{"user": person(index), "value": -1 if index == 3 else 1, "created_at": f"2026-10-{number:02d}T12:00:00Z"} for index in range(1, voters + 1)]
        item["reactions"] = [{"user": person(1), "emoji": "\U0001f525"}, {"user": person(2), "emoji": "\U0001f44d"}, {"user": person(3), "emoji": "nope"}]
    return item


def routes(count=4, **kwargs):
    return {
        "https://eng.test/": '<html lang="en"><head><title>Eng</title></head><body><p>Join 100 members discussing code every day on this site.</p></body></html>',
        "https://eng.test/api/posts": {"items": [post(number, **kwargs) for number in range(1, count + 1)], "total": count},
        "https://eng.test/about": "<html><body><p>About: 100 members.</p></body></html>",
        **{f"https://eng.test/img/p{number}.png": FakeBinary(PNG) for number in range(1, count + 1)},
        **{f"https://cdn.test/doc{number}.pdf": FakeBinary(b"%PDF-1.4 " + b"x" * 40, "application/pdf") for number in range(1, count + 1)},
        "https://eng.test/page": "<html><body>not a file</body></html>",
    }


def learned(source=None):
    result = run_async(CommunityDiscoverer(FakeFetcher(source or routes())).discover("https://eng.test/api/posts"))
    assert result.status == "ready", result.error
    return result.schema


def test_the_learner_finds_per_voter_lists_and_names_the_emoji_field():
    schema = learned()
    post_spec = schema["feeds"][0]["post"]
    assert post_spec["votes"]["list_path"] == "$.votes" and post_spec["votes"]["author"]["fields"]["username"] == ["$.user.username"]
    assert post_spec["votes"]["fields"]["value"] == ["$.value"]
    assert post_spec["reactions"]["fields"]["emoji"] == ["$.emoji"]
    assert schema["feeds"][0]["comments"]["votes"]["list_path"] == "$.votes"


def test_aggregate_counts_alone_learn_nothing():
    samples = [{"id": number, "votes": number * 3, "likes": 5, "reactions": {"fire": 2}} for number in range(1, 5)]
    assert learn_engagement(samples) == {}
    assert learn_engagement([{"id": 1, "votes": [{"user": {"username": "a"}}]}]) == {}
    assert learn_engagement([{"id": number, "votes": [{"score": 1}, {"score": 2}]} for number in range(3)]) == {}


def test_the_collector_turns_voter_lists_into_records_with_users():
    schema = learned()
    result = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep).run())
    records = result["records"]
    assert len(records["votes"]) == 4 * 3 + 4 * 2 and len(records["reactions"]) == 4 * 2
    assert {item["value"] for item in records["votes"]} == {1, -1}
    assert {item["emoji"] for item in records["reactions"]} == {"\U0001f525", "\U0001f44d"}
    voters = {item["voter_uid"] for item in records["votes"]}
    assert voters <= {user["origin_uid"] for user in records["users"]}
    for kind in ("votes", "reactions"):
        for row in records[kind]:
            RECORD_MODELS[kind].model_validate(row)
    assert len({item["origin_uid"] for item in records["votes"]}) == len(records["votes"])
    assert {item["target_kind"] for item in records["votes"]} == {"post", "comment"}


def test_attachments_are_off_by_default_and_bounded_when_on():
    schema = learned()
    off = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep).run())
    assert off["records"]["attachments"] == []
    on = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True}).run())
    files = on["records"]["attachments"]
    assert len(files) == 8 and {item["filename"].rsplit(".", 1)[-1] for item in files} == {"png", "pdf"}
    assert base64.b64decode(next(item for item in files if item["filename"] == "p1.png")["content_b64"]) == PNG
    for row in files:
        RECORD_MODELS["attachments"].model_validate(row)
    capped = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True, "max_attachments": 3}).run())
    assert len(capped["records"]["attachments"]) == 3
    one_each = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True, "max_attachments_per_post": 1}).run())
    assert len(one_each["records"]["attachments"]) == 4


def test_missing_oversize_and_wrong_type_files_become_notices_not_failures():
    source = routes(count=2)
    source["https://eng.test/img/p1.png"] = (404, "gone")
    source["https://cdn.test/doc1.pdf"] = FakeBinary(b"x" * 500, "application/pdf")
    source["https://eng.test/img/p2.png"] = FakeBinary(b"<html>", "text/html")
    schema = learned()
    result = run_async(Collector(FakeFetcher(source), schema, sleep=no_sleep, options={"attachments": True, "max_attachment_bytes": 100}).run())
    reasons = sorted(notice["reason"] for notice in result["notices"])
    assert reasons.count("attachment_unreachable") == 2 and reasons.count("attachment_too_large") == 1
    assert result["ok"] and [item["filename"] for item in result["records"]["attachments"]] == ["doc2.pdf"]


def test_a_generated_collector_collects_the_same_engagement(tmp_path):
    schema = learned()
    module = load(CodeGenerator(schema).write(tmp_path))
    generated = run_async(module.collect_all(FakeFetcher(routes()), sleep=no_sleep, options={"attachments": True}))
    direct = run_async(Collector(FakeFetcher(routes()), schema, sleep=no_sleep, options={"attachments": True}).run())
    for kind in ("votes", "reactions", "attachments"):
        assert len(generated["records"][kind]) == len(direct["records"][kind]) > 0


def test_an_old_generated_file_is_regenerated_keeping_its_hand_edited_schema(tmp_path):
    from devplacepy.services.community.files import CommunityFiles
    from devplacepy.services.community.generator import CodeGenerator

    schema = learned()
    files = CommunityFiles(tmp_path)
    path = files.write("old-site", schema)
    edited = path.read_text().replace(f"# generator-version: {CodeGenerator.VERSION}", "# generator-version: 1").replace('"max_posts": 150', '"max_posts": 7')
    path.write_text(edited)
    module = files.load("old-site", schema)
    assert module.SCHEMA["limits"]["max_posts"] == 7
    assert f"# generator-version: {CodeGenerator.VERSION}" in path.read_text()


@pytest.fixture
def world():
    init_db()
    previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins", "community_alias_enabled")}
    set_setting("sync_enabled", "1")
    set_setting("sync_origins", "")
    for table in ("communities", "community_runs"):
        get_table(table).delete()
    get_table("sync_links").delete(origin="eng.test")
    get_table("sync_identities").delete(origin="eng.test")
    yield
    for key, value in previous.items():
        set_setting(key, value)


def make_row(**options):
    runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes()), sleep=no_sleep)
    store.create("eng-test", "https://eng.test/api/posts", "tester", name="eng")
    run_async(runner.discover("https://eng.test/api/posts", "tester", relearn_slug="eng-test"))
    row = store.get("eng-test")
    if options:
        store.update(row["uid"], sync_options=SyncOptions.build(**options).to_dict())
    return runner, store.get("eng-test")


def test_a_live_run_syncs_votes_reactions_and_attachments(world):
    runner, row = make_row(collect_attachments=True)
    report = run_async(runner.run(row, dry_run=False, trigger="test"))
    assert report.status in ("ok", "partial"), report.errors
    assert report.pushed["votes"].get("created") == 20 and report.pushed["reactions"].get("created") == 8
    assert report.pushed["attachments"].get("created") == 8
    post_link = get_table("sync_links").find_one(origin="eng.test", kind="posts")
    stars = get_table("posts").find_one(uid=post_link["local_uid"])["stars"]
    assert stars == 1
    again = run_async(runner.run(store.get("eng-test"), dry_run=False, trigger="test"))
    assert again.pushed.get("votes", {}).get("created", 0) == 0


def test_without_the_option_no_file_is_downloaded_or_pushed(world):
    runner, row = make_row()
    report = run_async(runner.run(row, dry_run=False, trigger="test"))
    assert report.pushed["attachments"] == {}


def test_voters_are_never_stored_under_pseudonyms_so_votes_wait_while_aliasing_is_on(world):
    set_setting("community_alias_enabled", "1")
    runner, row = make_row(alias=True, collect_attachments=True)
    report = run_async(runner.run(row, dry_run=False, trigger="test"))
    assert report.pushed["votes"] == {} and report.pushed["reactions"] == {}
    assert report.pushed["skipped"]["alias_engagement"] == 20 + 8
    assert report.pushed["attachments"].get("created") == 8
    owners = {get_table("attachments").find_one(uid=link["local_uid"])["user_uid"] for link in get_table("sync_links").find(origin="eng.test", kind="attachments")}
    assert owners and all(get_table("users").find_one(uid=uid)["username"] != "writer" for uid in owners)