devplacepy/tests/unit/services/community/ledger.py

# retoor <retoor@molodetz.nl>

import json

import pytest

from tests.conftest import run_async

from devplacepy.database import db, get_setting, get_table, init_db, set_setting
from devplacepy.services.community.ledger import (
    LEDGER,
    Ledger,
    OUTCOME_KEYS,
    REASON_TEXT,
    explain,
    label_for,
    tally_of,
    verdict,
)
from devplacepy.services.community.ledger_store import TABLE, ledger_store
from devplacepy.services.community.pusher import SyncPusher
from devplacepy.services.community.runner import CommunityRunner
from devplacepy.services.community.store import store
from devplacepy.services.sync.options import SyncOptions
from devplacepy.services.sync.selection import Selector
from devplacepy.services.sync.spec import REASONS, SKIP_REASONS
from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep

ORIGIN = "ledger-unit.test"


@pytest.fixture
def world():
    init_db()
    previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins")}
    set_setting("sync_enabled", "1")
    set_setting("sync_origins", "")
    get_table("communities").delete()
    get_table("community_runs").delete()
    get_table(TABLE).delete()
    get_table("sync_links").delete(origin=ORIGIN)
    get_table("sync_links").delete(origin="forum.test")
    yield
    for key, value in previous.items():
        set_setting(key, value)


def records() -> dict:
    return {
        "users": [{"origin_uid": "u1", "username": "ledgerone", "created_at": "2026-10-01T10:00:00Z"}],
        "posts": [
            {"origin_uid": "p1", "author_uid": "u1", "title": "A good post", "content": "Body of a good post, long enough.", "created_at": "2026-10-01T10:00:00Z"},
            {"origin_uid": "p2", "author_uid": "ghost", "title": "Orphan post", "content": "Written by someone who was never sent.", "created_at": "2026-10-01T10:00:00Z"},
            {"origin_uid": "p3", "title": "Gone", "deleted": True, "author_uid": "u1", "content": "Deleted at the origin before it ever arrived.", "created_at": "2026-10-01T10:00:00Z"},
        ],
        "comments": [],
    }


def test_pusher_records_created_rejected_and_held_records_with_reasons(world):
    ledger = Ledger()
    report = SyncPusher(ORIGIN, ledger=ledger).push(records())
    by_origin = {item.origin_uid: item for item in ledger.items}
    assert by_origin["u1"].outcome == "created" and by_origin["u1"].label == "ledgerone" and by_origin["u1"].local_uid
    assert by_origin["p1"].outcome == "created" and by_origin["p1"].label == "A good post"
    assert by_origin["p2"].outcome == "rejected" and by_origin["p2"].reason == "unknown_author" and by_origin["p2"].label == "Orphan post"
    assert by_origin["p3"].outcome == "unchanged" and by_origin["p3"].reason == "tombstone_for_unknown"
    assert tally_of(report.counts, ledger) == {"created": 2, "unchanged": 1, "rejected": 1}


def test_repeating_the_same_batch_lists_nothing_that_is_merely_unchanged(world):
    SyncPusher(ORIGIN).push(records())
    ledger = Ledger()
    report = SyncPusher(ORIGIN, ledger=ledger).push(records())
    assert {item.origin_uid for item in ledger.items} == {"p2", "p3"}
    assert report.counts["users"] == {"unchanged": 1}


def test_a_dry_run_lists_what_it_would_create(world):
    ledger = Ledger()
    SyncPusher(ORIGIN, dry_run=True, ledger=ledger).push(records())
    created = [item for item in ledger.items if item.outcome == "created"]
    assert {item.origin_uid for item in created} == {"u1", "p1"}
    assert get_table("sync_links").count(origin=ORIGIN) == 0


def test_caps_never_let_routine_rows_push_out_problems_and_counts_stay_exact():
    ledger = Ledger(routine_limit=3, problem_limit=2)
    for number in range(10):
        ledger.add("posts", "created", origin_uid=f"c{number}")
    for number in range(5):
        ledger.add("posts", "rejected", reason="invalid", origin_uid=f"r{number}")
    kept = [item.outcome for item in ledger.items]
    assert kept.count("created") == 3 and kept.count("rejected") == 2
    assert ledger.dropped == 7 + 3
    assert ledger.counts["created"] == 10 and ledger.counts["rejected"] == 5


def test_every_reason_the_system_can_emit_has_a_plain_explanation():
    for reason in (*REASONS, *SKIP_REASONS, "article_unreachable", "truncated", "collect_error"):
        assert reason in REASON_TEXT, reason
        assert explain(reason)["title"] and explain(reason)["text"]
    assert explain("never_heard_of_it")["title"] == "Never heard of it"
    assert explain("identity_failed")["retried"] and not explain("sampled_out")["retried"]
    assert explain("") == {"title": "", "text": "", "retried": False}


@pytest.mark.parametrize(
    "status,dry,tally,level",
    [
        ("ok", False, {"created": 5}, "ok"),
        ("ok", False, {"unchanged": 40}, "idle"),
        ("ok", False, {}, "idle"),
        ("partial", False, {"created": 5, "rejected": 1}, "attention"),
        ("partial", False, {"created": 5, "skipped": 3}, "attention"),
        ("partial", False, {"created": 5, "warning": 1}, "attention"),
        ("partial", False, {"created": 5, "failed": 1}, "bad"),
        ("partial", False, {"error": 2}, "bad"),
        ("failed", False, {}, "bad"),
        ("skipped", False, {}, "attention"),
        ("running", False, {}, "running"),
        ("dry_run", True, {"created": 5}, "ok"),
    ],
)
def test_verdict_levels(status, dry, tally, level):
    outcome = verdict(status, dry, tally)
    assert outcome["level"] == level and outcome["headline"]
    assert outcome["headline"].endswith("(dry run).") == (dry and status != "running")


def test_tally_merges_sync_statuses_skips_and_collector_findings():
    ledger = Ledger()
    ledger.add("posts", "warning", reason="truncated")
    ledger.add("feed", "error", reason="collect_error")
    pushed = {"users": {"created": 2}, "posts": {"created": 1, "updated": 1, "rejected": 1}, "skipped": {"sampled_out": 4, "topic_not_allowed": 1}}
    assert tally_of(pushed, ledger) == {"created": 3, "updated": 1, "rejected": 1, "skipped": 5, "warning": 1, "error": 1}
    assert set(tally_of(pushed, ledger)) <= set(OUTCOME_KEYS)


def test_labels_are_clipped_single_lines():
    assert label_for("users", {"username": "bob"}) == "bob"
    assert label_for("posts", {"title": "", "content": "line one\n\nline   two"}) == "line one line two"
    assert len(label_for("comments", {"content": "x" * 900})) == LEDGER.label_chars


def test_selector_reports_every_record_it_drops():
    selector = Selector(ORIGIN, SyncOptions.build(sample_percent=1))
    batch = {
        "posts": [{"origin_uid": f"s{number}", "title": "t", "content": "c" * 20} for number in range(60)],
        "comments": [{"origin_uid": f"c{number}", "post_uid": f"s{number}", "content": "hey"} for number in range(60)],
    }
    kept, skipped = selector.select(batch)
    assert len(selector.dropped) == sum(skipped.values()) > 0
    assert {reason for _, _, reason in selector.dropped} <= {"sampled_out", "parent_skipped"}
    ledger = Ledger()
    for kind, record, reason in selector.dropped:
        ledger.add_dropped_record(kind, record, reason)
    assert ledger.counts["skipped"] == len(selector.dropped) and all(item.reason for item in ledger.items)


def test_a_run_stores_its_ledger_and_a_repeat_stores_nothing(world):
    routes = json_community()
    runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep)
    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
    run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test"))
    first = run_async(runner.run(store.get("forum-test"), dry_run=False, trigger="test"))
    saved = store.run(first.run_uid)
    assert saved["items_stored"] == get_table(TABLE).count(run_uid=first.run_uid) > 0
    assert saved["tally"]["created"] == ledger_store.counts(run_uid=first.run_uid)["created"]
    assert saved["verdict"] == "ok" and saved["items_dropped"] == 0
    second = run_async(runner.run(store.get("forum-test"), dry_run=False, trigger="test"))
    again = store.run(second.run_uid)
    assert again["items_stored"] == 0 and again["verdict"] == "idle" and again["tally"]["unchanged"] > 0


def test_query_plans_use_an_index_and_never_sort_or_scan(world):
    ledger = Ledger(routine_limit=4000, problem_limit=4000)
    for number in range(3000):
        ledger.add("posts", ("created", "rejected", "skipped", "updated")[number % 4], reason="invalid", origin_uid=f"o{number}", label=f"title {number}")
    ledger_store.save("plan-run", "plan-community", ledger)
    db.query("ANALYZE")
    cases = [
        ({}, {"run_uid": "plan-run"}),
        ({"outcome": "rejected"}, {"run_uid": "plan-run"}),
        ({"outcome": "rejected", "kind": "posts"}, {"run_uid": "plan-run"}),
        ({}, {"community_uid": "plan-community"}),
        ({"outcome": "skipped"}, {"community_uid": "plan-community"}),
    ]
    for filters, scope in cases:
        where, params, order = ledger_store.compile(filters, **scope)
        for sql in (
            f"SELECT * FROM {TABLE} WHERE {where} ORDER BY {order} LIMIT 50 OFFSET 0",
            f"SELECT COUNT(*) FROM (SELECT 1 FROM {TABLE} WHERE {where} LIMIT 10001)",
        ):
            plan = " | ".join(row["detail"] for row in db.query("EXPLAIN QUERY PLAN " + sql, **params))
            assert "USING INDEX" in plan or "USING COVERING INDEX" in plan, (filters, scope, plan)
            assert "SCAN community_run_items" not in plan, plan
            if not filters.get("outcome") or not scope.get("run_uid"):
                assert "TEMP B-TREE" not in plan, (filters, scope, plan)
    assert ledger_store.counts(run_uid="plan-run") == {"created": 750, "rejected": 750, "skipped": 750, "updated": 750}
    rows, pagination = ledger_store.page({"q": "title 7%"}, 1, run_uid="plan-run")
    assert rows == [] and pagination["total"] == 0
    rows, pagination = ledger_store.page({"q": "title 77"}, 1, run_uid="plan-run")
    assert pagination["total"] == 11 and len(rows) == 11
    rows, pagination = ledger_store.page({}, 10**6, run_uid="plan-run")
    assert pagination["page"] == pagination["total_pages"] and len(rows) == 50 and pagination["capped"] is False
    json.dumps(rows)


def test_record_detail_is_kept_for_the_newest_runs_only_but_their_counts_stay(world):
    from devplacepy.services.community.spec import LIMITS

    community = store.create("keep-test", "q", "tester", name="keep")
    uids = []
    for number in range(LIMITS.ledger_runs_kept + 3):
        run_uid = store.start_run(community["uid"], "test", False)
        ledger = Ledger()
        ledger.add("posts", "created", origin_uid=f"k{number}")
        ledger_store.save(run_uid, community["uid"], ledger)
        store.finish_run(run_uid, "ok", tally={"created": 1}, items_stored=1)
        uids.append(run_uid)
    store.prune_runs(community["uid"])
    kept = [get_table(TABLE).count(run_uid=uid) for uid in uids]
    assert sum(kept) == LIMITS.ledger_runs_kept and kept[0] == 0 and kept[-1] == 1
    assert len(store.run_summaries(community["uid"])) == LIMITS.ledger_runs_kept + 3
    assert store.run(uids[0])["tally"] == {"created": 1}