devplacepy/tests/unit/services/community/checkpoint.py

# retoor <retoor@molodetz.nl>

import asyncio
from dataclasses import replace

import pytest

from devplacepy.database import get_setting, get_table, init_db, set_setting
from devplacepy.services.community import checkpoint as checkpoint_module
from devplacepy.services.community.ledger import Ledger
from devplacepy.services.community.ledger_store import TABLE, ledger_store
from devplacepy.services.community.runner import CommunityRunner
from devplacepy.services.community.spec import LIMITS, RUN_INTERRUPTED
from devplacepy.services.community.store import iso, now, store
from tests.conftest import run_async
from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep


@pytest.fixture
def world():
    init_db()
    previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins")}
    set_setting("sync_enabled", "1")
    set_setting("sync_origins", "")
    for table in ("communities", "community_runs", TABLE):
        get_table(table).delete()
    get_table("sync_links").delete(origin="forum.test")
    yield
    for key, value in previous.items():
        set_setting(key, value)


class SlowFetcher(FakeFetcher):
    async def get(self, url, params=None, timeout=None, **kwargs):
        await asyncio.sleep(0.12)
        return await super().get(url, params, timeout, **kwargs)


def learned(runner):
    store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
    run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test"))
    return store.get("forum-test")


def test_a_running_run_exposes_its_log_and_ledger_before_it_finishes(world, monkeypatch):
    runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(json_community()), sleep=no_sleep)
    row = learned(runner)
    monkeypatch.setattr(checkpoint_module, "LIMITS", replace(LIMITS, checkpoint_seconds=0.05))
    seen = []
    original = store.update_run

    def spy(uid, **fields):
        original(uid, **fields)
        seen.append((get_table("community_runs").find_one(uid=uid)["status"], fields.get("log", ""), get_table(TABLE).count(run_uid=uid)))

    monkeypatch.setattr(store, "update_run", spy)
    runner.fetcher_factory = lambda: SlowFetcher(json_community())
    report = run_async(runner.run(row, dry_run=False, trigger="test"))
    assert seen, "no checkpoint was written while the run was going"
    assert all(status == "running" for status, _, _ in seen)
    assert any("collect" in log or "run[" in log for _, log, _ in seen)
    final = store.run(report.run_uid)
    assert final["status"] in ("ok", "partial") and final["verdict"] != "running"
    assert final["items_stored"] == get_table(TABLE).count(run_uid=report.run_uid) > 0


def test_checkpointed_ledger_rows_are_never_written_twice(world):
    community = store.create("keep-cp", "q", "tester", name="cp")
    run_uid = store.start_run(community["uid"], "test", False)
    ledger = Ledger()
    from devplacepy.services.community.checkpoint import RunCheckpoint
    from devplacepy.services.community.logbook import LogBook
    from devplacepy.services.community.pusher import PushReport
    from devplacepy.services.community.runner import RunReport

    report = RunReport(status="running", dry_run=False, run_uid=run_uid, ledger=ledger)
    live = PushReport(dry_run=False)
    with LogBook() as book:
        checkpoint = RunCheckpoint(store, ledger_store, run_uid, community["uid"], report, live, book)
        for number in range(3):
            ledger.add("posts", "created", origin_uid=f"a{number}")
        checkpoint.flush()
        checkpoint.flush()
        assert get_table(TABLE).count(run_uid=run_uid) == 3
        for number in range(2):
            ledger.add("posts", "rejected", reason="invalid", origin_uid=f"b{number}")
        checkpoint.flush()
        assert get_table(TABLE).count(run_uid=run_uid) == 5
        final = ledger_store.append(run_uid, community["uid"], ledger, checkpoint.saved)
    assert final == 0
    seqs = sorted(row["seq"] for row in get_table(TABLE).find(run_uid=run_uid))
    assert seqs == [0, 1, 2, 3, 4]


def test_a_killed_run_is_marked_interrupted_and_keeps_what_was_saved(world):
    community = store.create("keep-kill", "q", "tester", name="kill")
    run_uid = store.start_run(community["uid"], "test", False)
    store.update_run(run_uid, log="line one\nline two", tally={"created": 4}, items_stored=4)
    assert store.reap_interrupted() == 0
    stale = iso(now().replace(year=2000))
    get_table("community_runs").update({"uid": run_uid, "started_at": stale}, ["uid"])
    assert store.reap_interrupted() == 1
    found = store.run(run_uid)
    assert found["status"] == RUN_INTERRUPTED and found["verdict"] == "bad" and "Interrupted" in found["headline"]
    assert found["log"] == "line one\nline two" and found["tally"] == {"created": 4} and found["finished_at"]
    assert store.reap_interrupted() == 0
    assert not store.busy(community["uid"])


def test_the_interrupted_verdict_is_bad():
    from devplacepy.services.community.ledger import verdict

    assert verdict("interrupted", False, {})["level"] == "bad"


def test_the_search_index_follows_inserts_and_prunes(world):
    ledger = Ledger()
    ledger.add("posts", "created", origin_uid="needle-1", label="Alpha Beta Gamma", detail="")
    ledger.add("posts", "created", origin_uid="hay-2", label="Delta", detail="contains the NEEDLE word")
    ledger.add("posts", "created", origin_uid="hay-3", label="100% done", detail="")
    ledger_store.save("search-run", "search-community", ledger)
    assert ledger_store.search_ready()

    def find(text):
        rows, page = ledger_store.page({"q": text}, 1, community_uid="search-community")
        scoped, _ = ledger_store.page({"q": text}, 1, run_uid="search-run")
        assert sorted(row["origin_uid"] for row in scoped) == sorted(row["origin_uid"] for row in rows)
        return sorted(row["origin_uid"] for row in rows)

    assert find("needle") == ["hay-2", "needle-1"]
    assert find("beta g") == ["needle-1"]
    assert find("100%") == ["hay-3"]
    assert find("a\"b") == []
    assert find("ne") == ["hay-2", "hay-3", "needle-1"]
    get_table(TABLE).delete(run_uid="search-run")
    assert find("needle") == []


def test_a_community_wide_text_search_goes_through_the_index(world):
    from devplacepy.database import db

    ledger = Ledger()
    for number in range(50):
        ledger.add("posts", "created", origin_uid=f"p{number}", label=f"label {number}")
    ledger_store.save("plan-run-a", "plan-community-a", ledger)
    where, params, order = ledger_store.compile({"q": "label 4"}, community_uid="plan-community-a")
    plan = " | ".join(row["detail"] for row in db.query(f"EXPLAIN QUERY PLAN SELECT * FROM {TABLE} WHERE {where} ORDER BY {order} LIMIT 50", **params))
    assert "community_run_items_fts" in plan
    where, params, order = ledger_store.compile({"q": "la"}, community_uid="plan-community-a")
    assert "community_run_items_fts" not in where and "LIKE" in where
    where, params, order = ledger_store.compile({"q": "label 4"}, run_uid="plan-run-a")
    assert "community_run_items_fts" not in where