# retoor import asyncio from dataclasses import replace import pytest from devplacepy.database import get_setting, get_table, init_db, set_setting from devplacepy.services.community import checkpoint as checkpoint_module from devplacepy.services.community.ledger import Ledger from devplacepy.services.community.ledger_store import TABLE, ledger_store from devplacepy.services.community.runner import CommunityRunner from devplacepy.services.community.spec import LIMITS, RUN_INTERRUPTED from devplacepy.services.community.store import iso, now, store from tests.conftest import run_async from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep @pytest.fixture def world(): init_db() previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins")} set_setting("sync_enabled", "1") set_setting("sync_origins", "") for table in ("communities", "community_runs", TABLE): get_table(table).delete() get_table("sync_links").delete(origin="forum.test") yield for key, value in previous.items(): set_setting(key, value) class SlowFetcher(FakeFetcher): async def get(self, url, params=None, timeout=None, **kwargs): await asyncio.sleep(0.12) return await super().get(url, params, timeout, **kwargs) def learned(runner): store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum") run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test")) return store.get("forum-test") def test_a_running_run_exposes_its_log_and_ledger_before_it_finishes(world, monkeypatch): runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(json_community()), sleep=no_sleep) row = learned(runner) monkeypatch.setattr(checkpoint_module, "LIMITS", replace(LIMITS, checkpoint_seconds=0.05)) seen = [] original = store.update_run def spy(uid, **fields): original(uid, **fields) seen.append((get_table("community_runs").find_one(uid=uid)["status"], fields.get("log", ""), get_table(TABLE).count(run_uid=uid))) monkeypatch.setattr(store, "update_run", spy) runner.fetcher_factory = lambda: SlowFetcher(json_community()) report = run_async(runner.run(row, dry_run=False, trigger="test")) assert seen, "no checkpoint was written while the run was going" assert all(status == "running" for status, _, _ in seen) assert any("collect" in log or "run[" in log for _, log, _ in seen) final = store.run(report.run_uid) assert final["status"] in ("ok", "partial") and final["verdict"] != "running" assert final["items_stored"] == get_table(TABLE).count(run_uid=report.run_uid) > 0 def test_checkpointed_ledger_rows_are_never_written_twice(world): community = store.create("keep-cp", "q", "tester", name="cp") run_uid = store.start_run(community["uid"], "test", False) ledger = Ledger() from devplacepy.services.community.checkpoint import RunCheckpoint from devplacepy.services.community.logbook import LogBook from devplacepy.services.community.pusher import PushReport from devplacepy.services.community.runner import RunReport report = RunReport(status="running", dry_run=False, run_uid=run_uid, ledger=ledger) live = PushReport(dry_run=False) with LogBook() as book: checkpoint = RunCheckpoint(store, ledger_store, run_uid, community["uid"], report, live, book) for number in range(3): ledger.add("posts", "created", origin_uid=f"a{number}") checkpoint.flush() checkpoint.flush() assert get_table(TABLE).count(run_uid=run_uid) == 3 for number in range(2): ledger.add("posts", "rejected", reason="invalid", origin_uid=f"b{number}") checkpoint.flush() assert get_table(TABLE).count(run_uid=run_uid) == 5 final = ledger_store.append(run_uid, community["uid"], ledger, checkpoint.saved) assert final == 0 seqs = sorted(row["seq"] for row in get_table(TABLE).find(run_uid=run_uid)) assert seqs == [0, 1, 2, 3, 4] def test_a_killed_run_is_marked_interrupted_and_keeps_what_was_saved(world): community = store.create("keep-kill", "q", "tester", name="kill") run_uid = store.start_run(community["uid"], "test", False) store.update_run(run_uid, log="line one\nline two", tally={"created": 4}, items_stored=4) assert store.reap_interrupted() == 0 stale = iso(now().replace(year=2000)) get_table("community_runs").update({"uid": run_uid, "started_at": stale}, ["uid"]) assert store.reap_interrupted() == 1 found = store.run(run_uid) assert found["status"] == RUN_INTERRUPTED and found["verdict"] == "bad" and "Interrupted" in found["headline"] assert found["log"] == "line one\nline two" and found["tally"] == {"created": 4} and found["finished_at"] assert store.reap_interrupted() == 0 assert not store.busy(community["uid"]) def test_the_interrupted_verdict_is_bad(): from devplacepy.services.community.ledger import verdict assert verdict("interrupted", False, {})["level"] == "bad" def test_the_search_index_follows_inserts_and_prunes(world): ledger = Ledger() ledger.add("posts", "created", origin_uid="needle-1", label="Alpha Beta Gamma", detail="") ledger.add("posts", "created", origin_uid="hay-2", label="Delta", detail="contains the NEEDLE word") ledger.add("posts", "created", origin_uid="hay-3", label="100% done", detail="") ledger_store.save("search-run", "search-community", ledger) assert ledger_store.search_ready() def find(text): rows, page = ledger_store.page({"q": text}, 1, community_uid="search-community") scoped, _ = ledger_store.page({"q": text}, 1, run_uid="search-run") assert sorted(row["origin_uid"] for row in scoped) == sorted(row["origin_uid"] for row in rows) return sorted(row["origin_uid"] for row in rows) assert find("needle") == ["hay-2", "needle-1"] assert find("beta g") == ["needle-1"] assert find("100%") == ["hay-3"] assert find("a\"b") == [] assert find("ne") == ["hay-2", "hay-3", "needle-1"] get_table(TABLE).delete(run_uid="search-run") assert find("needle") == [] def test_a_community_wide_text_search_goes_through_the_index(world): from devplacepy.database import db ledger = Ledger() for number in range(50): ledger.add("posts", "created", origin_uid=f"p{number}", label=f"label {number}") ledger_store.save("plan-run-a", "plan-community-a", ledger) where, params, order = ledger_store.compile({"q": "label 4"}, community_uid="plan-community-a") plan = " | ".join(row["detail"] for row in db.query(f"EXPLAIN QUERY PLAN SELECT * FROM {TABLE} WHERE {where} ORDER BY {order} LIMIT 50", **params)) assert "community_run_items_fts" in plan where, params, order = ledger_store.compile({"q": "la"}, community_uid="plan-community-a") assert "community_run_items_fts" not in where and "LIKE" in where where, params, order = ledger_store.compile({"q": "label 4"}, run_uid="plan-run-a") assert "community_run_items_fts" not in where