retoorandClaude Sonnet 5.5
8d18994c2e
Fixes found by investigating the live lobste.rs run:
- Comments came back 0: the learner probed the external article, not the
discussion page. A comments_url field (RSS <comments>, JSON key, or an HTML
"comments" anchor) now drives comment learning; the article is still used
for the post body.
- HTML id selectors collapsed every story into one identity (constant /login
vote link). Id/url selectors must now be distinct; no distinct selector
rejects the feed.
- Over-long posts were rejected; title, post and comment are truncated to the
sync limits and reported as a warning.
- A failed article fetch is a warning, not a run error.
- A gate refusal (sync disabled, origin not allowed, aliasing off) ends the run
as skipped and no longer counts toward the failure floor.
- The same story from two feeds becomes one post, comments attached to it.
- HTML comments follow rel=next up to max_comment_pages.
- A dry run now predicts an unknown-record tombstone as unchanged, like a live
run.
- /nodes is exempt from the happy-404 decoy so a missing download is a real 404.
Sync log (admin): every run stores a verdict, exact per-outcome counts and one
row per record needing an explanation, with the reason in plain words, at
/admin/communities/{slug}/ledger (HTML, JSON, Devii tool, docs entry).
Bounded: 2000 routine + 2000 problem rows per run, 50 per page, detail kept for
the 10 newest runs, four indexes matching every ORDER BY (asserted by an
EXPLAIN QUERY PLAN test). Measured worst case 37 ms at the 40000-row ceiling.
Also: privacy disclosure for imported community content, status colour channel
tokens, and tests/unit/cli/nodes.py renamed to fix a duplicate-basename
collection error.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
96 lines
3.9 KiB
Python
96 lines
3.9 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
from tests.conftest import run_async
|
|
|
|
import pytest
|
|
|
|
from devplacepy.database import get_setting, get_table, init_db, set_setting
|
|
from devplacepy.services.community.runner import CommunityRunner
|
|
from devplacepy.services.community.service import DRY_RUN_KEY, CommunityService
|
|
from devplacepy.services.community.spec import LIMITS, RUN_SKIPPED, STATUS_FAILING, STATUS_READY
|
|
from devplacepy.services.community.store import store
|
|
from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep
|
|
|
|
|
|
@pytest.fixture
|
|
def world():
|
|
init_db()
|
|
previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins", DRY_RUN_KEY)}
|
|
set_setting("sync_enabled", "1")
|
|
set_setting("sync_origins", "")
|
|
get_table("communities").delete()
|
|
get_table("community_runs").delete()
|
|
get_table("sync_links").delete(origin="forum.test")
|
|
routes = json_community()
|
|
runner = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep)
|
|
yield routes, runner
|
|
for key, value in previous.items():
|
|
set_setting(key, value)
|
|
|
|
|
|
def seed(runner) -> dict:
|
|
store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
|
|
run_async(runner.discover("https://forum.test/api/posts", "tester", relearn_slug="forum-test"))
|
|
return store.get("forum-test")
|
|
|
|
|
|
def test_dry_run_service_tick_learns_nothing_and_runs_nothing(world):
|
|
_, runner = world
|
|
store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
|
|
set_setting(DRY_RUN_KEY, "1")
|
|
service = CommunityService(runner)
|
|
run_async(service.run_once())
|
|
assert store.get("forum-test")["status"] == "pending"
|
|
assert any("DRY RUN" in line for line in service.log_buffer)
|
|
|
|
|
|
def test_live_tick_learns_pending_then_runs_due(world):
|
|
_, runner = world
|
|
store.create("forum-test", "https://forum.test/api/posts", "tester", name="forum")
|
|
set_setting(DRY_RUN_KEY, "0")
|
|
service = CommunityService(runner)
|
|
run_async(service.run_once())
|
|
assert store.get("forum-test")["status"] == STATUS_READY
|
|
run_async(service.run_once())
|
|
row = store.get("forum-test")
|
|
assert row["run_count"] == 1
|
|
assert get_table("sync_links").count(origin="forum.test", kind="posts") == 10
|
|
run_async(service.run_once())
|
|
assert store.get("forum-test")["run_count"] == 1
|
|
|
|
|
|
def test_repeated_failures_disable_and_flag_the_community(world):
|
|
routes, runner = world
|
|
row = seed(runner)
|
|
routes["https://forum.test/api/posts"] = (500, "down")
|
|
failing = CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep)
|
|
for _ in range(6):
|
|
current = store.get("forum-test")
|
|
report = run_async(failing.run(current, dry_run=False, trigger="test"))
|
|
assert report.status == "failed"
|
|
final = store.get("forum-test")
|
|
assert final["status"] == STATUS_FAILING and not final["enabled"]
|
|
assert "relearn" in "".join(store.runs(row["uid"])[0]["log"]).lower() or final["failure_count"] >= 5
|
|
|
|
|
|
def test_run_history_is_bounded(world):
|
|
_, runner = world
|
|
row = seed(runner)
|
|
for _ in range(35):
|
|
run_async(runner.run(store.get("forum-test"), dry_run=True, trigger="test"))
|
|
assert len(store.runs(row["uid"], limit=100)) == 30
|
|
|
|
|
|
def test_gate_refusal_skips_the_run_without_counting_a_failure(world):
|
|
_, runner = world
|
|
row = seed(runner)
|
|
set_setting("sync_enabled", "0")
|
|
for _ in range(LIMITS.max_failures + 1):
|
|
report = run_async(runner.run(store.get("forum-test"), dry_run=False, trigger="test"))
|
|
assert report.status == RUN_SKIPPED
|
|
final = store.get("forum-test")
|
|
assert final["status"] == STATUS_READY and final["enabled"]
|
|
assert final["failure_count"] == 0 and final["last_status"] == RUN_SKIPPED
|
|
assert final["next_run_at"] and get_table("sync_links").count(origin="forum.test") == 0
|
|
assert store.runs(row["uid"])[0]["status"] == RUN_SKIPPED
|