retoorandClaude Sonnet 5.5
8d18994c2e
Fixes found by investigating the live lobste.rs run:
- Comments came back 0: the learner probed the external article, not the
discussion page. A comments_url field (RSS <comments>, JSON key, or an HTML
"comments" anchor) now drives comment learning; the article is still used
for the post body.
- HTML id selectors collapsed every story into one identity (constant /login
vote link). Id/url selectors must now be distinct; no distinct selector
rejects the feed.
- Over-long posts were rejected; title, post and comment are truncated to the
sync limits and reported as a warning.
- A failed article fetch is a warning, not a run error.
- A gate refusal (sync disabled, origin not allowed, aliasing off) ends the run
as skipped and no longer counts toward the failure floor.
- The same story from two feeds becomes one post, comments attached to it.
- HTML comments follow rel=next up to max_comment_pages.
- A dry run now predicts an unknown-record tombstone as unchanged, like a live
run.
- /nodes is exempt from the happy-404 decoy so a missing download is a real 404.
Sync log (admin): every run stores a verdict, exact per-outcome counts and one
row per record needing an explanation, with the reason in plain words, at
/admin/communities/{slug}/ledger (HTML, JSON, Devii tool, docs entry).
Bounded: 2000 routine + 2000 problem rows per run, 50 per page, detail kept for
the 10 newest runs, four indexes matching every ORDER BY (asserted by an
EXPLAIN QUERY PLAN test). Measured worst case 37 ms at the 40000-row ceiling.
Also: privacy disclosure for imported community content, status colour channel
tokens, and tests/unit/cli/nodes.py renamed to fix a duplicate-basename
collection error.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
123 lines
5.9 KiB
Python
123 lines
5.9 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
from devplacepy.database import get_table, set_setting
|
|
from devplacepy.services.community.ledger_store import ledger_store
|
|
from devplacepy.services.community.store import store
|
|
from tests.api.admin.communities.index import ( # noqa: F401
|
|
add_ready,
|
|
admin,
|
|
client,
|
|
fake_world,
|
|
wait_for,
|
|
)
|
|
|
|
LEDGER = "/admin/communities/forum-test/ledger"
|
|
|
|
|
|
def finished_runs(row):
|
|
return [item for item in store.runs(row["uid"]) if item["status"] != "running"]
|
|
|
|
|
|
def live_run(client, admin, row, count=1):
|
|
client.post("/admin/communities/forum-test/run", data={}, headers=admin["headers"])
|
|
return wait_for(lambda: len(finished_runs(row)) >= count and store.runs(row["uid"])[0])
|
|
|
|
|
|
def test_run_page_shows_verdict_exact_tiles_and_every_created_record(client, admin, fake_world):
|
|
row = add_ready(client, admin)
|
|
run = live_run(client, admin, row)
|
|
assert run["verdict"] == "ok" and run["headline"] == "Everything synced."
|
|
body = client.get(f"{LEDGER}?run={run['uid']}", headers=admin["headers"]).json()
|
|
assert body["scope"] == "run" and body["verdict"]["title"] == "All good"
|
|
tiles = {tile["key"]: tile for tile in body["tiles"]}
|
|
assert tiles["created"]["count"] == tiles["created"]["listed"] == run["tally"]["created"] > 10
|
|
assert tiles["rejected"]["count"] == 0 and tiles["failed"]["count"] == 0
|
|
assert body["pagination"]["total"] == tiles["created"]["listed"]
|
|
posts = [item for item in body["items"] if item["kind"] == "posts"]
|
|
assert posts and all(item["outcome"] == "created" and item["label"] for item in posts)
|
|
assert any(item["local_url"].startswith("/posts/") for item in posts)
|
|
users = [item for item in body["items"] if item["kind"] == "users"]
|
|
assert users and all(item["local_url"].startswith("/profile/") for item in users)
|
|
|
|
|
|
def test_second_run_lists_nothing_and_says_nothing_is_new(client, admin, fake_world):
|
|
row = add_ready(client, admin)
|
|
live_run(client, admin, row)
|
|
second = live_run(client, admin, row, count=2)
|
|
assert second["verdict"] == "idle" and second["items_stored"] == 0
|
|
body = client.get(f"{LEDGER}?run={second['uid']}", headers=admin["headers"]).json()
|
|
assert body["items"] == [] and {tile["key"]: tile["count"] for tile in body["tiles"]}["unchanged"] > 0
|
|
|
|
|
|
def test_kind_and_outcome_filters_narrow_the_list_and_are_echoed(client, admin, fake_world):
|
|
row = add_ready(client, admin)
|
|
run = live_run(client, admin, row)
|
|
uid = run["uid"]
|
|
assert ledger_store.counts(run_uid=uid)["created"] == run["tally"]["created"]
|
|
body = client.get(f"{LEDGER}?run={uid}&outcome=created&kind=posts", headers=admin["headers"]).json()
|
|
assert body["filters"] == {"kind": "posts", "outcome": "created"}
|
|
assert all(item["kind"] == "posts" for item in body["items"])
|
|
|
|
|
|
def test_filters_search_and_pagination_never_break(client, admin, fake_world):
|
|
row = add_ready(client, admin)
|
|
run = live_run(client, admin, row)
|
|
uid = run["uid"]
|
|
for query in (
|
|
f"?run={uid}&q=%25",
|
|
f"?run={uid}&q=_",
|
|
f"?run={uid}&q=" + "x" * 500,
|
|
f"?run={uid}&page=999999",
|
|
f"?run={uid}&page=-4",
|
|
f"?run={uid}&kind=nope&outcome=nope&reason=%27%3B--",
|
|
"",
|
|
"?page=999999&q=%25",
|
|
):
|
|
response = client.get(LEDGER + query, headers=admin["headers"])
|
|
assert response.status_code == 200, (query, response.text[:200])
|
|
assert response.json()["pagination"]["per_page"] == 50
|
|
html = client.get(f"{LEDGER}?run={uid}", headers={"X-API-KEY": admin["headers"]["X-API-KEY"]})
|
|
assert html.status_code == 200 and "All good" in html.text and "ledger-tile" in html.text
|
|
|
|
|
|
def test_unknown_run_and_another_communitys_run_are_404(client, admin, fake_world):
|
|
row = add_ready(client, admin)
|
|
assert client.get(f"{LEDGER}?run=nope", headers=admin["headers"]).status_code == 404
|
|
other = store.create("other-test", "other", "tester", name="other")
|
|
foreign = store.start_run(other["uid"], "test", False)
|
|
assert client.get(f"{LEDGER}?run={foreign}", headers=admin["headers"]).status_code == 404
|
|
assert client.get("/admin/communities/missing/ledger", headers=admin["headers"]).status_code == 404
|
|
assert row
|
|
|
|
|
|
def test_community_page_lists_runs_without_loading_their_logs(client, admin, fake_world):
|
|
row = add_ready(client, admin)
|
|
live_run(client, admin, row)
|
|
body = client.get("/admin/communities/forum-test", headers=admin["headers"]).json()
|
|
assert body["runs"] and all("log" not in run for run in body["runs"])
|
|
assert body["runs"][0]["verdict"] == "ok" and body["runs"][0]["tally"]["created"] > 0
|
|
html = client.get("/admin/communities/forum-test", headers={"X-API-KEY": admin["headers"]["X-API-KEY"]})
|
|
assert "Everything the sync did" in html.text and f"ledger?run={body['runs'][0]['uid']}" in html.text
|
|
|
|
|
|
def test_a_run_from_before_the_ledger_still_shows_a_tally(client, admin, fake_world):
|
|
row = add_ready(client, admin)
|
|
legacy = store.start_run(row["uid"], "admin", False)
|
|
store.finish_run(legacy, "ok", collected={"posts": 3}, pushed={"posts": {"created": 3}}, errors=[], log="old")
|
|
runs = client.get("/admin/communities/forum-test", headers=admin["headers"]).json()["runs"]
|
|
old = next(run for run in runs if run["uid"] == legacy)
|
|
assert old["tally"] == {"created": 3} and old["verdict"] == "ok"
|
|
page = client.get(f"{LEDGER}?run={legacy}", headers=admin["headers"])
|
|
assert page.status_code == 200 and page.json()["run_log"] == "old"
|
|
|
|
|
|
def test_the_ledger_is_gone_with_its_run(client, admin, fake_world):
|
|
row = add_ready(client, admin)
|
|
run = live_run(client, admin, row)
|
|
assert get_table("community_run_items").count(run_uid=run["uid"]) > 0
|
|
for _ in range(32):
|
|
store.finish_run(store.start_run(row["uid"], "test", True), "dry_run")
|
|
store.prune_runs(row["uid"])
|
|
assert get_table("community_run_items").count(run_uid=run["uid"]) == 0
|
|
set_setting("sync_enabled", "1")
|