retoorandClaude Sonnet 5.5
8d18994c2e
Fixes found by investigating the live lobste.rs run:
- Comments came back 0: the learner probed the external article, not the
discussion page. A comments_url field (RSS <comments>, JSON key, or an HTML
"comments" anchor) now drives comment learning; the article is still used
for the post body.
- HTML id selectors collapsed every story into one identity (constant /login
vote link). Id/url selectors must now be distinct; no distinct selector
rejects the feed.
- Over-long posts were rejected; title, post and comment are truncated to the
sync limits and reported as a warning.
- A failed article fetch is a warning, not a run error.
- A gate refusal (sync disabled, origin not allowed, aliasing off) ends the run
as skipped and no longer counts toward the failure floor.
- The same story from two feeds becomes one post, comments attached to it.
- HTML comments follow rel=next up to max_comment_pages.
- A dry run now predicts an unknown-record tombstone as unchanged, like a live
run.
- /nodes is exempt from the happy-404 decoy so a missing download is a real 404.
Sync log (admin): every run stores a verdict, exact per-outcome counts and one
row per record needing an explanation, with the reason in plain words, at
/admin/communities/{slug}/ledger (HTML, JSON, Devii tool, docs entry).
Bounded: 2000 routine + 2000 problem rows per run, 50 per page, detail kept for
the 10 newest runs, four indexes matching every ORDER BY (asserted by an
EXPLAIN QUERY PLAN test). Measured worst case 37 ms at the 40000-row ceiling.
Also: privacy disclosure for imported community content, status colour channel
tokens, and tests/unit/cli/nodes.py renamed to fix a duplicate-basename
collection error.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
250 lines
12 KiB
Python
250 lines
12 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
import time
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
from devplacepy.database import get_setting, get_table, set_setting
|
|
from devplacepy.main import app
|
|
from devplacepy.routers.admin import communities as router_module
|
|
from devplacepy.services.community.runner import CommunityRunner
|
|
from devplacepy.services.community.store import store
|
|
from devplacepy.utils import register_account
|
|
from tests.unit.services.community.fakes import FakeFetcher, json_community, no_sleep
|
|
|
|
_counter = [0]
|
|
|
|
|
|
def unique(prefix: str) -> str:
|
|
_counter[0] += 1
|
|
return f"{prefix}{int(time.time() * 1000)}{_counter[0]}"
|
|
|
|
|
|
def make_account(role: str = "Member") -> dict:
|
|
name = unique("cmty")
|
|
uid, _, _ = register_account(name, f"{name}@t.dev", "secret123", age_band="adult", accepted_terms=True)
|
|
if role != "Member":
|
|
get_table("users").update({"uid": uid, "role": role}, ["uid"])
|
|
row = get_table("users").find_one(uid=uid)
|
|
return {"username": name, "headers": {"X-API-KEY": row["api_key"], "Accept": "application/json"}}
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def client():
|
|
previous = {key: get_setting(key, "") for key in ("sync_enabled", "sync_origins")}
|
|
set_setting("sync_enabled", "1")
|
|
set_setting("sync_origins", "")
|
|
with TestClient(app) as test_client:
|
|
yield test_client
|
|
for key, value in previous.items():
|
|
set_setting(key, value)
|
|
|
|
|
|
@pytest.fixture
|
|
def fake_world(monkeypatch):
|
|
routes = json_community()
|
|
monkeypatch.setattr(
|
|
router_module,
|
|
"CommunityRunner",
|
|
lambda: CommunityRunner(fetcher_factory=lambda: FakeFetcher(routes), sleep=no_sleep),
|
|
)
|
|
get_table("communities").delete()
|
|
get_table("community_runs").delete()
|
|
get_table("sync_links").delete(origin="forum.test")
|
|
return routes
|
|
|
|
|
|
@pytest.fixture
|
|
def admin(client):
|
|
return make_account("Admin")
|
|
|
|
|
|
def wait_for(predicate, seconds: float = 10.0):
|
|
deadline = time.time() + seconds
|
|
while time.time() < deadline:
|
|
value = predicate()
|
|
if value:
|
|
return value
|
|
time.sleep(0.1)
|
|
raise AssertionError("condition never became true")
|
|
|
|
|
|
def add_ready(client, admin) -> dict:
|
|
response = client.post("/admin/communities", data={"query": "https://forum.test/api/posts"}, headers=admin["headers"])
|
|
assert response.status_code == 200, response.text
|
|
return wait_for(lambda: (row := store.get("forum-test")) and row["status"] == "ready" and row)
|
|
|
|
|
|
def test_member_is_refused(client, admin, fake_world):
|
|
member = make_account()
|
|
assert client.get("/admin/communities", headers=member["headers"], follow_redirects=False).status_code in (303, 403)
|
|
assert client.post("/admin/communities", data={"query": "x.test"}, headers=member["headers"], follow_redirects=False).status_code in (303, 403)
|
|
|
|
|
|
def test_add_learns_and_shows_detail(client, admin, fake_world):
|
|
row = add_ready(client, admin)
|
|
assert row["origin"] == "forum.test"
|
|
assert row["schema"]["feeds"][0]["type"] == "json_api"
|
|
assert row["description"] == "A friendly forum about code."
|
|
assert row["user_count"] == 1234
|
|
listing = client.get("/admin/communities", headers=admin["headers"])
|
|
assert listing.status_code == 200
|
|
assert listing.json()["items"][0]["slug"] == "forum-test"
|
|
detail = client.get("/admin/communities/forum-test", headers=admin["headers"])
|
|
assert detail.status_code == 200
|
|
body = detail.json()
|
|
assert body["feeds"][0]["pagination"] == "page"
|
|
assert body["collectors_lines"] > 100
|
|
html = client.get("/admin/communities/forum-test", headers={"X-API-KEY": admin["headers"]["X-API-KEY"]})
|
|
assert html.status_code == 200 and "Learned feeds" in html.text
|
|
|
|
|
|
def test_add_is_idempotent(client, admin, fake_world):
|
|
add_ready(client, admin)
|
|
again = client.post("/admin/communities", data={"query": "https://forum.test/api/posts"}, headers=admin["headers"])
|
|
assert again.status_code == 200
|
|
assert len(store.list()) == 1
|
|
|
|
|
|
def test_dry_run_writes_nothing_and_keeps_watermark(client, admin, fake_world):
|
|
row = add_ready(client, admin)
|
|
users_before = get_table("users").count()
|
|
response = client.post("/admin/communities/forum-test/run", data={"dry_run": "true"}, headers=admin["headers"])
|
|
assert response.status_code == 200
|
|
run = wait_for(lambda: next((item for item in store.runs(row["uid"]) if item["status"] == "dry_run"), None))
|
|
assert run["dry_run"] == 1
|
|
assert run["pushed"]["posts"]["created"] == 10
|
|
assert "DRY RUN" in run["log"]
|
|
assert get_table("users").count() == users_before
|
|
after = store.get("forum-test")
|
|
assert not after["watermark"]
|
|
assert after["run_count"] == 0
|
|
|
|
|
|
def test_live_run_creates_then_is_unchanged(client, admin, fake_world):
|
|
row = add_ready(client, admin)
|
|
client.post("/admin/communities/forum-test/run", data={}, headers=admin["headers"])
|
|
first = wait_for(lambda: next((item for item in store.runs(row["uid"]) if item["status"] in ("ok", "partial", "failed")), None))
|
|
assert first["status"] == "ok", first["errors"]
|
|
assert first["pushed"]["posts"]["created"] == 10
|
|
assert get_table("sync_links").count(origin="forum.test", kind="posts") == 10
|
|
done = store.get("forum-test")
|
|
assert done["run_count"] == 1 and done["watermark"]["feeds"]["feed1"]["since"]
|
|
assert done["next_run_at"]
|
|
client.post("/admin/communities/forum-test/run", data={}, headers=admin["headers"])
|
|
second = wait_for(lambda: len([item for item in store.runs(row["uid"]) if item["status"] != "running"]) >= 2 and store.runs(row["uid"])[0])
|
|
assert second["pushed"].get("posts", {}).get("created", 0) == 0
|
|
|
|
|
|
def test_live_run_refused_when_sync_disabled(client, admin, fake_world):
|
|
row = add_ready(client, admin)
|
|
set_setting("sync_enabled", "0")
|
|
try:
|
|
client.post("/admin/communities/forum-test/run", data={}, headers=admin["headers"])
|
|
run = wait_for(lambda: next((item for item in store.runs(row["uid"]) if item["status"] != "running"), None))
|
|
assert run["status"] == "skipped" and run["verdict"] == "attention"
|
|
assert any("disabled" in error for error in run["errors"])
|
|
assert store.get("forum-test")["failure_count"] == 0 and store.get("forum-test")["status"] == "ready"
|
|
finally:
|
|
set_setting("sync_enabled", "1")
|
|
|
|
|
|
def test_update_and_delete(client, admin, fake_world):
|
|
add_ready(client, admin)
|
|
client.post("/admin/communities/forum-test/update", data={"interval_seconds": "7200", "enabled": "on"}, headers=admin["headers"])
|
|
assert store.get("forum-test")["interval_seconds"] == 7200
|
|
bad = client.post("/admin/communities/forum-test/update", data={"interval_seconds": "10"}, headers=admin["headers"])
|
|
assert bad.status_code in (400, 422)
|
|
client.post("/admin/communities/forum-test/delete", headers=admin["headers"])
|
|
assert store.get("forum-test") is None
|
|
assert get_table("communities").find_one(slug="forum-test")["deleted_at"]
|
|
|
|
|
|
def test_unknown_community_is_404(client, admin, fake_world):
|
|
assert client.get("/admin/communities/nope", headers=admin["headers"]).status_code == 404
|
|
|
|
|
|
GATED = [
|
|
("get", "/admin/communities", {}),
|
|
("post", "/admin/communities", {"query": "https://forum.test/api/posts"}),
|
|
("get", "/admin/communities/forum-test", {}),
|
|
("get", "/admin/communities/forum-test/ledger", {}),
|
|
("post", "/admin/communities/forum-test/run", {}),
|
|
("post", "/admin/communities/forum-test/relearn", {}),
|
|
("post", "/admin/communities/forum-test/update", {"interval_seconds": "7200"}),
|
|
("post", "/admin/communities/forum-test/delete", {}),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("method,path,data", GATED)
|
|
def test_every_endpoint_refuses_guests_and_members_and_changes_nothing(client, admin, fake_world, method, path, data):
|
|
add_ready(client, admin)
|
|
before = store.get("forum-test")
|
|
member = make_account()
|
|
for headers in ({"Accept": "application/json"}, member["headers"]):
|
|
response = client.request(method.upper(), path, data=data or None, headers=headers, follow_redirects=False)
|
|
assert response.status_code in (303, 401, 403), (headers, response.status_code)
|
|
after = store.get("forum-test")
|
|
assert after is not None
|
|
assert after["interval_seconds"] == before["interval_seconds"] and len(store.list()) == 1
|
|
|
|
|
|
def test_password_is_stored_but_never_returned(client, admin, fake_world):
|
|
client.post(
|
|
"/admin/communities",
|
|
data={"query": "https://forum.test/api/posts", "login_username": "bot", "login_password": "s3cret-pw"},
|
|
headers=admin["headers"],
|
|
)
|
|
row = wait_for(lambda: (found := store.get("forum-test")) and found["status"] == "ready" and found)
|
|
assert row["login_password"] == "s3cret-pw"
|
|
for url in ("/admin/communities", "/admin/communities/forum-test"):
|
|
body = client.get(url, headers=admin["headers"])
|
|
assert "s3cret-pw" not in body.text and body.json()
|
|
page = client.get("/admin/communities/forum-test", headers={"X-API-KEY": admin["headers"]["X-API-KEY"]})
|
|
assert "s3cret-pw" not in page.text and "(stored)" in page.text
|
|
client.post("/admin/communities/forum-test/update", data={"interval_seconds": "7200", "login_username": "other"}, headers=admin["headers"])
|
|
assert store.get("forum-test")["login_password"] == "s3cret-pw"
|
|
client.post("/admin/communities/forum-test/update", data={"interval_seconds": "7200", "clear_login": "on"}, headers=admin["headers"])
|
|
assert store.get("forum-test")["login_password"] == ""
|
|
|
|
|
|
def test_devii_tools_are_admin_only_and_delete_needs_confirmation():
|
|
from devplacepy.services.devii.actions.catalog.community import COMMUNITY_ACTIONS
|
|
from devplacepy.services.devii.actions.dispatcher import CONFIRM_REQUIRED
|
|
from devplacepy.docs_api.groups import community
|
|
|
|
assert {action.name for action in COMMUNITY_ACTIONS} >= {"community_add", "community_run", "community_delete"}
|
|
assert all(action.requires_admin for action in COMMUNITY_ACTIONS)
|
|
assert "community_delete" in CONFIRM_REQUIRED
|
|
assert next(action for action in COMMUNITY_ACTIONS if action.name == "community_delete").params[-1].name == "confirm"
|
|
assert community.GROUP["admin"] is True
|
|
|
|
|
|
def test_update_saves_sync_options_and_keeps_unsent_ones(client, admin, fake_world):
|
|
add_ready(client, admin)
|
|
body = {"interval_seconds": 86400, "enabled": "true", "alias": "true", "sample_percent": 40, "topics": "devlog, rant"}
|
|
assert client.post("/admin/communities/forum-test/update", data=body, headers=admin["headers"]).status_code == 200
|
|
options = store.get("forum-test")["sync_options"]
|
|
assert options["alias"] is True and options["sample_percent"] == 40 and options["topics"] == ["devlog", "rant"]
|
|
client.post("/admin/communities/forum-test/update", data={"interval_seconds": 86400, "jitter_percent": 10}, headers=admin["headers"])
|
|
kept = store.get("forum-test")["sync_options"]
|
|
assert kept["alias"] is True and kept["jitter_percent"] == 10
|
|
|
|
|
|
def test_update_rejects_bad_options_and_late_aliasing(client, admin, fake_world):
|
|
add_ready(client, admin)
|
|
bad = client.post("/admin/communities/forum-test/update", data={"interval_seconds": 86400, "topics": "nonsense"}, headers=admin["headers"])
|
|
assert bad.status_code == 400
|
|
get_table("sync_links").insert({"uid": "late1", "origin": "forum.test", "kind": "users", "origin_uid": "1", "local_uid": "x", "fingerprint": "f", "origin_updated_at": "", "first_synced_at": "", "synced_at": ""})
|
|
late = client.post("/admin/communities/forum-test/update", data={"interval_seconds": 86400, "alias": "true"}, headers=admin["headers"])
|
|
assert late.status_code == 400
|
|
assert not store.get("forum-test")["sync_options"].get("alias")
|
|
|
|
|
|
def test_sync_options_are_admin_only(client, fake_world):
|
|
member = make_account("Member")
|
|
add_ready(client, make_account("Admin"))
|
|
assert client.get("/admin/communities/forum-test", headers=member["headers"]).status_code in (401, 403)
|
|
assert client.post("/admin/communities/forum-test/update", data={"interval_seconds": 86400, "alias": "true"}, headers=member["headers"]).status_code in (401, 403)
|