93 lines
4.0 KiB
Python
93 lines
4.0 KiB
Python
 
|
# retoor <retoor@molodetz.nl>
|
||
|
|
|
||
|
|
import json
|
||
|
|
|
||
|
|
from devplacepy.services.community.generator import CodeGenerator
|
||
|
|
from devplacepy.services.community.runtime import Collector
|
||
|
|
from tests.conftest import run_async
|
||
|
|
from tests.unit.services.community.fakes import FakeFetcher
|
||
|
|
from tests.unit.services.community.generator import load, no_sleep
|
||
|
|
|
||
|
|
|
||
|
|
def feed(identifier, url, comments="none"):
|
||
|
|
return {
|
||
|
|
"id": identifier,
|
||
|
|
"url": url,
|
||
|
|
"type": "json_api",
|
||
|
|
"list_path": "$.items",
|
||
|
|
"post": {"fields": {"id": ["$.id"], "title": ["$.title"], "content": ["$.body"], "created_at": ["$.at"], "url": ["$.link"]}},
|
||
|
|
"author": {"fields": {"username": ["$.who"]}},
|
||
|
|
"comments": {"mode": comments},
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def schema(limits=None):
|
||
|
|
return {
|
||
|
|
"community": "twins",
|
||
|
|
"origin": "twins.test",
|
||
|
|
"limits": {"delay_seconds": 0, "retries": 1, **(limits or {})},
|
||
|
|
"feeds": [feed("latest", "https://twins.test/latest"), feed("top", "https://twins.test/top")],
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def entry(identifier, link, title="A shared story", at="2026-10-01T10:00:00Z"):
|
||
|
|
return {"id": identifier, "title": title, "body": "a body that is long enough to sync", "at": at, "who": "alice", "link": link}
|
||
|
|
|
||
|
|
|
||
|
|
def routes(latest, top):
|
||
|
|
return {"https://twins.test/latest": {"items": latest}, "https://twins.test/top": {"items": top}}
|
||
|
|
|
||
|
|
|
||
|
|
def collect(fetcher_routes, state=None, only=None, limits=None):
|
||
|
|
return run_async(Collector(FakeFetcher(fetcher_routes), schema(limits), state, sleep=no_sleep).run(only))
|
||
|
|
|
||
|
|
|
||
|
|
def uids(result):
|
||
|
|
return {record["origin_uid"] for record in result["records"]["posts"]}
|
||
|
|
|
||
|
|
|
||
|
|
def test_the_same_story_in_a_second_feed_in_a_later_run_is_not_a_second_post():
|
||
|
|
shared = "https://news.test/story/42?utm=a"
|
||
|
|
sources = routes([entry("L1", shared)], [entry("T9", "https://www.news.test/story/42/?utm=b")])
|
||
|
|
first = collect(sources, only="latest")
|
||
|
|
assert len(first["records"]["posts"]) == 1 and first["state"]["links"]
|
||
|
|
second = collect(sources, state=json.loads(json.dumps(first["state"])), only="top")
|
||
|
|
assert second["records"]["posts"] == []
|
||
|
|
|
||
|
|
|
||
|
|
def test_without_the_remembered_map_the_twin_would_become_a_post():
|
||
|
|
shared = "https://news.test/story/42"
|
||
|
|
sources = routes([entry("L1", shared)], [entry("T9", shared)])
|
||
|
|
first = collect(sources, only="latest")
|
||
|
|
assert len(collect(sources, state={"feeds": first["state"]["feeds"]}, only="top")["records"]["posts"]) == 1
|
||
|
|
|
||
|
|
|
||
|
|
def test_one_feed_never_merges_its_own_posts_that_share_a_link():
|
||
|
|
shared = "https://news.test/root"
|
||
|
|
result = collect(routes([entry("A", shared, "first"), entry("B", shared, "second")], []))
|
||
|
|
assert len(result["records"]["posts"]) == 2
|
||
|
|
|
||
|
|
|
||
|
|
def test_the_same_feed_keeps_its_own_post_when_it_changes_in_a_later_run():
|
||
|
|
shared = "https://news.test/story/7"
|
||
|
|
first = collect(routes([entry("L1", shared)], []))
|
||
|
|
changed = routes([entry("L1", shared, "A shared story, edited", "2026-10-05T10:00:00Z")], [])
|
||
|
|
again = collect(changed, state=first["state"])
|
||
|
|
assert uids(again) == uids(first) and again["records"]["posts"][0]["title"].endswith("edited")
|
||
|
|
|
||
|
|
|
||
|
|
def test_the_remembered_map_is_bounded_and_keeps_the_newest():
|
||
|
|
items = [entry(f"L{number}", f"https://news.test/story/{number}") for number in range(40)]
|
||
|
|
result = collect(routes(items, []), limits={"max_remembered_links": 10})
|
||
|
|
kept = list(result["state"]["links"])
|
||
|
|
assert len(kept) == 10 and kept[-1].endswith("/story/39") and not any(key.endswith("/story/0") for key in kept)
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_generated_collector_remembers_links_too(tmp_path):
|
||
|
|
module = load(CodeGenerator(schema()).write(tmp_path))
|
||
|
|
shared = "https://news.test/story/42"
|
||
|
|
sources = routes([entry("L1", shared)], [entry("T9", shared)])
|
||
|
|
first = run_async(module.collect_all(FakeFetcher(sources), only="latest", sleep=no_sleep))
|
||
|
|
second = run_async(module.collect_all(FakeFetcher(sources), state=json.loads(json.dumps(first["state"])), only="top", sleep=no_sleep))
|
||
|
|
assert len(first["records"]["posts"]) == 1 and second["records"]["posts"] == []
|