# retoor import random from collections import Counter import pytest from devplacepy.database import get_table, init_db from devplacepy.services.sync import ai from devplacepy.services.sync.aliaser import AliasRefused, Aliaser from devplacepy.services.sync.identity import IdentityResolver, NameGenerator, fallback_name, source_key from devplacepy.services.sync.options import SyncOptions from devplacepy.services.sync.rewriter import MentionRewriter from devplacepy.services.sync.selection import Selector, bucket, jittered from devplacepy.services.sync.style import Style, plausible from devplacepy.services.sync.translator import Translator from tests.conftest import run_async ORIGIN = "alias.test" @pytest.fixture(autouse=True) def clean(monkeypatch): init_db() for table in ("sync_identities", "sync_links"): get_table(table).delete(origin=ORIGIN) async def unreachable(system, text, timeout): return None monkeypatch.setattr(ai, "complete", unreachable) yield for table in ("sync_identities", "sync_links"): get_table(table).delete(origin=ORIGIN) def test_rewriter_longest_first_and_single_pass(): rewriter = MentionRewriter({"alice": "bob", "alice_w": "carol", "bob": "dave"}) result = rewriter.rewrite("hi @alice_w and alice, also @bob") assert result.text == "hi @carol and bob, also @dave" assert len(result.substitutions) == 3 def test_rewriter_protects_code_urls_and_inline_code(): rewriter = MentionRewriter({"alice": "zed"}) text = "see `alice` and https://x.test/~alice\n```\nalice = 1\n```\nthen alice" assert rewriter.rewrite(text).text == text.rsplit("alice", 1)[0] + "zed" assert rewriter.rewrite("```\nalice unterminated").text == "```\nalice unterminated" def test_rewriter_standalone_floor_and_case_and_boundaries(): rewriter = MentionRewriter({"ann": "x1", "tef": "y2", "markus": "zeta"}) assert rewriter.rewrite("ann do, @ann, @tef, tef, Markus, markusx, a-markus").text == "ann do, @x1, @y2, tef, zeta, markusx, a-markus" def test_rewriter_mentioned_skips_code(): found = MentionRewriter({}).mentioned("hi @Alice `@code` @ab ```\n@fenced\n``` @alice_2") assert found == ["Alice", "alice_2"] def test_style_roundtrip_and_matching(): style = Style.of("Alice_99") assert Style.load(style.dump()) == style assert style.matches("Marta_12") and not style.matches("marta_12") and not style.matches("Marta12") assert not plausible("alice_99x", "Alice_99", None) assert not plausible("robotuser", "zzzz", None) assert not plausible("a!", "zzzz", None) def test_fallback_name_is_deterministic_and_style_matched(): for source in ("alice", "Bob_Smith", "dev99", "UPPER", "kebab-case"): style = Style.of(source) assert fallback_name(ORIGIN, source, style, 0) == fallback_name(ORIGIN, source, style, 0) assert any(plausible(fallback_name(ORIGIN, source, style, n), source, style) for n in range(60)) def test_generator_rejects_bad_ai_then_falls_back(monkeypatch): replies = iter(['{"username": "alice"}', "garbage", '{"username": "x"}']) async def scripted(system, text, timeout): return next(replies) monkeypatch.setattr(ai, "complete", scripted) name, by_ai = run_async(NameGenerator().generate(ORIGIN, "alice", Style.of("alice"), [])) assert not by_ai and plausible(name, "alice", Style.of("alice")) def test_generator_accepts_good_ai(monkeypatch): async def good(system, text, timeout): assert "alice" not in text return '```json\n{"username": "marta"}\n```' monkeypatch.setattr(ai, "complete", good) assert run_async(NameGenerator().generate(ORIGIN, "alice", Style.of("alice"), [])) == ("marta", True) def test_identity_is_permanent_unique_and_bidirectional(): first = IdentityResolver(ORIGIN) names = [run_async(first.ensure(n)) for n in ("alice", "Alice", "bobby", "carol")] assert names[0] == names[1] assert len({name.lower() for name in names[:1] + names[2:]}) == 3 second = IdentityResolver(ORIGIN) assert run_async(second.ensure("ALICE")) == names[0] assert second.created == 0 assert second.mapping()["alice"] == names[0] rewriter = MentionRewriter({value: key for key, value in second.mapping().items()}) assert rewriter.rewrite(f"@{names[0]}").text == "@alice" def test_identity_cap_and_invalid_names_fail_closed(): resolver = IdentityResolver(ORIGIN, cap=1) assert run_async(resolver.ensure("alice")) assert run_async(resolver.ensure("bobby")) is None assert run_async(resolver.ensure("not valid!")) is None assert source_key("x" * 100) is None def test_dry_run_resolver_persists_nothing(): resolver = IdentityResolver(ORIGIN, persist=False) assert run_async(resolver.ensure("alice")) assert get_table("sync_identities").count(origin=ORIGIN) == 0 def test_samples_are_capped(): resolver = IdentityResolver(ORIGIN) run_async(resolver.ensure("alice")) for n in range(6): resolver.remember("alice", f"sample number {n}") assert len(resolver.samples("alice")) == 3 assert len(IdentityResolver(ORIGIN).samples("alice")) == 3 def records(): users = [ {"origin_uid": "1", "username": "alice", "bio": "secret bio", "website": "https://alice.test", "created_at": "2025-01-01T00:00:00Z"}, {"origin_uid": "2", "username": "bobby", "created_at": "2025-01-02T00:00:00Z"}, ] posts = [ {"origin_uid": "p1", "author_uid": "1", "title": "Hello from alice", "content": "ping @bobby and @ghostly please", "created_at": "2025-02-01T00:00:00Z", "topic": "devlog"}, {"origin_uid": "p2", "author_uid": "9", "title": "orphan", "content": "no known author here", "created_at": "2025-02-02T00:00:00Z"}, ] comments = [ {"origin_uid": "c1", "author_uid": "2", "post_uid": "p1", "content": "thanks alice", "created_at": "2025-02-03T00:00:00Z"}, {"origin_uid": "c2", "author_uid": "2", "post_uid": "p1", "parent_uid": "cX", "content": "reply to missing", "created_at": "2025-02-03T00:00:00Z"}, ] return {"users": users, "posts": posts, "comments": comments} def leaked(outcome, names=("alice", "bobby")): blob = " ".join( str(item.get(name) or "") for kind in ("users", "posts", "comments") for item in outcome.records[kind] for name in ("username", "title", "content", "bio", "website") ).lower() return [name for name in names if name in blob] def test_aliaser_rewrites_everything_and_fails_closed(): outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records())) assert leaked(outcome) == [] assert len(outcome.records["posts"]) == 1 assert outcome.skipped["no_author"] == 1 assert [c["origin_uid"] for c in outcome.records["comments"]] == ["c1"] assert outcome.skipped["parent_skipped"] == 1 users = outcome.records["users"] assert len(users) == 3 assert all(set(user) == {"origin_uid", "username", "created_at", "deleted"} for user in users) assert {user["origin_uid"] for user in users} == {"alice", "bobby", "ghostly"} assert outcome.records["posts"][0]["author_uid"] == "alice" assert outcome.substitutions >= 4 def test_aliaser_is_stable_across_runs(): first = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records())) second = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records())) assert first.records == second.records assert second.identities_created == 0 def test_aliaser_refuses_origin_with_original_links(): get_table("sync_links").insert({"uid": "l1", "origin": ORIGIN, "kind": "users", "origin_uid": "1", "local_uid": "x", "fingerprint": "f", "origin_updated_at": "", "first_synced_at": "", "synced_at": ""}) with pytest.raises(AliasRefused): run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records())) def test_aliaser_cap_skips_item_as_transient(monkeypatch): resolver = IdentityResolver(ORIGIN, cap=1) outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True), resolver=resolver).apply(records())) assert outcome.transient() >= 1 assert leaked(outcome) == [] class FakeTranslator(Translator): def __init__(self, reply): super().__init__("nl") self.reply = reply async def translate(self, text, samples=None): return self.reply(text) if self.reply else None def test_translation_failure_skips_item_and_holds(): translator = FakeTranslator(None) outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True, translate_to="nl"), language="en", translator=translator).apply(records())) assert outcome.records["posts"] == [] and outcome.skipped["translation_failed"] == 1 assert outcome.transient() >= 1 def test_translation_output_is_rewritten_again(): translator = FakeTranslator(lambda text: text + " (met @alice)") outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True, translate_to="nl"), language="en", translator=translator).apply(records())) assert leaked(outcome) == [] assert outcome.translated >= 1 def test_translation_skipped_when_language_matches(): assert not Translator("en").needed("en-US") assert Translator("en").needed("") and Translator("nl").needed("en") def test_options_validation_and_roundtrip(): options = SyncOptions.build(alias=True, translate_to="NL", sample_percent=40, min_comments=2, topics="devlog, rant", jitter_percent=10) assert SyncOptions.from_row({"sync_options": options.to_dict()}) == options assert not SyncOptions().active() for bad in ({"sample_percent": 0}, {"jitter_percent": 99}, {"topics": "nonsense"}, {"translate_to": "x y"}, {"min_comments": -1}): with pytest.raises(ValueError): SyncOptions.build(**bad) assert options.merged(alias=False).alias is False and options.merged(alias=None).alias is True def test_selector_deterministic_sampling_and_filters(): posts = [{"origin_uid": f"p{n}", "author_uid": "1", "topic": "devlog"} for n in range(400)] comments = [{"origin_uid": "c", "author_uid": "1", "post_uid": "p0"}] data = {"users": [], "posts": posts, "comments": comments} options = SyncOptions(sample_percent=50) kept, skipped = Selector(ORIGIN, options).select(data) again, _ = Selector(ORIGIN, options).select(data) assert kept["posts"] == again["posts"] assert 150 < len(kept["posts"]) < 250 assert skipped["sampled_out"] == 400 - len(kept["posts"]) assert bucket(ORIGIN, "p0") == bucket(ORIGIN, "p0") kept, skipped = Selector(ORIGIN, SyncOptions(topics=("rant",))).select(data) assert kept["posts"] == [] and kept["comments"] == [] and skipped["topic_not_allowed"] == 400 kept, skipped = Selector(ORIGIN, SyncOptions(min_comments=1)).select(data) assert [post["origin_uid"] for post in kept["posts"]] == ["p0"] untouched, none = Selector(ORIGIN, SyncOptions()).select(data) assert untouched is data and not none def test_selector_never_drops_already_synced_posts(): get_table("sync_links").insert({"uid": "l2", "origin": ORIGIN, "kind": "posts", "origin_uid": "p0", "local_uid": "x", "fingerprint": "f", "origin_updated_at": "", "first_synced_at": "", "synced_at": ""}) data = {"users": [], "posts": [{"origin_uid": "p0", "author_uid": "1", "topic": "devlog"}], "comments": []} kept, _ = Selector(ORIGIN, SyncOptions(topics=("rant",))).select(data) assert len(kept["posts"]) == 1 def test_jitter_is_bounded(): rng = random.Random(1) values = [jittered(1000, 20, rng) for _ in range(500)] assert min(values) >= 800 and max(values) <= 1200 and len(set(values)) > 50 assert jittered(1000, 0) == 1000 assert Counter() == Counter() def test_mentions_are_case_insensitive_end_to_end(): rewriter = MentionRewriter({"Alice": "zed"}) assert rewriter.rewrite("@ALICE @alice @AlIcE and ALICE").text == "@zed @zed @zed and zed" data = records() data["posts"][0]["content"] = "ping @BOBBY and @Bobby and @bObBy" outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(data)) assert leaked(outcome) == [] assert len(outcome.records["users"]) == 2 + 0 or {u["origin_uid"] for u in outcome.records["users"]} >= {"alice", "bobby"}