Zo doen wijdat in oost

tests/unit/services/sync/__init__.py

@@ -0,0 +1 @@
+# retoor <retoor@molodetz.nl>

tests/unit/services/sync/alias.py

@@ -0,0 +1,273 @@
+# retoor <retoor@molodetz.nl>
+
+import random
+from collections import Counter
+
+import pytest
+
+from devplacepy.database import get_table, init_db
+from devplacepy.services.sync import ai
+from devplacepy.services.sync.aliaser import AliasRefused, Aliaser
+from devplacepy.services.sync.identity import IdentityResolver, NameGenerator, fallback_name, source_key
+from devplacepy.services.sync.options import SyncOptions
+from devplacepy.services.sync.rewriter import MentionRewriter
+from devplacepy.services.sync.selection import Selector, bucket, jittered
+from devplacepy.services.sync.style import Style, plausible
+from devplacepy.services.sync.translator import Translator
+from tests.conftest import run_async
+
+ORIGIN = "alias.test"
+
+
+@pytest.fixture(autouse=True)
+def clean(monkeypatch):
+    init_db()
+    for table in ("sync_identities", "sync_links"):
+        get_table(table).delete(origin=ORIGIN)
+
+    async def unreachable(system, text, timeout):
+        return None
+
+    monkeypatch.setattr(ai, "complete", unreachable)
+    yield
+    for table in ("sync_identities", "sync_links"):
+        get_table(table).delete(origin=ORIGIN)
+
+
+def test_rewriter_longest_first_and_single_pass():
+    rewriter = MentionRewriter({"alice": "bob", "alice_w": "carol", "bob": "dave"})
+    result = rewriter.rewrite("hi @alice_w and alice, also @bob")
+    assert result.text == "hi @carol and bob, also @dave"
+    assert len(result.substitutions) == 3
+
+
+def test_rewriter_protects_code_urls_and_inline_code():
+    rewriter = MentionRewriter({"alice": "zed"})
+    text = "see `alice` and https://x.test/~alice\n```\nalice = 1\n```\nthen alice"
+    assert rewriter.rewrite(text).text == text.rsplit("alice", 1)[0] + "zed"
+    assert rewriter.rewrite("```\nalice unterminated").text == "```\nalice unterminated"
+
+
+def test_rewriter_standalone_floor_and_case_and_boundaries():
+    rewriter = MentionRewriter({"ann": "x1", "tef": "y2", "markus": "zeta"})
+    assert rewriter.rewrite("ann do, @ann, @tef, tef, Markus, markusx, a-markus").text == "ann do, @x1, @y2, tef, zeta, markusx, a-markus"
+
+
+def test_rewriter_mentioned_skips_code():
+    found = MentionRewriter({}).mentioned("hi @Alice `@code` @ab ```\n@fenced\n``` @alice_2")
+    assert found == ["Alice", "alice_2"]
+
+
+def test_style_roundtrip_and_matching():
+    style = Style.of("Alice_99")
+    assert Style.load(style.dump()) == style
+    assert style.matches("Marta_12") and not style.matches("marta_12") and not style.matches("Marta12")
+    assert not plausible("alice_99x", "Alice_99", None)
+    assert not plausible("robotuser", "zzzz", None)
+    assert not plausible("a!", "zzzz", None)
+
+
+def test_fallback_name_is_deterministic_and_style_matched():
+    for source in ("alice", "Bob_Smith", "dev99", "UPPER", "kebab-case"):
+        style = Style.of(source)
+        assert fallback_name(ORIGIN, source, style, 0) == fallback_name(ORIGIN, source, style, 0)
+        assert any(plausible(fallback_name(ORIGIN, source, style, n), source, style) for n in range(60))
+
+
+def test_generator_rejects_bad_ai_then_falls_back(monkeypatch):
+    replies = iter(['{"username": "alice"}', "garbage", '{"username": "x"}'])
+
+    async def scripted(system, text, timeout):
+        return next(replies)
+
+    monkeypatch.setattr(ai, "complete", scripted)
+    name, by_ai = run_async(NameGenerator().generate(ORIGIN, "alice", Style.of("alice"), []))
+    assert not by_ai and plausible(name, "alice", Style.of("alice"))
+
+
+def test_generator_accepts_good_ai(monkeypatch):
+    async def good(system, text, timeout):
+        assert "alice" not in text
+        return '```json\n{"username": "marta"}\n```'
+
+    monkeypatch.setattr(ai, "complete", good)
+    assert run_async(NameGenerator().generate(ORIGIN, "alice", Style.of("alice"), [])) == ("marta", True)
+
+
+def test_identity_is_permanent_unique_and_bidirectional():
+    first = IdentityResolver(ORIGIN)
+    names = [run_async(first.ensure(n)) for n in ("alice", "Alice", "bobby", "carol")]
+    assert names[0] == names[1]
+    assert len({name.lower() for name in names[:1] + names[2:]}) == 3
+    second = IdentityResolver(ORIGIN)
+    assert run_async(second.ensure("ALICE")) == names[0]
+    assert second.created == 0
+    assert second.mapping()["alice"] == names[0]
+    rewriter = MentionRewriter({value: key for key, value in second.mapping().items()})
+    assert rewriter.rewrite(f"@{names[0]}").text == "@alice"
+
+
+def test_identity_cap_and_invalid_names_fail_closed():
+    resolver = IdentityResolver(ORIGIN, cap=1)
+    assert run_async(resolver.ensure("alice"))
+    assert run_async(resolver.ensure("bobby")) is None
+    assert run_async(resolver.ensure("not valid!")) is None
+    assert source_key("x" * 100) is None
+
+
+def test_dry_run_resolver_persists_nothing():
+    resolver = IdentityResolver(ORIGIN, persist=False)
+    assert run_async(resolver.ensure("alice"))
+    assert get_table("sync_identities").count(origin=ORIGIN) == 0
+
+
+def test_samples_are_capped():
+    resolver = IdentityResolver(ORIGIN)
+    run_async(resolver.ensure("alice"))
+    for n in range(6):
+        resolver.remember("alice", f"sample number {n}")
+    assert len(resolver.samples("alice")) == 3
+    assert len(IdentityResolver(ORIGIN).samples("alice")) == 3
+
+
+def records():
+    users = [
+        {"origin_uid": "1", "username": "alice", "bio": "secret bio", "website": "https://alice.test", "created_at": "2025-01-01T00:00:00Z"},
+        {"origin_uid": "2", "username": "bobby", "created_at": "2025-01-02T00:00:00Z"},
+    ]
+    posts = [
+        {"origin_uid": "p1", "author_uid": "1", "title": "Hello from alice", "content": "ping @bobby and @ghostly please", "created_at": "2025-02-01T00:00:00Z", "topic": "devlog"},
+        {"origin_uid": "p2", "author_uid": "9", "title": "orphan", "content": "no known author here", "created_at": "2025-02-02T00:00:00Z"},
+    ]
+    comments = [
+        {"origin_uid": "c1", "author_uid": "2", "post_uid": "p1", "content": "thanks alice", "created_at": "2025-02-03T00:00:00Z"},
+        {"origin_uid": "c2", "author_uid": "2", "post_uid": "p1", "parent_uid": "cX", "content": "reply to missing", "created_at": "2025-02-03T00:00:00Z"},
+    ]
+    return {"users": users, "posts": posts, "comments": comments}
+
+
+def leaked(outcome, names=("alice", "bobby")):
+    blob = " ".join(
+        str(item.get(name) or "")
+        for kind in ("users", "posts", "comments")
+        for item in outcome.records[kind]
+        for name in ("username", "title", "content", "bio", "website")
+    ).lower()
+    return [name for name in names if name in blob]
+
+
+def test_aliaser_rewrites_everything_and_fails_closed():
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records()))
+    assert leaked(outcome) == []
+    assert len(outcome.records["posts"]) == 1
+    assert outcome.skipped["no_author"] == 1
+    assert [c["origin_uid"] for c in outcome.records["comments"]] == ["c1"]
+    assert outcome.skipped["parent_skipped"] == 1
+    users = outcome.records["users"]
+    assert len(users) == 3
+    assert all(set(user) == {"origin_uid", "username", "created_at", "deleted"} for user in users)
+    assert {user["origin_uid"] for user in users} == {"alice", "bobby", "ghostly"}
+    assert outcome.records["posts"][0]["author_uid"] == "alice"
+    assert outcome.substitutions >= 4
+
+
+def test_aliaser_is_stable_across_runs():
+    first = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records()))
+    second = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records()))
+    assert first.records == second.records
+    assert second.identities_created == 0
+
+
+def test_aliaser_refuses_origin_with_original_links():
+    get_table("sync_links").insert({"uid": "l1", "origin": ORIGIN, "kind": "users", "origin_uid": "1", "local_uid": "x", "fingerprint": "f", "origin_updated_at": "", "first_synced_at": "", "synced_at": ""})
+    with pytest.raises(AliasRefused):
+        run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(records()))
+
+
+def test_aliaser_cap_skips_item_as_transient(monkeypatch):
+    resolver = IdentityResolver(ORIGIN, cap=1)
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True), resolver=resolver).apply(records()))
+    assert outcome.transient() >= 1
+    assert leaked(outcome) == []
+
+
+class FakeTranslator(Translator):
+    def __init__(self, reply):
+        super().__init__("nl")
+        self.reply = reply
+
+    async def translate(self, text, samples=None):
+        return self.reply(text) if self.reply else None
+
+
+def test_translation_failure_skips_item_and_holds():
+    translator = FakeTranslator(None)
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True, translate_to="nl"), language="en", translator=translator).apply(records()))
+    assert outcome.records["posts"] == [] and outcome.skipped["translation_failed"] == 1
+    assert outcome.transient() >= 1
+
+
+def test_translation_output_is_rewritten_again():
+    translator = FakeTranslator(lambda text: text + " (met @alice)")
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True, translate_to="nl"), language="en", translator=translator).apply(records()))
+    assert leaked(outcome) == []
+    assert outcome.translated >= 1
+
+
+def test_translation_skipped_when_language_matches():
+    assert not Translator("en").needed("en-US")
+    assert Translator("en").needed("") and Translator("nl").needed("en")
+
+
+def test_options_validation_and_roundtrip():
+    options = SyncOptions.build(alias=True, translate_to="NL", sample_percent=40, min_comments=2, topics="devlog, rant", jitter_percent=10)
+    assert SyncOptions.from_row({"sync_options": options.to_dict()}) == options
+    assert not SyncOptions().active()
+    for bad in ({"sample_percent": 0}, {"jitter_percent": 99}, {"topics": "nonsense"}, {"translate_to": "x y"}, {"min_comments": -1}):
+        with pytest.raises(ValueError):
+            SyncOptions.build(**bad)
+    assert options.merged(alias=False).alias is False and options.merged(alias=None).alias is True
+
+
+def test_selector_deterministic_sampling_and_filters():
+    posts = [{"origin_uid": f"p{n}", "author_uid": "1", "topic": "devlog"} for n in range(400)]
+    comments = [{"origin_uid": "c", "author_uid": "1", "post_uid": "p0"}]
+    data = {"users": [], "posts": posts, "comments": comments}
+    options = SyncOptions(sample_percent=50)
+    kept, skipped = Selector(ORIGIN, options).select(data)
+    again, _ = Selector(ORIGIN, options).select(data)
+    assert kept["posts"] == again["posts"]
+    assert 150 < len(kept["posts"]) < 250
+    assert skipped["sampled_out"] == 400 - len(kept["posts"])
+    assert bucket(ORIGIN, "p0") == bucket(ORIGIN, "p0")
+    kept, skipped = Selector(ORIGIN, SyncOptions(topics=("rant",))).select(data)
+    assert kept["posts"] == [] and kept["comments"] == [] and skipped["topic_not_allowed"] == 400
+    kept, skipped = Selector(ORIGIN, SyncOptions(min_comments=1)).select(data)
+    assert [post["origin_uid"] for post in kept["posts"]] == ["p0"]
+    untouched, none = Selector(ORIGIN, SyncOptions()).select(data)
+    assert untouched is data and not none
+
+
+def test_selector_never_drops_already_synced_posts():
+    get_table("sync_links").insert({"uid": "l2", "origin": ORIGIN, "kind": "posts", "origin_uid": "p0", "local_uid": "x", "fingerprint": "f", "origin_updated_at": "", "first_synced_at": "", "synced_at": ""})
+    data = {"users": [], "posts": [{"origin_uid": "p0", "author_uid": "1", "topic": "devlog"}], "comments": []}
+    kept, _ = Selector(ORIGIN, SyncOptions(topics=("rant",))).select(data)
+    assert len(kept["posts"]) == 1
+
+
+def test_jitter_is_bounded():
+    rng = random.Random(1)
+    values = [jittered(1000, 20, rng) for _ in range(500)]
+    assert min(values) >= 800 and max(values) <= 1200 and len(set(values)) > 50
+    assert jittered(1000, 0) == 1000
+    assert Counter() == Counter()
+
+
+def test_mentions_are_case_insensitive_end_to_end():
+    rewriter = MentionRewriter({"Alice": "zed"})
+    assert rewriter.rewrite("@ALICE @alice @AlIcE and ALICE").text == "@zed @zed @zed and zed"
+    data = records()
+    data["posts"][0]["content"] = "ping @BOBBY and @Bobby and @bObBy"
+    outcome = run_async(Aliaser(ORIGIN, SyncOptions(alias=True)).apply(data))
+    assert leaked(outcome) == []
+    assert len(outcome.records["users"]) == 2 + 0 or {u["origin_uid"] for u in outcome.records["users"]} >= {"alice", "bobby"}

tests/unit/services/sync/normalize.py

@@ -0,0 +1,97 @@
+# retoor <retoor@molodetz.nl>
+
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+from devplacepy.services.sync.normalize import (
+    clamp_instant,
+    clean_text,
+    fingerprint,
+    instant_iso,
+    normalize_origin,
+    normalize_origin_uid,
+    normalize_username,
+    parse_instant,
+    username_candidates,
+)
+from devplacepy.services.sync.spec import LIMITS
+
+
+@pytest.mark.parametrize(
+    "raw,expected",
+    [
+        ("Other.Example.ORG", "other.example.org"),
+        ("  a1  ", "a1"),
+        ("devplace-net", "devplace-net"),
+    ],
+)
+def test_origin_is_normalized(raw, expected):
+    assert normalize_origin(raw) == expected
+
+
+@pytest.mark.parametrize("raw", ["", "a", "-bad", "bad-", "a..b", "has space", "x" * 64, "ünï.org"])
+def test_origin_is_rejected(raw):
+    with pytest.raises(ValueError):
+        normalize_origin(raw)
+
+
+@pytest.mark.parametrize("raw", ["", "  ", "a b", "x" * 65, "a/b"])
+def test_origin_uid_is_rejected(raw):
+    with pytest.raises(ValueError):
+        normalize_origin_uid(raw)
+
+
+def test_origin_uid_accepts_integers_and_uuids():
+    assert normalize_origin_uid(42) == "42"
+    assert normalize_origin_uid("01a139e9-a2f0-7fa1-8635-1a98fbf95e3d")
+
+
+def test_clean_text_strips_control_characters_and_newlines():
+    assert clean_text("a\x00b\r\nc\rd\x07", multiline=True) == "ab\nc\nd"
+    assert clean_text("a \n  b\t c", multiline=False) == "a b c"
+
+
+def test_username_always_valid_across_inputs():
+    samples = ["", "a", "ab", "Ünï cøde!", "x" * 80, "--__--", "john.doe@mail", "日本語", "ok_name-1"]
+    for sample in samples:
+        name = normalize_username(sample, "uid-1")
+        assert LIMITS.min_username_length <= len(name) <= LIMITS.max_username_length
+        assert all(ch.isalnum() or ch in "_-" for ch in name) and name.isascii()
+
+
+def test_username_is_stable_for_same_input():
+    assert normalize_username("a", "u1") == normalize_username("a", "u1")
+    assert normalize_username("a", "u1") != normalize_username("a", "u2")
+
+
+def test_username_candidates_are_distinct_and_bounded():
+    seen = []
+    for candidate in username_candidates("x" * 32, "other.example.org"):
+        seen.append(candidate)
+        assert len(candidate) <= LIMITS.max_username_length
+        if len(seen) == 50:
+            break
+    assert len(set(seen)) == 50
+    assert seen[1].endswith("-other")
+
+
+def test_instants_are_utc_iso_and_clamped():
+    assert instant_iso("2025-01-01T01:00:00+01:00") == "2025-01-01T00:00:00+00:00"
+    assert instant_iso("2025-01-01").startswith("2025-01-01T00:00:00")
+    assert parse_instant(1700000000).year == 2023
+    now = datetime.now(timezone.utc)
+    future = now + timedelta(days=30)
+    assert clamp_instant(future, now) == now
+    assert instant_iso("1970-01-01").startswith("2000-01-01")
+
+
+@pytest.mark.parametrize("raw", ["", None, "nope", True, "2025-13-45"])
+def test_instant_rejects_garbage(raw):
+    with pytest.raises(ValueError):
+        parse_instant(raw)
+
+
+def test_fingerprint_ignores_key_order_and_detects_change():
+    assert fingerprint({"a": 1, "b": 2}) == fingerprint({"b": 2, "a": 1})
+    assert fingerprint({"a": 1}) != fingerprint({"a": 2})

tests/unit/services/sync/records.py

@@ -0,0 +1,85 @@
+# retoor <retoor@molodetz.nl>
+
+import pytest
+from pydantic import ValidationError
+
+from devplacepy.services.sync.records import (
+    CommentRecord,
+    PostRecord,
+    SyncBatchForm,
+    UserRecord,
+)
+from devplacepy.services.sync.spec import LIMITS
+
+BASE = {"origin_uid": "r1", "created_at": "2025-01-01T00:00:00Z"}
+
+
+def test_updated_at_defaults_to_created_and_never_precedes_it():
+    record = PostRecord.model_validate(
+        {**BASE, "author_uid": "u", "content": "ten chars!!"}
+    )
+    assert record.updated_at == record.created_at
+    earlier = PostRecord.model_validate(
+        {**BASE, "author_uid": "u", "content": "ten chars!!", "updated_at": "2024-01-01"}
+    )
+    assert earlier.updated_at == earlier.created_at
+
+
+def test_unknown_topic_becomes_random_and_known_is_kept():
+    base = {**BASE, "author_uid": "u", "content": "ten chars!!"}
+    assert PostRecord.model_validate({**base, "topic": "zzz"}).topic == "random"
+    assert PostRecord.model_validate({**base, "topic": "DevLog"}).topic == "devlog"
+
+
+@pytest.mark.parametrize(
+    "payload",
+    [
+        {"author_uid": "u", "content": "short"},
+        {"author_uid": "u", "content": "x" * (LIMITS.max_post_length + 1)},
+        {"author_uid": "", "content": "ten chars!!"},
+        {"content": "ten chars!!"},
+        {"author_uid": "u", "content": "ten chars!!", "title": "t" * (LIMITS.max_title_length + 1)},
+    ],
+)
+def test_invalid_posts_are_rejected(payload):
+    with pytest.raises(ValidationError):
+        PostRecord.model_validate({**BASE, **payload})
+
+
+def test_comment_parent_is_optional_and_post_is_required():
+    base = {**BASE, "author_uid": "u", "content": "fine"}
+    assert CommentRecord.model_validate({**base, "post_uid": "p"}).parent_uid is None
+    assert CommentRecord.model_validate({**base, "post_uid": "p", "parent_uid": ""}).parent_uid is None
+    with pytest.raises(ValidationError):
+        CommentRecord.model_validate(base)
+
+
+def test_user_urls_are_fixed_or_dropped():
+    record = UserRecord.model_validate(
+        {**BASE, "username": "bob", "website": "example.com", "git_link": "not a url"}
+    )
+    assert record.website == "https://example.com"
+    assert record.git_link == ""
+
+
+def test_user_bio_over_limit_is_rejected():
+    with pytest.raises(ValidationError):
+        UserRecord.model_validate({**BASE, "username": "bob", "bio": "x" * (LIMITS.max_bio_length + 1)})
+
+
+def test_user_counters_and_credentials_are_not_fields():
+    known = UserRecord.known_fields()
+    for name in ("email", "password_hash", "role", "xp", "level", "stars", "api_key"):
+        assert name not in known
+    assert "stars" not in PostRecord.known_fields()
+
+
+def test_envelope_normalizes_origin_and_bounds_batch():
+    form = SyncBatchForm(origin="Other.ORG", records=[{}])
+    assert form.origin == "other.org"
+    with pytest.raises(ValidationError):
+        SyncBatchForm(origin="other.org", records=[])
+    with pytest.raises(ValidationError):
+        SyncBatchForm(origin="other.org", records=[{}] * (LIMITS.max_batch_records + 1))
+    with pytest.raises(ValidationError):
+        SyncBatchForm(origin="bad origin", records=[{}])