Code Farm: per-plot auto-replant pause and crop-lock check; community sync and node work

Continuous Delivery could never be turned off once a crop was planted. Add a
one-shot per-plot "stop after harvest" flag (POST /game/plot/stop, Devii tool
game_plot_stop, game_plots.stop_after_harvest), cleared by the next plant,
autoreplant claim or Refactor. Autoreplant now also respects crop locks through
the shared _crop_lock predicate. Plan and measurements in happyplanting.md.

Also includes the pending community sync, node network and related work in the
tree; the full suite passed (4999) with all of it.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

tests/api/admin/communities/index.py

@@ -209,6 +209,27 @@ def test_password_is_stored_but_never_returned(client, admin, fake_world):
     assert store.get("forum-test")["login_password"] == ""
 
 
+def test_the_authenticator_secret_is_stored_but_never_returned(client, admin, fake_world):
+    client.post(
+        "/admin/communities",
+        data={"query": "https://forum.test/api/posts", "login_username": "bot", "login_password": "s3cret-pw", "login_totp_secret": "JBSWY3DPEHPK3PXP"},
+        headers=admin["headers"],
+    )
+    row = wait_for(lambda: (found := store.get("forum-test")) and found["status"] == "ready" and found)
+    assert row["login_totp_secret"] == "JBSWY3DPEHPK3PXP"
+    assert store.credentials(row)["totp"] == "JBSWY3DPEHPK3PXP"
+    for url in ("/admin/communities", "/admin/communities/forum-test", "/admin/communities/forum-test/ledger"):
+        for headers in (admin["headers"], {"X-API-KEY": admin["headers"]["X-API-KEY"]}):
+            body = client.get(url, headers=headers)
+            assert "JBSWY3DPEHPK3PXP" not in body.text
+    as_json = client.get("/admin/communities/forum-test", headers=admin["headers"]).json()
+    assert as_json["community"]["has_totp"] is True and "login_totp_secret" not in as_json["community"]
+    client.post("/admin/communities/forum-test/update", data={"interval_seconds": "7200", "login_username": "other"}, headers=admin["headers"])
+    assert store.get("forum-test")["login_totp_secret"] == "JBSWY3DPEHPK3PXP"
+    client.post("/admin/communities/forum-test/update", data={"interval_seconds": "7200", "clear_login": "on"}, headers=admin["headers"])
+    assert store.get("forum-test")["login_totp_secret"] == ""
+
+
 def test_devii_tools_are_admin_only_and_delete_needs_confirmation():
     from devplacepy.services.devii.actions.catalog.community import COMMUNITY_ACTIONS
     from devplacepy.services.devii.actions.dispatcher import CONFIRM_REQUIRED

tests/api/admin/communities/schema.py

@@ -0,0 +1,97 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+from devplacepy.database import get_table
+from devplacepy.services.community.files import files
+from devplacepy.services.community.store import store
+from tests.api.admin.communities.index import add_ready, admin, client, fake_world, make_account  # noqa: F401
+
+
+def save(client, admin, schema_text, slug="forum-test"):
+    return client.post(f"/admin/communities/{slug}/schema", data={"schema": schema_text}, headers=admin["headers"])
+
+
+def test_authz(client, admin, fake_world):
+    add_ready(client, admin)
+    member = make_account()
+    assert client.get("/admin/communities/forum-test/schema", headers={"Accept": "application/json"}).status_code in (401, 303)
+    assert client.get("/admin/communities/forum-test/schema", headers=member["headers"]).status_code == 403
+    assert client.post("/admin/communities/forum-test/schema", data={"schema": "{}"}, headers=member["headers"]).status_code == 403
+    assert client.post("/admin/communities/forum-test/schema/restore", headers=member["headers"]).status_code == 403
+
+
+def test_the_page_shows_the_schema_and_never_a_credential(client, admin, fake_world):
+    add_ready(client, admin)
+    store.update(store.get("forum-test")["uid"], login_password="topsecret-pw", login_totp_secret="TOTPSECRETVALUE")
+    body = client.get("/admin/communities/forum-test/schema", headers=admin["headers"]).json()
+    assert json.loads(body["schema_text"])["origin"] == "forum.test" and body["has_previous"] is False
+    page = client.get("/admin/communities/forum-test/schema", headers={"X-API-KEY": admin["headers"]["X-API-KEY"]})
+    assert page.status_code == 200 and "community-schema" in page.text
+    assert "topsecret-pw" not in page.text and "TOTPSECRETVALUE" not in page.text
+    assert "Edit schema" in client.get("/admin/communities/forum-test", headers={"X-API-KEY": admin["headers"]["X-API-KEY"]}).text
+
+
+def test_a_verified_schema_is_stored_regenerated_and_the_old_one_kept(client, admin, fake_world):
+    row = add_ready(client, admin)
+    schema = json.loads(json.dumps(row["schema"]))
+    schema["feeds"][0]["post"]["fields"]["title"] = ["$.title", "$.id"]
+    response = save(client, admin, json.dumps(schema))
+    assert response.status_code == 200, response.text
+    assert response.json()["data"]["posts"] >= 1
+    after = store.get("forum-test")
+    assert after["schema"]["feeds"][0]["post"]["fields"]["title"] == ["$.title", "$.id"]
+    assert after["schema_previous"] == row["schema"] and after["status"] == "ready"
+    assert files.load("forum-test", after["schema"]).SCHEMA["feeds"][0]["post"]["fields"]["title"] == ["$.title", "$.id"]
+    rows = list(get_table("audit_log").find(event_key="admin.community.schema", target_label="forum-test"))
+    assert rows
+
+
+def test_a_structurally_invalid_schema_is_refused_with_every_reason_and_changes_nothing(client, admin, fake_world):
+    row = add_ready(client, admin)
+    broken = json.loads(json.dumps(row["schema"]))
+    broken["feeds"][0]["type"] = "soap"
+    broken["origin"] = "NOT A HOST"
+    response = save(client, admin, json.dumps(broken))
+    assert response.status_code == 400
+    assert store.get("forum-test")["schema"] == row["schema"]
+    html = client.post("/admin/communities/forum-test/schema", data={"schema": json.dumps(broken)}, headers={"X-API-KEY": admin["headers"]["X-API-KEY"]})
+    assert html.status_code == 400 and "Not saved" in html.text and "feeds[0].type" in html.text and "origin" in html.text
+    assert 'name="schema"' in html.text and "NOT A HOST" in html.text
+
+
+def test_invalid_json_is_refused(client, admin, fake_world):
+    add_ready(client, admin)
+    html = client.post("/admin/communities/forum-test/schema", data={"schema": "{not json"}, headers={"X-API-KEY": admin["headers"]["X-API-KEY"]})
+    assert html.status_code == 400 and "not valid JSON" in html.text
+
+
+def test_a_schema_that_collects_nothing_live_is_not_saved(client, admin, fake_world):
+    row = add_ready(client, admin)
+    dead = json.loads(json.dumps(row["schema"]))
+    dead["feeds"][0]["list_path"] = "$.nothing_here"
+    response = client.post("/admin/communities/forum-test/schema", data={"schema": json.dumps(dead)}, headers={"X-API-KEY": admin["headers"]["X-API-KEY"]})
+    assert response.status_code == 422 and "no posts" in response.text and "not saved" in response.text
+    assert store.get("forum-test")["schema"] == row["schema"] and not store.get("forum-test")["schema_previous"]
+
+
+def test_restore_swaps_back_and_refuses_when_there_is_nothing(client, admin, fake_world):
+    row = add_ready(client, admin)
+    assert client.post("/admin/communities/forum-test/schema/restore", headers=admin["headers"]).status_code == 409
+    changed = json.loads(json.dumps(row["schema"]))
+    changed["feeds"][0]["post"]["fields"]["title"] = ["$.title", "$.id"]
+    assert save(client, admin, json.dumps(changed)).status_code == 200
+    assert client.post("/admin/communities/forum-test/schema/restore", headers=admin["headers"]).status_code == 200
+    restored = store.get("forum-test")
+    assert restored["schema"] == row["schema"] and restored["schema_previous"] == changed
+    assert list(get_table("audit_log").find(event_key="admin.community.schema_restore", target_label="forum-test"))
+
+
+def test_the_devii_tools_exist_and_the_save_needs_confirmation():
+    from devplacepy.services.devii.actions.catalog.community import COMMUNITY_ACTIONS
+    from devplacepy.services.devii.actions.dispatcher import CONFIRM_REQUIRED
+
+    tools = {action.name: action for action in COMMUNITY_ACTIONS}
+    assert tools["community_schema_get"].requires_admin and tools["community_schema_set"].requires_admin
+    assert "community_schema_set" in CONFIRM_REQUIRED
+    assert any(param.name == "confirm" for param in tools["community_schema_set"].params)

tests/api/admin/nodes/index.py

@@ -28,6 +28,8 @@ def test_admin_overview_shape(client, admin):
         "nodes_cache_ttl_seconds",
         "nodes_daily_social_user",
         "nodes_social_via_nodes",
+        "nodes_allow_unknown_class",
+        "nodes_redundancy_percent",
     }
     listed = {row["uid"]: row for row in body["devices"]}
     assert listed[device["uid"]]["owner_username"] == owner["username"]
@@ -40,3 +42,26 @@ def test_html_page_renders(client, admin):
     assert response.status_code == 200
     assert "Community nodes" in response.text
     assert "data-live=\"summary\"" in response.text
+
+
+def test_overview_carries_circuits_outcomes_lifetime_geo_and_cache(client, admin):
+    from devplacepy.database.nodes import flush_counters, save_circuits
+
+    flush_counters({"outcome:reddit:residential:ok": 3, "outcome:reddit:residential:blocked": 1, "kicked": 2})
+    save_circuits([{"key": "destination:reddit.com", "failures": 9, "state": "open", "opened_at": 4_000_000_000.0}])
+    try:
+        body = client.get("/admin/nodes", headers=admin_headers(admin)).json()
+        row = next(item for item in body["outcomes"] if item["adapter"] == "reddit" and item["asn_class"] == "residential")
+        assert row["ok"] >= 3 and row["blocked"] >= 1 and row["total"] == row["ok"] + row["blocked"] + row["failed"]
+        assert body["lifetime"]["kicked"] >= 2
+        assert not any(name.startswith("outcome:") for name in body["lifetime"])
+        assert body["circuits"][0]["key"] == "destination:reddit.com" and body["circuits"][0]["state"] == "open"
+        assert set(body["geo"]) >= {"available", "path"}
+        assert set(body["cache"]) == {"entries", "bytes"}
+    finally:
+        save_circuits([])
+
+
+def test_html_shows_the_new_sections_and_the_empty_outcome_state(client, admin):
+    text = client.get("/admin/nodes", headers=admin["headers"]).text
+    assert "Circuit breakers" in text and "Success by address class" in text and "Address database" in text

tests/api/admin/nodes/override.py

@@ -0,0 +1,53 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database.nodes import get_device
+from tests.api.admin.nodes.conftest import admin_headers, make_admin, make_device
+from tests.api.nodes.conftest import make_member
+
+
+def test_member_cannot_override(client):
+    member = make_member()
+    device = make_device(member["uid"])
+    response = client.post(f"/admin/nodes/devices/{device['uid']}/class", json={"value": "allow"}, headers=admin_headers(member))
+    assert response.status_code == 403
+
+
+def test_override_round_trip_is_visible_in_the_overview(client, admin):
+    owner = make_member()
+    device = make_device(owner["uid"])
+    url = f"/admin/nodes/devices/{device['uid']}/class"
+    response = client.post(url, json={"value": "allow"}, headers=admin_headers(admin))
+    assert response.status_code == 200 and response.json()["data"] == {"uid": device["uid"], "class_override": "allow"}
+    assert get_device(device["uid"])["class_override"] == "allow"
+    listed = {row["uid"]: row for row in client.get("/admin/nodes", headers=admin_headers(admin)).json()["devices"]}
+    assert listed[device["uid"]]["class_override"] == "allow"
+    client.post(url, json={"value": "clear"}, headers=admin_headers(admin))
+    assert not get_device(device["uid"])["class_override"]
+
+
+def test_only_known_values_are_accepted(client, admin):
+    owner = make_member()
+    device = make_device(owner["uid"])
+    response = client.post(f"/admin/nodes/devices/{device['uid']}/class", json={"value": "deny"}, headers=admin_headers(admin))
+    assert response.status_code in (400, 422)
+
+
+def test_unknown_device_is_404(client, admin):
+    assert client.post("/admin/nodes/devices/nope/class", json={"value": "allow"}, headers=admin_headers(admin)).status_code == 404
+
+
+def test_junior_admin_cannot_override_a_senior_admins_device(client):
+    senior, junior = make_admin(), make_admin()
+    device = make_device(senior["uid"])
+    client.post(f"/admin/nodes/devices/{device['uid']}/class", json={"value": "allow"}, headers=admin_headers(junior))
+    assert not get_device(device["uid"]).get("class_override")
+
+
+def test_the_change_is_audited(client, admin):
+    from devplacepy.database import get_table
+
+    owner = make_member()
+    device = make_device(owner["uid"])
+    client.post(f"/admin/nodes/devices/{device['uid']}/class", json={"value": "allow"}, headers=admin_headers(admin))
+    rows = list(get_table("audit_log").find(event_key="node.device.override", target_uid=device["uid"]))
+    assert rows and rows[0]["new_value"] == "allow"

tests/api/game/mutations.py

@@ -429,3 +429,62 @@ def test_grant_ineligible_when_rich_returns_400(app_server, seeded_db):
     )
     r = session.post(f"{BASE_URL}/game/grant", headers=JSON)
     assert r.status_code == 400
+
+
+def test_plot_stop_requires_auth(app_server, seeded_db):
+    r = requests.post(
+        f"{BASE_URL}/game/plot/stop", data={"slot": 0, "stop": "true"}, headers=JSON
+    )
+    assert r.status_code in (401, 303)
+
+
+def test_plot_stop_without_continuous_delivery_returns_400(app_server, seeded_db):
+    session, name = _signup()
+    _reset_farm(name)
+    session.post(f"{BASE_URL}/game/plant", data={"slot": 0, "crop": "shell"}, headers=JSON)
+    r = session.post(
+        f"{BASE_URL}/game/plot/stop", data={"slot": 0, "stop": "true"}, headers=JSON
+    )
+    assert r.status_code == 400
+
+
+def test_plot_stop_empty_plot_returns_400(app_server, seeded_db):
+    session, name = _signup()
+    _reset_farm(name)
+    _set_farm(name, mastery_autoreplant=1)
+    r = session.post(
+        f"{BASE_URL}/game/plot/stop", data={"slot": 0, "stop": "true"}, headers=JSON
+    )
+    assert r.status_code == 400
+
+
+def test_plot_stop_leaves_plot_empty_after_harvest(app_server, seeded_db):
+    session, name = _signup()
+    _reset_farm(name)
+    _set_farm(name, mastery_autoreplant=1)
+    assert session.post(
+        f"{BASE_URL}/game/plant", data={"slot": 0, "crop": "shell"}, headers=JSON
+    ).status_code == 200
+    r = session.post(
+        f"{BASE_URL}/game/plot/stop", data={"slot": 0, "stop": "true"}, headers=JSON
+    )
+    assert r.status_code == 200, r.text[:200]
+    plot = r.json()["farm"]["plots"][0]
+    assert plot["can_stop_replant"] is True and plot["stop_after_harvest"] is True
+    refresh_snapshot()
+    user = get_table("users").find_one(username=name)
+    farm = store.get_farm(user["uid"])
+    row = get_table("game_plots").find_one(farm_uid=farm["uid"], slot_index=0)
+    past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat()
+    get_table("game_plots").update({"uid": row["uid"], "ready_at": past}, ["uid"])
+    refresh_snapshot()
+    harvested = session.post(
+        f"{BASE_URL}/game/harvest", data={"slot": 0}, headers=JSON
+    )
+    assert harvested.status_code == 200, harvested.text[:200]
+    assert harvested.json()["farm"]["plots"][0]["state"] == "empty"
+    resumed = session.post(
+        f"{BASE_URL}/game/plant", data={"slot": 0, "crop": "python"}, headers=JSON
+    )
+    assert resumed.status_code == 200
+    assert resumed.json()["farm"]["plots"][0]["stop_after_harvest"] is False

tests/api/nodes/jobs.py

@@ -7,7 +7,7 @@ import time
 import pytest
 from starlette.websockets import WebSocketDisconnect
 
-from devplacepy.database import get_table
+from devplacepy.database import get_table, set_setting
 from devplacepy.database import nodes as store
 from devplacepy.services.manager import service_manager
 from devplacepy.services.nodes.broker import JobBroker
@@ -26,7 +26,9 @@ def relaxed(client):
     original = service.scheduler.destination_ready
     service.scheduler.destination_ready = lambda target: True
     get_table("node_requests").delete()
+    set_setting("nodes_allow_unknown_class", "1")
     yield service
+    set_setting("nodes_allow_unknown_class", "0")
     service.scheduler.destination_ready = original
     get_table("node_requests").delete()
 

tests/api/nodes/ticket.py

@@ -59,6 +59,7 @@ def test_a_suspended_device_gets_no_ticket(client, member):
     response = client.post("/nodes/ticket", headers=bearer(secret))
     assert response.status_code == 403
     assert response.json()["error"]["reason"] == "suspended"
+    assert response.json()["error"]["until"].startswith("20")
 
 
 def test_a_deleted_device_secret_stops_working(client, member):

tests/api/nodes/ws.py

@@ -55,6 +55,7 @@ def test_handshake_welcomes_with_the_published_numbers(client, member):
             item.suffix for item in SPEC.destinations
         }
         assert welcome["settings"]["enabled"] is True
+        assert welcome["device_label"] == "Home laptop"
         service = service_manager.get_service("nodes")
         assert service.registry.get(device_uid) is not None
         ws.send_text(json.dumps({"type": "pong"}))
@@ -153,3 +154,44 @@ def test_capacity_is_a_hard_ceiling(client, member):
         assert service.counters.snapshot()["rejected_capacity"] >= 1
     finally:
         set_setting("nodes_max_total", previous)
+
+
+def test_the_donor_address_is_classed_through_the_geo_database(client, member, tmp_path, monkeypatch):
+    from devplacepy import config
+    from devplacepy.database.nodes import get_device
+    from devplacepy.services.nodes import geo
+
+    monkeypatch.setattr(geo, "SPEC", geo.GeoSpec(min_asn_rows=1, min_country_rows=1, checks=()))
+    target = tmp_path / "geo.db"
+    built = geo.build(
+        ["8.8.8.0,8.8.8.255,15169,Google LLC", "20.0.0.0,20.0.0.255,64500,Home Fiber BV"],
+        ["8.8.8.0,8.8.8.255,US", "20.0.0.0,20.0.0.255,NL"],
+        target,
+        "2026-10",
+    )
+    geo.install(built, target)
+    monkeypatch.setattr(config, "NODE_GEO_DB", target)
+    for address, expected in (("8.8.8.8", ("US", "datacenter")), ("20.0.0.9", ("NL", "residential")), ("9.9.9.9", ("", "unknown"))):
+        device_uid, _, ticket = ticket_for(client, make_fresh(member))
+        with client.websocket_connect(f"/nodes/ws?ticket={ticket}", headers={"x-real-ip": address}) as ws:
+            ws.send_text(hello())
+            ws.receive_text()
+            session = service_manager.get_service("nodes").registry.get(device_uid)
+            assert session.asn_class == expected[1]
+        row = get_device(device_uid)
+        assert (row["country"], row["asn_class"]) == expected
+        assert "x-real-ip" not in str(row) and address not in str(row.values())
+
+
+def test_a_missing_geo_database_is_counted_and_classes_unknown(client, member, tmp_path, monkeypatch):
+    from devplacepy import config
+
+    monkeypatch.setattr(config, "NODE_GEO_DB", tmp_path / "absent.db")
+    device_uid, _, ticket = ticket_for(client, member)
+    service = service_manager.get_service("nodes")
+    before = service.counters.snapshot()["geo_unavailable"]
+    with client.websocket_connect(f"/nodes/ws?ticket={ticket}", headers={"x-real-ip": "8.8.8.8"}) as ws:
+        ws.send_text(hello())
+        ws.receive_text()
+        assert service.registry.get(device_uid).asn_class == "unknown"
+    assert service.counters.snapshot()["geo_unavailable"] == before + 1

tests/api/sync/attachments.py

@@ -0,0 +1,80 @@
+# retoor <retoor@molodetz.nl>
+
+import base64
+from pathlib import Path
+
+from devplacepy.config import ATTACHMENTS_DIR
+from devplacepy.database import get_table
+from tests.api.sync.conftest import PNG_1X1, attachment_record, seed_world, sync_ok
+
+
+def local(origin, kind, origin_uid):
+    return get_table("sync_links").find_one(origin=origin, kind=kind, origin_uid=origin_uid)["local_uid"]
+
+
+def test_an_attachment_is_stored_and_linked_to_its_target(client, admin, origin):
+    seed_world(client, admin, origin)
+    result = sync_ok(client, admin, "attachments", origin, [attachment_record()])["results"][0]
+    assert result["status"] == "created"
+    row = get_table("attachments").find_one(uid=result["local_uid"])
+    assert (row["target_type"], row["target_uid"]) == ("post", local(origin, "posts", "p1"))
+    assert row["user_uid"] == local(origin, "users", "u1") and row["mime_type"] == "image/png"
+    assert (Path(ATTACHMENTS_DIR) / row["directory"] / row["stored_name"]).read_bytes() == base64.b64decode(PNG_1X1)
+    assert row["created_at"].startswith("2025-01-05")
+
+
+def test_replay_is_unchanged_and_a_content_change_is_refused(client, admin, origin):
+    seed_world(client, admin, origin)
+    sync_ok(client, admin, "attachments", origin, [attachment_record()])
+    assert sync_ok(client, admin, "attachments", origin, [attachment_record()])["results"][0]["status"] == "unchanged"
+    other = base64.b64encode(b"GIF89a" + b"x" * 40).decode()
+    changed = sync_ok(client, admin, "attachments", origin, [attachment_record(content=other, filename="b.gif", updated_at="2025-02-01T00:00:00Z")])["results"][0]
+    assert changed["status"] == "rejected" and changed["reason"] == "immutable"
+
+
+def test_tombstone_soft_deletes_and_restores(client, admin, origin):
+    seed_world(client, admin, origin)
+    created = sync_ok(client, admin, "attachments", origin, [attachment_record()])["results"][0]
+    gone = attachment_record(updated_at="2025-02-01T00:00:00Z", deleted=True)
+    assert sync_ok(client, admin, "attachments", origin, [gone])["results"][0]["status"] == "updated"
+    assert get_table("attachments").find_one(uid=created["local_uid"])["deleted_by"] == "sync"
+    back = attachment_record(updated_at="2025-03-01T00:00:00Z")
+    sync_ok(client, admin, "attachments", origin, [back])
+    assert get_table("attachments").find_one(uid=created["local_uid"])["deleted_at"] is None
+
+
+def test_validation_and_type_rules(client, admin, origin):
+    seed_world(client, admin, origin)
+    body = sync_ok(
+        client, admin, "attachments", origin,
+        [
+            attachment_record("a1", content="not base64!!"),
+            attachment_record("a2", content=""),
+            attachment_record("a3", filename="run.exe", content=base64.b64encode(b"MZ" + b"0" * 10).decode()),
+            attachment_record("a4", filename="../../etc/passwd.png"),
+        ],
+    )
+    statuses = [item["status"] for item in body["results"]]
+    assert statuses[:3] == ["rejected", "rejected", "rejected"]
+    stored = get_table("attachments").find_one(uid=body["results"][3].get("local_uid") or "x")
+    assert statuses[3] == "created" and stored["original_filename"] == "passwd.png"
+
+
+def test_an_oversize_file_is_refused(client, admin, origin):
+    from devplacepy.services.sync.spec import LIMITS
+
+    seed_world(client, admin, origin)
+    big = base64.b64encode(b"\x00" * (LIMITS.max_attachment_bytes + 1)).decode()
+    result = sync_ok(client, admin, "attachments", origin, [attachment_record(content=big)])["results"][0]
+    assert result["status"] == "rejected" and "exceeds" in result["errors"][0]
+
+
+def test_an_unknown_target_waits_and_is_released_when_the_target_arrives(client, admin, origin):
+    from tests.api.sync.conftest import post_record, user_record
+
+    sync_ok(client, admin, "users", origin, [user_record("u1", "attuser1")])
+    waiting = sync_ok(client, admin, "attachments", origin, [attachment_record(target="p1")])["results"][0]
+    assert waiting["status"] == "deferred"
+    body = sync_ok(client, admin, "posts", origin, [post_record("p1", "u1")])
+    assert body["released"] == {"created": 1}
+    assert get_table("attachments").count(target_uid=local(origin, "posts", "p1")) == 1

tests/api/sync/comments.py

@@ -9,7 +9,7 @@ def seed_thread(client, admin, origin):
     sync_ok(client, admin, "posts", origin, [post_record("p1", "u1"), post_record("p2", "u1")])
 
 
-def test_unknown_references_are_rejected_with_specific_reasons(client, admin, origin):
+def test_unknown_references_are_deferred_with_specific_reasons(client, admin, origin):
     seed_thread(client, admin, origin)
     body = sync_ok(
         client, admin, "comments", origin,
@@ -20,6 +20,7 @@ def test_unknown_references_are_rejected_with_specific_reasons(client, admin, or
         ],
     )
     assert [r["reason"] for r in body["results"]] == ["unknown_author", "unknown_post", "unknown_parent"]
+    assert [r["status"] for r in body["results"]] == ["deferred", "deferred", "deferred"]
 
 
 def test_create_with_nested_reply_in_one_batch(client, admin, origin):

tests/api/sync/conftest.py

@@ -96,3 +96,96 @@ def comment_record(uid="c1", post="p1", author="u1", **extra) -> dict:
         "created_at": "2025-01-03T00:00:00Z",
         **extra,
     }
+
+
+def project_record(uid="pr1", author="u1", **extra) -> dict:
+    return {
+        "origin_uid": uid,
+        "author_uid": author,
+        "title": "Synced project",
+        "description": "A project that came in from another community",
+        "created_at": "2025-01-02T00:00:00Z",
+        **extra,
+    }
+
+
+def poll_record(uid="po1", post="p1", author="u1", **extra) -> dict:
+    return {
+        "origin_uid": uid,
+        "post_uid": post,
+        "author_uid": author,
+        "question": "Tabs or spaces?",
+        "options": [{"origin_uid": "a", "label": "Tabs"}, {"origin_uid": "b", "label": "Spaces"}],
+        "created_at": "2025-01-03T00:00:00Z",
+        **extra,
+    }
+
+
+def vote_record(uid="v1", voter="u2", target="p1", kind="post", value=1, **extra) -> dict:
+    return {
+        "origin_uid": uid,
+        "voter_uid": voter,
+        "target_kind": kind,
+        "target_uid": target,
+        "value": value,
+        "created_at": "2025-01-04T00:00:00Z",
+        **extra,
+    }
+
+
+def reaction_record(uid="r1", author="u2", target="p1", kind="post", emoji="🔥", **extra) -> dict:
+    return {
+        "origin_uid": uid,
+        "author_uid": author,
+        "target_kind": kind,
+        "target_uid": target,
+        "emoji": emoji,
+        "created_at": "2025-01-04T00:00:00Z",
+        **extra,
+    }
+
+
+def follow_record(uid="f1", follower="u2", following="u1", **extra) -> dict:
+    return {
+        "origin_uid": uid,
+        "follower_uid": follower,
+        "following_uid": following,
+        "created_at": "2025-01-04T00:00:00Z",
+        **extra,
+    }
+
+
+def poll_vote_record(uid="pv1", voter="u2", poll="po1", option="a", **extra) -> dict:
+    return {
+        "origin_uid": uid,
+        "voter_uid": voter,
+        "poll_uid": poll,
+        "option_uid": option,
+        "created_at": "2025-01-05T00:00:00Z",
+        **extra,
+    }
+
+
+PNG_1X1 = (
+    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
+)
+
+
+def attachment_record(uid="at1", author="u1", target="p1", kind="post", filename="pixel.png", content=PNG_1X1, **extra) -> dict:
+    return {
+        "origin_uid": uid,
+        "author_uid": author,
+        "target_kind": kind,
+        "target_uid": target,
+        "filename": filename,
+        "content_b64": content,
+        "created_at": "2025-01-05T00:00:00Z",
+        **extra,
+    }
+
+
+def seed_world(client, admin, origin):
+    sync_ok(client, admin, "users", origin, [user_record("u1", unique("wu1")), user_record("u2", unique("wu2")), user_record("u3", unique("wu3"))])
+    sync_ok(client, admin, "projects", origin, [project_record("pr1", "u1")])
+    sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", project_uid="pr1")])
+    sync_ok(client, admin, "comments", origin, [comment_record("c1", "p1", "u2")])

tests/api/sync/deletions.py

@@ -0,0 +1,109 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_table
+from tests.api.sync.conftest import comment_record, post_record, sync_ok, unique, user_record
+
+
+def listing(client, admin, origin, **params):
+    return client.get("/sync/deletions", params={"origin": origin, **params}, headers=admin["headers"])
+
+
+def ack(client, admin, origin, items):
+    return client.post("/sync/deletions/ack", json={"origin": origin, "items": items}, headers=admin["headers"])
+
+
+def seed(client, admin, origin):
+    sync_ok(client, admin, "users", origin, [user_record("u1", unique("del"))])
+    sync_ok(client, admin, "posts", origin, [post_record("p1", "u1"), post_record("p2", "u1"), post_record("p3", "u1")])
+    sync_ok(client, admin, "comments", origin, [comment_record("c1", "p1", "u1")])
+
+
+def local(origin, kind, origin_uid):
+    return get_table("sync_links").find_one(origin=origin, kind=kind, origin_uid=origin_uid)["local_uid"]
+
+
+def test_authz(client, admin):
+    from tests.api.sync.conftest import make_account
+
+    assert client.get("/sync/deletions", params={"origin": "a.b"}).status_code == 401
+    assert listing(client, make_account("Member"), "a.b").status_code == 403
+    assert client.get("/sync/deletions", headers=admin["headers"]).status_code == 422
+
+
+def test_nothing_is_listed_for_a_clean_origin_and_sync_own_deletions_never_are(client, admin, origin):
+    seed(client, admin, origin)
+    assert listing(client, admin, origin).json()["items"] == []
+    sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", updated_at="2025-05-01T00:00:00Z", deleted=True)])
+    assert listing(client, admin, origin).json()["items"] == []
+
+
+def test_a_member_deletion_a_system_deletion_and_a_purge_are_told_apart(client, admin, origin):
+    seed(client, admin, origin)
+    get_table("posts").update({"uid": local(origin, "posts", "p1"), "deleted_at": "2026-01-01T00:00:00+00:00", "deleted_by": "someone"}, ["uid"])
+    get_table("posts").update({"uid": local(origin, "posts", "p2"), "deleted_at": "2026-01-02T00:00:00+00:00", "deleted_by": "system"}, ["uid"])
+    purged_uid = local(origin, "posts", "p3")
+    get_table("posts").delete(uid=purged_uid)
+    body = listing(client, admin, origin).json()
+    found = {(item["kind"], item["origin_uid"]): item["by"] for item in body["items"]}
+    assert found == {("posts", "p1"): "member", ("posts", "p2"): "system", ("posts", "p3"): "purged"}
+    assert body["count"] == 3 and body["has_more"] is False
+
+
+def test_paging_with_a_cursor_covers_every_entry_once(client, admin, origin):
+    seed(client, admin, origin)
+    for origin_uid in ("p1", "p2", "p3"):
+        get_table("posts").update({"uid": local(origin, "posts", origin_uid), "deleted_at": "2026-01-01T00:00:00+00:00", "deleted_by": "m"}, ["uid"])
+    get_table("comments").update({"uid": local(origin, "comments", "c1"), "deleted_at": "2026-01-01T00:00:00+00:00", "deleted_by": "m"}, ["uid"])
+    seen, cursor = [], ""
+    while True:
+        body = listing(client, admin, origin, limit=2, cursor=cursor).json()
+        seen += [(item["kind"], item["origin_uid"]) for item in body["items"]]
+        cursor = body["next_cursor"]
+        if not body["has_more"]:
+            break
+    assert sorted(seen) == [("comments", "c1"), ("posts", "p1"), ("posts", "p2"), ("posts", "p3")]
+    assert listing(client, admin, origin, cursor="zz").status_code == 400
+
+
+def test_other_origins_are_never_listed(client, admin, origin):
+    seed(client, admin, origin)
+    get_table("posts").update({"uid": local(origin, "posts", "p1"), "deleted_at": "2026-01-01T00:00:00+00:00", "deleted_by": "m"}, ["uid"])
+    assert listing(client, admin, f"{unique('x')}.example.org").json()["items"] == []
+
+
+def test_acknowledging_stops_the_listing_and_quiets_the_resend(client, admin, origin):
+    seed(client, admin, origin)
+    purged = local(origin, "posts", "p3")
+    get_table("posts").delete(uid=purged)
+    resend = sync_ok(client, admin, "posts", origin, [post_record("p3", "u1", updated_at="2025-09-01T00:00:00Z")])["results"][0]
+    assert resend["status"] == "rejected" and resend["reason"] == "purged_locally"
+    response = ack(client, admin, origin, [{"kind": "posts", "origin_uid": "p3"}])
+    assert response.status_code == 200 and response.json()["acknowledged"] == 1
+    assert listing(client, admin, origin).json()["items"] == []
+    again = sync_ok(client, admin, "posts", origin, [post_record("p3", "u1", updated_at="2025-10-01T00:00:00Z")])["results"][0]
+    assert again["status"] == "unchanged" and again["reason"] == "acknowledged_deleted"
+    assert ack(client, admin, origin, [{"kind": "posts", "origin_uid": "p3"}]).json()["acknowledged"] == 0
+
+
+def test_a_locally_deleted_row_after_ack_answers_acknowledged(client, admin, origin):
+    seed(client, admin, origin)
+    get_table("posts").update({"uid": local(origin, "posts", "p1"), "deleted_at": "2026-01-01T00:00:00+00:00", "deleted_by": "m"}, ["uid"])
+    ack(client, admin, origin, [{"kind": "posts", "origin_uid": "p1"}])
+    result = sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", content="edited after local delete", updated_at="2025-09-01T00:00:00Z")])["results"][0]
+    assert result["status"] == "unchanged" and result["reason"] == "acknowledged_deleted"
+
+
+def test_ack_validation_and_audit(client, admin, origin):
+    assert ack(client, admin, origin, []).status_code == 422
+    assert ack(client, admin, origin, [{"kind": "gists", "origin_uid": "x"}]).status_code == 422
+    assert ack(client, admin, origin, [{"kind": "posts", "origin_uid": "bad id"}]).status_code == 422
+    assert ack(client, admin, origin, [{"kind": "posts", "origin_uid": "unknown-one"}]).json()["acknowledged"] == 0
+    rows = list(get_table("audit_log").find(event_key="sync.deletions.ack", actor_uid=admin["uid"]))
+    assert rows
+
+
+def test_a_purged_user_is_listed(client, admin, origin):
+    seed(client, admin, origin)
+    get_table("users").delete(uid=local(origin, "users", "u1"))
+    items = listing(client, admin, origin).json()["items"]
+    assert ("users", "u1", "purged") in {(item["kind"], item["origin_uid"], item["by"]) for item in items}

tests/api/sync/export.py

@@ -0,0 +1,203 @@
+# retoor <retoor@molodetz.nl>
+
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+from devplacepy.database import get_table, set_setting
+from devplacepy.utils import generate_uid, register_account
+from tests.api.sync.conftest import (
+    comment_record,
+    post_record,
+    sync,
+    sync_ok,
+    unique,
+    user_record,
+)
+
+
+def export(client, admin, kind, **params):
+    return client.get(f"/sync/export/{kind}", params=params, headers=admin["headers"])
+
+
+def pull_all(client, admin, kind, since=None):
+    records, cursor, pages = [], "", 0
+    while True:
+        params = {"cursor": cursor, "limit": 50}
+        if since:
+            params["since"] = since
+        response = export(client, admin, kind, **params)
+        assert response.status_code == 200, response.text
+        body = response.json()
+        records.extend(body["records"])
+        pages += 1
+        cursor = body["next_cursor"]
+        if not body["has_more"]:
+            return records, pages
+
+
+def native_user(prefix="expu"):
+    name = unique(prefix)
+    uid, _, _ = register_account(name, f"{name}@t.dev", "secret123", age_band="adult", accepted_terms=True)
+    return uid, name
+
+
+def native_post(user_uid, content="native post content here", stamp=None, **extra):
+    stamp = stamp or datetime.now(timezone.utc).isoformat()
+    uid = generate_uid()
+    get_table("posts").insert(
+        {
+            "uid": uid, "user_uid": user_uid, "slug": f"s-{uid[-12:]}", "title": "T", "content": content,
+            "topic": "random", "project_uid": None, "image": None, "stars": 0, "created_at": stamp,
+            "updated_at": stamp, "deleted_at": None, "deleted_by": None, **extra,
+        }
+    )
+    return uid
+
+
+def test_authz_and_kill_switch(client, admin):
+    assert client.get("/sync/export/posts").status_code == 401
+    member = {"headers": {**admin["headers"]}}
+    from tests.api.sync.conftest import make_account
+
+    plain = make_account("Member")
+    assert export(client, plain, "posts").status_code == 403
+    set_setting("sync_enabled", "0")
+    try:
+        assert export(client, admin, "posts").status_code == 503
+    finally:
+        set_setting("sync_enabled", "1")
+    assert member
+
+
+def test_unknown_and_private_kinds_are_not_exportable(client, admin):
+    for kind in ("votes", "reactions", "poll_votes", "nope"):
+        response = export(client, admin, kind)
+        assert response.status_code == 404 and response.json()["error"]["reason"] == "not_exportable"
+
+
+def test_a_post_exports_in_the_inbound_shape_and_no_private_user_field_leaks(client, admin):
+    uid, name = native_user()
+    post = native_post(uid, "exported post content here")
+    users, _ = pull_all(client, admin, "users")
+    row = next(item for item in users if item["origin_uid"] == uid)
+    assert row["username"] == name and row["deleted"] is False
+    assert not set(row) & {"email", "password_hash", "api_key", "role", "xp", "level"}
+    posts, _ = pull_all(client, admin, "posts")
+    found = next(item for item in posts if item["origin_uid"] == post)
+    assert found["author_uid"] == uid and found["content"] == "exported post content here"
+    from devplacepy.services.sync.records import PostRecord
+
+    assert PostRecord.model_validate(found).origin_uid == post
+
+
+def test_a_soft_deleted_post_is_a_tombstone_and_only_after_the_since_cursor(client, admin):
+    uid, _ = native_user()
+    post = native_post(uid)
+    before = datetime.now(timezone.utc).isoformat()
+    get_table("posts").update({"uid": post, "deleted_at": datetime.now(timezone.utc).isoformat(), "deleted_by": uid}, ["uid"])
+    posts, _ = pull_all(client, admin, "posts", since=before)
+    mine = [item for item in posts if item["origin_uid"] == post]
+    assert len(mine) == 1 and mine[0]["deleted"] is True
+
+
+def test_pagination_is_gapless_and_repeat_free(client, admin):
+    uid, _ = native_user()
+    base = datetime.now(timezone.utc) - timedelta(days=1)
+    made = {native_post(uid, f"paged post number {index}", (base + timedelta(seconds=index // 3)).isoformat()) for index in range(120)}
+    records, pages = pull_all(client, admin, "posts")
+    ids = [item["origin_uid"] for item in records]
+    assert len(ids) == len(set(ids)) and made <= set(ids) and pages >= 3
+
+
+def test_rows_that_arrived_by_sync_are_never_echoed(client, admin, origin):
+    sync_ok(client, admin, "users", origin, [user_record("u1", unique("echo"))])
+    sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", content="synced in, must not come back out")])
+    sync_ok(client, admin, "comments", origin, [comment_record("c1", "p1", "u1")])
+    posts, _ = pull_all(client, admin, "posts")
+    assert not any(item["content"] == "synced in, must not come back out" for item in posts)
+    local_post = get_table("sync_links").find_one(origin=origin, kind="posts", origin_uid="p1")["local_uid"]
+    comments, _ = pull_all(client, admin, "comments")
+    assert not any(item["post_uid"] == local_post for item in comments)
+
+
+def test_private_projects_are_left_out_and_post_links_to_them_are_cut(client, admin):
+    uid, _ = native_user()
+    public, private = generate_uid(), generate_uid()
+    now = datetime.now(timezone.utc).isoformat()
+    for project, is_private in ((public, 0), (private, 1)):
+        get_table("projects").insert(
+            {"uid": project, "user_uid": uid, "slug": f"p-{project[-10:]}", "title": "P", "description": "d", "project_type": "software",
+             "status": "x", "platforms": "", "website_url": "", "repo_url": "", "is_private": is_private, "read_only": 0, "stars": 0,
+             "created_at": now, "updated_at": now, "deleted_at": None, "deleted_by": None}
+        )
+    linked = native_post(uid, "post that points at a private project", project_uid=private)
+    open_post = native_post(uid, "post that points at a public project", project_uid=public)
+    projects, _ = pull_all(client, admin, "projects")
+    ids = {item["origin_uid"] for item in projects}
+    assert public in ids and private not in ids
+    posts, _ = pull_all(client, admin, "posts")
+    by_id = {item["origin_uid"]: item for item in posts}
+    assert by_id[linked]["project_uid"] is None and by_id[open_post]["project_uid"] == public
+
+
+def test_polls_and_follows_and_attachments_export(client, admin):
+    alice, _ = native_user()
+    bob, _ = native_user()
+    post = native_post(alice, "post that carries a poll for export")
+    now = datetime.now(timezone.utc).isoformat()
+    poll = generate_uid()
+    get_table("polls").insert({"uid": poll, "post_uid": post, "user_uid": alice, "question": "Q?", "created_at": now, "deleted_at": None, "deleted_by": None})
+    first, second = generate_uid(), generate_uid()
+    get_table("poll_options").insert_many([
+        {"uid": first, "poll_uid": poll, "label": "yes", "position": 0, "deleted_at": None, "deleted_by": None},
+        {"uid": second, "poll_uid": poll, "label": "no", "position": 1, "deleted_at": None, "deleted_by": None},
+    ])
+    follow = generate_uid()
+    get_table("follows").insert({"uid": follow, "follower_uid": bob, "following_uid": alice, "created_at": now, "deleted_at": None, "deleted_by": None})
+    attachment = generate_uid()
+    get_table("attachments").insert({"uid": attachment, "target_type": "post", "target_uid": post, "user_uid": alice, "original_filename": "a.png",
+        "stored_name": f"{attachment}.png", "directory": "ab/cd", "file_size": 10, "mime_type": "image/png", "created_at": now, "deleted_at": None})
+    polls, _ = pull_all(client, admin, "polls")
+    exported = next(item for item in polls if item["origin_uid"] == poll)
+    assert [option["label"] for option in exported["options"]] == ["yes", "no"]
+    from devplacepy.services.sync.records import PollRecord
+
+    assert PollRecord.model_validate(exported).question == "Q?"
+    follows, _ = pull_all(client, admin, "follows")
+    assert any(item["origin_uid"] == follow and item["follower_uid"] == bob for item in follows)
+    attachments, _ = pull_all(client, admin, "attachments")
+    found = next(item for item in attachments if item["origin_uid"] == attachment)
+    assert found["url"] == f"/static/uploads/attachments/ab/cd/{attachment}.png" and "content_b64" not in found
+
+
+def test_bad_cursor_since_and_limit(client, admin):
+    assert export(client, admin, "posts", cursor="!!!").status_code == 400
+    assert export(client, admin, "posts", since="yesterday-ish").status_code == 400
+    body = export(client, admin, "posts", limit=1000).json()
+    assert body["count"] <= 50
+
+
+def test_the_changed_expression_uses_an_index(local_db):
+    from devplacepy.database import db
+
+    plan = " ".join(
+        str(row)
+        for row in db.query(
+            "EXPLAIN QUERY PLAN SELECT * FROM posts t WHERE COALESCE(deleted_at, updated_at, created_at) > '2020' "
+            "ORDER BY COALESCE(deleted_at, updated_at, created_at), uid LIMIT 51"
+        )
+    )
+    assert "idx_posts_changed" in plan and "TEMP B-TREE" not in plan
+
+
+def test_every_export_is_audited(client, admin):
+    pull_all(client, admin, "follows")
+    rows = list(get_table("audit_log").find(event_key="sync.export", actor_uid=admin["uid"]))
+    assert rows
+
+
+@pytest.mark.parametrize("kind", ["users", "projects", "posts", "comments", "polls", "follows", "attachments"])
+def test_every_exportable_kind_answers(client, admin, kind):
+    body = export(client, admin, kind).json()
+    assert body["kind"] == kind and isinstance(body["records"], list)

tests/api/sync/follows.py

@@ -0,0 +1,31 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_table
+from tests.api.sync.conftest import follow_record, seed_world, sync_ok
+
+
+def local(origin, uid):
+    return get_table("sync_links").find_one(origin=origin, kind="users", origin_uid=uid)["local_uid"]
+
+
+def test_follow_unfollow_refollow_keeps_one_row_and_has_no_side_effects(client, admin, origin):
+    seed_world(client, admin, origin)
+    notifications = get_table("notifications").count()
+    xp = get_table("users").find_one(uid=local(origin, "u1"))["xp"]
+    created = sync_ok(client, admin, "follows", origin, [follow_record("f1")])["results"][0]
+    assert created["status"] == "created"
+    sync_ok(client, admin, "follows", origin, [follow_record("f1", updated_at="2025-02-01T00:00:00Z", deleted=True)])
+    assert get_table("follows").find_one(uid=created["local_uid"])["deleted_by"] == "sync"
+    sync_ok(client, admin, "follows", origin, [follow_record("f1", updated_at="2025-03-01T00:00:00Z")])
+    assert get_table("follows").count(follower_uid=local(origin, "u2"), following_uid=local(origin, "u1")) == 1
+    assert get_table("follows").find_one(uid=created["local_uid"])["deleted_at"] is None
+    assert get_table("notifications").count() == notifications
+    assert get_table("users").find_one(uid=local(origin, "u1"))["xp"] == xp
+
+
+def test_self_follow_and_unknown_users(client, admin, origin):
+    seed_world(client, admin, origin)
+    body = sync_ok(client, admin, "follows", origin, [follow_record("f1", "u1", "u1"), follow_record("f2", "ghost", "u1"), follow_record("f3", "u2", "ghost")])
+    assert [(item["status"], item["reason"]) for item in body["results"]] == [
+        ("rejected", "self_follow"), ("deferred", "unknown_author"), ("deferred", "unknown_author"),
+    ]

tests/api/sync/pending.py

@@ -0,0 +1,120 @@
+# retoor <retoor@molodetz.nl>
+
+import dataclasses
+import random
+
+from devplacepy.database import get_table
+from devplacepy.services.sync import processors
+from devplacepy.services.sync.pending import PendingQueue
+from devplacepy.services.sync.spec import LIMITS
+from tests.api.sync.conftest import comment_record, post_record, sync_ok, unique, user_record
+
+
+def link_count(origin, kind):
+    return get_table("sync_links").count(origin=origin, kind=kind)
+
+
+def test_a_chain_of_waiting_records_is_released_by_the_first_dependency(client, admin, origin):
+    reply = sync_ok(client, admin, "comments", origin, [comment_record("c2", "p1", "u1", parent_uid="c1")])["results"][0]
+    parent = sync_ok(client, admin, "comments", origin, [comment_record("c1", "p1", "u1")])["results"][0]
+    post = sync_ok(client, admin, "posts", origin, [post_record("p1", "u1")])["results"][0]
+    assert (reply["status"], parent["status"], post["status"]) == ("deferred", "deferred", "deferred")
+    assert get_table("sync_pending").count(origin=origin) == 3
+    body = sync_ok(client, admin, "users", origin, [user_record("u1", unique("chain"))])
+    assert body["results"][0]["status"] == "created"
+    assert body["released"] == {"created": 3}
+    assert get_table("sync_pending").count(origin=origin) == 0
+    assert [link_count(origin, kind) for kind in ("users", "posts", "comments")] == [1, 1, 2]
+    comments = {row["origin_uid"]: row for row in get_table("sync_links").find(origin=origin, kind="comments")}
+    reply_row = get_table("comments").find_one(uid=comments["c2"]["local_uid"])
+    assert reply_row["parent_uid"] == comments["c1"]["local_uid"]
+
+
+def test_any_arrival_order_ends_in_the_same_final_state(client, admin, origin):
+    records = [
+        ("users", user_record("u1", unique("order1"))),
+        ("users", user_record("u2", unique("order2"))),
+        ("posts", post_record("p1", "u1")),
+        *[("comments", comment_record(f"c{index}", "p1", "u2")) for index in range(15)],
+    ]
+    random.Random(7).shuffle(records)
+    for kind, record in records:
+        sync_ok(client, admin, kind, origin, [record])
+    assert [link_count(origin, kind) for kind in ("users", "posts", "comments")] == [2, 1, 15]
+    assert get_table("sync_pending").count(origin=origin) == 0
+
+
+def test_a_resend_replaces_the_waiting_row_instead_of_stacking(client, admin, origin):
+    for _ in range(3):
+        sync_ok(client, admin, "posts", origin, [post_record("p1", "ghost")])
+    assert get_table("sync_pending").count(origin=origin) == 1
+
+
+def test_a_resend_carries_the_newest_content(client, admin, origin):
+    sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", content="the first version of it")])
+    sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", content="the second version of it", updated_at="2025-02-01T00:00:00Z")])
+    sync_ok(client, admin, "users", origin, [user_record("u1", unique("newest"))])
+    link = get_table("sync_links").find_one(origin=origin, kind="posts", origin_uid="p1")
+    assert get_table("posts").find_one(uid=link["local_uid"])["content"] == "the second version of it"
+
+
+def test_the_waiting_list_is_capped_per_origin_and_says_so(client, admin, origin, monkeypatch):
+    monkeypatch.setattr(processors, "LIMITS", dataclasses.replace(LIMITS, pending_max_per_origin=2))
+    body = sync_ok(client, admin, "posts", origin, [post_record(f"p{index}", "ghost") for index in range(4)])
+    assert [item["status"] for item in body["results"]] == ["deferred", "deferred", "rejected", "rejected"]
+    assert "waiting list" in body["results"][2]["errors"][-1]
+    assert get_table("sync_pending").count(origin=origin) == 2
+
+
+def test_the_cap_is_per_origin(client, admin, origin, monkeypatch):
+    monkeypatch.setattr(processors, "LIMITS", dataclasses.replace(LIMITS, pending_max_per_origin=1))
+    other = f"{unique('o')}.example.org"
+    sync_ok(client, admin, "posts", origin, [post_record("p1", "ghost")])
+    assert sync_ok(client, admin, "posts", other, [post_record("p1", "ghost")])["results"][0]["status"] == "deferred"
+
+
+def test_a_dry_run_never_queues(client, admin, origin):
+    body = sync_ok(client, admin, "posts", origin, [post_record("p1", "ghost")], dry_run=True)
+    assert body["results"][0]["status"] == "rejected" and body["released"] == {}
+    assert get_table("sync_pending").count(origin=origin) == 0
+
+
+def test_a_tombstone_for_an_unknown_record_is_not_queued(client, admin, origin):
+    body = sync_ok(client, admin, "comments", origin, [comment_record("c1", "nopost", "nobody", deleted=True)])
+    assert body["results"][0]["status"] == "unchanged" and body["results"][0]["reason"] == "tombstone_for_unknown"
+    assert get_table("sync_pending").count(origin=origin) == 0
+
+
+def test_an_oversize_record_is_not_queued(client, admin, origin, monkeypatch):
+    monkeypatch.setattr(processors, "LIMITS", dataclasses.replace(LIMITS, pending_max_record_bytes=100))
+    result = sync_ok(client, admin, "posts", origin, [post_record("p1", "ghost")])["results"][0]
+    assert result["status"] == "rejected" and "too large" in result["errors"][-1]
+
+
+def test_expired_and_exhausted_rows_are_dropped_hard(client, admin, origin):
+    sync_ok(client, admin, "posts", origin, [post_record("p1", "ghost"), post_record("p2", "ghost")])
+    rows = list(get_table("sync_pending").find(origin=origin))
+    get_table("sync_pending").update({"uid": rows[0]["uid"], "expires_at": "2000-01-01T00:00:00+00:00"}, ["uid"])
+    get_table("sync_pending").update({"uid": rows[1]["uid"], "attempts": LIMITS.pending_max_attempts}, ["uid"])
+    assert PendingQueue().sweep() >= 2
+    assert get_table("sync_pending").count(origin=origin) == 0
+
+
+def test_the_release_budget_bounds_the_work_per_call(client, admin, origin):
+    for index in range(5):
+        sync_ok(client, admin, "posts", origin, [post_record(f"p{index}", "u1")])
+    from devplacepy.database.sync import claim_link
+
+    author = get_table("users").insert({"uid": "budget-author", "username": unique("budget"), "created_at": "2025-01-01T00:00:00+00:00"})
+    claim_link(origin, "users", "u1", "budget-author", "x", "2025-01-01T00:00:00+00:00")
+    released = PendingQueue().release(origin, cap=2)
+    assert sum(released.values()) == 2 and author
+    assert get_table("sync_pending").count(origin=origin) == 3
+
+
+def test_pending_rows_of_one_origin_are_never_released_by_another(client, admin, origin):
+    other = f"{unique('o')}.example.org"
+    sync_ok(client, admin, "posts", origin, [post_record("p1", "u1")])
+    sync_ok(client, admin, "users", other, [user_record("u1", unique("isolated"))])
+    assert get_table("sync_pending").count(origin=origin) == 1
+    assert link_count(origin, "posts") == 0

tests/api/sync/poll_votes.py

@@ -0,0 +1,41 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_table
+from tests.api.sync.conftest import poll_record, poll_vote_record, seed_world, sync_ok
+
+
+def setup(client, admin, origin):
+    seed_world(client, admin, origin)
+    return sync_ok(client, admin, "polls", origin, [poll_record()])["results"][0]["local_uid"]
+
+
+def test_a_ballot_changes_option_without_duplicating(client, admin, origin):
+    poll = setup(client, admin, origin)
+    first = sync_ok(client, admin, "poll_votes", origin, [poll_vote_record("pv1", option="a")])["results"][0]
+    assert first["status"] == "created"
+    sync_ok(client, admin, "poll_votes", origin, [poll_vote_record("pv1", option="b", updated_at="2025-02-01T00:00:00Z")])
+    rows = list(get_table("poll_votes").find(poll_uid=poll))
+    assert len(rows) == 1
+    option = get_table("poll_options").find_one(uid=rows[0]["option_uid"])
+    assert option["label"] == "Spaces"
+
+
+def test_unknown_poll_waits_but_an_unknown_option_is_final(client, admin, origin):
+    setup(client, admin, origin)
+    body = sync_ok(client, admin, "poll_votes", origin, [poll_vote_record("pv1", poll="ghost"), poll_vote_record("pv2", option="zzz")])
+    assert (body["results"][0]["status"], body["results"][0]["reason"]) == ("deferred", "unknown_poll")
+    assert (body["results"][1]["status"], body["results"][1]["reason"]) == ("rejected", "unknown_option")
+
+
+def test_one_ballot_per_voter_even_with_a_second_origin_id(client, admin, origin):
+    poll = setup(client, admin, origin)
+    sync_ok(client, admin, "poll_votes", origin, [poll_vote_record("pv1", option="a")])
+    sync_ok(client, admin, "poll_votes", origin, [poll_vote_record("pv7", option="b")])
+    assert get_table("poll_votes").count(poll_uid=poll) == 1
+
+
+def test_clearing_a_ballot(client, admin, origin):
+    poll = setup(client, admin, origin)
+    sync_ok(client, admin, "poll_votes", origin, [poll_vote_record("pv1")])
+    sync_ok(client, admin, "poll_votes", origin, [poll_vote_record("pv1", updated_at="2025-02-01T00:00:00Z", deleted=True)])
+    assert get_table("poll_votes").find_one(poll_uid=poll)["deleted_by"] == "sync"

tests/api/sync/polls.py

@@ -0,0 +1,75 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_table
+from tests.api.sync.conftest import poll_record, seed_world, sync_ok
+
+
+def options_of(poll_uid):
+    return {row["label"]: row for row in get_table("poll_options").find(poll_uid=poll_uid)}
+
+
+def test_a_poll_is_created_with_its_options(client, admin, origin):
+    seed_world(client, admin, origin)
+    result = sync_ok(client, admin, "polls", origin, [poll_record()])["results"][0]
+    assert result["status"] == "created"
+    poll = get_table("polls").find_one(uid=result["local_uid"])
+    assert poll["question"] == "Tabs or spaces?" and poll["deleted_at"] is None
+    assert sorted(options_of(poll["uid"])) == ["Spaces", "Tabs"]
+    assert sorted(row["position"] for row in options_of(poll["uid"]).values()) == [0, 1]
+
+
+def test_option_validation(client, admin, origin):
+    seed_world(client, admin, origin)
+    one = poll_record(options=[{"origin_uid": "a", "label": "Only"}])
+    dup = poll_record(options=[{"origin_uid": "a", "label": "X"}, {"origin_uid": "a", "label": "Y"}])
+    many = poll_record(options=[{"origin_uid": str(index), "label": f"o{index}"} for index in range(7)])
+    body = sync_ok(client, admin, "polls", origin, [one, dup, many])
+    assert [item["status"] for item in body["results"]] == ["rejected"] * 3
+
+
+def test_edit_adds_renames_and_removes_options(client, admin, origin):
+    seed_world(client, admin, origin)
+    created = sync_ok(client, admin, "polls", origin, [poll_record()])["results"][0]
+    edited = poll_record(
+        question="Tabs, spaces or both?",
+        options=[{"origin_uid": "a", "label": "Tabs!"}, {"origin_uid": "c", "label": "Both"}],
+        updated_at="2025-02-01T00:00:00Z",
+    )
+    assert sync_ok(client, admin, "polls", origin, [edited])["results"][0]["status"] == "updated"
+    rows = list(get_table("poll_options").find(poll_uid=created["local_uid"]))
+    live = sorted(row["label"] for row in rows if not row["deleted_at"])
+    assert live == ["Both", "Tabs!"]
+    assert [row["label"] for row in rows if row["deleted_at"]] == ["Spaces"]
+    assert get_table("polls").find_one(uid=created["local_uid"])["question"] == "Tabs, spaces or both?"
+
+
+def test_one_poll_per_post(client, admin, origin):
+    seed_world(client, admin, origin)
+    sync_ok(client, admin, "polls", origin, [poll_record("po1")])
+    second = sync_ok(client, admin, "polls", origin, [poll_record("po2")])["results"][0]
+    assert second["status"] == "rejected" and "already has a poll" in second["errors"][0]
+    assert get_table("sync_links").count(origin=origin, kind="polls") == 1
+
+
+def test_a_tombstone_removes_poll_options_and_votes_together_and_restores(client, admin, origin):
+    seed_world(client, admin, origin)
+    created = sync_ok(client, admin, "polls", origin, [poll_record()])["results"][0]
+    from tests.api.sync.conftest import poll_vote_record
+
+    sync_ok(client, admin, "poll_votes", origin, [poll_vote_record()])
+    gone = poll_record(updated_at="2025-02-01T00:00:00Z", deleted=True)
+    sync_ok(client, admin, "polls", origin, [gone])
+    uid = created["local_uid"]
+    assert get_table("polls").find_one(uid=uid)["deleted_at"]
+    assert all(row["deleted_at"] for row in get_table("poll_options").find(poll_uid=uid))
+    assert all(row["deleted_at"] for row in get_table("poll_votes").find(poll_uid=uid))
+    back = poll_record(updated_at="2025-03-01T00:00:00Z")
+    sync_ok(client, admin, "polls", origin, [back])
+    assert get_table("polls").find_one(uid=uid)["deleted_at"] is None
+    assert all(row["deleted_at"] is None for row in get_table("poll_options").find(poll_uid=uid))
+
+
+def test_an_unknown_post_waits(client, admin, origin):
+    seed_world(client, admin, origin)
+    result = sync_ok(client, admin, "polls", origin, [poll_record(post="nope")])["results"][0]
+    assert result["status"] == "deferred" and result["reason"] == "unknown_post"

tests/api/sync/posts.py

@@ -9,11 +9,13 @@ def seed_author(client, admin, origin, uid="u1"):
     return body["results"][0]["local_uid"]
 
 
-def test_unknown_author_is_rejected_with_reason(client, admin, origin):
+def test_unknown_author_is_deferred_with_the_reason_and_what_it_waits_for(client, admin, origin):
     body = sync_ok(client, admin, "posts", origin, [post_record(author="missing")])
     result = body["results"][0]
-    assert result["status"] == "rejected" and result["reason"] == "unknown_author"
+    assert result["status"] == "deferred" and result["reason"] == "unknown_author"
+    assert (result["needs_kind"], result["needs_uid"]) == ("users", "missing")
     assert get_table("sync_links").count(origin=origin) == 0
+    assert get_table("sync_pending").count(origin=origin) == 1
 
 
 def test_create_normalizes_and_has_no_side_effects(client, admin, origin):

tests/api/sync/projects.py

@@ -0,0 +1,64 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_table
+from tests.api.sync.conftest import post_record, project_record, sync_ok, unique, user_record
+
+
+def seed(client, admin, origin):
+    sync_ok(client, admin, "users", origin, [user_record("u1", unique("pj"))])
+
+
+def test_a_project_is_created_public_writable_and_without_side_effects(client, admin, origin):
+    seed(client, admin, origin)
+    before = get_table("notifications").count()
+    body = sync_ok(client, admin, "projects", origin, [project_record("pr1", project_type="game", stars=99)])
+    result = body["results"][0]
+    assert result["status"] == "created" and result["ignored"] == ["stars"]
+    row = get_table("projects").find_one(uid=result["local_uid"])
+    assert (row["is_private"], row["read_only"], row["stars"], row["project_type"]) == (0, 0, 0, "game")
+    assert row["slug"].endswith("synced-project") and row["deleted_at"] is None
+    assert get_table("notifications").count() == before
+
+
+def test_replay_edit_and_tombstone(client, admin, origin):
+    seed(client, admin, origin)
+    created = sync_ok(client, admin, "projects", origin, [project_record("pr1")])["results"][0]
+    assert sync_ok(client, admin, "projects", origin, [project_record("pr1")])["results"][0]["status"] == "unchanged"
+    edited = sync_ok(client, admin, "projects", origin, [project_record("pr1", title="Renamed", updated_at="2025-02-01T00:00:00Z")])["results"][0]
+    assert edited["status"] == "updated"
+    assert get_table("projects").find_one(uid=created["local_uid"])["title"] == "Renamed"
+    gone = sync_ok(client, admin, "projects", origin, [project_record("pr1", title="Renamed", updated_at="2025-03-01T00:00:00Z", deleted=True)])["results"][0]
+    assert gone["status"] == "updated"
+    row = get_table("projects").find_one(uid=created["local_uid"])
+    assert row["deleted_at"] and row["deleted_by"] == "sync"
+
+
+def test_unknown_author_and_bad_input(client, admin, origin):
+    body = sync_ok(client, admin, "projects", origin, [project_record("pr1", "ghost"), project_record("pr2", title="")])
+    assert body["results"][0]["status"] == "deferred"
+    assert body["results"][1]["status"] == "rejected" and body["results"][1]["reason"] == "invalid"
+
+
+def test_unknown_type_falls_back_and_urls_are_cleaned(client, admin, origin):
+    seed(client, admin, origin)
+    result = sync_ok(client, admin, "projects", origin, [project_record("pr1", project_type="quantum", website_url="not a url at all", repo_url="example.org/r")])["results"][0]
+    row = get_table("projects").find_one(uid=result["local_uid"])
+    assert row["project_type"] == "software" and row["website_url"] == "" and row["repo_url"].startswith("http")
+
+
+def test_a_post_links_to_its_project_and_waits_for_it(client, admin, origin):
+    seed(client, admin, origin)
+    waiting = sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", project_uid="pr1")])["results"][0]
+    assert waiting["status"] == "deferred" and waiting["reason"] == "unknown_project"
+    body = sync_ok(client, admin, "projects", origin, [project_record("pr1")])
+    assert body["released"] == {"created": 1}
+    post = get_table("posts").find_one(title=None, content="a synced post with enough content", user_uid=get_table("sync_links").find_one(origin=origin, kind="users", origin_uid="u1")["local_uid"])
+    project = get_table("sync_links").find_one(origin=origin, kind="projects", origin_uid="pr1")["local_uid"]
+    assert post["project_uid"] == project
+
+
+def test_a_post_without_a_project_keeps_the_old_fingerprint(client, admin, origin):
+    seed(client, admin, origin)
+    sync_ok(client, admin, "posts", origin, [post_record("p1", "u1")])
+    again = sync_ok(client, admin, "posts", origin, [post_record("p1", "u1", project_uid=None)])["results"][0]
+    assert again["status"] == "unchanged"

tests/api/sync/reactions.py

@@ -0,0 +1,33 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_table
+from tests.api.sync.conftest import reaction_record, seed_world, sync_ok
+
+
+def local(origin, kind, origin_uid):
+    return get_table("sync_links").find_one(origin=origin, kind=kind, origin_uid=origin_uid)["local_uid"]
+
+
+def test_reactions_are_created_toggled_and_revived(client, admin, origin):
+    seed_world(client, admin, origin)
+    created = sync_ok(client, admin, "reactions", origin, [reaction_record("r1")])["results"][0]
+    assert created["status"] == "created"
+    row = get_table("reactions").find_one(uid=created["local_uid"])
+    assert (row["emoji"], row["target_type"], row["target_uid"]) == ("🔥", "post", local(origin, "posts", "p1"))
+    sync_ok(client, admin, "reactions", origin, [reaction_record("r1", updated_at="2025-02-01T00:00:00Z", deleted=True)])
+    assert get_table("reactions").find_one(uid=created["local_uid"])["deleted_by"] == "sync"
+    sync_ok(client, admin, "reactions", origin, [reaction_record("r1", updated_at="2025-03-01T00:00:00Z")])
+    assert get_table("reactions").find_one(uid=created["local_uid"])["deleted_at"] is None
+
+
+def test_a_different_emoji_is_a_different_reaction_and_text_is_refused(client, admin, origin):
+    seed_world(client, admin, origin)
+    body = sync_ok(client, admin, "reactions", origin, [reaction_record("r1"), reaction_record("r2", emoji="👍"), reaction_record("r3", emoji="nope")])
+    assert [item["status"] for item in body["results"]] == ["created", "created", "rejected"]
+    assert get_table("reactions").count(user_uid=local(origin, "users", "u2"), target_uid=local(origin, "posts", "p1")) == 2
+
+
+def test_reactions_on_comments_and_projects_wait_for_their_target(client, admin, origin):
+    seed_world(client, admin, origin)
+    body = sync_ok(client, admin, "reactions", origin, [reaction_record("r1", target="c1", kind="comment"), reaction_record("r2", target="pr1", kind="project"), reaction_record("r3", target="ghost")])
+    assert [item["status"] for item in body["results"]] == ["created", "created", "deferred"]

tests/api/sync/votes.py

@@ -0,0 +1,82 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_table
+from tests.api.sync.conftest import seed_world, sync_ok, vote_record
+
+
+def local(origin, kind, origin_uid):
+    return get_table("sync_links").find_one(origin=origin, kind=kind, origin_uid=origin_uid)["local_uid"]
+
+
+def stars(table, uid):
+    return get_table(table).find_one(uid=uid)["stars"]
+
+
+def test_votes_derive_stars_from_synced_voters_only(client, admin, origin):
+    seed_world(client, admin, origin)
+    post = local(origin, "posts", "p1")
+    body = sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2"), vote_record("v2", "u3")])
+    assert [item["status"] for item in body["results"]] == ["created", "created"]
+    assert stars("posts", post) == 2
+    sync_ok(client, admin, "votes", origin, [vote_record("v3", "u1", value=-1)])
+    assert stars("posts", post) == 1
+    row = get_table("votes").find_one(uid=local(origin, "votes", "v1"))
+    assert row["target_type"] == "post" and row["user_uid"] == local(origin, "users", "u2")
+
+
+def test_a_changed_and_a_cleared_vote_move_the_total(client, admin, origin):
+    seed_world(client, admin, origin)
+    post = local(origin, "posts", "p1")
+    sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2")])
+    sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2", value=-1, updated_at="2025-02-01T00:00:00Z")])
+    assert stars("posts", post) == -1
+    sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2", value=-1, updated_at="2025-03-01T00:00:00Z", deleted=True)])
+    assert stars("posts", post) == 0
+    sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2", value=1, updated_at="2025-04-01T00:00:00Z")])
+    assert stars("posts", post) == 1
+    assert get_table("votes").count(user_uid=local(origin, "users", "u2"), target_uid=post) == 1
+
+
+def test_a_second_origin_id_for_the_same_voter_revives_the_same_row(client, admin, origin):
+    seed_world(client, admin, origin)
+    post = local(origin, "posts", "p1")
+    sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2")])
+    sync_ok(client, admin, "votes", origin, [vote_record("v9", "u2", value=-1)])
+    assert get_table("votes").count(user_uid=local(origin, "users", "u2"), target_uid=post, target_type="post") == 1
+    assert stars("posts", post) == -1
+
+
+def test_votes_on_comments_and_projects(client, admin, origin):
+    seed_world(client, admin, origin)
+    comment_votes = sync_ok(client, admin, "votes", origin, [vote_record("v1", "u3", "c1", "comment")])
+    assert comment_votes["results"][0]["status"] == "created"
+    project = local(origin, "projects", "pr1")
+    sync_ok(client, admin, "votes", origin, [vote_record("v2", "u2", "pr1", "project")])
+    assert stars("projects", project) == 1
+
+
+def test_value_target_and_voter_validation(client, admin, origin):
+    seed_world(client, admin, origin)
+    body = sync_ok(
+        client, admin, "votes", origin,
+        [vote_record("v1", value=5), vote_record("v2", kind="gist"), vote_record("v3", voter="ghost"), vote_record("v4", target="ghost")],
+    )
+    statuses = [(item["status"], item["reason"]) for item in body["results"]]
+    assert statuses == [("rejected", "invalid"), ("rejected", "invalid"), ("deferred", "unknown_author"), ("deferred", "unknown_target")]
+    assert body["results"][3]["needs_kind"] == "posts"
+
+
+def test_a_locally_deleted_vote_stays_deleted(client, admin, origin):
+    seed_world(client, admin, origin)
+    created = sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2")])["results"][0]
+    get_table("votes").update({"uid": created["local_uid"], "deleted_at": "2025-06-01T00:00:00+00:00", "deleted_by": "some-moderator"}, ["uid"])
+    again = sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2", value=-1, updated_at="2025-07-01T00:00:00Z")])["results"][0]
+    assert again["status"] == "unchanged" and again["reason"] == "locally_deleted"
+
+
+def test_dry_run_writes_nothing(client, admin, origin):
+    seed_world(client, admin, origin)
+    body = sync_ok(client, admin, "votes", origin, [vote_record("v1", "u2")], dry_run=True)
+    assert body["results"][0]["status"] == "created"
+    assert get_table("votes").count(target_uid=local(origin, "posts", "p1")) == 0
+    assert stars("posts", local(origin, "posts", "p1")) == 0