Add personal notes, DeepSearch history, backup offload, and gateway auth throttling

Also streamline the top navigation: drop the Tools dropdown, make Quizzes
and Battles icon-only entries, and remove the Workspace, Containers and
Editor entry points from the project detail page.

tests/api/docs/index.py

@@ -452,6 +452,29 @@ def test_profiles_api_doc_delete_is_member_safe(app_server):
     assert "purge" not in lowered
 
 
+def test_docs_content_response_fields_carry_correct_nullable_flags(app_server):
+    by_id = _configs_by_id(requests.get(f"{BASE_URL}/docs/content.html").text)
+    posts_detail = {p["name"]: p for p in by_id["posts-detail"]["params"] if p["location"] == "response"}
+    assert posts_detail["post.title"]["nullable"] is True
+    assert posts_detail["post.image"]["nullable"] is True
+    assert posts_detail["post.content"]["nullable"] is False
+    assert posts_detail["post.slug"]["nullable"] is False
+    feed_list = {p["name"]: p for p in by_id["feed-list"]["params"] if p["location"] == "response"}
+    assert feed_list["items[].poll"]["nullable"] is True
+    assert feed_list["items[].war"]["nullable"] is True
+    assert feed_list["items[].my_vote"]["nullable"] is False
+    assert feed_list["items[].comment_count"]["nullable"] is False
+
+
+def test_docs_profiles_response_fields_carry_correct_nullable_flags(app_server):
+    by_id = _configs_by_id(requests.get(f"{BASE_URL}/docs/profiles.html").text)
+    profile_detail = {p["name"]: p for p in by_id["profile-detail"]["params"] if p["location"] == "response"}
+    assert profile_detail["api_key"]["nullable"] is True
+    assert profile_detail["rank"]["nullable"] is True
+    assert profile_detail["is_following"]["nullable"] is False
+    assert profile_detail["profile_user.username"]["nullable"] is False
+
+
 def test_admin_api_doc_has_moderation(seeded_db):
     admin = get_table("users").find_one(username="alice_test")
     text = requests.get(

tests/api/feed.py

@@ -437,6 +437,44 @@ def test_strict_transport_security_header(app_server):
     assert "includeSubDomains" in r.headers.get("Strict-Transport-Security", "")
 
 
+def test_feed_card_truncation_does_not_break_a_straddling_image_url(app_server):
+    s, _ = _session_polls()
+    prefix = "word " * 58
+    url = "https://cdn.example.com/gallery/a-descriptive-name-for-the-photo.jpg"
+    content = prefix + url + " trailing words that must not leak into the truncated preview"
+    r = s.post(
+        f"{BASE_URL}/posts/create",
+        data={
+            "content": content,
+            "title": f"trunc-{int(time.time() * 1000)}",
+            "topic": "devlog",
+        },
+        allow_redirects=False,
+    )
+    assert r.status_code == 302, r.text
+    html = s.get(f"{BASE_URL}/feed").text
+    assert "cdn.example.com" not in html
+    assert "trailing words" not in html
+
+
+def test_feed_card_renders_full_image_embed_when_url_fits_before_truncation(app_server):
+    s, _ = _session_polls()
+    url = "https://cdn.example.com/photo.jpg"
+    content = url + " " + ("caption word " * 5)
+    r = s.post(
+        f"{BASE_URL}/posts/create",
+        data={
+            "content": content,
+            "title": f"embed-{int(time.time() * 1000)}",
+            "topic": "devlog",
+        },
+        allow_redirects=False,
+    )
+    assert r.status_code == 302, r.text
+    html = s.get(f"{BASE_URL}/feed").text
+    assert f'<img src="{url}"' in html
+
+
 def test_referrer_policy_header(app_server):
     r = requests.get(f"{BASE_URL}/feed", allow_redirects=True)
     assert r.headers.get("Referrer-Policy") == "strict-origin-when-cross-origin"

tests/api/messages/search.py

@@ -37,6 +37,39 @@ def _upload_uploads(s, name="a.png"):
     return r.json()["uid"]
 
 
+JSON_search = {"Accept": "application/json"}
+
+
+def test_search_recipients_includes_avatar_seed_for_custom_seed(app_server):
+    alice = _session_uploads()
+    bob, bob_name = _user_uploads("bob")
+
+    r = bob.post(
+        f"{BASE_URL}/profile/{bob_name}/regenerate-avatar",
+        headers=JSON_search,
+    )
+    assert r.status_code == 200, r.text[:300]
+    seed = r.json()["data"]["avatar_seed"]
+    assert seed
+
+    found = alice.get(f"{BASE_URL}/messages/search", params={"q": bob_name}).json()[
+        "results"
+    ]
+    match = next(x for x in found if x["username"] == bob_name)
+    assert match["avatar_seed"] == seed
+
+
+def test_search_recipients_avatar_seed_null_without_custom_seed(app_server):
+    alice = _session_uploads()
+    bob, bob_name = _user_uploads("bob")
+
+    found = alice.get(f"{BASE_URL}/messages/search", params={"q": bob_name}).json()[
+        "results"
+    ]
+    match = next(x for x in found if x["username"] == bob_name)
+    assert match.get("avatar_seed") is None
+
+
 def test_message_links_multiple_attachments(app_server):
     alice = _session_uploads()
     bob, bob_name = _user_uploads("bob")

tests/api/messages/ws.py

@@ -207,6 +207,56 @@ def test_ws_sync_replays_messages_since(app_server):
     asyncio.run(run())
 
 
+def test_ws_active_marker_suppresses_message_notification(app_server):
+    sender_session, sender_name = _member()
+    receiver_session, receiver_name = _member()
+    sender_uid = _db_user(sender_name)["uid"]
+    receiver_uid = _db_user(receiver_name)["uid"]
+
+    async def run():
+        async with websockets.connect(
+            WS_URL, additional_headers=_cookie_header(receiver_session)
+        ) as receiver_ws:
+            await _recv_until(receiver_ws, "ready")
+            await receiver_ws.send(
+                json.dumps({"type": "active", "with_uid": sender_uid})
+            )
+
+    asyncio.run(run())
+
+    resp = sender_session.post(
+        f"{BASE_URL}/messages/send",
+        headers={"Accept": "application/json"},
+        data={"content": "are you watching this", "receiver_uid": receiver_uid},
+    )
+    assert resp.status_code == 200
+
+    refresh_snapshot()
+    notifications = list(
+        get_table("notifications").find(user_uid=receiver_uid, type="message")
+    )
+    assert notifications == []
+
+
+def test_ws_without_active_marker_still_notifies(app_server):
+    sender_session, _ = _member()
+    receiver_session, receiver_name = _member()
+    receiver_uid = _db_user(receiver_name)["uid"]
+
+    resp = sender_session.post(
+        f"{BASE_URL}/messages/send",
+        headers={"Accept": "application/json"},
+        data={"content": "hello without watching", "receiver_uid": receiver_uid},
+    )
+    assert resp.status_code == 200
+
+    refresh_snapshot()
+    notifications = list(
+        get_table("notifications").find(user_uid=receiver_uid, type="message")
+    )
+    assert len(notifications) == 1
+
+
 def test_ws_ping_and_second_send_keep_the_socket_open(app_server):
     sender_session, _ = _member()
     receiver_session, receiver_name = _member()

tests/api/notes/index.py

@@ -0,0 +1,207 @@
+# retoor <retoor@molodetz.nl>
+
+import time
+from datetime import datetime, timezone
+import requests
+from tests.conftest import BASE_URL
+from devplacepy.database import get_table
+from devplacepy.utils import generate_uid
+
+_counter_notes = [0]
+AJAX_notes = {"X-Requested-With": "fetch"}
+
+
+def _session_notes():
+    _counter_notes[0] += 1
+    name = f"nte{int(time.time() * 1000)}{_counter_notes[0]}"
+    s = requests.Session()
+    s.post(
+        f"{BASE_URL}/auth/signup",
+        data={
+            "username": name,
+            "email": f"{name}@t.dev",
+            "password": "secret123",
+            "confirm_password": "secret123",
+            "birth_date": "1990-01-01",
+            "accept_terms": "1",
+        },
+        allow_redirects=True,
+    )
+    return s, name
+
+
+def _uid_notes(username):
+    return get_table("users").find_one(username=username)["uid"]
+
+
+def _make_post_notes(owner_uid, title):
+    uid = generate_uid()
+    get_table("posts").insert(
+        {
+            "deleted_at": None,
+            "deleted_by": None,
+            "uid": uid,
+            "user_uid": owner_uid,
+            "slug": f"{uid[-8:]}-note-post",
+            "title": title,
+            "content": "note target content",
+            "topic": "random",
+            "project_uid": None,
+            "image": None,
+            "stars": 0,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+    )
+    return uid
+
+
+def test_note_set_creates_row(app_server):
+    s, name = _session_notes()
+    post_uid = _make_post_notes(_uid_notes(name), "Note target")
+    r = s.post(
+        f"{BASE_URL}/notes/post/{post_uid}",
+        headers=AJAX_notes,
+        data={"content": "Remember to check this later."},
+    )
+    assert r.status_code == 200
+    assert r.json()["content"] == "Remember to check this later."
+    row = get_table("notes").find_one(
+        user_uid=_uid_notes(name), target_uid=post_uid, deleted_at=None
+    )
+    assert row is not None
+    assert row["content"] == "Remember to check this later."
+
+
+def test_note_set_replaces_existing_content(app_server):
+    s, name = _session_notes()
+    post_uid = _make_post_notes(_uid_notes(name), "Note replace target")
+    s.post(
+        f"{BASE_URL}/notes/post/{post_uid}",
+        headers=AJAX_notes,
+        data={"content": "First draft."},
+    )
+    s.post(
+        f"{BASE_URL}/notes/post/{post_uid}",
+        headers=AJAX_notes,
+        data={"content": "Second draft."},
+    )
+    rows = list(
+        get_table("notes").find(
+            user_uid=_uid_notes(name), target_uid=post_uid, deleted_at=None
+        )
+    )
+    assert len(rows) == 1
+    assert rows[0]["content"] == "Second draft."
+
+
+def test_note_delete_soft_deletes(app_server):
+    s, name = _session_notes()
+    post_uid = _make_post_notes(_uid_notes(name), "Note delete target")
+    s.post(
+        f"{BASE_URL}/notes/post/{post_uid}",
+        headers=AJAX_notes,
+        data={"content": "Delete me."},
+    )
+    r = s.post(f"{BASE_URL}/notes/post/{post_uid}/delete", headers=AJAX_notes)
+    assert r.json() == {"deleted": True}
+    assert (
+        get_table("notes").count(
+            user_uid=_uid_notes(name), target_uid=post_uid, deleted_at=None
+        )
+        == 0
+    )
+    row = get_table("notes").find_one(user_uid=_uid_notes(name), target_uid=post_uid)
+    assert row is not None
+    assert row["deleted_at"] is not None
+    assert row["deleted_by"] == _uid_notes(name)
+
+
+def test_note_revives_after_delete(app_server):
+    s, name = _session_notes()
+    post_uid = _make_post_notes(_uid_notes(name), "Note revive target")
+    s.post(
+        f"{BASE_URL}/notes/post/{post_uid}",
+        headers=AJAX_notes,
+        data={"content": "Before delete."},
+    )
+    s.post(f"{BASE_URL}/notes/post/{post_uid}/delete", headers=AJAX_notes)
+    s.post(
+        f"{BASE_URL}/notes/post/{post_uid}",
+        headers=AJAX_notes,
+        data={"content": "After revive."},
+    )
+    rows = list(get_table("notes").find(user_uid=_uid_notes(name), target_uid=post_uid))
+    assert len(rows) == 1
+    assert rows[0]["deleted_at"] is None
+    assert rows[0]["content"] == "After revive."
+
+
+def test_note_requires_login(app_server):
+    owner_s, owner_name = _session_notes()
+    post_uid = _make_post_notes(_uid_notes(owner_name), "Needs login")
+    anon = requests.Session()
+    r = anon.post(
+        f"{BASE_URL}/notes/post/{post_uid}",
+        headers=AJAX_notes,
+        data={"content": "sneaky"},
+        allow_redirects=False,
+    )
+    assert r.status_code == 303
+    assert get_table("notes").count(target_uid=post_uid) == 0
+
+
+def test_note_invalid_target_type_returns_400(app_server):
+    s, name = _session_notes()
+    post_uid = _make_post_notes(_uid_notes(name), "Bad target")
+    r = s.post(
+        f"{BASE_URL}/notes/widget/{post_uid}",
+        headers=AJAX_notes,
+        data={"content": "does not matter"},
+    )
+    assert r.status_code == 400
+
+
+def test_note_is_private_per_user(app_server):
+    alice, alice_name = _session_notes()
+    bob, bob_name = _session_notes()
+    post_uid = _make_post_notes(_uid_notes(alice_name), "Shared target")
+    alice.post(
+        f"{BASE_URL}/notes/post/{post_uid}",
+        headers=AJAX_notes,
+        data={"content": "Alice's private note."},
+    )
+    bob.post(
+        f"{BASE_URL}/notes/post/{post_uid}",
+        headers=AJAX_notes,
+        data={"content": "Bob's private note."},
+    )
+    alice_row = get_table("notes").find_one(
+        user_uid=_uid_notes(alice_name), target_uid=post_uid, deleted_at=None
+    )
+    bob_row = get_table("notes").find_one(
+        user_uid=_uid_notes(bob_name), target_uid=post_uid, deleted_at=None
+    )
+    assert alice_row["content"] == "Alice's private note."
+    assert bob_row["content"] == "Bob's private note."
+
+    bob_view = bob.get(
+        f"{BASE_URL}/posts/{post_uid}", headers={"Accept": "application/json"}
+    )
+    assert bob_view.json()["note_content"] == "Bob's private note."
+
+
+def test_note_delete_only_affects_own_row(app_server):
+    alice, alice_name = _session_notes()
+    bob, bob_name = _session_notes()
+    post_uid = _make_post_notes(_uid_notes(alice_name), "Delete isolation target")
+    alice.post(
+        f"{BASE_URL}/notes/post/{post_uid}",
+        headers=AJAX_notes,
+        data={"content": "Alice keeps this."},
+    )
+    bob.post(f"{BASE_URL}/notes/post/{post_uid}/delete", headers=AJAX_notes)
+    alice_row = get_table("notes").find_one(
+        user_uid=_uid_notes(alice_name), target_uid=post_uid, deleted_at=None
+    )
+    assert alice_row is not None
+    assert alice_row["content"] == "Alice keeps this."

tests/api/notes/saved.py

@@ -0,0 +1,89 @@
+# retoor <retoor@molodetz.nl>
+
+import time
+from datetime import datetime, timezone
+import requests
+from tests.conftest import BASE_URL
+from devplacepy.database import get_table
+from devplacepy.utils import generate_uid
+
+_counter_notes = [0]
+AJAX_notes = {"X-Requested-With": "fetch"}
+
+
+def _session_notes():
+    _counter_notes[0] += 1
+    name = f"ntesv{int(time.time() * 1000)}{_counter_notes[0]}"
+    s = requests.Session()
+    s.post(
+        f"{BASE_URL}/auth/signup",
+        data={
+            "username": name,
+            "email": f"{name}@t.dev",
+            "password": "secret123",
+            "confirm_password": "secret123",
+            "birth_date": "1990-01-01",
+            "accept_terms": "1",
+        },
+        allow_redirects=True,
+    )
+    return s, name
+
+
+def _uid_notes(username):
+    return get_table("users").find_one(username=username)["uid"]
+
+
+def _make_post_notes(owner_uid, title):
+    uid = generate_uid()
+    get_table("posts").insert(
+        {
+            "deleted_at": None,
+            "deleted_by": None,
+            "uid": uid,
+            "user_uid": owner_uid,
+            "slug": f"{uid[:8]}-note-saved-post",
+            "title": title,
+            "content": "note saved target content",
+            "topic": "random",
+            "project_uid": None,
+            "image": None,
+            "stars": 0,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+    )
+    return uid
+
+
+def test_notes_saved_page_lists_note_content(app_server):
+    s, name = _session_notes()
+    owner_uid = _uid_notes(name)
+    post_title = f"Note saved post {int(time.time() * 1000)}"
+    post_uid = _make_post_notes(owner_uid, post_title)
+    s.post(
+        f"{BASE_URL}/notes/post/{post_uid}",
+        headers=AJAX_notes,
+        data={"content": "A note worth remembering."},
+    )
+    html = s.get(f"{BASE_URL}/notes/saved").text
+    assert post_title in html
+    assert "A note worth remembering." in html
+
+
+def test_notes_saved_page_requires_login(app_server):
+    anon = requests.Session()
+    r = anon.get(f"{BASE_URL}/notes/saved", allow_redirects=False)
+    assert r.status_code == 303
+
+
+def test_notes_saved_page_only_shows_own_notes(app_server):
+    alice, alice_name = _session_notes()
+    bob, bob_name = _session_notes()
+    post_uid = _make_post_notes(_uid_notes(alice_name), "Isolation saved post")
+    alice.post(
+        f"{BASE_URL}/notes/post/{post_uid}",
+        headers=AJAX_notes,
+        data={"content": "Alice only note."},
+    )
+    html = bob.get(f"{BASE_URL}/notes/saved").text
+    assert "Alice only note." not in html

tests/api/notifications.py

@@ -1,7 +1,9 @@
 # retoor <retoor@molodetz.nl>
 
+import re
 import time
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
+from urllib.parse import parse_qs, urlparse
 import pytest
 import requests
 from tests.conftest import BASE_URL
@@ -128,3 +130,44 @@ def test_pagination_spans_pages(seeded_db):
     assert data["pagination"]["total_pages"] >= 2
     page2 = _audit(admin, event_key="security.authz.denied", page=2)
     assert len(page2["entries"]) > 0
+
+
+def _seed_notifications(user_uid, count=30):
+    notifications = get_table("notifications")
+    base = datetime.now(timezone.utc)
+    for i in range(count):
+        notifications.insert(
+            {
+                "uid": generate_uid(),
+                "user_uid": user_uid,
+                "type": "test",
+                "message": f"quota notification {i}",
+                "related_uid": user_uid,
+                "target_url": None,
+                "read": False,
+                "created_at": (base - timedelta(seconds=i)).isoformat(),
+            }
+        )
+    refresh_snapshot()
+
+
+def test_load_more_link_is_percent_encoded(seeded_db):
+    session, name = _member()
+    user = _db_user(name)
+    _seed_notifications(user["uid"])
+
+    r = session.get(f"{BASE_URL}/notifications")
+    assert r.status_code == 200
+    match = re.search(r'href="(/notifications\?before=[^"]*)"', r.text)
+    assert match, "expected a Load More link on the notifications page"
+    href = match.group(1)
+
+    assert "%2B" in href
+    assert "+" not in href
+
+    cursor = parse_qs(urlparse(href).query)["before"][0]
+    assert "+" in cursor
+
+    r2 = session.get(f"{BASE_URL}{href}")
+    assert r2.status_code == 200
+    assert "quota notification 29" in r2.text

tests/api/profile/search.py

@@ -280,6 +280,29 @@ def test_profile_renders_heatmap_and_streak(app_server):
     assert "1 day streak" in html
 
 
+def test_search_users_includes_avatar_seed_for_custom_seed(app_server):
+    s, name = _signup_media()
+    r = s.post(
+        f"{BASE_URL}/profile/{name}/regenerate-avatar",
+        headers=JSON_media,
+    )
+    assert r.status_code == 200, r.text[:300]
+    seed = r.json()["data"]["avatar_seed"]
+    assert seed
+
+    found = s.get(f"{BASE_URL}/profile/search", params={"q": name}).json()["results"]
+    match = next(x for x in found if x["username"] == name)
+    assert match["avatar_seed"] == seed
+
+
+def test_search_users_avatar_seed_null_without_custom_seed(app_server):
+    s, name = _signup_media()
+
+    found = s.get(f"{BASE_URL}/profile/search", params={"q": name}).json()["results"]
+    match = next(x for x in found if x["username"] == name)
+    assert match.get("avatar_seed") is None
+
+
 def test_profile_badges_json_has_non_null_names(app_server):
     import time
     from devplacepy.database import get_table, refresh_snapshot

tests/api/pushjson.py

@@ -121,6 +121,87 @@ def test_register_non_object_body_rejected(app_server):
     assert r.status_code == 400
 
 
+def test_unregister_requires_auth(app_server):
+    r = requests.delete(
+        f"{BASE_URL}/push.json",
+        json={"endpoint": "https://push.example.com/anon"},
+        allow_redirects=False,
+    )
+    assert r.status_code == 401
+
+
+def test_unregister_invalid_json_rejected(app_server):
+    s = _session_push()
+    r = s.delete(
+        f"{BASE_URL}/push.json",
+        data="not-json",
+        headers={"Content-Type": "application/json"},
+    )
+    assert r.status_code == 400
+
+
+def test_unregister_non_object_body_rejected(app_server):
+    s = _session_push()
+    r = s.delete(f"{BASE_URL}/push.json", json=["nope"])
+    assert r.status_code == 400
+
+
+def test_unregister_unknown_provider_rejected(app_server):
+    s = _session_push()
+    r = s.delete(
+        f"{BASE_URL}/push.json",
+        json={"provider": "carrier-pigeon", "endpoint": "https://push.example.com/x"},
+    )
+    assert r.status_code == 400
+
+
+def test_unregister_missing_identity_rejected(app_server):
+    s = _session_push()
+    r = s.delete(f"{BASE_URL}/push.json", json={})
+    assert r.status_code == 400
+
+
+def test_unregister_unknown_identity_is_idempotent(app_server):
+    s = _session_push()
+    r = s.delete(
+        f"{BASE_URL}/push.json",
+        json={"endpoint": "https://push.example.com/never-registered"},
+    )
+    assert r.status_code == 200, r.text
+    assert r.json().get("unregistered") is False
+
+
+def test_unregister_removes_a_registered_subscription(app_server):
+    s = _session_push()
+    body = {
+        "endpoint": "https://push.example.com/to-unregister",
+        "keys": {"p256dh": "p256dh_fake", "auth": "auth_fake"},
+    }
+    assert s.post(f"{BASE_URL}/push.json", json=body).status_code == 200
+
+    r = s.delete(f"{BASE_URL}/push.json", json={"endpoint": body["endpoint"]})
+    assert r.status_code == 200, r.text
+    assert r.json().get("unregistered") is True
+
+    r = s.delete(f"{BASE_URL}/push.json", json={"endpoint": body["endpoint"]})
+    assert r.status_code == 200, r.text
+    assert r.json().get("unregistered") is False
+
+
+def test_unregister_then_register_revives_the_row(app_server):
+    s = _session_push()
+    body = {
+        "endpoint": "https://push.example.com/revive-after-unregister",
+        "keys": {"p256dh": "p256dh_fake", "auth": "auth_fake"},
+    }
+    assert s.post(f"{BASE_URL}/push.json", json=body).status_code == 200
+    assert s.delete(f"{BASE_URL}/push.json", json={"endpoint": body["endpoint"]}).status_code == 200
+
+    r = s.post(f"{BASE_URL}/push.json", json=body)
+    assert r.status_code == 200, r.text
+    assert r.json().get("registered") is True
+
+
 def test_register_ignores_client_id_on_webpush(app_server):
     s = _session_push()
     r = s.post(

tests/api/tools/deepsearch/history.py

@@ -0,0 +1,173 @@
+# retoor <retoor@molodetz.nl>
+
+import time
+import pytest
+import requests
+from tests.conftest import BASE_URL
+from devplacepy.database import (
+    create_deepsearch_session,
+    get_table,
+    refresh_snapshot,
+    set_setting,
+    update_deepsearch_session,
+)
+from devplacepy.services.jobs import queue
+
+JSON = {"Accept": "application/json"}
+_counter_dsh = [0]
+
+
+@pytest.fixture(scope="module", autouse=True)
+def _settings(app_server):
+    set_setting("rate_limit_per_minute", "1000000")
+    set_setting("registration_open", "1")
+    yield
+
+
+def _user():
+    _counter_dsh[0] += 1
+    name = f"dsh{int(time.time() * 1000)}{_counter_dsh[0]}"
+    requests.post(
+        f"{BASE_URL}/auth/signup",
+        data={
+            "username": name,
+            "email": f"{name}@t.dev",
+            "password": "secret123",
+            "confirm_password": "secret123",
+            "birth_date": "1990-01-01",
+            "accept_terms": "1",
+        },
+        allow_redirects=True,
+    )
+    refresh_snapshot()
+    user = get_table("users").find_one(username=name)
+    return user["uid"], user["api_key"]
+
+
+def _seed_session(owner_id, status="done", keep_job=True, query="a research question"):
+    uid = queue.enqueue(
+        "deepsearch",
+        {"query": query, "depth": 1, "max_pages": 5},
+        "user",
+        owner_id,
+        f"DeepSearch: {query}",
+    )
+    create_deepsearch_session(
+        uid, "user", owner_id, query, 1, 5, f"ds_{uid.replace('-', '')}"
+    )
+    update_deepsearch_session(
+        uid,
+        {
+            "status": status,
+            "score": 80,
+            "confidence": 0.7,
+            "source_diversity": 0.6,
+            "page_count": 5,
+            "chunk_count": 40,
+            "summary": "A short summary of findings.",
+        },
+    )
+    if not keep_job:
+        get_table("jobs").delete(uid=uid)
+    refresh_snapshot()
+    return uid
+
+
+def test_history_lists_own_sessions_newest_first(app_server):
+    owner_uid, key = _user()
+    first = _seed_session(owner_uid, query="first question")
+    time.sleep(0.01)
+    second = _seed_session(owner_uid, query="second question")
+    r = requests.get(
+        f"{BASE_URL}/tools/deepsearch/history",
+        headers={"X-API-KEY": key, **JSON},
+    )
+    assert r.status_code == 200, r.text
+    body = r.json()
+    uids = [s["uid"] for s in body["sessions"]]
+    assert uids.index(second) < uids.index(first)
+
+
+def test_history_item_shape_for_done_session(app_server):
+    owner_uid, key = _user()
+    uid = _seed_session(owner_uid)
+    r = requests.get(
+        f"{BASE_URL}/tools/deepsearch/history",
+        headers={"X-API-KEY": key, **JSON},
+    )
+    assert r.status_code == 200, r.text
+    items = {s["uid"]: s for s in r.json()["sessions"]}
+    item = items[uid]
+    assert item["query"] == "a research question"
+    assert item["status"] == "done"
+    assert item["score"] == 80
+    assert item["reopen_url"] == f"/tools/deepsearch/{uid}/session"
+    assert item["chat_available"] is True
+    assert item["available"] is True
+
+
+def test_history_marks_expired_session_unavailable(app_server):
+    owner_uid, key = _user()
+    uid = _seed_session(owner_uid, keep_job=False)
+    r = requests.get(
+        f"{BASE_URL}/tools/deepsearch/history",
+        headers={"X-API-KEY": key, **JSON},
+    )
+    assert r.status_code == 200, r.text
+    items = {s["uid"]: s for s in r.json()["sessions"]}
+    item = items[uid]
+    assert item["available"] is False
+    assert item["chat_available"] is False
+
+
+def test_history_is_scoped_to_the_owner(app_server):
+    owner_uid, key = _user()
+    other_uid, other_key = _user()
+    mine = _seed_session(owner_uid, query="mine only")
+    _seed_session(other_uid, query="not mine")
+    r = requests.get(
+        f"{BASE_URL}/tools/deepsearch/history",
+        headers={"X-API-KEY": key, **JSON},
+    )
+    assert r.status_code == 200, r.text
+    uids = [s["uid"] for s in r.json()["sessions"]]
+    assert mine in uids
+    r_other = requests.get(
+        f"{BASE_URL}/tools/deepsearch/history",
+        headers={"X-API-KEY": other_key, **JSON},
+    )
+    other_uids = [s["uid"] for s in r_other.json()["sessions"]]
+    assert mine not in other_uids
+
+
+def test_history_scopes_guests_by_ip(app_server):
+    uid = queue.enqueue(
+        "deepsearch",
+        {"query": "guest question", "depth": 1, "max_pages": 5},
+        "guest",
+        "203.0.113.55",
+        "DeepSearch: guest question",
+    )
+    create_deepsearch_session(
+        uid, "guest", "203.0.113.55", "guest question", 1, 5, f"ds_{uid.replace('-', '')}"
+    )
+    refresh_snapshot()
+    r = requests.get(
+        f"{BASE_URL}/tools/deepsearch/history",
+        headers={"X-Real-IP": "203.0.113.55", **JSON},
+    )
+    assert r.status_code == 200, r.text
+    uids = [s["uid"] for s in r.json()["sessions"]]
+    assert uid in uids
+    r_other_ip = requests.get(
+        f"{BASE_URL}/tools/deepsearch/history",
+        headers={"X-Real-IP": "203.0.113.99", **JSON},
+    )
+    other_uids = [s["uid"] for s in r_other_ip.json()["sessions"]]
+    assert uid not in other_uids
+
+
+def test_history_renders_html(app_server):
+    r = requests.get(f"{BASE_URL}/tools/deepsearch/history")
+    assert r.status_code == 200
+    assert "DeepSearch history" in r.text

tests/api/votes.py

@@ -127,6 +127,35 @@ def test_self_vote_does_not_notify(app_server):
     assert after == before
 
 
+def test_vote_zero_retracts_existing_vote(app_server):
+    s_a, a_name = _session_votes()
+    s_b, _ = _session_votes()
+    post_uid = _make_post_votes(_uid_votes(a_name))
+    s_b.post(f"{BASE_URL}/votes/post/{post_uid}", data={"value": "1"}, headers=AJAX_votes)
+    r = s_b.post(f"{BASE_URL}/votes/post/{post_uid}", data={"value": "0"}, headers=AJAX_votes)
+    assert r.json() == {"net": 0, "up": 0, "down": 0, "value": 0}
+
+
+def test_vote_zero_with_no_prior_vote_is_a_noop(app_server):
+    s_a, a_name = _session_votes()
+    s_b, _ = _session_votes()
+    post_uid = _make_post_votes(_uid_votes(a_name))
+    r = s_b.post(f"{BASE_URL}/votes/post/{post_uid}", data={"value": "0"}, headers=AJAX_votes)
+    assert r.status_code == 200
+    assert r.json() == {"net": 0, "up": 0, "down": 0, "value": 0}
+    votes = get_table("votes")
+    assert votes.count(target_uid=post_uid, target_type="post") == 0
+
+
+def test_vote_zero_after_downvote_retracts(app_server):
+    s_a, a_name = _session_votes()
+    s_b, _ = _session_votes()
+    post_uid = _make_post_votes(_uid_votes(a_name))
+    s_b.post(f"{BASE_URL}/votes/post/{post_uid}", data={"value": "-1"}, headers=AJAX_votes)
+    r = s_b.post(f"{BASE_URL}/votes/post/{post_uid}", data={"value": "0"}, headers=AJAX_votes)
+    assert r.json() == {"net": 0, "up": 0, "down": 0, "value": 0}
+
+
 def test_non_ajax_vote_redirects_to_referer(app_server):
     s_a, a_name = _session_votes()
     s_b, _ = _session_votes()

tests/e2e/battles/index.py

@@ -52,7 +52,7 @@ def test_battles_page_lists_cards(alice):
     _seed_war("Vikings", "Knights")
     page.goto(f"{BASE_URL}/battles", wait_until="domcontentloaded")
     page.locator("dp-opinion-war").first.wait_for(state="visible")
-    expect(page.locator(".topnav-link.active:has-text('Battles')")).to_be_visible()
+    expect(page.locator(".topnav-icon.active[href='/battles']")).to_be_visible()
     expect(page.locator("dp-opinion-war .war-vs").first).to_have_text("VS")
 
 

tests/e2e/comments/copylink.py

@@ -0,0 +1,28 @@
+# retoor <retoor@molodetz.nl>
+
+from playwright.sync_api import expect
+from tests.e2e.post import create_post
+
+
+def test_comment_copy_link_button(alice):
+    page, _ = alice
+    create_post(page, "devlog", "Post for comment permalink test")
+    textarea = page.locator(".comment-form textarea[name='content']")
+    textarea.fill("Comment for permalink test")
+    page.locator(".comment-form button:has-text('Post')").click()
+    comment_text = page.locator(".comment-text:has-text('Comment for permalink test')")
+    expect(comment_text).to_be_visible()
+    wrapper = page.locator(".comment-body", has=comment_text)
+    comment_uid = wrapper.get_attribute("data-comment-uid")
+    assert comment_uid
+    copy_btn = page.locator(f".comment-action-btn[data-share='#comment-{comment_uid}']")
+    expect(copy_btn).to_be_visible()
+    copy_btn.click()
+    expect(copy_btn).to_have_text("Copied!", timeout=3000)
+    expect(copy_btn).not_to_have_text("Copied!", timeout=3000)
+    try:
+        clip = page.evaluate("navigator.clipboard.readText()")
+    except Exception:
+        clip = None
+    if clip:
+        assert clip.endswith(f"#comment-{comment_uid}"), f"clipboard={clip!r}"

tests/e2e/feed.py

@@ -1018,6 +1018,35 @@ def test_feed_scroll_not_restored_on_fresh_visit(alice):
     assert page.evaluate("window.scrollY") < 60
 
 
+def test_feed_right_rail_stays_plain_sticky_when_it_fits(alice):
+    page, user = alice
+    page.set_viewport_size({"width": 1400, "height": 2000})
+    page.goto(f"{BASE_URL}/feed", wait_until="domcontentloaded")
+    right = page.locator(".feed-right")
+    right.wait_for(state="visible")
+    page.wait_for_timeout(400)
+    assert right.evaluate("el => el.classList.contains('sticky-scroll-active')") is False
+    assert right.evaluate("el => getComputedStyle(el).maxHeight") == "none"
+
+
+def test_feed_right_rail_scrolls_independently_when_taller_than_viewport(alice):
+    page, user = alice
+    page.set_viewport_size({"width": 1400, "height": 400})
+    page.goto(f"{BASE_URL}/feed", wait_until="domcontentloaded")
+    right = page.locator(".feed-right")
+    right.wait_for(state="visible")
+    page.wait_for_function(
+        "document.querySelector('.feed-right').classList.contains('sticky-scroll-active')"
+    )
+    page.evaluate("window.scrollTo(0, 0)")
+    right.evaluate("el => el.scrollTo(0, el.scrollHeight)")
+    page.wait_for_timeout(200)
+    right_scroll_top = right.evaluate("el => el.scrollTop")
+    window_scroll_y = page.evaluate("window.scrollY")
+    assert right_scroll_top > 0
+    assert window_scroll_y == 0
+
+
 def _seed_post_with_long_code_line(topic="devlog"):
     owner = str(uuid4())
     get_table("users").insert(

tests/e2e/gists.py

@@ -230,8 +230,8 @@ def test_delete_gist(alice, app_server):
     page, _ = alice
     title = f"Delete Test {int(time.time())}"
     _create_gist(page, title=title, source_code="delete_me = True")
-    page.locator("button:has-text('Delete')").wait_for(state="visible", timeout=5000)
-    page.click("button:has-text('Delete')")
+    page.locator("button[data-confirm='Delete this gist?']").wait_for(state="visible", timeout=5000)
+    page.click("button[data-confirm='Delete this gist?']")
     page.locator(".dialog-overlay.visible .dialog-confirm").click()
     page.wait_for_url(f"{BASE_URL}/gists", timeout=10000, wait_until="domcontentloaded")
 
@@ -275,6 +275,44 @@ def test_gist_code_copy_button(alice, app_server):
     expect(btn).to_have_text("Copied!", timeout=3000)
 
 
+def test_markdown_gist_raw_rendered_toggle(alice, app_server):
+    from playwright.sync_api import expect
+
+    page, _ = alice
+    _create_gist(
+        page,
+        title=f"Markdown Toggle {int(time.time())}",
+        language="markdown",
+        source_code="# Heading\n\nSome **bold** text.",
+    )
+    toggle = page.locator("button[data-view-toggle='gist-view-raw']")
+    raw_view = page.locator("#gist-view-raw")
+    rendered_view = page.locator("#gist-view-rendered")
+    expect(toggle).to_have_text("View rendered")
+    expect(raw_view).to_be_visible()
+    expect(rendered_view).to_be_hidden()
+    toggle.click()
+    expect(toggle).to_have_text("View raw")
+    expect(rendered_view).to_be_visible()
+    expect(raw_view).to_be_hidden()
+    assert page.locator("#gist-view-rendered h1:has-text('Heading')").count() == 1
+    toggle.click()
+    expect(toggle).to_have_text("View rendered")
+    expect(raw_view).to_be_visible()
+    expect(rendered_view).to_be_hidden()
+
+
+def test_markdown_rendered_gist_has_no_toggle(alice, app_server):
+    page, _ = alice
+    _create_gist(
+        page,
+        title=f"No Toggle {int(time.time())}",
+        language="markdown_rendered",
+        source_code="# Already rendered",
+    )
+    assert page.locator("button[data-view-toggle]").count() == 0
+
+
 def test_language_filter(alice, app_server):
     page, _ = alice
     title = f"Lang Filter Test {int(time.time())}"

tests/e2e/messages.py

@@ -406,3 +406,120 @@ def test_live_incoming_message_shows_report_button(alice, bob):
     bubble = page_b.locator(f".message-bubble.theirs:has-text('{msg}')")
     bubble.wait_for(state="visible", timeout=10000)
     expect(bubble.locator(".message-report-btn")).to_be_visible()
+
+
+def test_send_button_locks_during_send_and_prevents_duplicate(alice, bob):
+    page_a, user_a = alice
+    page_b, user_b = bob
+    uid_b = get_table("users").find_one(username=user_b["username"])["uid"]
+
+    page_a.goto(
+        f"{BASE_URL}/messages?with_uid={uid_b}", wait_until="domcontentloaded"
+    )
+
+    msg = f"Locked send {int(time.time() * 1000)}"
+    textarea = page_a.locator(".messages-input-area textarea[name='content']")
+    textarea.wait_for(state="visible")
+    textarea.fill(msg)
+
+    send_btn = page_a.locator(".messages-send-btn")
+    send_btn.click()
+
+    # the composer refuses a duplicate submit while this send is unconfirmed
+    expect(send_btn).to_be_disabled()
+
+    bubble = page_a.locator(f".message-bubble.mine:has-text('{msg}')")
+    reconciled = page_a.locator(
+        f".message-bubble.mine:has-text('{msg}')[data-msg-uid]:not(.pending)"
+    )
+    reconciled.first.wait_for(state="visible", timeout=10000)
+
+    expect(send_btn).to_be_enabled()
+    assert bubble.count() == 1
+
+
+def test_scroll_up_is_not_yanked_by_incoming_message(alice, bob):
+    from datetime import datetime, timedelta, timezone
+    from devplacepy.utils import generate_uid
+
+    page_a, user_a = alice
+    page_b, user_b = bob
+    uid_a = get_table("users").find_one(username=user_a["username"])["uid"]
+    uid_b = get_table("users").find_one(username=user_b["username"])["uid"]
+
+    base = datetime.now(timezone.utc) - timedelta(minutes=5)
+    for i in range(40):
+        get_table("messages").insert(
+            {
+                "uid": generate_uid(),
+                "sender_uid": uid_a if i % 2 == 0 else uid_b,
+                "receiver_uid": uid_b if i % 2 == 0 else uid_a,
+                "content": f"scroll history line {i}",
+                "read": True,
+                "created_at": (base + timedelta(seconds=i)).isoformat(),
+                "updated_at": None,
+            }
+        )
+
+    page_a.goto(
+        f"{BASE_URL}/messages?with_uid={uid_b}", wait_until="domcontentloaded"
+    )
+    page_b.goto(
+        f"{BASE_URL}/messages?with_uid={uid_a}", wait_until="domcontentloaded"
+    )
+
+    thread = page_a.locator(".messages-thread")
+    thread.wait_for(state="visible")
+    thread.locator(".message-bubble").first.wait_for(state="visible")
+
+    thread.evaluate(
+        "el => { el.scrollTop = 0; el.dispatchEvent(new Event('scroll')); }"
+    )
+    page_a.wait_for_timeout(200)
+    scroll_before = thread.evaluate("el => el.scrollTop")
+    assert scroll_before < 50
+
+    msg = f"While scrolled up {int(time.time() * 1000)}"
+    textarea_b = page_b.locator(".messages-input-area textarea[name='content']")
+    textarea_b.wait_for(state="visible")
+    textarea_b.fill(msg)
+    page_b.locator(".messages-send-btn").click()
+
+    page_a.locator(f".message-bubble.theirs:has-text('{msg}')").first.wait_for(
+        state="visible", timeout=10000
+    )
+    scroll_after = thread.evaluate("el => el.scrollTop")
+    assert scroll_after < 50, (
+        "an incoming message must not yank a scrolled-up reader back to the bottom"
+    )
+
+
+def test_scroll_stays_pinned_to_bottom_for_active_reader(alice, bob):
+    page_a, user_a = alice
+    page_b, user_b = bob
+    uid_a = get_table("users").find_one(username=user_a["username"])["uid"]
+    uid_b = get_table("users").find_one(username=user_b["username"])["uid"]
+
+    page_a.goto(
+        f"{BASE_URL}/messages?with_uid={uid_b}", wait_until="domcontentloaded"
+    )
+    page_b.goto(
+        f"{BASE_URL}/messages?with_uid={uid_a}", wait_until="domcontentloaded"
+    )
+
+    thread = page_a.locator(".messages-thread")
+    thread.wait_for(state="visible")
+
+    msg = f"Pinned bottom {int(time.time() * 1000)}"
+    textarea_b = page_b.locator(".messages-input-area textarea[name='content']")
+    textarea_b.wait_for(state="visible")
+    textarea_b.fill(msg)
+    page_b.locator(".messages-send-btn").click()
+
+    page_a.locator(f".message-bubble.theirs:has-text('{msg}')").first.wait_for(
+        state="visible", timeout=10000
+    )
+    at_bottom = thread.evaluate(
+        "el => el.scrollHeight - el.scrollTop - el.clientHeight < 100"
+    )
+    assert at_bottom, "an active-at-bottom reader must still auto-scroll to new messages"

tests/e2e/projects/index.py

@@ -1007,25 +1007,6 @@ def _create_project_containers_menu(key, title, is_private=False):
     return r.json()["data"]["slug"]
 
 
-def test_containers_menu_visible_for_admin_on_own_private_project(page, app_server):
-    name, uid, key = _make_admin_containers_menu("cmown")
-    slug = _create_project_containers_menu(key, "Containers Menu Own Private", is_private=True)
-    login_user(page, {"email": f"{name}@t.dev", "password": "secret123"})
-    page.goto(f"{BASE_URL}/projects/{slug}", wait_until="domcontentloaded")
-    page.locator(".project-actions-more").click()
-    expect(page.locator(".context-menu-item:has-text('Containers')")).to_be_visible()
-
-
-def test_containers_menu_visible_for_any_admin_on_public_project(page, app_server):
-    _, _, owner_key = _make_admin_containers_menu("cmpubowner")
-    other_name, _, _ = _make_admin_containers_menu("cmpubother")
-    slug = _create_project_containers_menu(owner_key, "Containers Menu Public", is_private=False)
-    login_user(page, {"email": f"{other_name}@t.dev", "password": "secret123"})
-    page.goto(f"{BASE_URL}/projects/{slug}", wait_until="domcontentloaded")
-    page.locator(".project-actions-more").click()
-    expect(page.locator(".context-menu-item:has-text('Containers')")).to_be_visible()
-
-
 def test_containers_menu_hidden_for_non_owner_admin_on_member_private_project(page, app_server):
     _, member_uid, member_key = _signup_containers_menu("cmmember")
     slug = _create_project_containers_menu(member_key, "Containers Menu Member Private", is_private=True)

tests/e2e/projects/workspace.py

@@ -104,23 +104,6 @@ def _phase_view(page, phase: str):
     return page.locator(f".workspace-phase-view[data-phase-view~='{phase}']")
 
 
-def test_project_page_offers_the_workspace_entry_point_to_the_owner(alice):
-    page, user = alice
-    project = _project_for(_row_for(user)["uid"], "WS Entry")
-    page.goto(
-        f"{BASE_URL}/projects/{project['slug']}", wait_until="domcontentloaded"
-    )
-    page.locator(".project-actions-more").click()
-    entry = page.locator(".context-menu-item:has-text('Workspace')")
-    entry.wait_for(state="visible")
-    entry.click()
-    page.wait_for_url(
-        f"{BASE_URL}/projects/{project['slug']}/workspace",
-        wait_until="domcontentloaded",
-    )
-    page.locator(".workspace-page").wait_for(state="visible")
-
-
 def test_project_page_hides_the_workspace_entry_point_from_a_non_owner(bob):
     page, user = bob
     project = _project_for(str(uuid4()), "WS Entry Foreign")
@@ -131,25 +114,6 @@ def test_project_page_hides_the_workspace_entry_point_from_a_non_owner(bob):
     assert not page.locator(".context-menu-item:has-text('Workspace')").count()
 
 
-def test_project_page_shows_a_direct_vscode_button_for_a_running_workspace(
-    alice, editor_listener
-):
-    page, user = alice
-    row = _row_for(user)
-    project = _project_for(row["uid"], "WS Direct")
-    instance = _workspace_for(project, row["uid"], editor_port=editor_listener)
-    page.goto(
-        f"{BASE_URL}/projects/{project['slug']}", wait_until="domcontentloaded"
-    )
-    button = page.locator(".project-detail-actions a[data-editor-open]")
-    button.wait_for(state="visible")
-    expect(button).to_have_attribute("target", "_blank")
-    expect(button).to_have_attribute(
-        "href",
-        f"/projects/{project['slug']}/containers/instances/{instance['uid']}/code/",
-    )
-
-
 def test_direct_vscode_button_is_absent_while_the_workspace_is_stopped(alice):
     page, user = alice
     row = _row_for(user)

tests/e2e/quizzes/index.py

@@ -10,7 +10,7 @@ LAYOUT = ".feed-layout"
 LEFT = ".feed-layout > .sidebar-card"
 CENTRE = ".feed-layout > .feed-main"
 RIGHT = ".feed-layout > .feed-right"
-NAV_ENTRY = ".topnav-link[href='/quizzes']"
+NAV_ENTRY = ".topnav-icon[href='/quizzes']"
 NEW_QUIZ_BUTTON = ".quiz-actions a[href='/quizzes/new']"
 DEVII_BUTTON = ".quiz-actions button[data-devii-open]"
 SCOREBOARD = ".quiz-scoreboard-card"

tests/unit/docs_api/_shared.py

@@ -0,0 +1,36 @@
+from devplacepy.docs_api import field, endpoint
+
+
+def test_field_defaults_nullable_false():
+    spec = field("username", "path", "string", True, "alice", "A username.")
+    assert spec["nullable"] is False
+
+
+def test_field_nullable_true_is_threaded_into_spec():
+    spec = field(
+        "avatar_seed",
+        "response",
+        "string",
+        description="Null falls back to the username.",
+        nullable=True,
+    )
+    assert spec["nullable"] is True
+    assert spec["location"] == "response"
+
+
+def test_endpoint_carries_response_field_specs_in_params():
+    ep = endpoint(
+        id="example",
+        method="GET",
+        path="/example",
+        title="Example",
+        summary="An example endpoint.",
+        auth="public",
+        params=[
+            field("uid", "response", "string", nullable=False),
+            field("bio", "response", "string", nullable=True),
+        ],
+    )
+    by_name = {p["name"]: p for p in ep["params"]}
+    assert by_name["uid"]["nullable"] is False
+    assert by_name["bio"]["nullable"] is True

tests/unit/project_files.py

@@ -518,6 +518,30 @@ def test_sweep_orphan_project_file_blobs_keeps_blob_referenced_only_by_soft_dele
     assert (tmp_path / directory / stored_name).exists()
 
 
+def test_export_to_dir_summarizes_missing_blobs(local_db, tmp_path, monkeypatch, caplog):
+    import logging
+
+    monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
+    uid = _insert_binary_node("export-missing-proj", "ab/cd", "gone.bin")
+    dest = tmp_path / "out"
+    dest.mkdir()
+    pf._table().update(
+        {
+            "uid": uid,
+            "path": "gone.bin",
+            "name": "gone.bin",
+            "parent_path": "",
+        },
+        ["uid"],
+    )
+    with caplog.at_level(logging.WARNING, logger="devplacepy.project_files"):
+        written = pf.export_to_dir("export-missing-proj", "", dest)
+    assert written == 0
+    messages = [record.getMessage() for record in caplog.records]
+    assert any("Skipped 1 missing blob" in message for message in messages)
+    assert not any("Blob file missing" in message for message in messages)
+
+
 def test_sweep_orphan_project_file_blobs_dry_run_changes_nothing(local_db, tmp_path, monkeypatch):
     monkeypatch.setattr(pf, "PROJECT_FILES_DIR", tmp_path)
     directory = "ab/cd"

tests/unit/rendering.py

@@ -2,7 +2,12 @@
 
 from html.parser import HTMLParser
 
-from devplacepy.rendering import render_content, render_title
+from devplacepy.rendering import (
+    markdown_structure_signature,
+    render_content,
+    render_title,
+    safe_truncate,
+)
 
 
 def test_emoji_shortcodes():
@@ -12,6 +17,37 @@ def test_emoji_shortcodes():
     assert ":rocket:" not in out
 
 
+def test_markdown_structure_signature_counts_elements():
+    text = (
+        "# Title\n\n"
+        "- one\n- two\n\n"
+        "1. first\n2. second\n\n"
+        "```python\nprint(1)\n```\n\n"
+        "[link](https://example.com)\n"
+    )
+    assert markdown_structure_signature(text) == {
+        "code_fences": 1,
+        "list_items": 4,
+        "headers": 1,
+        "links": 1,
+    }
+
+
+def test_markdown_structure_signature_blank_text():
+    assert markdown_structure_signature("") == {
+        "code_fences": 0,
+        "list_items": 0,
+        "headers": 0,
+        "links": 0,
+    }
+
+
+def test_markdown_structure_signature_ignores_word_changes():
+    original = markdown_structure_signature("- alpha\n- beta\n- gamma\n")
+    reworded = markdown_structure_signature("- Alpha\n- beta\n- gamma\n")
+    assert original == reworded
+
+
 def test_markdown_bold_and_italic():
     out = str(render_content("this is **bold** and _em_"))
     assert "<strong>bold</strong>" in out
@@ -47,6 +83,38 @@ def test_plain_url_autolinked():
     assert '<a href="https://example.com/page" target="_blank" rel="noopener noreferrer">' in out
 
 
+def test_safe_truncate_short_text_unchanged():
+    text = "short content"
+    assert safe_truncate(text, 300) == text
+
+
+def test_safe_truncate_backs_off_to_word_boundary():
+    text = "a" * 45 + " " + "b" * 45
+    truncated = safe_truncate(text, 50)
+    assert truncated == "a" * 45
+
+
+def test_safe_truncate_does_not_split_a_url_straddling_the_cutoff():
+    prefix = "word " * 58
+    url = "https://example.com/gallery/a-very-long-descriptive-photo-name.jpg"
+    text = prefix + url + " more words after the link that get cut off entirely"
+    truncated = safe_truncate(text, 300)
+    assert url not in truncated
+    assert "example.com" not in truncated
+    out = str(render_content(truncated + "..."))
+    assert "<img" not in out
+    assert "<a href=" not in out
+
+
+def test_safe_truncate_keeps_a_url_that_fits_before_the_cutoff():
+    url = "https://example.com/image.png"
+    text = url + " " + ("word " * 60)
+    truncated = safe_truncate(text, 300)
+    assert url in truncated
+    out = str(render_content(truncated))
+    assert f'<img src="{url}"' in out
+
+
 def test_mention_links_to_profile():
     out = str(render_content("hello @alice_test there"))
     assert '<a href="/profile/alice_test" class="mention-link">@alice_test</a>' in out

tests/unit/services/ai_modifier.py

@@ -0,0 +1,32 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services import ai_modifier, correction
+
+
+class _FakeResponse:
+    def __init__(self, content: str) -> None:
+        self._content = content
+        self.headers: dict = {}
+
+    def raise_for_status(self) -> None:
+        return None
+
+    def json(self) -> dict:
+        return {"choices": [{"message": {"content": self._content}}]}
+
+
+class _FakeClient:
+    def __init__(self) -> None:
+        self.calls: list[dict] = []
+
+    def post(self, url, json=None, headers=None, timeout=None):
+        self.calls.append({"url": url, "json": json, "headers": headers})
+        return _FakeResponse("modified text")
+
+
+def test_modify_text_requests_the_preamble_bypass(local_db, monkeypatch):
+    fake = _FakeClient()
+    monkeypatch.setattr(correction, "_client", lambda: fake)
+    ai_modifier.modify_text("user-api-key", "", "@ai fix this")
+    sent = fake.calls[-1]
+    assert sent["json"]["bypass_preamble"] is True

tests/unit/services/backup/offload.py

@@ -0,0 +1,58 @@
+# retoor <retoor@molodetz.nl>
+
+import tempfile
+from pathlib import Path
+
+from tests.conftest import run_async
+from devplacepy.services.backup import offload, store
+
+
+def _make_backup(target):
+    job_uid = f"job-{store.generate_uid()}"
+    uid = store.create_backup(target=target, created_by="test", job_uid=job_uid)
+    path = Path(tempfile.gettempdir()) / f"backup-offload-test-{uid}.tar.gz"
+    path.write_bytes(b"x" * 10)
+    store.finalize_backup(
+        uid,
+        filename=path.name,
+        local_path=str(path),
+        stats={
+            "bytes_out": 10,
+            "bytes_in": 10,
+            "file_count": 1,
+            "dir_count": 0,
+            "sha256": "abc",
+        },
+    )
+    return uid, path
+
+
+def test_upload_pending_stops_after_auth_error(local_db, monkeypatch):
+    first_uid, first_path = _make_backup("database")
+    second_uid, second_path = _make_backup("uploads")
+    calls = []
+
+    async def fake_rclone(*args, timeout=1800.0):
+        calls.append(args)
+        return 1, "", "401 Unauthorized from Apache"
+
+    monkeypatch.setattr(offload, "_run_rclone", fake_rclone)
+    logs = []
+    try:
+        uploaded = run_async(offload.upload_pending(log=logs.append))
+        assert uploaded == 0
+        assert len(calls) == 1
+        assert any("halted" in line.lower() for line in logs)
+    finally:
+        first_path.unlink(missing_ok=True)
+        second_path.unlink(missing_ok=True)
+        store.delete_backup(first_uid)
+        store.delete_backup(second_uid)
+
+
+def test_is_auth_error_detects_rclone_html_401():
+    assert offload._is_auth_error(
+        'read metadata failed: <title>401 Unauthorized</title>: 401 Unauthorized'
+    )
+    assert offload._is_auth_error("didn't find section in config file")
+    assert not offload._is_auth_error("connection timed out")

tests/unit/services/backup/service.py

@@ -0,0 +1,72 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.backup import service, store
+
+
+def _stats(used_percent):
+    return {
+        "disk": {
+            "total_bytes": 100,
+            "used_bytes": used_percent,
+            "free_bytes": 100 - used_percent,
+            "total_human": "100 B",
+            "used_human": f"{used_percent} B",
+            "free_human": f"{100 - used_percent} B",
+            "used_percent": used_percent,
+        }
+    }
+
+
+def _disk(used_percent):
+    return _stats(used_percent)["disk"]
+
+
+def test_check_disk_usage_warns_once_and_recovers_once(local_db, monkeypatch):
+    svc = service.BackupService()
+    assert svc.disk_warn_percent_field.default == service.DEFAULT_DISK_WARN_PERCENT
+    logs = []
+    monkeypatch.setattr(svc, "log", lambda message: logs.append(message))
+
+    monkeypatch.setattr(store, "disk_usage", lambda: _disk(50))
+    svc._check_disk_usage()
+    assert logs == []
+    assert svc._disk_warned is False
+
+    monkeypatch.setattr(store, "disk_usage", lambda: _disk(95))
+    svc._check_disk_usage()
+    assert len(logs) == 1
+    assert "critical" in logs[0].lower()
+    assert svc._disk_warned is True
+
+    svc._check_disk_usage()
+    assert len(logs) == 1
+
+    monkeypatch.setattr(store, "disk_usage", lambda: _disk(40))
+    svc._check_disk_usage()
+    assert len(logs) == 2
+    assert "under threshold" in logs[1].lower()
+    assert svc._disk_warned is False
+
+
+def test_collect_metrics_includes_disk_usage_stat(local_db, monkeypatch):
+    svc = service.BackupService()
+    monkeypatch.setattr(store, "disk_usage", lambda: _disk(62))
+
+    metrics = svc.collect_metrics()
+
+    labels = [stat["label"] for stat in metrics["stats"]]
+    assert "Disk usage" in labels
+    disk_stat = next(stat for stat in metrics["stats"] if stat["label"] == "Disk usage")
+    assert "62%" in disk_stat["value"]
+
+
+def test_collect_metrics_never_walks_storage_tree(local_db, monkeypatch):
+    svc = service.BackupService()
+    monkeypatch.setattr(store, "disk_usage", lambda: _disk(10))
+
+    def boom():
+        raise AssertionError("compute_storage_stats must not run on the service tick")
+
+    monkeypatch.setattr(store, "compute_storage_stats", boom)
+    svc.collect_metrics()
+    svc._check_disk_usage()

tests/unit/services/backup/store.py

@@ -1,5 +1,6 @@
 # retoor <retoor@molodetz.nl>
 
+import os
 import tempfile
 from pathlib import Path
 
@@ -65,3 +66,90 @@ def test_rotate_schedule_never_removes_a_backup_without_a_confirmed_remote_copy(
     finally:
         store.delete_backup(older_uid)
         store.delete_backup(newer_uid)
+
+
+def _point_storage_at(monkeypatch, root: Path):
+    uploads = root / "uploads"
+    attachments = uploads / "attachments"
+    project_files = uploads / "project_files"
+    keys = root / "keys"
+    zips = root / "zips"
+    deepsearch = root / "deepsearch"
+    workspaces = root / "container_workspaces"
+    backups = root / "backups"
+    for path in (
+        attachments,
+        project_files,
+        keys,
+        zips,
+        deepsearch,
+        workspaces,
+        backups,
+    ):
+        path.mkdir(parents=True, exist_ok=True)
+    db_file = root / "devplace.db"
+    db_file.write_bytes(b"db")
+    tasks = root / "devii_tasks.db"
+    tasks.write_bytes(b"t")
+    lessons = root / "devii_lessons.db"
+    lessons.write_bytes(b"l")
+    (project_files / "a.bin").write_bytes(b"x" * 10)
+    (attachments / "b.bin").write_bytes(b"y" * 5)
+    monkeypatch.setattr(store.config, "DATA_DIR", root)
+    monkeypatch.setattr(store.config, "UPLOADS_DIR", uploads)
+    monkeypatch.setattr(store.config, "ATTACHMENTS_DIR", attachments)
+    monkeypatch.setattr(store.config, "PROJECT_FILES_DIR", project_files)
+    monkeypatch.setattr(store.config, "KEYS_DIR", keys)
+    monkeypatch.setattr(store.config, "ZIPS_DIR", zips)
+    monkeypatch.setattr(store.config, "DEEPSEARCH_DIR", deepsearch)
+    monkeypatch.setattr(store.config, "CONTAINER_WORKSPACES_DIR", workspaces)
+    monkeypatch.setattr(store.config, "BACKUPS_DIR", backups)
+    monkeypatch.setattr(store.config, "DEVII_TASKS_DB", tasks)
+    monkeypatch.setattr(store.config, "DEVII_LESSONS_DB", lessons)
+    monkeypatch.setattr(store.config, "DATABASE_URL", f"sqlite:///{db_file}")
+    store.clear_storage_stats_cache()
+    return {
+        "uploads": uploads,
+        "attachments": attachments,
+        "project_files": project_files,
+        "db_file": db_file,
+    }
+
+
+def test_disk_usage_does_not_walk(monkeypatch, tmp_path):
+    monkeypatch.setattr(store.config, "DATA_DIR", tmp_path)
+    store.clear_storage_stats_cache()
+
+    def boom(*_args, **_kwargs):
+        raise AssertionError("disk_usage must not walk the data tree")
+
+    monkeypatch.setattr(store.os, "walk", boom)
+    disk = store.disk_usage()
+    assert disk["total_bytes"] > 0
+    assert "used_percent" in disk
+
+
+def test_compute_storage_stats_walks_data_dir_once(local_db, monkeypatch, tmp_path):
+    root = tmp_path / "data"
+    _point_storage_at(monkeypatch, root)
+    walks = []
+    real_walk = os.walk
+
+    def counting_walk(path, *args, **kwargs):
+        walks.append(os.path.realpath(path))
+        return real_walk(path, *args, **kwargs)
+
+    monkeypatch.setattr(store.os, "walk", counting_walk)
+    stats = store.compute_storage_stats()
+    assert walks == [os.path.realpath(root)]
+    by_key = {row["key"]: row for row in stats["paths"]}
+    assert by_key["project_files"]["file_count"] == 1
+    assert by_key["project_files"]["size_bytes"] == 10
+    assert by_key["attachments"]["file_count"] == 1
+    assert by_key["uploads"]["file_count"] >= 2
+    assert by_key["database"]["file_count"] == 1
+    assert stats["data_dir"]["file_count"] >= 3
+    walks.clear()
+    again = store.compute_storage_stats()
+    assert walks == []
+    assert again["data_dir"]["file_count"] == stats["data_dir"]["file_count"]

tests/unit/services/bot/browser.py

@@ -0,0 +1,29 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.bot.browser import NATIVE_FILL_TYPES, uses_native_fill
+
+
+def test_uses_native_fill_for_date_widgets():
+    assert uses_native_fill("date")
+    assert uses_native_fill("DATE")
+    assert uses_native_fill("datetime-local")
+    assert uses_native_fill("month")
+    assert uses_native_fill("time")
+    assert uses_native_fill("week")
+
+
+def test_uses_native_fill_skips_text_fields():
+    assert not uses_native_fill("text")
+    assert not uses_native_fill("email")
+    assert not uses_native_fill("password")
+    assert not uses_native_fill("")
+
+
+def test_native_fill_types_are_the_date_family():
+    assert NATIVE_FILL_TYPES == {
+        "date",
+        "datetime-local",
+        "month",
+        "time",
+        "week",
+    }

tests/unit/services/correction.py

@@ -0,0 +1,158 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import set_setting
+from devplacepy.services import correction
+
+
+class _FakeResponse:
+    def __init__(self, content: str) -> None:
+        self._content = content
+        self.headers: dict = {}
+
+    def raise_for_status(self) -> None:
+        return None
+
+    def json(self) -> dict:
+        return {"choices": [{"message": {"content": self._content}}]}
+
+
+class _FakeClient:
+    def __init__(self) -> None:
+        self.calls: list[dict] = []
+
+    def post(self, url, json=None, headers=None, timeout=None):
+        self.calls.append({"url": url, "json": json, "headers": headers})
+        return _FakeResponse("corrected text")
+
+
+def test_gateway_complete_sets_bypass_preamble_and_internal_header(local_db, monkeypatch):
+    set_setting("gateway_internal_key", "test-internal-secret")
+    fake = _FakeClient()
+    monkeypatch.setattr(correction, "_client", lambda: fake)
+    try:
+        content, _usage = correction.gateway_complete(
+            "user-api-key",
+            "system prompt",
+            "hello world",
+            5.0,
+            bypass_preamble=True,
+        )
+    finally:
+        set_setting("gateway_internal_key", "")
+    assert content == "corrected text"
+    sent = fake.calls[-1]
+    assert sent["json"]["bypass_preamble"] is True
+    assert sent["headers"]["Authorization"] == "Bearer user-api-key"
+    assert sent["headers"]["X-Gateway-Internal-Key"] == "test-internal-secret"
+
+
+def test_gateway_complete_without_bypass_preamble_omits_it(local_db, monkeypatch):
+    fake = _FakeClient()
+    monkeypatch.setattr(correction, "_client", lambda: fake)
+    correction.gateway_complete("user-api-key", "system prompt", "hello world", 5.0)
+    sent = fake.calls[-1]
+    assert "bypass_preamble" not in sent["json"]
+    assert "X-Gateway-Internal-Key" not in sent["headers"]
+
+
+def test_correct_text_requests_the_preamble_bypass(local_db, monkeypatch):
+    fake = _FakeClient()
+    monkeypatch.setattr(correction, "_client", lambda: fake)
+    correction.correct_text("user-api-key", "", "hello world")
+    sent = fake.calls[-1]
+    assert sent["json"]["bypass_preamble"] is True
+
+
+class _ScriptedClient:
+    def __init__(self, content: str) -> None:
+        self._content = content
+        self.calls: list[dict] = []
+
+    def post(self, url, json=None, headers=None, timeout=None):
+        self.calls.append({"url": url, "json": json, "headers": headers})
+        return _FakeResponse(self._content)
+
+
+def test_structure_diverges_false_on_reworded_list_item():
+    original = "- alpha\n- beta\n- gamma\n"
+    corrected = "- Alpha\n- beta\n- gamma\n"
+    assert correction.structure_diverges(original, corrected) is False
+
+
+def test_structure_diverges_true_on_flattened_list():
+    original = "- alpha\n- beta\n- gamma\n"
+    corrected = "alpha, beta and gamma."
+    assert correction.structure_diverges(original, corrected) is True
+
+
+def test_gateway_complete_structure_check_disabled_by_default(local_db, monkeypatch):
+    original = "- alpha\n- beta\n"
+    corrected = "alpha and beta."
+    fake = _ScriptedClient(corrected)
+    monkeypatch.setattr(correction, "_client", lambda: fake)
+    content, _usage = correction.gateway_complete(
+        "user-api-key", "system prompt", original, 5.0
+    )
+    assert content == corrected
+
+
+def test_gateway_complete_structure_check_passes_typo_fix(local_db, monkeypatch):
+    original = "- alpha\n- beta\n- gamma\n"
+    corrected = "- Alpha\n- beta\n- gamma\n"
+    fake = _ScriptedClient(corrected)
+    monkeypatch.setattr(correction, "_client", lambda: fake)
+    content, _usage = correction.gateway_complete(
+        "user-api-key", "system prompt", original, 5.0, check_structure=True
+    )
+    assert content == corrected.strip()
+
+
+def test_gateway_complete_structure_check_rejects_flattened_list(local_db, monkeypatch):
+    original = "- alpha\n- beta\n- gamma\n"
+    corrected = "alpha, beta and gamma."
+    fake = _ScriptedClient(corrected)
+    monkeypatch.setattr(correction, "_client", lambda: fake)
+    content, _usage = correction.gateway_complete(
+        "user-api-key", "system prompt", original, 5.0, check_structure=True
+    )
+    assert content == original
+
+
+def test_gateway_complete_structure_check_rejects_dropped_code_fence(local_db, monkeypatch):
+    original = "Here is the fix:\n\n```python\nprint(1)\n```\n"
+    corrected = "Here is the fix:\n\nprint(1)\n"
+    fake = _ScriptedClient(corrected)
+    monkeypatch.setattr(correction, "_client", lambda: fake)
+    content, _usage = correction.gateway_complete(
+        "user-api-key", "system prompt", original, 5.0, check_structure=True
+    )
+    assert content == original
+
+
+def test_gateway_complete_structure_check_rejects_removed_link(local_db, monkeypatch):
+    original = "Check out [DevPlace](https://example.com) for details."
+    corrected = "Check out DevPlace for details."
+    fake = _ScriptedClient(corrected)
+    monkeypatch.setattr(correction, "_client", lambda: fake)
+    content, _usage = correction.gateway_complete(
+        "user-api-key", "system prompt", original, 5.0, check_structure=True
+    )
+    assert content == original
+
+
+def test_correct_text_rejects_corrupted_structure(local_db, monkeypatch):
+    original = "# Setup\n\n- step one\n- step two\n"
+    corrected = "Setup: step one, step two."
+    fake = _ScriptedClient(corrected)
+    monkeypatch.setattr(correction, "_client", lambda: fake)
+    content, _usage = correction.correct_text("user-api-key", "", original)
+    assert content == original
+
+
+def test_correct_text_keeps_grammar_only_correction(local_db, monkeypatch):
+    original = "# Setup\n\n- step one\n- step two\n"
+    corrected = "# Setup\n\n- Step one\n- Step two\n"
+    fake = _ScriptedClient(corrected)
+    monkeypatch.setattr(correction, "_client", lambda: fake)
+    content, _usage = correction.correct_text("user-api-key", "", original)
+    assert content == corrected.strip()

tests/unit/services/devii/config.py

@@ -190,3 +190,87 @@ def test_cli_reset_quota_all(local_db):
     _seed_ledger("user", user_id, 0.5, n=1)
     cmd_devii_reset_quota(SimpleNamespace(username=None, guests=False, all=True))
     assert get_table(LEDGER).count() == 0
+
+
+def _quota_warnings(uid):
+    from devplacepy.services.devii.service import QUOTA_WARNING_NOTIFICATION_TYPE
+
+    return list(
+        get_table("notifications").find(
+            user_uid=uid, type=QUOTA_WARNING_NOTIFICATION_TYPE
+        )
+    )
+
+
+def test_quota_warning_fires_once_past_80_percent(local_db):
+    svc = _ensure_devii()
+    set_setting("devii_user_daily_usd", "1.0")
+    try:
+        uid, _ = _make_user_devii_quota()
+        _seed_ledger("user", uid, 0.85)
+        svc.maybe_warn_quota_threshold("user", uid, False)
+        assert len(_quota_warnings(uid)) == 1
+    finally:
+        set_setting("devii_user_daily_usd", "1.0")
+
+
+def test_quota_warning_not_refired_within_same_window(local_db):
+    svc = _ensure_devii()
+    set_setting("devii_user_daily_usd", "1.0")
+    try:
+        uid, _ = _make_user_devii_quota()
+        _seed_ledger("user", uid, 0.85)
+        svc.maybe_warn_quota_threshold("user", uid, False)
+        _seed_ledger("user", uid, 0.05)
+        svc.maybe_warn_quota_threshold("user", uid, False)
+        assert len(_quota_warnings(uid)) == 1
+    finally:
+        set_setting("devii_user_daily_usd", "1.0")
+
+
+def test_quota_warning_skips_below_threshold(local_db):
+    svc = _ensure_devii()
+    set_setting("devii_user_daily_usd", "1.0")
+    try:
+        uid, _ = _make_user_devii_quota()
+        _seed_ledger("user", uid, 0.5)
+        svc.maybe_warn_quota_threshold("user", uid, False)
+        assert _quota_warnings(uid) == []
+    finally:
+        set_setting("devii_user_daily_usd", "1.0")
+
+
+def test_quota_warning_skips_admin(local_db):
+    svc = _ensure_devii()
+    set_setting("devii_admin_daily_usd", "1.0")
+    try:
+        uid, _ = _make_user_devii_quota(role="Admin")
+        _seed_ledger("user", uid, 0.95)
+        svc.maybe_warn_quota_threshold("user", uid, True)
+        assert _quota_warnings(uid) == []
+    finally:
+        set_setting("devii_admin_daily_usd", "0.0")
+
+
+def test_quota_warning_skips_unlimited_cap(local_db):
+    svc = _ensure_devii()
+    set_setting("devii_user_daily_usd", "0")
+    try:
+        uid, _ = _make_user_devii_quota()
+        _seed_ledger("user", uid, 50.0)
+        svc.maybe_warn_quota_threshold("user", uid, False)
+        assert _quota_warnings(uid) == []
+    finally:
+        set_setting("devii_user_daily_usd", "1.0")
+
+
+def test_quota_warning_skips_guests(local_db):
+    svc = _ensure_devii()
+    set_setting("devii_guest_daily_usd", "0.10")
+    try:
+        guest_id = f"guest-{generate_uid()[:8]}"
+        _seed_ledger("guest", guest_id, 0.09)
+        svc.maybe_warn_quota_threshold("guest", guest_id, False)
+        assert get_table("notifications").find_one(user_uid=guest_id) is None
+    finally:
+        set_setting("devii_guest_daily_usd", "0.05")

tests/unit/services/jobs/seo_meta_service.py

@@ -0,0 +1,41 @@
+# retoor <retoor@molodetz.nl>
+
+import json
+
+from devplacepy.seo_meta_text import plain_seo_defaults
+from devplacepy.services.jobs import seo_meta_service
+
+
+def test_generate_requests_the_preamble_bypass(local_db, monkeypatch):
+    seen = {}
+
+    def fake_gateway_complete(
+        api_key, system, text, timeout, model=None, bypass_preamble=False
+    ):
+        seen["bypass_preamble"] = bypass_preamble
+        return (
+            json.dumps(
+                {
+                    "seo_title": "A generated title",
+                    "seo_description": "A generated description that is reasonably long.",
+                    "seo_keywords": "one, two, three, four, five",
+                }
+            ),
+            None,
+        )
+
+    monkeypatch.setattr(seo_meta_service, "gateway_complete", fake_gateway_complete)
+    svc = seo_meta_service.SeoMetaService()
+    defaults = plain_seo_defaults("Title", "Body text")
+    totals = seo_meta_service.new_usage_totals()
+    svc._generate(
+        "post",
+        "nonexistent-uid",
+        {"user_uid": ""},
+        "Title",
+        "Body text",
+        defaults,
+        totals,
+        "molodetz",
+    )
+    assert seen["bypass_preamble"] is True

tests/unit/services/messaging/active_conversation.py

@@ -0,0 +1,105 @@
+# retoor <retoor@molodetz.nl>
+
+from datetime import datetime, timedelta, timezone
+
+import uuid_utils
+
+from devplacepy.database import get_table
+from devplacepy.services.messaging import active_conversation
+
+
+def _make_user():
+    uid = str(uuid_utils.uuid7())
+    get_table("users").insert(
+        {
+            "uid": uid,
+            "username": f"actconv_{uid[:8]}",
+            "email": f"{uid[:8]}@actconv.test",
+            "terms_version": "1",
+        }
+    )
+    return uid
+
+
+def test_touch_writes_marker_then_throttles(local_db):
+    viewer = _make_user()
+    other = _make_user()
+    active_conversation._last_write.pop(f"{viewer}:{other}", None)
+
+    active_conversation.touch_active_conversation(viewer, other)
+    row = get_table("users").find_one(uid=viewer)
+    assert row["active_conversation_uid"] == other
+    first_stamp = row["active_conversation_at"]
+    assert first_stamp
+
+    active_conversation.touch_active_conversation(viewer, other)
+    assert get_table("users").find_one(uid=viewer)["active_conversation_at"] == first_stamp
+
+    active_conversation._last_write[f"{viewer}:{other}"] = (
+        active_conversation.time.monotonic()
+        - active_conversation.WRITE_THROTTLE_SECONDS
+        - 1
+    )
+    active_conversation.touch_active_conversation(viewer, other)
+    second_stamp = get_table("users").find_one(uid=viewer)["active_conversation_at"]
+    assert second_stamp >= first_stamp
+
+
+def test_touch_is_a_noop_without_both_uids(local_db):
+    viewer = _make_user()
+    active_conversation.touch_active_conversation("", "someone")
+    active_conversation.touch_active_conversation(viewer, "")
+    row = get_table("users").find_one(uid=viewer)
+    assert not row.get("active_conversation_uid")
+
+
+def test_is_actively_viewing_true_when_fresh(local_db):
+    viewer = _make_user()
+    other = _make_user()
+    get_table("users").update(
+        {
+            "uid": viewer,
+            "active_conversation_uid": other,
+            "active_conversation_at": datetime.now(timezone.utc).isoformat(),
+        },
+        ["uid"],
+    )
+    assert active_conversation.is_actively_viewing(viewer, other) is True
+
+
+def test_is_actively_viewing_false_when_stale(local_db):
+    viewer = _make_user()
+    other = _make_user()
+    stale = datetime.now(timezone.utc) - timedelta(
+        seconds=active_conversation.FRESH_SECONDS + 5
+    )
+    get_table("users").update(
+        {
+            "uid": viewer,
+            "active_conversation_uid": other,
+            "active_conversation_at": stale.isoformat(),
+        },
+        ["uid"],
+    )
+    assert active_conversation.is_actively_viewing(viewer, other) is False
+
+
+def test_is_actively_viewing_false_for_a_different_conversation(local_db):
+    viewer = _make_user()
+    other = _make_user()
+    someone_else = _make_user()
+    get_table("users").update(
+        {
+            "uid": viewer,
+            "active_conversation_uid": someone_else,
+            "active_conversation_at": datetime.now(timezone.utc).isoformat(),
+        },
+        ["uid"],
+    )
+    assert active_conversation.is_actively_viewing(viewer, other) is False
+
+
+def test_is_actively_viewing_false_when_missing_or_unknown(local_db):
+    assert active_conversation.is_actively_viewing("", "other") is False
+    assert active_conversation.is_actively_viewing("viewer", "") is False
+    assert active_conversation.is_actively_viewing(str(uuid_utils.uuid7()), "other") is False

tests/unit/services/messaging/persist.py

@@ -4,7 +4,24 @@ from datetime import datetime, timezone
 
 from devplacepy.database import get_table
 from devplacepy.utils import generate_uid
-from devplacepy.services.messaging.persist import stamp_content_revision
+from devplacepy.services.messaging.persist import persist_message, stamp_content_revision
+
+
+def _make_user(prefix):
+    uid = generate_uid()
+    get_table("users").insert(
+        {
+            "uid": uid,
+            "username": f"{prefix}_{uid[:8]}",
+            "email": f"{uid[:8]}@{prefix}.test",
+            "terms_version": "1",
+        }
+    )
+    return uid
+
+
+def _sender(uid):
+    return {"uid": uid, "username": "sender", "role": "Member"}
 
 
 def test_stamp_content_revision_sets_updated_at(local_db):
@@ -26,3 +43,84 @@ def test_stamp_content_revision_sets_updated_at(local_db):
     stored = get_table("messages").find_one(uid=uid)
     assert stored["updated_at"] == row["updated_at"]
     assert stored["content"] == "hello"
+
+
+def test_persist_message_dedupes_same_sender_and_client_id(local_db):
+    sender_uid = _make_user("dedupe_sender")
+    receiver_uid = _make_user("dedupe_receiver")
+    sender = _sender(sender_uid)
+    client_id = f"cid-{generate_uid()}"
+
+    first = persist_message(sender, receiver_uid, "hello once", client_id=client_id)
+    assert first is not None
+
+    second = persist_message(sender, receiver_uid, "hello once", client_id=client_id)
+    assert second is not None
+    assert second["uid"] == first["uid"]
+
+    rows = list(
+        get_table("messages").find(sender_uid=sender_uid, receiver_uid=receiver_uid)
+    )
+    assert len(rows) == 1
+
+
+def test_persist_message_does_not_dedupe_without_client_id(local_db):
+    sender_uid = _make_user("nodedupe_sender")
+    receiver_uid = _make_user("nodedupe_receiver")
+    sender = _sender(sender_uid)
+
+    first = persist_message(sender, receiver_uid, "no client id here")
+    second = persist_message(sender, receiver_uid, "no client id here")
+    assert first["uid"] != second["uid"]
+
+    rows = list(
+        get_table("messages").find(sender_uid=sender_uid, receiver_uid=receiver_uid)
+    )
+    assert len(rows) == 2
+
+
+def test_persist_message_does_not_dedupe_a_different_client_id(local_db):
+    sender_uid = _make_user("diffid_sender")
+    receiver_uid = _make_user("diffid_receiver")
+    sender = _sender(sender_uid)
+
+    first = persist_message(sender, receiver_uid, "a", client_id="cid-a")
+    second = persist_message(sender, receiver_uid, "b", client_id="cid-b")
+    assert first["uid"] != second["uid"]
+
+
+def test_persist_message_suppresses_notification_when_receiver_is_actively_viewing(local_db):
+    sender_uid = _make_user("active_sender")
+    receiver_uid = _make_user("active_receiver")
+    sender = _sender(sender_uid)
+
+    get_table("users").update(
+        {
+            "uid": receiver_uid,
+            "active_conversation_uid": sender_uid,
+            "active_conversation_at": datetime.now(timezone.utc).isoformat(),
+        },
+        ["uid"],
+    )
+
+    result = persist_message(sender, receiver_uid, "are you watching?", client_id="cid-watch")
+    assert result is not None
+
+    notifications = list(
+        get_table("notifications").find(user_uid=receiver_uid, type="message")
+    )
+    assert notifications == []
+
+
+def test_persist_message_notifies_when_receiver_is_not_actively_viewing(local_db):
+    sender_uid = _make_user("inactive_sender")
+    receiver_uid = _make_user("inactive_receiver")
+    sender = _sender(sender_uid)
+
+    result = persist_message(sender, receiver_uid, "hello there", client_id="cid-plain")
+    assert result is not None
+
+    notifications = list(
+        get_table("notifications").find(user_uid=receiver_uid, type="message")
+    )
+    assert len(notifications) == 1

tests/unit/services/openai_gateway/auth_throttle.py

@@ -0,0 +1,69 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.services.openai_gateway import auth_throttle
+
+
+def test_is_throttled_false_below_threshold():
+    auth_throttle.clear()
+    try:
+        for _ in range(9):
+            auth_throttle.record_failure("1.1.1.1", 60)
+        assert auth_throttle.is_throttled("1.1.1.1", 10, 60) is False
+    finally:
+        auth_throttle.clear()
+
+
+def test_is_throttled_true_at_threshold():
+    auth_throttle.clear()
+    try:
+        for _ in range(10):
+            auth_throttle.record_failure("2.2.2.2", 60)
+        assert auth_throttle.is_throttled("2.2.2.2", 10, 60) is True
+    finally:
+        auth_throttle.clear()
+
+
+def test_is_throttled_scoped_per_ip():
+    auth_throttle.clear()
+    try:
+        for _ in range(10):
+            auth_throttle.record_failure("3.3.3.3", 60)
+        assert auth_throttle.is_throttled("3.3.3.3", 10, 60) is True
+        assert auth_throttle.is_throttled("4.4.4.4", 10, 60) is False
+    finally:
+        auth_throttle.clear()
+
+
+def test_record_failure_returns_running_count():
+    auth_throttle.clear()
+    try:
+        assert auth_throttle.record_failure("5.5.5.5", 60) == 1
+        assert auth_throttle.record_failure("5.5.5.5", 60) == 2
+        assert auth_throttle.record_failure("5.5.5.5", 60) == 3
+    finally:
+        auth_throttle.clear()
+
+
+def test_throttle_resets_after_window_passes(monkeypatch):
+    auth_throttle.clear()
+    try:
+        now = [1000.0]
+        monkeypatch.setattr(auth_throttle.time, "time", lambda: now[0])
+        for _ in range(10):
+            auth_throttle.record_failure("6.6.6.6", 60)
+        assert auth_throttle.is_throttled("6.6.6.6", 10, 60) is True
+        now[0] += 61
+        assert auth_throttle.is_throttled("6.6.6.6", 10, 60) is False
+    finally:
+        auth_throttle.clear()
+
+
+def test_reset_clears_one_ip():
+    auth_throttle.clear()
+    try:
+        for _ in range(10):
+            auth_throttle.record_failure("7.7.7.7", 60)
+        auth_throttle.reset("7.7.7.7")
+        assert auth_throttle.is_throttled("7.7.7.7", 10, 60) is False
+    finally:
+        auth_throttle.clear()

tests/unit/services/openai_gateway/gateway.py

@@ -7,6 +7,7 @@ from devplacepy.database import get_table, set_setting
 from devplacepy.utils import generate_uid
 import devplacepy.services.openai_gateway.gateway as gwmod
 from devplacepy.services.openai_gateway import GatewayService
+from devplacepy.services.openai_gateway import auth_throttle
 class FakeResp_openai_gateway:
     def __init__(
         self,
@@ -257,6 +258,7 @@ def test_authorize_static_access_key(local_db):
     finally:
         set_setting("gateway_access_key", "")
         set_setting("gateway_require_auth", "1")
+        auth_throttle.clear()
 
 
 def test_authorize_admin_and_user_toggles(local_db):
@@ -282,6 +284,7 @@ def test_authorize_admin_and_user_toggles(local_db):
         set_setting("gateway_allow_users", "1")
         set_setting("gateway_require_auth", "1")
         set_setting("gateway_access_key", "")
+        auth_throttle.clear()
 
 
 def test_authorize_require_auth_off_is_open(local_db):
@@ -293,6 +296,124 @@ def test_authorize_require_auth_off_is_open(local_db):
         set_setting("gateway_require_auth", "1")
 
 
+def test_authorize_throttles_repeated_failed_auth_from_one_ip(local_db):
+    import pytest
+    from fastapi import HTTPException
+
+    set_setting("gateway_require_auth", "1")
+    set_setting("gateway_access_key", "")
+    set_setting("gateway_auth_throttle_max_failures", "3")
+    set_setting("gateway_auth_throttle_window_seconds", "60")
+    auth_throttle.clear()
+    try:
+        svc = GatewayService()
+        ip = "203.0.113.10"
+        for _ in range(3):
+            assert (
+                svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
+                is False
+            )
+        with pytest.raises(HTTPException) as excinfo:
+            svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
+        assert excinfo.value.status_code == 429
+        assert "Retry-After" in excinfo.value.headers
+    finally:
+        set_setting("gateway_auth_throttle_max_failures", "10")
+        set_setting("gateway_auth_throttle_window_seconds", "60")
+        set_setting("gateway_require_auth", "1")
+        auth_throttle.clear()
+
+
+def test_authorize_throttle_does_not_affect_other_ips(local_db):
+    set_setting("gateway_require_auth", "1")
+    set_setting("gateway_access_key", "")
+    set_setting("gateway_auth_throttle_max_failures", "3")
+    set_setting("gateway_auth_throttle_window_seconds", "60")
+    auth_throttle.clear()
+    try:
+        svc = GatewayService()
+        throttled_ip = "203.0.113.20"
+        other_ip = "203.0.113.21"
+        for _ in range(3):
+            svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": throttled_ip}))
+        assert auth_throttle.is_throttled(throttled_ip, 3, 60) is True
+        assert (
+            svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": other_ip}))
+            is False
+        )
+        assert auth_throttle.is_throttled(other_ip, 3, 60) is False
+    finally:
+        set_setting("gateway_auth_throttle_max_failures", "10")
+        set_setting("gateway_auth_throttle_window_seconds", "60")
+        set_setting("gateway_require_auth", "1")
+        auth_throttle.clear()
+
+
+def test_authorize_valid_credentials_succeed_from_throttled_ip(local_db):
+    set_setting("gateway_require_auth", "1")
+    set_setting("gateway_access_key", "")
+    set_setting("gateway_allow_users", "1")
+    set_setting("gateway_auth_throttle_max_failures", "3")
+    set_setting("gateway_auth_throttle_window_seconds", "60")
+    auth_throttle.clear()
+    try:
+        svc = GatewayService()
+        ip = "203.0.113.30"
+        _, member_key = _make_admin_openai_gateway(role="Member")
+        for _ in range(3):
+            svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
+        assert auth_throttle.is_throttled(ip, 3, 60) is True
+        assert (
+            svc.authorize(
+                _make_request_openai_gateway(
+                    headers={"X-Real-IP": ip, "X-API-KEY": member_key}
+                )
+            )
+            is True
+        )
+        set_setting("gateway_access_key", "topsecret")
+        assert (
+            svc.authorize(
+                _make_request_openai_gateway(
+                    headers={"X-Real-IP": ip, "X-API-KEY": "topsecret"}
+                )
+            )
+            is True
+        )
+    finally:
+        set_setting("gateway_access_key", "")
+        set_setting("gateway_auth_throttle_max_failures", "10")
+        set_setting("gateway_auth_throttle_window_seconds", "60")
+        set_setting("gateway_require_auth", "1")
+        auth_throttle.clear()
+
+
+def test_authorize_throttle_resets_after_window(local_db, monkeypatch):
+    set_setting("gateway_require_auth", "1")
+    set_setting("gateway_access_key", "")
+    set_setting("gateway_auth_throttle_max_failures", "3")
+    set_setting("gateway_auth_throttle_window_seconds", "60")
+    auth_throttle.clear()
+    try:
+        now = [2000.0]
+        monkeypatch.setattr(auth_throttle.time, "time", lambda: now[0])
+        svc = GatewayService()
+        ip = "203.0.113.40"
+        for _ in range(3):
+            svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
+        assert auth_throttle.is_throttled(ip, 3, 60) is True
+        now[0] += 61
+        assert (
+            svc.authorize(_make_request_openai_gateway(headers={"X-Real-IP": ip}))
+            is False
+        )
+    finally:
+        set_setting("gateway_auth_throttle_max_failures", "10")
+        set_setting("gateway_auth_throttle_window_seconds", "60")
+        set_setting("gateway_require_auth", "1")
+        auth_throttle.clear()
+
+
 class FakeEmbedClient_openai_gateway:
     def __init__(self, *a, **k):
         self.calls = []
@@ -381,6 +502,36 @@ def test_embeddings_disabled_returns_503(local_db, monkeypatch):
     assert resp.status_code == 503
 
 
+def test_embeddings_missing_key_returns_clean_error_not_unauthenticated_call(
+    local_db, monkeypatch
+):
+    monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeEmbedClient_openai_gateway)
+    monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+    monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False)
+    set_setting("gateway_api_key", "")
+    set_setting("gateway_vision_key", "")
+    set_setting("gateway_embed_key", "")
+    try:
+        svc = GatewayService()
+        cfg = svc.effective_config()
+        assert cfg["gateway_embed_key"] == ""
+        cfg["gateway_embed_enabled"] = True
+        rt = svc.runtime()
+        resp = run_async(
+            rt.handle_embeddings(
+                {"input": "hello"}, cfg, ("guest", "no_key_probe"), "test", "default"
+            )
+        )
+        assert resp.status_code == 503
+        payload = json.loads(bytes(resp.body).decode())
+        assert payload["error"]["type"] == "embeddings_not_configured"
+        assert rt._client is None
+    finally:
+        set_setting("gateway_api_key", "")
+        set_setting("gateway_vision_key", "")
+        set_setting("gateway_embed_key", "")
+
+
 class FakeImageClient_openai_gateway:
     def __init__(self, *a, **k):
         self.calls = []
@@ -1942,3 +2093,121 @@ def test_hostile_upstream_model_header_is_ignored_end_to_end(local_db, monkeypat
         )
     )
     assert resp.headers["X-Gateway-Model"] == "deepseek-chat"
+
+
+def test_bypass_preamble_skips_the_operator_preamble_when_allowed(local_db, monkeypatch):
+    monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
+    svc = GatewayService()
+    cfg = svc.effective_config()
+    cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
+    rt = svc.runtime()
+    run_async(
+        rt.handle_chat(
+            {
+                "messages": [
+                    {"role": "system", "content": "Reply with strict JSON only."},
+                    {"role": "user", "content": "hi"},
+                ],
+                "bypass_preamble": True,
+            },
+            cfg,
+            ("internal", "devii"),
+            "test",
+            "default",
+            bypass_allowed=True,
+        )
+    )
+    sent = rt._client.calls[-1][1]
+    system_message = sent["messages"][0]
+    assert system_message["role"] == "system"
+    assert "OPERATOR-PREAMBLE-TEXT" not in system_message["content"]
+    assert "Current date" in system_message["content"]
+    assert "Reply with strict JSON only." in system_message["content"]
+    assert "bypass_preamble" not in sent
+
+
+def test_bypass_preamble_ignored_without_internal_credentials(local_db, monkeypatch):
+    monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
+    svc = GatewayService()
+    cfg = svc.effective_config()
+    cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
+    rt = svc.runtime()
+    run_async(
+        rt.handle_chat(
+            {"messages": [{"role": "user", "content": "hi"}], "bypass_preamble": True},
+            cfg,
+            ("user", "someuser"),
+            "test",
+            "default",
+            bypass_allowed=False,
+        )
+    )
+    sent = rt._client.calls[-1][1]
+    system_message = sent["messages"][0]
+    assert "OPERATOR-PREAMBLE-TEXT" in system_message["content"]
+    assert "bypass_preamble" not in sent
+
+
+def test_bypass_preamble_keeps_the_clients_own_system_content(local_db, monkeypatch):
+    monkeypatch.setattr(gwmod.httpx, "AsyncClient", FakeClient_openai_gateway)
+    svc = GatewayService()
+    cfg = svc.effective_config()
+    cfg["gateway_system_preamble"] = "OPERATOR-PREAMBLE-TEXT"
+    rt = svc.runtime()
+    run_async(
+        rt.handle_chat(
+            {
+                "messages": [
+                    {"role": "system", "content": "Grade strictly. Reply JSON only."},
+                    {"role": "user", "content": "hi"},
+                ],
+                "bypass_preamble": True,
+            },
+            cfg,
+            ("internal", "devii"),
+            "test",
+            "default",
+            bypass_allowed=True,
+        )
+    )
+    sent = rt._client.calls[-1][1]
+    system_message = sent["messages"][0]
+    assert "OPERATOR-PREAMBLE-TEXT" not in system_message["content"]
+    assert "Grade strictly. Reply JSON only." in system_message["content"]
+
+
+def test_internal_bypass_allowed_requires_the_exact_internal_key(local_db):
+    set_setting("gateway_internal_key", "test-internal-secret")
+    try:
+        svc = GatewayService()
+        cfg = svc.effective_config()
+        assert (
+            svc.internal_bypass_allowed(
+                _make_request_openai_gateway(
+                    headers={"X-Gateway-Internal-Key": "test-internal-secret"}
+                ),
+                cfg,
+            )
+            is True
+        )
+        assert (
+            svc.internal_bypass_allowed(
+                _make_request_openai_gateway(
+                    headers={"X-Gateway-Internal-Key": "wrong-secret"}
+                ),
+                cfg,
+            )
+            is False
+        )
+        assert (
+            svc.internal_bypass_allowed(
+                _make_request_openai_gateway(
+                    headers={"Authorization": "Bearer test-internal-secret"}
+                ),
+                cfg,
+            )
+            is False
+        )
+        assert svc.internal_bypass_allowed(_make_request_openai_gateway(), cfg) is False
+    finally:
+        set_setting("gateway_internal_key", "")

tests/unit/services/openai_gateway/routing.py

@@ -267,6 +267,32 @@ def test_embed_route_and_kind_isolation(local_db):
         _cleanup(["utemb"], ["ut-embed"])
 
 
+def test_embed_overlay_blank_provider_key_preserves_top_level_key(local_db):
+    r.provider_store.set(
+        r.ProviderIn(
+            name="utemblank",
+            base_url="https://emb.example/v1/chat/completions",
+            api_key="",
+        )
+    )
+    r.model_store.set(
+        r.ModelRouteIn(
+            source_model="ut-embed-blank",
+            provider="utemblank",
+            target_model="vendor/embed",
+            kind="embed",
+        )
+    )
+    try:
+        base_cfg = {"gateway_embed_key": "top-level-fallback-key"}
+        overlay = r.embed_overlay("ut-embed-blank", base_cfg)
+        assert "gateway_embed_key" not in overlay
+        merged = {**base_cfg, **overlay}
+        assert merged["gateway_embed_key"] == "top-level-fallback-key"
+    finally:
+        _cleanup(["utemblank"], ["ut-embed-blank"])
+
+
 def test_inactive_route_ignored(local_db):
     r.model_store.set(
         r.ModelRouteIn(

tests/unit/services/quiz/grading.py

@@ -134,8 +134,9 @@ def test_grade_free_text_uses_a_valid_verdict(monkeypatch):
 def test_grade_free_text_passes_the_answering_key_through(monkeypatch):
     seen = {}
 
-    def capture(api_key, system, text, timeout, model=None):
+    def capture(api_key, system, text, timeout, model=None, bypass_preamble=False):
         seen["api_key"] = api_key
+        seen["bypass_preamble"] = bypass_preamble
         return json.dumps({"score": 1.0}), None
 
     monkeypatch.setattr(grading, "gateway_complete", capture)
@@ -143,6 +144,18 @@ def test_grade_free_text_passes_the_answering_key_through(monkeypatch):
     assert seen["api_key"] == "member-key"
 
 
+def test_grade_free_text_bypasses_the_operator_preamble(monkeypatch):
+    seen = {}
+
+    def capture(api_key, system, text, timeout, model=None, bypass_preamble=False):
+        seen["bypass_preamble"] = bypass_preamble
+        return json.dumps({"score": 1.0}), None
+
+    monkeypatch.setattr(grading, "gateway_complete", capture)
+    grading.grade_free_text("member-key", QUESTION, "answer")
+    assert seen["bypass_preamble"] is True
+
+
 def test_the_fallback_score_is_the_deterministic_overlap():
     expected = scoring.token_overlap_score(QUESTION["expected_answer"], "one bucket index")
     result = grading.grade_free_text("", QUESTION, "one bucket index")