ticket #104 attempt 1

devplacepy/database/schema.py

@@ -30,26 +30,6 @@ def migrate_bug_tables_to_issue_tables() -> None:
         logger.info("Dropped table %s after migration", source_name)
 
 
-def _add_message_unique_index() -> None:
-    if "messages" not in db.tables:
-        return
-    with db:
-        db.query(
-            """
-            DELETE FROM messages WHERE id NOT IN (
-                SELECT MIN(id) FROM messages GROUP BY sender_uid, receiver_uid, content
-            )
-            """
-        )
-    _index(
-        db,
-        "messages",
-        "idx_messages_unique_sender_receiver_content",
-        ["sender_uid", "receiver_uid", "content"],
-        unique=True,
-    )
-
-
 def init_db():
     tables = db.tables
     _index(db, "users", "idx_users_username", ["username"])
@@ -151,7 +131,6 @@ def init_db():
         "idx_messages_conversation_rev",
         ["receiver_uid", "sender_uid"],
     )
-    _add_message_unique_index()
     _index(db, "notifications", "idx_notifications_user", ["user_uid"])
     _index(db, "notifications", "idx_notifications_user_read", ["user_uid", "read"])
     _index(db, "push_registration", "idx_push_registration_user", ["user_uid"])

devplacepy/services/correction.py

@@ -88,7 +88,7 @@ def gateway_complete(
             {"role": "system", "content": system},
             {"role": "user", "content": text},
         ],
-        "temperature": 0.0,
+        "temperature": 0.1,
     }
     headers = {
         "Content-Type": "application/json",

devplacepy/services/jobs/deepsearch/crawl.py

@@ -22,6 +22,8 @@ from .pdf import MAX_PDF_BYTES, extract_pdf_text, is_pdf
 
 logger = logging.getLogger(__name__)
 
+_pw_lock = asyncio.Lock()
+
 RSEARCH_URL = "https://rsearch.app.molodetz.nl"
 RSEARCH_TIMEOUT_SECONDS = 45.0
 FETCH_TIMEOUT_SECONDS = 20.0
@@ -172,13 +174,7 @@ async def _render_with_playwright(
 ) -> tuple[str, str, int, list[tuple[str, str]]]:
     from playwright.async_api import async_playwright
 
-    own_browser = browser is None
+    async def _render(browser) -> tuple[str, str, int, list[tuple[str, str]]]:
-    if own_browser:
-        pw = await async_playwright().__aenter__()
-        browser = await pw.chromium.launch(
-            headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
-        )
-    try:
         context = await browser.new_context(user_agent=USER_AGENT)
         page = await context.new_page()
         response = await page.goto(url, wait_until="load", timeout=30000)
@@ -189,10 +185,18 @@ async def _render_with_playwright(
         await context.close()
         extracted = extract_html(content, base_url=url)
         return extracted.title, extracted.text, status, extracted.links
+
+    if browser is None:
+        async with async_playwright() as pw:
+            browser = await pw.chromium.launch(
+                headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
+            )
+            try:
+                return await _render(browser)
             finally:
-        if own_browser:
                 await browser.close()
-            await pw.__aexit__(None, None, None)
+    else:
+        return await _render(browser)
 
 
 async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
@@ -242,8 +246,12 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
         except (LookupError, ValueError) as exc:
             logger.info("deepsearch decode failed for %s: %s", url, exc)
     if len(text) < MIN_PAGE_CHARS:
+        async with _pw_lock:
             try:
                 r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
+            except Exception as exc:
+                logger.info("deepsearch render failed for %s: %s", url, exc)
+                r_title, r_text, r_status, r_links = "", "", 0, []
         if len(r_text) > len(text):
             title, text, status, source, links = (
                 r_title or title,
@@ -252,8 +260,6 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
                 "playwright",
                 r_links,
             )
-        except Exception as exc:
-            logger.info("deepsearch render failed for %s: %s", url, exc)
     if len(text) < MIN_PAGE_CHARS:
         return None
     return CrawledPage(
@@ -296,10 +302,9 @@ async def crawl(
     total = min(len(level_candidates), max_pages)
     cancelled = False
 
-    pw = None
     browser = None
+    async with async_playwright() as pw:
         try:
-        pw = await async_playwright().__aenter__()
             browser = await pw.chromium.launch(
                 headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
             )
@@ -392,6 +397,4 @@ async def crawl(
         finally:
             if browser is not None:
                 await browser.close()
-        if pw is not None:
-            await pw.__aexit__(None, None, None)
     return outcome

devplacepy/services/messaging/persist.py

@@ -1,9 +1,6 @@
 # retoor <retoor@molodetz.nl>
 
-import hashlib
 import logging
-import time
-from collections import OrderedDict
 from datetime import datetime, timezone
 from typing import Any, Optional
 
@@ -18,16 +15,12 @@ from devplacepy.utils import (
     track_action,
 )
 from devplacepy.services.audit import record as audit
-from sqlalchemy.exc import IntegrityError
-
 from devplacepy.services.correction import schedule_correction
 from devplacepy.services.ai_modifier import schedule_modification
 
 logger = logging.getLogger("messaging.persist")
 
 MAX_CONTENT_LENGTH = 2000
-DEDUP_WINDOW_SECONDS = 3
-_content_cache: dict[str, tuple[float, str]] = OrderedDict()
 
 
 def _slim_attachment(attachment: dict[str, Any]) -> dict[str, Any]:
@@ -88,32 +81,9 @@ def persist_message(
 
     sender_uid = sender["uid"]
     sender_username = sender.get("username", "")
-
-    content_hash = hashlib.sha256(
-        f"{sender_uid}:{receiver_uid}:{content}".encode()
-    ).hexdigest()[:16]
-    now = time.time()
-    last_seen, cached_uid = _content_cache.get(content_hash, (0.0, None))
-    if now - last_seen < DEDUP_WINDOW_SECONDS and cached_uid is not None:
-        logger.debug(
-            "Dedup hit for message hash %s (original uid %s)", content_hash, cached_uid
-        )
-        cached = get_table("messages").find_one(uid=cached_uid)
-        if cached:
-            return {
-                "uid": cached["uid"],
-                "sender_uid": cached["sender_uid"],
-                "receiver_uid": cached["receiver_uid"],
-                "content": cached["content"],
-                "read": cached.get("read", False),
-                "created_at": cached["created_at"],
-            }
-
     messages_table = get_table("messages")
     msg_uid = generate_uid()
     created_at = datetime.now(timezone.utc).isoformat()
-
-    try:
     messages_table.insert(
         {
             "uid": msg_uid,
@@ -124,24 +94,6 @@ def persist_message(
             "created_at": created_at,
         }
     )
-    except IntegrityError:
-        existing = messages_table.find_one(
-            sender_uid=sender_uid, receiver_uid=receiver_uid, content=content
-        )
-        if not existing:
-            raise
-        logger.debug(
-            "Dedup via unique constraint for message (uid %s)", existing["uid"]
-        )
-        _content_cache[content_hash] = (time.time(), existing["uid"])
-        return {
-            "uid": existing["uid"],
-            "sender_uid": existing["sender_uid"],
-            "receiver_uid": existing["receiver_uid"],
-            "content": existing["content"],
-            "read": existing.get("read", False),
-            "created_at": existing["created_at"],
-        }
 
     link_attachments(attachment_uids, "message", msg_uid)
     schedule_correction(sender, "messages", msg_uid, request)
@@ -162,8 +114,6 @@ def persist_message(
     )
     track_action(sender_uid, "message")
 
-    _content_cache[content_hash] = (time.time(), msg_uid)
-
     logger.info(
         "Message %s sent from %s to %s via %s",
         msg_uid,

tests/api/messages/send.py

@@ -166,23 +166,3 @@ def test_send_attachment_only_empty_content_succeeds(seeded_db):
     refresh_snapshot()
     row = get_table("messages").find_one(uid=msg["uid"])
     assert row["content"] == ""
-
-
-def test_duplicate_message_returns_same_uid(seeded_db):
-    s, _ = _member()
-    receiver = _db_user("bob_test")["uid"]
-    content = _unique("dupmsg")
-
-    first = s.post(
-        f"{BASE_URL}/messages/send",
-        headers=JSON_audit_log,
-        data={"content": content, "receiver_uid": receiver},
-    ).json()["data"]
-
-    second = s.post(
-        f"{BASE_URL}/messages/send",
-        headers=JSON_audit_log,
-        data={"content": content, "receiver_uid": receiver},
-    ).json()["data"]
-
-    assert first["uid"] == second["uid"], "duplicate messages should return the same uid"