ticket #85 attempt 1

devplacepy/database/schema.py

@@ -30,6 +30,26 @@ def migrate_bug_tables_to_issue_tables() -> None:
         logger.info("Dropped table %s after migration", source_name)
 
 
+def _add_message_unique_index() -> None:
+    if "messages" not in db.tables:
+        return
+    with db:
+        db.query(
+            """
+            DELETE FROM messages WHERE id NOT IN (
+                SELECT MIN(id) FROM messages GROUP BY sender_uid, receiver_uid, content
+            )
+            """
+        )
+    _index(
+        db,
+        "messages",
+        "idx_messages_unique_sender_receiver_content",
+        ["sender_uid", "receiver_uid", "content"],
+        unique=True,
+    )
+
+
 def init_db():
     tables = db.tables
     _index(db, "users", "idx_users_username", ["username"])
@@ -131,6 +151,7 @@ def init_db():
         "idx_messages_conversation_rev",
         ["receiver_uid", "sender_uid"],
     )
+    _add_message_unique_index()
     _index(db, "notifications", "idx_notifications_user", ["user_uid"])
     _index(db, "notifications", "idx_notifications_user_read", ["user_uid", "read"])
     _index(db, "push_registration", "idx_push_registration_user", ["user_uid"])

devplacepy/services/correction.py

@@ -88,7 +88,7 @@ def gateway_complete(
             {"role": "system", "content": system},
             {"role": "user", "content": text},
         ],
-        "temperature": 0.1,
+        "temperature": 0.0,
     }
     headers = {
         "Content-Type": "application/json",

devplacepy/services/jobs/deepsearch/crawl.py

@@ -22,8 +22,6 @@ from .pdf import MAX_PDF_BYTES, extract_pdf_text, is_pdf
 
 logger = logging.getLogger(__name__)
 
-_pw_lock = asyncio.Lock()
-
 RSEARCH_URL = "https://rsearch.app.molodetz.nl"
 RSEARCH_TIMEOUT_SECONDS = 45.0
 FETCH_TIMEOUT_SECONDS = 20.0
@@ -174,7 +172,13 @@ async def _render_with_playwright(
 ) -> tuple[str, str, int, list[tuple[str, str]]]:
     from playwright.async_api import async_playwright
 
-    async def _render(browser) -> tuple[str, str, int, list[tuple[str, str]]]:
+    own_browser = browser is None
+    if own_browser:
+        pw = await async_playwright().__aenter__()
+        browser = await pw.chromium.launch(
+            headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
+        )
+    try:
         context = await browser.new_context(user_agent=USER_AGENT)
         page = await context.new_page()
         response = await page.goto(url, wait_until="load", timeout=30000)
@@ -185,18 +189,10 @@ async def _render_with_playwright(
         await context.close()
         extracted = extract_html(content, base_url=url)
         return extracted.title, extracted.text, status, extracted.links
-
-    if browser is None:
-        async with async_playwright() as pw:
-            browser = await pw.chromium.launch(
-                headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
-            )
-            try:
-                return await _render(browser)
-            finally:
-                await browser.close()
-    else:
-        return await _render(browser)
+    finally:
+        if own_browser:
+            await browser.close()
+            await pw.__aexit__(None, None, None)
 
 
 async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
@@ -246,20 +242,18 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
         except (LookupError, ValueError) as exc:
             logger.info("deepsearch decode failed for %s: %s", url, exc)
     if len(text) < MIN_PAGE_CHARS:
-        async with _pw_lock:
-            try:
-                r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
-            except Exception as exc:
-                logger.info("deepsearch render failed for %s: %s", url, exc)
-                r_title, r_text, r_status, r_links = "", "", 0, []
-        if len(r_text) > len(text):
-            title, text, status, source, links = (
-                r_title or title,
-                r_text,
-                r_status or status,
-                "playwright",
-                r_links,
-            )
+        try:
+            r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
+            if len(r_text) > len(text):
+                title, text, status, source, links = (
+                    r_title or title,
+                    r_text,
+                    r_status or status,
+                    "playwright",
+                    r_links,
+                )
+        except Exception as exc:
+            logger.info("deepsearch render failed for %s: %s", url, exc)
     if len(text) < MIN_PAGE_CHARS:
         return None
     return CrawledPage(
@@ -302,99 +296,102 @@ async def crawl(
     total = min(len(level_candidates), max_pages)
     cancelled = False
 
+    pw = None
     browser = None
-    async with async_playwright() as pw:
-        try:
-            browser = await pw.chromium.launch(
-                headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
-            )
-        except Exception as exc:
-            logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc)
+    try:
+        pw = await async_playwright().__aenter__()
+        browser = await pw.chromium.launch(
+            headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
+        )
+    except Exception as exc:
+        logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc)
 
-        try:
-            for level in range(max(1, depth)):
-                if cancelled or fetched >= max_pages or not level_candidates:
+    try:
+        for level in range(max(1, depth)):
+            if cancelled or fetched >= max_pages or not level_candidates:
+                break
+            next_candidates: list[dict] = []
+            for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
+                if fetched >= max_pages:
                     break
-                next_candidates: list[dict] = []
-                for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
+                if await should_stop():
+                    emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
+                    cancelled = True
+                    break
+                batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
+                for candidate in batch:
+                    emit(
+                        {
+                            "type": "progress",
+                            "done": fetched,
+                            "total": total,
+                            "url": candidate["url"],
+                            "depth": level,
+                            "message": f"Reading {candidate['url']}",
+                        }
+                    )
+                    if is_cached(candidate["url"]):
+                        emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
+                fetch_start = time.perf_counter()
+                results = await asyncio.gather(
+                    *(_resolve_candidate(candidate, level, browser) for candidate in batch),
+                    return_exceptions=True,
+                )
+                elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
+                for candidate, page in zip(batch, results):
+                    url = candidate["url"]
+                    if isinstance(page, BaseException):
+                        logger.info("deepsearch fetch crashed for %s: %s", url, page)
+                        page = None
+                    if page is None:
+                        emit(
+                            {
+                                "type": "page_skipped",
+                                "url": url,
+                                "reason": "no readable content",
+                                "elapsed_ms": elapsed_ms,
+                            }
+                        )
+                        continue
                     if fetched >= max_pages:
                         break
-                    if await should_stop():
-                        emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
-                        cancelled = True
-                        break
-                    batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
-                    for candidate in batch:
+                    digest = content_hash(page.text)
+                    if digest in outcome.seen_hashes:
                         emit(
                             {
-                                "type": "progress",
-                                "done": fetched,
-                                "total": total,
-                                "url": candidate["url"],
-                                "depth": level,
-                                "message": f"Reading {candidate['url']}",
-                            }
-                        )
-                        if is_cached(candidate["url"]):
-                            emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
-                    fetch_start = time.perf_counter()
-                    results = await asyncio.gather(
-                        *(_resolve_candidate(candidate, level, browser) for candidate in batch),
-                        return_exceptions=True,
-                    )
-                    elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
-                    for candidate, page in zip(batch, results):
-                        url = candidate["url"]
-                        if isinstance(page, BaseException):
-                            logger.info("deepsearch fetch crashed for %s: %s", url, page)
-                            page = None
-                        if page is None:
-                            emit(
-                                {
-                                    "type": "page_skipped",
-                                    "url": url,
-                                    "reason": "no readable content",
-                                    "elapsed_ms": elapsed_ms,
-                                }
-                            )
-                            continue
-                        if fetched >= max_pages:
-                            break
-                        digest = content_hash(page.text)
-                        if digest in outcome.seen_hashes:
-                            emit(
-                                {
-                                    "type": "page_duplicate",
-                                    "url": url,
-                                    "reason": "duplicate content",
-                                    "elapsed_ms": elapsed_ms,
-                                }
-                            )
-                            continue
-                        outcome.seen_hashes.add(digest)
-                        outcome.pages.append(page)
-                        fetched += 1
-                        emit(
-                            {
-                                "type": "page_loaded",
-                                "url": page.url,
-                                "title": page.title,
-                                "source": page.source,
-                                "depth": level,
-                                "render": page.source == "playwright",
+                                "type": "page_duplicate",
+                                "url": url,
+                                "reason": "duplicate content",
                                 "elapsed_ms": elapsed_ms,
-                                "done": fetched,
-                                "total": total,
                             }
                         )
-                        if level + 1 < depth:
-                            for link in relevant_links(page.links, query, LINKS_PER_PAGE):
-                                if link not in seen_urls:
-                                    seen_urls.add(link)
-                                    next_candidates.append({"url": link})
-                level_candidates = next_candidates
-                total = min(total + len(next_candidates), max_pages)
-        finally:
-            if browser is not None:
-                await browser.close()
+                        continue
+                    outcome.seen_hashes.add(digest)
+                    outcome.pages.append(page)
+                    fetched += 1
+                    emit(
+                        {
+                            "type": "page_loaded",
+                            "url": page.url,
+                            "title": page.title,
+                            "source": page.source,
+                            "depth": level,
+                            "render": page.source == "playwright",
+                            "elapsed_ms": elapsed_ms,
+                            "done": fetched,
+                            "total": total,
+                        }
+                    )
+                    if level + 1 < depth:
+                        for link in relevant_links(page.links, query, LINKS_PER_PAGE):
+                            if link not in seen_urls:
+                                seen_urls.add(link)
+                                next_candidates.append({"url": link})
+            level_candidates = next_candidates
+            total = min(total + len(next_candidates), max_pages)
+    finally:
+        if browser is not None:
+            await browser.close()
+        if pw is not None:
+            await pw.__aexit__(None, None, None)
     return outcome

devplacepy/services/messaging/persist.py

@@ -1,6 +1,9 @@
 # retoor <retoor@molodetz.nl>
 
+import hashlib
 import logging
+import time
+from collections import OrderedDict
 from datetime import datetime, timezone
 from typing import Any, Optional
 
@@ -15,12 +18,16 @@ from devplacepy.utils import (
     track_action,
 )
 from devplacepy.services.audit import record as audit
+from sqlalchemy.exc import IntegrityError
+
 from devplacepy.services.correction import schedule_correction
 from devplacepy.services.ai_modifier import schedule_modification
 
 logger = logging.getLogger("messaging.persist")
 
 MAX_CONTENT_LENGTH = 2000
+DEDUP_WINDOW_SECONDS = 3
+_content_cache: dict[str, tuple[float, str]] = OrderedDict()
 
 
 def _slim_attachment(attachment: dict[str, Any]) -> dict[str, Any]:
@@ -81,19 +88,60 @@ def persist_message(
 
     sender_uid = sender["uid"]
     sender_username = sender.get("username", "")
+
+    content_hash = hashlib.sha256(
+        f"{sender_uid}:{receiver_uid}:{content}".encode()
+    ).hexdigest()[:16]
+    now = time.time()
+    last_seen, cached_uid = _content_cache.get(content_hash, (0.0, None))
+    if now - last_seen < DEDUP_WINDOW_SECONDS and cached_uid is not None:
+        logger.debug(
+            "Dedup hit for message hash %s (original uid %s)", content_hash, cached_uid
+        )
+        cached = get_table("messages").find_one(uid=cached_uid)
+        if cached:
+            return {
+                "uid": cached["uid"],
+                "sender_uid": cached["sender_uid"],
+                "receiver_uid": cached["receiver_uid"],
+                "content": cached["content"],
+                "read": cached.get("read", False),
+                "created_at": cached["created_at"],
+            }
+
     messages_table = get_table("messages")
     msg_uid = generate_uid()
     created_at = datetime.now(timezone.utc).isoformat()
-    messages_table.insert(
-        {
-            "uid": msg_uid,
-            "sender_uid": sender_uid,
-            "receiver_uid": receiver_uid,
-            "content": content,
-            "read": False,
-            "created_at": created_at,
+
+    try:
+        messages_table.insert(
+            {
+                "uid": msg_uid,
+                "sender_uid": sender_uid,
+                "receiver_uid": receiver_uid,
+                "content": content,
+                "read": False,
+                "created_at": created_at,
+            }
+        )
+    except IntegrityError:
+        existing = messages_table.find_one(
+            sender_uid=sender_uid, receiver_uid=receiver_uid, content=content
+        )
+        if not existing:
+            raise
+        logger.debug(
+            "Dedup via unique constraint for message (uid %s)", existing["uid"]
+        )
+        _content_cache[content_hash] = (time.time(), existing["uid"])
+        return {
+            "uid": existing["uid"],
+            "sender_uid": existing["sender_uid"],
+            "receiver_uid": existing["receiver_uid"],
+            "content": existing["content"],
+            "read": existing.get("read", False),
+            "created_at": existing["created_at"],
         }
-    )
 
     link_attachments(attachment_uids, "message", msg_uid)
     schedule_correction(sender, "messages", msg_uid, request)
@@ -114,6 +162,8 @@ def persist_message(
     )
     track_action(sender_uid, "message")
 
+    _content_cache[content_hash] = (time.time(), msg_uid)
+
     logger.info(
         "Message %s sent from %s to %s via %s",
         msg_uid,

tests/api/messages/send.py

@@ -166,3 +166,23 @@ def test_send_attachment_only_empty_content_succeeds(seeded_db):
     refresh_snapshot()
     row = get_table("messages").find_one(uid=msg["uid"])
     assert row["content"] == ""
+
+
+def test_duplicate_message_returns_same_uid(seeded_db):
+    s, _ = _member()
+    receiver = _db_user("bob_test")["uid"]
+    content = _unique("dupmsg")
+
+    first = s.post(
+        f"{BASE_URL}/messages/send",
+        headers=JSON_audit_log,
+        data={"content": content, "receiver_uid": receiver},
+    ).json()["data"]
+
+    second = s.post(
+        f"{BASE_URL}/messages/send",
+        headers=JSON_audit_log,
+        data={"content": content, "receiver_uid": receiver},
+    ).json()["data"]
+
+    assert first["uid"] == second["uid"], "duplicate messages should return the same uid"