ticket #104 attempt 1

devplacepy/routers/profile/index.py

@@ -46,7 +46,6 @@ from devplacepy.utils import (
     track_action,
     build_achievements,
 )
-from devplacepy.utils.rewards import LEVEL_XP
 from devplacepy.responses import respond, action_result, wants_json
 from devplacepy.schemas import ProfileOut
 from devplacepy.avatar import avatar_url, avatar_seed
@@ -381,10 +380,6 @@ async def profile_page(
         ],
     )
 
-    xp = profile_user.get("xp") or 0
-    xp_progress_pct = xp % LEVEL_XP
-    xp_next_level = ((xp // LEVEL_XP) + 1) * LEVEL_XP
-
     return respond(
         request,
         "profile.html",
@@ -444,8 +439,6 @@ async def profile_page(
             "awards_pagination": awards_pagination,
             "awards_count": awards_count,
             "prominent_award": prominent_award,
-            "xp_progress_pct": xp_progress_pct,
-            "xp_next_level": xp_next_level,
             "can_give_award": can_give,
         },
         model=ProfileOut,

devplacepy/schemas/profile.py

@@ -80,8 +80,6 @@ class ProfileOut(_Out):
     awards_count: int = 0
     prominent_award: Optional[AwardOut] = None
     can_give_award: bool = False
-    xp_progress_pct: Optional[int] = None
-    xp_next_level: Optional[int] = None
 
 
 class TelegramPairOut(_Out):

devplacepy/services/jobs/deepsearch/crawl.py

@@ -22,6 +22,8 @@ from .pdf import MAX_PDF_BYTES, extract_pdf_text, is_pdf
 
 logger = logging.getLogger(__name__)
 
+_pw_lock = asyncio.Lock()
+
 RSEARCH_URL = "https://rsearch.app.molodetz.nl"
 RSEARCH_TIMEOUT_SECONDS = 45.0
 FETCH_TIMEOUT_SECONDS = 20.0
@@ -172,13 +174,7 @@ async def _render_with_playwright(
 ) -> tuple[str, str, int, list[tuple[str, str]]]:
     from playwright.async_api import async_playwright
 
-    own_browser = browser is None
-    if own_browser:
-        pw = await async_playwright().__aenter__()
-        browser = await pw.chromium.launch(
-            headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
-        )
-    try:
+    async def _render(browser) -> tuple[str, str, int, list[tuple[str, str]]]:
         context = await browser.new_context(user_agent=USER_AGENT)
         page = await context.new_page()
         response = await page.goto(url, wait_until="load", timeout=30000)
@@ -189,10 +185,18 @@ async def _render_with_playwright(
         await context.close()
         extracted = extract_html(content, base_url=url)
         return extracted.title, extracted.text, status, extracted.links
-    finally:
-        if own_browser:
-            await browser.close()
-            await pw.__aexit__(None, None, None)
+
+    if browser is None:
+        async with async_playwright() as pw:
+            browser = await pw.chromium.launch(
+                headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
+            )
+            try:
+                return await _render(browser)
+            finally:
+                await browser.close()
+    else:
+        return await _render(browser)
 
 
 async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
@@ -242,18 +246,20 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
         except (LookupError, ValueError) as exc:
             logger.info("deepsearch decode failed for %s: %s", url, exc)
     if len(text) < MIN_PAGE_CHARS:
-        try:
-            r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
-            if len(r_text) > len(text):
-                title, text, status, source, links = (
-                    r_title or title,
-                    r_text,
-                    r_status or status,
-                    "playwright",
-                    r_links,
-                )
-        except Exception as exc:
-            logger.info("deepsearch render failed for %s: %s", url, exc)
+        async with _pw_lock:
+            try:
+                r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
+            except Exception as exc:
+                logger.info("deepsearch render failed for %s: %s", url, exc)
+                r_title, r_text, r_status, r_links = "", "", 0, []
+        if len(r_text) > len(text):
+            title, text, status, source, links = (
+                r_title or title,
+                r_text,
+                r_status or status,
+                "playwright",
+                r_links,
+            )
     if len(text) < MIN_PAGE_CHARS:
         return None
     return CrawledPage(
@@ -296,102 +302,99 @@ async def crawl(
     total = min(len(level_candidates), max_pages)
     cancelled = False
 
-    pw = None
     browser = None
-    try:
-        pw = await async_playwright().__aenter__()
-        browser = await pw.chromium.launch(
-            headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
-        )
-    except Exception as exc:
-        logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc)
+    async with async_playwright() as pw:
+        try:
+            browser = await pw.chromium.launch(
+                headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
+            )
+        except Exception as exc:
+            logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc)
 
-    try:
-        for level in range(max(1, depth)):
-            if cancelled or fetched >= max_pages or not level_candidates:
-                break
-            next_candidates: list[dict] = []
-            for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
-                if fetched >= max_pages:
+        try:
+            for level in range(max(1, depth)):
+                if cancelled or fetched >= max_pages or not level_candidates:
                     break
-                if await should_stop():
-                    emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
-                    cancelled = True
-                    break
-                batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
-                for candidate in batch:
-                    emit(
-                        {
-                            "type": "progress",
-                            "done": fetched,
-                            "total": total,
-                            "url": candidate["url"],
-                            "depth": level,
-                            "message": f"Reading {candidate['url']}",
-                        }
-                    )
-                    if is_cached(candidate["url"]):
-                        emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
-                fetch_start = time.perf_counter()
-                results = await asyncio.gather(
-                    *(_resolve_candidate(candidate, level, browser) for candidate in batch),
-                    return_exceptions=True,
-                )
-                elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
-                for candidate, page in zip(batch, results):
-                    url = candidate["url"]
-                    if isinstance(page, BaseException):
-                        logger.info("deepsearch fetch crashed for %s: %s", url, page)
-                        page = None
-                    if page is None:
-                        emit(
-                            {
-                                "type": "page_skipped",
-                                "url": url,
-                                "reason": "no readable content",
-                                "elapsed_ms": elapsed_ms,
-                            }
-                        )
-                        continue
+                next_candidates: list[dict] = []
+                for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
                     if fetched >= max_pages:
                         break
-                    digest = content_hash(page.text)
-                    if digest in outcome.seen_hashes:
+                    if await should_stop():
+                        emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
+                        cancelled = True
+                        break
+                    batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
+                    for candidate in batch:
                         emit(
                             {
-                                "type": "page_duplicate",
-                                "url": url,
-                                "reason": "duplicate content",
-                                "elapsed_ms": elapsed_ms,
+                                "type": "progress",
+                                "done": fetched,
+                                "total": total,
+                                "url": candidate["url"],
+                                "depth": level,
+                                "message": f"Reading {candidate['url']}",
                             }
                         )
-                        continue
-                    outcome.seen_hashes.add(digest)
-                    outcome.pages.append(page)
-                    fetched += 1
-                    emit(
-                        {
-                            "type": "page_loaded",
-                            "url": page.url,
-                            "title": page.title,
-                            "source": page.source,
-                            "depth": level,
-                            "render": page.source == "playwright",
-                            "elapsed_ms": elapsed_ms,
-                            "done": fetched,
-                            "total": total,
-                        }
+                        if is_cached(candidate["url"]):
+                            emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
+                    fetch_start = time.perf_counter()
+                    results = await asyncio.gather(
+                        *(_resolve_candidate(candidate, level, browser) for candidate in batch),
+                        return_exceptions=True,
                     )
-                    if level + 1 < depth:
-                        for link in relevant_links(page.links, query, LINKS_PER_PAGE):
-                            if link not in seen_urls:
-                                seen_urls.add(link)
-                                next_candidates.append({"url": link})
-            level_candidates = next_candidates
-            total = min(total + len(next_candidates), max_pages)
-    finally:
-        if browser is not None:
-            await browser.close()
-        if pw is not None:
-            await pw.__aexit__(None, None, None)
+                    elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
+                    for candidate, page in zip(batch, results):
+                        url = candidate["url"]
+                        if isinstance(page, BaseException):
+                            logger.info("deepsearch fetch crashed for %s: %s", url, page)
+                            page = None
+                        if page is None:
+                            emit(
+                                {
+                                    "type": "page_skipped",
+                                    "url": url,
+                                    "reason": "no readable content",
+                                    "elapsed_ms": elapsed_ms,
+                                }
+                            )
+                            continue
+                        if fetched >= max_pages:
+                            break
+                        digest = content_hash(page.text)
+                        if digest in outcome.seen_hashes:
+                            emit(
+                                {
+                                    "type": "page_duplicate",
+                                    "url": url,
+                                    "reason": "duplicate content",
+                                    "elapsed_ms": elapsed_ms,
+                                }
+                            )
+                            continue
+                        outcome.seen_hashes.add(digest)
+                        outcome.pages.append(page)
+                        fetched += 1
+                        emit(
+                            {
+                                "type": "page_loaded",
+                                "url": page.url,
+                                "title": page.title,
+                                "source": page.source,
+                                "depth": level,
+                                "render": page.source == "playwright",
+                                "elapsed_ms": elapsed_ms,
+                                "done": fetched,
+                                "total": total,
+                            }
+                        )
+                        if level + 1 < depth:
+                            for link in relevant_links(page.links, query, LINKS_PER_PAGE):
+                                if link not in seen_urls:
+                                    seen_urls.add(link)
+                                    next_candidates.append({"url": link})
+                level_candidates = next_candidates
+                total = min(total + len(next_candidates), max_pages)
+        finally:
+            if browser is not None:
+                await browser.close()
     return outcome

tests/api/profile/index.py

@@ -267,37 +267,6 @@ def test_activity_comment_card_renders_overlay_link(app_server):
     assert f'class="card-link" href="/posts/{post_slug}#comment-{comment_uid}"' in r.text
 
 
-def test_profile_json_contains_xp_progress(app_server):
-    from devplacepy.database import get_table, refresh_snapshot
-    import time
-
-    name = f"xp{int(time.time() * 1000)}"
-    session = requests.Session()
-    session.post(
-        f"{BASE_URL}/auth/signup",
-        data={
-            "username": name,
-            "email": f"{name}@t.dev",
-            "password": "secret123",
-            "confirm_password": "secret123",
-        },
-        allow_redirects=True,
-    )
-
-    user = get_table("users").find_one(username=name)
-    assert user is not None
-    get_table("users").update({"uid": user["uid"], "xp": 250}, ["uid"])
-    refresh_snapshot()
-
-    r = session.get(
-        f"{BASE_URL}/profile/{name}", headers={"Accept": "application/json"}
-    )
-    assert r.status_code == 200, f"status {r.status_code}: {r.text[:200]}"
-    body = r.json()
-    assert body["xp_progress_pct"] == 50
-    assert body["xp_next_level"] == 300
-
-
 def test_profile_json_exposes_online_presence(app_server):
     import time