ticket #138 attempt 1

Reviewed-on: retoor/devplacepy#124

devplacepy/main.py

@@ -44,7 +44,6 @@ from devplacepy.templating import templates, jinja_unread_count
 from devplacepy.cache import TTLCache
 from devplacepy.responses import respond, wants_json, json_error
 from devplacepy.schemas import LandingOut, ValidationErrorOut
-from devplacepy.attachments import get_attachments_batch
 from fastapi.responses import JSONResponse
 from devplacepy.utils import get_current_user, time_ago, safe_next, client_ip
 from devplacepy.seo import base_seo_context, site_url, website_schema
@@ -672,7 +671,6 @@ def _landing_recent_posts(blocked):
             authors = get_users_by_uids(author_uids)
             comment_counts = get_comment_counts_by_post_uids(post_uids)
             upvotes, downvotes = get_vote_counts(post_uids)
-            attachments_map = get_attachments_batch("post", post_uids)
             for p in raw_posts:
                 posts.append(
                     {
@@ -682,7 +680,6 @@ def _landing_recent_posts(blocked):
                         "comment_count": comment_counts.get(p["uid"], 0),
                         "stars": upvotes.get(p["uid"], 0) - downvotes.get(p["uid"], 0),
                         "slug": p.get("slug", "") or p["uid"],
-                        "attachments": attachments_map.get(p["uid"], []),
                     }
                 )
     if not blocked:

devplacepy/routers/profile/index.py

@@ -7,6 +7,7 @@ from devplacepy.models import ProfileForm
 from fastapi.responses import HTMLResponse, JSONResponse
 from devplacepy.database import (
     get_table,
+    db,
     get_customization_prefs,
     get_notification_prefs,
     get_user_stars,
@@ -47,7 +48,6 @@ from devplacepy.utils import (
 )
 from devplacepy.responses import respond, action_result, wants_json
 from devplacepy.schemas import ProfileOut
-from devplacepy.attachments import get_attachments_batch
 from devplacepy.avatar import avatar_url, avatar_seed
 from devplacepy.seo import (
     base_seo_context,
@@ -188,10 +188,8 @@ async def profile_page(
             else set()
         )
         polls_map = get_polls_by_post_uids(post_uids, current_user)
-        attachments_map = get_attachments_batch("post", post_uids)
         for item in posts:
             uid = item["post"]["uid"]
-            item["attachments"] = attachments_map.get(uid, [])
             item["reactions"] = reactions_map.get(uid, {"counts": {}, "mine": []})
             item["bookmarked"] = uid in bookmark_set
             item["poll"] = polls_map.get(uid)

devplacepy/schemas/auth.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 from typing import Any, Optional
 
 from devplacepy.schemas.base import _Out
-from devplacepy.schemas.content import AttachmentOut, UserOut
+from devplacepy.schemas.content import UserOut
 
 
 class AuthPageOut(_Out):
@@ -38,7 +38,6 @@ class LandingPostOut(_Out):
     comment_count: int = 0
     stars: int = 0
     slug: str = ""
-    attachments: list[AttachmentOut] = []
 
 
 class TrendingTopicOut(_Out):

devplacepy/services/jobs/deepsearch/crawl.py

@@ -22,6 +22,8 @@ from .pdf import MAX_PDF_BYTES, extract_pdf_text, is_pdf
 
 logger = logging.getLogger(__name__)
 
+_pw_lock = asyncio.Lock()
+
 RSEARCH_URL = "https://rsearch.app.molodetz.nl"
 RSEARCH_TIMEOUT_SECONDS = 45.0
 FETCH_TIMEOUT_SECONDS = 20.0
@@ -172,13 +174,7 @@ async def _render_with_playwright(
 ) -> tuple[str, str, int, list[tuple[str, str]]]:
     from playwright.async_api import async_playwright
 
-    own_browser = browser is None
+    async def _render(browser) -> tuple[str, str, int, list[tuple[str, str]]]:
-    if own_browser:
-        pw = await async_playwright().__aenter__()
-        browser = await pw.chromium.launch(
-            headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
-        )
-    try:
         context = await browser.new_context(user_agent=USER_AGENT)
         page = await context.new_page()
         response = await page.goto(url, wait_until="load", timeout=30000)
@@ -189,10 +185,18 @@ async def _render_with_playwright(
         await context.close()
         extracted = extract_html(content, base_url=url)
         return extracted.title, extracted.text, status, extracted.links
+
+    if browser is None:
+        async with async_playwright() as pw:
+            browser = await pw.chromium.launch(
+                headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
+            )
+            try:
+                return await _render(browser)
             finally:
-        if own_browser:
                 await browser.close()
-            await pw.__aexit__(None, None, None)
+    else:
+        return await _render(browser)
 
 
 async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
@@ -242,8 +246,12 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
         except (LookupError, ValueError) as exc:
             logger.info("deepsearch decode failed for %s: %s", url, exc)
     if len(text) < MIN_PAGE_CHARS:
+        async with _pw_lock:
             try:
                 r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
+            except Exception as exc:
+                logger.info("deepsearch render failed for %s: %s", url, exc)
+                r_title, r_text, r_status, r_links = "", "", 0, []
         if len(r_text) > len(text):
             title, text, status, source, links = (
                 r_title or title,
@@ -252,8 +260,6 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
                 "playwright",
                 r_links,
             )
-        except Exception as exc:
-            logger.info("deepsearch render failed for %s: %s", url, exc)
     if len(text) < MIN_PAGE_CHARS:
         return None
     return CrawledPage(
@@ -296,10 +302,9 @@ async def crawl(
     total = min(len(level_candidates), max_pages)
     cancelled = False
 
-    pw = None
     browser = None
+    async with async_playwright() as pw:
         try:
-        pw = await async_playwright().__aenter__()
             browser = await pw.chromium.launch(
                 headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
             )
@@ -392,6 +397,4 @@ async def crawl(
         finally:
             if browser is not None:
                 await browser.close()
-        if pw is not None:
-            await pw.__aexit__(None, None, None)
     return outcome

devplacepy/static/css/base.css

@@ -620,6 +620,10 @@ img {
 .topnav-logo span { color: var(--accent); }
 .topnav-links { display: flex; gap: 0.25rem; }
 .topnav-link {
+    display: inline-flex;
+    align-items: center;
+    gap: 0.375rem;
+    white-space: nowrap;
     padding: 0.5rem 0.75rem;
     border-radius: var(--radius);
     font-size: 0.875rem;