ticket #104 attempt 1

Browse Source
This commit is contained in:
Typosaurus 2026-07-19 20:00:56 +00:00
parent ad1736ebf1
commit a0d573375a
1 changed files with 16 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
devplacepy/services/jobs/deepsearch/crawl.py
View File

@ -22,6 +22,8 @@ from .pdf import MAX_PDF_BYTES, extract_pdf_text, is_pdf
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_pw_lock = asyncio.Lock()
RSEARCH_URL = "https://rsearch.app.molodetz.nl" RSEARCH_URL = "https://rsearch.app.molodetz.nl"
RSEARCH_TIMEOUT_SECONDS = 45.0 RSEARCH_TIMEOUT_SECONDS = 45.0
FETCH_TIMEOUT_SECONDS = 20.0 FETCH_TIMEOUT_SECONDS = 20.0
@ -242,8 +244,12 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
except (LookupError, ValueError) as exc: except (LookupError, ValueError) as exc:
logger.info("deepsearch decode failed for %s: %s", url, exc) logger.info("deepsearch decode failed for %s: %s", url, exc)
if len(text) < MIN_PAGE_CHARS: if len(text) < MIN_PAGE_CHARS:
async with _pw_lock:
try: try:
r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser) r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
except Exception as exc:
logger.info("deepsearch render failed for %s: %s", url, exc)
r_title, r_text, r_status, r_links = "", "", 0, []
if len(r_text) > len(text): if len(r_text) > len(text):
title, text, status, source, links = ( title, text, status, source, links = (
r_title or title, r_title or title,
@ -252,8 +258,6 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
"playwright", "playwright",
r_links, r_links,
) )
except Exception as exc:
logger.info("deepsearch render failed for %s: %s", url, exc)
if len(text) < MIN_PAGE_CHARS: if len(text) < MIN_PAGE_CHARS:
return None return None
return CrawledPage( return CrawledPage(