From a0d573375a211281adf40608033a50b6633b01c0 Mon Sep 17 00:00:00 2001 From: Typosaurus Date: Sun, 19 Jul 2026 20:00:56 +0000 Subject: [PATCH 1/2] ticket #104 attempt 1 --- devplacepy/services/jobs/deepsearch/crawl.py | 28 +++++++++++--------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/devplacepy/services/jobs/deepsearch/crawl.py b/devplacepy/services/jobs/deepsearch/crawl.py index 0bba51a1..227edae5 100644 --- a/devplacepy/services/jobs/deepsearch/crawl.py +++ b/devplacepy/services/jobs/deepsearch/crawl.py @@ -22,6 +22,8 @@ from .pdf import MAX_PDF_BYTES, extract_pdf_text, is_pdf logger = logging.getLogger(__name__) +_pw_lock = asyncio.Lock() + RSEARCH_URL = "https://rsearch.app.molodetz.nl" RSEARCH_TIMEOUT_SECONDS = 45.0 FETCH_TIMEOUT_SECONDS = 20.0 @@ -242,18 +244,20 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None: except (LookupError, ValueError) as exc: logger.info("deepsearch decode failed for %s: %s", url, exc) if len(text) < MIN_PAGE_CHARS: - try: - r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser) - if len(r_text) > len(text): - title, text, status, source, links = ( - r_title or title, - r_text, - r_status or status, - "playwright", - r_links, - ) - except Exception as exc: - logger.info("deepsearch render failed for %s: %s", url, exc) + async with _pw_lock: + try: + r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser) + except Exception as exc: + logger.info("deepsearch render failed for %s: %s", url, exc) + r_title, r_text, r_status, r_links = "", "", 0, [] + if len(r_text) > len(text): + title, text, status, source, links = ( + r_title or title, + r_text, + r_status or status, + "playwright", + r_links, + ) if len(text) < MIN_PAGE_CHARS: return None return CrawledPage( From 1eeb54598f5da9be62d628ded3797c4b3570493d Mon Sep 17 00:00:00 2001 From: Typosaurus Date: Thu, 23 Jul 2026 02:32:33 +0000 Subject: [PATCH 2/2] ticket #104 attempt 1 --- devplacepy/services/jobs/deepsearch/crawl.py | 197 +++++++++---------- 1 file changed, 98 insertions(+), 99 deletions(-) diff --git a/devplacepy/services/jobs/deepsearch/crawl.py b/devplacepy/services/jobs/deepsearch/crawl.py index 227edae5..ca963031 100644 --- a/devplacepy/services/jobs/deepsearch/crawl.py +++ b/devplacepy/services/jobs/deepsearch/crawl.py @@ -174,13 +174,7 @@ async def _render_with_playwright( ) -> tuple[str, str, int, list[tuple[str, str]]]: from playwright.async_api import async_playwright - own_browser = browser is None - if own_browser: - pw = await async_playwright().__aenter__() - browser = await pw.chromium.launch( - headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"] - ) - try: + async def _render(browser) -> tuple[str, str, int, list[tuple[str, str]]]: context = await browser.new_context(user_agent=USER_AGENT) page = await context.new_page() response = await page.goto(url, wait_until="load", timeout=30000) @@ -191,10 +185,18 @@ async def _render_with_playwright( await context.close() extracted = extract_html(content, base_url=url) return extracted.title, extracted.text, status, extracted.links - finally: - if own_browser: - await browser.close() - await pw.__aexit__(None, None, None) + + if browser is None: + async with async_playwright() as pw: + browser = await pw.chromium.launch( + headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"] + ) + try: + return await _render(browser) + finally: + await browser.close() + else: + return await _render(browser) async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None: @@ -300,102 +302,99 @@ async def crawl( total = min(len(level_candidates), max_pages) cancelled = False - pw = None browser = None - try: - pw = await async_playwright().__aenter__() - browser = await pw.chromium.launch( - headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"] - ) - except Exception as exc: - logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc) + async with async_playwright() as pw: + try: + browser = await pw.chromium.launch( + headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"] + ) + except Exception as exc: + logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc) - try: - for level in range(max(1, depth)): - if cancelled or fetched >= max_pages or not level_candidates: - break - next_candidates: list[dict] = [] - for start in range(0, len(level_candidates), CRAWL_CONCURRENCY): - if fetched >= max_pages: + try: + for level in range(max(1, depth)): + if cancelled or fetched >= max_pages or not level_candidates: break - if await should_stop(): - emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"}) - cancelled = True - break - batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched] - for candidate in batch: - emit( - { - "type": "progress", - "done": fetched, - "total": total, - "url": candidate["url"], - "depth": level, - "message": f"Reading {candidate['url']}", - } - ) - if is_cached(candidate["url"]): - emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"}) - fetch_start = time.perf_counter() - results = await asyncio.gather( - *(_resolve_candidate(candidate, level, browser) for candidate in batch), - return_exceptions=True, - ) - elapsed_ms = int((time.perf_counter() - fetch_start) * 1000) - for candidate, page in zip(batch, results): - url = candidate["url"] - if isinstance(page, BaseException): - logger.info("deepsearch fetch crashed for %s: %s", url, page) - page = None - if page is None: - emit( - { - "type": "page_skipped", - "url": url, - "reason": "no readable content", - "elapsed_ms": elapsed_ms, - } - ) - continue + next_candidates: list[dict] = [] + for start in range(0, len(level_candidates), CRAWL_CONCURRENCY): if fetched >= max_pages: break - digest = content_hash(page.text) - if digest in outcome.seen_hashes: + if await should_stop(): + emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"}) + cancelled = True + break + batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched] + for candidate in batch: emit( { - "type": "page_duplicate", - "url": url, - "reason": "duplicate content", - "elapsed_ms": elapsed_ms, + "type": "progress", + "done": fetched, + "total": total, + "url": candidate["url"], + "depth": level, + "message": f"Reading {candidate['url']}", } ) - continue - outcome.seen_hashes.add(digest) - outcome.pages.append(page) - fetched += 1 - emit( - { - "type": "page_loaded", - "url": page.url, - "title": page.title, - "source": page.source, - "depth": level, - "render": page.source == "playwright", - "elapsed_ms": elapsed_ms, - "done": fetched, - "total": total, - } + if is_cached(candidate["url"]): + emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"}) + fetch_start = time.perf_counter() + results = await asyncio.gather( + *(_resolve_candidate(candidate, level, browser) for candidate in batch), + return_exceptions=True, ) - if level + 1 < depth: - for link in relevant_links(page.links, query, LINKS_PER_PAGE): - if link not in seen_urls: - seen_urls.add(link) - next_candidates.append({"url": link}) - level_candidates = next_candidates - total = min(total + len(next_candidates), max_pages) - finally: - if browser is not None: - await browser.close() - if pw is not None: - await pw.__aexit__(None, None, None) + elapsed_ms = int((time.perf_counter() - fetch_start) * 1000) + for candidate, page in zip(batch, results): + url = candidate["url"] + if isinstance(page, BaseException): + logger.info("deepsearch fetch crashed for %s: %s", url, page) + page = None + if page is None: + emit( + { + "type": "page_skipped", + "url": url, + "reason": "no readable content", + "elapsed_ms": elapsed_ms, + } + ) + continue + if fetched >= max_pages: + break + digest = content_hash(page.text) + if digest in outcome.seen_hashes: + emit( + { + "type": "page_duplicate", + "url": url, + "reason": "duplicate content", + "elapsed_ms": elapsed_ms, + } + ) + continue + outcome.seen_hashes.add(digest) + outcome.pages.append(page) + fetched += 1 + emit( + { + "type": "page_loaded", + "url": page.url, + "title": page.title, + "source": page.source, + "depth": level, + "render": page.source == "playwright", + "elapsed_ms": elapsed_ms, + "done": fetched, + "total": total, + } + ) + if level + 1 < depth: + for link in relevant_links(page.links, query, LINKS_PER_PAGE): + if link not in seen_urls: + seen_urls.add(link) + next_candidates.append({"url": link}) + level_candidates = next_candidates + total = min(total + len(next_candidates), max_pages) + finally: + if browser is not None: + await browser.close() return outcome