Merge pull request 'Fix #104: DeepSearch fails with Playwright async context manager error ('__aexit__' missing)' (#124) from typosaurus/ticket-104 into master

Browse Source 
Reviewed-on: #124
This commit is contained in:
typosaurus 2026-07-26 00:36:04 +02:00
commit 2620ecc0f1
1 changed files with 114 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

225
devplacepy/services/jobs/deepsearch/crawl.py
View File

@ -22,6 +22,8 @@ from .pdf import MAX_PDF_BYTES, extract_pdf_text, is_pdf
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_pw_lock = asyncio.Lock()
RSEARCH_URL = "https://rsearch.app.molodetz.nl" RSEARCH_URL = "https://rsearch.app.molodetz.nl"
RSEARCH_TIMEOUT_SECONDS = 45.0 RSEARCH_TIMEOUT_SECONDS = 45.0
FETCH_TIMEOUT_SECONDS = 20.0 FETCH_TIMEOUT_SECONDS = 20.0
@ -172,13 +174,7 @@ async def _render_with_playwright(
) -> tuple[str, str, int, list[tuple[str, str]]]: ) -> tuple[str, str, int, list[tuple[str, str]]]:
from playwright.async_api import async_playwright from playwright.async_api import async_playwright
own_browser = browser is None async def _render(browser) -> tuple[str, str, int, list[tuple[str, str]]]:
if own_browser:
pw = await async_playwright().__aenter__()
browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
)
try:
context = await browser.new_context(user_agent=USER_AGENT) context = await browser.new_context(user_agent=USER_AGENT)
page = await context.new_page() page = await context.new_page()
response = await page.goto(url, wait_until="load", timeout=30000) response = await page.goto(url, wait_until="load", timeout=30000)
@ -189,10 +185,18 @@ async def _render_with_playwright(
await context.close() await context.close()
extracted = extract_html(content, base_url=url) extracted = extract_html(content, base_url=url)
return extracted.title, extracted.text, status, extracted.links return extracted.title, extracted.text, status, extracted.links
finally:
if own_browser: if browser is None:
await browser.close() async with async_playwright() as pw:
await pw.__aexit__(None, None, None) browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
)
try:
return await _render(browser)
finally:
await browser.close()
else:
return await _render(browser)
async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None: async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
@ -242,18 +246,20 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
except (LookupError, ValueError) as exc: except (LookupError, ValueError) as exc:
logger.info("deepsearch decode failed for %s: %s", url, exc) logger.info("deepsearch decode failed for %s: %s", url, exc)
if len(text) < MIN_PAGE_CHARS: if len(text) < MIN_PAGE_CHARS:
try: async with _pw_lock:
r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser) try:
if len(r_text) > len(text): r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
title, text, status, source, links = ( except Exception as exc:
r_title or title, logger.info("deepsearch render failed for %s: %s", url, exc)
r_text, r_title, r_text, r_status, r_links = "", "", 0, []
r_status or status, if len(r_text) > len(text):
"playwright", title, text, status, source, links = (
r_links, r_title or title,
) r_text,
except Exception as exc: r_status or status,
logger.info("deepsearch render failed for %s: %s", url, exc) "playwright",
r_links,
)
if len(text) < MIN_PAGE_CHARS: if len(text) < MIN_PAGE_CHARS:
return None return None
return CrawledPage( return CrawledPage(
@ -296,102 +302,99 @@ async def crawl(
total = min(len(level_candidates), max_pages) total = min(len(level_candidates), max_pages)
cancelled = False cancelled = False
pw = None
browser = None browser = None
try: async with async_playwright() as pw:
pw = await async_playwright().__aenter__() try:
browser = await pw.chromium.launch( browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"] headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
) )
except Exception as exc: except Exception as exc:
logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc) logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc)
try: try:
for level in range(max(1, depth)): for level in range(max(1, depth)):
if cancelled or fetched >= max_pages or not level_candidates: if cancelled or fetched >= max_pages or not level_candidates:
break
next_candidates: list[dict] = []
for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
if fetched >= max_pages:
break break
if await should_stop(): next_candidates: list[dict] = []
emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"}) for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
cancelled = True
break
batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
for candidate in batch:
emit(
{
"type": "progress",
"done": fetched,
"total": total,
"url": candidate["url"],
"depth": level,
"message": f"Reading {candidate['url']}",
}
)
if is_cached(candidate["url"]):
emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
fetch_start = time.perf_counter()
results = await asyncio.gather(
*(_resolve_candidate(candidate, level, browser) for candidate in batch),
return_exceptions=True,
)
elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
for candidate, page in zip(batch, results):
url = candidate["url"]
if isinstance(page, BaseException):
logger.info("deepsearch fetch crashed for %s: %s", url, page)
page = None
if page is None:
emit(
{
"type": "page_skipped",
"url": url,
"reason": "no readable content",
"elapsed_ms": elapsed_ms,
}
)
continue
if fetched >= max_pages: if fetched >= max_pages:
break break
digest = content_hash(page.text) if await should_stop():
if digest in outcome.seen_hashes: emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
cancelled = True
break
batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
for candidate in batch:
emit( emit(
{ {
"type": "page_duplicate", "type": "progress",
"url": url, "done": fetched,
"reason": "duplicate content", "total": total,
"elapsed_ms": elapsed_ms, "url": candidate["url"],
"depth": level,
"message": f"Reading {candidate['url']}",
} }
) )
continue if is_cached(candidate["url"]):
outcome.seen_hashes.add(digest) emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
outcome.pages.append(page) fetch_start = time.perf_counter()
fetched += 1 results = await asyncio.gather(
emit( *(_resolve_candidate(candidate, level, browser) for candidate in batch),
{ return_exceptions=True,
"type": "page_loaded",
"url": page.url,
"title": page.title,
"source": page.source,
"depth": level,
"render": page.source == "playwright",
"elapsed_ms": elapsed_ms,
"done": fetched,
"total": total,
}
) )
if level + 1 < depth: elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
for link in relevant_links(page.links, query, LINKS_PER_PAGE): for candidate, page in zip(batch, results):
if link not in seen_urls: url = candidate["url"]
seen_urls.add(link) if isinstance(page, BaseException):
next_candidates.append({"url": link}) logger.info("deepsearch fetch crashed for %s: %s", url, page)
level_candidates = next_candidates page = None
total = min(total + len(next_candidates), max_pages) if page is None:
finally: emit(
if browser is not None: {
await browser.close() "type": "page_skipped",
if pw is not None: "url": url,
await pw.__aexit__(None, None, None) "reason": "no readable content",
"elapsed_ms": elapsed_ms,
}
)
continue
if fetched >= max_pages:
break
digest = content_hash(page.text)
if digest in outcome.seen_hashes:
emit(
{
"type": "page_duplicate",
"url": url,
"reason": "duplicate content",
"elapsed_ms": elapsed_ms,
}
)
continue
outcome.seen_hashes.add(digest)
outcome.pages.append(page)
fetched += 1
emit(
{
"type": "page_loaded",
"url": page.url,
"title": page.title,
"source": page.source,
"depth": level,
"render": page.source == "playwright",
"elapsed_ms": elapsed_ms,
"done": fetched,
"total": total,
}
)
if level + 1 < depth:
for link in relevant_links(page.links, query, LINKS_PER_PAGE):
if link not in seen_urls:
seen_urls.add(link)
next_candidates.append({"url": link})
level_candidates = next_candidates
total = min(total + len(next_candidates), max_pages)
finally:
if browser is not None:
await browser.close()
return outcome return outcome