Merge pull request 'Fix #104: DeepSearch fails with Playwright async context manager error ('__aexit__' missing)' (#124) from typosaurus/ticket-104 into master

Browse Source 
Reviewed-on: #124
This commit is contained in:
typosaurus 2026-07-26 00:36:04 +02:00
commit 2620ecc0f1
1 changed files with 114 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

225
devplacepy/services/jobs/deepsearch/crawl.py
View File

@ -22,6 +22,8 @@ from .pdf import MAX_PDF_BYTES, extract_pdf_text, is_pdf
logger = logging.getLogger(__name__)
_pw_lock = asyncio.Lock()
RSEARCH_URL = "https://rsearch.app.molodetz.nl"
RSEARCH_TIMEOUT_SECONDS = 45.0
FETCH_TIMEOUT_SECONDS = 20.0
@ -172,13 +174,7 @@ async def _render_with_playwright(
) -> tuple[str, str, int, list[tuple[str, str]]]:
from playwright.async_api import async_playwright
own_browser = browser is None
if own_browser:
pw = await async_playwright().__aenter__()
browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
)
try:
async def _render(browser) -> tuple[str, str, int, list[tuple[str, str]]]:
context = await browser.new_context(user_agent=USER_AGENT)
page = await context.new_page()
response = await page.goto(url, wait_until="load", timeout=30000)
@ -189,10 +185,18 @@ async def _render_with_playwright(
await context.close()
extracted = extract_html(content, base_url=url)
return extracted.title, extracted.text, status, extracted.links
finally:
if own_browser:
await browser.close()
await pw.__aexit__(None, None, None)
if browser is None:
async with async_playwright() as pw:
browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
)
try:
return await _render(browser)
finally:
await browser.close()
else:
return await _render(browser)
async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
@ -242,18 +246,20 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
except (LookupError, ValueError) as exc:
logger.info("deepsearch decode failed for %s: %s", url, exc)
if len(text) < MIN_PAGE_CHARS:
try:
r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
if len(r_text) > len(text):
title, text, status, source, links = (
r_title or title,
r_text,
r_status or status,
"playwright",
r_links,
)
except Exception as exc:
logger.info("deepsearch render failed for %s: %s", url, exc)
async with _pw_lock:
try:
r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
except Exception as exc:
logger.info("deepsearch render failed for %s: %s", url, exc)
r_title, r_text, r_status, r_links = "", "", 0, []
if len(r_text) > len(text):
title, text, status, source, links = (
r_title or title,
r_text,
r_status or status,
"playwright",
r_links,
)
if len(text) < MIN_PAGE_CHARS:
return None
return CrawledPage(
@ -296,102 +302,99 @@ async def crawl(
total = min(len(level_candidates), max_pages)
cancelled = False
pw = None
browser = None
try:
pw = await async_playwright().__aenter__()
browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
)
except Exception as exc:
logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc)
async with async_playwright() as pw:
try:
browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
)
except Exception as exc:
logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc)
try:
for level in range(max(1, depth)):
if cancelled or fetched >= max_pages or not level_candidates:
break
next_candidates: list[dict] = []
for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
if fetched >= max_pages:
try:
for level in range(max(1, depth)):
if cancelled or fetched >= max_pages or not level_candidates:
break
if await should_stop():
emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
cancelled = True
break
batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
for candidate in batch:
emit(
{
"type": "progress",
"done": fetched,
"total": total,
"url": candidate["url"],
"depth": level,
"message": f"Reading {candidate['url']}",
}
)
if is_cached(candidate["url"]):
emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
fetch_start = time.perf_counter()
results = await asyncio.gather(
*(_resolve_candidate(candidate, level, browser) for candidate in batch),
return_exceptions=True,
)
elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
for candidate, page in zip(batch, results):
url = candidate["url"]
if isinstance(page, BaseException):
logger.info("deepsearch fetch crashed for %s: %s", url, page)
page = None
if page is None:
emit(
{
"type": "page_skipped",
"url": url,
"reason": "no readable content",
"elapsed_ms": elapsed_ms,
}
)
continue
next_candidates: list[dict] = []
for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
if fetched >= max_pages:
break
digest = content_hash(page.text)
if digest in outcome.seen_hashes:
if await should_stop():
emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
cancelled = True
break
batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
for candidate in batch:
emit(
{
"type": "page_duplicate",
"url": url,
"reason": "duplicate content",
"elapsed_ms": elapsed_ms,
"type": "progress",
"done": fetched,
"total": total,
"url": candidate["url"],
"depth": level,
"message": f"Reading {candidate['url']}",
}
)
continue
outcome.seen_hashes.add(digest)
outcome.pages.append(page)
fetched += 1
emit(
{
"type": "page_loaded",
"url": page.url,
"title": page.title,
"source": page.source,
"depth": level,
"render": page.source == "playwright",
"elapsed_ms": elapsed_ms,
"done": fetched,
"total": total,
}
if is_cached(candidate["url"]):
emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
fetch_start = time.perf_counter()
results = await asyncio.gather(
*(_resolve_candidate(candidate, level, browser) for candidate in batch),
return_exceptions=True,
)
if level + 1 < depth:
for link in relevant_links(page.links, query, LINKS_PER_PAGE):
if link not in seen_urls:
seen_urls.add(link)
next_candidates.append({"url": link})
level_candidates = next_candidates
total = min(total + len(next_candidates), max_pages)
finally:
if browser is not None:
await browser.close()
if pw is not None:
await pw.__aexit__(None, None, None)
elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
for candidate, page in zip(batch, results):
url = candidate["url"]
if isinstance(page, BaseException):
logger.info("deepsearch fetch crashed for %s: %s", url, page)
page = None
if page is None:
emit(
{
"type": "page_skipped",
"url": url,
"reason": "no readable content",
"elapsed_ms": elapsed_ms,
}
)
continue
if fetched >= max_pages:
break
digest = content_hash(page.text)
if digest in outcome.seen_hashes:
emit(
{
"type": "page_duplicate",
"url": url,
"reason": "duplicate content",
"elapsed_ms": elapsed_ms,
}
)
continue
outcome.seen_hashes.add(digest)
outcome.pages.append(page)
fetched += 1
emit(
{
"type": "page_loaded",
"url": page.url,
"title": page.title,
"source": page.source,
"depth": level,
"render": page.source == "playwright",
"elapsed_ms": elapsed_ms,
"done": fetched,
"total": total,
}
)
if level + 1 < depth:
for link in relevant_links(page.links, query, LINKS_PER_PAGE):
if link not in seen_urls:
seen_urls.add(link)
next_candidates.append({"url": link})
level_candidates = next_candidates
total = min(total + len(next_candidates), max_pages)
finally:
if browser is not None:
await browser.close()
return outcome