Keep DeepSearch crawling when the Playwright driver cannot start

Browse Source 
The degradation guard in crawl() wrapped only chromium.launch(), while
the driver start sat outside it in the async context manager. A failure
to start the driver therefore propagated out of crawl() and failed the
whole DeepSearch job, contradicting the warning it logs on that path
("pages will use httpx only").

Start the driver inside the guarded block and stop it in the finally, so
an unavailable driver degrades to httpx-only fetching as intended. The
crawl loop body is unchanged apart from indentation.
This commit is contained in:
retoor 2026-07-26 19:05:21 +02:00
parent 3467f55df9
commit 535e9c5dc1
2 changed files with 113 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

173
devplacepy/services/jobs/deepsearch/crawl.py
View File

@ -302,99 +302,102 @@ async def crawl(
total = min(len(level_candidates), max_pages) total = min(len(level_candidates), max_pages)
cancelled = False cancelled = False
pw = None
browser = None browser = None
async with async_playwright() as pw: try:
try: pw = await async_playwright().start()
browser = await pw.chromium.launch( browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"] headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
) )
except Exception as exc: except Exception as exc:
logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc) logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc)
try: try:
for level in range(max(1, depth)): for level in range(max(1, depth)):
if cancelled or fetched >= max_pages or not level_candidates: if cancelled or fetched >= max_pages or not level_candidates:
break
next_candidates: list[dict] = []
for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
if fetched >= max_pages:
break break
next_candidates: list[dict] = [] if await should_stop():
for start in range(0, len(level_candidates), CRAWL_CONCURRENCY): emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
cancelled = True
break
batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
for candidate in batch:
emit(
{
"type": "progress",
"done": fetched,
"total": total,
"url": candidate["url"],
"depth": level,
"message": f"Reading {candidate['url']}",
}
)
if is_cached(candidate["url"]):
emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
fetch_start = time.perf_counter()
results = await asyncio.gather(
*(_resolve_candidate(candidate, level, browser) for candidate in batch),
return_exceptions=True,
)
elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
for candidate, page in zip(batch, results):
url = candidate["url"]
if isinstance(page, BaseException):
logger.info("deepsearch fetch crashed for %s: %s", url, page)
page = None
if page is None:
emit(
{
"type": "page_skipped",
"url": url,
"reason": "no readable content",
"elapsed_ms": elapsed_ms,
}
)
continue
if fetched >= max_pages: if fetched >= max_pages:
break break
if await should_stop(): digest = content_hash(page.text)
emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"}) if digest in outcome.seen_hashes:
cancelled = True
break
batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
for candidate in batch:
emit( emit(
{ {
"type": "progress", "type": "page_duplicate",
"done": fetched, "url": url,
"total": total, "reason": "duplicate content",
"url": candidate["url"],
"depth": level,
"message": f"Reading {candidate['url']}",
}
)
if is_cached(candidate["url"]):
emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
fetch_start = time.perf_counter()
results = await asyncio.gather(
*(_resolve_candidate(candidate, level, browser) for candidate in batch),
return_exceptions=True,
)
elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
for candidate, page in zip(batch, results):
url = candidate["url"]
if isinstance(page, BaseException):
logger.info("deepsearch fetch crashed for %s: %s", url, page)
page = None
if page is None:
emit(
{
"type": "page_skipped",
"url": url,
"reason": "no readable content",
"elapsed_ms": elapsed_ms,
}
)
continue
if fetched >= max_pages:
break
digest = content_hash(page.text)
if digest in outcome.seen_hashes:
emit(
{
"type": "page_duplicate",
"url": url,
"reason": "duplicate content",
"elapsed_ms": elapsed_ms,
}
)
continue
outcome.seen_hashes.add(digest)
outcome.pages.append(page)
fetched += 1
emit(
{
"type": "page_loaded",
"url": page.url,
"title": page.title,
"source": page.source,
"depth": level,
"render": page.source == "playwright",
"elapsed_ms": elapsed_ms, "elapsed_ms": elapsed_ms,
"done": fetched,
"total": total,
} }
) )
if level + 1 < depth: continue
for link in relevant_links(page.links, query, LINKS_PER_PAGE): outcome.seen_hashes.add(digest)
if link not in seen_urls: outcome.pages.append(page)
seen_urls.add(link) fetched += 1
next_candidates.append({"url": link}) emit(
level_candidates = next_candidates {
total = min(total + len(next_candidates), max_pages) "type": "page_loaded",
finally: "url": page.url,
if browser is not None: "title": page.title,
await browser.close() "source": page.source,
"depth": level,
"render": page.source == "playwright",
"elapsed_ms": elapsed_ms,
"done": fetched,
"total": total,
}
)
if level + 1 < depth:
for link in relevant_links(page.links, query, LINKS_PER_PAGE):
if link not in seen_urls:
seen_urls.add(link)
next_candidates.append({"url": link})
level_candidates = next_candidates
total = min(total + len(next_candidates), max_pages)
finally:
if browser is not None:
await browser.close()
if pw is not None:
await pw.stop()
return outcome return outcome

25
tests/unit/services/jobs/deepsearch/crawl.py
View File

@ -110,3 +110,28 @@ def test_crawl_falls_back_to_snippet_when_fetch_empty(monkeypatch):
) )
assert len(outcome.pages) == 1 assert len(outcome.pages) == 1
assert outcome.pages[0].source == "search" assert outcome.pages[0].source == "search"
def test_crawl_degrades_to_httpx_when_the_playwright_driver_cannot_start(monkeypatch):
import playwright.async_api as playwright_api
class FailingDriver:
async def start(self):
raise RuntimeError("driver missing")
monkeypatch.setattr(playwright_api, "async_playwright", lambda: FailingDriver())
async def fake_fetch(url, depth, browser=None):
return CrawledPage(
url=url, title="T", text="body " * 100, source="httpx", status=200, depth=depth
)
monkeypatch.setattr(crawl_module, "fetch_page", fake_fetch)
candidates = [
{"url": "https://example.com/a", "title": "A", "description": "", "content": ""}
]
outcome = run_async(
crawl(candidates, 6, lambda f: None, lambda u: False, _no_stop, query="q", depth=1)
)
assert len(outcome.pages) == 1
assert outcome.pages[0].source == "httpx"