Keep DeepSearch crawling when the Playwright driver cannot start

The degradation guard in crawl() wrapped only chromium.launch(), while
the driver start sat outside it in the async context manager. A failure
to start the driver therefore propagated out of crawl() and failed the
whole DeepSearch job, contradicting the warning it logs on that path
("pages will use httpx only").

Start the driver inside the guarded block and stop it in the finally, so
an unavailable driver degrades to httpx-only fetching as intended. The
crawl loop body is unchanged apart from indentation.
This commit is contained in:
retoor 2026-07-26 19:05:21 +02:00
parent 3467f55df9
commit 535e9c5dc1
2 changed files with 113 additions and 85 deletions

View File

@ -302,9 +302,10 @@ async def crawl(
total = min(len(level_candidates), max_pages) total = min(len(level_candidates), max_pages)
cancelled = False cancelled = False
pw = None
browser = None browser = None
async with async_playwright() as pw:
try: try:
pw = await async_playwright().start()
browser = await pw.chromium.launch( browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"] headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
) )
@ -397,4 +398,6 @@ async def crawl(
finally: finally:
if browser is not None: if browser is not None:
await browser.close() await browser.close()
if pw is not None:
await pw.stop()
return outcome return outcome

View File

@ -110,3 +110,28 @@ def test_crawl_falls_back_to_snippet_when_fetch_empty(monkeypatch):
) )
assert len(outcome.pages) == 1 assert len(outcome.pages) == 1
assert outcome.pages[0].source == "search" assert outcome.pages[0].source == "search"
def test_crawl_degrades_to_httpx_when_the_playwright_driver_cannot_start(monkeypatch):
import playwright.async_api as playwright_api
class FailingDriver:
async def start(self):
raise RuntimeError("driver missing")
monkeypatch.setattr(playwright_api, "async_playwright", lambda: FailingDriver())
async def fake_fetch(url, depth, browser=None):
return CrawledPage(
url=url, title="T", text="body " * 100, source="httpx", status=200, depth=depth
)
monkeypatch.setattr(crawl_module, "fetch_page", fake_fetch)
candidates = [
{"url": "https://example.com/a", "title": "A", "description": "", "content": ""}
]
outcome = run_async(
crawl(candidates, 6, lambda f: None, lambda u: False, _no_stop, query="q", depth=1)
)
assert len(outcome.pages) == 1
assert outcome.pages[0].source == "httpx"