Keep DeepSearch crawling when the Playwright driver cannot start
The degradation guard in crawl() wrapped only chromium.launch(), while
the driver start sat outside it in the async context manager. A failure
to start the driver therefore propagated out of crawl() and failed the
whole DeepSearch job, contradicting the warning it logs on that path
("pages will use httpx only").
Start the driver inside the guarded block and stop it in the finally, so
an unavailable driver degrades to httpx-only fetching as intended. The
crawl loop body is unchanged apart from indentation.
This commit is contained in:
parent
3467f55df9
commit
535e9c5dc1
@ -302,9 +302,10 @@ async def crawl(
|
|||||||
total = min(len(level_candidates), max_pages)
|
total = min(len(level_candidates), max_pages)
|
||||||
cancelled = False
|
cancelled = False
|
||||||
|
|
||||||
|
pw = None
|
||||||
browser = None
|
browser = None
|
||||||
async with async_playwright() as pw:
|
|
||||||
try:
|
try:
|
||||||
|
pw = await async_playwright().start()
|
||||||
browser = await pw.chromium.launch(
|
browser = await pw.chromium.launch(
|
||||||
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
|
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
|
||||||
)
|
)
|
||||||
@ -397,4 +398,6 @@ async def crawl(
|
|||||||
finally:
|
finally:
|
||||||
if browser is not None:
|
if browser is not None:
|
||||||
await browser.close()
|
await browser.close()
|
||||||
|
if pw is not None:
|
||||||
|
await pw.stop()
|
||||||
return outcome
|
return outcome
|
||||||
|
|||||||
@ -110,3 +110,28 @@ def test_crawl_falls_back_to_snippet_when_fetch_empty(monkeypatch):
|
|||||||
)
|
)
|
||||||
assert len(outcome.pages) == 1
|
assert len(outcome.pages) == 1
|
||||||
assert outcome.pages[0].source == "search"
|
assert outcome.pages[0].source == "search"
|
||||||
|
|
||||||
|
|
||||||
|
def test_crawl_degrades_to_httpx_when_the_playwright_driver_cannot_start(monkeypatch):
|
||||||
|
import playwright.async_api as playwright_api
|
||||||
|
|
||||||
|
class FailingDriver:
|
||||||
|
async def start(self):
|
||||||
|
raise RuntimeError("driver missing")
|
||||||
|
|
||||||
|
monkeypatch.setattr(playwright_api, "async_playwright", lambda: FailingDriver())
|
||||||
|
|
||||||
|
async def fake_fetch(url, depth, browser=None):
|
||||||
|
return CrawledPage(
|
||||||
|
url=url, title="T", text="body " * 100, source="httpx", status=200, depth=depth
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(crawl_module, "fetch_page", fake_fetch)
|
||||||
|
candidates = [
|
||||||
|
{"url": "https://example.com/a", "title": "A", "description": "", "content": ""}
|
||||||
|
]
|
||||||
|
outcome = run_async(
|
||||||
|
crawl(candidates, 6, lambda f: None, lambda u: False, _no_stop, query="q", depth=1)
|
||||||
|
)
|
||||||
|
assert len(outcome.pages) == 1
|
||||||
|
assert outcome.pages[0].source == "httpx"
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user