ticket #104 attempt 1

Browse Source
This commit is contained in:
Typosaurus 2026-07-23 02:32:33 +00:00
parent a0d573375a
commit 1eeb54598f
1 changed files with 98 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
devplacepy/services/jobs/deepsearch/crawl.py
View File

@ -174,13 +174,7 @@ async def _render_with_playwright(
) -> tuple[str, str, int, list[tuple[str, str]]]:
from playwright.async_api import async_playwright
own_browser = browser is None
if own_browser:
pw = await async_playwright().__aenter__()
browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
)
try:
async def _render(browser) -> tuple[str, str, int, list[tuple[str, str]]]:
context = await browser.new_context(user_agent=USER_AGENT)
page = await context.new_page()
response = await page.goto(url, wait_until="load", timeout=30000)
@ -191,10 +185,18 @@ async def _render_with_playwright(
await context.close()
extracted = extract_html(content, base_url=url)
return extracted.title, extracted.text, status, extracted.links
if browser is None:
async with async_playwright() as pw:
browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
)
try:
return await _render(browser)
finally:
if own_browser:
await browser.close()
await pw.__aexit__(None, None, None)
else:
return await _render(browser)
async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
@ -300,10 +302,9 @@ async def crawl(
total = min(len(level_candidates), max_pages)
cancelled = False
pw = None
browser = None
async with async_playwright() as pw:
try:
pw = await async_playwright().__aenter__()
browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
)
@ -396,6 +397,4 @@ async def crawl(
finally:
if browser is not None:
await browser.close()
if pw is not None:
await pw.__aexit__(None, None, None)
return outcome