ticket #104 attempt 1
Some checks failed
DevPlace CI / test (pull_request) Failing after 11s
Details

Browse Source
This commit is contained in:
Typosaurus 2026-07-23 02:32:33 +00:00
parent a0d573375a
commit 1eeb54598f
1 changed files with 98 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

197
devplacepy/services/jobs/deepsearch/crawl.py
View File

@ -174,13 +174,7 @@ async def _render_with_playwright(
) -> tuple[str, str, int, list[tuple[str, str]]]:
from playwright.async_api import async_playwright
own_browser = browser is None
if own_browser:
pw = await async_playwright().__aenter__()
browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
)
try:
async def _render(browser) -> tuple[str, str, int, list[tuple[str, str]]]:
context = await browser.new_context(user_agent=USER_AGENT)
page = await context.new_page()
response = await page.goto(url, wait_until="load", timeout=30000)
@ -191,10 +185,18 @@ async def _render_with_playwright(
await context.close()
extracted = extract_html(content, base_url=url)
return extracted.title, extracted.text, status, extracted.links
finally:
if own_browser:
await browser.close()
await pw.__aexit__(None, None, None)
if browser is None:
async with async_playwright() as pw:
browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
)
try:
return await _render(browser)
finally:
await browser.close()
else:
return await _render(browser)
async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
@ -300,102 +302,99 @@ async def crawl(
total = min(len(level_candidates), max_pages)
cancelled = False
pw = None
browser = None
try:
pw = await async_playwright().__aenter__()
browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
)
except Exception as exc:
logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc)
async with async_playwright() as pw:
try:
browser = await pw.chromium.launch(
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
)
except Exception as exc:
logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc)
try:
for level in range(max(1, depth)):
if cancelled or fetched >= max_pages or not level_candidates:
break
next_candidates: list[dict] = []
for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
if fetched >= max_pages:
try:
for level in range(max(1, depth)):
if cancelled or fetched >= max_pages or not level_candidates:
break
if await should_stop():
emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
cancelled = True
break
batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
for candidate in batch:
emit(
{
"type": "progress",
"done": fetched,
"total": total,
"url": candidate["url"],
"depth": level,
"message": f"Reading {candidate['url']}",
}
)
if is_cached(candidate["url"]):
emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
fetch_start = time.perf_counter()
results = await asyncio.gather(
*(_resolve_candidate(candidate, level, browser) for candidate in batch),
return_exceptions=True,
)
elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
for candidate, page in zip(batch, results):
url = candidate["url"]
if isinstance(page, BaseException):
logger.info("deepsearch fetch crashed for %s: %s", url, page)
page = None
if page is None:
emit(
{
"type": "page_skipped",
"url": url,
"reason": "no readable content",
"elapsed_ms": elapsed_ms,
}
)
continue
next_candidates: list[dict] = []
for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
if fetched >= max_pages:
break
digest = content_hash(page.text)
if digest in outcome.seen_hashes:
if await should_stop():
emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
cancelled = True
break
batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
for candidate in batch:
emit(
{
"type": "page_duplicate",
"url": url,
"reason": "duplicate content",
"elapsed_ms": elapsed_ms,
"type": "progress",
"done": fetched,
"total": total,
"url": candidate["url"],
"depth": level,
"message": f"Reading {candidate['url']}",
}
)
continue
outcome.seen_hashes.add(digest)
outcome.pages.append(page)
fetched += 1
emit(
{
"type": "page_loaded",
"url": page.url,
"title": page.title,
"source": page.source,
"depth": level,
"render": page.source == "playwright",
"elapsed_ms": elapsed_ms,
"done": fetched,
"total": total,
}
if is_cached(candidate["url"]):
emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
fetch_start = time.perf_counter()
results = await asyncio.gather(
*(_resolve_candidate(candidate, level, browser) for candidate in batch),
return_exceptions=True,
)
if level + 1 < depth:
for link in relevant_links(page.links, query, LINKS_PER_PAGE):
if link not in seen_urls:
seen_urls.add(link)
next_candidates.append({"url": link})
level_candidates = next_candidates
total = min(total + len(next_candidates), max_pages)
finally:
if browser is not None:
await browser.close()
if pw is not None:
await pw.__aexit__(None, None, None)
elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
for candidate, page in zip(batch, results):
url = candidate["url"]
if isinstance(page, BaseException):
logger.info("deepsearch fetch crashed for %s: %s", url, page)
page = None
if page is None:
emit(
{
"type": "page_skipped",
"url": url,
"reason": "no readable content",
"elapsed_ms": elapsed_ms,
}
)
continue
if fetched >= max_pages:
break
digest = content_hash(page.text)
if digest in outcome.seen_hashes:
emit(
{
"type": "page_duplicate",
"url": url,
"reason": "duplicate content",
"elapsed_ms": elapsed_ms,
}
)
continue
outcome.seen_hashes.add(digest)
outcome.pages.append(page)
fetched += 1
emit(
{
"type": "page_loaded",
"url": page.url,
"title": page.title,
"source": page.source,
"depth": level,
"render": page.source == "playwright",
"elapsed_ms": elapsed_ms,
"done": fetched,
"total": total,
}
)
if level + 1 < depth:
for link in relevant_links(page.links, query, LINKS_PER_PAGE):
if link not in seen_urls:
seen_urls.add(link)
next_candidates.append({"url": link})
level_candidates = next_candidates
total = min(total + len(next_candidates), max_pages)
finally:
if browser is not None:
await browser.close()
return outcome