forked from retoor/devplacepy
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Typosaurus
|
1eeb54598f | ||
|
Typosaurus
|
a0d573375a |
@@ -44,7 +44,6 @@ from devplacepy.templating import templates, jinja_unread_count
|
||||
from devplacepy.cache import TTLCache
|
||||
from devplacepy.responses import respond, wants_json, json_error
|
||||
from devplacepy.schemas import LandingOut, ValidationErrorOut
|
||||
from devplacepy.attachments import get_attachments_batch
|
||||
from fastapi.responses import JSONResponse
|
||||
from devplacepy.utils import get_current_user, time_ago, safe_next, client_ip
|
||||
from devplacepy.seo import base_seo_context, site_url, website_schema
|
||||
@@ -672,7 +671,6 @@ def _landing_recent_posts(blocked):
|
||||
authors = get_users_by_uids(author_uids)
|
||||
comment_counts = get_comment_counts_by_post_uids(post_uids)
|
||||
upvotes, downvotes = get_vote_counts(post_uids)
|
||||
attachments_map = get_attachments_batch("post", post_uids)
|
||||
for p in raw_posts:
|
||||
posts.append(
|
||||
{
|
||||
@@ -682,7 +680,6 @@ def _landing_recent_posts(blocked):
|
||||
"comment_count": comment_counts.get(p["uid"], 0),
|
||||
"stars": upvotes.get(p["uid"], 0) - downvotes.get(p["uid"], 0),
|
||||
"slug": p.get("slug", "") or p["uid"],
|
||||
"attachments": attachments_map.get(p["uid"], []),
|
||||
}
|
||||
)
|
||||
if not blocked:
|
||||
|
||||
@@ -7,6 +7,7 @@ from devplacepy.models import ProfileForm
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
from devplacepy.database import (
|
||||
get_table,
|
||||
db,
|
||||
get_customization_prefs,
|
||||
get_notification_prefs,
|
||||
get_user_stars,
|
||||
@@ -47,7 +48,6 @@ from devplacepy.utils import (
|
||||
)
|
||||
from devplacepy.responses import respond, action_result, wants_json
|
||||
from devplacepy.schemas import ProfileOut
|
||||
from devplacepy.attachments import get_attachments_batch
|
||||
from devplacepy.avatar import avatar_url, avatar_seed
|
||||
from devplacepy.seo import (
|
||||
base_seo_context,
|
||||
@@ -188,10 +188,8 @@ async def profile_page(
|
||||
else set()
|
||||
)
|
||||
polls_map = get_polls_by_post_uids(post_uids, current_user)
|
||||
attachments_map = get_attachments_batch("post", post_uids)
|
||||
for item in posts:
|
||||
uid = item["post"]["uid"]
|
||||
item["attachments"] = attachments_map.get(uid, [])
|
||||
item["reactions"] = reactions_map.get(uid, {"counts": {}, "mine": []})
|
||||
item["bookmarked"] = uid in bookmark_set
|
||||
item["poll"] = polls_map.get(uid)
|
||||
|
||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
from typing import Any, Optional
|
||||
|
||||
from devplacepy.schemas.base import _Out
|
||||
from devplacepy.schemas.content import AttachmentOut, UserOut
|
||||
from devplacepy.schemas.content import UserOut
|
||||
|
||||
|
||||
class AuthPageOut(_Out):
|
||||
@@ -38,7 +38,6 @@ class LandingPostOut(_Out):
|
||||
comment_count: int = 0
|
||||
stars: int = 0
|
||||
slug: str = ""
|
||||
attachments: list[AttachmentOut] = []
|
||||
|
||||
|
||||
class TrendingTopicOut(_Out):
|
||||
|
||||
@@ -22,6 +22,8 @@ from .pdf import MAX_PDF_BYTES, extract_pdf_text, is_pdf
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_pw_lock = asyncio.Lock()
|
||||
|
||||
RSEARCH_URL = "https://rsearch.app.molodetz.nl"
|
||||
RSEARCH_TIMEOUT_SECONDS = 45.0
|
||||
FETCH_TIMEOUT_SECONDS = 20.0
|
||||
@@ -172,13 +174,7 @@ async def _render_with_playwright(
|
||||
) -> tuple[str, str, int, list[tuple[str, str]]]:
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
own_browser = browser is None
|
||||
if own_browser:
|
||||
pw = await async_playwright().__aenter__()
|
||||
browser = await pw.chromium.launch(
|
||||
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
|
||||
)
|
||||
try:
|
||||
async def _render(browser) -> tuple[str, str, int, list[tuple[str, str]]]:
|
||||
context = await browser.new_context(user_agent=USER_AGENT)
|
||||
page = await context.new_page()
|
||||
response = await page.goto(url, wait_until="load", timeout=30000)
|
||||
@@ -189,10 +185,18 @@ async def _render_with_playwright(
|
||||
await context.close()
|
||||
extracted = extract_html(content, base_url=url)
|
||||
return extracted.title, extracted.text, status, extracted.links
|
||||
finally:
|
||||
if own_browser:
|
||||
await browser.close()
|
||||
await pw.__aexit__(None, None, None)
|
||||
|
||||
if browser is None:
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch(
|
||||
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
|
||||
)
|
||||
try:
|
||||
return await _render(browser)
|
||||
finally:
|
||||
await browser.close()
|
||||
else:
|
||||
return await _render(browser)
|
||||
|
||||
|
||||
async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
|
||||
@@ -242,18 +246,20 @@ async def fetch_page(url: str, depth: int, browser=None) -> CrawledPage | None:
|
||||
except (LookupError, ValueError) as exc:
|
||||
logger.info("deepsearch decode failed for %s: %s", url, exc)
|
||||
if len(text) < MIN_PAGE_CHARS:
|
||||
try:
|
||||
r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
|
||||
if len(r_text) > len(text):
|
||||
title, text, status, source, links = (
|
||||
r_title or title,
|
||||
r_text,
|
||||
r_status or status,
|
||||
"playwright",
|
||||
r_links,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.info("deepsearch render failed for %s: %s", url, exc)
|
||||
async with _pw_lock:
|
||||
try:
|
||||
r_title, r_text, r_status, r_links = await _render_with_playwright(url, browser)
|
||||
except Exception as exc:
|
||||
logger.info("deepsearch render failed for %s: %s", url, exc)
|
||||
r_title, r_text, r_status, r_links = "", "", 0, []
|
||||
if len(r_text) > len(text):
|
||||
title, text, status, source, links = (
|
||||
r_title or title,
|
||||
r_text,
|
||||
r_status or status,
|
||||
"playwright",
|
||||
r_links,
|
||||
)
|
||||
if len(text) < MIN_PAGE_CHARS:
|
||||
return None
|
||||
return CrawledPage(
|
||||
@@ -296,102 +302,99 @@ async def crawl(
|
||||
total = min(len(level_candidates), max_pages)
|
||||
cancelled = False
|
||||
|
||||
pw = None
|
||||
browser = None
|
||||
try:
|
||||
pw = await async_playwright().__aenter__()
|
||||
browser = await pw.chromium.launch(
|
||||
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc)
|
||||
async with async_playwright() as pw:
|
||||
try:
|
||||
browser = await pw.chromium.launch(
|
||||
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("deepsearch playwright launch failed, pages will use httpx only: %s", exc)
|
||||
|
||||
try:
|
||||
for level in range(max(1, depth)):
|
||||
if cancelled or fetched >= max_pages or not level_candidates:
|
||||
break
|
||||
next_candidates: list[dict] = []
|
||||
for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
|
||||
if fetched >= max_pages:
|
||||
try:
|
||||
for level in range(max(1, depth)):
|
||||
if cancelled or fetched >= max_pages or not level_candidates:
|
||||
break
|
||||
if await should_stop():
|
||||
emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
|
||||
cancelled = True
|
||||
break
|
||||
batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
|
||||
for candidate in batch:
|
||||
emit(
|
||||
{
|
||||
"type": "progress",
|
||||
"done": fetched,
|
||||
"total": total,
|
||||
"url": candidate["url"],
|
||||
"depth": level,
|
||||
"message": f"Reading {candidate['url']}",
|
||||
}
|
||||
)
|
||||
if is_cached(candidate["url"]):
|
||||
emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
|
||||
fetch_start = time.perf_counter()
|
||||
results = await asyncio.gather(
|
||||
*(_resolve_candidate(candidate, level, browser) for candidate in batch),
|
||||
return_exceptions=True,
|
||||
)
|
||||
elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
|
||||
for candidate, page in zip(batch, results):
|
||||
url = candidate["url"]
|
||||
if isinstance(page, BaseException):
|
||||
logger.info("deepsearch fetch crashed for %s: %s", url, page)
|
||||
page = None
|
||||
if page is None:
|
||||
emit(
|
||||
{
|
||||
"type": "page_skipped",
|
||||
"url": url,
|
||||
"reason": "no readable content",
|
||||
"elapsed_ms": elapsed_ms,
|
||||
}
|
||||
)
|
||||
continue
|
||||
next_candidates: list[dict] = []
|
||||
for start in range(0, len(level_candidates), CRAWL_CONCURRENCY):
|
||||
if fetched >= max_pages:
|
||||
break
|
||||
digest = content_hash(page.text)
|
||||
if digest in outcome.seen_hashes:
|
||||
if await should_stop():
|
||||
emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
|
||||
cancelled = True
|
||||
break
|
||||
batch = level_candidates[start : start + CRAWL_CONCURRENCY][: max_pages - fetched]
|
||||
for candidate in batch:
|
||||
emit(
|
||||
{
|
||||
"type": "page_duplicate",
|
||||
"url": url,
|
||||
"reason": "duplicate content",
|
||||
"elapsed_ms": elapsed_ms,
|
||||
"type": "progress",
|
||||
"done": fetched,
|
||||
"total": total,
|
||||
"url": candidate["url"],
|
||||
"depth": level,
|
||||
"message": f"Reading {candidate['url']}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
outcome.seen_hashes.add(digest)
|
||||
outcome.pages.append(page)
|
||||
fetched += 1
|
||||
emit(
|
||||
{
|
||||
"type": "page_loaded",
|
||||
"url": page.url,
|
||||
"title": page.title,
|
||||
"source": page.source,
|
||||
"depth": level,
|
||||
"render": page.source == "playwright",
|
||||
"elapsed_ms": elapsed_ms,
|
||||
"done": fetched,
|
||||
"total": total,
|
||||
}
|
||||
if is_cached(candidate["url"]):
|
||||
emit({"type": "page_cached", "url": candidate["url"], "reason": "seen in a prior run"})
|
||||
fetch_start = time.perf_counter()
|
||||
results = await asyncio.gather(
|
||||
*(_resolve_candidate(candidate, level, browser) for candidate in batch),
|
||||
return_exceptions=True,
|
||||
)
|
||||
if level + 1 < depth:
|
||||
for link in relevant_links(page.links, query, LINKS_PER_PAGE):
|
||||
if link not in seen_urls:
|
||||
seen_urls.add(link)
|
||||
next_candidates.append({"url": link})
|
||||
level_candidates = next_candidates
|
||||
total = min(total + len(next_candidates), max_pages)
|
||||
finally:
|
||||
if browser is not None:
|
||||
await browser.close()
|
||||
if pw is not None:
|
||||
await pw.__aexit__(None, None, None)
|
||||
elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
|
||||
for candidate, page in zip(batch, results):
|
||||
url = candidate["url"]
|
||||
if isinstance(page, BaseException):
|
||||
logger.info("deepsearch fetch crashed for %s: %s", url, page)
|
||||
page = None
|
||||
if page is None:
|
||||
emit(
|
||||
{
|
||||
"type": "page_skipped",
|
||||
"url": url,
|
||||
"reason": "no readable content",
|
||||
"elapsed_ms": elapsed_ms,
|
||||
}
|
||||
)
|
||||
continue
|
||||
if fetched >= max_pages:
|
||||
break
|
||||
digest = content_hash(page.text)
|
||||
if digest in outcome.seen_hashes:
|
||||
emit(
|
||||
{
|
||||
"type": "page_duplicate",
|
||||
"url": url,
|
||||
"reason": "duplicate content",
|
||||
"elapsed_ms": elapsed_ms,
|
||||
}
|
||||
)
|
||||
continue
|
||||
outcome.seen_hashes.add(digest)
|
||||
outcome.pages.append(page)
|
||||
fetched += 1
|
||||
emit(
|
||||
{
|
||||
"type": "page_loaded",
|
||||
"url": page.url,
|
||||
"title": page.title,
|
||||
"source": page.source,
|
||||
"depth": level,
|
||||
"render": page.source == "playwright",
|
||||
"elapsed_ms": elapsed_ms,
|
||||
"done": fetched,
|
||||
"total": total,
|
||||
}
|
||||
)
|
||||
if level + 1 < depth:
|
||||
for link in relevant_links(page.links, query, LINKS_PER_PAGE):
|
||||
if link not in seen_urls:
|
||||
seen_urls.add(link)
|
||||
next_candidates.append({"url": link})
|
||||
level_candidates = next_candidates
|
||||
total = min(total + len(next_candidates), max_pages)
|
||||
finally:
|
||||
if browser is not None:
|
||||
await browser.close()
|
||||
return outcome
|
||||
|
||||
Reference in New Issue
Block a user