retoor
d10f1af118
DevPlace CI / test (push) Failing after 2m13s
Implement a new `SeoMetaService` subservice that generates clean SEO title/description/keywords for published content items, distinct from the existing SEO diagnostics auditor. Add `seo_metadata` polymorphic table with soft-delete support, batch query methods, and usage tracking. Extend the CLI with `seo-meta prune` and `seo-meta clear` commands for job row lifecycle management. Wire `schedule_seo_meta_for_table` into content creation and editing flows in `content.py`. Document the new service in `AGENTS.md` and `README.md`, including the `extra_head` site setting for custom `<head>` injection.
265 lines
8.8 KiB
Python
265 lines
8.8 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import logging
|
|
import re
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import Awaitable, Callable
|
|
|
|
import httpx
|
|
|
|
from devplacepy import stealth
|
|
from devplacepy.net_guard import BlockedAddressError, guard_public_url, guarded_async_client
|
|
|
|
from .pdf import MAX_PDF_BYTES, extract_pdf_text, is_pdf
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
RSEARCH_URL = "https://rsearch.app.molodetz.nl"
|
|
RSEARCH_TIMEOUT_SECONDS = 45.0
|
|
FETCH_TIMEOUT_SECONDS = 20.0
|
|
MAX_FETCH_BYTES = 2_500_000
|
|
RESULTS_PER_QUERY = 8
|
|
CRAWL_CONCURRENCY = 4
|
|
USER_AGENT = (
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/131.0.0.0 Safari/537.36 DevPlaceDeepSearchBot/1.0"
|
|
)
|
|
SCRIPT_STYLE = re.compile(r"<(script|style)[^>]*>.*?</\1>", re.DOTALL | re.IGNORECASE)
|
|
TAG = re.compile(r"<[^>]+>")
|
|
TITLE = re.compile(r"<title[^>]*>(.*?)</title>", re.DOTALL | re.IGNORECASE)
|
|
SPACE = re.compile(r"\s+")
|
|
MIN_PAGE_CHARS = 200
|
|
|
|
|
|
@dataclass
|
|
class CrawledPage:
|
|
url: str
|
|
title: str
|
|
text: str
|
|
source: str
|
|
status: int
|
|
depth: int = 0
|
|
from_cache: bool = False
|
|
|
|
|
|
@dataclass
|
|
class CrawlOutcome:
|
|
pages: list[CrawledPage] = field(default_factory=list)
|
|
seen_hashes: set[str] = field(default_factory=set)
|
|
|
|
|
|
def url_hash(url: str) -> str:
|
|
return hashlib.sha256(url.strip().lower().encode("utf-8")).hexdigest()
|
|
|
|
|
|
def content_hash(text: str) -> str:
|
|
return hashlib.sha256(text.strip().encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _strip_html(raw: str) -> tuple[str, str]:
|
|
title_match = TITLE.search(raw)
|
|
title = SPACE.sub(" ", TAG.sub("", title_match.group(1))).strip() if title_match else ""
|
|
body = SCRIPT_STYLE.sub(" ", raw)
|
|
body = TAG.sub(" ", body)
|
|
body = SPACE.sub(" ", body).strip()
|
|
return title, body
|
|
|
|
|
|
async def search_queries(
|
|
queries: list[str], emit: Callable[[dict], None] = lambda frame: None
|
|
) -> list[dict]:
|
|
results: list[dict] = []
|
|
seen: set[str] = set()
|
|
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
|
|
timeout = httpx.Timeout(RSEARCH_TIMEOUT_SECONDS, connect=30.0)
|
|
async with stealth.stealth_async_client(
|
|
base_url=RSEARCH_URL, headers=headers, follow_redirects=True, timeout=timeout
|
|
) as client:
|
|
for query in queries:
|
|
try:
|
|
response = await client.get(
|
|
"/search",
|
|
params={"query": query, "count": RESULTS_PER_QUERY, "content": "false"},
|
|
)
|
|
emit({"type": "rsearch", "endpoint": "/search", "success": response.status_code < 400})
|
|
if response.status_code >= 400:
|
|
continue
|
|
data = response.json()
|
|
except (httpx.HTTPError, ValueError) as exc:
|
|
emit({"type": "rsearch", "endpoint": "/search", "success": False})
|
|
logger.warning("deepsearch rsearch failed for %r: %s", query, exc)
|
|
continue
|
|
for item in data.get("results") or []:
|
|
url = (item.get("url") or "").strip()
|
|
if not url or url in seen:
|
|
continue
|
|
seen.add(url)
|
|
results.append(
|
|
{
|
|
"url": url,
|
|
"title": item.get("title") or "",
|
|
"description": item.get("description") or "",
|
|
}
|
|
)
|
|
return results
|
|
|
|
|
|
async def _render_with_playwright(url: str) -> tuple[str, str, int]:
|
|
from playwright.async_api import async_playwright
|
|
|
|
async with async_playwright() as pw:
|
|
browser = await pw.chromium.launch(
|
|
headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
|
|
)
|
|
try:
|
|
context = await browser.new_context(user_agent=USER_AGENT)
|
|
page = await context.new_page()
|
|
response = await page.goto(url, wait_until="load", timeout=30000)
|
|
status = response.status if response else 0
|
|
for hop in [response.url] if response else []:
|
|
await guard_public_url(hop)
|
|
content = (await page.content())[:MAX_FETCH_BYTES]
|
|
await context.close()
|
|
title, text = _strip_html(content)
|
|
return title, text, status
|
|
finally:
|
|
await browser.close()
|
|
|
|
|
|
async def fetch_page(url: str, depth: int) -> CrawledPage | None:
|
|
try:
|
|
await guard_public_url(url)
|
|
except BlockedAddressError:
|
|
return None
|
|
title = ""
|
|
text = ""
|
|
status = 0
|
|
source = "httpx"
|
|
content_type = ""
|
|
encoding = "utf-8"
|
|
raw_bytes = b""
|
|
try:
|
|
async with guarded_async_client(
|
|
follow_redirects=True,
|
|
timeout=FETCH_TIMEOUT_SECONDS,
|
|
headers={"User-Agent": USER_AGENT},
|
|
) as client:
|
|
async with client.stream("GET", url) as response:
|
|
await guard_public_url(str(response.url))
|
|
status = response.status_code
|
|
content_type = response.headers.get("content-type", "").lower()
|
|
encoding = response.encoding or "utf-8"
|
|
buffer = bytearray()
|
|
async for chunk in response.aiter_bytes():
|
|
buffer.extend(chunk)
|
|
if len(buffer) >= MAX_PDF_BYTES:
|
|
break
|
|
raw_bytes = bytes(buffer)
|
|
except (httpx.HTTPError, BlockedAddressError) as exc:
|
|
logger.info("deepsearch httpx fetch failed for %s: %s", url, exc)
|
|
if is_pdf(content_type, url, raw_bytes[:5]):
|
|
title, text = extract_pdf_text(raw_bytes[:MAX_PDF_BYTES])
|
|
if len(text) < MIN_PAGE_CHARS:
|
|
return None
|
|
return CrawledPage(
|
|
url=url, title=title or url, text=text, source="pdf", status=status, depth=depth
|
|
)
|
|
if raw_bytes:
|
|
try:
|
|
raw = raw_bytes[:MAX_FETCH_BYTES].decode(encoding, errors="replace")
|
|
title, text = _strip_html(raw)
|
|
except (LookupError, ValueError) as exc:
|
|
logger.info("deepsearch decode failed for %s: %s", url, exc)
|
|
if len(text) < MIN_PAGE_CHARS:
|
|
try:
|
|
r_title, r_text, r_status = await _render_with_playwright(url)
|
|
if len(r_text) > len(text):
|
|
title, text, status, source = (
|
|
r_title or title,
|
|
r_text,
|
|
r_status or status,
|
|
"playwright",
|
|
)
|
|
except Exception as exc:
|
|
logger.info("deepsearch render failed for %s: %s", url, exc)
|
|
if len(text) < MIN_PAGE_CHARS:
|
|
return None
|
|
return CrawledPage(
|
|
url=url, title=title or url, text=text, source=source, status=status, depth=depth
|
|
)
|
|
|
|
|
|
async def crawl(
|
|
candidates: list[dict],
|
|
max_pages: int,
|
|
emit: Callable[[dict], None],
|
|
is_cached: Callable[[str], bool],
|
|
should_stop: Callable[[], Awaitable[bool]],
|
|
) -> CrawlOutcome:
|
|
outcome = CrawlOutcome()
|
|
fetched = 0
|
|
total = min(len(candidates), max_pages)
|
|
for index, candidate in enumerate(candidates):
|
|
if fetched >= max_pages:
|
|
break
|
|
if await should_stop():
|
|
emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
|
|
break
|
|
url = candidate["url"]
|
|
emit(
|
|
{
|
|
"type": "progress",
|
|
"done": fetched,
|
|
"total": total,
|
|
"url": url,
|
|
"message": f"Fetching {url}",
|
|
}
|
|
)
|
|
if is_cached(url):
|
|
emit({"type": "page_cached", "url": url, "reason": "seen in a prior run"})
|
|
fetch_start = time.perf_counter()
|
|
page = await fetch_page(url, depth=0)
|
|
elapsed_ms = int((time.perf_counter() - fetch_start) * 1000)
|
|
if page is None:
|
|
emit(
|
|
{
|
|
"type": "page_skipped",
|
|
"url": url,
|
|
"reason": "no readable content",
|
|
"elapsed_ms": elapsed_ms,
|
|
}
|
|
)
|
|
continue
|
|
digest = content_hash(page.text)
|
|
if digest in outcome.seen_hashes:
|
|
emit(
|
|
{
|
|
"type": "page_duplicate",
|
|
"url": url,
|
|
"reason": "duplicate content",
|
|
"elapsed_ms": elapsed_ms,
|
|
}
|
|
)
|
|
continue
|
|
outcome.seen_hashes.add(digest)
|
|
outcome.pages.append(page)
|
|
fetched += 1
|
|
emit(
|
|
{
|
|
"type": "page_loaded",
|
|
"url": page.url,
|
|
"title": page.title,
|
|
"source": page.source,
|
|
"render": page.source == "playwright",
|
|
"elapsed_ms": elapsed_ms,
|
|
"done": fetched,
|
|
"total": total,
|
|
}
|
|
)
|
|
return outcome
|