# retoor <retoor@molodetz.nl>

from __future__ import annotations

import asyncio
import hashlib
import logging
import re
import time
from dataclasses import dataclass, field
from typing import Awaitable, Callable

import httpx

from devplacepy import stealth
from devplacepy.net_guard import BlockedAddressError, guard_public_url, guarded_async_client

from .pdf import MAX_PDF_BYTES, extract_pdf_text, is_pdf
from .phases import PHASE_CRAWLING

logger = logging.getLogger(__name__)

RSEARCH_URL = "https://rsearch.app.molodetz.nl"
RSEARCH_TIMEOUT_SECONDS = 45.0
FETCH_TIMEOUT_SECONDS = 20.0
MAX_FETCH_BYTES = 2_500_000
RESULTS_PER_QUERY = 8
CRAWL_CONCURRENCY = 4
USER_AGENT = (
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
    "Chrome/131.0.0.0 Safari/537.36 DevPlaceDeepSearchBot/1.0"
)
SCRIPT_STYLE = re.compile(r"<(script|style)[^>]*>.*?</\1>", re.DOTALL | re.IGNORECASE)
TAG = re.compile(r"<[^>]+>")
TITLE = re.compile(r"<title[^>]*>(.*?)</title>", re.DOTALL | re.IGNORECASE)
SPACE = re.compile(r"\s+")
MIN_PAGE_CHARS = 200


@dataclass
class CrawledPage:
    url: str
    title: str
    text: str
    source: str
    status: int
    depth: int = 0
    from_cache: bool = False


@dataclass
class CrawlOutcome:
    pages: list[CrawledPage] = field(default_factory=list)
    seen_hashes: set[str] = field(default_factory=set)


def url_hash(url: str) -> str:
    return hashlib.sha256(url.strip().lower().encode("utf-8")).hexdigest()


def content_hash(text: str) -> str:
    return hashlib.sha256(text.strip().encode("utf-8")).hexdigest()


def _strip_html(raw: str) -> tuple[str, str]:
    title_match = TITLE.search(raw)
    title = SPACE.sub(" ", TAG.sub("", title_match.group(1))).strip() if title_match else ""
    body = SCRIPT_STYLE.sub(" ", raw)
    body = TAG.sub(" ", body)
    body = SPACE.sub(" ", body).strip()
    return title, body


async def search_queries(queries: list[str]) -> list[dict]:
    results: list[dict] = []
    seen: set[str] = set()
    headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
    timeout = httpx.Timeout(RSEARCH_TIMEOUT_SECONDS, connect=30.0)
    async with stealth.stealth_async_client(
        base_url=RSEARCH_URL, headers=headers, follow_redirects=True, timeout=timeout
    ) as client:
        for query in queries:
            try:
                response = await client.get(
                    "/search",
                    params={"query": query, "count": RESULTS_PER_QUERY, "content": "false"},
                )
                if response.status_code >= 400:
                    continue
                data = response.json()
            except (httpx.HTTPError, ValueError) as exc:
                logger.warning("deepsearch rsearch failed for %r: %s", query, exc)
                continue
            for item in data.get("results") or []:
                url = (item.get("url") or "").strip()
                if not url or url in seen:
                    continue
                seen.add(url)
                results.append(
                    {
                        "url": url,
                        "title": item.get("title") or "",
                        "description": item.get("description") or "",
                    }
                )
    return results


async def _render_with_playwright(url: str) -> tuple[str, str, int]:
    from playwright.async_api import async_playwright

    async with async_playwright() as pw:
        browser = await pw.chromium.launch(
            headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"]
        )
        try:
            context = await browser.new_context(user_agent=USER_AGENT)
            page = await context.new_page()
            response = await page.goto(url, wait_until="load", timeout=30000)
            status = response.status if response else 0
            for hop in [response.url] if response else []:
                await guard_public_url(hop)
            content = (await page.content())[:MAX_FETCH_BYTES]
            await context.close()
            title, text = _strip_html(content)
            return title, text, status
        finally:
            await browser.close()


async def fetch_page(url: str, depth: int) -> CrawledPage | None:
    try:
        await guard_public_url(url)
    except BlockedAddressError:
        return None
    title = ""
    text = ""
    status = 0
    source = "httpx"
    content_type = ""
    encoding = "utf-8"
    raw_bytes = b""
    try:
        async with guarded_async_client(
            follow_redirects=True,
            timeout=FETCH_TIMEOUT_SECONDS,
            headers={"User-Agent": USER_AGENT},
        ) as client:
            async with client.stream("GET", url) as response:
                await guard_public_url(str(response.url))
                status = response.status_code
                content_type = response.headers.get("content-type", "").lower()
                encoding = response.encoding or "utf-8"
                buffer = bytearray()
                async for chunk in response.aiter_bytes():
                    buffer.extend(chunk)
                    if len(buffer) >= MAX_PDF_BYTES:
                        break
                raw_bytes = bytes(buffer)
    except (httpx.HTTPError, BlockedAddressError) as exc:
        logger.info("deepsearch httpx fetch failed for %s: %s", url, exc)
    if is_pdf(content_type, url, raw_bytes[:5]):
        title, text = extract_pdf_text(raw_bytes[:MAX_PDF_BYTES])
        if len(text) < MIN_PAGE_CHARS:
            return None
        return CrawledPage(
            url=url, title=title or url, text=text, source="pdf", status=status, depth=depth
        )
    if raw_bytes:
        try:
            raw = raw_bytes[:MAX_FETCH_BYTES].decode(encoding, errors="replace")
            title, text = _strip_html(raw)
        except (LookupError, ValueError) as exc:
            logger.info("deepsearch decode failed for %s: %s", url, exc)
    if len(text) < MIN_PAGE_CHARS:
        try:
            r_title, r_text, r_status = await _render_with_playwright(url)
            if len(r_text) > len(text):
                title, text, status, source = (
                    r_title or title,
                    r_text,
                    r_status or status,
                    "playwright",
                )
        except Exception as exc:
            logger.info("deepsearch render failed for %s: %s", url, exc)
    if len(text) < MIN_PAGE_CHARS:
        return None
    return CrawledPage(
        url=url, title=title or url, text=text, source=source, status=status, depth=depth
    )


async def crawl(
    candidates: list[dict],
    max_pages: int,
    emit: Callable[[dict], None],
    is_cached: Callable[[str], bool],
    should_stop: Callable[[], Awaitable[bool]],
) -> CrawlOutcome:
    outcome = CrawlOutcome()
    fetched = 0
    total = min(len(candidates), max_pages)
    for index, candidate in enumerate(candidates):
        if fetched >= max_pages:
            break
        if await should_stop():
            emit({"type": "stage", "stage": "cancelled", "message": "Crawl cancelled"})
            break
        url = candidate["url"]
        emit(
            {
                "type": "progress",
                "done": fetched,
                "total": total,
                "url": url,
                "message": f"Fetching {url}",
            }
        )
        if is_cached(url):
            emit({"type": "page_cached", "url": url})
        page = await fetch_page(url, depth=0)
        if page is None:
            emit({"type": "page_skipped", "url": url})
            continue
        digest = content_hash(page.text)
        if digest in outcome.seen_hashes:
            emit({"type": "page_duplicate", "url": url})
            continue
        outcome.seen_hashes.add(digest)
        outcome.pages.append(page)
        fetched += 1
        emit(
            {
                "type": "page_loaded",
                "url": page.url,
                "title": page.title,
                "source": page.source,
                "done": fetched,
                "total": total,
            }
        )
    return outcome
