# retoor <retoor@molodetz.nl>
from __future__ import annotations

import asyncio
import logging
from dataclasses import dataclass, field
from html.parser import HTMLParser
from pathlib import Path
from typing import AsyncIterator
from urllib.parse import urldefrag, urljoin, urlparse

import httpx

from devplacepy.net_guard import BlockedAddressError, guard_public_url, guarded_async_client
from devplacepy.services.jobs.isslop.acquisition.domcapture import DomSnapshot
from devplacepy.services.jobs.isslop.acquisition.workspace import safe_relative_path
from devplacepy.services.jobs.isslop.config import (
    DOM_ANALYSIS_MAX_PAGES,
    WEBSITE_MAX_DEPTH,
    WEBSITE_MAX_FILE_BYTES,
    WEBSITE_MAX_FILES,
    WEBSITE_MAX_PAGES,
    WEBSITE_MAX_TOTAL_BYTES,
    WEBSITE_REQUEST_TIMEOUT_SECONDS,
)

logger = logging.getLogger(__name__)

USER_AGENT: str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"

TEXT_CONTENT_HINTS: tuple[str, ...] = ("text/", "javascript", "json", "xml", "svg", "css")
SKIP_SCHEMES: tuple[str, ...] = ("mailto:", "tel:", "javascript:", "data:", "#")


class LinkExtractor(HTMLParser):
    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self.page_links: list[str] = []
        self.asset_links: list[str] = []

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        mapping = dict(attrs)
        if tag == "a" and mapping.get("href"):
            self.page_links.append(str(mapping["href"]))
        if tag in ("script", "img", "source", "iframe") and mapping.get("src"):
            self.asset_links.append(str(mapping["src"]))
        if tag == "link" and mapping.get("href"):
            self.asset_links.append(str(mapping["href"]))


@dataclass
class CrawlState:
    pages_fetched: int = 0
    files_saved: int = 0
    bytes_saved: int = 0
    visited: set[str] = field(default_factory=set)


def _clean_url(base: str, href: str) -> str:
    joined = urljoin(base, href)
    return urldefrag(joined).url


def _local_path_for(workspace: Path, url: str) -> Path:
    parsed = urlparse(url)
    path = parsed.path or "/"
    if path.endswith("/"):
        path += "index.html"
    name = path.lstrip("/") or "index.html"
    if "." not in name.split("/")[-1]:
        name += ".html"
    if parsed.query:
        name += "-" + "".join(ch for ch in parsed.query if ch.isalnum())[:40]
    return safe_relative_path(workspace, name)


def _is_text_response(content_type: str) -> bool:
    lowered = content_type.lower()
    return any(hint in lowered for hint in TEXT_CONTENT_HINTS)


def _same_site(candidate: str, root_host: str | None) -> bool:
    host = urlparse(candidate).hostname
    if host is None or root_host is None:
        return False
    return host == root_host or host.endswith("." + root_host) or root_host.endswith("." + host)


def _extract_links(base: str, html: str, root_host: str | None) -> tuple[list[str], list[str]]:
    extractor = LinkExtractor()
    try:
        extractor.feed(html)
    except ValueError as error:
        logger.debug("HTML parse issue on %s: %s", base, error)
    pages: list[str] = []
    assets: list[str] = []
    for asset in extractor.asset_links:
        cleaned = _clean_url(base, asset)
        if _same_site(cleaned, root_host):
            assets.append(cleaned)
    for link in extractor.page_links:
        cleaned = _clean_url(base, link)
        if _same_site(cleaned, root_host):
            pages.append(cleaned)
    return pages, assets


def _save_bytes(workspace: Path, url: str, body: bytes) -> Path | None:
    try:
        target = _local_path_for(workspace, url)
    except ValueError as error:
        logger.warning("Rejected path for %s: %s", url, error)
        return None
    target.parent.mkdir(parents=True, exist_ok=True)
    target.write_bytes(body)
    return target


async def crawl_website(
    url: str,
    workspace: Path,
    *,
    dom_sink: list[DomSnapshot] | None = None,
    allow_private: bool = False,
) -> AsyncIterator[str]:
    from devplacepy.services.jobs.isslop.acquisition.browser import browser_available

    if browser_available():
        try:
            async for progress in _crawl_with_browser(
                url, workspace, dom_sink=dom_sink, allow_private=allow_private
            ):
                yield progress
            return
        except RuntimeError as error:
            logger.warning("Stealth browser crawl failed, falling back to HTTP client: %s", error)
            yield f"Stealth browser unavailable ({error}); falling back to HTTP client"
    async for progress in _crawl_with_http(url, workspace, allow_private=allow_private):
        yield progress


async def _crawl_with_browser(
    url: str,
    workspace: Path,
    *,
    dom_sink: list[DomSnapshot] | None = None,
    allow_private: bool = False,
) -> AsyncIterator[str]:
    from devplacepy.services.jobs.isslop.acquisition.browser import StealthBrowser

    root_host = urlparse(url).hostname
    state = CrawlState()
    yield f"Launching stealth browser for {url} (max depth {WEBSITE_MAX_DEPTH}, max {WEBSITE_MAX_FILES} files)"
    async with StealthBrowser(tab_concurrency=4) as browser:
        yield "Stealth browser ready; rendering pages with an undetectable Chromium fingerprint"
        current_level: list[str] = [url]
        for depth in range(WEBSITE_MAX_DEPTH + 1):
            pending = [
                candidate
                for candidate in current_level
                if candidate not in state.visited and _same_site(candidate, root_host)
                and not any(candidate.lower().startswith(scheme) for scheme in SKIP_SCHEMES)
            ]
            if not pending or state.files_saved >= WEBSITE_MAX_FILES:
                break
            pending = pending[: max(0, WEBSITE_MAX_PAGES - state.pages_fetched)]
            safe_candidates: list[str] = []
            for candidate in pending:
                try:
                    await guard_public_url(candidate, allow_private=allow_private)
                except BlockedAddressError:
                    state.visited.add(candidate)
                    yield f"Blocked private/local address: {candidate}"
                    continue
                safe_candidates.append(candidate)
            for candidate in safe_candidates:
                state.visited.add(candidate)

            async def _render_or_capture(index: int, candidate: str) -> object:
                if depth == 0 and dom_sink is not None and index < DOM_ANALYSIS_MAX_PAGES:
                    snapshot = await browser.capture(candidate)
                    dom_sink.append(snapshot)
                    return snapshot.render
                return await browser.render(candidate)

            renders = await asyncio.gather(
                *(_render_or_capture(index, candidate) for index, candidate in enumerate(safe_candidates))
            )
            asset_urls: set[str] = set()
            next_level: list[str] = []
            for render in renders:
                if not render.ok:
                    yield f"Render failed: {render.url} ({render.error or 'no content'})"
                    continue
                body = render.html.encode("utf-8", errors="replace")[:WEBSITE_MAX_FILE_BYTES]
                target = _save_bytes(workspace, render.final_url, body)
                if target is None:
                    continue
                state.files_saved += 1
                state.bytes_saved += len(body)
                state.pages_fetched += 1
                yield f"Rendered {target.relative_to(workspace)} ({len(body)} bytes, DOM after JS execution)"
                pages, assets = _extract_links(render.final_url, render.html, root_host)
                for asset in assets:
                    if asset not in state.visited:
                        asset_urls.add(asset)
                next_level.extend(pages)
            async for progress in _download_assets(browser, workspace, sorted(asset_urls), state):
                yield progress
            current_level = next_level
            if state.bytes_saved >= WEBSITE_MAX_TOTAL_BYTES:
                yield f"Byte budget reached: {state.bytes_saved / 1024:.1f} KB"
                break
    if state.files_saved == 0:
        raise RuntimeError(f"Stealth crawl produced no files for {url}")
    yield (
        f"Website download complete: {state.files_saved} files, {state.bytes_saved / 1024:.1f} KB, "
        f"{state.pages_fetched} rendered pages"
    )


async def _download_assets(
    browser: object,
    workspace: Path,
    asset_urls: list[str],
    state: CrawlState,
) -> AsyncIterator[str]:
    budget = min(len(asset_urls), max(0, WEBSITE_MAX_FILES - state.files_saved))
    if budget <= 0:
        return
    targets = [asset for asset in asset_urls if asset not in state.visited][:budget]
    for asset in targets:
        state.visited.add(asset)
    results = await asyncio.gather(*(browser.fetch(asset, WEBSITE_MAX_FILE_BYTES) for asset in targets))
    saved = 0
    for result in results:
        if not result.ok or not result.body:
            continue
        target = _save_bytes(workspace, result.url, result.body)
        if target is None:
            continue
        state.files_saved += 1
        state.bytes_saved += len(result.body)
        saved += 1
    if saved:
        yield f"Downloaded {saved} assets concurrently (scripts, styles, sources)"


async def _crawl_with_http(
    url: str, workspace: Path, *, allow_private: bool = False
) -> AsyncIterator[str]:
    root = urlparse(url)
    state = CrawlState()
    queue: list[tuple[str, int]] = [(url, 0)]
    headers = {"user-agent": USER_AGENT, "accept": "*/*"}
    yield f"Crawling website {url} with HTTP client (max depth {WEBSITE_MAX_DEPTH}, max {WEBSITE_MAX_FILES} files)"
    async with guarded_async_client(
        timeout=WEBSITE_REQUEST_TIMEOUT_SECONDS,
        follow_redirects=True,
        headers=headers,
    ) as client:
        while queue:
            if state.files_saved >= WEBSITE_MAX_FILES or state.bytes_saved >= WEBSITE_MAX_TOTAL_BYTES:
                yield f"Crawl limits reached: {state.files_saved} files, {state.bytes_saved} bytes"
                break
            current, depth = queue.pop(0)
            if current in state.visited:
                continue
            state.visited.add(current)
            parsed = urlparse(current)
            if parsed.hostname != root.hostname:
                continue
            if any(current.lower().startswith(scheme) for scheme in SKIP_SCHEMES):
                continue
            try:
                await guard_public_url(current, allow_private=allow_private)
            except BlockedAddressError:
                yield f"Blocked private/local address: {current}"
                continue
            try:
                response = await client.get(current)
            except BlockedAddressError as error:
                logger.warning("Fetch blocked %s: %s", current, error)
                yield f"Blocked private/local address on redirect: {current} ({error})"
                continue
            except httpx.HTTPError as error:
                logger.warning("Fetch failed %s: %s", current, error)
                yield f"Fetch failed: {current} ({error})"
                continue
            if response.status_code >= 400:
                yield f"Skipped {current}: HTTP {response.status_code}"
                continue
            content_type = response.headers.get("content-type", "")
            body = response.content[:WEBSITE_MAX_FILE_BYTES]
            try:
                target = _local_path_for(workspace, current)
            except ValueError as error:
                logger.warning("Rejected path for %s: %s", current, error)
                continue
            target.parent.mkdir(parents=True, exist_ok=True)
            target.write_bytes(body)
            state.files_saved += 1
            state.bytes_saved += len(body)
            yield f"Saved {target.relative_to(workspace)} ({len(body)} bytes, {content_type.split(';')[0] or 'unknown type'})"

            is_page = "text/html" in content_type.lower()
            if is_page:
                state.pages_fetched += 1
                if depth < WEBSITE_MAX_DEPTH and state.pages_fetched < WEBSITE_MAX_PAGES:
                    extractor = LinkExtractor()
                    try:
                        extractor.feed(body.decode("utf-8", errors="replace"))
                    except ValueError as error:
                        logger.debug("HTML parse issue on %s: %s", current, error)
                    for asset in extractor.asset_links:
                        cleaned = _clean_url(current, asset)
                        if urlparse(cleaned).hostname == root.hostname and cleaned not in state.visited:
                            queue.append((cleaned, depth + 1))
                    for link in extractor.page_links:
                        cleaned = _clean_url(current, link)
                        if urlparse(cleaned).hostname == root.hostname and cleaned not in state.visited:
                            queue.append((cleaned, depth + 1))
    if state.files_saved == 0:
        raise RuntimeError(f"Website download produced no files for {url}")
    yield f"Website download complete: {state.files_saved} files, {state.bytes_saved / 1024:.1f} KB, {state.pages_fetched} pages"
