# retoor <retoor@molodetz.nl>

from __future__ import annotations

import re
from dataclasses import dataclass, field
from html.parser import HTMLParser
from urllib.parse import urldefrag, urljoin, urlparse

SKIP_TAGS = {
    "script",
    "style",
    "noscript",
    "template",
    "svg",
    "iframe",
    "canvas",
    "form",
    "button",
    "select",
    "option",
    "nav",
    "header",
    "footer",
    "aside",
}
SKIP_ROLES = {"navigation", "banner", "contentinfo", "complementary", "search", "menu", "menubar"}
BLOCK_TAGS = {
    "p",
    "div",
    "section",
    "article",
    "main",
    "li",
    "ul",
    "ol",
    "td",
    "th",
    "tr",
    "table",
    "blockquote",
    "pre",
    "figure",
    "figcaption",
    "dd",
    "dt",
    "details",
    "summary",
    "h1",
    "h2",
    "h3",
    "h4",
    "h5",
    "h6",
}
CONTENT_TAGS = {"article", "main"}
HEADING_TAGS = {"h1", "h2", "h3", "h4", "h5", "h6"}
VOID_TAGS = {
    "br",
    "hr",
    "img",
    "meta",
    "link",
    "input",
    "source",
    "area",
    "base",
    "col",
    "embed",
    "track",
    "wbr",
}
NON_DOCUMENT_EXTENSIONS = (
    ".jpg",
    ".jpeg",
    ".png",
    ".gif",
    ".webp",
    ".svg",
    ".ico",
    ".css",
    ".js",
    ".json",
    ".xml",
    ".zip",
    ".gz",
    ".tar",
    ".mp3",
    ".mp4",
    ".webm",
    ".woff",
    ".woff2",
    ".exe",
    ".dmg",
)
SPACES = re.compile(r"[ \t\u00a0]+")
MULTI_NEWLINE = re.compile(r"\n{2,}")
MIN_BLOCK_CHARS = 30
MIN_HEADING_CHARS = 8
MAX_LINK_DENSITY = 0.55
MIN_CONTENT_TOTAL = 500
MAX_LINKS = 120


@dataclass
class Paragraph:
    text: str
    in_content: bool
    heading: bool


@dataclass
class ExtractedPage:
    title: str = ""
    text: str = ""
    links: list[tuple[str, str]] = field(default_factory=list)


@dataclass
class _Frame:
    tag: str
    in_content: bool
    parts: list[str] = field(default_factory=list)
    link_chars: int = 0


@dataclass
class _Entry:
    tag: str
    skip: bool
    in_content: bool
    frame: _Frame | None


class _Extractor(HTMLParser):
    def __init__(self, base_url: str) -> None:
        super().__init__(convert_charrefs=True)
        self.base_url = base_url
        self.title = ""
        self.first_heading = ""
        self.paragraphs: list[Paragraph] = []
        self.links: list[tuple[str, str]] = []
        self.stack: list[_Entry] = []
        self.root = _Frame(tag="root", in_content=False)
        self.in_title = False
        self.anchor_href = ""
        self.anchor_parts: list[str] = []
        self.in_anchor = False

    def _skipping(self) -> bool:
        return bool(self.stack) and self.stack[-1].skip

    def _in_content(self) -> bool:
        return bool(self.stack) and self.stack[-1].in_content

    def _frame(self) -> _Frame:
        for entry in reversed(self.stack):
            if entry.frame is not None:
                return entry.frame
        return self.root

    def handle_starttag(self, tag: str, attrs: list) -> None:
        if tag in VOID_TAGS:
            if tag in ("br", "hr") and not self._skipping():
                self._frame().parts.append("\n")
            return
        if tag == "title":
            self.in_title = True
            return
        attr_map = dict(attrs)
        skip = self._skipping() or tag in SKIP_TAGS or (attr_map.get("role") or "").lower() in SKIP_ROLES
        in_content = self._in_content() or tag in CONTENT_TAGS
        frame = _Frame(tag=tag, in_content=in_content) if tag in BLOCK_TAGS and not skip else None
        self.stack.append(_Entry(tag=tag, skip=skip, in_content=in_content, frame=frame))
        if tag == "a" and not skip and not self.in_anchor:
            href = (attr_map.get("href") or "").strip()
            if href and not href.startswith(("javascript:", "mailto:", "tel:", "#")):
                self.in_anchor = True
                self.anchor_href = href
                self.anchor_parts = []

    def handle_startendtag(self, tag: str, attrs: list) -> None:
        self.handle_starttag(tag, attrs)

    def handle_endtag(self, tag: str) -> None:
        if tag in VOID_TAGS:
            return
        if tag == "title":
            self.in_title = False
            return
        if tag == "a" and self.in_anchor:
            self._emit_link()
        if not any(entry.tag == tag for entry in self.stack):
            return
        while self.stack:
            entry = self.stack.pop()
            if entry.frame is not None:
                self._flush(entry.frame)
            if entry.tag == tag:
                break

    def handle_data(self, data: str) -> None:
        if self.in_title:
            self.title += data
            return
        if self._skipping() or not data:
            return
        self._frame().parts.append(data)
        if self.in_anchor:
            self.anchor_parts.append(data)
            self._frame().link_chars += len(data.strip())

    def _emit_link(self) -> None:
        text = SPACES.sub(" ", "".join(self.anchor_parts)).strip()
        absolute = urljoin(self.base_url, self.anchor_href) if self.base_url else self.anchor_href
        absolute = urldefrag(absolute).url
        if absolute.startswith(("http://", "https://")) and len(self.links) < MAX_LINKS:
            self.links.append((absolute, text))
        self.in_anchor = False
        self.anchor_href = ""
        self.anchor_parts = []

    def _flush(self, frame: _Frame) -> None:
        raw = "".join(frame.parts)
        if frame.tag == "pre":
            text = MULTI_NEWLINE.sub("\n", SPACES.sub(" ", raw)).strip()
        else:
            text = SPACES.sub(" ", raw.replace("\n", " ")).strip()
        if not text:
            return
        heading = frame.tag in HEADING_TAGS
        if heading:
            if len(text) < MIN_HEADING_CHARS:
                return
            if not self.first_heading:
                self.first_heading = text
        else:
            if len(text) < MIN_BLOCK_CHARS:
                return
            density = frame.link_chars / max(1, len(text))
            if density > MAX_LINK_DENSITY:
                return
        self.paragraphs.append(Paragraph(text=text, in_content=frame.in_content, heading=heading))

    def finish(self) -> None:
        if self.in_anchor:
            self._emit_link()
        while self.stack:
            entry = self.stack.pop()
            if entry.frame is not None:
                self._flush(entry.frame)
        self._flush(self.root)


def relevant_links(links: list[tuple[str, str]], query: str, limit: int) -> list[str]:
    tokens = {t for t in re.findall(r"[a-z0-9]+", query.lower()) if len(t) > 2}
    if not tokens:
        return []
    scored: list[tuple[int, str]] = []
    for url, text in links:
        path = urlparse(url).path.lower()
        if path.endswith(NON_DOCUMENT_EXTENSIONS):
            continue
        candidate_tokens = set(re.findall(r"[a-z0-9]+", f"{text} {path}".lower()))
        score = len(tokens & candidate_tokens)
        if score > 0:
            scored.append((score, url))
    scored.sort(key=lambda item: item[0], reverse=True)
    return [url for _score, url in scored[:limit]]


def extract_html(raw: str, base_url: str = "") -> ExtractedPage:
    parser = _Extractor(base_url)
    try:
        parser.feed(raw)
        parser.close()
    except Exception:
        pass
    parser.finish()
    title = SPACES.sub(" ", parser.title).strip() or parser.first_heading
    content = [p for p in parser.paragraphs if p.in_content]
    chosen = content if sum(len(p.text) for p in content) >= MIN_CONTENT_TOTAL else parser.paragraphs
    text = "\n\n".join(p.text for p in chosen)
    return ExtractedPage(title=title, text=text, links=parser.links)
