|
# retoor <retoor@molodetz.nl>
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
from .base import (
|
|
HIGH,
|
|
MEDIUM,
|
|
LOW,
|
|
INFO,
|
|
Check,
|
|
PageContext,
|
|
SiteContext,
|
|
ok_check,
|
|
page_check,
|
|
site_check,
|
|
)
|
|
|
|
CATEGORY = "ai_readiness"
|
|
|
|
_TAG = re.compile(r"<[^>]+>")
|
|
_SCRIPT_STYLE = re.compile(r"<(script|style)[^>]*>.*?</\1>", re.IGNORECASE | re.DOTALL)
|
|
|
|
|
|
def _text_words(html: str) -> int:
|
|
if not html:
|
|
return 0
|
|
stripped = _SCRIPT_STYLE.sub(" ", html)
|
|
stripped = _TAG.sub(" ", stripped)
|
|
return len([w for w in stripped.split() if w])
|
|
|
|
|
|
@page_check
|
|
def ssr_parity(page: PageContext, site: SiteContext) -> Check:
|
|
rendered = int(page.dom.get("wordCount", 0) or 0)
|
|
raw = _text_words(page.raw_html)
|
|
if rendered <= 0:
|
|
return None
|
|
ratio = raw / rendered if rendered else 0
|
|
passed = ratio >= 0.5
|
|
return ok_check(
|
|
"ai.ssr_parity",
|
|
CATEGORY,
|
|
"Content in initial HTML",
|
|
passed,
|
|
severity=HIGH,
|
|
value=f"{int(ratio * 100)}% server-rendered",
|
|
recommendation="Serve primary content in the initial HTML (SSR); JS-only content is invisible to many crawlers and AI agents.",
|
|
warn=ratio >= 0.25,
|
|
details={"raw_words": raw, "rendered_words": rendered},
|
|
url=page.url,
|
|
)
|
|
|
|
|
|
@page_check
|
|
def semantic_html(page: PageContext, site: SiteContext) -> Check:
|
|
semantic = page.dom.get("semantic", {}) or {}
|
|
has_main = int(semantic.get("main", 0) or 0) >= 1
|
|
has_landmarks = any(
|
|
int(semantic.get(tag, 0) or 0) >= 1 for tag in ("article", "nav", "header", "footer")
|
|
)
|
|
passed = has_main and has_landmarks
|
|
return ok_check(
|
|
"ai.semantic_html",
|
|
CATEGORY,
|
|
"Semantic HTML landmarks",
|
|
passed,
|
|
severity=LOW,
|
|
value="ok" if passed else "missing landmarks",
|
|
recommendation="Use <main>, <article>, <nav>, <header> and <footer> so machines can extract the main content.",
|
|
warn=True,
|
|
details=semantic,
|
|
url=page.url,
|
|
)
|
|
|
|
|
|
@site_check
|
|
def llms_txt(site: SiteContext) -> Check:
|
|
found = bool((site.llms_txt or {}).get("found"))
|
|
return ok_check(
|
|
"ai.llms_txt",
|
|
CATEGORY,
|
|
"llms.txt present",
|
|
found,
|
|
severity=LOW,
|
|
value="present" if found else "missing",
|
|
recommendation="Publish an /llms.txt manifest to guide AI crawlers to your key content (emerging standard).",
|
|
warn=True,
|
|
)
|