Files
devplacepy/devplacepy/services/community/feeds.py
T
retoor 6f80a820da
DevPlace CI / test (push) Failing after 3m13s
Zo doen wijdat in oost
2026-10-16 07:20:44 +02:00

263 lines
10 KiB
Python

# retoor <retoor@molodetz.nl>
from __future__ import annotations
import logging
import re
import statistics
from dataclasses import dataclass, field
from urllib.parse import urljoin, urlsplit
from devplacepy.services.community.extract import (
Node,
json_or_none,
parse_feed,
parse_html,
select_nodes,
)
from devplacepy.services.community.spec import (
FEED_ATOM,
FEED_HTML,
FEED_JSON,
FEED_PROBE_PATHS,
FEED_RSS,
LIMITS,
)
logger = logging.getLogger("community.feeds")
ITEM_KEYS = re.compile(
r"^(title|name|subject|headline|content|body|text|description|summary|id|uid|guid|url|link|"
r"created_at|created|published|date|author|user|username|creator|content_html|excerpt)$",
re.I,
)
ITEM_HINT = re.compile(r"post|thread|topic|item|entry|article|story|rant|card|result|row", re.I)
ACTIVITY_LINK = re.compile(r"\b(latest|recent|new|activity|posts|threads|topics|discussions|feed)\b", re.I)
LOGIN_WORDS = re.compile(r"(sign in|log in|login|password)", re.I)
MIN_REPEAT = 3
COMMENT_FEED = re.compile(r"/(comments?|repl(?:y|ies)|responses?)(?:[./_-][\w.]*)?/?$", re.I)
@dataclass
class FeedCandidate:
url: str
type: str
items: list = field(default_factory=list)
document: object = None
list_path: str | None = None
item_selector: str | None = None
login_required: bool = False
render: bool = False
def item_score(item) -> int:
if not isinstance(item, dict):
return 0
return sum(1 for key in item if ITEM_KEYS.match(str(key).lstrip("@")))
def find_item_list(document, path: str = "$", depth: int = 0):
best = (0, None, None)
if depth > 3:
return best
if isinstance(document, list):
dicts = [entry for entry in document if isinstance(entry, dict)]
if len(dicts) >= 2:
score = statistics.median(item_score(entry) for entry in dicts[:8]) * min(len(dicts), 20)
best = (score, path, document)
return best
if isinstance(document, dict):
for key, value in document.items():
if isinstance(value, (list, dict)):
found = find_item_list(value, f"{path}.{key}", depth + 1)
if found[0] > best[0]:
best = found
return best
def find_item_selector(root: Node) -> tuple[str, list[Node]] | None:
best_score = 0.0
best: tuple[str, list[Node]] | None = None
groups: dict[tuple, list[Node]] = {}
for node in root.descendants():
if node.tag in ("script", "style", "nav", "header", "footer", "select", "option", "a", "span", "td", "th"):
continue
key = (node.tag, tuple(sorted(node.classes())))
groups.setdefault(key, []).append(node)
for (tag, classes), nodes in groups.items():
if len(nodes) < MIN_REPEAT:
continue
linked = [node for node in nodes if any(child.tag == "a" and child.attrs.get("href") for child in node.descendants())]
if len(linked) < MIN_REPEAT:
continue
lengths = [len(node.text()) for node in linked]
median = statistics.median(lengths)
if median < 25:
continue
selector = tag + "".join(f".{name}" for name in classes)
hint = 2.0 if ITEM_HINT.search(selector) else 1.0
score = min(len(linked), 30) * min(median, 400) * hint
if score > best_score:
best_score = score
best = (selector, linked)
return best
def classify_text(url: str, text: str) -> FeedCandidate | None:
stripped = text.lstrip()
if stripped.startswith(("<?xml", "<rss", "<feed", "<rdf")):
try:
kind, _, items = parse_feed(text)
except Exception as exc:
logger.info("feed %s looked like XML but did not parse: %s", url, exc)
return None
if len(items) >= 1:
return FeedCandidate(url, FEED_RSS if kind == "rss" else FEED_ATOM, items=items)
return None
if stripped[:1] in "[{":
document = json_or_none(text)
if document is None:
return None
score, path, items = find_item_list(document)
if path and score >= 4:
return FeedCandidate(url, FEED_JSON, items=items, document=document, list_path=path)
return None
return None
COMMENT_HINT = re.compile(r"comment|repl(?:y|ies)|response|answer", re.I)
def find_comment_selector(roots: list[Node]) -> str | None:
tallies: dict[str, int] = {}
for root in roots:
for node in root.descendants():
if node.tag in ("script", "style", "a", "span", "form", "textarea", "input", "button", "ul", "ol", "section", "nav"):
continue
hinted = sorted(name for name in node.classes() if COMMENT_HINT.search(name) and not re.search(r"form|count|list|wrap|section|header|input|box|area|title|link|button", name, re.I))
if not hinted:
continue
selector = f"{node.tag}.{hinted[0]}"
if len(node.text()) < 4:
continue
tallies[selector] = tallies.get(selector, 0) + 1
ranked = sorted(tallies.items(), key=lambda entry: entry[1], reverse=True)
return ranked[0][0] if ranked and ranked[0][1] >= 2 else None
class FeedDetector:
def __init__(self, fetcher, ai=None):
self.fetcher = fetcher
self.ai = ai
async def fetch(self, url: str):
try:
response = await self.fetcher.get(url)
return response.status, await response.text()
except Exception as exc:
logger.warning("detect: %s unreachable: %s: %s", url, type(exc).__name__, exc)
return 0, ""
async def detect(self, base_url: str) -> tuple[list[FeedCandidate], dict]:
parts = urlsplit(base_url)
origin = f"{parts.scheme}://{parts.netloc}"
status, text = await self.fetch(base_url)
context = {"status": status, "html": text, "login_required": status in (401, 403)}
found: dict[str, FeedCandidate] = {}
direct = classify_text(base_url, text) if text else None
if direct:
found[base_url] = direct
root = parse_html(text) if text and not direct else None
queue: list[str] = []
if root is not None:
queue.extend(self.advertised(root, base_url))
queue.extend(self.activity_links(root, base_url))
context["login_required"] = context["login_required"] or self.looks_like_login(root)
for url in dict.fromkeys(queue):
if len(found) < LIMITS.max_candidates and url not in found and not COMMENT_FEED.search(urlsplit(url).path):
await self.consider(url, found, context)
if not found:
for path in FEED_PROBE_PATHS[: LIMITS.max_probe_paths]:
await self.consider(urljoin(origin, path), found, context)
if not found and root is not None:
candidate = await self.html_with_ai(base_url, text, root)
if candidate:
found[base_url] = candidate
if not found and not direct:
rendered = await self.rendered(base_url)
if rendered:
context["html"] = rendered
candidate = await self.html_with_ai(base_url, rendered, parse_html(rendered))
if candidate:
candidate.render = True
logger.info("detect: %s only renders with JavaScript, using the browser", base_url)
found[base_url] = candidate
return sorted(found.values(), key=self.preference), context
async def consider(self, url: str, found: dict, context: dict) -> None:
status, body = await self.fetch(url)
if status == 401:
context["login_required"] = True
if status >= 400 or not body:
return
candidate = classify_text(url, body)
if candidate is None and status < 300:
candidate = self.html_candidate(url, body)
if candidate:
logger.info("detect: %s feed at %s (%d sample items)", candidate.type, url, len(candidate.items))
found[url] = candidate
async def rendered(self, url: str) -> str:
try:
status, html = await self.fetcher.render_html(url)
except Exception as exc:
logger.info("detect: rendering %s failed: %s: %s", url, type(exc).__name__, exc)
return ""
return html if status < 400 else ""
async def html_with_ai(self, url: str, text: str, root: Node) -> FeedCandidate | None:
candidate = self.html_candidate(url, text)
if candidate or self.ai is None:
return candidate
selector = await self.ai.item_selector(root)
if not selector:
return None
return FeedCandidate(url, FEED_HTML, items=select_nodes(root, selector), document=root, item_selector=selector)
def preference(self, candidate: FeedCandidate) -> int:
order = {FEED_JSON: 0, FEED_ATOM: 1, FEED_RSS: 2, FEED_HTML: 3}
return order[candidate.type]
def advertised(self, root: Node, base_url: str) -> list[str]:
urls = []
for node in select_nodes(root, "link[rel=alternate]"):
kind = node.attrs.get("type", "").lower()
if any(marker in kind for marker in ("rss", "atom", "json")) and node.attrs.get("href"):
urls.append(urljoin(base_url, node.attrs["href"]))
return urls
def activity_links(self, root: Node, base_url: str) -> list[str]:
urls = []
host = urlsplit(base_url).hostname
for node in select_nodes(root, "a[href]"):
label = node.text()[:40]
target = urljoin(base_url, node.attrs["href"])
if urlsplit(target).hostname == host and ACTIVITY_LINK.search(label) and target != base_url:
urls.append(target)
return urls[: LIMITS.max_candidates]
def html_candidate(self, url: str, text: str) -> FeedCandidate | None:
if "<" not in text[:2000]:
return None
root = parse_html(text)
found = find_item_selector(root)
if found is None:
return None
selector, nodes = found
return FeedCandidate(url, FEED_HTML, items=nodes, document=root, item_selector=selector)
def looks_like_login(self, root: Node) -> bool:
if select_nodes(root, "input[type=password]") and not find_item_selector(root):
return True
return bool(LOGIN_WORDS.search(root.text()[:400])) and not find_item_selector(root)