263 lines
10 KiB
Python
263 lines
10 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import re
|
|
import statistics
|
|
from dataclasses import dataclass, field
|
|
from urllib.parse import urljoin, urlsplit
|
|
|
|
from devplacepy.services.community.extract import (
|
|
Node,
|
|
json_or_none,
|
|
parse_feed,
|
|
parse_html,
|
|
select_nodes,
|
|
)
|
|
from devplacepy.services.community.spec import (
|
|
FEED_ATOM,
|
|
FEED_HTML,
|
|
FEED_JSON,
|
|
FEED_PROBE_PATHS,
|
|
FEED_RSS,
|
|
LIMITS,
|
|
)
|
|
|
|
logger = logging.getLogger("community.feeds")
|
|
|
|
ITEM_KEYS = re.compile(
|
|
r"^(title|name|subject|headline|content|body|text|description|summary|id|uid|guid|url|link|"
|
|
r"created_at|created|published|date|author|user|username|creator|content_html|excerpt)$",
|
|
re.I,
|
|
)
|
|
ITEM_HINT = re.compile(r"post|thread|topic|item|entry|article|story|rant|card|result|row", re.I)
|
|
ACTIVITY_LINK = re.compile(r"\b(latest|recent|new|activity|posts|threads|topics|discussions|feed)\b", re.I)
|
|
LOGIN_WORDS = re.compile(r"(sign in|log in|login|password)", re.I)
|
|
MIN_REPEAT = 3
|
|
COMMENT_FEED = re.compile(r"/(comments?|repl(?:y|ies)|responses?)(?:[./_-][\w.]*)?/?$", re.I)
|
|
|
|
|
|
@dataclass
|
|
class FeedCandidate:
|
|
url: str
|
|
type: str
|
|
items: list = field(default_factory=list)
|
|
document: object = None
|
|
list_path: str | None = None
|
|
item_selector: str | None = None
|
|
login_required: bool = False
|
|
render: bool = False
|
|
|
|
|
|
def item_score(item) -> int:
|
|
if not isinstance(item, dict):
|
|
return 0
|
|
return sum(1 for key in item if ITEM_KEYS.match(str(key).lstrip("@")))
|
|
|
|
|
|
def find_item_list(document, path: str = "$", depth: int = 0):
|
|
best = (0, None, None)
|
|
if depth > 3:
|
|
return best
|
|
if isinstance(document, list):
|
|
dicts = [entry for entry in document if isinstance(entry, dict)]
|
|
if len(dicts) >= 2:
|
|
score = statistics.median(item_score(entry) for entry in dicts[:8]) * min(len(dicts), 20)
|
|
best = (score, path, document)
|
|
return best
|
|
if isinstance(document, dict):
|
|
for key, value in document.items():
|
|
if isinstance(value, (list, dict)):
|
|
found = find_item_list(value, f"{path}.{key}", depth + 1)
|
|
if found[0] > best[0]:
|
|
best = found
|
|
return best
|
|
|
|
|
|
def find_item_selector(root: Node) -> tuple[str, list[Node]] | None:
|
|
best_score = 0.0
|
|
best: tuple[str, list[Node]] | None = None
|
|
groups: dict[tuple, list[Node]] = {}
|
|
for node in root.descendants():
|
|
if node.tag in ("script", "style", "nav", "header", "footer", "select", "option", "a", "span", "td", "th"):
|
|
continue
|
|
key = (node.tag, tuple(sorted(node.classes())))
|
|
groups.setdefault(key, []).append(node)
|
|
for (tag, classes), nodes in groups.items():
|
|
if len(nodes) < MIN_REPEAT:
|
|
continue
|
|
linked = [node for node in nodes if any(child.tag == "a" and child.attrs.get("href") for child in node.descendants())]
|
|
if len(linked) < MIN_REPEAT:
|
|
continue
|
|
lengths = [len(node.text()) for node in linked]
|
|
median = statistics.median(lengths)
|
|
if median < 25:
|
|
continue
|
|
selector = tag + "".join(f".{name}" for name in classes)
|
|
hint = 2.0 if ITEM_HINT.search(selector) else 1.0
|
|
score = min(len(linked), 30) * min(median, 400) * hint
|
|
if score > best_score:
|
|
best_score = score
|
|
best = (selector, linked)
|
|
return best
|
|
|
|
|
|
def classify_text(url: str, text: str) -> FeedCandidate | None:
|
|
stripped = text.lstrip()
|
|
if stripped.startswith(("<?xml", "<rss", "<feed", "<rdf")):
|
|
try:
|
|
kind, _, items = parse_feed(text)
|
|
except Exception as exc:
|
|
logger.info("feed %s looked like XML but did not parse: %s", url, exc)
|
|
return None
|
|
if len(items) >= 1:
|
|
return FeedCandidate(url, FEED_RSS if kind == "rss" else FEED_ATOM, items=items)
|
|
return None
|
|
if stripped[:1] in "[{":
|
|
document = json_or_none(text)
|
|
if document is None:
|
|
return None
|
|
score, path, items = find_item_list(document)
|
|
if path and score >= 4:
|
|
return FeedCandidate(url, FEED_JSON, items=items, document=document, list_path=path)
|
|
return None
|
|
return None
|
|
|
|
|
|
COMMENT_HINT = re.compile(r"comment|repl(?:y|ies)|response|answer", re.I)
|
|
|
|
|
|
def find_comment_selector(roots: list[Node]) -> str | None:
|
|
tallies: dict[str, int] = {}
|
|
for root in roots:
|
|
for node in root.descendants():
|
|
if node.tag in ("script", "style", "a", "span", "form", "textarea", "input", "button", "ul", "ol", "section", "nav"):
|
|
continue
|
|
hinted = sorted(name for name in node.classes() if COMMENT_HINT.search(name) and not re.search(r"form|count|list|wrap|section|header|input|box|area|title|link|button", name, re.I))
|
|
if not hinted:
|
|
continue
|
|
selector = f"{node.tag}.{hinted[0]}"
|
|
if len(node.text()) < 4:
|
|
continue
|
|
tallies[selector] = tallies.get(selector, 0) + 1
|
|
ranked = sorted(tallies.items(), key=lambda entry: entry[1], reverse=True)
|
|
return ranked[0][0] if ranked and ranked[0][1] >= 2 else None
|
|
|
|
|
|
class FeedDetector:
|
|
def __init__(self, fetcher, ai=None):
|
|
self.fetcher = fetcher
|
|
self.ai = ai
|
|
|
|
async def fetch(self, url: str):
|
|
try:
|
|
response = await self.fetcher.get(url)
|
|
return response.status, await response.text()
|
|
except Exception as exc:
|
|
logger.warning("detect: %s unreachable: %s: %s", url, type(exc).__name__, exc)
|
|
return 0, ""
|
|
|
|
async def detect(self, base_url: str) -> tuple[list[FeedCandidate], dict]:
|
|
parts = urlsplit(base_url)
|
|
origin = f"{parts.scheme}://{parts.netloc}"
|
|
status, text = await self.fetch(base_url)
|
|
context = {"status": status, "html": text, "login_required": status in (401, 403)}
|
|
found: dict[str, FeedCandidate] = {}
|
|
direct = classify_text(base_url, text) if text else None
|
|
if direct:
|
|
found[base_url] = direct
|
|
root = parse_html(text) if text and not direct else None
|
|
queue: list[str] = []
|
|
if root is not None:
|
|
queue.extend(self.advertised(root, base_url))
|
|
queue.extend(self.activity_links(root, base_url))
|
|
context["login_required"] = context["login_required"] or self.looks_like_login(root)
|
|
for url in dict.fromkeys(queue):
|
|
if len(found) < LIMITS.max_candidates and url not in found and not COMMENT_FEED.search(urlsplit(url).path):
|
|
await self.consider(url, found, context)
|
|
if not found:
|
|
for path in FEED_PROBE_PATHS[: LIMITS.max_probe_paths]:
|
|
await self.consider(urljoin(origin, path), found, context)
|
|
if not found and root is not None:
|
|
candidate = await self.html_with_ai(base_url, text, root)
|
|
if candidate:
|
|
found[base_url] = candidate
|
|
if not found and not direct:
|
|
rendered = await self.rendered(base_url)
|
|
if rendered:
|
|
context["html"] = rendered
|
|
candidate = await self.html_with_ai(base_url, rendered, parse_html(rendered))
|
|
if candidate:
|
|
candidate.render = True
|
|
logger.info("detect: %s only renders with JavaScript, using the browser", base_url)
|
|
found[base_url] = candidate
|
|
return sorted(found.values(), key=self.preference), context
|
|
|
|
async def consider(self, url: str, found: dict, context: dict) -> None:
|
|
status, body = await self.fetch(url)
|
|
if status == 401:
|
|
context["login_required"] = True
|
|
if status >= 400 or not body:
|
|
return
|
|
candidate = classify_text(url, body)
|
|
if candidate is None and status < 300:
|
|
candidate = self.html_candidate(url, body)
|
|
if candidate:
|
|
logger.info("detect: %s feed at %s (%d sample items)", candidate.type, url, len(candidate.items))
|
|
found[url] = candidate
|
|
|
|
async def rendered(self, url: str) -> str:
|
|
try:
|
|
status, html = await self.fetcher.render_html(url)
|
|
except Exception as exc:
|
|
logger.info("detect: rendering %s failed: %s: %s", url, type(exc).__name__, exc)
|
|
return ""
|
|
return html if status < 400 else ""
|
|
|
|
async def html_with_ai(self, url: str, text: str, root: Node) -> FeedCandidate | None:
|
|
candidate = self.html_candidate(url, text)
|
|
if candidate or self.ai is None:
|
|
return candidate
|
|
selector = await self.ai.item_selector(root)
|
|
if not selector:
|
|
return None
|
|
return FeedCandidate(url, FEED_HTML, items=select_nodes(root, selector), document=root, item_selector=selector)
|
|
|
|
def preference(self, candidate: FeedCandidate) -> int:
|
|
order = {FEED_JSON: 0, FEED_ATOM: 1, FEED_RSS: 2, FEED_HTML: 3}
|
|
return order[candidate.type]
|
|
|
|
def advertised(self, root: Node, base_url: str) -> list[str]:
|
|
urls = []
|
|
for node in select_nodes(root, "link[rel=alternate]"):
|
|
kind = node.attrs.get("type", "").lower()
|
|
if any(marker in kind for marker in ("rss", "atom", "json")) and node.attrs.get("href"):
|
|
urls.append(urljoin(base_url, node.attrs["href"]))
|
|
return urls
|
|
|
|
def activity_links(self, root: Node, base_url: str) -> list[str]:
|
|
urls = []
|
|
host = urlsplit(base_url).hostname
|
|
for node in select_nodes(root, "a[href]"):
|
|
label = node.text()[:40]
|
|
target = urljoin(base_url, node.attrs["href"])
|
|
if urlsplit(target).hostname == host and ACTIVITY_LINK.search(label) and target != base_url:
|
|
urls.append(target)
|
|
return urls[: LIMITS.max_candidates]
|
|
|
|
def html_candidate(self, url: str, text: str) -> FeedCandidate | None:
|
|
if "<" not in text[:2000]:
|
|
return None
|
|
root = parse_html(text)
|
|
found = find_item_selector(root)
|
|
if found is None:
|
|
return None
|
|
selector, nodes = found
|
|
return FeedCandidate(url, FEED_HTML, items=nodes, document=root, item_selector=selector)
|
|
|
|
def looks_like_login(self, root: Node) -> bool:
|
|
if select_nodes(root, "input[type=password]") and not find_item_selector(root):
|
|
return True
|
|
return bool(LOGIN_WORDS.search(root.text()[:400])) and not find_item_selector(root)
|