Files
devplacepy/devplacepy/rendering.py
T
retoorandClaude Sonnet 5.5 f705099928 Inline YouTube Music embeds with one shared embed spec
YouTube Music has no player of its own and music.youtube.com sends
X-Frame-Options SAMEORIGIN, so music links render YouTube's own player
with an Open in YouTube Music caption. Tracks, playlists and OLAK5uy
albums embed; browse, channel, search and library stay plain links.

- embeds.py is the single URL spec (exact host allowlists, query-order
  independent, validated ids only); embed-spec.js is generated from it by
  the embeds-sync CLI command and drift is tested
- rendering.py and ContentRenderer.js both follow it; the old duplicate
  YouTube regexes are removed
- max 3 music embeds per content item, overflow stays a plain link
- fixes evilyoutube.com matching and watch?feature=share&v=ID links
- fixes the invalid unicode CSS escape that printed u2197 after every
  external link in rendered content
- docs, privacy third-party row, parser table tests, server/client parity

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-01 11:53:55 +01:00

434 lines
13 KiB
Python
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# retoor <retoor@molodetz.nl>
import html
import json
import re
from functools import lru_cache
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import unquote, urlparse
import emoji
import mistune
from markupsafe import Markup
from devplacepy.embeds import EMBED_ALLOW, MAX_MUSIC_EMBEDS, MUSIC_CAPTION, EmbedParser
EMOJI_JS_PATH = Path(__file__).parent / "static" / "js" / "emoji-shortcodes.js"
CUSTOM_SHORTCODES = {
"party": "\U0001F973",
"tools": "\U0001F6E0️",
}
def build_emoji_shortcodes() -> dict[str, str]:
shortcodes: dict[str, str] = {}
for char, data in emoji.EMOJI_DATA.items():
if data.get("status", 0) > emoji.STATUS["fully_qualified"]:
continue
names = [data["en"]] if data.get("en") else []
names += data.get("alias", [])
for name in names:
shortcodes.setdefault(name.strip(":").lower(), char)
shortcodes.update(CUSTOM_SHORTCODES)
return shortcodes
def write_emoji_module(path: Path = EMOJI_JS_PATH) -> int:
shortcodes = build_emoji_shortcodes()
body = json.dumps(shortcodes, ensure_ascii=False, sort_keys=True)
path.write_text(
f"// retoor <retoor@molodetz.nl>\n\nexport const EMOJI_SHORTCODES = {body};\n",
encoding="utf-8",
)
return len(shortcodes)
EMOJI_MAP = build_emoji_shortcodes()
def is_single_emoji(value: str) -> bool:
text = (value or "").strip()
return emoji.emoji_count(text) == 1 and emoji.purely_emoji(text)
_WIDGET_RE = re.compile(r"<dp-widget>(.*?)</dp-widget>", re.DOTALL | re.IGNORECASE)
_WIDGET_PH = "\x00WIDGET_{}\x00"
_SHORTCODE_RE = re.compile(r":([A-Za-z0-9_+\-]+):")
_IMAGE_RE = re.compile(r"\.(jpg|jpeg|png|gif|webp|svg|bmp)(\?.*)?$", re.I)
_VIDEO_RE = re.compile(r"\.(mp4|webm|ogg|ogv|mov|m4v)(\?.*)?$", re.I)
_AUDIO_RE = re.compile(r"\.(mp3|wav|flac|ogg|aac|m4a|wma|opus)(\?.*)?$", re.I)
_TOKEN_RE = re.compile(
r"(?P<url>https?://[^\s<]+)"
r"|(?<![A-Za-z0-9_@])@(?P<mention>[A-Za-z0-9_-]+)"
)
_SCHEME_RE = re.compile(r"^([a-z][a-z0-9+.\-]*):", re.I)
_ALLOWED_SCHEMES = {"http", "https", "mailto", "tel"}
_TRAILING_PUNCT = ".,;:!?"
_TRAILING_CLOSERS = {")": "(", "]": "[", "}": "{"}
def _split_trailing_punct(url: str) -> tuple[str, str]:
trailing: list[str] = []
while url:
char = url[-1]
if char in _TRAILING_PUNCT:
trailing.append(char)
url = url[:-1]
continue
opener = _TRAILING_CLOSERS.get(char)
if opener is not None and url.count(char) > url.count(opener):
trailing.append(char)
url = url[:-1]
continue
break
return url, "".join(reversed(trailing))
_EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b")
_EMAIL_KEEP_DOMAIN = "molodetz.nl"
_MEDIA_SKIP_TAGS = {"a", "code", "pre"}
_TITLE_INLINE_TAGS = {
"b", "strong", "i", "em", "code", "del", "s", "mark", "sub", "sup", "span", "br",
}
def _safe_url(url: str) -> str:
value = re.sub(r"[\x00-\x20]", "", url or "")
match = _SCHEME_RE.match(value)
if match and match.group(1).lower() not in _ALLOWED_SCHEMES:
return "#"
return value
class _ContentRenderer(mistune.HTMLRenderer):
def link(self, text: str, url: str, title=None) -> str:
return super().link(text, _safe_url(url), title)
def image(self, text: str, url: str, title=None) -> str:
src = html.escape(_safe_url(url), quote=True)
alt = html.escape(text or "", quote=True)
title_attr = f' title="{html.escape(title, quote=True)}"' if title else ""
return (
f'<img src="{src}" alt="{alt}"{title_attr} '
f'loading="lazy" data-lightbox>'
)
_content_markdown = mistune.create_markdown(
hard_wrap=True,
renderer=_ContentRenderer(escape=True),
plugins=["strikethrough", "table"],
)
_structure_markdown = mistune.create_markdown(
renderer=None,
plugins=["strikethrough", "table"],
)
_STRUCTURE_TOKEN_KEYS = {
"block_code": "code_fences",
"list_item": "list_items",
"heading": "headers",
"link": "links",
}
def _count_structure_tokens(tokens, signature: dict[str, int]) -> None:
for token in tokens or []:
key = _STRUCTURE_TOKEN_KEYS.get(token.get("type"))
if key:
signature[key] += 1
children = token.get("children")
if children:
_count_structure_tokens(children, signature)
def markdown_structure_signature(text: str) -> dict[str, int]:
signature = {key: 0 for key in _STRUCTURE_TOKEN_KEYS.values()}
if not text or not text.strip():
return signature
try:
tokens = _structure_markdown(text)
except (TypeError, ValueError):
return signature
_count_structure_tokens(tokens, signature)
return signature
def _normalize_dashes(text: str) -> str:
text = text.replace("\u2014", "-")
text = text.replace("\u2013", "-")
text = text.replace("&mdash;", "-")
text = text.replace("&ndash;", "-")
text = text.replace("&#8212;", "-")
text = text.replace("&#8211;", "-")
text = text.replace("&#x2014;", "-")
text = text.replace("&#X2014;", "-")
text = text.replace("&#x2013;", "-")
text = text.replace("&#X2013;", "-")
return text
def _replace_shortcodes(text: str) -> str:
return _SHORTCODE_RE.sub(
lambda m: EMOJI_MAP.get(m.group(1).lower(), m.group(0)), text
)
def _alt_from_url(url: str) -> str:
name = Path(unquote(urlparse(url).path)).stem.replace("-", " ").replace("_", " ").strip()
return name if name else "Embedded image"
class _EmbedBudget:
def __init__(self) -> None:
self.music = 0
def allows(self, music: bool) -> bool:
if not music:
return True
if self.music >= MAX_MUSIC_EMBEDS:
return False
self.music += 1
return True
def _youtube_embed(url: str, budget: _EmbedBudget) -> str | None:
embed = EmbedParser.parse(url)
if embed is None or not budget.allows(embed.music):
return None
lazy = " loading=\"lazy\"" if embed.music else ""
frame = (
f'<div class="embed-youtube"><iframe '
f'src="{html.escape(EmbedParser.src(embed), quote=True)}" '
f'frameborder="0" allowfullscreen{lazy} allow="{EMBED_ALLOW}">'
f"</iframe></div>"
)
if not embed.music:
return frame
return (
f'<div class="embed-music">{frame}'
f'<a class="embed-music-caption" href="{html.escape(url, quote=True)}" '
f'target="_blank" rel="noopener noreferrer">{html.escape(MUSIC_CAPTION)}</a>'
f"</div>"
)
def _embed_url(url: str, budget: _EmbedBudget) -> str:
youtube = _youtube_embed(url, budget)
if youtube:
return youtube
escaped = html.escape(url, quote=True)
if _IMAGE_RE.search(url):
alt = html.escape(_alt_from_url(url), quote=True)
return f'<img src="{escaped}" alt="{alt}" loading="lazy" data-lightbox>'
if _VIDEO_RE.search(url):
return f'<video src="{escaped}" controls preload="metadata"></video>'
if _AUDIO_RE.search(url):
return f'<audio src="{escaped}" controls preload="metadata"></audio>'
return (
f'<a href="{escaped}" target="_blank" rel="noopener noreferrer">'
f"{html.escape(url)}</a>"
)
def _mask_email(match: re.Match) -> str:
email = match.group(0)
local, _, domain = email.partition("@")
lowered = domain.lower()
if lowered == _EMAIL_KEEP_DOMAIN or lowered.endswith("." + _EMAIL_KEEP_DOMAIN):
return email
reveal = max(1, len(local) - round(len(local) * 0.8))
return f"{local[:reveal]}{'*' * (len(local) - reveal)}@{domain}"
def _mask_emails(text: str) -> str:
return _EMAIL_RE.sub(_mask_email, text)
def _transform_text(text: str, budget: _EmbedBudget) -> str:
out: list[str] = []
pos = 0
for match in _TOKEN_RE.finditer(text):
if match.start() > pos:
out.append(html.escape(_mask_emails(text[pos:match.start()])))
if match.group("url"):
url, trailing = _split_trailing_punct(match.group("url"))
out.append(_embed_url(url, budget))
if trailing:
out.append(html.escape(trailing))
else:
user = match.group("mention")
out.append(
f'<a href="/profile/{html.escape(user, quote=True)}" '
f'class="mention-link">@{html.escape(user)}</a>'
)
pos = match.end()
if pos < len(text):
out.append(html.escape(_mask_emails(text[pos:])))
return "".join(out)
def _emit_tag(tag: str, attrs: list, self_closing: bool = False) -> str:
parts = [f"<{tag}"]
for key, value in attrs:
if value is None:
parts.append(f" {key}")
else:
parts.append(f' {key}="{html.escape(value, quote=True)}"')
parts.append(" />" if self_closing else ">")
return "".join(parts)
class _MediaProcessor(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self._out: list[str] = []
self._skip_depth = 0
self._budget = _EmbedBudget()
def handle_starttag(self, tag: str, attrs: list) -> None:
self._out.append(_emit_tag(tag, attrs))
if tag in _MEDIA_SKIP_TAGS:
self._skip_depth += 1
def handle_startendtag(self, tag: str, attrs: list) -> None:
self._out.append(_emit_tag(tag, attrs, self_closing=True))
def handle_endtag(self, tag: str) -> None:
if tag in _MEDIA_SKIP_TAGS and self._skip_depth > 0:
self._skip_depth -= 1
self._out.append(f"</{tag}>")
def handle_data(self, data: str) -> None:
if self._skip_depth > 0:
self._out.append(html.escape(_mask_emails(data)))
else:
self._out.append(_transform_text(data, self._budget))
def result(self) -> str:
return "".join(self._out)
class _InlineFilter(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self._out: list[str] = []
def handle_starttag(self, tag: str, attrs: list) -> None:
if tag in _TITLE_INLINE_TAGS:
self._out.append(f"<{tag}>")
def handle_startendtag(self, tag: str, attrs: list) -> None:
if tag in _TITLE_INLINE_TAGS:
self._out.append(f"<{tag}>")
def handle_endtag(self, tag: str) -> None:
if tag in _TITLE_INLINE_TAGS and tag != "br":
self._out.append(f"</{tag}>")
def handle_data(self, data: str) -> None:
self._out.append(html.escape(_mask_emails(data)))
def result(self) -> str:
return "".join(self._out).strip()
def _process_media(rendered: str) -> str:
processor = _MediaProcessor()
processor.feed(rendered)
processor.close()
return processor.result()
def _keep_inline(rendered: str) -> str:
inline = _InlineFilter()
inline.feed(rendered)
inline.close()
return inline.result()
@lru_cache(maxsize=4096)
def _render_content(text: str) -> str:
text = _normalize_dashes(text)
text = _replace_shortcodes(text)
return _process_media(_content_markdown(text))
@lru_cache(maxsize=4096)
def _render_title(text: str) -> str:
text = _normalize_dashes(text)
text = _replace_shortcodes(text)
return _keep_inline(_content_markdown(text))
def _extract_widgets(text: str) -> tuple[str, list[str]]:
widgets: list[str] = []
def _replacer(m: re.Match) -> str:
widgets.append(m.group(1))
return _WIDGET_PH.format(len(widgets) - 1)
return _WIDGET_RE.sub(_replacer, text), widgets
def _reinsert_widgets(text: str, widgets: list[str]) -> str:
for i, widget in enumerate(widgets):
text = text.replace(_WIDGET_PH.format(i), widget)
return text
def render_content(text, author_is_admin: bool = False) -> Markup:
if not text:
return Markup("")
text_str = str(text)
if author_is_admin and _WIDGET_RE.search(text_str):
modified, widgets = _extract_widgets(text_str)
rendered = _render_content(modified)
result = _reinsert_widgets(rendered, widgets)
return Markup(result)
return Markup(_render_content(text_str))
def render_title(text, author_is_admin: bool = False) -> Markup:
if not text:
return Markup("")
text_str = str(text)
if author_is_admin and _WIDGET_RE.search(text_str):
modified, widgets = _extract_widgets(text_str)
rendered = _render_title(modified)
result = _reinsert_widgets(rendered, widgets)
return Markup(result)
return Markup(_render_title(text_str))
def content_preview(text, length: int = 60) -> str:
if not text:
return ""
from devplacepy.utils import strip_html
plain = strip_html(_render_content(str(text)))
if len(plain) <= length:
return plain
return plain[:length].rstrip() + "..."
def safe_truncate(text, length: int = 300) -> str:
if not text:
return ""
text_str = str(text)
if len(text_str) <= length:
return text_str
cutoff = length
for match in _TOKEN_RE.finditer(text_str):
if match.start() < cutoff < match.end():
cutoff = match.start()
break
if cutoff > 0 and not text_str[cutoff - 1].isspace() and not text_str[cutoff].isspace():
boundary = text_str.rfind(" ", 0, cutoff)
if boundary > 0:
cutoff = boundary
return text_str[:cutoff].rstrip()