332 lines
10 KiB
Python
332 lines
10 KiB
Python
# retoor <retoor@molodetz.nl>
|
||
|
||
import html
|
||
import json
|
||
import re
|
||
from functools import lru_cache
|
||
from html.parser import HTMLParser
|
||
from pathlib import Path
|
||
from urllib.parse import unquote, urlparse
|
||
|
||
import emoji
|
||
import mistune
|
||
from markupsafe import Markup
|
||
|
||
EMOJI_JS_PATH = Path(__file__).parent / "static" / "js" / "emoji-shortcodes.js"
|
||
|
||
CUSTOM_SHORTCODES = {
|
||
"party": "\U0001F973",
|
||
"tools": "\U0001F6E0️",
|
||
}
|
||
|
||
|
||
def build_emoji_shortcodes() -> dict[str, str]:
|
||
shortcodes: dict[str, str] = {}
|
||
for char, data in emoji.EMOJI_DATA.items():
|
||
if data.get("status", 0) > emoji.STATUS["fully_qualified"]:
|
||
continue
|
||
names = [data["en"]] if data.get("en") else []
|
||
names += data.get("alias", [])
|
||
for name in names:
|
||
shortcodes.setdefault(name.strip(":").lower(), char)
|
||
shortcodes.update(CUSTOM_SHORTCODES)
|
||
return shortcodes
|
||
|
||
|
||
def write_emoji_module(path: Path = EMOJI_JS_PATH) -> int:
|
||
shortcodes = build_emoji_shortcodes()
|
||
body = json.dumps(shortcodes, ensure_ascii=False, sort_keys=True)
|
||
path.write_text(
|
||
f"// retoor <retoor@molodetz.nl>\n\nexport const EMOJI_SHORTCODES = {body};\n",
|
||
encoding="utf-8",
|
||
)
|
||
return len(shortcodes)
|
||
|
||
|
||
EMOJI_MAP = build_emoji_shortcodes()
|
||
|
||
_WIDGET_RE = re.compile(r"<dp-widget>(.*?)</dp-widget>", re.DOTALL | re.IGNORECASE)
|
||
_WIDGET_PH = "\x00WIDGET_{}\x00"
|
||
|
||
_SHORTCODE_RE = re.compile(r":([A-Za-z0-9_+\-]+):")
|
||
_YOUTUBE_RE = re.compile(
|
||
r"(?:https?://)?(?:www\.)?"
|
||
r"(?:youtube\.com/(?:watch\?v=|embed/|v/|shorts/)|youtu\.be/)"
|
||
r"([A-Za-z0-9_-]{11})"
|
||
)
|
||
_IMAGE_RE = re.compile(r"\.(jpg|jpeg|png|gif|webp|svg|bmp)(\?.*)?$", re.I)
|
||
_VIDEO_RE = re.compile(r"\.(mp4|webm|ogg|ogv|mov|m4v)(\?.*)?$", re.I)
|
||
_AUDIO_RE = re.compile(r"\.(mp3|wav|flac|ogg|aac|m4a|wma|opus)(\?.*)?$", re.I)
|
||
_TOKEN_RE = re.compile(
|
||
r"(?P<url>https?://[^\s<]+)"
|
||
r"|(?<![A-Za-z0-9_@])@(?P<mention>[A-Za-z0-9_-]+)"
|
||
)
|
||
_SCHEME_RE = re.compile(r"^([a-z][a-z0-9+.\-]*):", re.I)
|
||
_ALLOWED_SCHEMES = {"http", "https", "mailto", "tel"}
|
||
|
||
_YOUTUBE_ALLOW = (
|
||
"accelerometer; autoplay; clipboard-write; encrypted-media; "
|
||
"gyroscope; picture-in-picture"
|
||
)
|
||
|
||
_EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b")
|
||
_EMAIL_KEEP_DOMAIN = "molodetz.nl"
|
||
|
||
_MEDIA_SKIP_TAGS = {"a", "code", "pre"}
|
||
_TITLE_INLINE_TAGS = {
|
||
"b", "strong", "i", "em", "code", "del", "s", "mark", "sub", "sup", "span", "br",
|
||
}
|
||
|
||
|
||
def _safe_url(url: str) -> str:
|
||
value = re.sub(r"[\x00-\x20]", "", url or "")
|
||
match = _SCHEME_RE.match(value)
|
||
if match and match.group(1).lower() not in _ALLOWED_SCHEMES:
|
||
return "#"
|
||
return value
|
||
|
||
|
||
class _ContentRenderer(mistune.HTMLRenderer):
|
||
def link(self, text: str, url: str, title=None) -> str:
|
||
return super().link(text, _safe_url(url), title)
|
||
|
||
def image(self, text: str, url: str, title=None) -> str:
|
||
src = html.escape(_safe_url(url), quote=True)
|
||
alt = html.escape(text or "", quote=True)
|
||
title_attr = f' title="{html.escape(title, quote=True)}"' if title else ""
|
||
return (
|
||
f'<img src="{src}" alt="{alt}"{title_attr} '
|
||
f'loading="lazy" data-lightbox>'
|
||
)
|
||
|
||
|
||
_content_markdown = mistune.create_markdown(
|
||
hard_wrap=True,
|
||
renderer=_ContentRenderer(escape=True),
|
||
plugins=["strikethrough", "table"],
|
||
)
|
||
|
||
|
||
def _normalize_dashes(text: str) -> str:
|
||
text = text.replace("\u2014", "-")
|
||
text = text.replace("\u2013", "-")
|
||
text = text.replace("—", "-")
|
||
text = text.replace("–", "-")
|
||
text = text.replace("—", "-")
|
||
text = text.replace("–", "-")
|
||
text = text.replace("—", "-")
|
||
text = text.replace("—", "-")
|
||
text = text.replace("–", "-")
|
||
text = text.replace("–", "-")
|
||
return text
|
||
|
||
|
||
def _replace_shortcodes(text: str) -> str:
|
||
return _SHORTCODE_RE.sub(
|
||
lambda m: EMOJI_MAP.get(m.group(1).lower(), m.group(0)), text
|
||
)
|
||
|
||
|
||
def _alt_from_url(url: str) -> str:
|
||
name = Path(unquote(urlparse(url).path)).stem.replace("-", " ").replace("_", " ").strip()
|
||
return name if name else "Embedded image"
|
||
|
||
|
||
def _embed_url(url: str) -> str:
|
||
youtube = _YOUTUBE_RE.search(url)
|
||
if youtube:
|
||
video_id = youtube.group(1)
|
||
return (
|
||
f'<div class="embed-youtube"><iframe '
|
||
f'src="https://www.youtube.com/embed/{video_id}" '
|
||
f'frameborder="0" allowfullscreen allow="{_YOUTUBE_ALLOW}">'
|
||
f"</iframe></div>"
|
||
)
|
||
escaped = html.escape(url, quote=True)
|
||
if _IMAGE_RE.search(url):
|
||
alt = html.escape(_alt_from_url(url), quote=True)
|
||
return f'<img src="{escaped}" alt="{alt}" loading="lazy" data-lightbox>'
|
||
if _VIDEO_RE.search(url):
|
||
return f'<video src="{escaped}" controls preload="metadata"></video>'
|
||
if _AUDIO_RE.search(url):
|
||
return f'<audio src="{escaped}" controls preload="metadata"></audio>'
|
||
return (
|
||
f'<a href="{escaped}" target="_blank" rel="noopener noreferrer">'
|
||
f"{html.escape(url)}</a>"
|
||
)
|
||
|
||
|
||
def _mask_email(match: re.Match) -> str:
|
||
email = match.group(0)
|
||
local, _, domain = email.partition("@")
|
||
lowered = domain.lower()
|
||
if lowered == _EMAIL_KEEP_DOMAIN or lowered.endswith("." + _EMAIL_KEEP_DOMAIN):
|
||
return email
|
||
reveal = max(1, len(local) - round(len(local) * 0.8))
|
||
return f"{local[:reveal]}{'*' * (len(local) - reveal)}@{domain}"
|
||
|
||
|
||
def _mask_emails(text: str) -> str:
|
||
return _EMAIL_RE.sub(_mask_email, text)
|
||
|
||
|
||
def _transform_text(text: str) -> str:
|
||
out: list[str] = []
|
||
pos = 0
|
||
for match in _TOKEN_RE.finditer(text):
|
||
if match.start() > pos:
|
||
out.append(html.escape(_mask_emails(text[pos:match.start()])))
|
||
if match.group("url"):
|
||
out.append(_embed_url(match.group("url")))
|
||
else:
|
||
user = match.group("mention")
|
||
out.append(
|
||
f'<a href="/profile/{html.escape(user, quote=True)}" '
|
||
f'class="mention-link">@{html.escape(user)}</a>'
|
||
)
|
||
pos = match.end()
|
||
if pos < len(text):
|
||
out.append(html.escape(_mask_emails(text[pos:])))
|
||
return "".join(out)
|
||
|
||
|
||
def _emit_tag(tag: str, attrs: list, self_closing: bool = False) -> str:
|
||
parts = [f"<{tag}"]
|
||
for key, value in attrs:
|
||
if value is None:
|
||
parts.append(f" {key}")
|
||
else:
|
||
parts.append(f' {key}="{html.escape(value, quote=True)}"')
|
||
parts.append(" />" if self_closing else ">")
|
||
return "".join(parts)
|
||
|
||
|
||
class _MediaProcessor(HTMLParser):
|
||
def __init__(self) -> None:
|
||
super().__init__(convert_charrefs=True)
|
||
self._out: list[str] = []
|
||
self._skip_depth = 0
|
||
|
||
def handle_starttag(self, tag: str, attrs: list) -> None:
|
||
self._out.append(_emit_tag(tag, attrs))
|
||
if tag in _MEDIA_SKIP_TAGS:
|
||
self._skip_depth += 1
|
||
|
||
def handle_startendtag(self, tag: str, attrs: list) -> None:
|
||
self._out.append(_emit_tag(tag, attrs, self_closing=True))
|
||
|
||
def handle_endtag(self, tag: str) -> None:
|
||
if tag in _MEDIA_SKIP_TAGS and self._skip_depth > 0:
|
||
self._skip_depth -= 1
|
||
self._out.append(f"</{tag}>")
|
||
|
||
def handle_data(self, data: str) -> None:
|
||
if self._skip_depth > 0:
|
||
self._out.append(html.escape(_mask_emails(data)))
|
||
else:
|
||
self._out.append(_transform_text(data))
|
||
|
||
def result(self) -> str:
|
||
return "".join(self._out)
|
||
|
||
|
||
class _InlineFilter(HTMLParser):
|
||
def __init__(self) -> None:
|
||
super().__init__(convert_charrefs=True)
|
||
self._out: list[str] = []
|
||
|
||
def handle_starttag(self, tag: str, attrs: list) -> None:
|
||
if tag in _TITLE_INLINE_TAGS:
|
||
self._out.append(f"<{tag}>")
|
||
|
||
def handle_startendtag(self, tag: str, attrs: list) -> None:
|
||
if tag in _TITLE_INLINE_TAGS:
|
||
self._out.append(f"<{tag}>")
|
||
|
||
def handle_endtag(self, tag: str) -> None:
|
||
if tag in _TITLE_INLINE_TAGS and tag != "br":
|
||
self._out.append(f"</{tag}>")
|
||
|
||
def handle_data(self, data: str) -> None:
|
||
self._out.append(html.escape(_mask_emails(data)))
|
||
|
||
def result(self) -> str:
|
||
return "".join(self._out).strip()
|
||
|
||
|
||
def _process_media(rendered: str) -> str:
|
||
processor = _MediaProcessor()
|
||
processor.feed(rendered)
|
||
processor.close()
|
||
return processor.result()
|
||
|
||
|
||
def _keep_inline(rendered: str) -> str:
|
||
inline = _InlineFilter()
|
||
inline.feed(rendered)
|
||
inline.close()
|
||
return inline.result()
|
||
|
||
|
||
@lru_cache(maxsize=4096)
|
||
def _render_content(text: str) -> str:
|
||
text = _normalize_dashes(text)
|
||
text = _replace_shortcodes(text)
|
||
return _process_media(_content_markdown(text))
|
||
|
||
|
||
@lru_cache(maxsize=4096)
|
||
def _render_title(text: str) -> str:
|
||
text = _normalize_dashes(text)
|
||
text = _replace_shortcodes(text)
|
||
return _keep_inline(_content_markdown(text))
|
||
|
||
|
||
def _extract_widgets(text: str) -> tuple[str, list[str]]:
|
||
widgets: list[str] = []
|
||
def _replacer(m: re.Match) -> str:
|
||
widgets.append(m.group(1))
|
||
return _WIDGET_PH.format(len(widgets) - 1)
|
||
return _WIDGET_RE.sub(_replacer, text), widgets
|
||
|
||
|
||
def _reinsert_widgets(text: str, widgets: list[str]) -> str:
|
||
for i, widget in enumerate(widgets):
|
||
text = text.replace(_WIDGET_PH.format(i), widget)
|
||
return text
|
||
|
||
|
||
def render_content(text, author_is_admin: bool = False) -> Markup:
|
||
if not text:
|
||
return Markup("")
|
||
text_str = str(text)
|
||
if author_is_admin and _WIDGET_RE.search(text_str):
|
||
modified, widgets = _extract_widgets(text_str)
|
||
rendered = _render_content(modified)
|
||
result = _reinsert_widgets(rendered, widgets)
|
||
return Markup(result)
|
||
return Markup(_render_content(text_str))
|
||
|
||
|
||
def render_title(text, author_is_admin: bool = False) -> Markup:
|
||
if not text:
|
||
return Markup("")
|
||
text_str = str(text)
|
||
if author_is_admin and _WIDGET_RE.search(text_str):
|
||
modified, widgets = _extract_widgets(text_str)
|
||
rendered = _render_title(modified)
|
||
result = _reinsert_widgets(rendered, widgets)
|
||
return Markup(result)
|
||
return Markup(_render_title(text_str))
|
||
|
||
|
||
def content_preview(text, length: int = 60) -> str:
|
||
if not text:
|
||
return ""
|
||
from devplacepy.utils import strip_html
|
||
|
||
plain = strip_html(_render_content(str(text)))
|
||
if len(plain) <= length:
|
||
return plain
|
||
return plain[:length].rstrip() + "..."
|