# retoor import html import json import re from functools import lru_cache from html.parser import HTMLParser from pathlib import Path from urllib.parse import unquote, urlparse import emoji import mistune from markupsafe import Markup EMOJI_JS_PATH = Path(__file__).parent / "static" / "js" / "emoji-shortcodes.js" CUSTOM_SHORTCODES = { "party": "\U0001F973", "tools": "\U0001F6E0️", } def build_emoji_shortcodes() -> dict[str, str]: shortcodes: dict[str, str] = {} for char, data in emoji.EMOJI_DATA.items(): if data.get("status", 0) > emoji.STATUS["fully_qualified"]: continue names = [data["en"]] if data.get("en") else [] names += data.get("alias", []) for name in names: shortcodes.setdefault(name.strip(":").lower(), char) shortcodes.update(CUSTOM_SHORTCODES) return shortcodes def write_emoji_module(path: Path = EMOJI_JS_PATH) -> int: shortcodes = build_emoji_shortcodes() body = json.dumps(shortcodes, ensure_ascii=False, sort_keys=True) path.write_text( f"// retoor \n\nexport const EMOJI_SHORTCODES = {body};\n", encoding="utf-8", ) return len(shortcodes) EMOJI_MAP = build_emoji_shortcodes() _WIDGET_RE = re.compile(r"(.*?)", re.DOTALL | re.IGNORECASE) _WIDGET_PH = "\x00WIDGET_{}\x00" _SHORTCODE_RE = re.compile(r":([A-Za-z0-9_+\-]+):") _YOUTUBE_RE = re.compile( r"(?:https?://)?(?:www\.)?" r"(?:youtube\.com/(?:watch\?v=|embed/|v/|shorts/)|youtu\.be/)" r"([A-Za-z0-9_-]{11})" ) _IMAGE_RE = re.compile(r"\.(jpg|jpeg|png|gif|webp|svg|bmp)(\?.*)?$", re.I) _VIDEO_RE = re.compile(r"\.(mp4|webm|ogg|ogv|mov|m4v)(\?.*)?$", re.I) _AUDIO_RE = re.compile(r"\.(mp3|wav|flac|ogg|aac|m4a|wma|opus)(\?.*)?$", re.I) _TOKEN_RE = re.compile( r"(?Phttps?://[^\s<]+)" r"|(?[A-Za-z0-9_-]+)" ) _SCHEME_RE = re.compile(r"^([a-z][a-z0-9+.\-]*):", re.I) _ALLOWED_SCHEMES = {"http", "https", "mailto", "tel"} _YOUTUBE_ALLOW = ( "accelerometer; autoplay; clipboard-write; encrypted-media; " "gyroscope; picture-in-picture" ) _EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b") _EMAIL_KEEP_DOMAIN = "molodetz.nl" _MEDIA_SKIP_TAGS = {"a", "code", "pre"} _TITLE_INLINE_TAGS = { "b", "strong", "i", "em", "code", "del", "s", "mark", "sub", "sup", "span", "br", } def _safe_url(url: str) -> str: value = re.sub(r"[\x00-\x20]", "", url or "") match = _SCHEME_RE.match(value) if match and match.group(1).lower() not in _ALLOWED_SCHEMES: return "#" return value class _ContentRenderer(mistune.HTMLRenderer): def link(self, text: str, url: str, title=None) -> str: return super().link(text, _safe_url(url), title) def image(self, text: str, url: str, title=None) -> str: src = html.escape(_safe_url(url), quote=True) alt = html.escape(text or "", quote=True) title_attr = f' title="{html.escape(title, quote=True)}"' if title else "" return ( f'{alt}' ) _content_markdown = mistune.create_markdown( hard_wrap=True, renderer=_ContentRenderer(escape=True), plugins=["strikethrough", "table"], ) def _normalize_dashes(text: str) -> str: text = text.replace("\u2014", "-") text = text.replace("\u2013", "-") text = text.replace("—", "-") text = text.replace("–", "-") text = text.replace("—", "-") text = text.replace("–", "-") text = text.replace("—", "-") text = text.replace("—", "-") text = text.replace("–", "-") text = text.replace("–", "-") return text def _replace_shortcodes(text: str) -> str: return _SHORTCODE_RE.sub( lambda m: EMOJI_MAP.get(m.group(1).lower(), m.group(0)), text ) def _alt_from_url(url: str) -> str: name = Path(unquote(urlparse(url).path)).stem.replace("-", " ").replace("_", " ").strip() return name if name else "Embedded image" def _embed_url(url: str) -> str: youtube = _YOUTUBE_RE.search(url) if youtube: video_id = youtube.group(1) return ( f'
" ) escaped = html.escape(url, quote=True) if _IMAGE_RE.search(url): alt = html.escape(_alt_from_url(url), quote=True) return f'{alt}' if _VIDEO_RE.search(url): return f'' if _AUDIO_RE.search(url): return f'' return ( f'' f"{html.escape(url)}" ) def _mask_email(match: re.Match) -> str: email = match.group(0) local, _, domain = email.partition("@") lowered = domain.lower() if lowered == _EMAIL_KEEP_DOMAIN or lowered.endswith("." + _EMAIL_KEEP_DOMAIN): return email reveal = max(1, len(local) - round(len(local) * 0.8)) return f"{local[:reveal]}{'*' * (len(local) - reveal)}@{domain}" def _mask_emails(text: str) -> str: return _EMAIL_RE.sub(_mask_email, text) def _transform_text(text: str) -> str: out: list[str] = [] pos = 0 for match in _TOKEN_RE.finditer(text): if match.start() > pos: out.append(html.escape(_mask_emails(text[pos:match.start()]))) if match.group("url"): out.append(_embed_url(match.group("url"))) else: user = match.group("mention") out.append( f'@{html.escape(user)}' ) pos = match.end() if pos < len(text): out.append(html.escape(_mask_emails(text[pos:]))) return "".join(out) def _emit_tag(tag: str, attrs: list, self_closing: bool = False) -> str: parts = [f"<{tag}"] for key, value in attrs: if value is None: parts.append(f" {key}") else: parts.append(f' {key}="{html.escape(value, quote=True)}"') parts.append(" />" if self_closing else ">") return "".join(parts) class _MediaProcessor(HTMLParser): def __init__(self) -> None: super().__init__(convert_charrefs=True) self._out: list[str] = [] self._skip_depth = 0 def handle_starttag(self, tag: str, attrs: list) -> None: self._out.append(_emit_tag(tag, attrs)) if tag in _MEDIA_SKIP_TAGS: self._skip_depth += 1 def handle_startendtag(self, tag: str, attrs: list) -> None: self._out.append(_emit_tag(tag, attrs, self_closing=True)) def handle_endtag(self, tag: str) -> None: if tag in _MEDIA_SKIP_TAGS and self._skip_depth > 0: self._skip_depth -= 1 self._out.append(f"") def handle_data(self, data: str) -> None: if self._skip_depth > 0: self._out.append(html.escape(_mask_emails(data))) else: self._out.append(_transform_text(data)) def result(self) -> str: return "".join(self._out) class _InlineFilter(HTMLParser): def __init__(self) -> None: super().__init__(convert_charrefs=True) self._out: list[str] = [] def handle_starttag(self, tag: str, attrs: list) -> None: if tag in _TITLE_INLINE_TAGS: self._out.append(f"<{tag}>") def handle_startendtag(self, tag: str, attrs: list) -> None: if tag in _TITLE_INLINE_TAGS: self._out.append(f"<{tag}>") def handle_endtag(self, tag: str) -> None: if tag in _TITLE_INLINE_TAGS and tag != "br": self._out.append(f"") def handle_data(self, data: str) -> None: self._out.append(html.escape(_mask_emails(data))) def result(self) -> str: return "".join(self._out).strip() def _process_media(rendered: str) -> str: processor = _MediaProcessor() processor.feed(rendered) processor.close() return processor.result() def _keep_inline(rendered: str) -> str: inline = _InlineFilter() inline.feed(rendered) inline.close() return inline.result() @lru_cache(maxsize=4096) def _render_content(text: str) -> str: text = _normalize_dashes(text) text = _replace_shortcodes(text) return _process_media(_content_markdown(text)) @lru_cache(maxsize=4096) def _render_title(text: str) -> str: text = _normalize_dashes(text) text = _replace_shortcodes(text) return _keep_inline(_content_markdown(text)) def _extract_widgets(text: str) -> tuple[str, list[str]]: widgets: list[str] = [] def _replacer(m: re.Match) -> str: widgets.append(m.group(1)) return _WIDGET_PH.format(len(widgets) - 1) return _WIDGET_RE.sub(_replacer, text), widgets def _reinsert_widgets(text: str, widgets: list[str]) -> str: for i, widget in enumerate(widgets): text = text.replace(_WIDGET_PH.format(i), widget) return text def render_content(text, author_is_admin: bool = False) -> Markup: if not text: return Markup("") text_str = str(text) if author_is_admin and _WIDGET_RE.search(text_str): modified, widgets = _extract_widgets(text_str) rendered = _render_content(modified) result = _reinsert_widgets(rendered, widgets) return Markup(result) return Markup(_render_content(text_str)) def render_title(text, author_is_admin: bool = False) -> Markup: if not text: return Markup("") text_str = str(text) if author_is_admin and _WIDGET_RE.search(text_str): modified, widgets = _extract_widgets(text_str) rendered = _render_title(modified) result = _reinsert_widgets(rendered, widgets) return Markup(result) return Markup(_render_title(text_str)) def content_preview(text, length: int = 60) -> str: if not text: return "" from devplacepy.utils import strip_html plain = strip_html(_render_content(str(text))) if len(plain) <= length: return plain return plain[:length].rstrip() + "..."