devplacepy/devplacepy/seo.py

# retoor <retoor@molodetz.nl>

import json
import logging
import os
import time
from urllib.parse import urlencode
from xml.etree.ElementTree import Element, tostring
from xml.dom import minidom
from devplacepy.config import SITE_URL
from devplacepy.utils import strip_html

logger = logging.getLogger(__name__)


SITE_NAME = "DevPlace"
SITEMAP_URL_LIMIT = 5000

LEGAL_DOC_SLUGS = (
    "terms",
    "community-guidelines",
    "privacy",
    "content-moderation",
    "intellectual-property",
    "contact",
)
SITEMAP_TTL = int(os.environ.get("DEVPLACE_SITEMAP_TTL", "3600"))
_sitemap_cache = {}


def plain_markdown(text: str) -> str:
    if not text:
        return ""
    from devplacepy.seo_meta_text import plain_text_from_markdown

    return plain_text_from_markdown(text)


def truncate(text: str, max_len: int = 160) -> str:
    if not text:
        return ""
    text = " ".join(text.split())
    if len(text) <= max_len:
        return text
    text = text[: max_len - 3].rsplit(" ", 1)[0]
    return text + "..."


def public_base_url() -> str:
    from devplacepy.database import get_setting

    return (get_setting("site_url", "").strip() or SITE_URL or "").rstrip("/")


def site_url(request):
    return public_base_url() or str(request.base_url).rstrip("/")


def website_schema(base_url):
    return {
        "@type": "WebSite",
        "name": SITE_NAME,
        "url": base_url,
        "potentialAction": {
            "@type": "SearchAction",
            "target": {
                "@type": "EntryPoint",
                "urlTemplate": f"{base_url}/feed?search={{query}}",
            },
            "query-input": "required name=query",
        },
    }


def breadcrumb_schema(items, base_url):
    return {
        "@type": "BreadcrumbList",
        "itemListElement": [
            {
                "@type": "ListItem",
                "position": i + 1,
                "item": {
                    "@id": item["url"]
                    if item["url"].startswith("http")
                    else f"{base_url}{item['url']}",
                    "name": item["name"],
                },
            }
            for i, item in enumerate(items)
        ],
    }


def discussion_forum_posting(post, author, comment_count, star_count, base_url):
    schema = {
        "@type": "DiscussionForumPosting",
        "headline": post.get("title") or "Untitled",
        "text": truncate(plain_markdown(post.get("content", "")), 500),
        "url": f"{base_url}/posts/{post.get('slug') or post['uid']}",
        "author": {
            "@type": "Person",
            "name": author["username"] if author else "Unknown",
            "url": f"{base_url}/profile/{author['username']}" if author else "",
        },
        "datePublished": post.get("created_at", ""),
        "dateModified": post.get("updated_at") or post.get("created_at", ""),
        "interactionStatistic": [
            {
                "@type": "InteractionCounter",
                "interactionType": "https://schema.org/LikeAction",
                "userInteractionCount": star_count,
            },
            {
                "@type": "InteractionCounter",
                "interactionType": "https://schema.org/CommentAction",
                "userInteractionCount": comment_count,
            },
        ],
    }
    return schema


def profile_page_schema(profile_user, post_count, base_url):
    return {
        "@type": "ProfilePage",
        "mainEntity": {
            "@type": "Person",
            "name": profile_user.get("username", ""),
            "alternateName": profile_user.get("username", ""),
            "description": profile_user.get("bio", "") or f"Developer on {SITE_NAME}",
            "interactionStatistic": [
                {
                    "@type": "InteractionCounter",
                    "interactionType": "https://schema.org/WriteAction",
                    "userInteractionCount": post_count,
                }
            ],
        },
    }


def software_application_schema(project, base_url):
    return {
        "@type": "SoftwareApplication",
        "name": project.get("title", "Untitled"),
        "description": truncate(plain_markdown(project.get("description", "")), 300),
        "url": f"{base_url}/projects/{project.get('slug') or project['uid']}",
        "applicationCategory": "DeveloperApplication",
        "operatingSystem": project.get("platforms", "Cross-platform"),
        "author": {"@type": "Person", "name": project.get("author_name", "Unknown")},
        "datePublished": project.get("created_at", ""),
        "dateModified": project.get("updated_at") or project.get("created_at", ""),
        "offers": {"@type": "Offer", "price": "0", "priceCurrency": "USD"},
    }


def web_application_schema(name, description, path, base_url, category="DeveloperApplication"):
    return {
        "@type": "WebApplication",
        "name": name,
        "description": truncate(description, 300),
        "url": f"{base_url}{path}",
        "applicationCategory": category,
        "operatingSystem": "All",
        "browserRequirements": "Requires JavaScript",
        "offers": {"@type": "Offer", "price": "0", "priceCurrency": "USD"},
        "provider": {"@type": "Organization", "name": SITE_NAME, "url": base_url},
    }


def organization_schema(base_url):
    return {
        "@type": "Organization",
        "name": SITE_NAME,
        "url": base_url,
        "logo": f"{base_url}{DEFAULT_OG_IMAGE}",
    }


def news_article_schema(article, base_url, image_url=""):
    url = f"{base_url}/news/{article.get('slug') or article['uid']}"
    schema = {
        "@type": "NewsArticle",
        "headline": (article.get("title") or "Untitled")[:110],
        "description": truncate(plain_markdown(article.get("description", "") or ""), 200),
        "url": url,
        "datePublished": article.get("synced_at", "") or article.get("created_at", ""),
        "dateModified": article.get("synced_at", "") or article.get("created_at", ""),
        "mainEntityOfPage": {"@type": "WebPage", "@id": url},
        "author": {
            "@type": "Organization",
            "name": article.get("source_name") or SITE_NAME,
        },
        "publisher": organization_schema(base_url),
    }
    if image_url:
        schema["image"] = image_url
    return schema


def software_source_code_schema(gist, base_url):
    return {
        "@type": "SoftwareSourceCode",
        "name": gist.get("title") or "Gist",
        "description": truncate(plain_markdown(gist.get("description", "") or ""), 200),
        "url": f"{base_url}/gists/{gist.get('slug') or gist['uid']}",
        "programmingLanguage": gist.get("language", "") or "text",
        "dateCreated": gist.get("created_at", ""),
        "dateModified": gist.get("updated_at") or gist.get("created_at", ""),
    }


def quiz_schema(quiz, author, base_url):
    return {
        "@type": "Quiz",
        "name": quiz.get("title") or "Quiz",
        "description": truncate(plain_markdown(quiz.get("description", "") or ""), 200),
        "url": f"{base_url}/quizzes/{quiz.get('slug') or quiz['uid']}",
        "about": quiz.get("title") or "Quiz",
        "educationalLevel": "beginner",
        "numberOfQuestions": int(quiz.get("question_count") or 0),
        "author": {
            "@type": "Person",
            "name": (author or {}).get("username", "") or "DevPlace member",
            "url": f"{base_url}/profile/{(author or {}).get('username', '')}",
        },
        "dateCreated": quiz.get("created_at", ""),
    }


def _json_ld_dumps(payload):
    raw = json.dumps(payload, ensure_ascii=False)
    return (
        raw.replace("<", "\\u003c")
        .replace(">", "\\u003e")
        .replace("&", "\\u0026")
        .replace("
", "\\u2028")
        .replace("
", "\\u2029")
    )


def combine(schemas):
    if not schemas:
        return None
    cleaned = []
    for s in schemas:
        if s is not None:
            s.pop("@context", None)
            cleaned.append(s)
    if not cleaned:
        return None
    payload = (
        {"@context": "https://schema.org", **cleaned[0]}
        if len(cleaned) == 1
        else {"@context": "https://schema.org", "@graph": cleaned}
    )
    return _json_ld_dumps(payload)


DEFAULT_OG_IMAGE = "/static/og-default.png"


def absolute_url(base, path):
    if not path:
        return ""
    return path if path.startswith("http") else f"{base}{path}"


def _clean_breadcrumbs(breadcrumbs):
    if not breadcrumbs:
        return breadcrumbs
    from devplacepy.seo_meta_text import plain_text_from_markdown

    cleaned = []
    for crumb in breadcrumbs:
        name = crumb.get("name", "")
        plain = plain_text_from_markdown(name) or name
        cleaned.append({**crumb, "name": plain})
    return cleaned


def base_seo_context(
    request,
    title="",
    description="",
    robots="index,follow",
    og_type="website",
    og_image=None,
    breadcrumbs=None,
    schemas=None,
    prev_url=None,
    next_url=None,
    keywords="",
    seo_target=None,
):
    from devplacepy.seo_meta_text import plain_seo_defaults, plain_text_from_markdown

    base = site_url(request)
    seo_title = title
    clean_description = truncate(plain_text_from_markdown(description), 160)
    meta_keywords = (keywords or "").strip()

    ready = _ready_seo_metadata(seo_target)
    if ready:
        seo_title = ready.get("seo_title") or seo_title
        clean_description = ready.get("seo_description") or clean_description
        meta_keywords = ready.get("seo_keywords") or meta_keywords
    elif seo_target:
        defaults = plain_seo_defaults(title, description)
        seo_title = seo_title or defaults["title"]
        clean_description = clean_description or defaults["description"]
        meta_keywords = meta_keywords or defaults["keywords"]

    page_title = f"{seo_title} - {SITE_NAME}" if seo_title else SITE_NAME
    canonical = f"{base}{request.url.path}"
    page = request.query_params.get("page")
    if page and page not in ("", "1"):
        canonical = f"{canonical}?page={page}"
    og_img = absolute_url(base, og_image) or f"{base}{DEFAULT_OG_IMAGE}"
    breadcrumbs = _clean_breadcrumbs(breadcrumbs)
    page_schemas = list(schemas or [])
    if breadcrumbs:
        page_schemas.append(breadcrumb_schema(breadcrumbs, base))
    return {
        "page_title": page_title,
        "meta_description": clean_description,
        "meta_keywords": meta_keywords,
        "meta_robots": robots,
        "canonical_url": canonical,
        "og_title": seo_title or SITE_NAME,
        "og_description": clean_description,
        "og_image": og_img,
        "og_type": og_type,
        "breadcrumbs": breadcrumbs or [],
        "page_schema": combine(page_schemas),
        "prev_url": absolute_url(base, prev_url),
        "next_url": absolute_url(base, next_url),
    }


def _ready_seo_metadata(seo_target):
    if not seo_target:
        return None
    target_type, target_uid = seo_target
    if not target_type or not target_uid:
        return None
    try:
        from devplacepy.database import get_seo_metadata

        return get_seo_metadata(target_type, str(target_uid))
    except Exception:
        return None


def next_page_url(request, next_cursor):
    if not next_cursor:
        return None
    params = dict(request.query_params)
    params["before"] = next_cursor
    return f"{request.url.path}?{urlencode(params)}"


def list_page_seo(
    request, title="", description="", breadcrumbs=None, prev_url=None, next_url=None
):
    base = site_url(request)
    return base_seo_context(
        request,
        title=title,
        description=description,
        breadcrumbs=breadcrumbs,
        schemas=[website_schema(base)],
        prev_url=prev_url,
        next_url=next_url,
    )


def make_sitemap(base_url):
    cached = _sitemap_cache.get(base_url)
    if cached and time.time() - cached[0] < SITEMAP_TTL:
        return cached[1]
    xml = _build_sitemap(base_url)
    _sitemap_cache[base_url] = (time.time(), xml)
    return xml


def _collect(table, limit, label, **query):
    if table.has_column("deleted_at") and "deleted_at" not in query:
        query["deleted_at"] = None
    rows = list(table.find(_limit=limit, **query))
    if len(rows) >= limit:
        logger.warning("sitemap: %s truncated at %d entries", label, limit)
    return rows


def _build_sitemap(base_url):
    from devplacepy.database import get_table, db

    def url_element(loc, lastmod=None, changefreq=None, priority=None):
        u = Element("url")
        loc_el = Element("loc")
        loc_el.text = loc
        u.append(loc_el)
        if lastmod:
            lm = Element("lastmod")
            lm.text = lastmod
            u.append(lm)
        if changefreq:
            cf = Element("changefreq")
            cf.text = changefreq
            u.append(cf)
        if priority is not None:
            pr = Element("priority")
            pr.text = str(priority)
            u.append(pr)
        return u

    urlset = Element("urlset")
    urlset.set("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9")

    urlset.append(url_element(f"{base_url}/", changefreq="daily", priority="1.0"))
    urlset.append(url_element(f"{base_url}/feed", changefreq="hourly", priority="0.9"))
    urlset.append(url_element(f"{base_url}/news", changefreq="hourly", priority="0.9"))
    urlset.append(
        url_element(f"{base_url}/projects", changefreq="daily", priority="0.8")
    )
    urlset.append(url_element(f"{base_url}/gists", changefreq="daily", priority="0.8"))
    urlset.append(url_element(f"{base_url}/quizzes", changefreq="daily", priority="0.8"))
    urlset.append(url_element(f"{base_url}/battles", changefreq="daily", priority="0.7"))
    urlset.append(
        url_element(f"{base_url}/leaderboard", changefreq="daily", priority="0.7")
    )
    urlset.append(url_element(f"{base_url}/issues", changefreq="daily", priority="0.6"))
    urlset.append(url_element(f"{base_url}/tools", changefreq="monthly", priority="0.5"))
    urlset.append(url_element(f"{base_url}/tools/seo", changefreq="monthly", priority="0.5"))
    urlset.append(url_element(f"{base_url}/tools/deepsearch", changefreq="monthly", priority="0.5"))
    urlset.append(
        url_element(f"{base_url}/workspaces/index", changefreq="daily", priority="0.6")
    )
    urlset.append(
        url_element(f"{base_url}/reports/reasons", changefreq="monthly", priority="0.4")
    )
    for slug in LEGAL_DOC_SLUGS:
        urlset.append(
            url_element(
                f"{base_url}/docs/{slug}.html", changefreq="monthly", priority="0.5"
            )
        )

    if "posts" in db.tables:
        posts = _collect(
            get_table("posts"), SITEMAP_URL_LIMIT, "posts", order_by=["-created_at"]
        )
        for p in posts:
            urlset.append(
                url_element(
                    f"{base_url}/posts/{p.get('slug') or p['uid']}",
                    lastmod=p.get("created_at", ""),
                    changefreq="weekly",
                    priority="0.7",
                )
            )

    if "projects" in db.tables:
        projects = _collect(
            get_table("projects"),
            SITEMAP_URL_LIMIT,
            "projects",
            order_by=["-created_at"],
        )
        for p in projects:
            if p.get("is_private"):
                continue
            urlset.append(
                url_element(
                    f"{base_url}/projects/{p.get('slug') or p['uid']}",
                    lastmod=p.get("created_at", ""),
                    changefreq="weekly",
                    priority="0.6",
                )
            )

    if "gists" in db.tables:
        gists = _collect(
            get_table("gists"), SITEMAP_URL_LIMIT, "gists", order_by=["-created_at"]
        )
        for g in gists:
            urlset.append(
                url_element(
                    f"{base_url}/gists/{g.get('slug') or g['uid']}",
                    lastmod=g.get("created_at", ""),
                    changefreq="weekly",
                    priority="0.6",
                )
            )

    if "quizzes" in db.tables:
        quizzes = _collect(
            get_table("quizzes"),
            SITEMAP_URL_LIMIT,
            "quizzes",
            status="published",
            order_by=["-created_at"],
        )
        for quiz in quizzes:
            urlset.append(
                url_element(
                    f"{base_url}/quizzes/{quiz.get('slug') or quiz['uid']}",
                    lastmod=quiz.get("published_at", "") or quiz.get("created_at", ""),
                    changefreq="weekly",
                    priority="0.6",
                )
            )

    if "news" in db.tables:
        articles = _collect(
            get_table("news"),
            SITEMAP_URL_LIMIT,
            "news",
            status="published",
            order_by=["-synced_at"],
        )
        for a in articles:
            urlset.append(
                url_element(
                    f"{base_url}/news/{a.get('slug') or a['uid']}",
                    lastmod=a.get("synced_at", "") or a.get("created_at", ""),
                    changefreq="weekly",
                    priority="0.7",
                )
            )

    if "users" in db.tables:
        post_counts = {}
        if "posts" in db.tables:
            for row in db.query(
                "SELECT user_uid, COUNT(*) AS c FROM posts WHERE deleted_at IS NULL GROUP BY user_uid"
            ):
                post_counts[row["user_uid"]] = row["c"]
        users = _collect(
            get_table("users"), SITEMAP_URL_LIMIT, "users", order_by=["-created_at"]
        )
        for u in users:
            if post_counts.get(u["uid"], 0) < 2:
                continue
            urlset.append(
                url_element(
                    f"{base_url}/profile/{u['username']}",
                    lastmod=u.get("created_at", ""),
                    changefreq="weekly",
                    priority="0.4",
                )
            )

    try:
        from devplacepy.routers.docs.pages import DOCS_PAGES

        listed = set(LEGAL_DOC_SLUGS)
        for page in DOCS_PAGES:
            if page.get("admin") or page.get("kind") == "live":
                continue
            if page["slug"] in listed:
                continue
            listed.add(page["slug"])
            urlset.append(
                url_element(
                    f"{base_url}/docs/{page['slug']}.html",
                    changefreq="weekly",
                    priority="0.5",
                )
            )
    except Exception:
        logger.warning("sitemap: could not add docs pages")

    rough = tostring(urlset, encoding="unicode")
    dom = minidom.parseString(rough)
    return dom.toprettyxml(indent="  ")