# retoor import json import logging import os import time from urllib.parse import urlencode from xml.etree.ElementTree import Element, tostring from xml.dom import minidom from devplacepy.config import SITE_URL from devplacepy.utils import strip_html logger = logging.getLogger(__name__) SITE_NAME = "DevPlace" SITEMAP_URL_LIMIT = 5000 SITEMAP_TTL = int(os.environ.get("DEVPLACE_SITEMAP_TTL", "3600")) _sitemap_cache = {} def plain_markdown(text: str) -> str: if not text: return "" from devplacepy.seo_meta_text import plain_text_from_markdown return plain_text_from_markdown(text) def truncate(text: str, max_len: int = 160) -> str: if not text: return "" text = " ".join(text.split()) if len(text) <= max_len: return text text = text[: max_len - 3].rsplit(" ", 1)[0] return text + "..." def public_base_url() -> str: from devplacepy.database import get_setting return (get_setting("site_url", "").strip() or SITE_URL or "").rstrip("/") def site_url(request): return public_base_url() or str(request.base_url).rstrip("/") def website_schema(base_url): return { "@type": "WebSite", "name": SITE_NAME, "url": base_url, "potentialAction": { "@type": "SearchAction", "target": { "@type": "EntryPoint", "urlTemplate": f"{base_url}/feed?search={{query}}", }, "query-input": "required name=query", }, } def breadcrumb_schema(items, base_url): return { "@type": "BreadcrumbList", "itemListElement": [ { "@type": "ListItem", "position": i + 1, "item": { "@id": item["url"] if item["url"].startswith("http") else f"{base_url}{item['url']}", "name": item["name"], }, } for i, item in enumerate(items) ], } def discussion_forum_posting(post, author, comment_count, star_count, base_url): schema = { "@type": "DiscussionForumPosting", "headline": post.get("title") or "Untitled", "text": truncate(plain_markdown(post.get("content", "")), 500), "url": f"{base_url}/posts/{post.get('slug') or post['uid']}", "author": { "@type": "Person", "name": author["username"] if author else "Unknown", "url": f"{base_url}/profile/{author['username']}" if author else "", }, "datePublished": post.get("created_at", ""), "dateModified": post.get("updated_at") or post.get("created_at", ""), "interactionStatistic": [ { "@type": "InteractionCounter", "interactionType": "https://schema.org/LikeAction", "userInteractionCount": star_count, }, { "@type": "InteractionCounter", "interactionType": "https://schema.org/CommentAction", "userInteractionCount": comment_count, }, ], } return schema def profile_page_schema(profile_user, post_count, base_url): return { "@type": "ProfilePage", "mainEntity": { "@type": "Person", "name": profile_user.get("username", ""), "alternateName": profile_user.get("username", ""), "description": profile_user.get("bio", "") or f"Developer on {SITE_NAME}", "interactionStatistic": [ { "@type": "InteractionCounter", "interactionType": "https://schema.org/WriteAction", "userInteractionCount": post_count, } ], }, } def software_application_schema(project, base_url): return { "@type": "SoftwareApplication", "name": project.get("title", "Untitled"), "description": truncate(plain_markdown(project.get("description", "")), 300), "url": f"{base_url}/projects/{project.get('slug') or project['uid']}", "applicationCategory": "DeveloperApplication", "operatingSystem": project.get("platforms", "Cross-platform"), "author": {"@type": "Person", "name": project.get("author_name", "Unknown")}, "datePublished": project.get("created_at", ""), "dateModified": project.get("updated_at") or project.get("created_at", ""), "offers": {"@type": "Offer", "price": "0", "priceCurrency": "USD"}, } def web_application_schema(name, description, path, base_url, category="DeveloperApplication"): return { "@type": "WebApplication", "name": name, "description": truncate(description, 300), "url": f"{base_url}{path}", "applicationCategory": category, "operatingSystem": "All", "browserRequirements": "Requires JavaScript", "offers": {"@type": "Offer", "price": "0", "priceCurrency": "USD"}, "provider": {"@type": "Organization", "name": SITE_NAME, "url": base_url}, } def organization_schema(base_url): return { "@type": "Organization", "name": SITE_NAME, "url": base_url, "logo": f"{base_url}{DEFAULT_OG_IMAGE}", } def news_article_schema(article, base_url, image_url=""): url = f"{base_url}/news/{article.get('slug') or article['uid']}" schema = { "@type": "NewsArticle", "headline": (article.get("title") or "Untitled")[:110], "description": truncate(plain_markdown(article.get("description", "") or ""), 200), "url": url, "datePublished": article.get("synced_at", "") or article.get("created_at", ""), "dateModified": article.get("synced_at", "") or article.get("created_at", ""), "mainEntityOfPage": {"@type": "WebPage", "@id": url}, "author": { "@type": "Organization", "name": article.get("source_name") or SITE_NAME, }, "publisher": organization_schema(base_url), } if image_url: schema["image"] = image_url return schema def software_source_code_schema(gist, base_url): return { "@type": "SoftwareSourceCode", "name": gist.get("title") or "Gist", "description": truncate(plain_markdown(gist.get("description", "") or ""), 200), "url": f"{base_url}/gists/{gist.get('slug') or gist['uid']}", "programmingLanguage": gist.get("language", "") or "text", "dateCreated": gist.get("created_at", ""), "dateModified": gist.get("updated_at") or gist.get("created_at", ""), } def _json_ld_dumps(payload): raw = json.dumps(payload, ensure_ascii=False) return ( raw.replace("<", "\\u003c") .replace(">", "\\u003e") .replace("&", "\\u0026") .replace("
", "\\u2028") .replace("
", "\\u2029") ) def combine(schemas): if not schemas: return None cleaned = [] for s in schemas: if s is not None: s.pop("@context", None) cleaned.append(s) if not cleaned: return None payload = ( {"@context": "https://schema.org", **cleaned[0]} if len(cleaned) == 1 else {"@context": "https://schema.org", "@graph": cleaned} ) return _json_ld_dumps(payload) DEFAULT_OG_IMAGE = "/static/og-default.png" def absolute_url(base, path): if not path: return "" return path if path.startswith("http") else f"{base}{path}" def _clean_breadcrumbs(breadcrumbs): if not breadcrumbs: return breadcrumbs from devplacepy.seo_meta_text import plain_text_from_markdown cleaned = [] for crumb in breadcrumbs: name = crumb.get("name", "") plain = plain_text_from_markdown(name) or name cleaned.append({**crumb, "name": plain}) return cleaned def base_seo_context( request, title="", description="", robots="index,follow", og_type="website", og_image=None, breadcrumbs=None, schemas=None, prev_url=None, next_url=None, keywords="", seo_target=None, ): from devplacepy.seo_meta_text import plain_seo_defaults, plain_text_from_markdown base = site_url(request) seo_title = title clean_description = truncate(plain_text_from_markdown(description), 160) meta_keywords = (keywords or "").strip() ready = _ready_seo_metadata(seo_target) if ready: seo_title = ready.get("seo_title") or seo_title clean_description = ready.get("seo_description") or clean_description meta_keywords = ready.get("seo_keywords") or meta_keywords elif seo_target: defaults = plain_seo_defaults(title, description) seo_title = seo_title or defaults["title"] clean_description = clean_description or defaults["description"] meta_keywords = meta_keywords or defaults["keywords"] page_title = f"{seo_title} - {SITE_NAME}" if seo_title else SITE_NAME canonical = f"{base}{request.url.path}" page = request.query_params.get("page") if page and page not in ("", "1"): canonical = f"{canonical}?page={page}" og_img = absolute_url(base, og_image) or f"{base}{DEFAULT_OG_IMAGE}" breadcrumbs = _clean_breadcrumbs(breadcrumbs) page_schemas = list(schemas or []) if breadcrumbs: page_schemas.append(breadcrumb_schema(breadcrumbs, base)) return { "page_title": page_title, "meta_description": clean_description, "meta_keywords": meta_keywords, "meta_robots": robots, "canonical_url": canonical, "og_title": seo_title or SITE_NAME, "og_description": clean_description, "og_image": og_img, "og_type": og_type, "breadcrumbs": breadcrumbs or [], "page_schema": combine(page_schemas), "prev_url": absolute_url(base, prev_url), "next_url": absolute_url(base, next_url), } def _ready_seo_metadata(seo_target): if not seo_target: return None target_type, target_uid = seo_target if not target_type or not target_uid: return None try: from devplacepy.database import get_seo_metadata return get_seo_metadata(target_type, str(target_uid)) except Exception: return None def next_page_url(request, next_cursor): if not next_cursor: return None params = dict(request.query_params) params["before"] = next_cursor return f"{request.url.path}?{urlencode(params)}" def list_page_seo( request, title="", description="", breadcrumbs=None, prev_url=None, next_url=None ): base = site_url(request) return base_seo_context( request, title=title, description=description, breadcrumbs=breadcrumbs, schemas=[website_schema(base)], prev_url=prev_url, next_url=next_url, ) def make_sitemap(base_url): cached = _sitemap_cache.get(base_url) if cached and time.time() - cached[0] < SITEMAP_TTL: return cached[1] xml = _build_sitemap(base_url) _sitemap_cache[base_url] = (time.time(), xml) return xml def _collect(table, limit, label, **query): if table.has_column("deleted_at") and "deleted_at" not in query: query["deleted_at"] = None rows = list(table.find(_limit=limit, **query)) if len(rows) >= limit: logger.warning("sitemap: %s truncated at %d entries", label, limit) return rows def _build_sitemap(base_url): from devplacepy.database import get_table, db def url_element(loc, lastmod=None, changefreq=None, priority=None): u = Element("url") loc_el = Element("loc") loc_el.text = loc u.append(loc_el) if lastmod: lm = Element("lastmod") lm.text = lastmod u.append(lm) if changefreq: cf = Element("changefreq") cf.text = changefreq u.append(cf) if priority is not None: pr = Element("priority") pr.text = str(priority) u.append(pr) return u urlset = Element("urlset") urlset.set("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9") urlset.append(url_element(f"{base_url}/", changefreq="daily", priority="1.0")) urlset.append(url_element(f"{base_url}/feed", changefreq="hourly", priority="0.9")) urlset.append(url_element(f"{base_url}/news", changefreq="hourly", priority="0.9")) urlset.append( url_element(f"{base_url}/projects", changefreq="daily", priority="0.8") ) urlset.append(url_element(f"{base_url}/gists", changefreq="daily", priority="0.8")) urlset.append( url_element(f"{base_url}/leaderboard", changefreq="daily", priority="0.7") ) urlset.append(url_element(f"{base_url}/issues", changefreq="daily", priority="0.6")) urlset.append(url_element(f"{base_url}/tools", changefreq="monthly", priority="0.5")) urlset.append(url_element(f"{base_url}/tools/seo", changefreq="monthly", priority="0.5")) urlset.append(url_element(f"{base_url}/tools/deepsearch", changefreq="monthly", priority="0.5")) if "posts" in db.tables: posts = _collect( get_table("posts"), SITEMAP_URL_LIMIT, "posts", order_by=["-created_at"] ) for p in posts: urlset.append( url_element( f"{base_url}/posts/{p.get('slug') or p['uid']}", lastmod=p.get("created_at", ""), changefreq="weekly", priority="0.7", ) ) if "projects" in db.tables: projects = _collect( get_table("projects"), SITEMAP_URL_LIMIT, "projects", order_by=["-created_at"], ) for p in projects: if p.get("is_private"): continue urlset.append( url_element( f"{base_url}/projects/{p.get('slug') or p['uid']}", lastmod=p.get("created_at", ""), changefreq="weekly", priority="0.6", ) ) if "gists" in db.tables: gists = _collect( get_table("gists"), SITEMAP_URL_LIMIT, "gists", order_by=["-created_at"] ) for g in gists: urlset.append( url_element( f"{base_url}/gists/{g.get('slug') or g['uid']}", lastmod=g.get("created_at", ""), changefreq="weekly", priority="0.6", ) ) if "news" in db.tables: articles = _collect( get_table("news"), SITEMAP_URL_LIMIT, "news", status="published", order_by=["-synced_at"], ) for a in articles: urlset.append( url_element( f"{base_url}/news/{a.get('slug') or a['uid']}", lastmod=a.get("synced_at", "") or a.get("created_at", ""), changefreq="weekly", priority="0.7", ) ) if "users" in db.tables: post_counts = {} if "posts" in db.tables: for row in db.query( "SELECT user_uid, COUNT(*) AS c FROM posts WHERE deleted_at IS NULL GROUP BY user_uid" ): post_counts[row["user_uid"]] = row["c"] users = _collect( get_table("users"), SITEMAP_URL_LIMIT, "users", order_by=["-created_at"] ) for u in users: if post_counts.get(u["uid"], 0) < 2: continue urlset.append( url_element( f"{base_url}/profile/{u['username']}", lastmod=u.get("created_at", ""), changefreq="weekly", priority="0.4", ) ) try: from devplacepy.routers.docs.pages import DOCS_PAGES for page in DOCS_PAGES: if page.get("admin") or page.get("kind") == "live": continue urlset.append( url_element( f"{base_url}/docs/{page['slug']}.html", changefreq="weekly", priority="0.5", ) ) except Exception: logger.warning("sitemap: could not add docs pages") rough = tostring(urlset, encoding="unicode") dom = minidom.parseString(rough) return dom.toprettyxml(indent=" ")