# retoor <retoor@molodetz.nl>
import json
import logging
import os
import time
from urllib . parse import urlencode
from xml . etree . ElementTree import Element , tostring
from xml . dom import minidom
from devplacepy . config import SITE_URL
from devplacepy . utils import strip_html
logger = logging . getLogger ( __name__ )
SITE_NAME = " DevPlace "
SITEMAP_URL_LIMIT = 5000
SITEMAP_TTL = int ( os . environ . get ( " DEVPLACE_SITEMAP_TTL " , " 3600 " ) )
_sitemap_cache = { }
def truncate ( text , max_len = 160 ) :
if not text :
return " "
text = " " . join ( text . split ( ) )
if len ( text ) < = max_len :
return text
text = text [ : max_len - 3 ] . rsplit ( " " , 1 ) [ 0 ]
return text + " ... "
def public_base_url ( ) - > str :
from devplacepy . database import get_setting
return ( get_setting ( " site_url " , " " ) . strip ( ) or SITE_URL or " " ) . rstrip ( " / " )
def site_url ( request ) :
return public_base_url ( ) or str ( request . base_url ) . rstrip ( " / " )
def website_schema ( base_url ) :
return {
" @type " : " WebSite " ,
" name " : SITE_NAME ,
" url " : base_url ,
" potentialAction " : {
" @type " : " SearchAction " ,
" target " : {
" @type " : " EntryPoint " ,
" urlTemplate " : f " { base_url } /feed?search= {{ query }} " ,
} ,
" query-input " : " required name=query " ,
} ,
}
def breadcrumb_schema ( items , base_url ) :
return {
" @type " : " BreadcrumbList " ,
" itemListElement " : [
{
" @type " : " ListItem " ,
" position " : i + 1 ,
" item " : {
" @id " : item [ " url " ]
if item [ " url " ] . startswith ( " http " )
else f " { base_url } { item [ ' url ' ] } " ,
" name " : item [ " name " ] ,
} ,
}
for i , item in enumerate ( items )
] ,
}
def discussion_forum_posting ( post , author , comment_count , star_count , base_url ) :
schema = {
" @type " : " DiscussionForumPosting " ,
" headline " : post . get ( " title " ) or " Untitled " ,
" text " : truncate ( post . get ( " content " , " " ) , 500 ) ,
" url " : f " { base_url } /posts/ { post . get ( ' slug ' ) or post [ ' uid ' ] } " ,
" author " : {
" @type " : " Person " ,
" name " : author [ " username " ] if author else " Unknown " ,
" url " : f " { base_url } /profile/ { author [ ' username ' ] } " if author else " " ,
} ,
" datePublished " : post . get ( " created_at " , " " ) ,
" dateModified " : post . get ( " updated_at " ) or post . get ( " created_at " , " " ) ,
" interactionStatistic " : [
{
" @type " : " InteractionCounter " ,
" interactionType " : " https://schema.org/LikeAction " ,
" userInteractionCount " : star_count ,
} ,
{
" @type " : " InteractionCounter " ,
" interactionType " : " https://schema.org/CommentAction " ,
" userInteractionCount " : comment_count ,
} ,
] ,
}
return schema
def profile_page_schema ( profile_user , post_count , base_url ) :
return {
" @type " : " ProfilePage " ,
" mainEntity " : {
" @type " : " Person " ,
" name " : profile_user . get ( " username " , " " ) ,
" alternateName " : profile_user . get ( " username " , " " ) ,
" description " : profile_user . get ( " bio " , " " ) or f " Developer on { SITE_NAME } " ,
" interactionStatistic " : [
{
" @type " : " InteractionCounter " ,
" interactionType " : " https://schema.org/WriteAction " ,
" userInteractionCount " : post_count ,
}
] ,
} ,
}
def software_application_schema ( project , base_url ) :
return {
" @type " : " SoftwareApplication " ,
" name " : project . get ( " title " , " Untitled " ) ,
" description " : truncate ( project . get ( " description " , " " ) , 300 ) ,
" url " : f " { base_url } /projects/ { project . get ( ' slug ' ) or project [ ' uid ' ] } " ,
" applicationCategory " : " DeveloperApplication " ,
" operatingSystem " : project . get ( " platforms " , " Cross-platform " ) ,
" author " : { " @type " : " Person " , " name " : project . get ( " author_name " , " Unknown " ) } ,
" datePublished " : project . get ( " created_at " , " " ) ,
" dateModified " : project . get ( " updated_at " ) or project . get ( " created_at " , " " ) ,
" offers " : { " @type " : " Offer " , " price " : " 0 " , " priceCurrency " : " USD " } ,
}
def web_application_schema ( name , description , path , base_url , category = " DeveloperApplication " ) :
return {
" @type " : " WebApplication " ,
" name " : name ,
" description " : truncate ( description , 300 ) ,
" url " : f " { base_url } { path } " ,
" applicationCategory " : category ,
" operatingSystem " : " All " ,
" browserRequirements " : " Requires JavaScript " ,
" offers " : { " @type " : " Offer " , " price " : " 0 " , " priceCurrency " : " USD " } ,
" provider " : { " @type " : " Organization " , " name " : SITE_NAME , " url " : base_url } ,
}
def organization_schema ( base_url ) :
return {
" @type " : " Organization " ,
" name " : SITE_NAME ,
" url " : base_url ,
" logo " : f " { base_url } { DEFAULT_OG_IMAGE } " ,
}
def news_article_schema ( article , base_url , image_url = " " ) :
url = f " { base_url } /news/ { article . get ( ' slug ' ) or article [ ' uid ' ] } "
schema = {
" @type " : " NewsArticle " ,
" headline " : ( article . get ( " title " ) or " Untitled " ) [ : 110 ] ,
" description " : truncate ( strip_html ( article . get ( " description " , " " ) or " " ) , 200 ) ,
" url " : url ,
" datePublished " : article . get ( " synced_at " , " " ) or article . get ( " created_at " , " " ) ,
" dateModified " : article . get ( " synced_at " , " " ) or article . get ( " created_at " , " " ) ,
" mainEntityOfPage " : { " @type " : " WebPage " , " @id " : url } ,
" author " : {
" @type " : " Organization " ,
" name " : article . get ( " source_name " ) or SITE_NAME ,
} ,
" publisher " : organization_schema ( base_url ) ,
}
if image_url :
schema [ " image " ] = image_url
return schema
def software_source_code_schema ( gist , base_url ) :
return {
" @type " : " SoftwareSourceCode " ,
" name " : gist . get ( " title " ) or " Gist " ,
" description " : truncate ( strip_html ( gist . get ( " description " , " " ) or " " ) , 200 ) ,
" url " : f " { base_url } /gists/ { gist . get ( ' slug ' ) or gist [ ' uid ' ] } " ,
" programmingLanguage " : gist . get ( " language " , " " ) or " text " ,
" dateCreated " : gist . get ( " created_at " , " " ) ,
" dateModified " : gist . get ( " updated_at " ) or gist . get ( " created_at " , " " ) ,
}
def _json_ld_dumps ( payload ) :
raw = json . dumps ( payload , ensure_ascii = False )
return (
raw . replace ( " < " , " \\ u003c " )
. replace ( " > " , " \\ u003e " )
. replace ( " & " , " \\ u0026 " )
. replace ( " 
 " , " \\ u2028 " )
. replace ( " 
 " , " \\ u2029 " )
)
def combine ( schemas ) :
if not schemas :
return None
cleaned = [ ]
for s in schemas :
if s is not None :
s . pop ( " @context " , None )
cleaned . append ( s )
if not cleaned :
return None
payload = (
{ " @context " : " https://schema.org " , * * cleaned [ 0 ] }
if len ( cleaned ) == 1
else { " @context " : " https://schema.org " , " @graph " : cleaned }
)
return _json_ld_dumps ( payload )
DEFAULT_OG_IMAGE = " /static/og-default.png "
def absolute_url ( base , path ) :
if not path :
return " "
return path if path . startswith ( " http " ) else f " { base } { path } "
def base_seo_context (
request ,
title = " " ,
description = " " ,
robots = " index,follow " ,
og_type = " website " ,
og_image = None ,
breadcrumbs = None ,
schemas = None ,
prev_url = None ,
next_url = None ,
) :
base = site_url ( request )
page_title = f " { title } - { SITE_NAME } " if title else SITE_NAME
canonical = f " { base } { request . url . path } "
page = request . query_params . get ( " page " )
if page and page not in ( " " , " 1 " ) :
canonical = f " { canonical } ?page= { page } "
clean_description = truncate ( strip_html ( description ) , 160 )
og_img = absolute_url ( base , og_image ) or f " { base } { DEFAULT_OG_IMAGE } "
page_schemas = list ( schemas or [ ] )
if breadcrumbs :
page_schemas . append ( breadcrumb_schema ( breadcrumbs , base ) )
return {
" page_title " : page_title ,
" meta_description " : clean_description ,
" meta_robots " : robots ,
" canonical_url " : canonical ,
" og_title " : title or SITE_NAME ,
" og_description " : clean_description ,
" og_image " : og_img ,
" og_type " : og_type ,
" breadcrumbs " : breadcrumbs or [ ] ,
" page_schema " : combine ( page_schemas ) ,
" prev_url " : absolute_url ( base , prev_url ) ,
" next_url " : absolute_url ( base , next_url ) ,
}
def next_page_url ( request , next_cursor ) :
if not next_cursor :
return None
params = dict ( request . query_params )
params [ " before " ] = next_cursor
return f " { request . url . path } ? { urlencode ( params ) } "
def list_page_seo (
request , title = " " , description = " " , breadcrumbs = None , prev_url = None , next_url = None
) :
base = site_url ( request )
return base_seo_context (
request ,
title = title ,
description = description ,
breadcrumbs = breadcrumbs ,
schemas = [ website_schema ( base ) ] ,
prev_url = prev_url ,
next_url = next_url ,
)
def make_sitemap ( base_url ) :
cached = _sitemap_cache . get ( base_url )
if cached and time . time ( ) - cached [ 0 ] < SITEMAP_TTL :
return cached [ 1 ]
xml = _build_sitemap ( base_url )
_sitemap_cache [ base_url ] = ( time . time ( ) , xml )
return xml
def _collect ( table , limit , label , * * query ) :
if table . has_column ( " deleted_at " ) and " deleted_at " not in query :
query [ " deleted_at " ] = None
rows = list ( table . find ( _limit = limit , * * query ) )
if len ( rows ) > = limit :
logger . warning ( " sitemap: %s truncated at %d entries " , label , limit )
return rows
def _build_sitemap ( base_url ) :
from devplacepy . database import get_table , db
def url_element ( loc , lastmod = None , changefreq = None , priority = None ) :
u = Element ( " url " )
loc_el = Element ( " loc " )
loc_el . text = loc
u . append ( loc_el )
if lastmod :
lm = Element ( " lastmod " )
lm . text = lastmod
u . append ( lm )
if changefreq :
cf = Element ( " changefreq " )
cf . text = changefreq
u . append ( cf )
if priority is not None :
pr = Element ( " priority " )
pr . text = str ( priority )
u . append ( pr )
return u
urlset = Element ( " urlset " )
urlset . set ( " xmlns " , " http://www.sitemaps.org/schemas/sitemap/0.9 " )
urlset . append ( url_element ( f " { base_url } / " , changefreq = " daily " , priority = " 1.0 " ) )
urlset . append ( url_element ( f " { base_url } /feed " , changefreq = " hourly " , priority = " 0.9 " ) )
urlset . append ( url_element ( f " { base_url } /news " , changefreq = " hourly " , priority = " 0.9 " ) )
urlset . append (
url_element ( f " { base_url } /projects " , changefreq = " daily " , priority = " 0.8 " )
)
urlset . append ( url_element ( f " { base_url } /gists " , changefreq = " daily " , priority = " 0.8 " ) )
urlset . append (
url_element ( f " { base_url } /leaderboard " , changefreq = " daily " , priority = " 0.7 " )
)
urlset . append ( url_element ( f " { base_url } /issues " , changefreq = " daily " , priority = " 0.6 " ) )
urlset . append ( url_element ( f " { base_url } /tools " , changefreq = " monthly " , priority = " 0.5 " ) )
urlset . append ( url_element ( f " { base_url } /tools/seo " , changefreq = " monthly " , priority = " 0.5 " ) )
urlset . append ( url_element ( f " { base_url } /tools/deepsearch " , changefreq = " monthly " , priority = " 0.5 " ) )
if " posts " in db . tables :
posts = _collect (
get_table ( " posts " ) , SITEMAP_URL_LIMIT , " posts " , order_by = [ " -created_at " ]
)
for p in posts :
urlset . append (
url_element (
f " { base_url } /posts/ { p . get ( ' slug ' ) or p [ ' uid ' ] } " ,
lastmod = p . get ( " created_at " , " " ) ,
changefreq = " weekly " ,
priority = " 0.7 " ,
)
)
if " projects " in db . tables :
projects = _collect (
get_table ( " projects " ) ,
SITEMAP_URL_LIMIT ,
" projects " ,
order_by = [ " -created_at " ] ,
)
for p in projects :
if p . get ( " is_private " ) :
continue
urlset . append (
url_element (
f " { base_url } /projects/ { p . get ( ' slug ' ) or p [ ' uid ' ] } " ,
lastmod = p . get ( " created_at " , " " ) ,
changefreq = " weekly " ,
priority = " 0.6 " ,
)
)
if " gists " in db . tables :
gists = _collect (
get_table ( " gists " ) , SITEMAP_URL_LIMIT , " gists " , order_by = [ " -created_at " ]
)
for g in gists :
urlset . append (
url_element (
f " { base_url } /gists/ { g . get ( ' slug ' ) or g [ ' uid ' ] } " ,
lastmod = g . get ( " created_at " , " " ) ,
changefreq = " weekly " ,
priority = " 0.6 " ,
)
)
if " news " in db . tables :
articles = _collect (
get_table ( " news " ) ,
SITEMAP_URL_LIMIT ,
" news " ,
status = " published " ,
order_by = [ " -synced_at " ] ,
)
for a in articles :
urlset . append (
url_element (
f " { base_url } /news/ { a . get ( ' slug ' ) or a [ ' uid ' ] } " ,
lastmod = a . get ( " synced_at " , " " ) or a . get ( " created_at " , " " ) ,
changefreq = " weekly " ,
priority = " 0.7 " ,
)
)
if " users " in db . tables :
post_counts = { }
if " posts " in db . tables :
for row in db . query (
" SELECT user_uid, COUNT(*) AS c FROM posts WHERE deleted_at IS NULL GROUP BY user_uid "
) :
post_counts [ row [ " user_uid " ] ] = row [ " c " ]
users = _collect (
get_table ( " users " ) , SITEMAP_URL_LIMIT , " users " , order_by = [ " -created_at " ]
)
for u in users :
if post_counts . get ( u [ " uid " ] , 0 ) < 2 :
continue
urlset . append (
url_element (
f " { base_url } /profile/ { u [ ' username ' ] } " ,
lastmod = u . get ( " created_at " , " " ) ,
changefreq = " weekly " ,
priority = " 0.4 " ,
)
)
try :
from devplacepy . routers . docs . pages import DOCS_PAGES
for page in DOCS_PAGES :
if page . get ( " admin " ) or page . get ( " kind " ) == " live " :
continue
urlset . append (
url_element (
f " { base_url } /docs/ { page [ ' slug ' ] } .html " ,
changefreq = " weekly " ,
priority = " 0.5 " ,
)
)
except Exception :
logger . warning ( " sitemap: could not add docs pages " )
rough = tostring ( urlset , encoding = " unicode " )
dom = minidom . parseString ( rough )
return dom . toprettyxml ( indent = " " )