ticket #75 attempt 1

devplacepy/main.py

@@ -4,12 +4,13 @@ import asyncio
 import fcntl
 import logging
 import os
+import re
 import time
 from collections import defaultdict
 from contextlib import asynccontextmanager, contextmanager
 from pathlib import Path
 from fastapi import FastAPI, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.responses import HTMLResponse, RedirectResponse, Response
 from fastapi.staticfiles import StaticFiles
 from fastapi.exceptions import RequestValidationError
 from starlette.middleware.gzip import GZipMiddleware
@@ -611,6 +612,22 @@ async def response_timing(request: Request, call_next):
     return response
 
 
+_KNOWN_CRAWLERS = re.compile(
+    r"(google.*read.*aloud|googlebot|bingbot|slurp|duckduckbot|baiduspider|yandexbot)",
+    re.IGNORECASE,
+)
+_AUTH_GATED_PATHS = frozenset({"/messages", "/notifications", "/game", "/admin"})
+
+
+@app.middleware("http")
+async def crawler_detection(request: Request, call_next):
+    if request.method == "GET" and request.url.path in _AUTH_GATED_PATHS:
+        ua = request.headers.get("user-agent", "")
+        if _KNOWN_CRAWLERS.search(ua):
+            return Response(status_code=204)
+    return await call_next(request)
+
+
 app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=5)
 
 

devplacepy/routers/seo.py

@@ -17,6 +17,7 @@ async def robots_txt(request: Request):
 Disallow: /auth/
 Disallow: /messages/
 Disallow: /notifications/
+Disallow: /game/
 Disallow: /votes/
 Disallow: /avatar/
 Disallow: /follow/

devplacepy/templates/_comment.html

@@ -1,4 +1,4 @@
-{% macro comment_cell(item, depth, parent_author=None, parent_url=None) %}
+{% macro comment_cell(item, depth, parent_author=None) %}
 <div class="comment" style="--comment-depth: {{ depth }};" data-depth="{{ depth }}">
     <div class="comment-votes" role="group" aria-label="Comment votes">
         <form method="POST" action="/votes/comment/{{ item.comment['uid'] }}">
@@ -26,7 +26,6 @@
         {% endif %}
         <div class="comment-actions">
             <button type="button" class="comment-action-btn" data-action="reply"{{ guest_disabled(user) }}><span class="icon">💬</span><span class="label"> Reply</span></button>
-            <button type="button" class="comment-action-btn" data-share="{{ parent_url }}#comment-{{ item.comment['uid'] }}"><span class="icon">&#x1F517;</span><span class="label"> Share</span></button>
             {% if owns(item.comment, user) %}
             <button type="button" class="comment-action-btn" data-action="edit" data-edit-url="/comments/edit/{{ item.comment['uid'] }}"><span class="icon">✏️</span><span class="label"> Edit</span></button>
             <form method="POST" action="/comments/delete/{{ item.comment['uid'] }}" class="inline-form comment-delete-form" data-comment-uid="{{ item.comment['uid'] }}">
@@ -40,21 +39,21 @@
 </div>
 {% endmacro %}
 
-{% macro render_comment(item, depth=0, parent_url=None) %}
+{% macro render_comment(item, depth=0) %}
 {% set max_depth = 2 %}
 {% set visual_depth = depth if depth < max_depth else max_depth %}
-{% call comment_cell(item, visual_depth, parent_url=parent_url) %}
+{% call comment_cell(item, visual_depth) %}
     {% if item.children %}
     {% if depth < max_depth %}
     <div class="comment-replies">
         {% for child in item.children %}
-        {{ render_comment(child, depth + 1, parent_url) }}
+        {{ render_comment(child, depth + 1) }}
         {% endfor %}
     </div>
     {% else %}
     <div class="comment-replies comment-replies-flat">
         {% for child in item.children %}
-        {{ render_comment_flat(child, max_depth, item.author, parent_url) }}
+        {{ render_comment_flat(child, max_depth, item.author) }}
         {% endfor %}
     </div>
     {% endif %}
@@ -62,9 +61,9 @@
 {% endcall %}
 {% endmacro %}
 
-{% macro render_comment_flat(item, depth, parent_author, parent_url=None) %}
+{% macro render_comment_flat(item, depth, parent_author) %}
-{% call comment_cell(item, depth, parent_author, parent_url=parent_url) %}{% endcall %}
+{% call comment_cell(item, depth, parent_author) %}{% endcall %}
 {% for child in item.children %}
-{{ render_comment_flat(child, depth, item.author, parent_url) }}
+{{ render_comment_flat(child, depth, item.author) }}
 {% endfor %}
 {% endmacro %}

devplacepy/templates/_comment_section.html

@@ -3,10 +3,8 @@
 
     {% from "_comment.html" import render_comment with context %}
 
-    {% set parent_url = resolve_object_url(target_type, target_uid) %}
-
     {% for item in comments %}
-    {{ render_comment(item, 0, parent_url) }}
+    {{ render_comment(item, 0) }}
     {% else %}
     <p class="no-comments-msg">No comments yet. Start the discussion.</p>
     {% endfor %}

devplacepy/templating.py

@@ -9,7 +9,6 @@ from devplacepy.cache import TTLCache
 from devplacepy.config import STATIC_VERSION, TEMPLATES_DIR, TEMPLATE_AUTO_RELOAD, PRESENCE_TIMEOUT_SECONDS
 from devplacepy.constants import TOPICS, REACTION_EMOJI
 from devplacepy.database import get_int_setting, get_setting, get_table
-from devplacepy.database.content import resolve_object_url
 from devplacepy.avatar import avatar_url, avatar_seed
 from devplacepy.utils import format_date as _format_date
 from devplacepy.utils import time_ago as _time_ago
@@ -50,7 +49,6 @@ templates.env.globals["is_self"] = is_self
 templates.env.globals["guest_disabled"] = guest_disabled
 templates.env.globals["is_online"] = presence.is_online
 templates.env.globals["presence_timeout"] = PRESENCE_TIMEOUT_SECONDS
-templates.env.globals["resolve_object_url"] = resolve_object_url
 
 from devplacepy.docs_devrant import devrant_endpoints
 

tests/api/robotstxt.py

@@ -196,7 +196,8 @@ def test_robots_txt_exists(app_server):
     assert "Sitemap:" in r.text
 
 
-def test_robots_disallows_admin_and_uploads(app_server):
+def test_robots_disallows_auth_gated_paths(app_server):
     r = requests.get(f"{BASE_URL}/robots.txt")
-    assert "Disallow: /admin/" in r.text
+    required = ["/auth/", "/game/", "/messages/", "/notifications/", "/admin/", "/uploads/"]
-    assert "Disallow: /uploads/" in r.text
+    for path in required:
+        assert f"Disallow: {path}" in r.text, f"Missing Disallow: {path}"

tests/api/test_crawler_middleware.py

@@ -0,0 +1,50 @@
+# retoor <retoor@molodetz.nl>
+
+import requests
+from tests.conftest import BASE_URL
+
+
+def test_bot_blocked_on_auth_path(app_server):
+    headers = {"User-Agent": "Google-Read-Aloud"}
+    r = requests.get(f"{BASE_URL}/messages", headers=headers)
+    assert r.status_code == 204
+    assert r.content == b""
+
+
+def test_bot_blocked_on_game_path(app_server):
+    headers = {"User-Agent": "Google-Read-Aloud"}
+    r = requests.get(f"{BASE_URL}/game", headers=headers)
+    assert r.status_code == 204
+    assert r.content == b""
+
+
+def test_bot_blocked_on_admin_path(app_server):
+    headers = {"User-Agent": "Google-Read-Aloud"}
+    r = requests.get(f"{BASE_URL}/admin", headers=headers)
+    assert r.status_code == 204
+    assert r.content == b""
+
+
+def test_bot_not_blocked_on_public_path(app_server):
+    headers = {"User-Agent": "Google-Read-Aloud"}
+    r = requests.get(f"{BASE_URL}/", headers=headers)
+    assert r.status_code == 200
+
+
+def test_normal_user_not_blocked(app_server):
+    r = requests.get(f"{BASE_URL}/messages", allow_redirects=False)
+    assert r.status_code == 303
+
+
+def test_known_crawler_user_agent_variants(app_server):
+    ua_list = [
+        "Googlebot",
+        "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+        "bingbot/2.0; +http://www.bing.com/bingbot.htm",
+        "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; ...)",
+        "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
+    ]
+    for ua in ua_list:
+        headers = {"User-Agent": ua}
+        r = requests.get(f"{BASE_URL}/admin", headers=headers)
+        assert r.status_code == 204, f"Expected 204 for UA: {ua}"