ticket #75 attempt 1

devplacepy/database/notifications.py

@@ -18,7 +18,6 @@ NOTIFICATION_TYPES = [
     {"key": "reminder", "label": "Reminders", "description": "A reminder or scheduled task you asked Devii to run fires"},
     {"key": "harvest_stolen", "label": "Farm raids", "description": "Someone steals a ready build from your Code Farm"},
     {"key": "award", "label": "Awards", "description": "Someone gives you an award on your profile"},
-    {"key": "admin_alert", "label": "Admin alerts", "description": "System alerts requiring admin attention"},
 ]
 
 

devplacepy/main.py

@@ -4,12 +4,13 @@ import asyncio
 import fcntl
 import logging
 import os
+import re
 import time
 from collections import defaultdict
 from contextlib import asynccontextmanager, contextmanager
 from pathlib import Path
 from fastapi import FastAPI, Request
-from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.responses import HTMLResponse, RedirectResponse, Response
 from fastapi.staticfiles import StaticFiles
 from fastapi.exceptions import RequestValidationError
 from starlette.middleware.gzip import GZipMiddleware
@@ -611,6 +612,22 @@ async def response_timing(request: Request, call_next):
     return response
 
 
+_KNOWN_CRAWLERS = re.compile(
+    r"(google.*read.*aloud|googlebot|bingbot|slurp|duckduckbot|baiduspider|yandexbot)",
+    re.IGNORECASE,
+)
+_AUTH_GATED_PATHS = frozenset({"/messages", "/notifications", "/game", "/admin"})
+
+
+@app.middleware("http")
+async def crawler_detection(request: Request, call_next):
+    if request.method == "GET" and request.url.path in _AUTH_GATED_PATHS:
+        ua = request.headers.get("user-agent", "")
+        if _KNOWN_CRAWLERS.search(ua):
+            return Response(status_code=204)
+    return await call_next(request)
+
+
 app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=5)
 
 

devplacepy/routers/devii.py

@@ -23,7 +23,6 @@ from devplacepy.utils import (
     require_user,
 )
 from devplacepy.services.audit import record as audit
-from devplacepy.services.devii.quota import notify_admins_quota_blocked
 
 logger = logging.getLogger("devii.router")
 
@@ -280,7 +279,6 @@ async def devii_ws(websocket: WebSocket):
                         summary=f"AI request by {username or owner_id} blocked - 24h quota reached",
                         metadata={"limit_usd": limit, "owner_kind": owner_kind},
                     )
-                    notify_admins_quota_blocked(owner_kind, owner_id, username)
                     await websocket.send_json(
                         {
                             "type": "error",

devplacepy/routers/seo.py

@@ -17,6 +17,7 @@ async def robots_txt(request: Request):
 Disallow: /auth/
 Disallow: /messages/
 Disallow: /notifications/
+Disallow: /game/
 Disallow: /votes/
 Disallow: /avatar/
 Disallow: /follow/

devplacepy/services/devii/quota.py

@@ -1,26 +0,0 @@
-# retoor <retoor@molodetz.nl>
-
-import logging
-
-from devplacepy.database import get_table
-from devplacepy.utils.notifications import create_notification
-
-logger = logging.getLogger("devii.quota")
-
-ADMIN_ALERT_TYPE = "admin_alert"
-
-
-def notify_admins_quota_blocked(owner_kind: str, owner_id: str, username: str) -> None:
-    if owner_kind != "user":
-        return
-    admins = get_table("users").find(role="Admin")
-    for admin in admins:
-        if admin["uid"] == owner_id:
-            continue
-        create_notification(
-            admin["uid"],
-            ADMIN_ALERT_TYPE,
-            f"User {username} blocked - 24h AI quota reached",
-            owner_id,
-            "/admin/users",
-        )

devplacepy/services/telegram/bridge.py

@@ -10,7 +10,6 @@ from typing import Any
 
 from devplacepy.database import get_table
 from devplacepy.services.audit import record as audit
-from devplacepy.services.devii.quota import notify_admins_quota_blocked
 from devplacepy.services.manager import service_manager
 from devplacepy.utils import is_admin, is_primary_admin
 
@@ -401,7 +400,6 @@ class TelegramBridge:
                 summary=f"Telegram AI request by {user.get('username', owner_id)} blocked - 24h quota reached",
                 metadata={"limit_usd": limit},
             )
-            notify_admins_quota_blocked("user", owner_id, user.get("username", ""))
             await self._service.send(
                 chat_id, "Your daily AI quota is reached (100%). Please try again later."
             )

tests/api/robotstxt.py

@@ -196,7 +196,8 @@ def test_robots_txt_exists(app_server):
     assert "Sitemap:" in r.text
 
 
-def test_robots_disallows_admin_and_uploads(app_server):
+def test_robots_disallows_auth_gated_paths(app_server):
     r = requests.get(f"{BASE_URL}/robots.txt")
-    assert "Disallow: /admin/" in r.text
+    required = ["/auth/", "/game/", "/messages/", "/notifications/", "/admin/", "/uploads/"]
-    assert "Disallow: /uploads/" in r.text
+    for path in required:
+        assert f"Disallow: {path}" in r.text, f"Missing Disallow: {path}"

tests/api/test_crawler_middleware.py

@@ -0,0 +1,50 @@
+# retoor <retoor@molodetz.nl>
+
+import requests
+from tests.conftest import BASE_URL
+
+
+def test_bot_blocked_on_auth_path(app_server):
+    headers = {"User-Agent": "Google-Read-Aloud"}
+    r = requests.get(f"{BASE_URL}/messages", headers=headers)
+    assert r.status_code == 204
+    assert r.content == b""
+
+
+def test_bot_blocked_on_game_path(app_server):
+    headers = {"User-Agent": "Google-Read-Aloud"}
+    r = requests.get(f"{BASE_URL}/game", headers=headers)
+    assert r.status_code == 204
+    assert r.content == b""
+
+
+def test_bot_blocked_on_admin_path(app_server):
+    headers = {"User-Agent": "Google-Read-Aloud"}
+    r = requests.get(f"{BASE_URL}/admin", headers=headers)
+    assert r.status_code == 204
+    assert r.content == b""
+
+
+def test_bot_not_blocked_on_public_path(app_server):
+    headers = {"User-Agent": "Google-Read-Aloud"}
+    r = requests.get(f"{BASE_URL}/", headers=headers)
+    assert r.status_code == 200
+
+
+def test_normal_user_not_blocked(app_server):
+    r = requests.get(f"{BASE_URL}/messages", allow_redirects=False)
+    assert r.status_code == 303
+
+
+def test_known_crawler_user_agent_variants(app_server):
+    ua_list = [
+        "Googlebot",
+        "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+        "bingbot/2.0; +http://www.bing.com/bingbot.htm",
+        "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; ...)",
+        "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
+    ]
+    for ua in ua_list:
+        headers = {"User-Agent": ua}
+        r = requests.get(f"{BASE_URL}/admin", headers=headers)
+        assert r.status_code == 204, f"Expected 204 for UA: {ua}"