diff --git a/devplacepy/main.py b/devplacepy/main.py index dcaa7787..3382f8c2 100644 --- a/devplacepy/main.py +++ b/devplacepy/main.py @@ -4,12 +4,13 @@ import asyncio import fcntl import logging import os +import re import time from collections import defaultdict from contextlib import asynccontextmanager, contextmanager from pathlib import Path from fastapi import FastAPI, Request -from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.responses import HTMLResponse, RedirectResponse, Response from fastapi.staticfiles import StaticFiles from fastapi.exceptions import RequestValidationError from starlette.middleware.gzip import GZipMiddleware @@ -611,6 +612,22 @@ async def response_timing(request: Request, call_next): return response +_KNOWN_CRAWLERS = re.compile( + r"(google.*read.*aloud|googlebot|bingbot|slurp|duckduckbot|baiduspider|yandexbot)", + re.IGNORECASE, +) +_AUTH_GATED_PATHS = frozenset({"/messages", "/notifications", "/game", "/admin"}) + + +@app.middleware("http") +async def crawler_detection(request: Request, call_next): + if request.method == "GET" and request.url.path in _AUTH_GATED_PATHS: + ua = request.headers.get("user-agent", "") + if _KNOWN_CRAWLERS.search(ua): + return Response(status_code=204) + return await call_next(request) + + app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=5) diff --git a/devplacepy/routers/seo.py b/devplacepy/routers/seo.py index 77d7da4e..698c0473 100644 --- a/devplacepy/routers/seo.py +++ b/devplacepy/routers/seo.py @@ -17,6 +17,7 @@ async def robots_txt(request: Request): Disallow: /auth/ Disallow: /messages/ Disallow: /notifications/ +Disallow: /game/ Disallow: /votes/ Disallow: /avatar/ Disallow: /follow/ diff --git a/tests/api/robotstxt.py b/tests/api/robotstxt.py index 26979954..999c39f2 100644 --- a/tests/api/robotstxt.py +++ b/tests/api/robotstxt.py @@ -196,7 +196,8 @@ def test_robots_txt_exists(app_server): assert "Sitemap:" in r.text -def test_robots_disallows_admin_and_uploads(app_server): +def test_robots_disallows_auth_gated_paths(app_server): r = requests.get(f"{BASE_URL}/robots.txt") - assert "Disallow: /admin/" in r.text - assert "Disallow: /uploads/" in r.text + required = ["/auth/", "/game/", "/messages/", "/notifications/", "/admin/", "/uploads/"] + for path in required: + assert f"Disallow: {path}" in r.text, f"Missing Disallow: {path}" diff --git a/tests/api/test_crawler_middleware.py b/tests/api/test_crawler_middleware.py new file mode 100644 index 00000000..37487cc7 --- /dev/null +++ b/tests/api/test_crawler_middleware.py @@ -0,0 +1,50 @@ +# retoor + +import requests +from tests.conftest import BASE_URL + + +def test_bot_blocked_on_auth_path(app_server): + headers = {"User-Agent": "Google-Read-Aloud"} + r = requests.get(f"{BASE_URL}/messages", headers=headers) + assert r.status_code == 204 + assert r.content == b"" + + +def test_bot_blocked_on_game_path(app_server): + headers = {"User-Agent": "Google-Read-Aloud"} + r = requests.get(f"{BASE_URL}/game", headers=headers) + assert r.status_code == 204 + assert r.content == b"" + + +def test_bot_blocked_on_admin_path(app_server): + headers = {"User-Agent": "Google-Read-Aloud"} + r = requests.get(f"{BASE_URL}/admin", headers=headers) + assert r.status_code == 204 + assert r.content == b"" + + +def test_bot_not_blocked_on_public_path(app_server): + headers = {"User-Agent": "Google-Read-Aloud"} + r = requests.get(f"{BASE_URL}/", headers=headers) + assert r.status_code == 200 + + +def test_normal_user_not_blocked(app_server): + r = requests.get(f"{BASE_URL}/messages", allow_redirects=False) + assert r.status_code == 303 + + +def test_known_crawler_user_agent_variants(app_server): + ua_list = [ + "Googlebot", + "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "bingbot/2.0; +http://www.bing.com/bingbot.htm", + "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; ...)", + "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)", + ] + for ua in ua_list: + headers = {"User-Agent": ua} + r = requests.get(f"{BASE_URL}/admin", headers=headers) + assert r.status_code == 204, f"Expected 204 for UA: {ua}"