ticket #102 attempt 1

devplacepy/main.py

@@ -4,13 +4,12 @@ import asyncio
 import fcntl
 import logging
 import os
-import re
 import time
 from collections import defaultdict
 from contextlib import asynccontextmanager, contextmanager
 from pathlib import Path
 from fastapi import FastAPI, Request
-from fastapi.responses import HTMLResponse, RedirectResponse, Response
+from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.exceptions import RequestValidationError
 from starlette.middleware.gzip import GZipMiddleware
@@ -612,22 +611,6 @@ async def response_timing(request: Request, call_next):
     return response
 
 
-_KNOWN_CRAWLERS = re.compile(
-    r"(google.*read.*aloud|googlebot|bingbot|slurp|duckduckbot|baiduspider|yandexbot)",
-    re.IGNORECASE,
-)
-_AUTH_GATED_PATHS = frozenset({"/messages", "/notifications", "/game", "/admin"})
-
-
-@app.middleware("http")
-async def crawler_detection(request: Request, call_next):
-    if request.method == "GET" and request.url.path in _AUTH_GATED_PATHS:
-        ua = request.headers.get("user-agent", "")
-        if _KNOWN_CRAWLERS.search(ua):
-            return Response(status_code=204)
-    return await call_next(request)
-
-
 app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=5)
 
 

devplacepy/routers/seo.py

@@ -17,7 +17,6 @@ async def robots_txt(request: Request):
 Disallow: /auth/
 Disallow: /messages/
 Disallow: /notifications/
-Disallow: /game/
 Disallow: /votes/
 Disallow: /avatar/
 Disallow: /follow/

devplacepy/services/xmlrpc/__init__.py

@@ -2,8 +2,10 @@
 
 from __future__ import annotations
 
+import socket
 import subprocess
 import sys
+import time
 
 from devplacepy.config import XMLRPC_BIND, XMLRPC_PORT
 from devplacepy.services.base import BaseService
@@ -25,20 +27,48 @@ class XmlrpcService(BaseService):
     def __init__(self) -> None:
         super().__init__("xmlrpc", interval_seconds=XMLRPC_INTERVAL_SECONDS)
         self._process: subprocess.Popen | None = None
+        self._last_stderr: str | None = None
 
     def _alive(self) -> bool:
         return self._process is not None and self._process.poll() is None
 
     def _spawn(self) -> None:
+        self._last_stderr = None
         self._process = subprocess.Popen(
             [sys.executable, "-m", SERVER_MODULE],
             stdout=subprocess.DEVNULL,
-            stderr=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
         )
         self.log(
-            f"Forking XML-RPC server started (pid {self._process.pid}) on "
+            f"Forking XML-RPC server spawning (pid {self._process.pid}) on "
             f"{XMLRPC_BIND}:{XMLRPC_PORT}"
         )
+        time.sleep(0.5)
+        if self._process.poll() is not None:
+            stderr_data = self._process.communicate()[1]
+            if stderr_data:
+                self._last_stderr = stderr_data.decode("utf-8", errors="replace")
+            self.log(
+                f"Forking XML-RPC server died after spawn (pid {self._process.pid}, "
+                f"exit code {self._process.returncode})"
+            )
+            if self._last_stderr:
+                for line in self._last_stderr.strip().split("\n"):
+                    self.log(f"  stderr: {line}")
+            return
+        try:
+            sock = socket.create_connection((XMLRPC_BIND, XMLRPC_PORT), timeout=1)
+            sock.close()
+        except (OSError, socket.timeout):
+            self.log(
+                f"Forking XML-RPC server not yet ready (pid {self._process.pid}) "
+                f"on {XMLRPC_BIND}:{XMLRPC_PORT}"
+            )
+        else:
+            self.log(
+                f"Forking XML-RPC server started (pid {self._process.pid}) on "
+                f"{XMLRPC_BIND}:{XMLRPC_PORT}"
+            )
 
     def _terminate(self) -> None:
         if not self._alive():
@@ -65,6 +95,13 @@ class XmlrpcService(BaseService):
             self.log(f"XML-RPC server healthy (pid {self._process.pid})")
             return
         self.log("XML-RPC server not running, starting it")
+        if self._process is not None:
+            self.log(
+                f"Previous process exited with code {self._process.returncode}"
+            )
+            if self._last_stderr:
+                for line in self._last_stderr.strip().split("\n"):
+                    self.log(f"  last stderr: {line}")
         self._spawn()
 
     def collect_metrics(self) -> dict:

devplacepy/services/xmlrpc/server.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import logging
+import sys
 from socketserver import ForkingMixIn
 from xmlrpc.server import SimpleXMLRPCDispatcher, SimpleXMLRPCRequestHandler, SimpleXMLRPCServer
 
@@ -116,6 +117,9 @@ def main() -> None:
         server.serve_forever()
     except KeyboardInterrupt:
         logger.info("XML-RPC server interrupted")
+    except Exception:
+        logging.exception("XML-RPC server crashed with unhandled exception")
+        sys.exit(1)
     finally:
         server.server_close()
 

tests/api/robotstxt.py

@@ -196,8 +196,7 @@ def test_robots_txt_exists(app_server):
     assert "Sitemap:" in r.text
 
 
-def test_robots_disallows_auth_gated_paths(app_server):
+def test_robots_disallows_admin_and_uploads(app_server):
     r = requests.get(f"{BASE_URL}/robots.txt")
-    required = ["/auth/", "/game/", "/messages/", "/notifications/", "/admin/", "/uploads/"]
+    assert "Disallow: /admin/" in r.text
-    for path in required:
+    assert "Disallow: /uploads/" in r.text
-        assert f"Disallow: {path}" in r.text, f"Missing Disallow: {path}"

tests/api/test_crawler_middleware.py

@@ -1,50 +0,0 @@
-# retoor <retoor@molodetz.nl>
-
-import requests
-from tests.conftest import BASE_URL
-
-
-def test_bot_blocked_on_auth_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/messages", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_blocked_on_game_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/game", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_blocked_on_admin_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/admin", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_not_blocked_on_public_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/", headers=headers)
-    assert r.status_code == 200
-
-
-def test_normal_user_not_blocked(app_server):
-    r = requests.get(f"{BASE_URL}/messages", allow_redirects=False)
-    assert r.status_code == 303
-
-
-def test_known_crawler_user_agent_variants(app_server):
-    ua_list = [
-        "Googlebot",
-        "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-        "bingbot/2.0; +http://www.bing.com/bingbot.htm",
-        "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; ...)",
-        "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
-    ]
-    for ua in ua_list:
-        headers = {"User-Agent": ua}
-        r = requests.get(f"{BASE_URL}/admin", headers=headers)
-        assert r.status_code == 204, f"Expected 204 for UA: {ua}"