ticket #87 attempt 1

devplacepy/main.py

@@ -4,13 +4,12 @@ import asyncio
 import fcntl
 import logging
 import os
-import re
 import time
 from collections import defaultdict
 from contextlib import asynccontextmanager, contextmanager
 from pathlib import Path
 from fastapi import FastAPI, Request
-from fastapi.responses import HTMLResponse, RedirectResponse, Response
+from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.exceptions import RequestValidationError
 from starlette.middleware.gzip import GZipMiddleware
@@ -612,22 +611,6 @@ async def response_timing(request: Request, call_next):
     return response
 
 
-_KNOWN_CRAWLERS = re.compile(
-    r"(google.*read.*aloud|googlebot|bingbot|slurp|duckduckbot|baiduspider|yandexbot)",
-    re.IGNORECASE,
-)
-_AUTH_GATED_PATHS = frozenset({"/messages", "/notifications", "/game", "/admin"})
-
-
-@app.middleware("http")
-async def crawler_detection(request: Request, call_next):
-    if request.method == "GET" and request.url.path in _AUTH_GATED_PATHS:
-        ua = request.headers.get("user-agent", "")
-        if _KNOWN_CRAWLERS.search(ua):
-            return Response(status_code=204)
-    return await call_next(request)
-
-
 app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=5)
 
 

devplacepy/routers/messages.py

@@ -19,7 +19,6 @@ from devplacepy.templating import clear_messages_cache
 from devplacepy.utils import (
     require_user,
     time_ago,
-    is_admin,
     _user_from_session,
     _user_from_api_key,
 )
@@ -153,7 +152,6 @@ async def messages_page(request: Request, with_uid: str = None, search: str = ""
     other_last_seen = None
     if with_uid:
         messages, other_user = get_conversation_messages(user["uid"], with_uid)
-        mark_conversation_read(user["uid"], with_uid)
         mark_notifications_read_by_target(
             user["uid"], f"/messages?with_uid={with_uid}"
         )

devplacepy/routers/seo.py

@@ -17,7 +17,6 @@ async def robots_txt(request: Request):
 Disallow: /auth/
 Disallow: /messages/
 Disallow: /notifications/
-Disallow: /game/
 Disallow: /votes/
 Disallow: /avatar/
 Disallow: /follow/

tests/api/messages/index.py

@@ -115,7 +115,6 @@ def _seed_news_audit_log():
     )
     refresh_snapshot()
     return uid
-from devplacepy.database import get_table
 JSON_content_negotiation = {"Accept": "application/json"}
 _counter_content_negotiation = [0]
 def _session_content_negotiation(password="secret123"):
@@ -180,3 +179,56 @@ def test_messages_conversation_list_avatar_has_presence_dot(app_server):
     assert "conversation-item" in html
     assert "presence-dot" in html
     assert f'data-presence-uid="{other_uid}"' in html
+
+
+def test_notification_open_does_not_mark_messages_read(app_server, seeded_db):
+    """Regression: opening a DM notification must NOT mark the conversation read.
+    The GET /messages?with_uid=... handler should NOT call mark_conversation_read;
+    only the WebSocket markRead mechanism should do that.
+    """
+    sender, sender_name = _member()
+    receiver, receiver_name = _member()
+
+    refresh_snapshot()
+    sender_user = _db_user(sender_name)
+    receiver_user = _db_user(receiver_name)
+
+    send_resp = sender.post(
+        f"{BASE_URL}/messages/send",
+        data={"receiver_uid": receiver_user["uid"], "content": "DM that must stay unread"},
+        allow_redirects=True,
+    )
+    assert send_resp.status_code == 200, send_resp.text[:300]
+
+    refresh_snapshot()
+    msg = get_table("messages").find_one(
+        sender_uid=sender_user["uid"], receiver_uid=receiver_user["uid"]
+    )
+    assert msg is not None, "message should exist"
+    assert msg["read"] == 0, f"expected read=0 after send, got {msg['read']}"
+
+    n_uid = generate_uid()
+    get_table("notifications").insert({
+        "uid": n_uid,
+        "user_uid": receiver_user["uid"],
+        "type": "message",
+        "message": "sent you a message",
+        "related_uid": sender_user["uid"],
+        "target_url": f"/messages?with_uid={sender_user['uid']}",
+        "read": False,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+    })
+
+    receiver.get(f"{BASE_URL}/notifications/open/{n_uid}", allow_redirects=True)
+
+    refresh_snapshot()
+    n = get_table("notifications").find_one(uid=n_uid)
+    assert n is not None, "notification should exist"
+    assert n["read"] in (1, True, "1"), "notification should be marked read"
+
+    refresh_snapshot()
+    msg_after = get_table("messages").find_one(uid=msg["uid"])
+    assert msg_after is not None, "message should still exist"
+    assert msg_after["read"] == 0, (
+        f"expected read=0 after notification open, got {msg_after['read']}"
+    )

tests/api/robotstxt.py

@@ -196,8 +196,7 @@ def test_robots_txt_exists(app_server):
     assert "Sitemap:" in r.text
 
 
-def test_robots_disallows_auth_gated_paths(app_server):
+def test_robots_disallows_admin_and_uploads(app_server):
     r = requests.get(f"{BASE_URL}/robots.txt")
-    required = ["/auth/", "/game/", "/messages/", "/notifications/", "/admin/", "/uploads/"]
+    assert "Disallow: /admin/" in r.text
-    for path in required:
+    assert "Disallow: /uploads/" in r.text
-        assert f"Disallow: {path}" in r.text, f"Missing Disallow: {path}"

tests/api/test_crawler_middleware.py

@@ -1,50 +0,0 @@
-# retoor <retoor@molodetz.nl>
-
-import requests
-from tests.conftest import BASE_URL
-
-
-def test_bot_blocked_on_auth_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/messages", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_blocked_on_game_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/game", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_blocked_on_admin_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/admin", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_not_blocked_on_public_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/", headers=headers)
-    assert r.status_code == 200
-
-
-def test_normal_user_not_blocked(app_server):
-    r = requests.get(f"{BASE_URL}/messages", allow_redirects=False)
-    assert r.status_code == 303
-
-
-def test_known_crawler_user_agent_variants(app_server):
-    ua_list = [
-        "Googlebot",
-        "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-        "bingbot/2.0; +http://www.bing.com/bingbot.htm",
-        "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; ...)",
-        "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
-    ]
-    for ua in ua_list:
-        headers = {"User-Agent": ua}
-        r = requests.get(f"{BASE_URL}/admin", headers=headers)
-        assert r.status_code == 204, f"Expected 204 for UA: {ua}"