ticket #26 attempt 1

devplacepy/database/comments.py

@@ -93,6 +93,36 @@ def get_recent_comments_by_target_uids(target_type, target_uids, limit=3, user=N
     raw = _drop_blocked(raw, user)
     if not raw:
         return {}
+
+    # Fetch missing parent comments so the threaded hierarchy is preserved even
+    # when a parent is outside the row-number limit.
+    if "comments" in db.tables:
+        comments_table = db["comments"]
+        current_uids = {c["uid"] for c in raw}
+        seen = set(current_uids)
+        to_fetch = []
+        for c in raw:
+            puid = c.get("parent_uid")
+            if puid and puid not in seen:
+                seen.add(puid)
+                to_fetch.append(puid)
+        if to_fetch:
+            ancestors = []
+            queue = list(to_fetch)
+            fetched = set()
+            while queue:
+                uid = queue.pop(0)
+                if uid in fetched or uid in current_uids:
+                    continue
+                row = comments_table.find_one(uid=uid, deleted_at=None)
+                if row:
+                    ancestors.append(row)
+                    fetched.add(uid)
+                    grandparent = row.get("parent_uid")
+                    if grandparent and grandparent not in fetched and grandparent not in current_uids:
+                        queue.append(grandparent)
+            raw.extend(ancestors)
+
     items = _build_comment_items(raw, user)
     by_target = defaultdict(list)
     for c in raw:

devplacepy/main.py

@@ -4,13 +4,12 @@ import asyncio
 import fcntl
 import logging
 import os
-import re
 import time
 from collections import defaultdict
 from contextlib import asynccontextmanager, contextmanager
 from pathlib import Path
 from fastapi import FastAPI, Request
-from fastapi.responses import HTMLResponse, RedirectResponse, Response
+from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.exceptions import RequestValidationError
 from starlette.middleware.gzip import GZipMiddleware
@@ -612,22 +611,6 @@ async def response_timing(request: Request, call_next):
     return response
 
 
-_KNOWN_CRAWLERS = re.compile(
-    r"(google.*read.*aloud|googlebot|bingbot|slurp|duckduckbot|baiduspider|yandexbot)",
-    re.IGNORECASE,
-)
-_AUTH_GATED_PATHS = frozenset({"/messages", "/notifications", "/game", "/admin"})
-
-
-@app.middleware("http")
-async def crawler_detection(request: Request, call_next):
-    if request.method == "GET" and request.url.path in _AUTH_GATED_PATHS:
-        ua = request.headers.get("user-agent", "")
-        if _KNOWN_CRAWLERS.search(ua):
-            return Response(status_code=204)
-    return await call_next(request)
-
-
 app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=5)
 
 

devplacepy/routers/seo.py

@@ -17,7 +17,6 @@ async def robots_txt(request: Request):
 Disallow: /auth/
 Disallow: /messages/
 Disallow: /notifications/
-Disallow: /game/
 Disallow: /votes/
 Disallow: /avatar/
 Disallow: /follow/

tests/api/robotstxt.py

@@ -196,8 +196,7 @@ def test_robots_txt_exists(app_server):
     assert "Sitemap:" in r.text
 
 
-def test_robots_disallows_auth_gated_paths(app_server):
+def test_robots_disallows_admin_and_uploads(app_server):
     r = requests.get(f"{BASE_URL}/robots.txt")
-    required = ["/auth/", "/game/", "/messages/", "/notifications/", "/admin/", "/uploads/"]
-    for path in required:
-        assert f"Disallow: {path}" in r.text, f"Missing Disallow: {path}"
+    assert "Disallow: /admin/" in r.text
+    assert "Disallow: /uploads/" in r.text

tests/api/test_crawler_middleware.py

@@ -1,50 +0,0 @@
-# retoor <retoor@molodetz.nl>
-
-import requests
-from tests.conftest import BASE_URL
-
-
-def test_bot_blocked_on_auth_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/messages", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_blocked_on_game_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/game", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_blocked_on_admin_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/admin", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_not_blocked_on_public_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/", headers=headers)
-    assert r.status_code == 200
-
-
-def test_normal_user_not_blocked(app_server):
-    r = requests.get(f"{BASE_URL}/messages", allow_redirects=False)
-    assert r.status_code == 303
-
-
-def test_known_crawler_user_agent_variants(app_server):
-    ua_list = [
-        "Googlebot",
-        "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-        "bingbot/2.0; +http://www.bing.com/bingbot.htm",
-        "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; ...)",
-        "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
-    ]
-    for ua in ua_list:
-        headers = {"User-Agent": ua}
-        r = requests.get(f"{BASE_URL}/admin", headers=headers)
-        assert r.status_code == 204, f"Expected 204 for UA: {ua}"

tests/e2e/feed.py

@@ -959,3 +959,72 @@ def test_feed_scroll_not_restored_on_fresh_visit(alice):
     page.goto(f"{BASE_URL}/feed", wait_until="domcontentloaded")
     page.wait_for_timeout(600)
     assert page.evaluate("window.scrollY") < 60
+
+
+def _seed_post_with_orphaned_parent():
+    owner = str(uuid4())
+    get_table("users").insert(
+        {
+            "uid": owner,
+            "username": f"opseed_{owner[:8]}",
+            "email": f"{owner[:8]}@op.seed",
+            "password_hash": "x",
+            "role": "Member",
+            "is_active": True,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+    )
+    post_uid = str(uuid4())
+    marker = f"oppost-{post_uid[:8]}"
+    get_table("posts").insert(
+        {
+            "deleted_at": None,
+            "deleted_by": None,
+            "uid": post_uid,
+            "user_uid": owner,
+            "slug": make_combined_slug(marker, post_uid),
+            "title": None,
+            "content": marker,
+            "topic": "devlog",
+            "project_uid": None,
+            "image": None,
+            "stars": 0,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+    )
+    orphan_parent = str(uuid4())
+    base = datetime.now(timezone.utc)
+    rows = [
+        (f"{marker}-orphan-parent", orphan_parent, None, -3),
+        (f"{marker}-older", str(uuid4()), None, -2),
+        (f"{marker}-flat", str(uuid4()), None, -1),
+        (f"{marker}-reply", str(uuid4()), orphan_parent, 0),
+    ]
+    comments = get_table("comments")
+    for content, cuid, par, off in rows:
+        comments.insert(
+            {
+                "deleted_at": None,
+                "deleted_by": None,
+                "uid": cuid,
+                "target_type": "post",
+                "target_uid": post_uid,
+                "post_uid": post_uid,
+                "user_uid": owner,
+                "content": content,
+                "parent_uid": par,
+                "created_at": (base + timedelta(seconds=off)).isoformat(),
+            }
+        )
+    return marker
+
+
+def test_feed_preserves_comment_hierarchy_with_orphaned_parent(page, app_server):
+    marker = _seed_post_with_orphaned_parent()
+    page.goto(f"{BASE_URL}/feed", wait_until="domcontentloaded")
+    card = page.locator(".post-card").filter(has_text=marker).first
+    card.wait_for(state="visible")
+    expect(card.locator(".post-card-comments")).to_contain_text(f"{marker}-orphan-parent")
+    expect(card.locator(".post-card-comments")).to_contain_text(f"{marker}-reply")
+    nested = card.locator(".post-card-comments .comment-replies .comment-text")
+    expect(nested).to_contain_text(f"{marker}-reply")

tests/e2e/gists.py

@@ -468,3 +468,70 @@ def test_gists_list_preserves_comment_hierarchy(page, app_server):
     expect(card.locator(".post-card-comments")).not_to_contain_text(f"{marker}-excluded")
     nested = card.locator(".post-card-comments .comment-replies .comment-text")
     expect(nested).to_contain_text(f"{marker}-reply")
+
+
+def _seed_gist_with_orphaned_parent():
+    owner = str(uuid4())
+    get_table("users").insert(
+        {
+            "uid": owner,
+            "username": f"gopseed_{owner[:8]}",
+            "email": f"{owner[:8]}@gop.seed",
+            "password_hash": "x",
+            "role": "Member",
+            "is_active": True,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+    )
+    uid = str(uuid4())
+    marker = f"gistop-{uid[:8]}"
+    get_table("gists").insert(
+        {
+            "deleted_at": None,
+            "deleted_by": None,
+            "uid": uid,
+            "user_uid": owner,
+            "slug": make_combined_slug(marker, uid),
+            "title": marker,
+            "description": "Gist with orphaned parent hierarchy.",
+            "source_code": "print('x')",
+            "language": "python",
+            "stars": 0,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+    )
+    orphan_parent = str(uuid4())
+    base = datetime.now(timezone.utc)
+    rows = [
+        (f"{marker}-orphan-parent", orphan_parent, None, -3),
+        (f"{marker}-older", str(uuid4()), None, -2),
+        (f"{marker}-flat", str(uuid4()), None, -1),
+        (f"{marker}-reply", str(uuid4()), orphan_parent, 0),
+    ]
+    comments = get_table("comments")
+    for content, cuid, par, off in rows:
+        comments.insert(
+            {
+                "deleted_at": None,
+                "deleted_by": None,
+                "uid": cuid,
+                "target_type": "gist",
+                "target_uid": uid,
+                "user_uid": owner,
+                "content": content,
+                "parent_uid": par,
+                "created_at": (base + timedelta(seconds=off)).isoformat(),
+            }
+        )
+    return marker
+
+
+def test_gists_list_preserves_comment_hierarchy_with_orphaned_parent(page, app_server):
+    marker = _seed_gist_with_orphaned_parent()
+    page.goto(f"{BASE_URL}/gists", wait_until="domcontentloaded")
+    card = page.locator(".gist-card").filter(has_text=marker).first
+    card.wait_for(state="visible")
+    expect(card.locator(".post-card-comments")).to_contain_text(f"{marker}-orphan-parent")
+    expect(card.locator(".post-card-comments")).to_contain_text(f"{marker}-reply")
+    nested = card.locator(".post-card-comments .comment-replies .comment-text")
+    expect(nested).to_contain_text(f"{marker}-reply")

tests/e2e/news.py

@@ -387,3 +387,75 @@ def test_news_list_preserves_comment_hierarchy(page, app_server):
     expect(card.locator(".post-card-comments")).not_to_contain_text(f"{marker}-excluded")
     nested = card.locator(".post-card-comments .comment-replies .comment-text")
     expect(nested).to_contain_text(f"{marker}-reply")
+
+
+def _seed_news_with_orphaned_parent():
+    owner = str(uuid4())
+    get_table("users").insert(
+        {
+            "uid": owner,
+            "username": f"nopseed_{owner[:8]}",
+            "email": f"{owner[:8]}@nop.seed",
+            "password_hash": "x",
+            "role": "Member",
+            "is_active": True,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+    )
+    uid = str(uuid4())
+    marker = f"newsop-{uid[:8]}"
+    get_table("news").insert(
+        {
+            "deleted_at": None,
+            "deleted_by": None,
+            "uid": uid,
+            "slug": make_combined_slug(marker, uid),
+            "title": marker,
+            "description": "News with orphaned parent hierarchy.",
+            "content": "Body.",
+            "url": "https://example.com/nop",
+            "source_name": "NOPSource",
+            "external_id": f"nop_{uid[:8]}",
+            "status": "published",
+            "show_on_landing": 0,
+            "grade": 10,
+            "synced_at": datetime.now(timezone.utc).isoformat(),
+        }
+    )
+    orphan_parent = str(uuid4())
+    base = datetime.now(timezone.utc)
+    rows = [
+        (f"{marker}-orphan-parent", orphan_parent, None, -3),
+        (f"{marker}-older", str(uuid4()), None, -2),
+        (f"{marker}-flat", str(uuid4()), None, -1),
+        (f"{marker}-reply", str(uuid4()), orphan_parent, 0),
+    ]
+    comments = get_table("comments")
+    for content, cuid, par, off in rows:
+        comments.insert(
+            {
+                "deleted_at": None,
+                "deleted_by": None,
+                "uid": cuid,
+                "target_type": "news",
+                "target_uid": uid,
+                "user_uid": owner,
+                "content": content,
+                "parent_uid": par,
+                "created_at": (base + timedelta(seconds=off)).isoformat(),
+            }
+        )
+    return marker
+
+
+def test_news_list_preserves_comment_hierarchy_with_orphaned_parent(page, app_server):
+    from playwright.sync_api import expect
+
+    marker = _seed_news_with_orphaned_parent()
+    page.goto(f"{BASE_URL}/news", wait_until="domcontentloaded")
+    card = page.locator(".news-card").filter(has_text=marker).first
+    card.wait_for(state="visible")
+    expect(card.locator(".post-card-comments")).to_contain_text(f"{marker}-orphan-parent")
+    expect(card.locator(".post-card-comments")).to_contain_text(f"{marker}-reply")
+    nested = card.locator(".post-card-comments .comment-replies .comment-text")
+    expect(nested).to_contain_text(f"{marker}-reply")

tests/e2e/projects/index.py

@@ -1015,3 +1015,71 @@ def test_containers_menu_hidden_for_non_owner_admin_on_member_private_project(pa
     page.goto(f"{BASE_URL}/projects/{slug}", wait_until="domcontentloaded")
     page.locator(".project-actions-more").click()
     expect(page.locator(".context-menu-item:has-text('Containers')")).to_have_count(0)
+
+
+def _seed_project_with_orphaned_parent():
+    owner = str(uuid4())
+    get_table("users").insert(
+        {
+            "uid": owner,
+            "username": f"popseed_{owner[:8]}",
+            "email": f"{owner[:8]}@pop.seed",
+            "password_hash": "x",
+            "role": "Member",
+            "is_active": True,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+    )
+    uid = str(uuid4())
+    marker = f"projop-{uid[:8]}"
+    get_table("projects").insert(
+        {
+            "deleted_at": None,
+            "deleted_by": None,
+            "uid": uid,
+            "user_uid": owner,
+            "slug": make_combined_slug(marker, uid),
+            "title": marker,
+            "description": "Project with orphaned parent hierarchy.",
+            "project_type": "software",
+            "platforms": "Linux",
+            "status": "Released",
+            "stars": 0,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        }
+    )
+    orphan_parent = str(uuid4())
+    base = datetime.now(timezone.utc)
+    rows = [
+        (f"{marker}-orphan-parent", orphan_parent, None, -3),
+        (f"{marker}-older", str(uuid4()), None, -2),
+        (f"{marker}-flat", str(uuid4()), None, -1),
+        (f"{marker}-reply", str(uuid4()), orphan_parent, 0),
+    ]
+    comments = get_table("comments")
+    for content, cuid, par, off in rows:
+        comments.insert(
+            {
+                "deleted_at": None,
+                "deleted_by": None,
+                "uid": cuid,
+                "target_type": "project",
+                "target_uid": uid,
+                "user_uid": owner,
+                "content": content,
+                "parent_uid": par,
+                "created_at": (base + timedelta(seconds=off)).isoformat(),
+            }
+        )
+    return marker
+
+
+def test_projects_list_preserves_comment_hierarchy_with_orphaned_parent(page, app_server):
+    marker = _seed_project_with_orphaned_parent()
+    page.goto(f"{BASE_URL}/projects", wait_until="domcontentloaded")
+    card = page.locator(".project-card").filter(has_text=marker).first
+    card.wait_for(state="visible")
+    expect(card.locator(".post-card-comments")).to_contain_text(f"{marker}-orphan-parent")
+    expect(card.locator(".post-card-comments")).to_contain_text(f"{marker}-reply")
+    nested = card.locator(".post-card-comments .comment-replies .comment-text")
+    expect(nested).to_contain_text(f"{marker}-reply")