ticket #66 attempt 1

devplacepy/main.py

@@ -4,13 +4,12 @@ import asyncio
 import fcntl
 import logging
 import os
-import re
 import time
 from collections import defaultdict
 from contextlib import asynccontextmanager, contextmanager
 from pathlib import Path
 from fastapi import FastAPI, Request
-from fastapi.responses import HTMLResponse, RedirectResponse, Response
+from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.exceptions import RequestValidationError
 from starlette.middleware.gzip import GZipMiddleware
@@ -612,22 +611,6 @@ async def response_timing(request: Request, call_next):
     return response
 
 
-_KNOWN_CRAWLERS = re.compile(
-    r"(google.*read.*aloud|googlebot|bingbot|slurp|duckduckbot|baiduspider|yandexbot)",
-    re.IGNORECASE,
-)
-_AUTH_GATED_PATHS = frozenset({"/messages", "/notifications", "/game", "/admin"})
-
-
-@app.middleware("http")
-async def crawler_detection(request: Request, call_next):
-    if request.method == "GET" and request.url.path in _AUTH_GATED_PATHS:
-        ua = request.headers.get("user-agent", "")
-        if _KNOWN_CRAWLERS.search(ua):
-            return Response(status_code=204)
-    return await call_next(request)
-
-
 app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=5)
 
 

devplacepy/routers/seo.py

@@ -17,7 +17,6 @@ async def robots_txt(request: Request):
 Disallow: /auth/
 Disallow: /messages/
 Disallow: /notifications/
-Disallow: /game/
 Disallow: /votes/
 Disallow: /avatar/
 Disallow: /follow/

devplacepy/static/css/gists.css

@@ -328,3 +328,44 @@
         padding: 0.75rem;
     }
 }
+
+/* ---- Markdown source/preview toggle ---- */
+.gist-markdown-toggle .gist-md-tabbar {
+    display: flex;
+    gap: 0;
+    border-bottom: 1px solid var(--border);
+    margin-bottom: 0;
+}
+
+.gist-markdown-toggle .gist-md-tab {
+    padding: 0.5em 1em;
+    cursor: pointer;
+    color: var(--text-muted);
+    border: 1px solid transparent;
+    border-bottom: none;
+    border-radius: var(--radius) var(--radius) 0 0;
+    background: transparent;
+    font-size: 0.875rem;
+}
+
+.gist-markdown-toggle .gist-md-tab.active {
+    color: var(--text-primary);
+    background: var(--bg-card);
+    border-color: var(--border) var(--border) transparent;
+}
+
+.gist-markdown-toggle .tab-pane {
+    display: none;
+}
+
+.gist-markdown-toggle .tab-pane.active {
+    display: block;
+}
+
+.gist-markdown-toggle .gist-markdown-preview {
+    padding: 1em;
+    border: 1px solid var(--border);
+    border-radius: 0 0 var(--radius) var(--radius);
+    min-height: 4em;
+    background: var(--bg-card);
+}

devplacepy/templates/gist_detail.html

@@ -35,6 +35,23 @@
 
         {% if gist['language'] == 'markdown_rendered' %}
         <div class="gist-rendered-content rendered-content">{{ render_content(gist['source_code'], author_is_admin=is_admin(author)) }}</div>
+        {% elif gist['language'] == 'markdown' %}
+        <div class="gist-code-block gist-markdown-toggle" data-tabs>
+            <nav class="gist-md-tabbar" role="tablist">
+                <button type="button" class="gist-md-tab" data-tab="source" role="tab">Source</button>
+                <button type="button" class="gist-md-tab" data-tab="preview" role="tab">Preview</button>
+            </nav>
+            <div data-tab-pane="source" class="tab-pane active" role="tabpanel">
+                <div class="gist-code-header">
+                    <span class="gist-language-name">{{ language_name(gist['language']) }}</span>
+                    <button type="button" class="gist-copy-btn" data-copy="gist-code-content">Copy</button>
+                </div>
+                <pre><code id="gist-code-content" class="language-markdown">{{ gist['source_code'] }}</code></pre>
+            </div>
+            <div data-tab-pane="preview" class="tab-pane gist-markdown-preview rendered-content" role="tabpanel">
+                {{ render_content(gist['source_code'], author_is_admin=is_admin(author)) }}
+            </div>
+        </div>
         {% else %}
         <div class="gist-code-block">
             <div class="gist-code-header">

tests/api/robotstxt.py

@@ -196,8 +196,7 @@ def test_robots_txt_exists(app_server):
     assert "Sitemap:" in r.text
 
 
-def test_robots_disallows_auth_gated_paths(app_server):
+def test_robots_disallows_admin_and_uploads(app_server):
     r = requests.get(f"{BASE_URL}/robots.txt")
-    required = ["/auth/", "/game/", "/messages/", "/notifications/", "/admin/", "/uploads/"]
-    for path in required:
-        assert f"Disallow: {path}" in r.text, f"Missing Disallow: {path}"
+    assert "Disallow: /admin/" in r.text
+    assert "Disallow: /uploads/" in r.text

tests/api/test_crawler_middleware.py

@@ -1,50 +0,0 @@
-# retoor <retoor@molodetz.nl>
-
-import requests
-from tests.conftest import BASE_URL
-
-
-def test_bot_blocked_on_auth_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/messages", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_blocked_on_game_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/game", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_blocked_on_admin_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/admin", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_not_blocked_on_public_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/", headers=headers)
-    assert r.status_code == 200
-
-
-def test_normal_user_not_blocked(app_server):
-    r = requests.get(f"{BASE_URL}/messages", allow_redirects=False)
-    assert r.status_code == 303
-
-
-def test_known_crawler_user_agent_variants(app_server):
-    ua_list = [
-        "Googlebot",
-        "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-        "bingbot/2.0; +http://www.bing.com/bingbot.htm",
-        "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; ...)",
-        "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
-    ]
-    for ua in ua_list:
-        headers = {"User-Agent": ua}
-        r = requests.get(f"{BASE_URL}/admin", headers=headers)
-        assert r.status_code == 204, f"Expected 204 for UA: {ua}"