ticket #95 attempt 2

devplacepy/main.py

@@ -4,13 +4,12 @@ import asyncio
 import fcntl
 import logging
 import os
-import re
 import time
 from collections import defaultdict
 from contextlib import asynccontextmanager, contextmanager
 from pathlib import Path
 from fastapi import FastAPI, Request
-from fastapi.responses import HTMLResponse, RedirectResponse, Response
+from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.exceptions import RequestValidationError
 from starlette.middleware.gzip import GZipMiddleware
@@ -612,22 +611,6 @@ async def response_timing(request: Request, call_next):
     return response
 
 
-_KNOWN_CRAWLERS = re.compile(
-    r"(google.*read.*aloud|googlebot|bingbot|slurp|duckduckbot|baiduspider|yandexbot)",
-    re.IGNORECASE,
-)
-_AUTH_GATED_PATHS = frozenset({"/messages", "/notifications", "/game", "/admin"})
-
-
-@app.middleware("http")
-async def crawler_detection(request: Request, call_next):
-    if request.method == "GET" and request.url.path in _AUTH_GATED_PATHS:
-        ua = request.headers.get("user-agent", "")
-        if _KNOWN_CRAWLERS.search(ua):
-            return Response(status_code=204)
-    return await call_next(request)
-
-
 app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=5)
 
 

devplacepy/routers/seo.py

@@ -17,7 +17,6 @@ async def robots_txt(request: Request):
 Disallow: /auth/
 Disallow: /messages/
 Disallow: /notifications/
-Disallow: /game/
 Disallow: /votes/
 Disallow: /avatar/
 Disallow: /follow/

devplacepy/static/css/base.css

@@ -513,6 +513,24 @@ img {
     display: inline-block;
 }
 
+.post-action-btn.is-loading,
+.comment-vote-btn.is-loading,
+.comment-action-btn.is-loading,
+.comment-form-submit.is-loading,
+.vote-star.is-loading,
+.poll-option.is-loading {
+    pointer-events: none;
+}
+
+.post-action-btn.is-loading .btn-spinner,
+.comment-vote-btn.is-loading .btn-spinner,
+.comment-action-btn.is-loading .btn-spinner,
+.comment-form-submit.is-loading .btn-spinner,
+.vote-star.is-loading .btn-spinner,
+.poll-option.is-loading .btn-spinner {
+    display: inline-block;
+}
+
 .badge {
     display: inline-flex;
     align-items: center;

devplacepy/static/js/BookmarkManager.js

@@ -22,6 +22,6 @@ export class BookmarkManager extends OptimisticAction {
             if (label) {
                 label.textContent = result.saved ? "Saved" : "Save";
             }
-        });
+        }, button);
     }
 }

devplacepy/static/js/CommentManager.js

@@ -23,7 +23,10 @@ export class CommentManager {
         const wrapper = form.closest(".comment");
         if (!wrapper) return;
         const button = form.querySelector("button[type='submit']");
-        if (button) button.disabled = true;
+        if (button) {
+            button.disabled = true;
+            button.classList.add("is-loading");
+        }
 
         const anchor = wrapper.previousElementSibling
             && wrapper.previousElementSibling.classList.contains("comment")
@@ -38,7 +41,10 @@ export class CommentManager {
             wrapper.remove();
             window.scrollTo({ top: Math.max(anchorTop, 0) });
         } catch (err) {
-            if (button) button.disabled = false;
+            if (button) {
+                button.disabled = false;
+                button.classList.remove("is-loading");
+            }
         }
     }
 
@@ -118,7 +124,10 @@ export class CommentManager {
 
     async submitEdit(form, text) {
         const save = form.querySelector("button[type='submit']");
-        if (save) save.disabled = true;
+        if (save) {
+            save.disabled = true;
+            save.classList.add("is-loading");
+        }
         const content = form.querySelector("textarea").value;
         try {
             const result = await Http.send(form.action, { content });
@@ -132,7 +141,10 @@ export class CommentManager {
             form.remove();
             text.style.display = "";
         } catch (err) {
-            if (save) save.disabled = false;
+            if (save) {
+                save.disabled = false;
+                save.classList.remove("is-loading");
+            }
         }
     }
 
@@ -205,7 +217,10 @@ export class CommentManager {
         }
         form.addEventListener("submit", () => {
             const btn = form.querySelector("button[type='submit']");
-            if (btn) btn.disabled = true;
+            if (btn) {
+                btn.disabled = true;
+                btn.classList.add("is-loading");
+            }
         });
     }
 }

devplacepy/static/js/OptimisticAction.js

@@ -4,7 +4,11 @@ import { Http } from "./Http.js";
 import { Toast } from "./Toast.js";
 
 export class OptimisticAction {
-    async submit(url, params, errorTarget, render) {
+    async submit(url, params, errorTarget, render, button = null) {
+        if (button) {
+            button.disabled = true;
+            button.classList.add("is-loading");
+        }
         try {
             const result = await Http.sendForm(url, params, { silent: true });
             if (render) render(result);
@@ -13,10 +17,19 @@ export class OptimisticAction {
             console.error("optimistic action failed", error);
             if (errorTarget) Toast.flash(errorTarget, "Error", 1500);
             return null;
+        } finally {
+            if (button) {
+                button.disabled = false;
+                button.classList.remove("is-loading");
+            }
         }
     }
 
-    async submitOptimistic(url, params, errorTarget, apply, revert, reconcile) {
+    async submitOptimistic(url, params, errorTarget, apply, revert, reconcile, button = null) {
+        if (button) {
+            button.disabled = true;
+            button.classList.add("is-loading");
+        }
         apply();
         try {
             const result = await Http.sendForm(url, params, { silent: true });
@@ -27,6 +40,11 @@ export class OptimisticAction {
             revert();
             if (errorTarget) Toast.flash(errorTarget, "Error", 1500);
             return null;
+        } finally {
+            if (button) {
+                button.disabled = false;
+                button.classList.remove("is-loading");
+            }
         }
     }
 }

devplacepy/static/js/PollManager.js

@@ -42,7 +42,7 @@ export class PollManager extends OptimisticAction {
         }
         const pollUid = poll.dataset.pollUid;
         const optionUid = option.dataset.optionUid;
-        await this.submit(`/polls/${pollUid}/vote`, { option_uid: optionUid }, null, (result) => this.render(poll, result));
+        await this.submit(`/polls/${pollUid}/vote`, { option_uid: optionUid }, null, (result) => this.render(poll, result), option);
     }
 
     render(poll, result) {

devplacepy/static/js/ReactionBar.js

@@ -75,7 +75,7 @@ export class ReactionBar extends OptimisticAction {
         const type = bar.dataset.reactionType;
         const uid = bar.dataset.reactionUid;
         const emoji = trigger.dataset.reactionEmoji;
-        await this.submit(`/reactions/${type}/${uid}`, { emoji }, null, (result) => this.render(bar, result));
+        await this.submit(`/reactions/${type}/${uid}`, { emoji }, null, (result) => this.render(bar, result), trigger);
         this.closeAllPalettes();
     }
 

devplacepy/static/js/VoteManager.js

@@ -34,6 +34,7 @@ export class VoteManager extends OptimisticAction {
             () => this.render(action, { net: predictedNet, value: newValue }),
             () => this.render(action, { net: prevNet, value: prevValue }),
             (result) => this.render(action, result),
+            button,
         );
     }
 

devplacepy/templates/_bookmark_button.html

@@ -1,3 +1,3 @@
 <button type="button" class="post-action-btn bookmark-btn{% if _bookmarked %} bookmarked{% endif %}" data-bookmark-type="{{ _type }}" data-bookmark-uid="{{ _uid }}" title="Save" aria-label="Save" aria-pressed="{% if _bookmarked %}true{% else %}false{% endif %}"{{ guest_disabled(user) }}>
-    <span class="bookmark-icon">&#x1F516;</span> <span class="bookmark-label">{% if _bookmarked %}Saved{% else %}Save{% endif %}</span>
+    <span class="btn-spinner" aria-hidden="true"></span><span class="bookmark-icon">&#x1F516;</span> <span class="bookmark-label">{% if _bookmarked %}Saved{% else %}Save{% endif %}</span>
 </button>

devplacepy/templates/_comment.html

@@ -3,12 +3,12 @@
     <div class="comment-votes" role="group" aria-label="Comment votes">
         <form method="POST" action="/votes/comment/{{ item.comment['uid'] }}">
             <input type="hidden" name="value" value="1">
-            <button type="submit" class="comment-vote-btn vote-up{% if item.my_vote == 1 %} voted{% endif %}" aria-label="Upvote" title="Upvote" aria-pressed="{% if item.my_vote == 1 %}true{% else %}false{% endif %}"{{ guest_disabled(user) }}>+</button>
+            <button type="submit" class="comment-vote-btn vote-up{% if item.my_vote == 1 %} voted{% endif %}" aria-label="Upvote" title="Upvote" aria-pressed="{% if item.my_vote == 1 %}true{% else %}false{% endif %}"{{ guest_disabled(user) }}><span class="btn-spinner" aria-hidden="true"></span>+</button>
         </form>
         <span class="comment-vote-count" data-vote-count="{{ item.comment['uid'] }}">{{ item.votes.up - item.votes.down }}</span>
         <form method="POST" action="/votes/comment/{{ item.comment['uid'] }}">
             <input type="hidden" name="value" value="-1">
-            <button type="submit" class="comment-vote-btn vote-down{% if item.my_vote == -1 %} voted{% endif %}" aria-label="Downvote" title="Downvote" aria-pressed="{% if item.my_vote == -1 %}true{% else %}false{% endif %}"{{ guest_disabled(user) }}>-</button>
+            <button type="submit" class="comment-vote-btn vote-down{% if item.my_vote == -1 %} voted{% endif %}" aria-label="Downvote" title="Downvote" aria-pressed="{% if item.my_vote == -1 %}true{% else %}false{% endif %}"{{ guest_disabled(user) }}><span class="btn-spinner" aria-hidden="true"></span>-</button>
         </form>
     </div>
 
@@ -25,11 +25,11 @@
         {% include "_attachment_display.html" %}
         {% endif %}
         <div class="comment-actions">
-            <button type="button" class="comment-action-btn" data-action="reply"{{ guest_disabled(user) }}><span class="icon">💬</span><span class="label"> Reply</span></button>
+            <button type="button" class="comment-action-btn" data-action="reply"{{ guest_disabled(user) }}><span class="btn-spinner" aria-hidden="true"></span><span class="icon">💬</span><span class="label"> Reply</span></button>
             {% if owns(item.comment, user) %}
-            <button type="button" class="comment-action-btn" data-action="edit" data-edit-url="/comments/edit/{{ item.comment['uid'] }}"><span class="icon">✏️</span><span class="label"> Edit</span></button>
+            <button type="button" class="comment-action-btn" data-action="edit" data-edit-url="/comments/edit/{{ item.comment['uid'] }}"><span class="btn-spinner" aria-hidden="true"></span><span class="icon">✏️</span><span class="label"> Edit</span></button>
             <form method="POST" action="/comments/delete/{{ item.comment['uid'] }}" class="inline-form comment-delete-form" data-comment-uid="{{ item.comment['uid'] }}">
-                <button type="submit" class="comment-action-btn" data-confirm="Delete this comment?"><span class="icon">🗑️</span><span class="label"> Delete</span></button>
+                <button type="submit" class="comment-action-btn" data-confirm="Delete this comment?"><span class="btn-spinner" aria-hidden="true"></span><span class="icon">🗑️</span><span class="label"> Delete</span></button>
             </form>
             {% endif %}
             {% set _type = "comment" %}{% set _uid = item.comment['uid'] %}{% set _reactions = item.reactions %}{% include "_reaction_bar.html" %}

devplacepy/templates/_comment_form.html

@@ -9,7 +9,7 @@
             max-size="{{ max_upload_size_mb() }}"
             max-files="{{ max_attachments_per_resource() }}"
             allowed-types="{{ allowed_file_types() }}"></dp-upload>
-        <button type="submit" class="comment-form-submit"><span class="icon">&#x1F4E4;</span><span class="label"> Post</span></button>
+        <button type="submit" class="comment-form-submit"><span class="btn-spinner" aria-hidden="true"></span><span class="icon">&#x1F4E4;</span><span class="label"> Post</span></button>
     </div>
 </form>
 {% else %}

devplacepy/templates/_poll.html

@@ -4,6 +4,7 @@
     <div class="poll-options" role="group" aria-label="Poll options">
         {% for opt in _poll.options %}
         <button type="button" class="poll-option{% if _poll.my_choice == opt.uid %} chosen{% endif %}" data-option-uid="{{ opt.uid }}" aria-pressed="{% if _poll.my_choice == opt.uid %}true{% else %}false{% endif %}"{{ guest_disabled(user) }}>
+            <span class="btn-spinner" aria-hidden="true"></span>
             <span class="poll-option-bar" style="--poll-pct: {{ opt.pct }}%;"></span>
             <span class="poll-option-label">{{ render_title(opt.label) }}</span>
             <span class="poll-option-meta">

devplacepy/templates/_post_votes.html

@@ -1,11 +1,11 @@
 <div class="post-votes" role="group" aria-label="Post votes">
     <form method="POST" action="/votes/post/{{ _uid }}" class="inline-form">
         <input type="hidden" name="value" value="-1">
-        <button type="submit" class="post-action-btn vote-down{% if _my_vote == -1 %} voted{% endif %}" aria-label="Downvote" title="Downvote" aria-pressed="{% if _my_vote == -1 %}true{% else %}false{% endif %}"{{ guest_disabled(user) }}>−</button>
+        <button type="submit" class="post-action-btn vote-down{% if _my_vote == -1 %} voted{% endif %}" aria-label="Downvote" title="Downvote" aria-pressed="{% if _my_vote == -1 %}true{% else %}false{% endif %}"{{ guest_disabled(user) }}><span class="btn-spinner" aria-hidden="true"></span>−</button>
     </form>
     <span class="post-vote-count" data-vote-count="{{ _uid }}">{{ _count }}</span>
     <form method="POST" action="/votes/post/{{ _uid }}" class="inline-form">
         <input type="hidden" name="value" value="1">
-        <button type="submit" class="post-action-btn vote-up{% if _my_vote == 1 %} voted{% endif %}" aria-label="Upvote" title="Upvote" aria-pressed="{% if _my_vote == 1 %}true{% else %}false{% endif %}"{{ guest_disabled(user) }}>+</button>
+        <button type="submit" class="post-action-btn vote-up{% if _my_vote == 1 %} voted{% endif %}" aria-label="Upvote" title="Upvote" aria-pressed="{% if _my_vote == 1 %}true{% else %}false{% endif %}"{{ guest_disabled(user) }}><span class="btn-spinner" aria-hidden="true"></span>+</button>
     </form>
 </div>

devplacepy/templates/_star_vote.html

@@ -1,4 +1,4 @@
 <form method="POST" action="/votes/{{ _type }}/{{ _uid }}" class="inline-form"{% if _stop %} data-stop-propagation{% endif %}>
     <input type="hidden" name="value" value="1">
-    <button type="submit" class="{{ _btn_class }} vote-star{% if _my_vote == 1 %} voted{% endif %}" aria-label="Star" aria-pressed="{% if _my_vote == 1 %}true{% else %}false{% endif %}"{{ guest_disabled(user) }}><span class="vote-count-value" data-vote-count="{{ _uid }}">{{ _count }}</span></button>
+    <button type="submit" class="{{ _btn_class }} vote-star{% if _my_vote == 1 %} voted{% endif %}" aria-label="Star" aria-pressed="{% if _my_vote == 1 %}true{% else %}false{% endif %}"{{ guest_disabled(user) }}><span class="btn-spinner" aria-hidden="true"></span><span class="vote-count-value" data-vote-count="{{ _uid }}">{{ _count }}</span></button>
 </form>

tests/api/robotstxt.py

@@ -196,8 +196,7 @@ def test_robots_txt_exists(app_server):
     assert "Sitemap:" in r.text
 
 
-def test_robots_disallows_auth_gated_paths(app_server):
+def test_robots_disallows_admin_and_uploads(app_server):
     r = requests.get(f"{BASE_URL}/robots.txt")
-    required = ["/auth/", "/game/", "/messages/", "/notifications/", "/admin/", "/uploads/"]
+    assert "Disallow: /admin/" in r.text
-    for path in required:
+    assert "Disallow: /uploads/" in r.text
-        assert f"Disallow: {path}" in r.text, f"Missing Disallow: {path}"

tests/api/test_crawler_middleware.py

@@ -1,50 +0,0 @@
-# retoor <retoor@molodetz.nl>
-
-import requests
-from tests.conftest import BASE_URL
-
-
-def test_bot_blocked_on_auth_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/messages", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_blocked_on_game_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/game", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_blocked_on_admin_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/admin", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_not_blocked_on_public_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/", headers=headers)
-    assert r.status_code == 200
-
-
-def test_normal_user_not_blocked(app_server):
-    r = requests.get(f"{BASE_URL}/messages", allow_redirects=False)
-    assert r.status_code == 303
-
-
-def test_known_crawler_user_agent_variants(app_server):
-    ua_list = [
-        "Googlebot",
-        "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-        "bingbot/2.0; +http://www.bing.com/bingbot.htm",
-        "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; ...)",
-        "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
-    ]
-    for ua in ua_list:
-        headers = {"User-Agent": ua}
-        r = requests.get(f"{BASE_URL}/admin", headers=headers)
-        assert r.status_code == 204, f"Expected 204 for UA: {ua}"