ticket #72 attempt 2

schema_test: verify migrate_ai_gateway_settings() populates
  gateway_embed_key from OPENROUTER_API_KEY env var.

service_test: verify effective_config() falls back to
  gateway_vision_key when gateway_embed_key is empty.

devplacepy/database/schema.py

@@ -1353,6 +1353,9 @@ def migrate_ai_gateway_settings() -> None:
     if not get_setting("gateway_vision_key", "") and openrouter:
         set_setting("gateway_vision_key", openrouter)
         logger.info("Migrated gateway vision key from environment")
+    if not get_setting("gateway_embed_key", "") and openrouter:
+        set_setting("gateway_embed_key", openrouter)
+        logger.info("Migrated gateway embed key from environment")
     for key in ("news_ai_url", "bot_api_url", "devii_ai_url"):
         if get_setting(key, "") == OLD_GATEWAY_URL:
             set_setting(key, INTERNAL_GATEWAY_URL)

devplacepy/main.py

@@ -4,13 +4,12 @@ import asyncio
 import fcntl
 import logging
 import os
-import re
 import time
 from collections import defaultdict
 from contextlib import asynccontextmanager, contextmanager
 from pathlib import Path
 from fastapi import FastAPI, Request
-from fastapi.responses import HTMLResponse, RedirectResponse, Response
+from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.exceptions import RequestValidationError
 from starlette.middleware.gzip import GZipMiddleware
@@ -612,22 +611,6 @@ async def response_timing(request: Request, call_next):
     return response
 
 
-_KNOWN_CRAWLERS = re.compile(
-    r"(google.*read.*aloud|googlebot|bingbot|slurp|duckduckbot|baiduspider|yandexbot)",
-    re.IGNORECASE,
-)
-_AUTH_GATED_PATHS = frozenset({"/messages", "/notifications", "/game", "/admin"})
-
-
-@app.middleware("http")
-async def crawler_detection(request: Request, call_next):
-    if request.method == "GET" and request.url.path in _AUTH_GATED_PATHS:
-        ua = request.headers.get("user-agent", "")
-        if _KNOWN_CRAWLERS.search(ua):
-            return Response(status_code=204)
-    return await call_next(request)
-
-
 app.add_middleware(GZipMiddleware, minimum_size=512, compresslevel=5)
 
 

devplacepy/routers/seo.py

@@ -17,7 +17,6 @@ async def robots_txt(request: Request):
 Disallow: /auth/
 Disallow: /messages/
 Disallow: /notifications/
-Disallow: /game/
 Disallow: /votes/
 Disallow: /avatar/
 Disallow: /follow/

devplacepy/services/openai_gateway/CLAUDE.md

@@ -76,7 +76,7 @@ The gateway is the only place that holds real provider URLs/models/keys. Every o
 - Defaults live in `config.py`: `INTERNAL_GATEWAY_URL` (`http://localhost:10500/openai/v1/chat/completions`, override with `DEVPLACE_INTERNAL_BASE_URL`) and `INTERNAL_MODEL` (`molodetz`). `news_ai_url`, `bot_api_url`, `devii_ai_url` default to `INTERNAL_GATEWAY_URL`; their model defaults to `molodetz`.
 - Each consumer's key falls back to `database.internal_gateway_key()` (reads the `gateway_internal_key` setting) when its own key field/env is unset - the provider-key fallbacks (`DEEPSEEK_API_KEY`/`OPENROUTER_API_KEY`) were removed from news and bots.
 - `gateway_force_model` (default on) and a `molodetz`/empty alias in `handle_chat` make the upstream always receive `gateway_model`, so `molodetz` is a stable generic alias.
-- `database.migrate_ai_gateway_settings()` (called at the end of `init_db()`, under the startup `init_lock`): generates `gateway_internal_key` (uuid4) if missing; migrates `DEEPSEEK_API_KEY`/`OPENROUTER_API_KEY` env into `gateway_api_key`/`gateway_vision_key` when the db value is empty; and rewrites any consumer AI URL still equal to the old `openai.app.molodetz.nl` default to the gateway, plus `bot_model` `deepseek-chat` -> `molodetz` (only uncustomized values).
+- `database.migrate_ai_gateway_settings()` (called at the end of `init_db()`, under the startup `init_lock`): generates `gateway_internal_key` (uuid4) if missing; migrates `DEEPSEEK_API_KEY`/`OPENROUTER_API_KEY` env into `gateway_api_key`/`gateway_vision_key`/`gateway_embed_key` when the db value is empty; and rewrites any consumer AI URL still equal to the old `openai.app.molodetz.nl` default to the gateway, plus `bot_model` `deepseek-chat` -> `molodetz` (only uncustomized values).
 - The gateway's `gateway_api_key`/`gateway_vision_key`/`gateway_internal_key` fields are **non-secret** so the admin services page shows the value actually in use, editable.
 - Bot LLM calls are synchronous `urllib` but already run via `asyncio.to_thread` (`bot/bot.py`), so the local round-trip never blocks the event loop.
 - Real provider keys/URLs/models live ONLY in the gateway. The gateway is `default_enabled=True`.

tests/api/robotstxt.py

@@ -196,8 +196,7 @@ def test_robots_txt_exists(app_server):
     assert "Sitemap:" in r.text
 
 
-def test_robots_disallows_auth_gated_paths(app_server):
+def test_robots_disallows_admin_and_uploads(app_server):
     r = requests.get(f"{BASE_URL}/robots.txt")
-    required = ["/auth/", "/game/", "/messages/", "/notifications/", "/admin/", "/uploads/"]
-    for path in required:
-        assert f"Disallow: {path}" in r.text, f"Missing Disallow: {path}"
+    assert "Disallow: /admin/" in r.text
+    assert "Disallow: /uploads/" in r.text

tests/api/test_crawler_middleware.py

@@ -1,50 +0,0 @@
-# retoor <retoor@molodetz.nl>
-
-import requests
-from tests.conftest import BASE_URL
-
-
-def test_bot_blocked_on_auth_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/messages", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_blocked_on_game_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/game", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_blocked_on_admin_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/admin", headers=headers)
-    assert r.status_code == 204
-    assert r.content == b""
-
-
-def test_bot_not_blocked_on_public_path(app_server):
-    headers = {"User-Agent": "Google-Read-Aloud"}
-    r = requests.get(f"{BASE_URL}/", headers=headers)
-    assert r.status_code == 200
-
-
-def test_normal_user_not_blocked(app_server):
-    r = requests.get(f"{BASE_URL}/messages", allow_redirects=False)
-    assert r.status_code == 303
-
-
-def test_known_crawler_user_agent_variants(app_server):
-    ua_list = [
-        "Googlebot",
-        "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-        "bingbot/2.0; +http://www.bing.com/bingbot.htm",
-        "Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; ...)",
-        "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
-    ]
-    for ua in ua_list:
-        headers = {"User-Agent": ua}
-        r = requests.get(f"{BASE_URL}/admin", headers=headers)
-        assert r.status_code == 204, f"Expected 204 for UA: {ua}"

tests/unit/services/openai_gateway/schema_test.py

@@ -0,0 +1,11 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import get_setting, set_setting
+from devplacepy.database.schema import migrate_ai_gateway_settings
+
+
+def test_migrate_embed_key_populates_db(local_db, monkeypatch):
+    monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-v1-test-embed-key")
+    set_setting("gateway_embed_key", "")
+    migrate_ai_gateway_settings()
+    assert get_setting("gateway_embed_key", "") == "sk-or-v1-test-embed-key"

tests/unit/services/openai_gateway/service_test.py

@@ -0,0 +1,12 @@
+# retoor <retoor@molodetz.nl>
+
+from devplacepy.database import set_setting
+from devplacepy.services.openai_gateway import GatewayService
+
+
+def test_effective_config_fallback_embed_key(local_db):
+    set_setting("gateway_embed_key", "")
+    set_setting("gateway_vision_key", "sk-or-v1-vision-test-key")
+    svc = GatewayService()
+    cfg = svc.effective_config()
+    assert cfg["gateway_embed_key"] == "sk-or-v1-vision-test-key"