Update

.gitea/workflows/test.yaml

@@ -37,14 +37,14 @@ jobs:
 
       - name: Publish coverage HTML
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: coverage-html
           path: htmlcov/
 
       - name: Upload test screenshots
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
           name: failure-screenshots
           path: /tmp/devplace_test_screenshots/

devplacepy/database.py

@@ -26,6 +26,12 @@ db = dataset.connect(
 )
 
 
+def refresh_snapshot() -> None:
+    connection = db.executable
+    if connection.in_transaction() and not db.in_transaction:
+        connection.commit()
+
+
 def _index(db, table, name, columns):
     try:
         if table in db.tables:
@@ -73,6 +79,13 @@ def init_db():
     _index(db, "news", "idx_news_external_id", ["external_id"])
     _index(db, "news", "idx_news_synced_at", ["synced_at"])
     _index(db, "news", "idx_news_status", ["status"])
+
+    news_images = get_table("news_images")
+    if not news_images.has_column("news_uid"):
+        news_images.create_column_by_example("news_uid", "")
+    if not news_images.has_column("url"):
+        news_images.create_column_by_example("url", "")
+
     _index(db, "news_images", "idx_news_images_news_uid", ["news_uid"])
     _index(db, "news_sync", "idx_news_sync_external_id", ["external_id"])
 
@@ -332,6 +345,8 @@ def get_news_images_by_uids(news_uids: list) -> dict:
     if not news_uids or "news_images" not in db.tables:
         return {}
     images_table = db["news_images"]
+    if not images_table.has_column("news_uid"):
+        return {}
     rows = images_table.find(images_table.table.columns.news_uid.in_(news_uids), order_by=["uid"])
     result = {}
     for r in rows:

devplacepy/main.py

@@ -9,7 +9,7 @@ from fastapi.responses import HTMLResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.exceptions import RequestValidationError
 from devplacepy.config import STATIC_DIR, PORT, SERVICE_LOCK_FILE
-from devplacepy.database import init_db, get_table, db, get_users_by_uids, get_comment_counts_by_post_uids, get_vote_counts, get_news_images_by_uids
+from devplacepy.database import init_db, get_table, db, refresh_snapshot, get_users_by_uids, get_comment_counts_by_post_uids, get_vote_counts, get_news_images_by_uids
 from devplacepy.templating import templates
 from devplacepy.utils import get_current_user, time_ago
 from devplacepy.seo import base_seo_context, site_url, website_schema
@@ -132,6 +132,12 @@ app.include_router(services_router.router, prefix="/admin/services")
 app.include_router(uploads.router, prefix="/uploads")
 
 
+@app.middleware("http")
+async def refresh_db_snapshot(request: Request, call_next):
+    refresh_snapshot()
+    return await call_next(request)
+
+
 @app.middleware("http")
 async def add_security_headers(request: Request, call_next):
     response = await call_next(request)

devplacepy/seo.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import os
 import time
 from urllib.parse import urlencode
 from xml.etree.ElementTree import Element, tostring
@@ -12,7 +13,7 @@ logger = logging.getLogger(__name__)
 
 SITE_NAME = "DevPlace"
 SITEMAP_URL_LIMIT = 5000
-SITEMAP_TTL = 3600
+SITEMAP_TTL = int(os.environ.get("DEVPLACE_SITEMAP_TTL", "3600"))
 _sitemap_cache = {}
 
 

tests/conftest.py

@@ -1,4 +1,4 @@
-import os, sys, tempfile, subprocess, time
+import os, sys, tempfile, subprocess, time, asyncio, threading
 from pathlib import Path
 import pytest
 
@@ -26,6 +26,33 @@ os.environ["DEVPLACE_DATABASE_URL"] = f"sqlite:///{_TEST_DB.name}"
 os.environ["SECRET_KEY"] = "test-secret-key"
 os.environ["DEVPLACE_DISABLE_SERVICES"] = "1"
 os.environ["DEVPLACE_RATE_LIMIT"] = "1000000"
+os.environ["DEVPLACE_SITEMAP_TTL"] = "0"
+
+
+def run_async(coro):
+    box = {}
+
+    def runner():
+        try:
+            box["value"] = asyncio.run(coro)
+        except BaseException as exc:
+            box["error"] = exc
+
+    thread = threading.Thread(target=runner)
+    thread.start()
+    thread.join()
+    from devplacepy.database import refresh_snapshot
+    refresh_snapshot()
+    if "error" in box:
+        raise box["error"]
+    return box["value"]
+
+
+@pytest.fixture(autouse=True)
+def _fresh_db_snapshot():
+    from devplacepy.database import refresh_snapshot
+    refresh_snapshot()
+    yield
 
 
 def save_failure_screenshot(page, test_name):

tests/test_news_service.py

@@ -1,11 +1,10 @@
-import asyncio
-
 import httpx
 
 from devplacepy.services import news as news_mod
 from devplacepy.services.news import NewsService, _extract_grade, _get_ai_key, _get_article_images
 from devplacepy.database import get_table
 from devplacepy.utils import generate_uid
+from tests.conftest import run_async
 
 API_URL = "http://news.test/api"
 AI_URL = "http://ai.test/v1/chat"
@@ -120,14 +119,14 @@ def test_get_ai_key_empty_when_unset(monkeypatch):
 
 def test_grade_article_empty_content_returns_none(local_db, monkeypatch):
     monkeypatch.setattr(news_mod, "get_setting", _settings_stub())
-    grade = asyncio.run(NewsService()._grade_article(
+    grade = run_async(NewsService()._grade_article(
         {"title": "EmptyArticle", "description": "d", "content": "c"}, AI_URL, "m", FakeClient([])))
     assert grade is None
 
 
 def test_grade_article_unparseable_returns_none(local_db, monkeypatch):
     monkeypatch.setattr(news_mod, "get_setting", _settings_stub())
-    grade = asyncio.run(NewsService()._grade_article(
+    grade = run_async(NewsService()._grade_article(
         {"title": "BadArticle", "description": "d", "content": "c"}, AI_URL, "m", FakeClient([])))
     assert grade is None
 
@@ -135,7 +134,7 @@ def test_grade_article_unparseable_returns_none(local_db, monkeypatch):
 def test_run_once_handles_api_failure(local_db, monkeypatch):
     monkeypatch.setattr(news_mod, "get_setting", _settings_stub())
     monkeypatch.setattr(news_mod.httpx, "AsyncClient", lambda *a, **k: FailingApiClient([]))
-    asyncio.run(NewsService().run_once())
+    run_async(NewsService().run_once())
 
 
 def test_run_once_updates_existing_news_row(local_db, monkeypatch):
@@ -150,7 +149,7 @@ def test_run_once_updates_existing_news_row(local_db, monkeypatch):
     monkeypatch.setattr(news_mod, "get_setting", _settings_stub(threshold="7"))
     monkeypatch.setattr(news_mod.httpx, "AsyncClient", lambda *a, **k: FakeClient(articles))
 
-    asyncio.run(NewsService().run_once())
+    run_async(NewsService().run_once())
 
     row = get_table("news").find_one(uid=existing_uid)
     assert row["title"] == "HighArticle"
@@ -171,7 +170,7 @@ def test_get_article_images_filters_and_dedupes():
         return FakeResp(text=html)
 
     client.get = fake_get
-    images = asyncio.run(_get_article_images("http://news.test/page", client))
+    images = run_async(_get_article_images("http://news.test/page", client))
     assert [img["url"] for img in images] == ["http://img.test/a.png"]
 
 
@@ -182,7 +181,7 @@ def test_get_article_images_network_error_returns_empty():
         raise httpx.HTTPError("down")
 
     client.get = failing_get
-    assert asyncio.run(_get_article_images("http://news.test/page", client)) == []
+    assert run_async(_get_article_images("http://news.test/page", client)) == []
 
 
 def test_run_once_publishes_grades_and_is_idempotent(local_db, monkeypatch):
@@ -200,7 +199,7 @@ def test_run_once_publishes_grades_and_is_idempotent(local_db, monkeypatch):
     monkeypatch.setattr(news_mod, "get_setting", _settings_stub(threshold="7"))
     monkeypatch.setattr(news_mod.httpx, "AsyncClient", lambda *a, **k: FakeClient(articles))
 
-    asyncio.run(NewsService().run_once())
+    run_async(NewsService().run_once())
 
     news = get_table("news")
     sync = get_table("news_sync")
@@ -214,6 +213,6 @@ def test_run_once_publishes_grades_and_is_idempotent(local_db, monkeypatch):
     assert sync.find_one(external_id=g_fail)["status"] == "grading_failed"
     assert sync.find_one(external_id=g_high)["status"] == "graded"
 
-    asyncio.run(NewsService().run_once())
+    run_async(NewsService().run_once())
     assert news.count(external_id=g_high) == 1
     assert news.count(external_id=g_fail) == 1

tests/test_seo.py

@@ -152,11 +152,12 @@ def _seed_news():
     from devplacepy.database import get_table
     from devplacepy.utils import generate_uid, make_combined_slug
     uid = generate_uid()
-    slug = make_combined_slug("SEO Test News Article", uid)
+    title = f"SEO Test News Article {uid.split('-')[-1]}"
+    slug = make_combined_slug(title, uid)
     get_table("news").insert({
         "uid": uid,
         "slug": slug,
-        "title": "SEO Test News Article",
+        "title": title,
         "description": "A seeded news article for SEO tests.",
         "content": "Body content for the seeded article.",
         "url": "https://example.com/article",