From 2eb1e6a004f4b161491dd10168db47ad7e74f0fb Mon Sep 17 00:00:00 2001 From: retoor Date: Fri, 5 Jun 2026 21:51:36 +0200 Subject: [PATCH] Update --- .gitea/workflows/test.yaml | 4 ++-- devplacepy/database.py | 15 +++++++++++++++ devplacepy/main.py | 8 +++++++- devplacepy/seo.py | 3 ++- tests/conftest.py | 29 ++++++++++++++++++++++++++++- tests/test_news_service.py | 19 +++++++++---------- tests/test_seo.py | 5 +++-- 7 files changed, 66 insertions(+), 17 deletions(-) diff --git a/.gitea/workflows/test.yaml b/.gitea/workflows/test.yaml index 3c7532f..3726302 100644 --- a/.gitea/workflows/test.yaml +++ b/.gitea/workflows/test.yaml @@ -37,14 +37,14 @@ jobs: - name: Publish coverage HTML if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: coverage-html path: htmlcov/ - name: Upload test screenshots if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: failure-screenshots path: /tmp/devplace_test_screenshots/ diff --git a/devplacepy/database.py b/devplacepy/database.py index bfd5e34..9a287be 100644 --- a/devplacepy/database.py +++ b/devplacepy/database.py @@ -26,6 +26,12 @@ db = dataset.connect( ) +def refresh_snapshot() -> None: + connection = db.executable + if connection.in_transaction() and not db.in_transaction: + connection.commit() + + def _index(db, table, name, columns): try: if table in db.tables: @@ -73,6 +79,13 @@ def init_db(): _index(db, "news", "idx_news_external_id", ["external_id"]) _index(db, "news", "idx_news_synced_at", ["synced_at"]) _index(db, "news", "idx_news_status", ["status"]) + + news_images = get_table("news_images") + if not news_images.has_column("news_uid"): + news_images.create_column_by_example("news_uid", "") + if not news_images.has_column("url"): + news_images.create_column_by_example("url", "") + _index(db, "news_images", "idx_news_images_news_uid", ["news_uid"]) _index(db, "news_sync", "idx_news_sync_external_id", ["external_id"]) @@ -332,6 +345,8 @@ def get_news_images_by_uids(news_uids: list) -> dict: if not news_uids or "news_images" not in db.tables: return {} images_table = db["news_images"] + if not images_table.has_column("news_uid"): + return {} rows = images_table.find(images_table.table.columns.news_uid.in_(news_uids), order_by=["uid"]) result = {} for r in rows: diff --git a/devplacepy/main.py b/devplacepy/main.py index 348e243..fdf6a97 100644 --- a/devplacepy/main.py +++ b/devplacepy/main.py @@ -9,7 +9,7 @@ from fastapi.responses import HTMLResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from fastapi.exceptions import RequestValidationError from devplacepy.config import STATIC_DIR, PORT, SERVICE_LOCK_FILE -from devplacepy.database import init_db, get_table, db, get_users_by_uids, get_comment_counts_by_post_uids, get_vote_counts, get_news_images_by_uids +from devplacepy.database import init_db, get_table, db, refresh_snapshot, get_users_by_uids, get_comment_counts_by_post_uids, get_vote_counts, get_news_images_by_uids from devplacepy.templating import templates from devplacepy.utils import get_current_user, time_ago from devplacepy.seo import base_seo_context, site_url, website_schema @@ -132,6 +132,12 @@ app.include_router(services_router.router, prefix="/admin/services") app.include_router(uploads.router, prefix="/uploads") +@app.middleware("http") +async def refresh_db_snapshot(request: Request, call_next): + refresh_snapshot() + return await call_next(request) + + @app.middleware("http") async def add_security_headers(request: Request, call_next): response = await call_next(request) diff --git a/devplacepy/seo.py b/devplacepy/seo.py index 92e8fe3..5e97a29 100644 --- a/devplacepy/seo.py +++ b/devplacepy/seo.py @@ -1,5 +1,6 @@ import json import logging +import os import time from urllib.parse import urlencode from xml.etree.ElementTree import Element, tostring @@ -12,7 +13,7 @@ logger = logging.getLogger(__name__) SITE_NAME = "DevPlace" SITEMAP_URL_LIMIT = 5000 -SITEMAP_TTL = 3600 +SITEMAP_TTL = int(os.environ.get("DEVPLACE_SITEMAP_TTL", "3600")) _sitemap_cache = {} diff --git a/tests/conftest.py b/tests/conftest.py index 705c196..85dac73 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -import os, sys, tempfile, subprocess, time +import os, sys, tempfile, subprocess, time, asyncio, threading from pathlib import Path import pytest @@ -26,6 +26,33 @@ os.environ["DEVPLACE_DATABASE_URL"] = f"sqlite:///{_TEST_DB.name}" os.environ["SECRET_KEY"] = "test-secret-key" os.environ["DEVPLACE_DISABLE_SERVICES"] = "1" os.environ["DEVPLACE_RATE_LIMIT"] = "1000000" +os.environ["DEVPLACE_SITEMAP_TTL"] = "0" + + +def run_async(coro): + box = {} + + def runner(): + try: + box["value"] = asyncio.run(coro) + except BaseException as exc: + box["error"] = exc + + thread = threading.Thread(target=runner) + thread.start() + thread.join() + from devplacepy.database import refresh_snapshot + refresh_snapshot() + if "error" in box: + raise box["error"] + return box["value"] + + +@pytest.fixture(autouse=True) +def _fresh_db_snapshot(): + from devplacepy.database import refresh_snapshot + refresh_snapshot() + yield def save_failure_screenshot(page, test_name): diff --git a/tests/test_news_service.py b/tests/test_news_service.py index 3ce45f2..51c5614 100644 --- a/tests/test_news_service.py +++ b/tests/test_news_service.py @@ -1,11 +1,10 @@ -import asyncio - import httpx from devplacepy.services import news as news_mod from devplacepy.services.news import NewsService, _extract_grade, _get_ai_key, _get_article_images from devplacepy.database import get_table from devplacepy.utils import generate_uid +from tests.conftest import run_async API_URL = "http://news.test/api" AI_URL = "http://ai.test/v1/chat" @@ -120,14 +119,14 @@ def test_get_ai_key_empty_when_unset(monkeypatch): def test_grade_article_empty_content_returns_none(local_db, monkeypatch): monkeypatch.setattr(news_mod, "get_setting", _settings_stub()) - grade = asyncio.run(NewsService()._grade_article( + grade = run_async(NewsService()._grade_article( {"title": "EmptyArticle", "description": "d", "content": "c"}, AI_URL, "m", FakeClient([]))) assert grade is None def test_grade_article_unparseable_returns_none(local_db, monkeypatch): monkeypatch.setattr(news_mod, "get_setting", _settings_stub()) - grade = asyncio.run(NewsService()._grade_article( + grade = run_async(NewsService()._grade_article( {"title": "BadArticle", "description": "d", "content": "c"}, AI_URL, "m", FakeClient([]))) assert grade is None @@ -135,7 +134,7 @@ def test_grade_article_unparseable_returns_none(local_db, monkeypatch): def test_run_once_handles_api_failure(local_db, monkeypatch): monkeypatch.setattr(news_mod, "get_setting", _settings_stub()) monkeypatch.setattr(news_mod.httpx, "AsyncClient", lambda *a, **k: FailingApiClient([])) - asyncio.run(NewsService().run_once()) + run_async(NewsService().run_once()) def test_run_once_updates_existing_news_row(local_db, monkeypatch): @@ -150,7 +149,7 @@ def test_run_once_updates_existing_news_row(local_db, monkeypatch): monkeypatch.setattr(news_mod, "get_setting", _settings_stub(threshold="7")) monkeypatch.setattr(news_mod.httpx, "AsyncClient", lambda *a, **k: FakeClient(articles)) - asyncio.run(NewsService().run_once()) + run_async(NewsService().run_once()) row = get_table("news").find_one(uid=existing_uid) assert row["title"] == "HighArticle" @@ -171,7 +170,7 @@ def test_get_article_images_filters_and_dedupes(): return FakeResp(text=html) client.get = fake_get - images = asyncio.run(_get_article_images("http://news.test/page", client)) + images = run_async(_get_article_images("http://news.test/page", client)) assert [img["url"] for img in images] == ["http://img.test/a.png"] @@ -182,7 +181,7 @@ def test_get_article_images_network_error_returns_empty(): raise httpx.HTTPError("down") client.get = failing_get - assert asyncio.run(_get_article_images("http://news.test/page", client)) == [] + assert run_async(_get_article_images("http://news.test/page", client)) == [] def test_run_once_publishes_grades_and_is_idempotent(local_db, monkeypatch): @@ -200,7 +199,7 @@ def test_run_once_publishes_grades_and_is_idempotent(local_db, monkeypatch): monkeypatch.setattr(news_mod, "get_setting", _settings_stub(threshold="7")) monkeypatch.setattr(news_mod.httpx, "AsyncClient", lambda *a, **k: FakeClient(articles)) - asyncio.run(NewsService().run_once()) + run_async(NewsService().run_once()) news = get_table("news") sync = get_table("news_sync") @@ -214,6 +213,6 @@ def test_run_once_publishes_grades_and_is_idempotent(local_db, monkeypatch): assert sync.find_one(external_id=g_fail)["status"] == "grading_failed" assert sync.find_one(external_id=g_high)["status"] == "graded" - asyncio.run(NewsService().run_once()) + run_async(NewsService().run_once()) assert news.count(external_id=g_high) == 1 assert news.count(external_id=g_fail) == 1 diff --git a/tests/test_seo.py b/tests/test_seo.py index ae6399e..9b55fd3 100644 --- a/tests/test_seo.py +++ b/tests/test_seo.py @@ -152,11 +152,12 @@ def _seed_news(): from devplacepy.database import get_table from devplacepy.utils import generate_uid, make_combined_slug uid = generate_uid() - slug = make_combined_slug("SEO Test News Article", uid) + title = f"SEO Test News Article {uid.split('-')[-1]}" + slug = make_combined_slug(title, uid) get_table("news").insert({ "uid": uid, "slug": slug, - "title": "SEO Test News Article", + "title": title, "description": "A seeded news article for SEO tests.", "content": "Body content for the seeded article.", "url": "https://example.com/article",