diff --git a/devplacepy/services/news/service.py b/devplacepy/services/news/service.py index ff9234a4..7c5db1c3 100644 --- a/devplacepy/services/news/service.py +++ b/devplacepy/services/news/service.py @@ -18,7 +18,9 @@ from devplacepy.services.openai_gateway.usage import ( usage_metric_cards, ) from devplacepy.utils import generate_uid, make_combined_slug +from devplacepy.utils.notifications import create_notification from devplacepy.services.audit import record as audit +from devplacepy.services.openai_gateway.reliability import retry_send from devplacepy.services.seo_meta import schedule_seo_meta from . import _get_ai_key @@ -151,6 +153,32 @@ class NewsService(BaseService): ), group="AI formatting", ), + ConfigField( + "news_fetch_retries", + "News API fetch retries", + type="int", + default=3, + minimum=0, + maximum=10, + help=( + "How many times to retry the upstream news API on failure. " + "Each retry waits longer (linear backoff). Set 0 for no retries." + ), + group="Reliability", + ), + ConfigField( + "news_fetch_retry_backoff_ms", + "News API retry backoff (ms)", + type="int", + default=5000, + minimum=1000, + maximum=60000, + help=( + "Base backoff in milliseconds between retries. The actual " + "delay is backoff * attempt number." + ), + group="Reliability", + ), ] def __init__(self): @@ -165,13 +193,38 @@ class NewsService(BaseService): format_enabled = config["news_format_enabled"] self.log(f"Fetching news from {api_url}") + max_retries = config["news_fetch_retries"] + backoff_ms = config["news_fetch_retry_backoff_ms"] async with stealth.stealth_async_client(timeout=30.0) as client: try: - resp = await client.get(api_url) - resp.raise_for_status() + resp, exc, attempts = await retry_send( + do_call=lambda: client.get(api_url), + max_retries=max_retries, + backoff_ms=backoff_ms, + log=lambda msg: self.log(msg), + ) + if exc is not None: + raise exc + if resp is None or resp.status_code >= 400: + raise httpx.HTTPError(f"status {resp.status_code if resp else 0}") data = resp.json() except Exception as e: - self.log(f"Failed to fetch news API: {e}") + self.log(f"Failed to fetch news API after {attempts} attempts: {e}") + audit.record_system( + "news.service.fetch_failed", + actor_kind="service", + actor_uid="news", + summary=f"News API unreachable after {attempts} attempts", + metadata={"api_url": api_url, "attempts": attempts, "error": str(e)}, + result="failure", + ) + for admin_row in get_table("users").find(role="Admin"): + create_notification( + admin_row["uid"], + "system", + f"News API unreachable after {attempts} attempts", + related_uid="", + ) return articles = data.get("articles", []) diff --git a/tests/unit/services/news.py b/tests/unit/services/news.py index 34698491..7c58fda4 100644 --- a/tests/unit/services/news.py +++ b/tests/unit/services/news.py @@ -67,9 +67,14 @@ class FakeClient_news_service: grade = "9" if "HighArticle" in prompt else "3" return FakeResp_news_service(json_data={"choices": [{"message": {"content": grade}}]}) class FailingApiClient(FakeClient_news_service): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.call_count = 0 + async def get(self, url, timeout=None): + self.call_count += 1 if url == API_URL: - raise httpx.HTTPError("api down") + raise httpx.RequestError("api down") return FakeResp_news_service(text="") GATEWAY_HEADERS = { "X-Gateway-Cost-USD": "0.00010000", @@ -98,7 +103,7 @@ class UsageClient(FakeClient_news_service): json_data={"choices": [{"message": {"content": grade}}]}, headers=GATEWAY_HEADERS, ) -def _settings_stub(threshold="7"): +def _settings_stub(threshold="7", retries="3", backoff="5"): def fake_get_setting(key, default=None): return { "news_api_url": API_URL, @@ -106,6 +111,8 @@ def _settings_stub(threshold="7"): "news_ai_model": "test-model", "news_grade_threshold": threshold, "news_ai_key": "", + "news_fetch_retries": retries, + "news_fetch_retry_backoff_ms": backoff, }.get(key, default) return fake_get_setting @@ -171,12 +178,47 @@ def test_grade_article_unparseable_returns_none(local_db, monkeypatch): def test_run_once_handles_api_failure(local_db, monkeypatch): + admin_uid = generate_uid() + get_table("users").insert({ + "uid": admin_uid, + "username": "testadmin", + "role": "Admin", + "email": "admin@test.test", + "password": "hash", + "api_key": generate_uid(), + "created_at": "2025-01-01T00:00:00Z", + }) monkeypatch.setattr(news_mod, "get_setting", _settings_stub()) monkeypatch.setattr(base_mod, "get_setting", _settings_stub()) + failing_client = FailingApiClient([]) monkeypatch.setattr( - news_mod.httpx, "AsyncClient", lambda *a, **k: FailingApiClient([]) + news_mod.stealth, "stealth_async_client", + lambda *a, **k: failing_client, + ) + notifications = [] + monkeypatch.setattr( + news_mod, "create_notification", + lambda user_uid, notification_type, message, related_uid, target_url=None: ( + notifications.append((user_uid, message)) + ), ) run_async(NewsService().run_once()) + assert failing_client.call_count == 4 + assert len(notifications) == 1 + assert notifications[0][0] == admin_uid + assert "News API unreachable" in notifications[0][1] + + +def test_run_once_handles_api_failure_zero_retries(local_db, monkeypatch): + monkeypatch.setattr(news_mod, "get_setting", _settings_stub(retries="0")) + monkeypatch.setattr(base_mod, "get_setting", _settings_stub(retries="0")) + failing_client = FailingApiClient([]) + monkeypatch.setattr( + news_mod.stealth, "stealth_async_client", + lambda *a, **k: failing_client, + ) + run_async(NewsService().run_once()) + assert failing_client.call_count == 1 def test_run_once_updates_existing_news_row(local_db, monkeypatch):