devplacepy/tests/unit/services/news.py

# retoor <retoor@molodetz.nl>

import httpx
from devplacepy.services import news as news_mod
from devplacepy.services import base as base_mod
from devplacepy.services.news import (
    NewsService,
    ImageCandidate,
    LANDING_MIN_SCORE,
    LANDING_MAX,
    UNIQUE_IMAGE_BONUS,
    THIN_CONTENT_PENALTY,
    _extract_grade,
    _get_ai_key,
    _get_article_images,
    _flag_shared_placeholders,
    _primary_image,
    clean_news_text,
    reliability_reason,
    effective_score,
)
from datetime import datetime, timezone
from devplacepy.database import get_table
from devplacepy.utils import generate_uid
from tests.conftest import run_async
API_URL = "http://news.test/api"
AI_URL = "http://ai.test/v1/chat"
LINK_HIGH = "http://news.test/high"
LINK_LOW = "http://news.test/low"
class FakeResp_news_service:
    def __init__(self, json_data=None, text="", status=200):
        self._json = json_data
        self.text = text
        self.status_code = status

    def raise_for_status(self):
        if self.status_code >= 400:
            raise httpx.HTTPError(f"status {self.status_code}")

    def json(self):
        return self._json
class FakeClient_news_service:
    def __init__(self, articles):
        self.articles = articles

    async def __aenter__(self):
        return self

    async def __aexit__(self, *exc):
        return False

    async def get(self, url, timeout=None):
        if url == API_URL:
            return FakeResp_news_service(json_data={"articles": self.articles})
        return FakeResp_news_service(text='<img src="http://img.test/a.png">')

    async def post(self, url, json=None, headers=None, timeout=None):
        prompt = json["messages"][0]["content"]
        if "FailArticle" in prompt:
            return FakeResp_news_service(status=500, text="err")
        if "EmptyArticle" in prompt:
            return FakeResp_news_service(json_data={"choices": [{"message": {"content": ""}}]})
        if "BadArticle" in prompt:
            return FakeResp_news_service(
                json_data={"choices": [{"message": {"content": "no number"}}]}
            )
        grade = "9" if "HighArticle" in prompt else "3"
        return FakeResp_news_service(json_data={"choices": [{"message": {"content": grade}}]})
class FailingApiClient(FakeClient_news_service):
    async def get(self, url, timeout=None):
        if url == API_URL:
            raise httpx.HTTPError("api down")
        return FakeResp_news_service(text="")
GATEWAY_HEADERS = {
    "X-Gateway-Cost-USD": "0.00010000",
    "X-Gateway-Model": "molodetz",
    "X-Gateway-Prompt-Tokens": "500",
    "X-Gateway-Completion-Tokens": "100",
    "X-Gateway-Total-Tokens": "600",
    "X-Gateway-Upstream-Latency-Ms": "400",
    "X-Gateway-Total-Latency-Ms": "450",
}
class FakeRespHeaders(FakeResp_news_service):
    def __init__(self, json_data=None, text="", status=200, headers=None):
        super().__init__(json_data, text, status)
        self.headers = headers or {}
class UsageClient(FakeClient_news_service):
    async def post(self, url, json=None, headers=None, timeout=None):
        prompt = json["messages"][0]["content"]
        if "Reformat" in prompt:
            body = "## Heading\n\n" + ("word " * 80)
            return FakeRespHeaders(
                json_data={"choices": [{"message": {"content": body}}]},
                headers=GATEWAY_HEADERS,
            )
        grade = "9" if "HighArticle" in prompt else "3"
        return FakeRespHeaders(
            json_data={"choices": [{"message": {"content": grade}}]},
            headers=GATEWAY_HEADERS,
        )
def _settings_stub(threshold="7"):
    def fake_get_setting(key, default=None):
        return {
            "news_api_url": API_URL,
            "news_ai_url": AI_URL,
            "news_ai_model": "test-model",
            "news_grade_threshold": threshold,
            "news_ai_key": "",
        }.get(key, default)

    return fake_get_setting


def test_extract_grade_parsing():
    assert _extract_grade("8") == 8
    assert _extract_grade("Grade: 9") == 9
    assert _extract_grade("Score is 7/10") == 7
    assert _extract_grade("0") is None
    assert _extract_grade("11") is None
    assert _extract_grade("not a number") is None


def test_get_ai_key_env_precedence(monkeypatch):
    monkeypatch.setattr(news_mod, "get_setting", _settings_stub())
    monkeypatch.setenv("NEWS_AI_KEY", "primary")
    assert _get_ai_key() == "primary"


def test_get_ai_key_setting_fallback(monkeypatch):
    monkeypatch.delenv("NEWS_AI_KEY", raising=False)
    monkeypatch.setattr(
        news_mod,
        "get_setting",
        lambda key, default=None: "from-setting" if key == "news_ai_key" else default,
    )
    assert _get_ai_key() == "from-setting"


def test_get_ai_key_internal_gateway_fallback(local_db, monkeypatch):
    from devplacepy.database import internal_gateway_key

    monkeypatch.delenv("NEWS_AI_KEY", raising=False)
    monkeypatch.setattr(news_mod, "get_setting", _settings_stub())
    assert _get_ai_key() == internal_gateway_key()


def test_grade_article_empty_content_returns_none(local_db, monkeypatch):
    monkeypatch.setattr(news_mod, "get_setting", _settings_stub())
    grade = run_async(
        NewsService()._grade_article(
            {"title": "EmptyArticle", "description": "d", "content": "c"},
            AI_URL,
            "m",
            FakeClient_news_service([]),
        )
    )
    assert grade is None


def test_grade_article_unparseable_returns_none(local_db, monkeypatch):
    monkeypatch.setattr(news_mod, "get_setting", _settings_stub())
    grade = run_async(
        NewsService()._grade_article(
            {"title": "BadArticle", "description": "d", "content": "c"},
            AI_URL,
            "m",
            FakeClient_news_service([]),
        )
    )
    assert grade is None


def test_run_once_handles_api_failure(local_db, monkeypatch):
    monkeypatch.setattr(news_mod, "get_setting", _settings_stub())
    monkeypatch.setattr(base_mod, "get_setting", _settings_stub())
    monkeypatch.setattr(
        news_mod.httpx, "AsyncClient", lambda *a, **k: FailingApiClient([])
    )
    run_async(NewsService().run_once())


def test_run_once_updates_existing_news_row(local_db, monkeypatch):
    external_id = f"news-{generate_uid()}"
    existing_uid = generate_uid()
    get_table("news").insert(
        {
            "deleted_at": None,
            "deleted_by": None,
            "uid": existing_uid,
            "external_id": external_id,
            "slug": "",
            "title": "Old Title",
            "status": "draft",
            "grade": 0,
            "synced_at": "2020-01-01",
        }
    )
    articles = [
        {
            "guid": external_id,
            "title": "HighArticle deep technical analysis",
            "description": "d" * 150,
            "content": "c" * 150,
            "link": LINK_HIGH,
            "feed_name": "Feed",
            "author": "A",
            "published": "2026-01-01",
        }
    ]
    monkeypatch.setattr(news_mod, "get_setting", _settings_stub(threshold="7"))
    monkeypatch.setattr(base_mod, "get_setting", _settings_stub(threshold="7"))
    monkeypatch.setattr(
        news_mod.httpx, "AsyncClient", lambda *a, **k: FakeClient_news_service(articles)
    )

    run_async(NewsService().run_once())

    row = get_table("news").find_one(uid=existing_uid)
    assert row["title"] == "HighArticle deep technical analysis"
    assert row["status"] == "published"
    assert get_table("news").count(external_id=external_id) == 1


def test_get_article_images_filters_and_dedupes(monkeypatch):
    async def _allow(url):
        return None

    monkeypatch.setattr(news_mod.net_guard, "guard_public_url", _allow)
    html = (
        '<img src="http://img.test/a.png">'
        "<img src='http://img.test/a.png'>"
        '<img src="http://img.test/b.svg">'
        '<img src="/relative.png">'
    )
    client = FakeClient_news_service([])

    async def fake_get(url, timeout=None):
        return FakeResp_news_service(text=html)

    client.get = fake_get
    images = run_async(_get_article_images("http://news.test/page", client))
    assert [img["url"] for img in images] == ["http://img.test/a.png"]


def test_run_once_publishes_grades_and_is_idempotent(local_db, monkeypatch):
    g_high, g_low, g_fail = (f"news-{generate_uid()}" for _ in range(3))
    articles = [
        {
            "guid": g_high,
            "title": "HighArticle definitive guide",
            "description": "d" * 250,
            "content": "c" * 250,
            "link": LINK_HIGH,
            "feed_name": "Feed",
            "author": "A",
            "published": "2026-01-01",
        },
        {
            "guid": g_low,
            "title": "LowArticle filler thoughts",
            "description": "d" * 250,
            "content": "c" * 250,
            "link": LINK_LOW,
            "feed_name": "Feed",
            "author": "A",
            "published": "2026-01-01",
        },
        {
            "guid": g_fail,
            "title": "FailArticle broken feed",
            "description": "d" * 250,
            "content": "c" * 250,
            "link": LINK_HIGH,
            "feed_name": "Feed",
            "author": "A",
            "published": "2026-01-01",
        },
        {
            "guid": "",
            "title": "NoGuid placeholder entry",
            "description": "d" * 250,
            "content": "c" * 250,
            "link": "",
            "feed_name": "Feed",
            "author": "A",
            "published": "2026-01-01",
        },
    ]
    monkeypatch.setattr(news_mod, "get_setting", _settings_stub(threshold="7"))
    monkeypatch.setattr(base_mod, "get_setting", _settings_stub(threshold="7"))
    monkeypatch.setattr(
        news_mod.httpx, "AsyncClient", lambda *a, **k: FakeClient_news_service(articles)
    )

    run_async(NewsService().run_once())

    news = get_table("news")
    sync = get_table("news_sync")
    assert news.find_one(external_id=g_high)["status"] == "published"
    assert news.find_one(external_id=g_high)["grade"] == 9
    assert news.find_one(external_id=g_low)["status"] == "draft"
    assert news.find_one(external_id=g_low)["grade"] == 3
    fail_row = news.find_one(external_id=g_fail)
    assert fail_row["status"] == "draft"
    assert fail_row["grade"] == 0
    assert sync.find_one(external_id=g_fail)["status"] == "grading_failed"
    assert sync.find_one(external_id=g_high)["status"] == "graded"

    run_async(NewsService().run_once())
    assert news.count(external_id=g_high) == 1
    assert news.count(external_id=g_fail) == 1


def test_clean_news_text_strips_reddit_boilerplate():
    raw = (
        "Great article body that is long enough. submitted by /u/someone to "
        "/r/programming [link] [12 comments]"
    )
    cleaned = clean_news_text(raw)
    assert "submitted by" not in cleaned
    assert "[link]" not in cleaned
    assert "comments]" not in cleaned
    assert "Great article body" in cleaned
    assert "  " not in cleaned


def test_clean_news_text_collapses_whitespace_and_html():
    assert clean_news_text("<p>hello   world</p>\n\n[link]") == "hello world"
    assert clean_news_text("") == ""


def test_reliability_reason_gate():
    body = "x" * 300
    assert reliability_reason("A proper title here", body, "https://a.test") == ""
    assert reliability_reason("short", body, "https://a.test") == "short_title"
    assert (
        reliability_reason("A proper title here", "tiny", "https://a.test")
        == "thin_body"
    )
    assert (
        reliability_reason("A proper title here", body, "") == "invalid_url"
    )
    assert (
        reliability_reason("A proper title here", body, "ftp://a.test")
        == "invalid_url"
    )
    assert (
        reliability_reason("THIS IS ALL SHOUTING TITLE", body, "https://a.test")
        == "shouting_title"
    )


def test_effective_score_bonus_and_penalty():
    assert effective_score(6, True, False) == 6 + UNIQUE_IMAGE_BONUS
    assert effective_score(6, False, True) == 6 - THIN_CONTENT_PENALTY
    assert effective_score(6, False, False) == 6
    assert effective_score(10, True, False) == 10
    assert effective_score(1, False, True) == 1


def _stripes_phash():
    from PIL import Image, ImageDraw
    import imagehash

    image = Image.new("RGB", (200, 200), (255, 255, 255))
    draw = ImageDraw.Draw(image)
    for x in range(0, 200, 20):
        draw.rectangle([x, 0, x + 10, 200], fill=(0, 0, 0))
    return str(imagehash.phash(image))


def _circle_phash():
    from PIL import Image, ImageDraw
    import imagehash

    image = Image.new("RGB", (200, 200), (255, 255, 255))
    draw = ImageDraw.Draw(image)
    draw.ellipse([40, 40, 160, 160], fill=(0, 0, 0))
    return str(imagehash.phash(image))


def test_flag_shared_placeholders_marks_cross_article_duplicates():
    shared = _stripes_phash()
    distinct = _circle_phash()
    by_article = {
        "a": [ImageCandidate(url="u1", phash=shared, width=200, height=200, is_placeholder=False)],
        "b": [ImageCandidate(url="u2", phash=shared, width=200, height=200, is_placeholder=False)],
        "c": [ImageCandidate(url="u3", phash=distinct, width=200, height=200, is_placeholder=False)],
    }
    _flag_shared_placeholders(by_article)
    assert by_article["a"][0].is_placeholder is True
    assert by_article["b"][0].is_placeholder is True
    assert by_article["c"][0].is_placeholder is False


def test_flag_shared_placeholders_keeps_same_article_duplicates():
    shared = _stripes_phash()
    by_article = {
        "a": [
            ImageCandidate(url="u1", phash=shared, width=200, height=200, is_placeholder=False),
            ImageCandidate(url="u2", phash=shared, width=200, height=200, is_placeholder=False),
        ]
    }
    _flag_shared_placeholders(by_article)
    assert by_article["a"][0].is_placeholder is False
    assert by_article["a"][1].is_placeholder is False


def test_primary_image_picks_first_non_placeholder():
    candidates = [
        ImageCandidate(url="ph", is_placeholder=True),
        ImageCandidate(url="good", is_placeholder=False),
    ]
    assert _primary_image(candidates) == "good"
    assert _primary_image([ImageCandidate(url="x", is_placeholder=True)]) == ""


def test_grade_article_full_unique_image_bonus():
    service = NewsService()
    article = {
        "title": "A solid technical headline",
        "description": "d" * 250,
        "content": "c" * 250,
        "link": "https://example.test/post",
    }
    unique = [ImageCandidate(url="img", phash="abc", width=400, height=400, is_placeholder=False)]
    result = service._grade_article_full(article, 7, unique)
    assert result.valid is True
    assert result.has_unique_image is True
    assert result.image_url == "img"
    assert result.effective_score == 7 + UNIQUE_IMAGE_BONUS

    placeholder = [ImageCandidate(url="img", is_placeholder=True)]
    result2 = service._grade_article_full(article, 7, placeholder)
    assert result2.has_unique_image is False
    assert result2.effective_score == 7


def test_grade_article_full_gate_invalidates():
    service = NewsService()
    article = {
        "title": "tiny",
        "description": "d",
        "content": "c",
        "link": "https://example.test/post",
    }
    result = service._grade_article_full(article, 9, [])
    assert result.valid is False
    assert result.reject_reason == "short_title"


def _insert_landing_row(news_table, *, grade, featured, unique, locked=0, landing=0):
    uid = generate_uid()
    news_table.insert(
        {
            "deleted_at": None,
            "deleted_by": None,
            "uid": uid,
            "external_id": f"land-{uid}",
            "slug": uid,
            "title": "Landing candidate",
            "status": "published",
            "grade": grade,
            "featured": featured,
            "has_unique_image": unique,
            "landing_locked": locked,
            "show_on_landing": landing,
            "synced_at": datetime.now(timezone.utc).isoformat(),
        }
    )
    return uid


def test_apply_landing_selection_promotes_top_and_respects_lock(local_db):
    news_table = get_table("news")
    news_table.delete()
    top = _insert_landing_row(news_table, grade=10, featured=1, unique=1)
    mid = _insert_landing_row(news_table, grade=LANDING_MIN_SCORE, featured=1, unique=1)
    too_low = _insert_landing_row(news_table, grade=LANDING_MIN_SCORE - 1, featured=1, unique=1)
    locked = _insert_landing_row(
        news_table, grade=10, featured=1, unique=1, locked=1, landing=0
    )

    NewsService()._apply_landing_selection(news_table, 7)

    assert news_table.find_one(uid=top)["show_on_landing"] == 1
    assert news_table.find_one(uid=mid)["show_on_landing"] == 1
    assert news_table.find_one(uid=too_low)["show_on_landing"] == 0
    assert news_table.find_one(uid=locked)["show_on_landing"] == 0


def test_apply_landing_selection_caps_at_landing_max(local_db):
    news_table = get_table("news")
    news_table.delete()
    uids = [
        _insert_landing_row(news_table, grade=10, featured=1, unique=1)
        for _ in range(LANDING_MAX + 3)
    ]
    NewsService()._apply_landing_selection(news_table, 7)
    promoted = sum(
        1 for u in uids if news_table.find_one(uid=u)["show_on_landing"] == 1
    )
    assert promoted == LANDING_MAX


def test_strip_md_fence_unwraps_only_full_fence():
    from devplacepy.services.news import _strip_md_fence

    assert _strip_md_fence("```markdown\n# Hi\n\nBody\n```") == "# Hi\n\nBody"
    assert _strip_md_fence("# Hi\n\nBody") == "# Hi\n\nBody"
    embedded = "Intro\n\n```\ncode\n```"
    assert _strip_md_fence(embedded) == embedded


def test_grade_article_accumulates_usage(local_db, monkeypatch):
    from devplacepy.services.openai_gateway.usage import new_usage_totals

    monkeypatch.setattr(news_mod, "get_setting", _settings_stub())
    totals = new_usage_totals()
    grade = run_async(
        NewsService()._grade_article(
            {"title": "HighArticle deep dive", "description": "d", "content": "c"},
            AI_URL,
            "m",
            UsageClient([]),
            totals,
        )
    )
    assert grade == 9
    assert totals["calls"] == 1
    assert totals["total_tokens"] == 600


def test_format_article_returns_markdown_and_accumulates(local_db, monkeypatch):
    from devplacepy.services.openai_gateway.usage import new_usage_totals

    monkeypatch.setattr(news_mod, "get_setting", _settings_stub())
    totals = new_usage_totals()
    article = {
        "title": "A solid technical headline",
        "description": "d" * 120,
        "content": "c" * 120,
        "link": "https://example.test/post",
    }
    out = run_async(
        NewsService()._format_article(article, AI_URL, "m", UsageClient([]), totals)
    )
    assert out.startswith("## Heading")
    assert totals["calls"] == 1
    assert totals["cost_usd"] > 0


def test_format_article_empty_body_returns_empty(local_db, monkeypatch):
    monkeypatch.setattr(news_mod, "get_setting", _settings_stub())
    out = run_async(
        NewsService()._format_article(
            {"title": "t", "description": "", "content": ""},
            AI_URL,
            "m",
            UsageClient([]),
        )
    )
    assert out == ""


def test_collect_metrics_reports_ai_usage(local_db):
    from devplacepy.database import add_news_usage

    get_table("news_usage").delete()
    add_news_usage(
        {
            "calls": 2,
            "prompt_tokens": 100,
            "completion_tokens": 20,
            "total_tokens": 120,
            "cost_usd": 0.0002,
            "upstream_latency_ms": 300.0,
            "total_latency_ms": 350.0,
        }
    )
    metrics = NewsService().collect_metrics()
    labels = {stat["label"]: stat["value"] for stat in metrics["stats"]}
    assert labels["AI calls"] == 2
    assert labels["Total tokens"] == 120
    assert labels["Total cost"].startswith("$")
    assert labels["Avg cost/call"].startswith("$")