From b57e657cf8033928972f3130353788e977d03758 Mon Sep 17 00:00:00 2001 From: retoor Date: Sat, 4 Jul 2026 19:23:51 +0000 Subject: [PATCH] refactor: split news.py into package (ref.md 3.9) Co-Authored-By: Claude Opus 4.8 (1M context) --- devplacepy/services/news/__init__.py | 96 +++++ devplacepy/services/news/clean.py | 79 ++++ devplacepy/services/news/constants.py | 153 +++++++ devplacepy/services/news/images.py | 113 ++++++ devplacepy/services/news/models.py | 24 ++ .../services/{news.py => news/service.py} | 381 ++---------------- 6 files changed, 499 insertions(+), 347 deletions(-) create mode 100644 devplacepy/services/news/__init__.py create mode 100644 devplacepy/services/news/clean.py create mode 100644 devplacepy/services/news/constants.py create mode 100644 devplacepy/services/news/images.py create mode 100644 devplacepy/services/news/models.py rename devplacepy/services/{news.py => news/service.py} (63%) diff --git a/devplacepy/services/news/__init__.py b/devplacepy/services/news/__init__.py new file mode 100644 index 00000000..dfe74932 --- /dev/null +++ b/devplacepy/services/news/__init__.py @@ -0,0 +1,96 @@ +# retoor + +import os +import logging + +from devplacepy.database import get_setting, internal_gateway_key + +logger = logging.getLogger(__name__) + + +def _get_ai_key() -> str: + key = os.environ.get("NEWS_AI_KEY") + if key: + return key + key = get_setting("news_ai_key", "") + if key: + return key + return internal_gateway_key() + + +from .constants import ( + AI_MODEL_DEFAULT, + AI_URL_DEFAULT, + FEATURE_MIN_SCORE, + FORMAT_INPUT_MAX_CHARS, + FORMAT_MAX_TOKENS, + FORMAT_OUTPUT_MAX_CHARS, + FORMAT_PROMPT_SPEC, + GRADE_MAX_TOKENS, + GRADE_PROMPT_SPEC, + GRADE_THRESHOLD_DEFAULT, + GRADING_RULES_DESCRIPTION, + IMG_FETCH_MAX_BYTES, + IMG_FETCH_TIMEOUT, + IMG_PER_ARTICLE, + INTERNAL_GATEWAY_URL, + INTERNAL_MODEL, + JUNK_PATTERNS, + LANDING_MAX, + LANDING_MIN_SCORE, + LANDING_RECENCY_DAYS, + MAX_TITLE_CAPS_RATIO, + MIN_BODY_CHARS, + MIN_IMAGE_DIMENSION, + MIN_TITLE_CHARS, + NEWS_API_URL_DEFAULT, + NEWS_SUMMARY, + PHASH_DISTANCE, + THIN_CONTENT_PENALTY, + UNIQUE_IMAGE_BONUS, + WHITESPACE_PATTERN, + re, +) +from .models import ArticleGrade, ImageCandidate, dataclass, field +from .clean import ( + _caps_ratio, + _extract_grade, + _is_valid_url, + _strip_md_fence, + clean_news_text, + effective_score, + reliability_reason, + strip_html, +) +from .images import ( + Image, + _decode_phash, + _fetch_image_candidate, + _flag_shared_placeholders, + _get_article_images, + _primary_image, + asyncio, + httpx, + imagehash, + io, + net_guard, +) +from .service import ( + BaseService, + ConfigField, + NewsService, + accumulate_usage, + add_news_usage, + audit, + datetime, + generate_uid, + get_news_usage, + get_table, + make_combined_slug, + new_usage_totals, + schedule_seo_meta, + stealth, + timedelta, + timezone, + usage_metric_cards, +) diff --git a/devplacepy/services/news/clean.py b/devplacepy/services/news/clean.py new file mode 100644 index 00000000..2360f41d --- /dev/null +++ b/devplacepy/services/news/clean.py @@ -0,0 +1,79 @@ +# retoor + +import re + +from devplacepy.utils import strip_html + +from .constants import ( + JUNK_PATTERNS, + MAX_TITLE_CAPS_RATIO, + MIN_BODY_CHARS, + MIN_TITLE_CHARS, + THIN_CONTENT_PENALTY, + UNIQUE_IMAGE_BONUS, + WHITESPACE_PATTERN, +) + + +def _extract_grade(text: str) -> int | None: + match = re.search(r"\d+", text.strip()) + if match: + val = int(match.group()) + if 1 <= val <= 10: + return val + return None + + +def _strip_md_fence(text: str) -> str: + stripped = text.strip() + if not stripped.startswith("```"): + return stripped + lines = stripped.splitlines() + if len(lines) < 2 or not lines[-1].strip().startswith("```"): + return stripped + return "\n".join(lines[1:-1]).strip() + + +def clean_news_text(text: str) -> str: + if not text: + return "" + cleaned = strip_html(text) + for pattern in JUNK_PATTERNS: + cleaned = pattern.sub(" ", cleaned) + cleaned = WHITESPACE_PATTERN.sub(" ", cleaned) + return cleaned.strip() + + +def _caps_ratio(title: str) -> float: + letters = [c for c in title if c.isalpha()] + if not letters: + return 0.0 + uppers = sum(1 for c in letters if c.isupper()) + return uppers / len(letters) + + +def _is_valid_url(url: str) -> bool: + return url.startswith("http://") or url.startswith("https://") + + +def reliability_reason(title: str, body: str, url: str) -> str: + if not url or not _is_valid_url(url): + return "invalid_url" + if len(title) < MIN_TITLE_CHARS: + return "short_title" + if len(body) < MIN_BODY_CHARS: + return "thin_body" + if _caps_ratio(title) > MAX_TITLE_CAPS_RATIO: + return "shouting_title" + return "" + + +def effective_score( + ai_grade: int, has_unique_image: bool, body_marginal: bool +) -> int: + score = ai_grade + if has_unique_image: + score += UNIQUE_IMAGE_BONUS + if body_marginal: + score -= THIN_CONTENT_PENALTY + return max(1, min(10, score)) diff --git a/devplacepy/services/news/constants.py b/devplacepy/services/news/constants.py new file mode 100644 index 00000000..6dc1e76c --- /dev/null +++ b/devplacepy/services/news/constants.py @@ -0,0 +1,153 @@ +# retoor + +import re + +from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL + +NEWS_API_URL_DEFAULT = "https://news.app.molodetz.nl/api" +AI_URL_DEFAULT = INTERNAL_GATEWAY_URL +AI_MODEL_DEFAULT = INTERNAL_MODEL +GRADE_THRESHOLD_DEFAULT = 7 +GRADE_MAX_TOKENS = 2000 +FORMAT_MAX_TOKENS = 6000 +FORMAT_INPUT_MAX_CHARS = 14000 +FORMAT_OUTPUT_MAX_CHARS = 30000 + +MIN_TITLE_CHARS = 12 +MIN_BODY_CHARS = 200 +MAX_TITLE_CAPS_RATIO = 0.6 +MIN_IMAGE_DIMENSION = 100 +PHASH_DISTANCE = 5 +IMG_PER_ARTICLE = 5 +IMG_FETCH_MAX_BYTES = 5_000_000 +IMG_FETCH_TIMEOUT = 8.0 +UNIQUE_IMAGE_BONUS = 2 +THIN_CONTENT_PENALTY = 2 +FEATURE_MIN_SCORE = 8 +LANDING_MIN_SCORE = 9 +LANDING_MAX = 6 +LANDING_RECENCY_DAYS = 4 + +JUNK_PATTERNS = ( + re.compile(r"submitted by\s+/?u/\S+(?:\s+to\s+/?r/\S+)?", re.IGNORECASE), + re.compile(r"submitted by\s+\S+\s+to\s+/?r/\S+", re.IGNORECASE), + re.compile(r"\[\s*link\s*\]", re.IGNORECASE), + re.compile(r"\[\s*\d*\s*comments?\s*\]", re.IGNORECASE), + re.compile(r"submitted by[^\[]*\[link\]\s*\[comments?\]", re.IGNORECASE), +) +WHITESPACE_PATTERN = re.compile(r"\s+") + +NEWS_SUMMARY = ( + "Fully automatic, zero-maintenance news pipeline: fetches the feed, cleans " + "each article, fetches and perceptually compares images to reject " + "placeholders, grades deterministically, and auto-rotates Featured and " + "Landing. The full grading algorithm is detailed below." +) + +GRADE_PROMPT_SPEC = ( + "You are an expert content strategist and editor for DevPlace, a " + "server-rendered social network for software developers. Decide how " + "strongly a given article should be published, expressed as a single " + "grade.\n\n" + "Site context:\n" + "- Audience: software developers, hobbyists and professionals, " + "interested in software development, AI agents, open-source tools, " + "systems programming, and self-hosting.\n" + "- Mission: deliver deep technical insight, critical analysis, and " + "practical value; build authority and reader trust; avoid low-effort, " + "rehashed, or clickbait content.\n" + "- Tone: technical, practical, nuanced, skeptical of hype; no " + "sensationalism or unverified claims.\n\n" + "Evaluate the article against these weighted criteria and weigh them " + "internally:\n" + "1. Relevance and alignment to the developer niche and audience needs " + "(20%).\n" + "2. Quality and accuracy: depth, originality, factual correctness, " + "sourcing, technical correctness, clear writing (25%).\n" + "3. Engagement and readability: strong title/intro/flow, useful " + "examples, appropriate length (15%).\n" + "4. Originality and uniqueness: new perspective, data, or analysis; not " + "duplicated generic web content (15%).\n" + "5. SEO and discoverability: keyword opportunity without stuffing, " + "shareable (10%).\n" + "6. Ethics, legality and risk: no misinformation, plagiarism, or " + "brand-safety issues; controversial topics handled with nuance and " + "evidence (10%).\n" + "7. Strategic fit for the site (5%).\n\n" + "Convert your overall judgment to a single integer grade from 1 to 10:\n" + "- 9-10: excellent, publish as-is.\n" + "- 7-8: good, worth publishing.\n" + "- 5-6: marginal, weak fit or quality.\n" + "- 1-4: reject, low quality or off-topic.\n\n" + "Return ONLY a single integer from 1 to 10, nothing else." +) + +FORMAT_PROMPT_SPEC = ( + "You are an expert editor for DevPlace, a social network for software " + "developers. You are given the raw body of a news article that arrived as " + "an undifferentiated wall of text. Reformat it into clean, readable " + "Markdown for a technical audience.\n\n" + "Rules:\n" + "- Break the text into short, well-structured paragraphs separated by a " + "blank line.\n" + "- Add Markdown section headings (## Heading) where the topic clearly " + "shifts, so the article scans well.\n" + "- Use bullet or numbered lists for enumerations, and inline code or " + "fenced code blocks where code, commands, or identifiers appear.\n" + "- Preserve every fact, name, number, and quotation exactly as given. " + "Never invent, add, remove, or reorder information.\n" + "- Only restructure and lightly polish wording for flow and grammar; do " + "not add an introduction, conclusion, opinion, or commentary of your " + "own.\n" + "- Do not repeat the article title as a heading and do not wrap the whole " + "answer in a code fence.\n" + "- Output only the reformatted article body as Markdown, nothing else." +) + +GRADING_RULES_DESCRIPTION = ( + "Each run fetches the feed, cleans every article, fetches and perceptually " + "compares images, grades deterministically, and auto-rotates Featured and " + "Landing.\n\n" + "Content cleaning: after HTML stripping it removes Reddit boilerplate " + "(submitted by /u/, submitted by ... to /r/, standalone [link], " + "[comments], [N comments] and trailing submitted-by [link] [comments]) and " + "collapses whitespace, applied to title (light), description and content " + "before grading and before storage.\n\n" + "Reliability gate (forces draft, never Featured or Landing) when: the url is " + f"empty or invalid; the cleaned title is shorter than {MIN_TITLE_CHARS} chars; " + f"the combined cleaned body is shorter than {MIN_BODY_CHARS} chars; or the " + f"title uppercase-letter ratio exceeds {MAX_TITLE_CAPS_RATIO}.\n\n" + "Image uniqueness: up to " + f"{IMG_PER_ARTICLE} candidate images per article are fetched (max " + f"{IMG_FETCH_MAX_BYTES} bytes), decoded and perceptually hashed (phash). An " + f"image under {MIN_IMAGE_DIMENSION}px on either side, or one that fails to " + "fetch or decode, is a placeholder. Hashes within Hamming distance " + f"{PHASH_DISTANCE} that span two or more different articles are a shared " + "logo or placeholder and are rejected for all of them. An article with at " + "least one genuinely unique image is marked has_unique_image and its first " + "such image becomes the primary image.\n\n" + "Grading prompt: the exact rubric is the editable news_grade_prompt field " + "(Configuration tab), sent to the model at temperature 0 with the cleaned " + "Title, Description and Content appended; it must return a single integer " + "from 1 to 10.\n\n" + "Formatting: after grading, every valid article is reformatted by the AI " + "into clean Markdown (paragraphs, section headings, lists and code spans) " + "using the editable news_format_prompt field, preserving every fact while " + "turning the source wall of text into a readable article. The formatted " + "Markdown replaces the stored content; on any failure the cleaned original " + "is kept. Disable with the news_format_enabled toggle.\n\n" + "Scoring: the AI grade (1-10, temperature 0) is the base. effective_score = " + f"clamp(ai_grade + {UNIQUE_IMAGE_BONUS} if unique image - " + f"{THIN_CONTENT_PENALTY} if the body is marginal but not gated, 1..10). The " + "effective score is stored in grade (so the listing reflects final quality) " + "and the raw AI grade in ai_grade.\n\n" + "Promotion: an article is published when valid and effective_score is at or " + "above the grade threshold. It is Featured when published, has a unique " + f"image and scores at least {FEATURE_MIN_SCORE}. After the run the service " + "rotates Landing: the top " + f"{LANDING_MAX} published, featured, unique-image articles from the last " + f"{LANDING_RECENCY_DAYS} days scoring at least {LANDING_MIN_SCORE} are shown " + "on the landing page and the rest of the service-managed set is cleared. " + "Rows an admin has manually toggled are locked (featured_locked or " + "landing_locked) and the service never auto-manages them again." +) diff --git a/devplacepy/services/news/images.py b/devplacepy/services/news/images.py new file mode 100644 index 00000000..09caac17 --- /dev/null +++ b/devplacepy/services/news/images.py @@ -0,0 +1,113 @@ +# retoor + +import asyncio +import io +import re + +import httpx +import imagehash +from PIL import Image + +from devplacepy import net_guard + +from .constants import ( + IMG_FETCH_MAX_BYTES, + IMG_FETCH_TIMEOUT, + MIN_IMAGE_DIMENSION, + PHASH_DISTANCE, +) +from .models import ImageCandidate + + +def _decode_phash(data: bytes) -> tuple[str, int, int] | None: + try: + with Image.open(io.BytesIO(data)) as image: + image.load() + width, height = image.size + phash = str(imagehash.phash(image)) + return phash, width, height + except Exception: + return None + + +async def _get_article_images(url: str, client: httpx.AsyncClient) -> list[dict]: + try: + await net_guard.guard_public_url(url) + except Exception: + return [] + try: + resp = await client.get(url, timeout=10.0) + resp.raise_for_status() + html = resp.text + pattern = re.compile(r']+src=["\']([^"\']+)["\']', re.IGNORECASE) + matches = pattern.findall(html) + images = [] + seen = set() + for src in matches: + if src in seen: + continue + seen.add(src) + if src.startswith("http") and not any( + ext in src.lower() for ext in [".svg", ".ico"] + ): + images.append({"url": src, "alt_text": ""}) + return images[:10] + except Exception: + return [] + + +async def _fetch_image_candidate( + src: dict, image_client: httpx.AsyncClient +) -> ImageCandidate: + candidate = ImageCandidate(url=src["url"], alt_text=src.get("alt_text", "")) + try: + await net_guard.guard_public_url(candidate.url) + except Exception: + return candidate + try: + resp = await image_client.get(candidate.url, timeout=IMG_FETCH_TIMEOUT) + resp.raise_for_status() + data = resp.content[:IMG_FETCH_MAX_BYTES] + except Exception: + return candidate + if len(data) >= IMG_FETCH_MAX_BYTES: + return candidate + loop = asyncio.get_running_loop() + decoded = await loop.run_in_executor(None, _decode_phash, data) + if decoded is None: + return candidate + phash, width, height = decoded + candidate.phash = phash + candidate.width = width + candidate.height = height + candidate.is_placeholder = ( + width < MIN_IMAGE_DIMENSION or height < MIN_IMAGE_DIMENSION + ) + return candidate + + +def _flag_shared_placeholders(by_article: dict[str, list[ImageCandidate]]) -> None: + entries: list[tuple[str, ImageCandidate, imagehash.ImageHash]] = [] + for article_uid, candidates in by_article.items(): + for candidate in candidates: + if candidate.is_placeholder or not candidate.phash: + continue + entries.append( + (article_uid, candidate, imagehash.hex_to_hash(candidate.phash)) + ) + for i in range(len(entries)): + uid_a, cand_a, hash_a = entries[i] + for j in range(i + 1, len(entries)): + uid_b, cand_b, hash_b = entries[j] + if uid_a == uid_b: + continue + if (hash_a - hash_b) <= PHASH_DISTANCE: + cand_a.is_placeholder = True + cand_b.is_placeholder = True + + +def _primary_image(candidates: list[ImageCandidate]) -> str: + for candidate in candidates: + if not candidate.is_placeholder: + return candidate.url + return "" diff --git a/devplacepy/services/news/models.py b/devplacepy/services/news/models.py new file mode 100644 index 00000000..b34105be --- /dev/null +++ b/devplacepy/services/news/models.py @@ -0,0 +1,24 @@ +# retoor + +from dataclasses import dataclass, field + + +@dataclass +class ImageCandidate: + url: str + alt_text: str = "" + phash: str = "" + width: int = 0 + height: int = 0 + is_placeholder: bool = True + + +@dataclass +class ArticleGrade: + ai_grade: int | None + effective_score: int + has_unique_image: bool + image_url: str + valid: bool + reject_reason: str + candidates: list[ImageCandidate] = field(default_factory=list) diff --git a/devplacepy/services/news.py b/devplacepy/services/news/service.py similarity index 63% rename from devplacepy/services/news.py rename to devplacepy/services/news/service.py index 03f1c000..7ac5b26e 100644 --- a/devplacepy/services/news.py +++ b/devplacepy/services/news/service.py @@ -1,26 +1,15 @@ # retoor -import asyncio -import io -import logging -import os -import re -from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone import httpx -import imagehash -from PIL import Image from devplacepy import stealth from devplacepy import net_guard -from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL from devplacepy.database import ( add_news_usage, get_news_usage, - get_setting, get_table, - internal_gateway_key, ) from devplacepy.services.base import BaseService, ConfigField from devplacepy.services.openai_gateway.usage import ( @@ -28,348 +17,46 @@ from devplacepy.services.openai_gateway.usage import ( new_usage_totals, usage_metric_cards, ) -from devplacepy.utils import generate_uid, make_combined_slug, strip_html +from devplacepy.utils import generate_uid, make_combined_slug from devplacepy.services.audit import record as audit from devplacepy.services.seo_meta import schedule_seo_meta -logger = logging.getLogger(__name__) - -NEWS_API_URL_DEFAULT = "https://news.app.molodetz.nl/api" -AI_URL_DEFAULT = INTERNAL_GATEWAY_URL -AI_MODEL_DEFAULT = INTERNAL_MODEL -GRADE_THRESHOLD_DEFAULT = 7 -GRADE_MAX_TOKENS = 2000 -FORMAT_MAX_TOKENS = 6000 -FORMAT_INPUT_MAX_CHARS = 14000 -FORMAT_OUTPUT_MAX_CHARS = 30000 - -MIN_TITLE_CHARS = 12 -MIN_BODY_CHARS = 200 -MAX_TITLE_CAPS_RATIO = 0.6 -MIN_IMAGE_DIMENSION = 100 -PHASH_DISTANCE = 5 -IMG_PER_ARTICLE = 5 -IMG_FETCH_MAX_BYTES = 5_000_000 -IMG_FETCH_TIMEOUT = 8.0 -UNIQUE_IMAGE_BONUS = 2 -THIN_CONTENT_PENALTY = 2 -FEATURE_MIN_SCORE = 8 -LANDING_MIN_SCORE = 9 -LANDING_MAX = 6 -LANDING_RECENCY_DAYS = 4 - -JUNK_PATTERNS = ( - re.compile(r"submitted by\s+/?u/\S+(?:\s+to\s+/?r/\S+)?", re.IGNORECASE), - re.compile(r"submitted by\s+\S+\s+to\s+/?r/\S+", re.IGNORECASE), - re.compile(r"\[\s*link\s*\]", re.IGNORECASE), - re.compile(r"\[\s*\d*\s*comments?\s*\]", re.IGNORECASE), - re.compile(r"submitted by[^\[]*\[link\]\s*\[comments?\]", re.IGNORECASE), +from . import _get_ai_key +from .constants import ( + AI_MODEL_DEFAULT, + AI_URL_DEFAULT, + FEATURE_MIN_SCORE, + FORMAT_INPUT_MAX_CHARS, + FORMAT_MAX_TOKENS, + FORMAT_OUTPUT_MAX_CHARS, + FORMAT_PROMPT_SPEC, + GRADE_MAX_TOKENS, + GRADE_PROMPT_SPEC, + GRADE_THRESHOLD_DEFAULT, + GRADING_RULES_DESCRIPTION, + IMG_FETCH_TIMEOUT, + IMG_PER_ARTICLE, + LANDING_MAX, + LANDING_MIN_SCORE, + LANDING_RECENCY_DAYS, + MIN_BODY_CHARS, + NEWS_API_URL_DEFAULT, + NEWS_SUMMARY, ) -WHITESPACE_PATTERN = re.compile(r"\s+") - -NEWS_SUMMARY = ( - "Fully automatic, zero-maintenance news pipeline: fetches the feed, cleans " - "each article, fetches and perceptually compares images to reject " - "placeholders, grades deterministically, and auto-rotates Featured and " - "Landing. The full grading algorithm is detailed below." +from .clean import ( + _extract_grade, + _strip_md_fence, + clean_news_text, + effective_score, + reliability_reason, ) - -GRADE_PROMPT_SPEC = ( - "You are an expert content strategist and editor for DevPlace, a " - "server-rendered social network for software developers. Decide how " - "strongly a given article should be published, expressed as a single " - "grade.\n\n" - "Site context:\n" - "- Audience: software developers, hobbyists and professionals, " - "interested in software development, AI agents, open-source tools, " - "systems programming, and self-hosting.\n" - "- Mission: deliver deep technical insight, critical analysis, and " - "practical value; build authority and reader trust; avoid low-effort, " - "rehashed, or clickbait content.\n" - "- Tone: technical, practical, nuanced, skeptical of hype; no " - "sensationalism or unverified claims.\n\n" - "Evaluate the article against these weighted criteria and weigh them " - "internally:\n" - "1. Relevance and alignment to the developer niche and audience needs " - "(20%).\n" - "2. Quality and accuracy: depth, originality, factual correctness, " - "sourcing, technical correctness, clear writing (25%).\n" - "3. Engagement and readability: strong title/intro/flow, useful " - "examples, appropriate length (15%).\n" - "4. Originality and uniqueness: new perspective, data, or analysis; not " - "duplicated generic web content (15%).\n" - "5. SEO and discoverability: keyword opportunity without stuffing, " - "shareable (10%).\n" - "6. Ethics, legality and risk: no misinformation, plagiarism, or " - "brand-safety issues; controversial topics handled with nuance and " - "evidence (10%).\n" - "7. Strategic fit for the site (5%).\n\n" - "Convert your overall judgment to a single integer grade from 1 to 10:\n" - "- 9-10: excellent, publish as-is.\n" - "- 7-8: good, worth publishing.\n" - "- 5-6: marginal, weak fit or quality.\n" - "- 1-4: reject, low quality or off-topic.\n\n" - "Return ONLY a single integer from 1 to 10, nothing else." +from .images import ( + _fetch_image_candidate, + _flag_shared_placeholders, + _get_article_images, + _primary_image, ) - -FORMAT_PROMPT_SPEC = ( - "You are an expert editor for DevPlace, a social network for software " - "developers. You are given the raw body of a news article that arrived as " - "an undifferentiated wall of text. Reformat it into clean, readable " - "Markdown for a technical audience.\n\n" - "Rules:\n" - "- Break the text into short, well-structured paragraphs separated by a " - "blank line.\n" - "- Add Markdown section headings (## Heading) where the topic clearly " - "shifts, so the article scans well.\n" - "- Use bullet or numbered lists for enumerations, and inline code or " - "fenced code blocks where code, commands, or identifiers appear.\n" - "- Preserve every fact, name, number, and quotation exactly as given. " - "Never invent, add, remove, or reorder information.\n" - "- Only restructure and lightly polish wording for flow and grammar; do " - "not add an introduction, conclusion, opinion, or commentary of your " - "own.\n" - "- Do not repeat the article title as a heading and do not wrap the whole " - "answer in a code fence.\n" - "- Output only the reformatted article body as Markdown, nothing else." -) - -GRADING_RULES_DESCRIPTION = ( - "Each run fetches the feed, cleans every article, fetches and perceptually " - "compares images, grades deterministically, and auto-rotates Featured and " - "Landing.\n\n" - "Content cleaning: after HTML stripping it removes Reddit boilerplate " - "(submitted by /u/, submitted by ... to /r/, standalone [link], " - "[comments], [N comments] and trailing submitted-by [link] [comments]) and " - "collapses whitespace, applied to title (light), description and content " - "before grading and before storage.\n\n" - "Reliability gate (forces draft, never Featured or Landing) when: the url is " - f"empty or invalid; the cleaned title is shorter than {MIN_TITLE_CHARS} chars; " - f"the combined cleaned body is shorter than {MIN_BODY_CHARS} chars; or the " - f"title uppercase-letter ratio exceeds {MAX_TITLE_CAPS_RATIO}.\n\n" - "Image uniqueness: up to " - f"{IMG_PER_ARTICLE} candidate images per article are fetched (max " - f"{IMG_FETCH_MAX_BYTES} bytes), decoded and perceptually hashed (phash). An " - f"image under {MIN_IMAGE_DIMENSION}px on either side, or one that fails to " - "fetch or decode, is a placeholder. Hashes within Hamming distance " - f"{PHASH_DISTANCE} that span two or more different articles are a shared " - "logo or placeholder and are rejected for all of them. An article with at " - "least one genuinely unique image is marked has_unique_image and its first " - "such image becomes the primary image.\n\n" - "Grading prompt: the exact rubric is the editable news_grade_prompt field " - "(Configuration tab), sent to the model at temperature 0 with the cleaned " - "Title, Description and Content appended; it must return a single integer " - "from 1 to 10.\n\n" - "Formatting: after grading, every valid article is reformatted by the AI " - "into clean Markdown (paragraphs, section headings, lists and code spans) " - "using the editable news_format_prompt field, preserving every fact while " - "turning the source wall of text into a readable article. The formatted " - "Markdown replaces the stored content; on any failure the cleaned original " - "is kept. Disable with the news_format_enabled toggle.\n\n" - "Scoring: the AI grade (1-10, temperature 0) is the base. effective_score = " - f"clamp(ai_grade + {UNIQUE_IMAGE_BONUS} if unique image - " - f"{THIN_CONTENT_PENALTY} if the body is marginal but not gated, 1..10). The " - "effective score is stored in grade (so the listing reflects final quality) " - "and the raw AI grade in ai_grade.\n\n" - "Promotion: an article is published when valid and effective_score is at or " - "above the grade threshold. It is Featured when published, has a unique " - f"image and scores at least {FEATURE_MIN_SCORE}. After the run the service " - "rotates Landing: the top " - f"{LANDING_MAX} published, featured, unique-image articles from the last " - f"{LANDING_RECENCY_DAYS} days scoring at least {LANDING_MIN_SCORE} are shown " - "on the landing page and the rest of the service-managed set is cleared. " - "Rows an admin has manually toggled are locked (featured_locked or " - "landing_locked) and the service never auto-manages them again." -) - - -@dataclass -class ImageCandidate: - url: str - alt_text: str = "" - phash: str = "" - width: int = 0 - height: int = 0 - is_placeholder: bool = True - - -@dataclass -class ArticleGrade: - ai_grade: int | None - effective_score: int - has_unique_image: bool - image_url: str - valid: bool - reject_reason: str - candidates: list[ImageCandidate] = field(default_factory=list) - - -def _get_ai_key() -> str: - key = os.environ.get("NEWS_AI_KEY") - if key: - return key - key = get_setting("news_ai_key", "") - if key: - return key - return internal_gateway_key() - - -def _extract_grade(text: str) -> int | None: - match = re.search(r"\d+", text.strip()) - if match: - val = int(match.group()) - if 1 <= val <= 10: - return val - return None - - -def _strip_md_fence(text: str) -> str: - stripped = text.strip() - if not stripped.startswith("```"): - return stripped - lines = stripped.splitlines() - if len(lines) < 2 or not lines[-1].strip().startswith("```"): - return stripped - return "\n".join(lines[1:-1]).strip() - - -def clean_news_text(text: str) -> str: - if not text: - return "" - cleaned = strip_html(text) - for pattern in JUNK_PATTERNS: - cleaned = pattern.sub(" ", cleaned) - cleaned = WHITESPACE_PATTERN.sub(" ", cleaned) - return cleaned.strip() - - -def _caps_ratio(title: str) -> float: - letters = [c for c in title if c.isalpha()] - if not letters: - return 0.0 - uppers = sum(1 for c in letters if c.isupper()) - return uppers / len(letters) - - -def _is_valid_url(url: str) -> bool: - return url.startswith("http://") or url.startswith("https://") - - -def reliability_reason(title: str, body: str, url: str) -> str: - if not url or not _is_valid_url(url): - return "invalid_url" - if len(title) < MIN_TITLE_CHARS: - return "short_title" - if len(body) < MIN_BODY_CHARS: - return "thin_body" - if _caps_ratio(title) > MAX_TITLE_CAPS_RATIO: - return "shouting_title" - return "" - - -def effective_score( - ai_grade: int, has_unique_image: bool, body_marginal: bool -) -> int: - score = ai_grade - if has_unique_image: - score += UNIQUE_IMAGE_BONUS - if body_marginal: - score -= THIN_CONTENT_PENALTY - return max(1, min(10, score)) - - -def _decode_phash(data: bytes) -> tuple[str, int, int] | None: - try: - with Image.open(io.BytesIO(data)) as image: - image.load() - width, height = image.size - phash = str(imagehash.phash(image)) - return phash, width, height - except Exception: - return None - - -async def _get_article_images(url: str, client: httpx.AsyncClient) -> list[dict]: - try: - await net_guard.guard_public_url(url) - except Exception: - return [] - try: - resp = await client.get(url, timeout=10.0) - resp.raise_for_status() - html = resp.text - pattern = re.compile(r']+src=["\']([^"\']+)["\']', re.IGNORECASE) - matches = pattern.findall(html) - images = [] - seen = set() - for src in matches: - if src in seen: - continue - seen.add(src) - if src.startswith("http") and not any( - ext in src.lower() for ext in [".svg", ".ico"] - ): - images.append({"url": src, "alt_text": ""}) - return images[:10] - except Exception: - return [] - - -async def _fetch_image_candidate( - src: dict, image_client: httpx.AsyncClient -) -> ImageCandidate: - candidate = ImageCandidate(url=src["url"], alt_text=src.get("alt_text", "")) - try: - await net_guard.guard_public_url(candidate.url) - except Exception: - return candidate - try: - resp = await image_client.get(candidate.url, timeout=IMG_FETCH_TIMEOUT) - resp.raise_for_status() - data = resp.content[:IMG_FETCH_MAX_BYTES] - except Exception: - return candidate - if len(data) >= IMG_FETCH_MAX_BYTES: - return candidate - loop = asyncio.get_running_loop() - decoded = await loop.run_in_executor(None, _decode_phash, data) - if decoded is None: - return candidate - phash, width, height = decoded - candidate.phash = phash - candidate.width = width - candidate.height = height - candidate.is_placeholder = ( - width < MIN_IMAGE_DIMENSION or height < MIN_IMAGE_DIMENSION - ) - return candidate - - -def _flag_shared_placeholders(by_article: dict[str, list[ImageCandidate]]) -> None: - entries: list[tuple[str, ImageCandidate, imagehash.ImageHash]] = [] - for article_uid, candidates in by_article.items(): - for candidate in candidates: - if candidate.is_placeholder or not candidate.phash: - continue - entries.append( - (article_uid, candidate, imagehash.hex_to_hash(candidate.phash)) - ) - for i in range(len(entries)): - uid_a, cand_a, hash_a = entries[i] - for j in range(i + 1, len(entries)): - uid_b, cand_b, hash_b = entries[j] - if uid_a == uid_b: - continue - if (hash_a - hash_b) <= PHASH_DISTANCE: - cand_a.is_placeholder = True - cand_b.is_placeholder = True - - -def _primary_image(candidates: list[ImageCandidate]) -> str: - for candidate in candidates: - if not candidate.is_placeholder: - return candidate.url - return "" +from .models import ArticleGrade, ImageCandidate class NewsService(BaseService):