forked from retoor/devplacepy
feat: add featured/locked columns and auto-rotation logic to news pipeline
- Add `ai_grade`, `featured`, `featured_locked`, `landing_locked`, `author`, `article_published`, `image_url`, `has_unique_image` columns to news table - Extend `news_images` schema with `alt_text`, `phash`, `width`, `height`, `is_placeholder` columns - Create `idx_news_featured` index for efficient featured queries - Update admin toggle endpoints to set `featured_locked`/`landing_locked` when manually toggling - Propagate `featured` and `image_url` fields through landing page, news list, and detail page rendering - Update `get_featured_news` to return `featured` and `image_url` fields - Document in AGENTS.md the full zero-maintenance pipeline: image perceptual hashing, placeholder detection, AI grading on cleaned text, reliability gate, effective score computation, and post-loop landing rotation - Update README.md service description to reflect automatic image comparison and landing rotation capabilities - Clarify docs_api.py summaries that toggling featured/landing now locks articles from auto-rotation
This commit is contained in:
+22
-4
@@ -464,7 +464,15 @@ def init_db():
|
||||
("content", ""),
|
||||
("synced_at", ""),
|
||||
("grade", 0),
|
||||
("ai_grade", 0),
|
||||
("show_on_landing", 0),
|
||||
("featured", 0),
|
||||
("author", ""),
|
||||
("article_published", ""),
|
||||
("image_url", ""),
|
||||
("has_unique_image", 0),
|
||||
("featured_locked", 0),
|
||||
("landing_locked", 0),
|
||||
):
|
||||
if not news.has_column(column):
|
||||
news.create_column_by_example(column, example)
|
||||
@@ -472,12 +480,20 @@ def init_db():
|
||||
_index(db, "news", "idx_news_external_id", ["external_id"])
|
||||
_index(db, "news", "idx_news_synced_at", ["synced_at"])
|
||||
_index(db, "news", "idx_news_status", ["status"])
|
||||
_index(db, "news", "idx_news_featured", ["featured"])
|
||||
|
||||
news_images = get_table("news_images")
|
||||
if not news_images.has_column("news_uid"):
|
||||
news_images.create_column_by_example("news_uid", "")
|
||||
if not news_images.has_column("url"):
|
||||
news_images.create_column_by_example("url", "")
|
||||
for column, example in (
|
||||
("news_uid", ""),
|
||||
("url", ""),
|
||||
("alt_text", ""),
|
||||
("phash", ""),
|
||||
("width", 0),
|
||||
("height", 0),
|
||||
("is_placeholder", 0),
|
||||
):
|
||||
if not news_images.has_column(column):
|
||||
news_images.create_column_by_example(column, example)
|
||||
|
||||
news_sync = get_table("news_sync")
|
||||
if not news_sync.has_column("external_id"):
|
||||
@@ -2753,6 +2769,8 @@ def get_featured_news(limit=5):
|
||||
"slug": article.get("slug", ""),
|
||||
"url": article.get("url", ""),
|
||||
"source_name": article.get("source_name", ""),
|
||||
"featured": article.get("featured", 0),
|
||||
"image_url": article.get("image_url", "") or "",
|
||||
"time_ago": time_ago(article["synced_at"])
|
||||
if article.get("synced_at")
|
||||
else "",
|
||||
|
||||
@@ -4504,7 +4504,7 @@ four ways to sign requests.
|
||||
method="POST",
|
||||
path="/admin/news/{uid}/toggle",
|
||||
title="Toggle featured",
|
||||
summary="Toggle an article's featured flag.",
|
||||
summary="Toggle an article's featured flag and lock it from the news service's auto-rotation.",
|
||||
auth="admin",
|
||||
destructive=True,
|
||||
params=[
|
||||
@@ -4528,7 +4528,7 @@ four ways to sign requests.
|
||||
method="POST",
|
||||
path="/admin/news/{uid}/landing",
|
||||
title="Toggle landing",
|
||||
summary="Toggle whether an article shows on the landing page.",
|
||||
summary="Toggle whether an article shows on the landing page and lock it from the news service's auto-rotation.",
|
||||
auth="admin",
|
||||
destructive=True,
|
||||
params=[
|
||||
|
||||
+2
-1
@@ -522,9 +522,10 @@ async def landing(request: Request):
|
||||
"url": a.get("url", ""),
|
||||
"source_name": a.get("source_name", ""),
|
||||
"grade": a.get("grade", 0),
|
||||
"featured": a.get("featured", 0),
|
||||
"synced_at": a.get("synced_at", "") or "",
|
||||
"time_ago": time_ago(a["synced_at"]),
|
||||
"image_url": images_by_news.get(a["uid"], ""),
|
||||
"image_url": a.get("image_url", "") or images_by_news.get(a["uid"], ""),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -85,7 +85,9 @@ async def admin_news_toggle(request: Request, uid: str):
|
||||
if article:
|
||||
current = article.get("featured", 0)
|
||||
new_value = 0 if current else 1
|
||||
news_table.update({"uid": uid, "featured": new_value}, ["uid"])
|
||||
news_table.update(
|
||||
{"uid": uid, "featured": new_value, "featured_locked": 1}, ["uid"]
|
||||
)
|
||||
logger.info(
|
||||
f"Admin {admin['username']} toggled news {uid} featured={'off' if current else 'on'}"
|
||||
)
|
||||
@@ -137,7 +139,9 @@ async def admin_news_landing(request: Request, uid: str):
|
||||
if article:
|
||||
current = article.get("show_on_landing", 0)
|
||||
new_value = 0 if current else 1
|
||||
news_table.update({"uid": uid, "show_on_landing": new_value}, ["uid"])
|
||||
news_table.update(
|
||||
{"uid": uid, "show_on_landing": new_value, "landing_locked": 1}, ["uid"]
|
||||
)
|
||||
logger.info(
|
||||
f"Admin {admin['username']} toggled news {uid} landing={'on' if not current else 'off'}"
|
||||
)
|
||||
|
||||
@@ -62,8 +62,9 @@ async def news_page(request: Request, before: str = None):
|
||||
{
|
||||
"article": a,
|
||||
"time_ago": time_ago(a["synced_at"]),
|
||||
"image_url": images_by_news.get(a["uid"]),
|
||||
"image_url": a.get("image_url", "") or images_by_news.get(a["uid"]),
|
||||
"grade": a.get("grade", 0),
|
||||
"featured": a.get("featured", 0),
|
||||
"recent_comments": recent_comments.get(a["uid"], []),
|
||||
}
|
||||
)
|
||||
@@ -104,10 +105,10 @@ async def news_detail_page(request: Request, news_slug: str):
|
||||
if redirect:
|
||||
return redirect
|
||||
|
||||
image_url = ""
|
||||
if "news_images" in db.tables:
|
||||
image_url = article.get("image_url", "") or ""
|
||||
if not image_url and "news_images" in db.tables:
|
||||
img = get_table("news_images").find_one(
|
||||
news_uid=article["uid"], deleted_at=None
|
||||
news_uid=article["uid"], deleted_at=None, is_placeholder=0
|
||||
)
|
||||
if img:
|
||||
image_url = img["url"]
|
||||
|
||||
@@ -397,8 +397,11 @@ class NewsOut(_Out):
|
||||
source_name: Optional[str] = None
|
||||
author: Optional[str] = None
|
||||
grade: Optional[int] = None
|
||||
ai_grade: Optional[int] = None
|
||||
status: Optional[str] = None
|
||||
image_url: Optional[str] = None
|
||||
featured: Optional[int] = None
|
||||
has_unique_image: Optional[int] = None
|
||||
article_published: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
synced_at: Optional[str] = None
|
||||
@@ -485,6 +488,7 @@ class NewsListItemOut(_Out):
|
||||
time_ago: Optional[str] = None
|
||||
image_url: Optional[str] = None
|
||||
grade: Optional[int] = None
|
||||
featured: Optional[int] = None
|
||||
recent_comments: list[CommentItemOut] = []
|
||||
|
||||
|
||||
@@ -944,6 +948,7 @@ class LandingArticleOut(_Out):
|
||||
url: Optional[str] = None
|
||||
source_name: Optional[str] = None
|
||||
grade: int = 0
|
||||
featured: int = 0
|
||||
synced_at: Optional[str] = None
|
||||
time_ago: Optional[str] = None
|
||||
image_url: Optional[str] = None
|
||||
|
||||
@@ -120,6 +120,7 @@ class BaseService(ABC):
|
||||
default_enabled = True
|
||||
title = ""
|
||||
description = ""
|
||||
details = ""
|
||||
DEFAULT_LOG_SIZE = 20
|
||||
TICK_SECONDS = 1
|
||||
PERSIST_SECONDS = 3
|
||||
@@ -414,6 +415,7 @@ class BaseService(ABC):
|
||||
"name": self.name,
|
||||
"title": self.title or self.name.capitalize(),
|
||||
"description": self.description,
|
||||
"details": self.details,
|
||||
"enabled": enabled,
|
||||
"status": status,
|
||||
"interval_seconds": self.current_interval(),
|
||||
|
||||
+610
-184
@@ -1,13 +1,19 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import httpx
|
||||
import imagehash
|
||||
from PIL import Image
|
||||
|
||||
from devplacepy import stealth
|
||||
from devplacepy import net_guard
|
||||
from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
|
||||
from devplacepy.database import get_table, get_setting, internal_gateway_key
|
||||
from devplacepy.services.base import BaseService, ConfigField
|
||||
@@ -22,6 +28,138 @@ AI_MODEL_DEFAULT = INTERNAL_MODEL
|
||||
GRADE_THRESHOLD_DEFAULT = 7
|
||||
GRADE_MAX_TOKENS = 2000
|
||||
|
||||
MIN_TITLE_CHARS = 12
|
||||
MIN_BODY_CHARS = 200
|
||||
MAX_TITLE_CAPS_RATIO = 0.6
|
||||
MIN_IMAGE_DIMENSION = 100
|
||||
PHASH_DISTANCE = 5
|
||||
IMG_PER_ARTICLE = 5
|
||||
IMG_FETCH_MAX_BYTES = 5_000_000
|
||||
IMG_FETCH_TIMEOUT = 8.0
|
||||
UNIQUE_IMAGE_BONUS = 2
|
||||
THIN_CONTENT_PENALTY = 2
|
||||
FEATURE_MIN_SCORE = 8
|
||||
LANDING_MIN_SCORE = 9
|
||||
LANDING_MAX = 6
|
||||
LANDING_RECENCY_DAYS = 4
|
||||
|
||||
JUNK_PATTERNS = (
|
||||
re.compile(r"submitted by\s+/?u/\S+(?:\s+to\s+/?r/\S+)?", re.IGNORECASE),
|
||||
re.compile(r"submitted by\s+\S+\s+to\s+/?r/\S+", re.IGNORECASE),
|
||||
re.compile(r"\[\s*link\s*\]", re.IGNORECASE),
|
||||
re.compile(r"\[\s*\d*\s*comments?\s*\]", re.IGNORECASE),
|
||||
re.compile(r"submitted by[^\[]*\[link\]\s*\[comments?\]", re.IGNORECASE),
|
||||
)
|
||||
WHITESPACE_PATTERN = re.compile(r"\s+")
|
||||
|
||||
NEWS_SUMMARY = (
|
||||
"Fully automatic, zero-maintenance news pipeline: fetches the feed, cleans "
|
||||
"each article, fetches and perceptually compares images to reject "
|
||||
"placeholders, grades deterministically, and auto-rotates Featured and "
|
||||
"Landing. The full grading algorithm is detailed below."
|
||||
)
|
||||
|
||||
GRADE_PROMPT_SPEC = (
|
||||
"You are an expert content strategist and editor for DevPlace, a "
|
||||
"server-rendered social network for software developers. Decide how "
|
||||
"strongly a given article should be published, expressed as a single "
|
||||
"grade.\n\n"
|
||||
"Site context:\n"
|
||||
"- Audience: software developers, hobbyists and professionals, "
|
||||
"interested in software development, AI agents, open-source tools, "
|
||||
"systems programming, and self-hosting.\n"
|
||||
"- Mission: deliver deep technical insight, critical analysis, and "
|
||||
"practical value; build authority and reader trust; avoid low-effort, "
|
||||
"rehashed, or clickbait content.\n"
|
||||
"- Tone: technical, practical, nuanced, skeptical of hype; no "
|
||||
"sensationalism or unverified claims.\n\n"
|
||||
"Evaluate the article against these weighted criteria and weigh them "
|
||||
"internally:\n"
|
||||
"1. Relevance and alignment to the developer niche and audience needs "
|
||||
"(20%).\n"
|
||||
"2. Quality and accuracy: depth, originality, factual correctness, "
|
||||
"sourcing, technical correctness, clear writing (25%).\n"
|
||||
"3. Engagement and readability: strong title/intro/flow, useful "
|
||||
"examples, appropriate length (15%).\n"
|
||||
"4. Originality and uniqueness: new perspective, data, or analysis; not "
|
||||
"duplicated generic web content (15%).\n"
|
||||
"5. SEO and discoverability: keyword opportunity without stuffing, "
|
||||
"shareable (10%).\n"
|
||||
"6. Ethics, legality and risk: no misinformation, plagiarism, or "
|
||||
"brand-safety issues; controversial topics handled with nuance and "
|
||||
"evidence (10%).\n"
|
||||
"7. Strategic fit for the site (5%).\n\n"
|
||||
"Convert your overall judgment to a single integer grade from 1 to 10:\n"
|
||||
"- 9-10: excellent, publish as-is.\n"
|
||||
"- 7-8: good, worth publishing.\n"
|
||||
"- 5-6: marginal, weak fit or quality.\n"
|
||||
"- 1-4: reject, low quality or off-topic.\n\n"
|
||||
"Return ONLY a single integer from 1 to 10, nothing else."
|
||||
)
|
||||
|
||||
GRADING_RULES_DESCRIPTION = (
|
||||
"Each run fetches the feed, cleans every article, fetches and perceptually "
|
||||
"compares images, grades deterministically, and auto-rotates Featured and "
|
||||
"Landing.\n\n"
|
||||
"Content cleaning: after HTML stripping it removes Reddit boilerplate "
|
||||
"(submitted by /u/<user>, submitted by ... to /r/<sub>, standalone [link], "
|
||||
"[comments], [N comments] and trailing submitted-by [link] [comments]) and "
|
||||
"collapses whitespace, applied to title (light), description and content "
|
||||
"before grading and before storage.\n\n"
|
||||
"Reliability gate (forces draft, never Featured or Landing) when: the url is "
|
||||
f"empty or invalid; the cleaned title is shorter than {MIN_TITLE_CHARS} chars; "
|
||||
f"the combined cleaned body is shorter than {MIN_BODY_CHARS} chars; or the "
|
||||
f"title uppercase-letter ratio exceeds {MAX_TITLE_CAPS_RATIO}.\n\n"
|
||||
"Image uniqueness: up to "
|
||||
f"{IMG_PER_ARTICLE} candidate images per article are fetched (max "
|
||||
f"{IMG_FETCH_MAX_BYTES} bytes), decoded and perceptually hashed (phash). An "
|
||||
f"image under {MIN_IMAGE_DIMENSION}px on either side, or one that fails to "
|
||||
"fetch or decode, is a placeholder. Hashes within Hamming distance "
|
||||
f"{PHASH_DISTANCE} that span two or more different articles are a shared "
|
||||
"logo or placeholder and are rejected for all of them. An article with at "
|
||||
"least one genuinely unique image is marked has_unique_image and its first "
|
||||
"such image becomes the primary image.\n\n"
|
||||
"Grading prompt: the exact rubric is the editable news_grade_prompt field "
|
||||
"(Configuration tab), sent to the model at temperature 0 with the cleaned "
|
||||
"Title, Description and Content appended; it must return a single integer "
|
||||
"from 1 to 10.\n\n"
|
||||
"Scoring: the AI grade (1-10, temperature 0) is the base. effective_score = "
|
||||
f"clamp(ai_grade + {UNIQUE_IMAGE_BONUS} if unique image - "
|
||||
f"{THIN_CONTENT_PENALTY} if the body is marginal but not gated, 1..10). The "
|
||||
"effective score is stored in grade (so the listing reflects final quality) "
|
||||
"and the raw AI grade in ai_grade.\n\n"
|
||||
"Promotion: an article is published when valid and effective_score is at or "
|
||||
"above the grade threshold. It is Featured when published, has a unique "
|
||||
f"image and scores at least {FEATURE_MIN_SCORE}. After the run the service "
|
||||
"rotates Landing: the top "
|
||||
f"{LANDING_MAX} published, featured, unique-image articles from the last "
|
||||
f"{LANDING_RECENCY_DAYS} days scoring at least {LANDING_MIN_SCORE} are shown "
|
||||
"on the landing page and the rest of the service-managed set is cleared. "
|
||||
"Rows an admin has manually toggled are locked (featured_locked or "
|
||||
"landing_locked) and the service never auto-manages them again."
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImageCandidate:
|
||||
url: str
|
||||
alt_text: str = ""
|
||||
phash: str = ""
|
||||
width: int = 0
|
||||
height: int = 0
|
||||
is_placeholder: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArticleGrade:
|
||||
ai_grade: int | None
|
||||
effective_score: int
|
||||
has_unique_image: bool
|
||||
image_url: str
|
||||
valid: bool
|
||||
reject_reason: str
|
||||
candidates: list[ImageCandidate] = field(default_factory=list)
|
||||
|
||||
|
||||
def _get_ai_key() -> str:
|
||||
key = os.environ.get("NEWS_AI_KEY")
|
||||
@@ -42,6 +180,62 @@ def _extract_grade(text: str) -> int | None:
|
||||
return None
|
||||
|
||||
|
||||
def clean_news_text(text: str) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
cleaned = strip_html(text)
|
||||
for pattern in JUNK_PATTERNS:
|
||||
cleaned = pattern.sub(" ", cleaned)
|
||||
cleaned = WHITESPACE_PATTERN.sub(" ", cleaned)
|
||||
return cleaned.strip()
|
||||
|
||||
|
||||
def _caps_ratio(title: str) -> float:
|
||||
letters = [c for c in title if c.isalpha()]
|
||||
if not letters:
|
||||
return 0.0
|
||||
uppers = sum(1 for c in letters if c.isupper())
|
||||
return uppers / len(letters)
|
||||
|
||||
|
||||
def _is_valid_url(url: str) -> bool:
|
||||
return url.startswith("http://") or url.startswith("https://")
|
||||
|
||||
|
||||
def reliability_reason(title: str, body: str, url: str) -> str:
|
||||
if not url or not _is_valid_url(url):
|
||||
return "invalid_url"
|
||||
if len(title) < MIN_TITLE_CHARS:
|
||||
return "short_title"
|
||||
if len(body) < MIN_BODY_CHARS:
|
||||
return "thin_body"
|
||||
if _caps_ratio(title) > MAX_TITLE_CAPS_RATIO:
|
||||
return "shouting_title"
|
||||
return ""
|
||||
|
||||
|
||||
def effective_score(
|
||||
ai_grade: int, has_unique_image: bool, body_marginal: bool
|
||||
) -> int:
|
||||
score = ai_grade
|
||||
if has_unique_image:
|
||||
score += UNIQUE_IMAGE_BONUS
|
||||
if body_marginal:
|
||||
score -= THIN_CONTENT_PENALTY
|
||||
return max(1, min(10, score))
|
||||
|
||||
|
||||
def _decode_phash(data: bytes) -> tuple[str, int, int] | None:
|
||||
try:
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
image.load()
|
||||
width, height = image.size
|
||||
phash = str(imagehash.phash(image))
|
||||
return phash, width, height
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def _get_article_images(url: str, client: httpx.AsyncClient) -> list[dict]:
|
||||
try:
|
||||
resp = await client.get(url, timeout=10.0)
|
||||
@@ -64,15 +258,70 @@ async def _get_article_images(url: str, client: httpx.AsyncClient) -> list[dict]
|
||||
return []
|
||||
|
||||
|
||||
async def _fetch_image_candidate(
|
||||
src: dict, image_client: httpx.AsyncClient
|
||||
) -> ImageCandidate:
|
||||
candidate = ImageCandidate(url=src["url"], alt_text=src.get("alt_text", ""))
|
||||
try:
|
||||
await net_guard.guard_public_url(candidate.url)
|
||||
except Exception:
|
||||
return candidate
|
||||
try:
|
||||
resp = await image_client.get(candidate.url, timeout=IMG_FETCH_TIMEOUT)
|
||||
resp.raise_for_status()
|
||||
data = resp.content[:IMG_FETCH_MAX_BYTES]
|
||||
except Exception:
|
||||
return candidate
|
||||
if len(data) >= IMG_FETCH_MAX_BYTES:
|
||||
return candidate
|
||||
loop = asyncio.get_running_loop()
|
||||
decoded = await loop.run_in_executor(None, _decode_phash, data)
|
||||
if decoded is None:
|
||||
return candidate
|
||||
phash, width, height = decoded
|
||||
candidate.phash = phash
|
||||
candidate.width = width
|
||||
candidate.height = height
|
||||
candidate.is_placeholder = (
|
||||
width < MIN_IMAGE_DIMENSION or height < MIN_IMAGE_DIMENSION
|
||||
)
|
||||
return candidate
|
||||
|
||||
|
||||
def _flag_shared_placeholders(by_article: dict[str, list[ImageCandidate]]) -> None:
|
||||
entries: list[tuple[str, ImageCandidate, imagehash.ImageHash]] = []
|
||||
for article_uid, candidates in by_article.items():
|
||||
for candidate in candidates:
|
||||
if candidate.is_placeholder or not candidate.phash:
|
||||
continue
|
||||
entries.append(
|
||||
(article_uid, candidate, imagehash.hex_to_hash(candidate.phash))
|
||||
)
|
||||
for i in range(len(entries)):
|
||||
uid_a, cand_a, hash_a = entries[i]
|
||||
for j in range(i + 1, len(entries)):
|
||||
uid_b, cand_b, hash_b = entries[j]
|
||||
if uid_a == uid_b:
|
||||
continue
|
||||
if (hash_a - hash_b) <= PHASH_DISTANCE:
|
||||
cand_a.is_placeholder = True
|
||||
cand_b.is_placeholder = True
|
||||
|
||||
|
||||
def _primary_image(candidates: list[ImageCandidate]) -> str:
|
||||
for candidate in candidates:
|
||||
if not candidate.is_placeholder:
|
||||
return candidate.url
|
||||
return ""
|
||||
|
||||
|
||||
class NewsService(BaseService):
|
||||
interval_key = "news_service_interval"
|
||||
min_interval = 60
|
||||
default_enabled = True
|
||||
title = "News"
|
||||
description = (
|
||||
"Periodically fetches developer news articles from the configured feed, "
|
||||
"grades each one with an AI model, and stores them - auto-publishing those "
|
||||
"at or above the grade threshold and saving the rest as drafts."
|
||||
)
|
||||
description = NEWS_SUMMARY
|
||||
details = GRADING_RULES_DESCRIPTION
|
||||
config_fields = [
|
||||
ConfigField(
|
||||
"news_api_url",
|
||||
@@ -87,7 +336,7 @@ class NewsService(BaseService):
|
||||
"AI grading URL",
|
||||
type="url",
|
||||
default=AI_URL_DEFAULT,
|
||||
help="Chat-completions endpoint used to grade each article.",
|
||||
help="Chat-completions endpoint used to grade each cleaned article.",
|
||||
group="AI grading",
|
||||
),
|
||||
ConfigField(
|
||||
@@ -98,6 +347,19 @@ class NewsService(BaseService):
|
||||
help="Model name sent to the grading endpoint.",
|
||||
group="AI grading",
|
||||
),
|
||||
ConfigField(
|
||||
"news_grade_prompt",
|
||||
"Grading prompt specification",
|
||||
type="text",
|
||||
default=GRADE_PROMPT_SPEC,
|
||||
help=(
|
||||
"The exact rubric sent to the grading model at temperature 0. "
|
||||
"The cleaned Title, Description and Content are appended "
|
||||
"automatically. It must instruct the model to return only a "
|
||||
"single integer from 1 to 10."
|
||||
),
|
||||
group="AI grading",
|
||||
),
|
||||
ConfigField(
|
||||
"news_grade_threshold",
|
||||
"Grade threshold (1-10)",
|
||||
@@ -105,7 +367,11 @@ class NewsService(BaseService):
|
||||
default=GRADE_THRESHOLD_DEFAULT,
|
||||
minimum=1,
|
||||
maximum=10,
|
||||
help="Articles graded at or above this are auto-published; below go to draft.",
|
||||
help=(
|
||||
"Articles whose effective score (AI grade plus the unique-image "
|
||||
"bonus minus the thin-content penalty) reaches this are "
|
||||
"auto-published; below go to draft."
|
||||
),
|
||||
group="AI grading",
|
||||
),
|
||||
ConfigField(
|
||||
@@ -154,205 +420,365 @@ class NewsService(BaseService):
|
||||
updated_count = 0
|
||||
draft_count = 0
|
||||
failed_count = 0
|
||||
rejected_count = 0
|
||||
skipped_count = 0
|
||||
|
||||
for article in articles:
|
||||
external_id = article.get("guid", "")
|
||||
if not external_id:
|
||||
continue
|
||||
async with net_guard.guarded_async_client(
|
||||
timeout=IMG_FETCH_TIMEOUT
|
||||
) as image_client:
|
||||
candidates_by_article: dict[str, list[ImageCandidate]] = {}
|
||||
pending: list[tuple[dict, str, ImageCandidate | None]] = []
|
||||
|
||||
if external_id in synced_ids:
|
||||
skipped_count += 1
|
||||
continue
|
||||
for article in articles:
|
||||
external_id = article.get("guid", "")
|
||||
if not external_id:
|
||||
continue
|
||||
if external_id in synced_ids:
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
grade = await self._grade_article(article, ai_url, ai_model, client)
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
article_uid, is_new = self._resolve_uid(news_table, external_id)
|
||||
link = article.get("link", "")
|
||||
candidates: list[ImageCandidate] = []
|
||||
if link:
|
||||
raw_images = await _get_article_images(link, client)
|
||||
for src in raw_images[:IMG_PER_ARTICLE]:
|
||||
candidates.append(
|
||||
await _fetch_image_candidate(src, image_client)
|
||||
)
|
||||
candidates_by_article[article_uid] = candidates
|
||||
pending.append((article, article_uid, None))
|
||||
|
||||
if grade is None:
|
||||
grade_val = 0
|
||||
auto_published = False
|
||||
failed_count += 1
|
||||
sync_status = "grading_failed"
|
||||
elif grade < threshold:
|
||||
grade_val = grade
|
||||
auto_published = False
|
||||
draft_count += 1
|
||||
sync_status = "graded"
|
||||
else:
|
||||
grade_val = grade
|
||||
auto_published = True
|
||||
sync_status = "graded"
|
||||
_flag_shared_placeholders(candidates_by_article)
|
||||
|
||||
is_published = "published" if auto_published else "draft"
|
||||
existing = news_table.find_one(external_id=external_id)
|
||||
|
||||
if existing:
|
||||
existing_slug = existing.get("slug", "")
|
||||
news_table.update(
|
||||
{
|
||||
"id": existing["id"],
|
||||
"grade": grade_val,
|
||||
"status": is_published,
|
||||
"title": article.get("title", ""),
|
||||
"slug": existing_slug
|
||||
or make_combined_slug(
|
||||
article.get("title", "") or "news", existing["uid"]
|
||||
),
|
||||
"description": strip_html(
|
||||
article.get("description", "") or ""
|
||||
)[:5000],
|
||||
"url": article.get("link", ""),
|
||||
"source_name": article.get("feed_name", ""),
|
||||
"content": strip_html(article.get("content", "") or "")[
|
||||
:10000
|
||||
],
|
||||
"author": article.get("author", ""),
|
||||
"article_published": article.get("published", ""),
|
||||
"synced_at": now,
|
||||
},
|
||||
["id"],
|
||||
for article, article_uid, _ in pending:
|
||||
candidates = candidates_by_article[article_uid]
|
||||
ai_grade = await self._grade_article(
|
||||
article, ai_url, ai_model, client
|
||||
)
|
||||
updated_count += 1
|
||||
images_table.delete(news_uid=existing["uid"])
|
||||
article_uid = existing["uid"]
|
||||
else:
|
||||
article_uid = generate_uid()
|
||||
article_slug = make_combined_slug(
|
||||
article.get("title", "") or "news", article_uid
|
||||
)
|
||||
news_table.insert(
|
||||
{
|
||||
"uid": article_uid,
|
||||
"slug": article_slug,
|
||||
"external_id": external_id,
|
||||
"title": article.get("title", ""),
|
||||
"description": strip_html(
|
||||
article.get("description", "") or ""
|
||||
)[:5000],
|
||||
"url": article.get("link", ""),
|
||||
"image_url": "",
|
||||
"source_name": article.get("feed_name", ""),
|
||||
"grade": grade_val,
|
||||
"status": is_published,
|
||||
"content": strip_html(article.get("content", "") or "")[
|
||||
:10000
|
||||
],
|
||||
"author": article.get("author", ""),
|
||||
"article_published": article.get("published", ""),
|
||||
"synced_at": now,
|
||||
"deleted_at": None,
|
||||
"deleted_by": None,
|
||||
}
|
||||
)
|
||||
new_count += 1
|
||||
title = article.get("title", "")
|
||||
audit.record_system(
|
||||
"news.service.ingest",
|
||||
actor_kind="service",
|
||||
target_type="news",
|
||||
target_uid=article_uid,
|
||||
target_label=title,
|
||||
metadata={"source": article.get("feed_name", ""), "grade": grade_val},
|
||||
summary=f"news article {title} ingested",
|
||||
links=[audit.target("news", article_uid, title)],
|
||||
)
|
||||
audit.record_system(
|
||||
"news.service.publish" if auto_published else "news.service.draft",
|
||||
actor_kind="service",
|
||||
target_type="news",
|
||||
target_uid=article_uid,
|
||||
target_label=title,
|
||||
metadata={"grade": grade_val, "threshold": threshold},
|
||||
summary=f"news article {title} {'auto-published' if auto_published else 'held as draft'}",
|
||||
links=[audit.target("news", article_uid, title)],
|
||||
result = self._grade_article_full(
|
||||
article, ai_grade, candidates
|
||||
)
|
||||
|
||||
existing_sync = sync_table.find_one(external_id=external_id)
|
||||
if existing_sync:
|
||||
sync_table.update(
|
||||
{
|
||||
"id": existing_sync["id"],
|
||||
"status": sync_status,
|
||||
"synced_at": now,
|
||||
},
|
||||
["id"],
|
||||
if result.ai_grade is None:
|
||||
failed_count += 1
|
||||
sync_status = "grading_failed"
|
||||
elif not result.valid:
|
||||
rejected_count += 1
|
||||
sync_status = f"rejected_quality:{result.reject_reason}"
|
||||
elif result.effective_score < threshold:
|
||||
draft_count += 1
|
||||
sync_status = "graded"
|
||||
else:
|
||||
sync_status = "graded"
|
||||
|
||||
published = (
|
||||
result.valid and result.effective_score >= threshold
|
||||
)
|
||||
else:
|
||||
sync_table.insert(
|
||||
{
|
||||
"uid": generate_uid(),
|
||||
"external_id": external_id,
|
||||
"status": sync_status,
|
||||
"synced_at": now,
|
||||
}
|
||||
featured = (
|
||||
published
|
||||
and result.has_unique_image
|
||||
and result.effective_score >= FEATURE_MIN_SCORE
|
||||
)
|
||||
|
||||
synced_ids.add(external_id)
|
||||
saved_new = self._store_article(
|
||||
news_table,
|
||||
images_table,
|
||||
article,
|
||||
article_uid,
|
||||
result,
|
||||
published,
|
||||
featured,
|
||||
threshold,
|
||||
candidates,
|
||||
)
|
||||
if saved_new:
|
||||
new_count += 1
|
||||
else:
|
||||
updated_count += 1
|
||||
|
||||
link = article.get("link", "")
|
||||
if link:
|
||||
fresh_images = await _get_article_images(link, client)
|
||||
for img in fresh_images:
|
||||
images_table.insert(
|
||||
{
|
||||
"uid": generate_uid(),
|
||||
"news_uid": article_uid,
|
||||
"url": img["url"],
|
||||
"alt_text": img.get("alt_text", ""),
|
||||
"deleted_at": None,
|
||||
"deleted_by": None,
|
||||
}
|
||||
)
|
||||
self._record_sync(sync_table, article["guid"], sync_status)
|
||||
synced_ids.add(article["guid"])
|
||||
|
||||
self._apply_landing_selection(news_table, threshold)
|
||||
|
||||
self.log(
|
||||
f"New {new_count}, updated {updated_count}, draft {draft_count}, "
|
||||
f"grading failed {failed_count}, skipped {skipped_count}"
|
||||
f"rejected {rejected_count}, grading failed {failed_count}, "
|
||||
f"skipped {skipped_count}"
|
||||
)
|
||||
|
||||
def _resolve_uid(self, news_table, external_id: str) -> tuple[str, bool]:
|
||||
existing = news_table.find_one(external_id=external_id)
|
||||
if existing:
|
||||
return existing["uid"], False
|
||||
return generate_uid(), True
|
||||
|
||||
def _grade_article_full(
|
||||
self,
|
||||
article: dict,
|
||||
ai_grade: int | None,
|
||||
candidates: list[ImageCandidate],
|
||||
) -> ArticleGrade:
|
||||
title = clean_news_text(article.get("title", "") or "")
|
||||
description = clean_news_text(article.get("description", "") or "")
|
||||
content = clean_news_text(article.get("content", "") or "")
|
||||
body = f"{description} {content}".strip()
|
||||
url = article.get("link", "") or ""
|
||||
|
||||
has_unique_image = any(not c.is_placeholder for c in candidates)
|
||||
image_url = _primary_image(candidates)
|
||||
|
||||
if ai_grade is None:
|
||||
return ArticleGrade(
|
||||
ai_grade=None,
|
||||
effective_score=0,
|
||||
has_unique_image=has_unique_image,
|
||||
image_url=image_url,
|
||||
valid=False,
|
||||
reject_reason="grading_failed",
|
||||
candidates=candidates,
|
||||
)
|
||||
|
||||
reason = reliability_reason(title, body, url)
|
||||
if reason:
|
||||
return ArticleGrade(
|
||||
ai_grade=ai_grade,
|
||||
effective_score=ai_grade,
|
||||
has_unique_image=has_unique_image,
|
||||
image_url=image_url,
|
||||
valid=False,
|
||||
reject_reason=reason,
|
||||
candidates=candidates,
|
||||
)
|
||||
|
||||
body_marginal = len(body) < (MIN_BODY_CHARS * 2)
|
||||
score = effective_score(ai_grade, has_unique_image, body_marginal)
|
||||
return ArticleGrade(
|
||||
ai_grade=ai_grade,
|
||||
effective_score=score,
|
||||
has_unique_image=has_unique_image,
|
||||
image_url=image_url,
|
||||
valid=True,
|
||||
reject_reason="",
|
||||
candidates=candidates,
|
||||
)
|
||||
|
||||
def _store_article(
|
||||
self,
|
||||
news_table,
|
||||
images_table,
|
||||
article: dict,
|
||||
article_uid: str,
|
||||
result: ArticleGrade,
|
||||
published: bool,
|
||||
featured: bool,
|
||||
threshold: int,
|
||||
candidates: list[ImageCandidate],
|
||||
) -> bool:
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
external_id = article.get("guid", "")
|
||||
title = clean_news_text(article.get("title", "") or "")[:500] or "news"
|
||||
description = clean_news_text(article.get("description", "") or "")[:5000]
|
||||
content = clean_news_text(article.get("content", "") or "")[:10000]
|
||||
status = "published" if published else "draft"
|
||||
existing = news_table.find_one(external_id=external_id)
|
||||
|
||||
if existing:
|
||||
existing_slug = existing.get("slug", "")
|
||||
featured_locked = existing.get("featured_locked", 0)
|
||||
landing_locked = existing.get("landing_locked", 0)
|
||||
update_row = {
|
||||
"id": existing["id"],
|
||||
"grade": result.effective_score,
|
||||
"ai_grade": result.ai_grade or 0,
|
||||
"status": status,
|
||||
"title": title,
|
||||
"slug": existing_slug or make_combined_slug(title, existing["uid"]),
|
||||
"description": description,
|
||||
"url": article.get("link", ""),
|
||||
"source_name": article.get("feed_name", ""),
|
||||
"content": content,
|
||||
"author": article.get("author", ""),
|
||||
"article_published": article.get("published", ""),
|
||||
"image_url": result.image_url,
|
||||
"has_unique_image": 1 if result.has_unique_image else 0,
|
||||
"synced_at": now,
|
||||
}
|
||||
if not featured_locked:
|
||||
update_row["featured"] = 1 if featured else 0
|
||||
news_table.update(update_row, ["id"])
|
||||
images_table.delete(news_uid=existing["uid"])
|
||||
self._store_images(images_table, existing["uid"], candidates)
|
||||
return False
|
||||
|
||||
slug = make_combined_slug(title, article_uid)
|
||||
news_table.insert(
|
||||
{
|
||||
"uid": article_uid,
|
||||
"slug": slug,
|
||||
"external_id": external_id,
|
||||
"title": title,
|
||||
"description": description,
|
||||
"url": article.get("link", ""),
|
||||
"image_url": result.image_url,
|
||||
"has_unique_image": 1 if result.has_unique_image else 0,
|
||||
"source_name": article.get("feed_name", ""),
|
||||
"grade": result.effective_score,
|
||||
"ai_grade": result.ai_grade or 0,
|
||||
"status": status,
|
||||
"featured": 1 if featured else 0,
|
||||
"featured_locked": 0,
|
||||
"landing_locked": 0,
|
||||
"show_on_landing": 0,
|
||||
"content": content,
|
||||
"author": article.get("author", ""),
|
||||
"article_published": article.get("published", ""),
|
||||
"synced_at": now,
|
||||
"deleted_at": None,
|
||||
"deleted_by": None,
|
||||
}
|
||||
)
|
||||
self._store_images(images_table, article_uid, candidates)
|
||||
audit.record_system(
|
||||
"news.service.ingest",
|
||||
actor_kind="service",
|
||||
target_type="news",
|
||||
target_uid=article_uid,
|
||||
target_label=title,
|
||||
metadata={
|
||||
"source": article.get("feed_name", ""),
|
||||
"grade": result.effective_score,
|
||||
"ai_grade": result.ai_grade,
|
||||
"unique_image": result.has_unique_image,
|
||||
},
|
||||
summary=f"news article {title} ingested",
|
||||
links=[audit.target("news", article_uid, title)],
|
||||
)
|
||||
if result.valid:
|
||||
audit.record_system(
|
||||
"news.service.publish" if published else "news.service.draft",
|
||||
actor_kind="service",
|
||||
target_type="news",
|
||||
target_uid=article_uid,
|
||||
target_label=title,
|
||||
metadata={
|
||||
"grade": result.effective_score,
|
||||
"threshold": threshold,
|
||||
"featured": featured,
|
||||
},
|
||||
summary=(
|
||||
f"news article {title} "
|
||||
f"{'auto-published' if published else 'held as draft'}"
|
||||
),
|
||||
links=[audit.target("news", article_uid, title)],
|
||||
)
|
||||
else:
|
||||
audit.record_system(
|
||||
"news.service.reject",
|
||||
actor_kind="service",
|
||||
target_type="news",
|
||||
target_uid=article_uid,
|
||||
target_label=title,
|
||||
metadata={"reason": result.reject_reason},
|
||||
summary=f"news article {title} rejected ({result.reject_reason})",
|
||||
links=[audit.target("news", article_uid, title)],
|
||||
)
|
||||
return True
|
||||
|
||||
def _store_images(
|
||||
self, images_table, news_uid: str, candidates: list[ImageCandidate]
|
||||
) -> None:
|
||||
for candidate in candidates:
|
||||
images_table.insert(
|
||||
{
|
||||
"uid": generate_uid(),
|
||||
"news_uid": news_uid,
|
||||
"url": candidate.url,
|
||||
"alt_text": candidate.alt_text,
|
||||
"phash": candidate.phash,
|
||||
"width": candidate.width,
|
||||
"height": candidate.height,
|
||||
"is_placeholder": 1 if candidate.is_placeholder else 0,
|
||||
"deleted_at": None,
|
||||
"deleted_by": None,
|
||||
}
|
||||
)
|
||||
|
||||
def _record_sync(self, sync_table, external_id: str, status: str) -> None:
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
existing = sync_table.find_one(external_id=external_id)
|
||||
if existing:
|
||||
sync_table.update(
|
||||
{"id": existing["id"], "status": status, "synced_at": now}, ["id"]
|
||||
)
|
||||
else:
|
||||
sync_table.insert(
|
||||
{
|
||||
"uid": generate_uid(),
|
||||
"external_id": external_id,
|
||||
"status": status,
|
||||
"synced_at": now,
|
||||
}
|
||||
)
|
||||
|
||||
def _apply_landing_selection(self, news_table, threshold: int) -> None:
|
||||
cutoff = (
|
||||
datetime.now(timezone.utc) - timedelta(days=LANDING_RECENCY_DAYS)
|
||||
).isoformat()
|
||||
managed = list(
|
||||
news_table.find(
|
||||
deleted_at=None,
|
||||
status="published",
|
||||
featured=1,
|
||||
has_unique_image=1,
|
||||
landing_locked=0,
|
||||
synced_at={">=": cutoff},
|
||||
order_by=["-grade", "-synced_at"],
|
||||
)
|
||||
)
|
||||
chosen: list[str] = []
|
||||
for article in managed:
|
||||
if (
|
||||
len(chosen) < LANDING_MAX
|
||||
and article.get("grade", 0) >= LANDING_MIN_SCORE
|
||||
):
|
||||
chosen.append(article["uid"])
|
||||
chosen_set = set(chosen)
|
||||
for article in managed:
|
||||
target = 1 if article["uid"] in chosen_set else 0
|
||||
if article.get("show_on_landing", 0) != target:
|
||||
news_table.update(
|
||||
{"uid": article["uid"], "show_on_landing": target}, ["uid"]
|
||||
)
|
||||
if target:
|
||||
audit.record_system(
|
||||
"news.service.landing",
|
||||
actor_kind="service",
|
||||
target_type="news",
|
||||
target_uid=article["uid"],
|
||||
target_label=article.get("title"),
|
||||
metadata={"grade": article.get("grade", 0)},
|
||||
summary=(
|
||||
f"news article {article.get('title')} promoted to landing"
|
||||
),
|
||||
links=[
|
||||
audit.target(
|
||||
"news", article["uid"], article.get("title")
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
async def _grade_article(
|
||||
self, article: dict, ai_url: str, ai_model: str, client: httpx.AsyncClient
|
||||
) -> int | None:
|
||||
title = (article.get("title", "") or "")[:500]
|
||||
description = strip_html(article.get("description", "") or "")[:1000]
|
||||
content = strip_html(article.get("content", "") or "")[:1500]
|
||||
title = clean_news_text(article.get("title", "") or "")[:500]
|
||||
description = clean_news_text(article.get("description", "") or "")[:1000]
|
||||
content = clean_news_text(article.get("content", "") or "")[:1500]
|
||||
|
||||
spec = self.get_config().get("news_grade_prompt", "") or GRADE_PROMPT_SPEC
|
||||
prompt = (
|
||||
"You are an expert content strategist and editor for DevPlace, a "
|
||||
"server-rendered social network for software developers. Decide how "
|
||||
"strongly a given article should be published, expressed as a single "
|
||||
"grade.\n\n"
|
||||
"Site context:\n"
|
||||
"- Audience: software developers, hobbyists and professionals, "
|
||||
"interested in software development, AI agents, open-source tools, "
|
||||
"systems programming, and self-hosting.\n"
|
||||
"- Mission: deliver deep technical insight, critical analysis, and "
|
||||
"practical value; build authority and reader trust; avoid low-effort, "
|
||||
"rehashed, or clickbait content.\n"
|
||||
"- Tone: technical, practical, nuanced, skeptical of hype; no "
|
||||
"sensationalism or unverified claims.\n\n"
|
||||
"Evaluate the article against these weighted criteria and weigh them "
|
||||
"internally:\n"
|
||||
"1. Relevance & alignment to the developer niche and audience needs "
|
||||
"(20%).\n"
|
||||
"2. Quality & accuracy: depth, originality, factual correctness, "
|
||||
"sourcing, technical correctness, clear writing (25%).\n"
|
||||
"3. Engagement & readability: strong title/intro/flow, useful "
|
||||
"examples, appropriate length (15%).\n"
|
||||
"4. Originality & uniqueness: new perspective, data, or analysis; not "
|
||||
"duplicated generic web content (15%).\n"
|
||||
"5. SEO & discoverability: keyword opportunity without stuffing, "
|
||||
"shareable (10%).\n"
|
||||
"6. Ethics, legality & risk: no misinformation, plagiarism, or "
|
||||
"brand-safety issues; controversial topics handled with nuance and "
|
||||
"evidence (10%).\n"
|
||||
"7. Strategic fit for the site (5%).\n\n"
|
||||
"Convert your overall judgment to a single integer grade from 1 to "
|
||||
"10:\n"
|
||||
"- 9-10: excellent, publish as-is.\n"
|
||||
"- 7-8: good, worth publishing.\n"
|
||||
"- 5-6: marginal, weak fit or quality.\n"
|
||||
"- 1-4: reject, low quality or off-topic.\n\n"
|
||||
"Return ONLY a single integer from 1 to 10, nothing else.\n\n"
|
||||
f"{spec}\n\n"
|
||||
f"Title: {title}\n"
|
||||
f"Description: {description}\n"
|
||||
f"Content: {content}"
|
||||
|
||||
@@ -98,6 +98,20 @@
|
||||
color: var(--warning);
|
||||
}
|
||||
|
||||
.news-featured-badge {
|
||||
font-size: 0.625rem;
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
padding: 0.125rem 0.5rem;
|
||||
border-radius: 999px;
|
||||
background: var(--accent);
|
||||
color: var(--bg-primary);
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.25rem;
|
||||
}
|
||||
|
||||
.news-time {
|
||||
font-size: 0.75rem;
|
||||
color: var(--text-muted);
|
||||
|
||||
@@ -307,6 +307,30 @@
|
||||
max-width: 70ch;
|
||||
}
|
||||
|
||||
.service-detail-details {
|
||||
margin: 0 0 0.75rem;
|
||||
max-width: 80ch;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
background: var(--surface-2, var(--surface));
|
||||
padding: 0.5rem 0.75rem;
|
||||
}
|
||||
|
||||
.service-detail-details > summary {
|
||||
cursor: pointer;
|
||||
font-weight: 600;
|
||||
color: var(--text-primary);
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.service-detail-details-body {
|
||||
white-space: pre-line;
|
||||
color: var(--text-secondary);
|
||||
font-size: 0.8125rem;
|
||||
line-height: 1.5;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
/* ---- Tabs ---- */
|
||||
.svc-tabbar {
|
||||
display: flex;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<div class="docs-content" data-render>
|
||||
# NewsService
|
||||
|
||||
`name` `news` - enabled by default - 3600s interval (minimum 60s). It fetches developer news from a configured feed, grades each article with an AI model, and stores every article, auto-publishing those at or above the grade threshold and saving the rest as drafts. Source: `services/news.py`. See also [Reading service data](/docs/services-data.html).
|
||||
`name` `news` - enabled by default - 3600s interval (minimum 60s). It is a fully automatic, zero-maintenance import pipeline: it fetches developer news from a configured feed, cleans each article, fetches and perceptually compares the images to reject placeholders and detect uniqueness, grades each article deterministically, and auto-rotates the best ones to Featured and Landing. Source: `services/news.py`. See also [Reading service data](/docs/services-data.html).
|
||||
|
||||
> Audience: administrators and maintainers.
|
||||
|
||||
@@ -10,28 +10,70 @@
|
||||
1. Reads the source URL, grading URL, model, and threshold from configuration.
|
||||
2. Fetches the feed over HTTP (30s timeout) and reads its `articles` list. On error it logs and returns.
|
||||
3. Loads the set of already-synced external ids from `news_sync` so seen articles are skipped.
|
||||
4. For each new article it grades the article, decides published versus draft, upserts the `news` row, records the sync state, and stores any extracted images.
|
||||
5. Logs a summary: new, updated, draft, grading-failed, and skipped counts.
|
||||
4. For each new article it cleans the text, fetches up to 5 candidate images, perceptually hashes them, grades the article on the cleaned text, computes a final score, decides published versus draft and Featured, upserts the `news` row, records the sync state, and stores the images with their hashes.
|
||||
5. After the per-article loop it rotates Landing: the top Featured unique-image articles from the recent window are shown on the landing page and the rest of the service-managed set is cleared.
|
||||
6. Logs a summary: new, updated, draft, rejected, grading-failed, and skipped counts.
|
||||
|
||||
**Grading:** the article title, description, and content (each length-capped) are sent to the grading endpoint with a content-strategy prompt asking for a single integer from 1 to 10, and the first integer in the response is parsed. An article graded at or above `news_grade_threshold` is published; below threshold it is a draft; a grading failure stores it as a draft with grade 0 and sync status `grading_failed`. Nothing is silently discarded.
|
||||
**Content cleaning:** after HTML stripping the title (light), description, and content have Reddit boilerplate removed (`submitted by /u/<user>`, `submitted by ... to /r/<sub>`, standalone `[link]`, `[comments]`, `[N comments]`, and trailing `submitted by ... [link] [comments]`) and whitespace collapsed, both before grading and before storage.
|
||||
|
||||
**Image uniqueness:** each candidate image is fetched through the SSRF-guarded client (capped at 5 MB, short timeout), decoded with Pillow off the event loop, and perceptually hashed (`imagehash` phash) with its width and height recorded. An image under 100px on either side, or one that fails to fetch or decode, is a placeholder. Hashes within Hamming distance 5 that span two or more different articles are a shared logo or placeholder and are rejected for all of them. An article with at least one genuinely unique image is marked `has_unique_image` and its first such image becomes `image_url`.
|
||||
|
||||
**Grading:** the cleaned title, description, and content (each length-capped) are sent to the grading endpoint at temperature 0 asking for a single integer 1 to 10 (the raw AI grade, stored in `ai_grade`). A reliability gate forces draft (never Featured or Landing) when the url is empty or invalid, the cleaned title is shorter than 12 chars, the combined cleaned body is shorter than 200 chars, or the title uppercase-letter ratio exceeds 0.6 (the gate reason is recorded in `news_sync.status` as `rejected_quality:<reason>`). The effective score is `clamp(ai_grade + 2 if unique image - 2 if the body is marginal but not gated, 1..10)`, stored in `grade`. An article is published when valid and its effective score reaches `news_grade_threshold`; it is Featured when published, has a unique image, and scores at least 8. A grading failure stores it as a draft with grade 0 and sync status `grading_failed`. Nothing is silently discarded.
|
||||
|
||||
**Auto Featured and Landing with admin lock:** after the run the service selects the top 6 published, Featured, unique-image articles from the last 4 days scoring at least 9 for the landing page and clears the rest of its managed set (self-rotating). When an admin manually toggles Featured or Landing in the admin UI, that row's `featured_locked` or `landing_locked` is set to 1 and the service never auto-manages it again.
|
||||
|
||||
## Configuration fields
|
||||
|
||||
- `news_api_url` (url, default `https://news.app.molodetz.nl/api`) - the source feed, group Source.
|
||||
- `news_ai_url` (url, default the internal gateway) - the chat-completions endpoint used to grade, group AI grading.
|
||||
- `news_ai_model` (str, default the internal model) - the model sent to the grading endpoint.
|
||||
- `news_grade_threshold` (int, default 7, range 1 to 10) - at or above publishes, below drafts.
|
||||
- `news_grade_prompt` (text) - the exact grading rubric sent to the model; defaults to the specification below and is editable on the service page. The cleaned Title, Description and Content are appended automatically.
|
||||
- `news_grade_threshold` (int, default 7, range 1 to 10) - articles whose effective score (AI grade plus the unique-image bonus minus the thin-content penalty) reaches this publish, below drafts.
|
||||
- `news_ai_key` (secret) - defaults to the `NEWS_AI_KEY` env var, then the gateway internal key.
|
||||
|
||||
Plus the inherited Enabled, Run interval, and Log buffer size fields.
|
||||
|
||||
## Grading prompt specification
|
||||
|
||||
This is the default value of the editable `news_grade_prompt` field. It is sent verbatim to the grading model at temperature 0, with the cleaned `Title`, `Description` and `Content` of the article appended on their own lines. The model must return only a single integer from 1 to 10, which becomes the raw `ai_grade`.
|
||||
|
||||
```
|
||||
You are an expert content strategist and editor for DevPlace, a server-rendered social network for software developers. Decide how strongly a given article should be published, expressed as a single grade.
|
||||
|
||||
Site context:
|
||||
- Audience: software developers, hobbyists and professionals, interested in software development, AI agents, open-source tools, systems programming, and self-hosting.
|
||||
- Mission: deliver deep technical insight, critical analysis, and practical value; build authority and reader trust; avoid low-effort, rehashed, or clickbait content.
|
||||
- Tone: technical, practical, nuanced, skeptical of hype; no sensationalism or unverified claims.
|
||||
|
||||
Evaluate the article against these weighted criteria and weigh them internally:
|
||||
1. Relevance and alignment to the developer niche and audience needs (20%).
|
||||
2. Quality and accuracy: depth, originality, factual correctness, sourcing, technical correctness, clear writing (25%).
|
||||
3. Engagement and readability: strong title/intro/flow, useful examples, appropriate length (15%).
|
||||
4. Originality and uniqueness: new perspective, data, or analysis; not duplicated generic web content (15%).
|
||||
5. SEO and discoverability: keyword opportunity without stuffing, shareable (10%).
|
||||
6. Ethics, legality and risk: no misinformation, plagiarism, or brand-safety issues; controversial topics handled with nuance and evidence (10%).
|
||||
7. Strategic fit for the site (5%).
|
||||
|
||||
Convert your overall judgment to a single integer grade from 1 to 10:
|
||||
- 9-10: excellent, publish as-is.
|
||||
- 7-8: good, worth publishing.
|
||||
- 5-6: marginal, weak fit or quality.
|
||||
- 1-4: reject, low quality or off-topic.
|
||||
|
||||
Return ONLY a single integer from 1 to 10, nothing else.
|
||||
|
||||
Title: ...
|
||||
Description: ...
|
||||
Content: ...
|
||||
```
|
||||
|
||||
## Tables it writes
|
||||
|
||||
| Table | Purpose |
|
||||
|-------|---------|
|
||||
| `news` | The article rows: `uid`, `slug`, `external_id`, `title`, `description` (stripped, capped 5000), `url`, `image_url`, `source_name`, `grade`, `status` (`published` or `draft`), `content` (stripped, capped 10000), `author`, `article_published`, `synced_at`. Existing rows are updated in place by `external_id`. |
|
||||
| `news_sync` | One row per external id tracking `status` (`graded` or `grading_failed`) and `synced_at`, so an article is graded only once. |
|
||||
| `news_images` | Images extracted per article, keyed by `news_uid`; replaced wholesale when the article is updated. |
|
||||
| `news` | The article rows: `uid`, `slug`, `external_id`, `title`, `description` (cleaned, capped 5000), `url`, `image_url` (the chosen unique image), `has_unique_image`, `source_name`, `grade` (effective score), `ai_grade` (raw AI grade), `status` (`published` or `draft`), `featured`, `featured_locked`, `landing_locked`, `show_on_landing`, `content` (cleaned, capped 10000), `author`, `article_published`, `synced_at`. Existing rows are updated in place by `external_id` (locked Featured stays untouched). |
|
||||
| `news_sync` | One row per external id tracking `status` (`graded`, `grading_failed`, or `rejected_quality:<reason>`) and `synced_at`, so an article is graded only once. |
|
||||
| `news_images` | Images fetched per article, keyed by `news_uid`, each with `url`, `alt_text`, `phash`, `width`, `height`, and `is_placeholder`; replaced wholesale when the article is updated. |
|
||||
|
||||
## Data it offers
|
||||
|
||||
|
||||
@@ -124,6 +124,9 @@
|
||||
<div class="news-card-body">
|
||||
<div class="news-card-meta">
|
||||
<span class="news-source">{{ item.source_name }}</span>
|
||||
{% if item.featured %}
|
||||
<span class="news-featured-badge">★ Featured</span>
|
||||
{% endif %}
|
||||
{% if item.grade >= 9 %}
|
||||
<span class="news-grade news-grade-top">Grade {{ item.grade }}</span>
|
||||
{% elif item.grade >= 7 %}
|
||||
|
||||
@@ -23,6 +23,9 @@
|
||||
<div class="news-card-body">
|
||||
<div class="news-card-meta">
|
||||
<span class="news-source">{{ item.article['source_name'] }}</span>
|
||||
{% if item.article.get('featured') %}
|
||||
<span class="news-featured-badge">★ Featured</span>
|
||||
{% endif %}
|
||||
{% if item.grade >= 9 %}
|
||||
<span class="news-grade news-grade-top">Grade {{ item.grade }}</span>
|
||||
{% elif item.grade >= 7 %}
|
||||
|
||||
@@ -15,6 +15,9 @@
|
||||
<div class="news-detail-body">
|
||||
<div class="news-detail-meta">
|
||||
<span class="news-source">{{ article['source_name'] }}</span>
|
||||
{% if article.get('featured') %}
|
||||
<span class="news-featured-badge">★ Featured</span>
|
||||
{% endif %}
|
||||
{% if grade >= 9 %}
|
||||
<span class="news-grade news-grade-top">Grade {{ grade }}</span>
|
||||
{% elif grade >= 7 %}
|
||||
|
||||
@@ -14,6 +14,12 @@
|
||||
<span class="service-status {{ svc.status }}">{{ svc.status }}</span>
|
||||
</div>
|
||||
{% if svc.description %}<p class="service-detail-desc">{{ svc.description }}</p>{% endif %}
|
||||
{% if svc.details %}
|
||||
<details class="service-detail-details" open>
|
||||
<summary>Grading algorithm</summary>
|
||||
<div class="service-detail-details-body">{{ svc.details }}</div>
|
||||
</details>
|
||||
{% endif %}
|
||||
<div class="service-controls">
|
||||
<button type="button" class="service-btn" data-action="start" data-service="{{ svc.name }}" {% if svc.enabled %}disabled{% endif %}>Start</button>
|
||||
<button type="button" class="service-btn" data-action="stop" data-service="{{ svc.name }}" data-confirm="Stop the {{ svc.title }} service?" {% if not svc.enabled %}disabled{% endif %}>Stop</button>
|
||||
|
||||
Reference in New Issue
Block a user