blindxfish/devplacepy
1
0
Fork 0
forked from retoor/devplacepy
Code Pull Requests Activity

feat: add AI markdown reformatting and usage metering to NewsService

Browse Source 
Add AI-powered body reformatting for valid articles in the news pipeline, converting raw text walls into clean Markdown with paragraphs, headings, and lists. Introduce `news_usage` database table and `add_news_usage`/`get_news_usage` helpers to track per-cycle gateway costs (calls, tokens, latency, USD) from response headers, reported on the admin Services page. Remove unused `.sidebar-more` CSS and apply `render_title()` to poll question/label fields.
This commit is contained in:
retoor
2026-06-18 23:46:53 +00:00
parent 6ceca3d0d4
commit f3a4667fce
13 changed files with 427 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devplacepy/database.py
+34
View File
@@ -215,6 +215,17 @@ def get_modifier_usage(user_uid: str) -> dict:
return _get_usage("modifier_usage", user_uid)
NEWS_USAGE_KEY = "news"
def add_news_usage(totals: dict) -> None:
_add_usage("news_usage", NEWS_USAGE_KEY, totals)
def get_news_usage() -> dict:
return _get_usage("news_usage", NEWS_USAGE_KEY)
def record_activity(user_uid: str, action: str) -> int:
if not user_uid or not action or "user_activity" not in db.tables:
return 0
@@ -802,6 +813,29 @@ def init_db():
except Exception as e:
logger.warning(f"Could not create unique index on modifier_usage: {e}")
news_usage = get_table("news_usage")
for column, example in (
("user_uid", ""),
("calls", 0),
("prompt_tokens", 0),
("completion_tokens", 0),
("total_tokens", 0),
("cost_usd", 0.0),
("upstream_latency_ms", 0.0),
("total_latency_ms", 0.0),
("updated_at", ""),
):
if not news_usage.has_column(column):
news_usage.create_column_by_example(column, example)
try:
if "news_usage" in db.tables:
db.query(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_news_usage_user "
"ON news_usage (user_uid)"
)
except Exception as e:
logger.warning(f"Could not create unique index on news_usage: {e}")
email_accounts = get_table("email_accounts")
for column, example in (
("uid", ""),
devplacepy/services/news.py
+198 -4
View File
@@ -15,8 +15,15 @@ from PIL import Image
from devplacepy import stealth
from devplacepy import net_guard
from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
from devplacepy.database import get_table, get_setting, internal_gateway_key
from devplacepy.database import (
add_news_usage,
get_news_usage,
get_setting,
get_table,
internal_gateway_key,
)
from devplacepy.services.base import BaseService, ConfigField
from devplacepy.services.openai_gateway.usage import parse_usage_headers
from devplacepy.utils import generate_uid, make_combined_slug, strip_html
from devplacepy.services.audit import record as audit
@@ -27,6 +34,9 @@ AI_URL_DEFAULT = INTERNAL_GATEWAY_URL
AI_MODEL_DEFAULT = INTERNAL_MODEL
GRADE_THRESHOLD_DEFAULT = 7
GRADE_MAX_TOKENS = 2000
FORMAT_MAX_TOKENS = 6000
FORMAT_INPUT_MAX_CHARS = 14000
FORMAT_OUTPUT_MAX_CHARS = 30000
MIN_TITLE_CHARS = 12
MIN_BODY_CHARS = 200
@@ -97,6 +107,28 @@ GRADE_PROMPT_SPEC = (
"Return ONLY a single integer from 1 to 10, nothing else."
)
FORMAT_PROMPT_SPEC = (
"You are an expert editor for DevPlace, a social network for software "
"developers. You are given the raw body of a news article that arrived as "
"an undifferentiated wall of text. Reformat it into clean, readable "
"Markdown for a technical audience.\n\n"
"Rules:\n"
"- Break the text into short, well-structured paragraphs separated by a "
"blank line.\n"
"- Add Markdown section headings (## Heading) where the topic clearly "
"shifts, so the article scans well.\n"
"- Use bullet or numbered lists for enumerations, and inline code or "
"fenced code blocks where code, commands, or identifiers appear.\n"
"- Preserve every fact, name, number, and quotation exactly as given. "
"Never invent, add, remove, or reorder information.\n"
"- Only restructure and lightly polish wording for flow and grammar; do "
"not add an introduction, conclusion, opinion, or commentary of your "
"own.\n"
"- Do not repeat the article title as a heading and do not wrap the whole "
"answer in a code fence.\n"
"- Output only the reformatted article body as Markdown, nothing else."
)
GRADING_RULES_DESCRIPTION = (
"Each run fetches the feed, cleans every article, fetches and perceptually "
"compares images, grades deterministically, and auto-rotates Featured and "
@@ -123,6 +155,12 @@ GRADING_RULES_DESCRIPTION = (
"(Configuration tab), sent to the model at temperature 0 with the cleaned "
"Title, Description and Content appended; it must return a single integer "
"from 1 to 10.\n\n"
"Formatting: after grading, every valid article is reformatted by the AI "
"into clean Markdown (paragraphs, section headings, lists and code spans) "
"using the editable news_format_prompt field, preserving every fact while "
"turning the source wall of text into a readable article. The formatted "
"Markdown replaces the stored content; on any failure the cleaned original "
"is kept. Disable with the news_format_enabled toggle.\n\n"
"Scoring: the AI grade (1-10, temperature 0) is the base. effective_score = "
f"clamp(ai_grade + {UNIQUE_IMAGE_BONUS} if unique image - "
f"{THIN_CONTENT_PENALTY} if the body is marginal but not gated, 1..10). The "
@@ -171,6 +209,31 @@ def _get_ai_key() -> str:
return internal_gateway_key()
USAGE_FIELDS = (
"calls",
"prompt_tokens",
"completion_tokens",
"total_tokens",
"cost_usd",
"upstream_latency_ms",
"total_latency_ms",
)
def _new_usage_totals() -> dict:
return {field: 0 for field in USAGE_FIELDS}
def _accumulate_usage(totals: dict | None, response) -> None:
if totals is None:
return
parsed = parse_usage_headers(getattr(response, "headers", None))
if not parsed:
return
for field in USAGE_FIELDS:
totals[field] += parsed[field]
def _extract_grade(text: str) -> int | None:
match = re.search(r"\d+", text.strip())
if match:
@@ -180,6 +243,16 @@ def _extract_grade(text: str) -> int | None:
return None
def _strip_md_fence(text: str) -> str:
stripped = text.strip()
if not stripped.startswith("```"):
return stripped
lines = stripped.splitlines()
if len(lines) < 2 or not lines[-1].strip().startswith("```"):
return stripped
return "\n".join(lines[1:-1]).strip()
def clean_news_text(text: str) -> str:
if not text:
return ""
@@ -383,6 +456,30 @@ class NewsService(BaseService):
help="Defaults to the NEWS_AI_KEY env var, then the gateway's internal key.",
group="AI grading",
),
ConfigField(
"news_format_enabled",
"Reformat content with AI",
type="bool",
default=True,
help=(
"When enabled, every valid article is reformatted into clean "
"Markdown (paragraphs, headings, lists) after grading."
),
group="AI formatting",
),
ConfigField(
"news_format_prompt",
"Formatting prompt specification",
type="text",
default=FORMAT_PROMPT_SPEC,
help=(
"The instruction sent to the AI to reformat each cleaned "
"article into Markdown. The Title and the article body are "
"appended automatically. It must preserve every fact and "
"output only the reformatted Markdown body."
),
group="AI formatting",
),
]
def __init__(self):
@@ -394,6 +491,7 @@ class NewsService(BaseService):
ai_url = config["news_ai_url"]
ai_model = config["news_ai_model"]
threshold = config["news_grade_threshold"]
format_enabled = config["news_format_enabled"]
self.log(f"Fetching news from {api_url}")
async with stealth.stealth_async_client(timeout=30.0) as client:
@@ -422,6 +520,7 @@ class NewsService(BaseService):
failed_count = 0
rejected_count = 0
skipped_count = 0
usage_totals = _new_usage_totals()
async with net_guard.guarded_async_client(
timeout=IMG_FETCH_TIMEOUT
@@ -454,7 +553,7 @@ class NewsService(BaseService):
for article, article_uid, _ in pending:
candidates = candidates_by_article[article_uid]
ai_grade = await self._grade_article(
article, ai_url, ai_model, client
article, ai_url, ai_model, client, usage_totals
)
result = self._grade_article_full(
article, ai_grade, candidates
@@ -481,6 +580,12 @@ class NewsService(BaseService):
and result.effective_score >= FEATURE_MIN_SCORE
)
formatted_content = ""
if result.valid and format_enabled:
formatted_content = await self._format_article(
article, ai_url, ai_model, client, usage_totals
)
saved_new = self._store_article(
news_table,
images_table,
@@ -491,6 +596,7 @@ class NewsService(BaseService):
featured,
threshold,
candidates,
formatted_content,
)
if saved_new:
new_count += 1
@@ -502,12 +608,38 @@ class NewsService(BaseService):
self._apply_landing_selection(news_table, threshold)
if usage_totals["calls"]:
add_news_usage(usage_totals)
self.log(
f"AI usage: {usage_totals['calls']} calls, "
f"{usage_totals['total_tokens']} tokens, "
f"${usage_totals['cost_usd']:.4f}"
)
self.log(
f"New {new_count}, updated {updated_count}, draft {draft_count}, "
f"rejected {rejected_count}, grading failed {failed_count}, "
f"skipped {skipped_count}"
)
def collect_metrics(self) -> dict:
usage = get_news_usage()
stats = [
{"label": "AI calls", "value": usage["calls"]},
{"label": "Total tokens", "value": usage["total_tokens"]},
{"label": "Prompt tokens", "value": usage["prompt_tokens"]},
{"label": "Completion tokens", "value": usage["completion_tokens"]},
{"label": "Total cost", "value": f"${usage['cost_usd']:.4f}"},
{"label": "Avg tokens/call", "value": usage["avg_tokens"]},
{"label": "Avg cost/call", "value": f"${usage['avg_cost_usd']:.6f}"},
{
"label": "Avg latency",
"value": f"{usage['avg_upstream_latency_ms']:.0f}ms",
},
{"label": "Avg tokens/sec", "value": usage["avg_tokens_per_second"]},
]
return {"stats": stats}
def _resolve_uid(self, news_table, external_id: str) -> tuple[str, bool]:
existing = news_table.find_one(external_id=external_id)
if existing:
@@ -575,12 +707,16 @@ class NewsService(BaseService):
featured: bool,
threshold: int,
candidates: list[ImageCandidate],
formatted_content: str = "",
) -> bool:
now = datetime.now(timezone.utc).isoformat()
external_id = article.get("guid", "")
title = clean_news_text(article.get("title", "") or "")[:500] or "news"
description = clean_news_text(article.get("description", "") or "")[:5000]
content = clean_news_text(article.get("content", "") or "")[:10000]
if formatted_content:
content = formatted_content[:FORMAT_OUTPUT_MAX_CHARS]
else:
content = clean_news_text(article.get("content", "") or "")[:10000]
status = "published" if published else "draft"
existing = news_table.find_one(external_id=external_id)
@@ -770,7 +906,12 @@ class NewsService(BaseService):
)
async def _grade_article(
self, article: dict, ai_url: str, ai_model: str, client: httpx.AsyncClient
self,
article: dict,
ai_url: str,
ai_model: str,
client: httpx.AsyncClient,
totals: dict | None = None,
) -> int | None:
title = clean_news_text(article.get("title", "") or "")[:500]
description = clean_news_text(article.get("description", "") or "")[:1000]
@@ -805,6 +946,7 @@ class NewsService(BaseService):
if resp.status_code != 200:
self.log(f"AI grading returned {resp.status_code}: {resp.text[:200]}")
resp.raise_for_status()
_accumulate_usage(totals, resp)
result = resp.json()
text = result.get("choices", [{}])[0].get("message", {}).get("content", "")
if not text:
@@ -817,3 +959,55 @@ class NewsService(BaseService):
except Exception as e:
self.log(f"AI grading failed for '{title[:50]}': {e}")
return None
async def _format_article(
self,
article: dict,
ai_url: str,
ai_model: str,
client: httpx.AsyncClient,
totals: dict | None = None,
) -> str:
title = clean_news_text(article.get("title", "") or "")[:500]
description = clean_news_text(article.get("description", "") or "")
content = clean_news_text(article.get("content", "") or "")
body = f"{description}\n\n{content}".strip()
if not body:
return ""
body = body[:FORMAT_INPUT_MAX_CHARS]
spec = self.get_config().get("news_format_prompt", "") or FORMAT_PROMPT_SPEC
prompt = f"{spec}\n\nTitle: {title}\n\nArticle:\n{body}"
payload = {
"model": ai_model,
"messages": [{"role": "user", "content": prompt}],
"max_tokens": FORMAT_MAX_TOKENS,
"temperature": 0.3,
}
headers = {"Content-Type": "application/json"}
ai_key = _get_ai_key()
if ai_key:
headers["Authorization"] = f"Bearer {ai_key}"
try:
resp = await client.post(
ai_url, json=payload, headers=headers, timeout=60.0
)
if resp.status_code != 200:
self.log(
f"AI formatting returned {resp.status_code}: {resp.text[:200]}"
)
resp.raise_for_status()
_accumulate_usage(totals, resp)
result = resp.json()
text = result.get("choices", [{}])[0].get("message", {}).get("content", "")
text = _strip_md_fence(text or "")
if len(text) < MIN_BODY_CHARS:
self.log(f"AI formatting returned too little for: {title[:60]}")
return ""
return text
except Exception as e:
self.log(f"AI formatting failed for '{title[:50]}': {e}")
return ""
devplacepy/static/css/sidebar.css
-28
View File
@@ -82,34 +82,6 @@
box-shadow: var(--glow-accent);
}
.sidebar-more {
display: flex;
flex-direction: column;
gap: 0.125rem;
}
.sidebar-more-toggle {
list-style: none;
cursor: pointer;
user-select: none;
}
.sidebar-more-toggle::-webkit-details-marker {
display: none;
}
.sidebar-more-chevron {
margin-left: auto;
font-size: 1rem;
line-height: 1;
color: var(--text-muted);
transition: transform 0.15s ease;
}
.sidebar-more[open] > .sidebar-more-toggle .sidebar-more-chevron {
transform: rotate(90deg);
}
.sidebar-dot {
width: 8px;
height: 8px;
devplacepy/templates/_poll.html
+2 -2
View File
@@ -1,11 +1,11 @@
{% if _poll %}
<div class="poll" data-poll-uid="{{ _poll.uid }}">
<div class="poll-question">{{ _poll.question }}</div>
<div class="poll-question">{{ render_title(_poll.question) }}</div>
<div class="poll-options" role="group" aria-label="Poll options">
{% for opt in _poll.options %}
<button type="button" class="poll-option{% if _poll.my_choice == opt.uid %} chosen{% endif %}" data-option-uid="{{ opt.uid }}" aria-pressed="{% if _poll.my_choice == opt.uid %}true{% else %}false{% endif %}"{{ guest_disabled(user) }}>
<span class="poll-option-bar" style="width: {{ opt.pct }}%;"></span>
<span class="poll-option-label">{{ opt.label }}</span>
<span class="poll-option-label">{{ render_title(opt.label) }}</span>
<span class="poll-option-meta">
<span class="poll-option-check"{% if _poll.my_choice != opt.uid %} hidden{% endif %}>&#x2713;</span>
<span class="poll-option-pct">{{ opt.pct }}%</span>
devplacepy/templates/feed.html
+12 -19
View File
@@ -29,25 +29,18 @@
<span class="icon">&#x2753;</span>
Question
</a>
<details class="sidebar-more" {% if current_topic in ['rant', 'fun', 'signals'] %}open{% endif %}>
<summary class="sidebar-link sidebar-more-toggle">
<span class="icon">&#x2026;</span>
More
<span class="sidebar-more-chevron">&#x203A;</span>
</summary>
<a href="/feed?topic=rant" class="sidebar-link {% if current_topic == 'rant' %}active{% endif %}" data-topic="rant">
<span class="icon">&#x1F4A2;</span>
Rant
</a>
<a href="/feed?topic=fun" class="sidebar-link {% if current_topic == 'fun' %}active{% endif %}" data-topic="fun">
<span class="icon">&#x1F3AE;</span>
Fun
</a>
<a href="/feed?topic=signals" class="sidebar-link {% if current_topic == 'signals' %}active{% endif %}" data-topic="signals">
<span class="icon">&#x1F4E1;</span>
Signals
</a>
</details>
<a href="/feed?topic=rant" class="sidebar-link {% if current_topic == 'rant' %}active{% endif %}" data-topic="rant">
<span class="icon">&#x1F4A2;</span>
Rant
</a>
<a href="/feed?topic=fun" class="sidebar-link {% if current_topic == 'fun' %}active{% endif %}" data-topic="fun">
<span class="icon">&#x1F3AE;</span>
Fun
</a>
<a href="/feed?topic=signals" class="sidebar-link {% if current_topic == 'signals' %}active{% endif %}" data-topic="signals">
<span class="icon">&#x1F4E1;</span>
Signals
</a>
</div>
<div class="sidebar-section">
devplacepy/templates/landing.html
+2
View File
@@ -124,6 +124,7 @@
<div class="news-card-body">
<div class="news-card-meta">
<span class="news-source">{{ item.source_name }}</span>
{% if is_admin(user) %}
{% if item.featured %}
<span class="news-featured-badge">&#x2605; Featured</span>
{% endif %}
@@ -134,6 +135,7 @@
{% else %}
<span class="news-grade">Grade {{ item.grade }}</span>
{% endif %}
{% endif %}
<span class="news-time">{{ dt_ago(item.synced_at) if item.synced_at else item.time_ago }}</span>
</div>
<h3 class="news-card-title">
devplacepy/templates/news.html
+2
View File
@@ -23,6 +23,7 @@
<div class="news-card-body">
<div class="news-card-meta">
<span class="news-source">{{ item.article['source_name'] }}</span>
{% if is_admin(user) %}
{% if item.article.get('featured') %}
<span class="news-featured-badge">&#x2605; Featured</span>
{% endif %}
@@ -33,6 +34,7 @@
{% else %}
<span class="news-grade">Grade {{ item.grade }}</span>
{% endif %}
{% endif %}
<span class="news-time">{{ dt_ago(item.synced_at) if item.synced_at else item.time_ago }}</span>
</div>
<h3 class="news-card-title">
devplacepy/templates/news_detail.html
+2
View File
@@ -15,6 +15,7 @@
<div class="news-detail-body">
<div class="news-detail-meta">
<span class="news-source">{{ article['source_name'] }}</span>
{% if is_admin(user) %}
{% if article.get('featured') %}
<span class="news-featured-badge">&#x2605; Featured</span>
{% endif %}
@@ -25,6 +26,7 @@
{% else %}
<span class="news-grade">Grade {{ grade }}</span>
{% endif %}
{% endif %}
<span class="news-time">{{ dt_ago(article.synced_at) if article.get('synced_at') else time_ago }}</span>
</div>