feat: add api key auth, devii agent, openai gateway, and admin service management
This commit introduces a comprehensive set of new features including API key authentication with CLI management commands (get, reset, backfill), a Devii agentic assistant with WebSocket terminal and session bootstrap, an OpenAI-compatible LLM gateway service, and an admin service management panel. It also adds Playwright browser automation for bot support, configures internal gateway URLs, refactors content editing/deletion to support JSON API responses, and updates documentation across AGENTS.md, README.md, and the developer docs site.
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
from devplacepy.services.bot.service import BotsService
|
||||
|
||||
__all__ = ["BotsService"]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,162 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
from typing import Optional
|
||||
|
||||
from playwright.async_api import Page, Playwright, async_playwright
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BotBrowser:
|
||||
def __init__(self, headless: bool = True):
|
||||
self._headless = headless
|
||||
self._playwright: Optional[Playwright] = None
|
||||
self._browser = None
|
||||
self._context = None
|
||||
self._page: Optional[Page] = None
|
||||
|
||||
async def launch(self) -> None:
|
||||
self._playwright = await async_playwright().start()
|
||||
self._browser = await self._playwright.chromium.launch(
|
||||
headless=self._headless,
|
||||
args=[
|
||||
"--disable-blink-features=AutomationControlled", "--no-first-run",
|
||||
"--disable-dev-shm-usage", "--window-size=1280,900",
|
||||
],
|
||||
)
|
||||
self._context = await self._browser.new_context(
|
||||
user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36",
|
||||
viewport={"width": 1280, "height": 900},
|
||||
locale="en-US",
|
||||
)
|
||||
self._page = await self._context.new_page()
|
||||
|
||||
async def close(self) -> None:
|
||||
for label, resource, shutdown in (
|
||||
("context", self._context, lambda r: r.close()),
|
||||
("browser", self._browser, lambda r: r.close()),
|
||||
("playwright", self._playwright, lambda r: r.stop()),
|
||||
):
|
||||
if resource is None:
|
||||
continue
|
||||
try:
|
||||
await shutdown(resource)
|
||||
except Exception as e:
|
||||
logger.debug("Browser %s close error: %s", label, e)
|
||||
self._page = None
|
||||
self._context = None
|
||||
self._browser = None
|
||||
self._playwright = None
|
||||
|
||||
@property
|
||||
def page(self) -> Page:
|
||||
return self._page
|
||||
|
||||
async def goto(self, url: str) -> None:
|
||||
try:
|
||||
await self._page.goto(url, wait_until="domcontentloaded", timeout=15000)
|
||||
except Exception as e:
|
||||
logger.debug("goto (domcontentloaded) failed for %s: %s", url[:80], e)
|
||||
try:
|
||||
await self._page.goto(url, timeout=20000)
|
||||
except Exception as e2:
|
||||
logger.warning("goto failed for %s: %s", url[:80], e2)
|
||||
await self._idle(0.5, 1.5)
|
||||
|
||||
async def _idle(self, lo: float = 0.5, hi: float = 2.0) -> None:
|
||||
await asyncio.sleep(random.uniform(lo, hi))
|
||||
|
||||
async def scroll(self, amount: Optional[int] = None) -> None:
|
||||
if amount is None:
|
||||
amount = random.randint(300, 800)
|
||||
try:
|
||||
await self._page.evaluate(f"window.scrollBy(0, {amount})")
|
||||
except Exception as e:
|
||||
logger.debug("scroll failed: %s", e)
|
||||
await self._idle(0.2, 0.6)
|
||||
|
||||
async def scroll_to_bottom(self) -> None:
|
||||
try:
|
||||
await self._page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
||||
except Exception as e:
|
||||
logger.debug("scroll_to_bottom failed: %s", e)
|
||||
|
||||
async def fill(self, sel: str, val: str) -> bool:
|
||||
try:
|
||||
el = self._page.locator(sel).first
|
||||
if await el.count() == 0:
|
||||
logger.debug("fill: selector '%s' not found", sel)
|
||||
return False
|
||||
await el.click(timeout=3000)
|
||||
await self._idle(0.05, 0.15)
|
||||
await el.fill("")
|
||||
for ch in val:
|
||||
await self._page.keyboard.type(ch, delay=random.randint(20, 60))
|
||||
if ch == " ":
|
||||
await asyncio.sleep(random.uniform(0.02, 0.08))
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug("fill failed for '%s': %s", sel, e)
|
||||
return False
|
||||
|
||||
async def click(self, sel: str) -> bool:
|
||||
try:
|
||||
el = self._page.locator(sel).first
|
||||
if await el.count() == 0:
|
||||
logger.debug("click: selector '%s' not found", sel)
|
||||
return False
|
||||
box = await el.bounding_box()
|
||||
if box:
|
||||
tx = box["x"] + box["width"] / 2 + random.uniform(-3, 3)
|
||||
ty = box["y"] + box["height"] / 2 + random.uniform(-3, 3)
|
||||
await self._page.mouse.move(tx, ty, steps=random.randint(8, 18))
|
||||
await self._idle(0.02, 0.06)
|
||||
await el.click(timeout=5000)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug("click failed for '%s': %s", sel, e)
|
||||
return False
|
||||
|
||||
async def text(self, sel: str) -> str:
|
||||
try:
|
||||
return (await self._page.locator(sel).first.inner_text()) or ""
|
||||
except Exception as e:
|
||||
logger.debug("text failed for '%s': %s", sel, e)
|
||||
return ""
|
||||
|
||||
async def attr(self, sel: str, attr: str) -> str:
|
||||
try:
|
||||
val = await self._page.locator(sel).first.get_attribute(attr)
|
||||
return val or ""
|
||||
except Exception as e:
|
||||
logger.debug("attr '%s' on '%s' failed: %s", attr, sel, e)
|
||||
return ""
|
||||
|
||||
async def html(self) -> str:
|
||||
try:
|
||||
return await self._page.content()
|
||||
except Exception as e:
|
||||
logger.debug("html() failed: %s", e)
|
||||
return ""
|
||||
|
||||
async def url(self) -> str:
|
||||
try:
|
||||
return self._page.url
|
||||
except Exception as e:
|
||||
logger.debug("url() failed: %s", e)
|
||||
return ""
|
||||
|
||||
async def title(self) -> str:
|
||||
try:
|
||||
return await self._page.title()
|
||||
except Exception as e:
|
||||
logger.debug("title() failed: %s", e)
|
||||
return ""
|
||||
|
||||
async def reload(self) -> None:
|
||||
try:
|
||||
await self._page.reload(timeout=15000)
|
||||
await self._idle()
|
||||
except Exception as e:
|
||||
logger.debug("reload failed: %s", e)
|
||||
@@ -0,0 +1,58 @@
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
|
||||
|
||||
BASE_URL_DEFAULT = "https://pravda.education"
|
||||
API_URL_DEFAULT = INTERNAL_GATEWAY_URL
|
||||
NEWS_API_DEFAULT = "https://news.app.molodetz.nl/api"
|
||||
MODEL_DEFAULT = INTERNAL_MODEL
|
||||
INPUT_COST_PER_1M_DEFAULT = 0.27
|
||||
OUTPUT_COST_PER_1M_DEFAULT = 1.10
|
||||
|
||||
STATE_DIR = Path.home() / ".devplace_bots"
|
||||
ARTICLE_REGISTRY_PATH = Path.home() / ".dpbot_article_registry.json"
|
||||
|
||||
PROJECT_STATUSES = ["In Development", "Released"]
|
||||
|
||||
PERSONAS = [
|
||||
"enthusiastic_junior",
|
||||
"grumpy_senior",
|
||||
"hobbyist_maker",
|
||||
"academic_type",
|
||||
"minimalist",
|
||||
"storyteller",
|
||||
"rebel",
|
||||
"mentor",
|
||||
]
|
||||
|
||||
CATEGORIES = ["devlog", "showcase", "question", "rant", "fun", "random"]
|
||||
|
||||
GIST_LANGUAGES = [
|
||||
"python", "javascript", "typescript", "bash", "go", "rust",
|
||||
"sql", "html", "css", "java", "c", "cpp", "ruby", "php", "lua",
|
||||
]
|
||||
|
||||
FEED_TOPICS = ["devlog", "showcase", "question", "rant", "fun"]
|
||||
PROJECT_TYPES = ["game", "game_asset", "software", "mobile_app", "website"]
|
||||
|
||||
SEARCH_TERMS = {
|
||||
"enthusiastic_junior": ["vibe coding", "first app", "react", "python", "ai tools"],
|
||||
"grumpy_senior": ["rust", "perl", "legacy", "tech debt", "kubernetes"],
|
||||
"hobbyist_maker": ["arduino", "raspberry pi", "side project", "3d printing", "game"],
|
||||
"academic_type": ["algorithms", "type theory", "distributed systems", "compilers"],
|
||||
"minimalist": ["cli", "vim", "golang", "suckless"],
|
||||
"storyteller": ["postmortem", "war story", "migration", "outage"],
|
||||
"rebel": ["hot take", "overrated", "monorepo", "microservices"],
|
||||
"mentor": ["best practices", "code review", "testing", "career"],
|
||||
}
|
||||
|
||||
MIN_COMMENT_LEN = 40
|
||||
MIN_POST_LEN = 120
|
||||
RESTATEMENT_OVERLAP_THRESHOLD = 0.6
|
||||
|
||||
GENERIC_COMMENT_PHRASES = [
|
||||
"great point", "good point", "i agree", "totally agree", "well said",
|
||||
"nice post", "great post", "thanks for sharing", "interesting read",
|
||||
"this is interesting", "so true", "couldn't agree more", "spot on",
|
||||
"great article", "love this", "this is great", "good read",
|
||||
]
|
||||
@@ -0,0 +1,274 @@
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from devplacepy.services.bot.config import CATEGORIES, GIST_LANGUAGES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LLMClient:
|
||||
def __init__(self, api_key: str, api_url: str, model: str,
|
||||
input_cost_per_1m: float, output_cost_per_1m: float):
|
||||
if not api_key:
|
||||
raise RuntimeError("LLM API key not set")
|
||||
self.api_key = api_key
|
||||
self.api_url = api_url
|
||||
self.model = model
|
||||
self.input_cost_per_1m = input_cost_per_1m
|
||||
self.output_cost_per_1m = output_cost_per_1m
|
||||
self.total_cost = 0.0
|
||||
self.total_calls = 0
|
||||
self.total_in_tokens = 0
|
||||
self.total_out_tokens = 0
|
||||
|
||||
def _call(self, system: str, prompt: str, temperature: float = 0.7) -> str:
|
||||
for attempt in range(3):
|
||||
try:
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}],
|
||||
"temperature": temperature,
|
||||
}
|
||||
logger.info("LLM >>> model=%s system=%s prompt=%s",
|
||||
self.model, json.dumps(system[:500]), json.dumps(prompt[:500]))
|
||||
data = json.dumps(payload).encode()
|
||||
req = urllib.request.Request(
|
||||
self.api_url, data=data,
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
logger.info("LLM <<< %s", raw[:2000].decode(errors="replace"))
|
||||
result = json.loads(raw)
|
||||
usage = result.get("usage", {})
|
||||
in_tokens = usage.get("prompt_tokens", 0)
|
||||
out_tokens = usage.get("completion_tokens", 0)
|
||||
cost = (in_tokens * self.input_cost_per_1m / 1_000_000) + \
|
||||
(out_tokens * self.output_cost_per_1m / 1_000_000)
|
||||
self.total_cost += cost
|
||||
self.total_calls += 1
|
||||
self.total_in_tokens += in_tokens
|
||||
self.total_out_tokens += out_tokens
|
||||
return self.clean(result["choices"][0]["message"]["content"])
|
||||
except (urllib.error.HTTPError, urllib.error.URLError, json.JSONDecodeError, KeyError) as e:
|
||||
logger.warning("LLM call attempt %d/3 failed: %s", attempt + 1, e)
|
||||
if attempt == 2:
|
||||
raise
|
||||
time.sleep(2 ** attempt)
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def clean(text: str, preserve_md: bool = False) -> str:
|
||||
if not preserve_md:
|
||||
text = re.sub(r"\*\*|__|\*|_", "", text)
|
||||
text = text.replace("-", "-").replace("–", "-")
|
||||
text = text.replace("‘", "'").replace("’", "'")
|
||||
text = text.replace("“", '"').replace("”", '"')
|
||||
text = re.sub(r"\s{2,}", " ", text)
|
||||
return text.strip()
|
||||
|
||||
@staticmethod
|
||||
def strip_md(text: str) -> str:
|
||||
return LLMClient.clean(text)[:200]
|
||||
|
||||
@staticmethod
|
||||
def strip_code_fences(text: str) -> str:
|
||||
text = text.strip()
|
||||
if text.startswith("```"):
|
||||
lines = text.split("\n")
|
||||
if lines and lines[0].lstrip().startswith("```"):
|
||||
lines = lines[1:]
|
||||
if lines and lines[-1].strip().startswith("```"):
|
||||
lines = lines[:-1]
|
||||
text = "\n".join(lines)
|
||||
return text.strip()
|
||||
|
||||
def generate_post(self, title: str, desc: str, persona: str = "", category: str = "") -> str:
|
||||
persona_extras = {
|
||||
"enthusiastic_junior": "Be excited. Use **bold** for emphasis. Short excited sentences. End with a question sometimes.",
|
||||
"grumpy_senior": "Be slightly cynical but helpful. Short blunt sentences. No fluff. Call out bad practices.",
|
||||
"hobbyist_maker": "Be casual and friendly. Mention side projects and tinkering. Use *italics* for tools/libraries.",
|
||||
"academic_type": "Be precise and structured. Use *italics* for terminology. Well thought out arguments.",
|
||||
"storyteller": "Use **bold** for key points. Write anecdotal, narrative style. Longer flowing sentences.",
|
||||
"rebel": "Be informal. Skip punctuation sometimes. Use slang. Type like you're in a hurry.",
|
||||
"mentor": "Be helpful and explanatory. Use **bold** for key takeaways. Include practical advice.",
|
||||
"minimalist": "One paragraph. Short blunt declarative sentences. No markdown. No fluff.",
|
||||
}
|
||||
category_extras = {
|
||||
"devlog": "Write it as a personal devlog entry - share your learning process and insights about this topic.",
|
||||
"showcase": "Write it as a showcase - express genuine excitement about what makes this impressive.",
|
||||
"question": "Write it as a discussion starter - ask thoughtful questions, invite others to share perspectives.",
|
||||
"rant": "Write it as an opinionated rant - strong viewpoint, passionate criticism, but keep it substantive.",
|
||||
"fun": "Write it lighthearted and playful, but stay tied to the actual tech topic. Humor about the technology itself, not off-topic jokes or lyrics.",
|
||||
"random": "Be natural and conversational - share your thoughts like any casual discussion.",
|
||||
}
|
||||
persona_extra = f" {persona_extras.get(persona, 'Be casual. No markdown.')}" if persona else " Be casual. No markdown."
|
||||
category_extra = f" {category_extras.get(category, '')}" if category else ""
|
||||
preserve = persona in ("enthusiastic_junior", "hobbyist_maker", "academic_type", "storyteller", "mentor")
|
||||
text = self._call(
|
||||
f"You are a dev writing a social media post reacting to tech news. Write 2-4 short paragraphs."
|
||||
f"{persona_extra}{category_extra} Do not summarize the article; assume the reader already saw it. "
|
||||
f"Add your own value: a concrete opinion, an implication, a personal experience, or a pointed question. "
|
||||
f"Reference a specific detail rather than restating the headline. No em dashes. Under 300 words.",
|
||||
f"News: {title}\n\n{desc[:800]}",
|
||||
)
|
||||
return self.clean(text, preserve_md=preserve)
|
||||
|
||||
def select_category(self, title: str, desc: str) -> str:
|
||||
text = self._call(
|
||||
"You are a classifier. Given tech news, choose the single best post category. Reply with ONLY the category name.",
|
||||
f"Categories:\n- devlog: learning journey, building something, personal dev experience\n- showcase: impressive release, new tool, achievement worth highlighting\n- question: uncertainty, asking for advice, curiosity about implications\n- rant: frustration, bad practices, criticism of industry trends\n- fun: amusing, surprising, entertaining but not deeply serious\n- random: anything else that doesn't clearly fit above\n\nNews: {title}\n\n{desc[:600]}",
|
||||
temperature=0.2,
|
||||
)
|
||||
for c in CATEGORIES:
|
||||
if c in text.lower():
|
||||
return c
|
||||
return "random"
|
||||
|
||||
def generate_comment(self, post_snippet: str, persona: str = "",
|
||||
mention_target: str = "", parent_context: str = "") -> str:
|
||||
extras = {
|
||||
"enthusiastic_junior": "Be excited. Use **bold** for agreement. Short replies.",
|
||||
"grumpy_senior": "Be blunt and short. No markdown. One sarcastic remark or actual advice.",
|
||||
"rebel": "Super casual. Skip caps sometimes. Short.",
|
||||
"mentor": "Be helpful. Use **bold** for key point.",
|
||||
"storyteller": "Share a quick related story. Use *italics* for emphasis.",
|
||||
"minimalist": "Shortest possible reply. One sentence max.",
|
||||
}
|
||||
extra = f" {extras.get(persona, 'Be casual. No markdown.')}" if persona else " Be casual. No markdown."
|
||||
preserve = persona in ("enthusiastic_junior", "mentor", "storyteller")
|
||||
mention_rule = ""
|
||||
if mention_target:
|
||||
mention_rule = (
|
||||
f" Address @{mention_target} naturally in the first sentence "
|
||||
f"(weave the @{mention_target} mention inline, do not append it at the end)."
|
||||
)
|
||||
ctx = ""
|
||||
if parent_context:
|
||||
ctx = f"\n\nReplying to this comment:\n{parent_context[:800]}"
|
||||
system = (
|
||||
f"You are a dev replying to a post. Write 1-3 short sentences.{extra}{mention_rule} "
|
||||
"No em dashes. Reference a specific detail from the post. "
|
||||
"Do not open with or rely on generic filler like 'great point', 'I agree', 'nice', "
|
||||
"'thanks for sharing', or 'interesting'. Add something real: a concrete experience, "
|
||||
"a caveat, a counterexample, respectful disagreement, or a pointed question. "
|
||||
"Never just paraphrase or agree blandly."
|
||||
)
|
||||
reply = self.clean(
|
||||
self._call(system, f"Post:\n{post_snippet[:1500]}{ctx}").strip(),
|
||||
preserve_md=preserve,
|
||||
)
|
||||
return reply[:2000]
|
||||
|
||||
@staticmethod
|
||||
def _overlap_ratio(text: str, context: str) -> float:
|
||||
def tokens(s: str) -> list[str]:
|
||||
return [w for w in re.findall(r"[a-z0-9]+", s.lower()) if len(w) > 3]
|
||||
|
||||
candidate = tokens(text)
|
||||
source = set(tokens(context))
|
||||
if not candidate or not source:
|
||||
return 0.0
|
||||
hits = sum(1 for w in candidate if w in source)
|
||||
return hits / len(candidate)
|
||||
|
||||
def quality_check(self, kind: str, text: str, context: str = "") -> tuple[bool, str]:
|
||||
from devplacepy.services.bot.config import (
|
||||
MIN_COMMENT_LEN, MIN_POST_LEN, RESTATEMENT_OVERLAP_THRESHOLD, GENERIC_COMMENT_PHRASES,
|
||||
)
|
||||
stripped = self.clean(text or "").strip()
|
||||
min_len = MIN_COMMENT_LEN if kind == "comment" else MIN_POST_LEN
|
||||
if len(stripped) < min_len:
|
||||
return False, f"too short ({len(stripped)} < {min_len})"
|
||||
lowered = stripped.lower()
|
||||
if kind == "comment":
|
||||
for phrase in GENERIC_COMMENT_PHRASES:
|
||||
if phrase in lowered:
|
||||
return False, f"generic phrase '{phrase}'"
|
||||
if context and self._overlap_ratio(stripped, context) > RESTATEMENT_OVERLAP_THRESHOLD:
|
||||
return False, "restates the source"
|
||||
verdict = self._call(
|
||||
"You are a strict content quality reviewer for a developer community. "
|
||||
"Reject text that is generic, low-effort, pure filler, or merely summarizes its "
|
||||
"source without adding an opinion, experience, or insight. "
|
||||
"Reply with exactly PASS or 'FAIL: <short reason>'.",
|
||||
f"Type: {kind}\nText:\n{stripped[:1200]}",
|
||||
temperature=0.0,
|
||||
)
|
||||
if verdict.strip().upper().startswith("PASS"):
|
||||
return True, "ok"
|
||||
return False, verdict.strip()[:120] or "judge rejected"
|
||||
|
||||
def generate_bug(self, topic: str) -> tuple[str, str]:
|
||||
text = self._call(
|
||||
"Write a bug report. One line title. Then 2-3 sentences describing what happened and what should have happened. Like a real dev reporting a bug. No em dashes.",
|
||||
f"Bug: {topic}",
|
||||
)
|
||||
lines = text.strip().split("\n")
|
||||
title = lines[0].strip().lstrip("#").strip()[:120] or f"Bug report: {topic}"
|
||||
desc = " ".join(line.lstrip("-* ").strip() for line in lines[1:] if line.strip())
|
||||
return title, desc[:2000] or text[:500]
|
||||
|
||||
def generate_bio(self) -> str:
|
||||
return self._call(
|
||||
"Write a short bio for a developer profile. 1-2 sentences. What they work on, what they like. Sound human. No em dashes.",
|
||||
"Bio:",
|
||||
)
|
||||
|
||||
def generate_profile_fields(self, handle: str) -> tuple[str, str, str]:
|
||||
location = self.clean(self._call(
|
||||
"Name one plausible city and country for a developer. Reply with ONLY 'City, Country'. No extra words.",
|
||||
"Location:", temperature=0.9,
|
||||
))[:80]
|
||||
slug = re.sub(r"[^a-z0-9_-]", "", handle.lower()) or "dev"
|
||||
git_link = f"https://github.com/{slug}"
|
||||
website = f"https://{slug}.dev"
|
||||
return location, git_link, website
|
||||
|
||||
def generate_dm(self, persona: str = "", context: str = "") -> str:
|
||||
extra = {
|
||||
"enthusiastic_junior": "Be friendly and excited.",
|
||||
"grumpy_senior": "Be blunt but not rude.",
|
||||
"rebel": "Be super casual.",
|
||||
"mentor": "Be warm and encouraging.",
|
||||
"storyteller": "Open with a small hook.",
|
||||
"minimalist": "One short sentence.",
|
||||
}.get(persona, "Be casual and friendly.")
|
||||
ctx = f"\n\nEarlier message:\n{context[:400]}" if context else ""
|
||||
return self.clean(self._call(
|
||||
f"Write one short, friendly direct message to another developer. 1-2 sentences. {extra} No em dashes.",
|
||||
f"Write a DM to start or continue a chat.{ctx}",
|
||||
))[:500]
|
||||
|
||||
def generate_project_title(self) -> str:
|
||||
return self._call("Come up with a project name. 2-4 words. A tool, game, or app idea. No em dashes.", "Name:", temperature=0.9)
|
||||
|
||||
def generate_project_desc(self, title: str) -> str:
|
||||
return self._call(
|
||||
"You are a developer. Write a compelling 2-3 sentence project description including tech stack and purpose.",
|
||||
f"Project: {title}",
|
||||
)
|
||||
|
||||
def generate_gist(self, persona: str = "") -> tuple[str, str, str, str]:
|
||||
language = random.choice(GIST_LANGUAGES)
|
||||
title = self.clean(self._call(
|
||||
"Name a short, useful code snippet. 2-5 words. No quotes. No markdown.",
|
||||
f"Language: {language}. Snippet name:",
|
||||
temperature=0.9,
|
||||
))[:120]
|
||||
description = self.clean(self._call(
|
||||
"Write a one-sentence description of what a code snippet does, like a dev sharing something handy. No em dashes.",
|
||||
f"Snippet: {title}\nLanguage: {language}",
|
||||
))[:400]
|
||||
code = self.strip_code_fences(self._call(
|
||||
f"Write a short, correct, self-contained {language} snippet of 5 to 20 lines for the description. "
|
||||
"Output ONLY raw code. No markdown fences. No commentary.",
|
||||
f"Title: {title}\nDescription: {description}\nLanguage: {language}",
|
||||
temperature=0.4,
|
||||
))[:4000]
|
||||
return title, description, language, code
|
||||
@@ -0,0 +1,31 @@
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NewsFetcher:
|
||||
TTL_SECONDS = 1800
|
||||
|
||||
def __init__(self, news_api: str):
|
||||
self.news_api = news_api
|
||||
self.articles: list[dict] = []
|
||||
self.fetched_at: float = 0.0
|
||||
|
||||
def fetch(self) -> list[dict]:
|
||||
fresh = self.articles and (time.monotonic() - self.fetched_at) < self.TTL_SECONDS
|
||||
if fresh:
|
||||
return self.articles
|
||||
try:
|
||||
req = urllib.request.Request(self.news_api, headers={"User-Agent": "dpbot/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
articles = json.loads(resp.read()).get("articles", [])
|
||||
if articles:
|
||||
self.articles = articles
|
||||
self.fetched_at = time.monotonic()
|
||||
logger.info("Fetched %d news articles", len(self.articles))
|
||||
except Exception as e:
|
||||
logger.warning("News fetch failed: %s", e)
|
||||
return self.articles
|
||||
@@ -0,0 +1,94 @@
|
||||
import fcntl
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ArticleRegistry:
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
|
||||
def _lock(self) -> tuple:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
fd = os.open(str(self.path), os.O_RDWR | os.O_CREAT)
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
data = {}
|
||||
if os.path.getsize(str(self.path)) > 0:
|
||||
chunks = []
|
||||
while True:
|
||||
chunk = os.read(fd, 65536)
|
||||
if not chunk:
|
||||
break
|
||||
chunks.append(chunk)
|
||||
raw = b"".join(chunks).decode(errors="replace")
|
||||
if raw.strip():
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
data = {}
|
||||
if not isinstance(data, dict):
|
||||
data = {}
|
||||
now = datetime.now()
|
||||
kept: dict = {}
|
||||
for title, meta in data.items():
|
||||
if not isinstance(meta, dict):
|
||||
continue
|
||||
ts = meta.get("time", "")
|
||||
try:
|
||||
age_days = (now - datetime.fromisoformat(ts)).total_seconds() / 86400
|
||||
except (ValueError, TypeError):
|
||||
age_days = 0
|
||||
if age_days < 7:
|
||||
kept[title] = meta
|
||||
return fd, kept
|
||||
except Exception as e:
|
||||
logger.warning("Failed to lock article registry: %s", e)
|
||||
return None, {}
|
||||
|
||||
@staticmethod
|
||||
def _unlock(fd) -> None:
|
||||
if fd is not None:
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
os.close(fd)
|
||||
except Exception as e:
|
||||
logger.debug("Failed to unlock article registry: %s", e)
|
||||
|
||||
def reserve(self, title: str, owner: str) -> bool:
|
||||
fd, registry = self._lock()
|
||||
if fd is None:
|
||||
return True
|
||||
try:
|
||||
if title in registry:
|
||||
return False
|
||||
registry[title] = {"bot": owner, "time": datetime.now().isoformat()}
|
||||
self.path.write_text(json.dumps(registry, indent=2, default=str))
|
||||
return True
|
||||
finally:
|
||||
self._unlock(fd)
|
||||
|
||||
def reserve_unused(self, articles: list[dict], owner: str, clean) -> Optional[dict]:
|
||||
if not articles:
|
||||
return None
|
||||
fd, registry = self._lock()
|
||||
if fd is None:
|
||||
return random.choice(articles) if articles else None
|
||||
try:
|
||||
used = set(registry.keys())
|
||||
random.shuffle(articles)
|
||||
for a in articles:
|
||||
title = clean(a.get("title", "")[:200])
|
||||
if title and title not in used:
|
||||
registry[title] = {"bot": owner, "time": datetime.now().isoformat()}
|
||||
self.path.write_text(json.dumps(registry, indent=2, default=str))
|
||||
return a
|
||||
return None
|
||||
finally:
|
||||
self._unlock(fd)
|
||||
@@ -0,0 +1,18 @@
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.services.bot import config
|
||||
|
||||
|
||||
@dataclass
|
||||
class BotRuntimeConfig:
|
||||
state_path: Path
|
||||
base_url: str = config.BASE_URL_DEFAULT
|
||||
api_url: str = config.API_URL_DEFAULT
|
||||
news_api: str = config.NEWS_API_DEFAULT
|
||||
model: str = config.MODEL_DEFAULT
|
||||
api_key: str = ""
|
||||
input_cost_per_1m: float = config.INPUT_COST_PER_1M_DEFAULT
|
||||
output_cost_per_1m: float = config.OUTPUT_COST_PER_1M_DEFAULT
|
||||
headless: bool = True
|
||||
registry_path: Path = config.ARTICLE_REGISTRY_PATH
|
||||
@@ -0,0 +1,215 @@
|
||||
import asyncio
|
||||
import importlib
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from devplacepy.services.base import BaseService, ConfigField
|
||||
from devplacepy.services.bot import config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BotsService(BaseService):
|
||||
default_enabled = False
|
||||
min_interval = 5
|
||||
title = "Bots"
|
||||
description = (
|
||||
"Runs a fleet of Playwright-driven AI personas that browse a DevPlace "
|
||||
"instance and post, comment, vote, and react like real users. Fleet size, "
|
||||
"target site, and LLM settings are configurable, with live cost and usage "
|
||||
"metrics per bot."
|
||||
)
|
||||
config_fields = [
|
||||
ConfigField("bot_fleet_size", "Fleet size (bots)", type="int", default=1, minimum=0, maximum=20,
|
||||
help="Number of concurrent bot accounts to run.", group="Fleet"),
|
||||
ConfigField("bot_headless", "Headless browser", type="bool", default=True,
|
||||
help="Run Chromium headless. Disable only for local debugging on a desktop.",
|
||||
group="Fleet"),
|
||||
ConfigField("bot_max_actions", "Max actions per bot", type="int", default=0, minimum=0,
|
||||
help="Stop a bot after this many actions (0 = run indefinitely).", group="Fleet"),
|
||||
ConfigField("bot_base_url", "Target site URL", type="url", default=config.BASE_URL_DEFAULT,
|
||||
help="DevPlace instance the bots browse and post to.", group="Target"),
|
||||
ConfigField("bot_news_api", "News API URL", type="url", default=config.NEWS_API_DEFAULT,
|
||||
help="Source of articles the bots react to.", group="Target"),
|
||||
ConfigField("bot_api_url", "LLM API URL", type="url", default=config.API_URL_DEFAULT,
|
||||
help="OpenAI-compatible chat-completions endpoint.", group="LLM"),
|
||||
ConfigField("bot_model", "LLM model", type="str", default=config.MODEL_DEFAULT,
|
||||
help="Model name sent to the LLM API.", group="LLM"),
|
||||
ConfigField("bot_api_key", "LLM API key", type="password", default="", secret=True,
|
||||
help="Defaults to the gateway's internal key (the bots use the local AI gateway).",
|
||||
group="LLM"),
|
||||
ConfigField("bot_input_cost_per_1m", "Input cost per 1M tokens ($)", type="float",
|
||||
default=config.INPUT_COST_PER_1M_DEFAULT, minimum=0,
|
||||
help="Used to compute live spend per bot and across the fleet.", group="LLM"),
|
||||
ConfigField("bot_output_cost_per_1m", "Output cost per 1M tokens ($)", type="float",
|
||||
default=config.OUTPUT_COST_PER_1M_DEFAULT, minimum=0,
|
||||
help="Used to compute live spend per bot and across the fleet.", group="LLM"),
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="bots", interval_seconds=15)
|
||||
self._fleet = {}
|
||||
self._state_dir = config.STATE_DIR
|
||||
self._cost_baseline = None
|
||||
self._cost_baseline_at = None
|
||||
|
||||
def _resolve_api_key(self, cfg: dict) -> str:
|
||||
from devplacepy.database import internal_gateway_key
|
||||
return cfg["bot_api_key"] or internal_gateway_key()
|
||||
|
||||
@staticmethod
|
||||
def _ensure_importable() -> None:
|
||||
importlib.import_module("devplacepy.services.bot.bot")
|
||||
|
||||
async def run_once(self) -> None:
|
||||
cfg = self.get_config()
|
||||
desired = cfg["bot_fleet_size"]
|
||||
api_key = self._resolve_api_key(cfg)
|
||||
if not api_key:
|
||||
if self._fleet:
|
||||
await self._stop_fleet()
|
||||
self.log("No LLM API key available (bot_api_key / gateway internal key); fleet idle")
|
||||
return
|
||||
try:
|
||||
self._ensure_importable()
|
||||
except ImportError as e:
|
||||
if self._fleet:
|
||||
await self._stop_fleet()
|
||||
self.log(f"Bot dependencies unavailable ({e}); install the 'bots' extra")
|
||||
return
|
||||
|
||||
for slot in list(self._fleet):
|
||||
task = self._fleet[slot]["task"]
|
||||
if task.done():
|
||||
self._report_exit(slot, task)
|
||||
del self._fleet[slot]
|
||||
|
||||
while len(self._fleet) > desired:
|
||||
await self._stop_slot(max(self._fleet))
|
||||
|
||||
for slot in range(desired):
|
||||
if slot not in self._fleet:
|
||||
self._launch_slot(slot, cfg, api_key)
|
||||
|
||||
self.log(f"Fleet: {len(self._fleet)}/{desired} bots active")
|
||||
|
||||
def _report_exit(self, slot: int, task) -> None:
|
||||
if task.cancelled():
|
||||
return
|
||||
exc = task.exception()
|
||||
if exc:
|
||||
self.log(f"bot{slot} exited with error: {exc}; relaunching")
|
||||
else:
|
||||
self.log(f"bot{slot} finished; relaunching")
|
||||
|
||||
def _launch_slot(self, slot: int, cfg: dict, api_key: str) -> None:
|
||||
from devplacepy.services.bot.bot import DevPlaceBot
|
||||
from devplacepy.services.bot.runtime import BotRuntimeConfig
|
||||
rc = BotRuntimeConfig(
|
||||
state_path=self._state_dir / f"state_slot{slot}.json",
|
||||
base_url=cfg["bot_base_url"],
|
||||
api_url=cfg["bot_api_url"],
|
||||
news_api=cfg["bot_news_api"],
|
||||
model=cfg["bot_model"],
|
||||
api_key=api_key,
|
||||
input_cost_per_1m=cfg["bot_input_cost_per_1m"],
|
||||
output_cost_per_1m=cfg["bot_output_cost_per_1m"],
|
||||
headless=cfg["bot_headless"],
|
||||
registry_path=config.ARTICLE_REGISTRY_PATH,
|
||||
)
|
||||
bot = DevPlaceBot(rc, on_event=self.log)
|
||||
task = asyncio.create_task(self._run_slot(slot, bot, cfg["bot_max_actions"]))
|
||||
self._fleet[slot] = {"bot": bot, "task": task}
|
||||
self.log(f"Launched bot{slot}")
|
||||
|
||||
async def _run_slot(self, slot: int, bot, max_actions: int) -> None:
|
||||
try:
|
||||
await bot.run_forever(action_limit=max_actions)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.log(f"bot{slot} crashed: {e}")
|
||||
|
||||
async def on_disable(self) -> None:
|
||||
await self._stop_fleet()
|
||||
|
||||
async def _stop_fleet(self) -> None:
|
||||
for slot in list(self._fleet):
|
||||
await self._stop_slot(slot)
|
||||
|
||||
async def _stop_slot(self, slot: int) -> None:
|
||||
entry = self._fleet.pop(slot, None)
|
||||
if not entry:
|
||||
return
|
||||
task = entry["task"]
|
||||
task.cancel()
|
||||
try:
|
||||
await asyncio.wait_for(task, timeout=30)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError):
|
||||
pass
|
||||
except Exception as e:
|
||||
self.log(f"bot{slot} stop error: {e}")
|
||||
self.log(f"Stopped bot{slot}")
|
||||
|
||||
def collect_metrics(self) -> dict:
|
||||
rows = []
|
||||
total_cost = 0.0
|
||||
total_calls = total_in = total_out = 0
|
||||
posts = comments = votes = running = 0
|
||||
for slot in sorted(self._fleet):
|
||||
entry = self._fleet[slot]
|
||||
bot = entry["bot"]
|
||||
alive = not entry["task"].done()
|
||||
running += 1 if alive else 0
|
||||
llm = getattr(bot, "llm", None)
|
||||
st = getattr(bot, "state", None)
|
||||
cost = llm.total_cost if llm else 0.0
|
||||
calls = llm.total_calls if llm else 0
|
||||
total_cost += cost
|
||||
total_calls += calls
|
||||
total_in += llm.total_in_tokens if llm else 0
|
||||
total_out += llm.total_out_tokens if llm else 0
|
||||
if st:
|
||||
posts += st.created_posts
|
||||
comments += st.comments_posted
|
||||
votes += st.votes_cast
|
||||
rows.append([
|
||||
f"bot{slot}",
|
||||
(st.username if st else "") or "-",
|
||||
(st.persona if st else "") or "-",
|
||||
"running" if alive else "stopped",
|
||||
st.created_posts if st else 0,
|
||||
st.comments_posted if st else 0,
|
||||
st.votes_cast if st else 0,
|
||||
calls,
|
||||
f"${cost:.4f}",
|
||||
])
|
||||
now = datetime.now(timezone.utc)
|
||||
if self._started_at is None:
|
||||
self._cost_baseline = None
|
||||
self._cost_baseline_at = None
|
||||
elif self._cost_baseline is None or self._cost_baseline[0] != self._started_at:
|
||||
self._cost_baseline = (self._started_at, total_cost)
|
||||
self._cost_baseline_at = now
|
||||
window = (now - self._cost_baseline_at).total_seconds() if self._cost_baseline_at else 0.0
|
||||
observed_cost = max(0.0, total_cost - self._cost_baseline[1]) if self._cost_baseline else 0.0
|
||||
cost_per_hour = observed_cost / window * 3600 if window > 0 else 0.0
|
||||
projected_24h = observed_cost / window * 86400 if window > 0 else 0.0
|
||||
stats = [
|
||||
{"label": "Bots running", "value": running},
|
||||
{"label": "Fleet cost", "value": f"${total_cost:.4f}"},
|
||||
{"label": "Observed window", "value": f"{window:.0f}s"},
|
||||
{"label": "Cost rate", "value": f"${cost_per_hour:.4f}/h"},
|
||||
{"label": "Projected 24h cost", "value": f"${projected_24h:.2f}"},
|
||||
{"label": "LLM calls", "value": total_calls},
|
||||
{"label": "Tokens in", "value": total_in},
|
||||
{"label": "Tokens out", "value": total_out},
|
||||
{"label": "Posts", "value": posts},
|
||||
{"label": "Comments", "value": comments},
|
||||
{"label": "Votes", "value": votes},
|
||||
]
|
||||
table = {
|
||||
"columns": ["Bot", "User", "Persona", "Status", "Posts", "Comments", "Votes", "Calls", "Cost"],
|
||||
"rows": rows,
|
||||
}
|
||||
return {"stats": stats, "table": table}
|
||||
@@ -0,0 +1,65 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BotState:
|
||||
username: str = ""
|
||||
email: str = ""
|
||||
password: str = ""
|
||||
uid: str = ""
|
||||
persona: str = ""
|
||||
log: list[str] = field(default_factory=list)
|
||||
known_users: list[str] = field(default_factory=list)
|
||||
known_posts: list[str] = field(default_factory=list)
|
||||
created_posts: int = 0
|
||||
posted_content_hashes: list[str] = field(default_factory=list)
|
||||
comments_posted: int = 0
|
||||
quality_rejections: int = 0
|
||||
votes_cast: int = 0
|
||||
bugs_filed: int = 0
|
||||
projects_created: int = 0
|
||||
gists_created: int = 0
|
||||
profiles_viewed: int = 0
|
||||
started_at: str = ""
|
||||
own_post_urls: list[str] = field(default_factory=list)
|
||||
voted_post_ids: list[str] = field(default_factory=list)
|
||||
commented_post_ids: list[str] = field(default_factory=list)
|
||||
notifications_seen: list[str] = field(default_factory=list)
|
||||
mentions_received: int = 0
|
||||
mentions_replied: int = 0
|
||||
replies_to_own_posts: int = 0
|
||||
news_comments: int = 0
|
||||
projects_starred: int = 0
|
||||
messages_sent: int = 0
|
||||
searches_run: int = 0
|
||||
poll_votes_cast: int = 0
|
||||
reactions_cast: int = 0
|
||||
polls_voted: list[str] = field(default_factory=list)
|
||||
reacted_targets: list[str] = field(default_factory=list)
|
||||
reading_speed_wpm: int = 0
|
||||
total_cost: float = 0.0
|
||||
total_calls: int = 0
|
||||
total_in_tokens: int = 0
|
||||
total_out_tokens: int = 0
|
||||
|
||||
def save(self, path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(asdict(self), indent=2, default=str))
|
||||
os.replace(str(tmp), str(path))
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> "BotState":
|
||||
if path.exists():
|
||||
try:
|
||||
data = json.loads(path.read_text())
|
||||
return cls(**data)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load state: %s", e)
|
||||
return cls()
|
||||
Reference in New Issue
Block a user