retoor/devplacepy
1
0
Fork 1
Code Issues 1 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

feat: add TTLCache for get_cache_version and TEMPLATE_AUTO_RELOAD config with Makefile worker count variables

Browse Source
This commit is contained in:
retoor
2026-06-14 14:46:36 +00:00
parent c151325916
commit d75d5dc6a8
149 changed files with 9811 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devplacepy/services/bot/bot.py
+13 -1
View File
@@ -46,11 +46,12 @@ logger = logging.getLogger(__name__)
class DevPlaceBot:
def __init__(self, cfg: BotRuntimeConfig, on_event=None):
def __init__(self, cfg: BotRuntimeConfig, on_event=None, slot: int = 0):
self.cfg = cfg
self.base_url = cfg.base_url
self.state_path = cfg.state_path
self.on_event = on_event
self._slot = slot
self.state = BotState.load(cfg.state_path)
if not self.state.persona:
self.state.persona = random.choice(PERSONAS)
@@ -139,6 +140,15 @@ class DevPlaceBot:
self.state.log = self.state.log[-250:]
logger.info("[%s] %s (%s)", self._identity(), line, self._cost_tag())
self._notify(f"[{self._identity()}] {line}")
self.b.note(action=line)
asyncio.create_task(self.b.capture(tag.lower(), force=True))
def _bind_monitor(self) -> None:
self.b.bind_monitor(
self._slot,
username=self.state.username,
persona=self.state.persona,
)
async def _generate(self, fn: Any, *args: Any) -> Any:
try:
@@ -2773,6 +2783,8 @@ class DevPlaceBot:
await asyncio.sleep(60)
continue
self._bind_monitor()
if self._ai_decisions:
await self._ensure_identity()
use_ai = self._ai_decisions and bool(self.state.identity)
devplacepy/services/bot/browser.py
+52
View File
@@ -3,10 +3,18 @@
import asyncio
import logging
import random
import time
from typing import Optional
from playwright.async_api import Page, Playwright, async_playwright
from devplacepy.services.bot.config import (
MONITOR_JPEG_QUALITY,
MONITOR_MIN_INTERVAL_SECONDS,
MONITOR_SCALE,
)
from devplacepy.services.bot.monitor import monitor
logger = logging.getLogger(__name__)
DEFAULT_USER_AGENT = (
@@ -30,6 +38,8 @@ class BotBrowser:
self._browser = None
self._context = None
self._page: Optional[Page] = None
self._monitor_slot: Optional[int] = None
self._last_capture: float = 0.0
async def launch(self) -> None:
self._playwright = await async_playwright().start()
@@ -70,6 +80,44 @@ class BotBrowser:
def page(self) -> Page:
return self._page
def bind_monitor(
self, slot: int, *, username: str = "", persona: str = ""
) -> None:
self._monitor_slot = slot
monitor.update_meta(slot, username=username, persona=persona)
def note(self, *, action: str = "", status: str = "") -> None:
if self._monitor_slot is None:
return
monitor.update_meta(self._monitor_slot, action=action, status=status)
async def capture(self, reason: str = "", *, force: bool = False) -> None:
if self._monitor_slot is None or self._page is None:
return
now = time.time()
if not force and now - self._last_capture < MONITOR_MIN_INTERVAL_SECONDS:
return
self._last_capture = now
try:
current_url = ""
try:
current_url = self._page.url
except Exception:
current_url = ""
monitor.update_meta(
self._monitor_slot, status=reason, url=current_url
)
image = await self._page.screenshot(
type="jpeg",
quality=MONITOR_JPEG_QUALITY,
scale=MONITOR_SCALE,
full_page=False,
timeout=4000,
)
monitor.store_image(self._monitor_slot, image)
except Exception as e:
logger.debug("monitor capture (%s) failed: %s", reason, e)
async def goto(self, url: str) -> None:
try:
await self._page.goto(url, wait_until="domcontentloaded", timeout=15000)
@@ -80,6 +128,7 @@ class BotBrowser:
except Exception as e2:
logger.warning("goto failed for %s: %s", url[:80], e2)
await self._idle(0.5, 1.5)
await self.capture("page load", force=True)
async def _idle(self, lo: float = 0.5, hi: float = 2.0) -> None:
await asyncio.sleep(random.uniform(lo, hi))
@@ -92,6 +141,7 @@ class BotBrowser:
except Exception as e:
logger.debug("scroll failed: %s", e)
await self._idle(0.2, 0.6)
await self.capture("scroll")
async def scroll_to_bottom(self) -> None:
try:
@@ -112,6 +162,7 @@ class BotBrowser:
await self._page.keyboard.type(ch, delay=random.randint(20, 60))
if ch == " ":
await asyncio.sleep(random.uniform(0.02, 0.08))
await self.capture("field input")
return True
except Exception as e:
logger.debug("fill failed for '%s': %s", sel, e)
@@ -175,5 +226,6 @@ class BotBrowser:
try:
await self._page.reload(timeout=15000)
await self._idle()
await self.capture("reload", force=True)
except Exception as e:
logger.debug("reload failed: %s", e)
devplacepy/services/bot/config.py
+4
View File
@@ -14,6 +14,10 @@ OUTPUT_COST_PER_1M_DEFAULT = 1.10
COST_WINDOW_SECONDS = 600
COST_WARMUP_SECONDS = 120
MONITOR_JPEG_QUALITY = 35
MONITOR_SCALE = "css"
MONITOR_MIN_INTERVAL_SECONDS = 1.0
STATE_DIR = BOT_DIR
ARTICLE_REGISTRY_PATH = BOT_DIR / "article_registry.json"
devplacepy/services/bot/monitor.py
+122
View File
@@ -0,0 +1,122 @@
# retoor <retoor@molodetz.nl>
import logging
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional
from devplacepy.config import BOT_DIR
logger = logging.getLogger(__name__)
MONITOR_DIR: Path = BOT_DIR / "monitor"
FRAME_SUFFIX = ".jpg"
ACTIVE_WINDOW_SECONDS = 90
@dataclass
class BotFrame:
slot: int
username: str = ""
persona: str = ""
action: str = ""
status: str = ""
url: str = ""
captured_at: float = 0.0
has_image: bool = False
def label(self) -> str:
return self.username or f"bot{self.slot}"
def age_seconds(self) -> float:
return max(0.0, time.time() - self.captured_at) if self.captured_at else 0.0
def is_active(self) -> bool:
return self.has_image and self.age_seconds() <= ACTIVE_WINDOW_SECONDS
def as_dict(self) -> dict:
return {
"slot": self.slot,
"username": self.username,
"persona": self.persona,
"action": self.action,
"status": self.status,
"url": self.url,
"label": self.label(),
"captured_at": int(self.captured_at) if self.captured_at else 0,
"age_seconds": int(self.age_seconds()),
"has_image": self.has_image,
"active": self.is_active(),
}
@dataclass
class BotMonitor:
frames: dict[int, BotFrame] = field(default_factory=dict)
def frame_path(self, slot: int) -> Path:
return MONITOR_DIR / f"slot{slot}{FRAME_SUFFIX}"
def update_meta(
self,
slot: int,
*,
username: str = "",
persona: str = "",
action: str = "",
status: str = "",
url: str = "",
) -> None:
frame = self.frames.get(slot) or BotFrame(slot=slot)
if username:
frame.username = username
if persona:
frame.persona = persona
if action:
frame.action = action
if status:
frame.status = status
if url:
frame.url = url
self.frames[slot] = frame
def store_image(self, slot: int, image: bytes) -> None:
if not image:
return
frame = self.frames.get(slot) or BotFrame(slot=slot)
try:
MONITOR_DIR.mkdir(parents=True, exist_ok=True)
path = self.frame_path(slot)
tmp = path.with_suffix(path.suffix + ".tmp")
tmp.write_bytes(image)
tmp.replace(path)
except OSError as e:
logger.debug("monitor store_image slot %s failed: %s", slot, e)
return
frame.has_image = True
frame.captured_at = time.time()
self.frames[slot] = frame
def drop(self, slot: int) -> None:
self.frames.pop(slot, None)
try:
self.frame_path(slot).unlink(missing_ok=True)
except OSError as e:
logger.debug("monitor drop slot %s failed: %s", slot, e)
def read_image(self, slot: int) -> Optional[bytes]:
path = self.frame_path(slot)
if not path.exists():
return None
try:
return path.read_bytes()
except OSError as e:
logger.debug("monitor read_image slot %s failed: %s", slot, e)
return None
def snapshot(self) -> list[dict]:
return [self.frames[slot].as_dict() for slot in sorted(self.frames)]
monitor = BotMonitor()
devplacepy/services/bot/service.py
+4 -2
View File
@@ -8,6 +8,7 @@ from datetime import datetime, timedelta, timezone
from devplacepy.services.base import BaseService, ConfigField
from devplacepy.services.bot import config
from devplacepy.services.bot.monitor import monitor
logger = logging.getLogger(__name__)
@@ -283,7 +284,7 @@ class BotsService(BaseService):
ai_decisions=cfg["bot_ai_decisions"],
decision_temperature=cfg["bot_decision_temperature"],
)
bot = DevPlaceBot(rc, on_event=self.log)
bot = DevPlaceBot(rc, on_event=self.log, slot=slot)
task = asyncio.create_task(self._run_slot(slot, bot, cfg["bot_max_actions"]))
self._fleet[slot] = {"bot": bot, "task": task}
self.log(f"Launched bot{slot}")
@@ -319,6 +320,7 @@ class BotsService(BaseService):
except Exception as e:
self.log(f"bot{slot} stop error: {e}")
await self._close_bot(slot, entry.get("bot"))
monitor.drop(slot)
self.log(f"Stopped bot{slot}")
async def _close_bot(self, slot: int, bot) -> None:
@@ -401,7 +403,7 @@ class BotsService(BaseService):
],
"rows": rows,
}
return {"stats": stats, "table": table}
return {"stats": stats, "table": table, "frames": monitor.snapshot()}
def _sample_cost(
self, now: datetime, total_cost: float