feat: add api key auth, devii agent, openai gateway, and admin service management
This commit introduces a comprehensive set of new features including API key authentication with CLI management commands (get, reset, backfill), a Devii agentic assistant with WebSocket terminal and session bootstrap, an OpenAI-compatible LLM gateway service, and an admin service management panel. It also adds Playwright browser automation for bot support, configures internal gateway URLs, refactors content editing/deletion to support JSON API responses, and updates documentation across AGENTS.md, README.md, and the developer docs site.
This commit is contained in:
+354
-56
@@ -1,93 +1,391 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import deque
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from devplacepy.database import db, get_table, get_setting, get_int_setting
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ConfigField:
|
||||
def __init__(self, key, label, type="str", default="", help="",
|
||||
options=None, secret=False, minimum=None, maximum=None, group="General"):
|
||||
self.key = key
|
||||
self.label = label
|
||||
self.type = type
|
||||
self.default = default
|
||||
self.help = help
|
||||
self.options = options or []
|
||||
self.secret = secret
|
||||
self.minimum = minimum
|
||||
self.maximum = maximum
|
||||
self.group = group
|
||||
|
||||
def coerce(self, raw):
|
||||
if self.type == "int":
|
||||
try:
|
||||
value = int(str(raw).strip())
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError(f"{self.label} must be a whole number")
|
||||
if self.minimum is not None and value < self.minimum:
|
||||
raise ValueError(f"{self.label} must be at least {self.minimum}")
|
||||
if self.maximum is not None and value > self.maximum:
|
||||
raise ValueError(f"{self.label} must be at most {self.maximum}")
|
||||
return value
|
||||
if self.type == "float":
|
||||
try:
|
||||
value = float(str(raw).strip())
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError(f"{self.label} must be a number")
|
||||
if self.minimum is not None and value < self.minimum:
|
||||
raise ValueError(f"{self.label} must be at least {self.minimum}")
|
||||
if self.maximum is not None and value > self.maximum:
|
||||
raise ValueError(f"{self.label} must be at most {self.maximum}")
|
||||
return value
|
||||
if self.type == "bool":
|
||||
text = str(raw).strip()
|
||||
if text in ("0", "1"):
|
||||
return text == "1"
|
||||
raise ValueError(f"{self.label} must be enabled or disabled")
|
||||
if self.type == "select":
|
||||
allowed = [option["value"] for option in self.options]
|
||||
if raw not in allowed:
|
||||
raise ValueError(f"{self.label} has an invalid selection")
|
||||
return raw
|
||||
if self.type == "url":
|
||||
text = str(raw).strip()
|
||||
if text and not (text.startswith("http://") or text.startswith("https://")):
|
||||
raise ValueError(f"{self.label} must be a http(s) URL")
|
||||
return text
|
||||
return str(raw)
|
||||
|
||||
def to_storage(self, value):
|
||||
if isinstance(value, bool):
|
||||
return "1" if value else "0"
|
||||
return str(value)
|
||||
|
||||
def read(self):
|
||||
raw = get_setting(self.key, "")
|
||||
if raw == "":
|
||||
return self.default
|
||||
try:
|
||||
return self.coerce(raw)
|
||||
except ValueError:
|
||||
return self.default
|
||||
|
||||
def display_value(self):
|
||||
if self.secret:
|
||||
return ""
|
||||
raw = get_setting(self.key, "")
|
||||
if raw == "":
|
||||
return self.to_storage(self.default)
|
||||
return raw
|
||||
|
||||
def spec(self):
|
||||
return {
|
||||
"key": self.key,
|
||||
"label": self.label,
|
||||
"type": self.type,
|
||||
"help": self.help,
|
||||
"options": self.options,
|
||||
"secret": self.secret,
|
||||
"minimum": self.minimum,
|
||||
"maximum": self.maximum,
|
||||
"value": self.display_value(),
|
||||
"is_set": bool(get_setting(self.key, "")) if self.secret else None,
|
||||
"group": self.group,
|
||||
}
|
||||
|
||||
|
||||
class BaseService(ABC):
|
||||
config_fields = []
|
||||
interval_key = None
|
||||
min_interval = 1
|
||||
default_enabled = True
|
||||
title = ""
|
||||
description = ""
|
||||
DEFAULT_LOG_SIZE = 20
|
||||
TICK_SECONDS = 1
|
||||
PERSIST_SECONDS = 3
|
||||
STALE_SECONDS = 15
|
||||
|
||||
def __init__(self, name: str, interval_seconds: int = 3600):
|
||||
self.name = name
|
||||
self.interval_seconds = interval_seconds
|
||||
self.log_buffer = deque(maxlen=20)
|
||||
self.interval_key = self.interval_key or f"service_{name}_interval"
|
||||
self.enabled_key = f"service_{name}_enabled"
|
||||
self.command_key = f"service_{name}_command"
|
||||
self.log_size_key = f"service_{name}_log_size"
|
||||
self.log_buffer = deque(maxlen=self.DEFAULT_LOG_SIZE)
|
||||
self._task = None
|
||||
self._running = False
|
||||
self._shutdown = False
|
||||
self._started_at = None
|
||||
self._last_run = None
|
||||
self._next_run = None
|
||||
self._started_at = None
|
||||
self._next_due = None
|
||||
self._run_pending = False
|
||||
self._last_command = None
|
||||
self._last_persist = None
|
||||
self.enabled_field = ConfigField(
|
||||
self.enabled_key, "Enabled", type="bool", default=self.default_enabled,
|
||||
help="When enabled the service runs on its interval and starts on boot.",
|
||||
group="General")
|
||||
self.interval_field = ConfigField(
|
||||
self.interval_key, "Run interval (seconds)", type="int",
|
||||
default=interval_seconds, minimum=self.min_interval,
|
||||
help=f"Delay between runs. Minimum {self.min_interval} seconds.",
|
||||
group="General")
|
||||
self.log_size_field = ConfigField(
|
||||
self.log_size_key, "Log buffer size", type="int",
|
||||
default=self.DEFAULT_LOG_SIZE, minimum=1, maximum=200,
|
||||
help="Number of recent log lines to keep.",
|
||||
group="Advanced")
|
||||
|
||||
def all_fields(self) -> list:
|
||||
return [self.enabled_field, self.interval_field, *self.config_fields, self.log_size_field]
|
||||
|
||||
def _grouped_fields(self) -> list:
|
||||
groups: dict = {}
|
||||
order: list = []
|
||||
for field in self.all_fields():
|
||||
spec = field.spec()
|
||||
group = spec["group"]
|
||||
if group not in groups:
|
||||
groups[group] = []
|
||||
order.append(group)
|
||||
groups[group].append(spec)
|
||||
return [{"name": name, "fields": groups[name]} for name in order]
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
return "running" if self._running else "stopped"
|
||||
|
||||
@property
|
||||
def last_run(self) -> str | None:
|
||||
return self._last_run
|
||||
def is_enabled(self) -> bool:
|
||||
return get_setting(self.enabled_key, "1" if self.default_enabled else "0") == "1"
|
||||
|
||||
@property
|
||||
def next_run(self) -> str | None:
|
||||
return self._next_run
|
||||
def current_interval(self) -> int:
|
||||
return max(self.min_interval, get_int_setting(self.interval_key, self.interval_seconds))
|
||||
|
||||
@property
|
||||
def uptime(self) -> str | None:
|
||||
if self._started_at is None:
|
||||
return None
|
||||
delta = datetime.now(timezone.utc) - self._started_at
|
||||
return str(delta).split(".")[0]
|
||||
def get_config(self) -> dict:
|
||||
return {field.key: field.read() for field in self.all_fields()}
|
||||
|
||||
def log(self, message: str) -> None:
|
||||
stamp = datetime.now(timezone.utc).strftime("%H:%M:%S")
|
||||
entry = f"[{stamp}] {message}"
|
||||
self.log_buffer.append(entry)
|
||||
self.log_buffer.append(f"[{stamp}] {message}")
|
||||
logger.info(f"[{self.name}] {message}")
|
||||
|
||||
@abstractmethod
|
||||
async def run_once(self) -> None:
|
||||
pass
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
self._running = True
|
||||
self._started_at = datetime.now(timezone.utc)
|
||||
self.log(f"Service started (interval={self.interval_seconds}s)")
|
||||
while self._running:
|
||||
try:
|
||||
self._last_run = datetime.now(timezone.utc).isoformat()
|
||||
self._next_run = None
|
||||
await self.run_once()
|
||||
except asyncio.CancelledError:
|
||||
self.log("Service cancelled")
|
||||
break
|
||||
except Exception as e:
|
||||
self.log(f"Error in run_once: {e}")
|
||||
if not self._running:
|
||||
break
|
||||
self._next_run = datetime.now(timezone.utc).isoformat()
|
||||
self.log(f"Sleeping for {self.interval_seconds}s")
|
||||
try:
|
||||
await asyncio.sleep(self.interval_seconds)
|
||||
except asyncio.CancelledError:
|
||||
self.log("Service cancelled during sleep")
|
||||
break
|
||||
self._running = False
|
||||
self.log("Service stopped")
|
||||
async def on_enable(self) -> None:
|
||||
pass
|
||||
|
||||
async def start(self) -> None:
|
||||
if self._running:
|
||||
self.log("Already running")
|
||||
async def on_disable(self) -> None:
|
||||
pass
|
||||
|
||||
def collect_metrics(self) -> dict:
|
||||
return {}
|
||||
|
||||
def _safe_metrics(self) -> dict:
|
||||
try:
|
||||
return self.collect_metrics()
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not collect metrics for {self.name}: {e}")
|
||||
return {}
|
||||
|
||||
def start_supervisor(self) -> None:
|
||||
if self._task is not None:
|
||||
return
|
||||
self._running = True
|
||||
self._shutdown = False
|
||||
self._task = asyncio.create_task(self._run_loop())
|
||||
|
||||
async def stop(self) -> None:
|
||||
if not self._running:
|
||||
return
|
||||
self._running = False
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
def request_shutdown(self) -> None:
|
||||
self._shutdown = True
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
self._last_command = get_setting(self.command_key, "")
|
||||
self.log("Supervisor started")
|
||||
self._persist_state(force=True)
|
||||
while not self._shutdown:
|
||||
try:
|
||||
await asyncio.wait_for(self._task, timeout=10.0)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError):
|
||||
pass
|
||||
self._task = None
|
||||
await self._tick()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
self.log(f"Loop error: {e}")
|
||||
try:
|
||||
await asyncio.sleep(self.TICK_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
if self._started_at is not None:
|
||||
await self._safe_disable()
|
||||
self._running = False
|
||||
self._started_at = None
|
||||
self._persist_state(force=True)
|
||||
self.log("Supervisor stopped")
|
||||
|
||||
async def _tick(self) -> None:
|
||||
self._sync_log_size()
|
||||
self._handle_commands()
|
||||
if self.is_enabled():
|
||||
if self._started_at is None:
|
||||
self._started_at = datetime.now(timezone.utc)
|
||||
self._running = True
|
||||
self.log("Service enabled")
|
||||
await self.on_enable()
|
||||
due = self._next_due is None or datetime.now(timezone.utc) >= self._next_due
|
||||
if self._run_pending or due:
|
||||
self._run_pending = False
|
||||
await self._execute_run()
|
||||
elif self._started_at is not None:
|
||||
await self._safe_disable()
|
||||
self._started_at = None
|
||||
self._running = False
|
||||
self._next_due = None
|
||||
self._next_run = None
|
||||
self.log("Service disabled")
|
||||
self._persist_state()
|
||||
|
||||
async def _safe_disable(self) -> None:
|
||||
try:
|
||||
await self.on_disable()
|
||||
except Exception as e:
|
||||
self.log(f"on_disable error: {e}")
|
||||
|
||||
async def _execute_run(self) -> None:
|
||||
self.interval_seconds = self.current_interval()
|
||||
self._last_run = datetime.now(timezone.utc).isoformat()
|
||||
self._next_run = None
|
||||
self._persist_state(force=True)
|
||||
try:
|
||||
await self.run_once()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.log(f"Error in run_once: {e}")
|
||||
self.interval_seconds = self.current_interval()
|
||||
self._next_due = datetime.now(timezone.utc) + timedelta(seconds=self.interval_seconds)
|
||||
self._next_run = self._next_due.isoformat()
|
||||
self.log(f"Next run in {self.interval_seconds}s")
|
||||
self._persist_state(force=True)
|
||||
|
||||
def _handle_commands(self) -> None:
|
||||
raw = get_setting(self.command_key, "")
|
||||
if not raw or raw == self._last_command:
|
||||
return
|
||||
self._last_command = raw
|
||||
verb = raw.split(":", 1)[0]
|
||||
if verb == "run":
|
||||
self._run_pending = True
|
||||
self.log("Run requested")
|
||||
elif verb == "clear":
|
||||
self.log_buffer.clear()
|
||||
self.log("Logs cleared")
|
||||
|
||||
def _sync_log_size(self) -> None:
|
||||
size = max(1, get_int_setting(self.log_size_key, self.DEFAULT_LOG_SIZE))
|
||||
if size != self.log_buffer.maxlen:
|
||||
self.log_buffer = deque(self.log_buffer, maxlen=size)
|
||||
|
||||
def _persist_state(self, force: bool = False) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
if not force and self._last_persist is not None:
|
||||
if (now - self._last_persist).total_seconds() < self.PERSIST_SECONDS:
|
||||
return
|
||||
self._last_persist = now
|
||||
record = {
|
||||
"name": self.name,
|
||||
"status": self.status,
|
||||
"last_run": self._last_run or "",
|
||||
"next_run": self._next_run or "",
|
||||
"started_at": self._started_at.isoformat() if self._started_at else "",
|
||||
"heartbeat": now.isoformat(),
|
||||
"logs": json.dumps(list(self.log_buffer)),
|
||||
"metrics": json.dumps(self._safe_metrics()),
|
||||
"updated_at": now.isoformat(),
|
||||
}
|
||||
try:
|
||||
table = get_table("service_state")
|
||||
existing = table.find_one(name=self.name)
|
||||
if existing:
|
||||
table.update({**record, "id": existing["id"]}, ["id"])
|
||||
else:
|
||||
table.insert(record)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not persist state for {self.name}: {e}")
|
||||
|
||||
def _read_state(self) -> dict:
|
||||
if "service_state" not in db.tables:
|
||||
return {}
|
||||
row = db["service_state"].find_one(name=self.name)
|
||||
return row or {}
|
||||
|
||||
def _display_status(self, enabled: bool, state: dict) -> str:
|
||||
if not enabled:
|
||||
return "stopped"
|
||||
heartbeat = state.get("heartbeat")
|
||||
if not heartbeat:
|
||||
return "stalled"
|
||||
try:
|
||||
beat = datetime.fromisoformat(heartbeat)
|
||||
except ValueError:
|
||||
return "stalled"
|
||||
if (datetime.now(timezone.utc) - beat).total_seconds() > self.STALE_SECONDS:
|
||||
return "stalled"
|
||||
return "running"
|
||||
|
||||
def _uptime(self, state: dict, status: str) -> str | None:
|
||||
if status != "running":
|
||||
return None
|
||||
started = state.get("started_at")
|
||||
if not started:
|
||||
return None
|
||||
try:
|
||||
since = datetime.fromisoformat(started)
|
||||
except ValueError:
|
||||
return None
|
||||
return str(datetime.now(timezone.utc) - since).split(".")[0]
|
||||
|
||||
def _logs(self, state: dict) -> list:
|
||||
raw = state.get("logs")
|
||||
if not raw:
|
||||
return []
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except (ValueError, TypeError):
|
||||
return []
|
||||
|
||||
def _metrics(self, state: dict) -> dict:
|
||||
raw = state.get("metrics")
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except (ValueError, TypeError):
|
||||
return {}
|
||||
|
||||
def describe(self) -> dict:
|
||||
state = self._read_state()
|
||||
enabled = self.is_enabled()
|
||||
status = self._display_status(enabled, state)
|
||||
return {
|
||||
"name": self.name,
|
||||
"title": self.title or self.name.capitalize(),
|
||||
"description": self.description,
|
||||
"enabled": enabled,
|
||||
"status": status,
|
||||
"interval_seconds": self.current_interval(),
|
||||
"last_run": state.get("last_run") or None,
|
||||
"next_run": state.get("next_run") or None,
|
||||
"heartbeat": state.get("heartbeat") or None,
|
||||
"uptime": self._uptime(state, status),
|
||||
"log_buffer": self._logs(state),
|
||||
"metrics": self._metrics(state),
|
||||
"fields": [field.spec() for field in self.all_fields()],
|
||||
"field_groups": self._grouped_fields(),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user