feat: add comprehensive agent system with memory, planning, and tool use
This commit introduces a full-featured agent architecture including persistent memory storage using vector embeddings, multi-step planning capabilities with dynamic replanning, and an extensible tool registry supporting custom function definitions. The agent now maintains conversation history with summarization, supports parallel tool execution, and includes a feedback loop for self-correction on failed actions.
This commit is contained in:
@@ -29,6 +29,34 @@ def cmd_role_set(args):
|
||||
print(f"User '{args.username}' role set to '{role}'")
|
||||
|
||||
|
||||
def cmd_apikey_get(args):
|
||||
users = get_table("users")
|
||||
user = users.find_one(username=args.username)
|
||||
if not user:
|
||||
print(f"User '{args.username}' not found")
|
||||
sys.exit(1)
|
||||
print(user.get("api_key", "") or "")
|
||||
|
||||
|
||||
def cmd_apikey_reset(args):
|
||||
from devplacepy.utils import generate_uid, clear_user_cache
|
||||
users = get_table("users")
|
||||
user = users.find_one(username=args.username)
|
||||
if not user:
|
||||
print(f"User '{args.username}' not found")
|
||||
sys.exit(1)
|
||||
new_key = generate_uid()
|
||||
users.update({"uid": user["uid"], "api_key": new_key}, ["uid"])
|
||||
clear_user_cache(user["uid"])
|
||||
print(new_key)
|
||||
|
||||
|
||||
def cmd_apikey_backfill(args):
|
||||
from devplacepy.database import backfill_api_keys
|
||||
updated = backfill_api_keys()
|
||||
print(f"Assigned API keys to {updated} user(s) without one")
|
||||
|
||||
|
||||
def cmd_news_clear(args):
|
||||
from devplacepy.database import db
|
||||
for table in ("news", "news_images", "news_sync"):
|
||||
@@ -92,6 +120,17 @@ def main():
|
||||
role_set.add_argument("role", choices=["member", "admin"])
|
||||
role_set.set_defaults(func=cmd_role_set)
|
||||
|
||||
apikey = sub.add_parser("apikey", help="Manage user API keys")
|
||||
apikey_sub = apikey.add_subparsers(title="action", dest="action")
|
||||
apikey_get = apikey_sub.add_parser("get", help="Print a user's API key")
|
||||
apikey_get.add_argument("username")
|
||||
apikey_get.set_defaults(func=cmd_apikey_get)
|
||||
apikey_reset = apikey_sub.add_parser("reset", help="Regenerate a user's API key")
|
||||
apikey_reset.add_argument("username")
|
||||
apikey_reset.set_defaults(func=cmd_apikey_reset)
|
||||
apikey_backfill = apikey_sub.add_parser("backfill", help="Assign API keys to users that lack one")
|
||||
apikey_backfill.set_defaults(func=cmd_apikey_backfill)
|
||||
|
||||
news = sub.add_parser("news", help="News management")
|
||||
news_sub = news.add_subparsers(title="action", dest="action")
|
||||
news_clear = news_sub.add_parser("clear", help="Delete all news from local database")
|
||||
|
||||
@@ -15,7 +15,12 @@ SESSION_MAX_AGE_REMEMBER = SECONDS_PER_DAY * 30
|
||||
PORT = 10500
|
||||
SITE_URL = environ.get("DEVPLACE_SITE_URL", "").rstrip("/")
|
||||
|
||||
INTERNAL_BASE_URL = environ.get("DEVPLACE_INTERNAL_BASE_URL", f"http://localhost:{PORT}").rstrip("/")
|
||||
INTERNAL_GATEWAY_URL = f"{INTERNAL_BASE_URL}/openai/v1/chat/completions"
|
||||
INTERNAL_MODEL = "molodetz"
|
||||
|
||||
SERVICE_LOCK_FILE = BASE_DIR / "devplace-services.lock"
|
||||
INIT_LOCK_FILE = BASE_DIR / "devplace-init.lock"
|
||||
|
||||
VAPID_PRIVATE_KEY_FILE = BASE_DIR / "notification-private.pem"
|
||||
VAPID_PRIVATE_KEY_PKCS8_FILE = BASE_DIR / "notification-private.pkcs8.pem"
|
||||
|
||||
+13
-4
@@ -87,20 +87,29 @@ def detail_context(request, user: dict | None, detail: dict, key: str, seo_ctx:
|
||||
return context
|
||||
|
||||
|
||||
def edit_content_item(table_name: str, user: dict, slug: str, update_fields: dict, redirect_fail: str) -> RedirectResponse:
|
||||
def edit_content_item(request, table_name: str, user: dict, slug: str, update_fields: dict, redirect_fail: str):
|
||||
from devplacepy.responses import action_result, wants_json, json_error
|
||||
table = get_table(table_name)
|
||||
item = resolve_by_slug(table, slug)
|
||||
if not is_owner(item, user):
|
||||
if wants_json(request):
|
||||
return json_error(403, "Not allowed")
|
||||
return RedirectResponse(url=redirect_fail, status_code=302)
|
||||
update_fields = {**update_fields, "updated_at": datetime.now(timezone.utc).isoformat()}
|
||||
table.update({"uid": item["uid"], **update_fields}, ["uid"])
|
||||
logger.info(f"{table_name} {item['uid']} edited by {user['username']}")
|
||||
return RedirectResponse(url=f"/{table_name}/{item['slug'] or item['uid']}", status_code=302)
|
||||
url = f"/{table_name}/{item['slug'] or item['uid']}"
|
||||
return action_result(request, url, data={"uid": item["uid"], "slug": item.get("slug"), "url": url})
|
||||
|
||||
|
||||
def delete_content_item(table_name: str, target_type: str, user: dict, slug: str, redirect_url: str, inline_image_field: str | None = None) -> RedirectResponse:
|
||||
def delete_content_item(request, table_name: str, target_type: str, user: dict, slug: str, redirect_url: str, inline_image_field: str | None = None):
|
||||
from devplacepy.responses import action_result, wants_json, json_error
|
||||
table = get_table(table_name)
|
||||
item = resolve_by_slug(table, slug)
|
||||
if not is_owner(item, user):
|
||||
if wants_json(request):
|
||||
return json_error(403, "Not allowed")
|
||||
return RedirectResponse(url=redirect_url, status_code=302)
|
||||
if is_owner(item, user):
|
||||
delete_attachments_for(target_type, [item["uid"]])
|
||||
comment_uids = []
|
||||
@@ -121,7 +130,7 @@ def delete_content_item(table_name: str, target_type: str, user: dict, slug: str
|
||||
delete_inline_image(item.get(inline_image_field))
|
||||
table.delete(id=item["id"])
|
||||
logger.info(f"{table_name} {item['uid']} deleted by {user['username']}")
|
||||
return RedirectResponse(url=redirect_url, status_code=302)
|
||||
return action_result(request, redirect_url)
|
||||
|
||||
|
||||
def load_detail(table_name: str, target_type: str, slug: str, user: dict | None) -> dict | None:
|
||||
|
||||
+325
-36
@@ -3,7 +3,7 @@ import logging
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from devplacepy.cache import TTLCache
|
||||
from devplacepy.config import DATABASE_URL
|
||||
from devplacepy.config import DATABASE_URL, INTERNAL_GATEWAY_URL
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -32,6 +32,60 @@ def refresh_snapshot() -> None:
|
||||
connection.commit()
|
||||
|
||||
|
||||
_cache_version_cache = TTLCache(ttl=2)
|
||||
_local_cache_versions: dict = {}
|
||||
_cache_state_ready = False
|
||||
|
||||
|
||||
def _ensure_cache_state() -> None:
|
||||
global _cache_state_ready
|
||||
if _cache_state_ready:
|
||||
return
|
||||
db.query(
|
||||
"CREATE TABLE IF NOT EXISTS cache_state "
|
||||
"(name TEXT PRIMARY KEY, version INTEGER NOT NULL DEFAULT 0)"
|
||||
)
|
||||
_cache_state_ready = True
|
||||
|
||||
|
||||
def get_cache_version(name: str) -> int:
|
||||
cached = _cache_version_cache.get(name)
|
||||
if cached is not None:
|
||||
return cached
|
||||
try:
|
||||
_ensure_cache_state()
|
||||
row = next(iter(db.query("SELECT version FROM cache_state WHERE name = :name", name=name)), None)
|
||||
version = int(row["version"]) if row else 0
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not read cache version {name}: {e}")
|
||||
version = 0
|
||||
_cache_version_cache.set(name, version)
|
||||
return version
|
||||
|
||||
|
||||
def bump_cache_version(name: str) -> None:
|
||||
try:
|
||||
_ensure_cache_state()
|
||||
db.query("INSERT OR IGNORE INTO cache_state (name, version) VALUES (:name, 0)", name=name)
|
||||
db.query("UPDATE cache_state SET version = version + 1 WHERE name = :name", name=name)
|
||||
connection = db.executable
|
||||
if connection.in_transaction():
|
||||
connection.commit()
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not bump cache version {name}: {e}")
|
||||
_cache_version_cache.pop(name)
|
||||
|
||||
|
||||
def sync_local_cache(name: str, cache) -> None:
|
||||
current = get_cache_version(name)
|
||||
if name not in _local_cache_versions:
|
||||
_local_cache_versions[name] = current
|
||||
return
|
||||
if _local_cache_versions[name] != current:
|
||||
cache.clear()
|
||||
_local_cache_versions[name] = current
|
||||
|
||||
|
||||
def _index(db, table, name, columns):
|
||||
try:
|
||||
if table in db.tables:
|
||||
@@ -45,6 +99,7 @@ def init_db():
|
||||
tables = db.tables
|
||||
_index(db, "users", "idx_users_username", ["username"])
|
||||
_index(db, "users", "idx_users_email", ["email"])
|
||||
_index(db, "users", "idx_users_api_key", ["api_key"])
|
||||
_index(db, "posts", "idx_posts_user_uid", ["user_uid"])
|
||||
_index(db, "posts", "idx_posts_created_at", ["created_at"])
|
||||
_index(db, "posts", "idx_posts_topic", ["topic"])
|
||||
@@ -96,6 +151,19 @@ def init_db():
|
||||
|
||||
_index(db, "news_images", "idx_news_images_news_uid", ["news_uid"])
|
||||
_index(db, "news_sync", "idx_news_sync_external_id", ["external_id"])
|
||||
_index(db, "service_state", "idx_service_state_name", ["name"])
|
||||
_index(db, "devii_conversations", "idx_devii_conv_owner", ["owner_kind", "owner_id"])
|
||||
_index(db, "devii_usage_ledger", "idx_devii_ledger_owner_time", ["owner_kind", "owner_id", "created_at"])
|
||||
_index(db, "devii_turns", "idx_devii_turns_owner_time", ["owner_kind", "owner_id", "started_at"])
|
||||
_index(db, "devii_tasks", "idx_devii_tasks_owner", ["owner_kind", "owner_id"])
|
||||
_index(db, "devii_tasks", "idx_devii_tasks_due", ["enabled", "status", "next_run_at"])
|
||||
_index(db, "devii_lessons", "idx_devii_lessons_owner", ["owner_kind", "owner_id"])
|
||||
_index(db, "gateway_usage_ledger", "idx_gw_usage_time", ["created_at"])
|
||||
_index(db, "gateway_usage_ledger", "idx_gw_usage_backend_time", ["backend", "created_at"])
|
||||
_index(db, "gateway_usage_ledger", "idx_gw_usage_model_time", ["model", "created_at"])
|
||||
_index(db, "gateway_usage_ledger", "idx_gw_usage_owner_time", ["owner_kind", "owner_id", "created_at"])
|
||||
_index(db, "gateway_usage_ledger", "idx_gw_usage_endpoint_time", ["endpoint", "created_at"])
|
||||
_index(db, "gateway_concurrency_samples", "idx_gw_conc_time", ["created_at"])
|
||||
|
||||
if "news" in tables:
|
||||
for article in db["news"].find(status=None):
|
||||
@@ -130,7 +198,7 @@ def init_db():
|
||||
news_defaults = {
|
||||
"news_grade_threshold": "7",
|
||||
"news_api_url": "https://news.app.molodetz.nl/api",
|
||||
"news_ai_url": "https://openai.app.molodetz.nl/v1/chat/completions",
|
||||
"news_ai_url": INTERNAL_GATEWAY_URL,
|
||||
"news_ai_model": "molodetz",
|
||||
}
|
||||
for key, value in news_defaults.items():
|
||||
@@ -164,10 +232,57 @@ def init_db():
|
||||
db["site_settings"].insert({"uid": f"default_{key}", "key": key, "value": value})
|
||||
|
||||
_backfill_gamification()
|
||||
backfill_api_keys()
|
||||
migrate_ai_gateway_settings()
|
||||
|
||||
logger.info("Database initialized")
|
||||
|
||||
|
||||
def internal_gateway_key() -> str:
|
||||
return get_setting("gateway_internal_key", "")
|
||||
|
||||
|
||||
OLD_GATEWAY_URL = "https://openai.app.molodetz.nl/v1/chat/completions"
|
||||
|
||||
|
||||
def migrate_ai_gateway_settings() -> None:
|
||||
import os
|
||||
import uuid
|
||||
if not get_setting("gateway_internal_key", ""):
|
||||
set_setting("gateway_internal_key", str(uuid.uuid4()))
|
||||
logger.info("Generated internal gateway key")
|
||||
deepseek = os.environ.get("DEEPSEEK_API_KEY", "")
|
||||
openrouter = os.environ.get("OPENROUTER_API_KEY", "")
|
||||
if not get_setting("gateway_api_key", "") and (deepseek or openrouter):
|
||||
set_setting("gateway_api_key", deepseek or openrouter)
|
||||
logger.info("Migrated gateway upstream key from environment")
|
||||
if not get_setting("gateway_vision_key", "") and openrouter:
|
||||
set_setting("gateway_vision_key", openrouter)
|
||||
logger.info("Migrated gateway vision key from environment")
|
||||
for key in ("news_ai_url", "bot_api_url", "devii_ai_url"):
|
||||
if get_setting(key, "") == OLD_GATEWAY_URL:
|
||||
set_setting(key, INTERNAL_GATEWAY_URL)
|
||||
logger.info(f"Migrated {key} to the internal gateway")
|
||||
if get_setting("bot_model", "") == "deepseek-chat":
|
||||
set_setting("bot_model", "molodetz")
|
||||
logger.info("Migrated bot_model to molodetz")
|
||||
|
||||
|
||||
def backfill_api_keys() -> int:
|
||||
if "users" not in db.tables:
|
||||
return 0
|
||||
users = db["users"]
|
||||
if not users.has_column("api_key"):
|
||||
users.create_column_by_example("api_key", "")
|
||||
import uuid_utils
|
||||
updated = 0
|
||||
for user in users.find():
|
||||
if not user.get("api_key"):
|
||||
users.update({"uid": user["uid"], "api_key": str(uuid_utils.uuid7())}, ["uid"])
|
||||
updated += 1
|
||||
return updated
|
||||
|
||||
|
||||
def _backfill_gamification():
|
||||
if "users" not in db.tables:
|
||||
return
|
||||
@@ -503,6 +618,7 @@ _settings_cache = TTLCache(ttl=60)
|
||||
|
||||
|
||||
def get_setting(key: str, default: str = "") -> str:
|
||||
sync_local_cache("settings", _settings_cache)
|
||||
cached = _settings_cache.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
@@ -523,8 +639,20 @@ def get_int_setting(key: str, default: int) -> int:
|
||||
return default
|
||||
|
||||
|
||||
def set_setting(key: str, value: str) -> None:
|
||||
settings = get_table("site_settings")
|
||||
existing = settings.find_one(key=key)
|
||||
if existing:
|
||||
settings.update({"id": existing["id"], "key": key, "value": value}, ["id"])
|
||||
else:
|
||||
settings.insert({"uid": f"setting_{key}", "key": key, "value": value})
|
||||
_settings_cache.set(key, value)
|
||||
bump_cache_version("settings")
|
||||
|
||||
|
||||
def clear_settings_cache() -> None:
|
||||
_settings_cache.clear()
|
||||
bump_cache_version("settings")
|
||||
|
||||
|
||||
_stats_cache = TTLCache(ttl=30)
|
||||
@@ -545,6 +673,83 @@ def get_site_stats() -> dict:
|
||||
return stats
|
||||
|
||||
|
||||
_analytics_cache = TTLCache(ttl=60)
|
||||
|
||||
|
||||
def get_platform_analytics(top_n: int = 10) -> dict:
|
||||
top_n = max(1, min(int(top_n or 10), 50))
|
||||
cache_key = f"analytics:{top_n}"
|
||||
cached = _analytics_cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
d1 = (now - timedelta(days=1)).isoformat()
|
||||
d7 = (now - timedelta(days=7)).isoformat()
|
||||
d30 = (now - timedelta(days=30)).isoformat()
|
||||
|
||||
active_24h = active_7d = active_30d = 0
|
||||
sources = [table for table in ("posts", "comments", "gists", "projects") if table in db.tables]
|
||||
if sources:
|
||||
union = " UNION ALL ".join(f"SELECT user_uid, created_at FROM {table}" for table in sources)
|
||||
rows = db.query(
|
||||
"SELECT "
|
||||
"COUNT(DISTINCT CASE WHEN created_at >= :d1 THEN user_uid END) AS a1, "
|
||||
"COUNT(DISTINCT CASE WHEN created_at >= :d7 THEN user_uid END) AS a7, "
|
||||
"COUNT(DISTINCT CASE WHEN created_at >= :d30 THEN user_uid END) AS a30 "
|
||||
f"FROM ({union})",
|
||||
d1=d1, d7=d7, d30=d30,
|
||||
)
|
||||
for row in rows:
|
||||
active_24h = row["a1"] or 0
|
||||
active_7d = row["a7"] or 0
|
||||
active_30d = row["a30"] or 0
|
||||
|
||||
signed_in_now = 0
|
||||
if "sessions" in db.tables:
|
||||
for row in db.query(
|
||||
"SELECT COUNT(DISTINCT user_uid) AS n FROM sessions WHERE expires_at > :now",
|
||||
now=now.isoformat(),
|
||||
):
|
||||
signed_in_now = row["n"] or 0
|
||||
|
||||
def _new_since(cutoff: str) -> int:
|
||||
return db["users"].count(created_at={">=": cutoff}) if "users" in db.tables else 0
|
||||
|
||||
site = get_site_stats()
|
||||
totals = {
|
||||
"posts": db["posts"].count() if "posts" in db.tables else 0,
|
||||
"comments": db["comments"].count() if "comments" in db.tables else 0,
|
||||
"gists": site["total_gists"],
|
||||
"projects": site["total_projects"],
|
||||
"news": db["news"].count() if "news" in db.tables else 0,
|
||||
}
|
||||
top_authors = [
|
||||
{"username": author.get("username", ""), "stars": author.get("stars", 0)}
|
||||
for author in get_top_authors(top_n)
|
||||
]
|
||||
|
||||
result = {
|
||||
"total_members": site["total_members"],
|
||||
"active_24h": active_24h,
|
||||
"active_7d": active_7d,
|
||||
"active_30d": active_30d,
|
||||
"signed_in_now": signed_in_now,
|
||||
"new_24h": _new_since(d1),
|
||||
"new_7d": _new_since(d7),
|
||||
"new_30d": _new_since(d30),
|
||||
"posts_today": site["posts_today"],
|
||||
"totals": totals,
|
||||
"top_authors": top_authors,
|
||||
"active_definition": (
|
||||
"Active = created a post, comment, gist, or project within the window. "
|
||||
"Automated/bot accounts are not separately flagged."
|
||||
),
|
||||
}
|
||||
_analytics_cache.set(cache_key, result)
|
||||
return result
|
||||
|
||||
|
||||
_activity_cache = TTLCache(ttl=120)
|
||||
|
||||
_ACTIVITY_TABLES = ("posts", "comments", "gists", "projects")
|
||||
@@ -582,27 +787,54 @@ def _activity_level(count: int) -> int:
|
||||
return 4
|
||||
|
||||
|
||||
def get_first_activity_date(user_uid: str):
|
||||
sources = [table for table in _ACTIVITY_TABLES if table in db.tables]
|
||||
if not sources:
|
||||
return None
|
||||
union = " UNION ALL ".join(f"SELECT MIN(created_at) AS m FROM {table} WHERE user_uid = :u" for table in sources)
|
||||
for row in db.query(f"SELECT MIN(m) AS first FROM ({union})", u=user_uid):
|
||||
if row["first"]:
|
||||
return datetime.fromisoformat(row["first"]).date()
|
||||
return None
|
||||
|
||||
|
||||
HEATMAP_WEEKS = 53
|
||||
|
||||
|
||||
def get_activity_heatmap(user_uid: str) -> list:
|
||||
calendar = get_activity_calendar(user_uid)
|
||||
today = datetime.now(timezone.utc).date()
|
||||
start = today - timedelta(days=363)
|
||||
start = start - timedelta(days=start.weekday())
|
||||
week_start = today - timedelta(days=today.weekday())
|
||||
start = week_start - timedelta(weeks=HEATMAP_WEEKS - 1)
|
||||
first = get_first_activity_date(user_uid)
|
||||
if first:
|
||||
first_week = first - timedelta(days=first.weekday())
|
||||
if first_week > start:
|
||||
start = first_week
|
||||
weeks = []
|
||||
week = []
|
||||
day = start
|
||||
while day <= today:
|
||||
iso = day.isoformat()
|
||||
count = calendar.get(iso, 0)
|
||||
week.append({"date": iso, "count": count, "level": _activity_level(count)})
|
||||
if len(week) == 7:
|
||||
weeks.append(week)
|
||||
week = []
|
||||
day = day + timedelta(days=1)
|
||||
if week:
|
||||
for w in range(HEATMAP_WEEKS):
|
||||
week = []
|
||||
for d in range(7):
|
||||
day = start + timedelta(days=w * 7 + d)
|
||||
iso = day.isoformat()
|
||||
count = calendar.get(iso, 0)
|
||||
week.append({"date": iso, "count": count, "level": _activity_level(count)})
|
||||
weeks.append(week)
|
||||
return weeks
|
||||
|
||||
|
||||
def get_activity_months(weeks: list) -> list:
|
||||
if not weeks:
|
||||
return []
|
||||
last = len(weeks) - 1
|
||||
labels = []
|
||||
for i in range(6):
|
||||
column = round(i * last / 5)
|
||||
iso = weeks[column][0]["date"]
|
||||
labels.append(datetime.fromisoformat(iso).strftime("%b"))
|
||||
return labels
|
||||
|
||||
|
||||
def get_streaks(user_uid: str) -> dict:
|
||||
calendar = get_activity_calendar(user_uid)
|
||||
if not calendar:
|
||||
@@ -645,7 +877,14 @@ def get_gist_languages() -> set[str]:
|
||||
return codes
|
||||
|
||||
|
||||
_STARRED_CONTENT_TABLES = ("posts", "projects", "gists")
|
||||
VOTABLE_TARGETS: dict[str, str] = {
|
||||
"post": "posts",
|
||||
"project": "projects",
|
||||
"gist": "gists",
|
||||
"comment": "comments",
|
||||
}
|
||||
|
||||
STAR_TARGETS: set[str] = {"post", "project", "gist"}
|
||||
|
||||
_authors_cache = TTLCache(ttl=60)
|
||||
|
||||
@@ -654,13 +893,18 @@ def _ranked_authors() -> list:
|
||||
cached = _authors_cache.get("ranked")
|
||||
if cached is not None:
|
||||
return cached
|
||||
sources = [table for table in _STARRED_CONTENT_TABLES if table in db.tables]
|
||||
if not sources:
|
||||
sources = [(target_type, table_name) for target_type, table_name in VOTABLE_TARGETS.items() if table_name in db.tables]
|
||||
if "votes" not in db.tables or not sources:
|
||||
_authors_cache.set("ranked", [])
|
||||
return []
|
||||
union = " UNION ALL ".join(f"SELECT user_uid, stars FROM {table}" for table in sources)
|
||||
union = " UNION ALL ".join(
|
||||
f"SELECT t.user_uid AS user_uid, v.value AS value "
|
||||
f"FROM votes v JOIN {table_name} t ON v.target_uid = t.uid "
|
||||
f"WHERE v.target_type = '{target_type}'"
|
||||
for target_type, table_name in sources
|
||||
)
|
||||
rows = db.query(
|
||||
f"SELECT user_uid, SUM(stars) AS total FROM ({union}) "
|
||||
f"SELECT user_uid, SUM(value) AS total FROM ({union}) "
|
||||
f"GROUP BY user_uid HAVING total > 0 ORDER BY total DESC"
|
||||
)
|
||||
ranked = [(row["user_uid"], row["total"]) for row in rows]
|
||||
@@ -698,11 +942,19 @@ def get_user_rank(user_uid: str):
|
||||
|
||||
|
||||
def get_user_stars(user_uid: str) -> int:
|
||||
if "votes" not in db.tables:
|
||||
return 0
|
||||
total = 0
|
||||
for table in _STARRED_CONTENT_TABLES:
|
||||
if table in db.tables:
|
||||
for row in db.query(f"SELECT COALESCE(SUM(stars), 0) AS s FROM {table} WHERE user_uid = :u", u=user_uid):
|
||||
total += row["s"] or 0
|
||||
for target_type, table_name in VOTABLE_TARGETS.items():
|
||||
if table_name not in db.tables:
|
||||
continue
|
||||
for row in db.query(
|
||||
f"SELECT COALESCE(SUM(v.value), 0) AS s "
|
||||
f"FROM votes v JOIN {table_name} t ON v.target_uid = t.uid "
|
||||
f"WHERE v.target_type = :tt AND t.user_uid = :u",
|
||||
tt=target_type, u=user_uid,
|
||||
):
|
||||
total += row["s"] or 0
|
||||
return total
|
||||
|
||||
|
||||
@@ -739,21 +991,12 @@ def resolve_object_url(target_type: str, target_uid: str) -> str:
|
||||
return "/feed"
|
||||
|
||||
|
||||
VOTABLE_TARGETS: dict[str, str] = {
|
||||
"post": "posts",
|
||||
"project": "projects",
|
||||
"gist": "gists",
|
||||
"comment": "comments",
|
||||
}
|
||||
|
||||
STAR_TARGETS: set[str] = {"post", "project", "gist"}
|
||||
|
||||
|
||||
def update_target_stars(target_type: str, target_uid: str, net_stars: int) -> None:
|
||||
table_name = VOTABLE_TARGETS.get(target_type)
|
||||
if not table_name or target_type not in STAR_TARGETS:
|
||||
if not table_name:
|
||||
return
|
||||
get_table(table_name).update({"uid": target_uid, "stars": net_stars}, ["uid"])
|
||||
if target_type in STAR_TARGETS:
|
||||
get_table(table_name).update({"uid": target_uid, "stars": net_stars}, ["uid"])
|
||||
_authors_cache.clear()
|
||||
|
||||
|
||||
@@ -817,6 +1060,52 @@ def build_pagination(page, total, per_page=25):
|
||||
}
|
||||
|
||||
|
||||
def get_follow_counts(user_uid: str) -> dict:
|
||||
if "follows" not in db.tables:
|
||||
return {"followers": 0, "following": 0}
|
||||
follows = get_table("follows")
|
||||
return {
|
||||
"followers": follows.count(following_uid=user_uid),
|
||||
"following": follows.count(follower_uid=user_uid),
|
||||
}
|
||||
|
||||
|
||||
def get_follow_list(user_uid: str, mode: str, page: int = 1, per_page: int = 25) -> tuple:
|
||||
if "follows" not in db.tables:
|
||||
return [], build_pagination(page, 0, per_page)
|
||||
follows = get_table("follows")
|
||||
key = "following_uid" if mode == "followers" else "follower_uid"
|
||||
other = "follower_uid" if mode == "followers" else "following_uid"
|
||||
total = follows.count(**{key: user_uid})
|
||||
pagination = build_pagination(page, total, per_page)
|
||||
offset = (pagination["page"] - 1) * pagination["per_page"]
|
||||
rows = list(follows.find(order_by=["-created_at"], _limit=pagination["per_page"], _offset=offset, **{key: user_uid}))
|
||||
users_map = get_users_by_uids([row[other] for row in rows])
|
||||
people = []
|
||||
for row in rows:
|
||||
person = users_map.get(row[other])
|
||||
if person:
|
||||
people.append({
|
||||
"uid": person["uid"],
|
||||
"username": person["username"],
|
||||
"bio": (person.get("bio") or "")[:140],
|
||||
"followed_at": row.get("created_at"),
|
||||
})
|
||||
return people, pagination
|
||||
|
||||
|
||||
def get_following_among(follower_uid: str, target_uids: list) -> set:
|
||||
if not follower_uid or not target_uids or "follows" not in db.tables:
|
||||
return set()
|
||||
placeholders, params = _in_clause(target_uids)
|
||||
params["f"] = follower_uid
|
||||
rows = db.query(
|
||||
f"SELECT following_uid FROM follows WHERE follower_uid = :f AND following_uid IN ({placeholders})",
|
||||
**params,
|
||||
)
|
||||
return {row["following_uid"] for row in rows}
|
||||
|
||||
|
||||
def get_daily_topic():
|
||||
if "news" in db.tables:
|
||||
article = db["news"].find_one(status="published", order_by=["-synced_at"])
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,81 @@
|
||||
from typing import Any, Union, get_args, get_origin
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
_PLACEHOLDERS = {
|
||||
"uid": "UID",
|
||||
"username": "username",
|
||||
"slug": "slug",
|
||||
"url": "/path",
|
||||
"email": "user@example.com",
|
||||
"title": "Title",
|
||||
"content": "text",
|
||||
"description": "text",
|
||||
"bio": "text",
|
||||
"language": "python",
|
||||
"topic": "random",
|
||||
"status": "published",
|
||||
"next_cursor": "2026-01-01T00:00:00+00:00",
|
||||
"created_at": "2026-01-01T00:00:00+00:00",
|
||||
"updated_at": "2026-01-01T00:00:00+00:00",
|
||||
"synced_at": "2026-01-01T00:00:00+00:00",
|
||||
"time_ago": "2 hours ago",
|
||||
}
|
||||
|
||||
|
||||
def _placeholder(name: str) -> str:
|
||||
for token, value in _PLACEHOLDERS.items():
|
||||
if token in name:
|
||||
return value
|
||||
return "string"
|
||||
|
||||
|
||||
def _unwrap_optional(annotation):
|
||||
if get_origin(annotation) is Union:
|
||||
args = [arg for arg in get_args(annotation) if arg is not type(None)]
|
||||
return args[0] if args else Any
|
||||
return annotation
|
||||
|
||||
|
||||
def _value(name, annotation, stack):
|
||||
annotation = _unwrap_optional(annotation)
|
||||
origin = get_origin(annotation)
|
||||
if origin in (list, set, tuple):
|
||||
args = get_args(annotation)
|
||||
inner = args[0] if args else str
|
||||
if isinstance(inner, type) and issubclass(inner, BaseModel) and inner in stack:
|
||||
return []
|
||||
return [_value(name, inner, stack)]
|
||||
if origin is dict or annotation is dict:
|
||||
return {}
|
||||
if annotation in (list, set, tuple):
|
||||
return []
|
||||
if isinstance(annotation, type) and issubclass(annotation, BaseModel):
|
||||
return _from_model(annotation, stack)
|
||||
if annotation is bool:
|
||||
return False
|
||||
if annotation is int:
|
||||
return 0
|
||||
if annotation is float:
|
||||
return 0.0
|
||||
if annotation is Any:
|
||||
return None
|
||||
return _placeholder(name)
|
||||
|
||||
|
||||
def _from_model(model, stack):
|
||||
if model in stack:
|
||||
return {}
|
||||
stack = stack | {model}
|
||||
example = {}
|
||||
for name, info in model.model_fields.items():
|
||||
example[name] = _value(name, info.annotation, stack)
|
||||
return example
|
||||
|
||||
|
||||
def schema_example(model):
|
||||
return _from_model(model, frozenset())
|
||||
|
||||
|
||||
def action_example(redirect, data=None):
|
||||
return {"ok": True, "redirect": redirect, "data": data}
|
||||
@@ -0,0 +1,159 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.config import STATIC_DIR
|
||||
from devplacepy import docs_api
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _vendor(name: str) -> str:
|
||||
return (Path(STATIC_DIR) / "vendor" / name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _prose_markdown(slug: str, ctx: dict) -> str:
|
||||
from devplacepy.templating import templates
|
||||
try:
|
||||
rendered = templates.env.get_template(f"docs/{slug}.html").render(**ctx)
|
||||
except Exception:
|
||||
return ""
|
||||
rendered = re.sub(r"^\s*<div[^>]*>", "", rendered.strip())
|
||||
rendered = re.sub(r"</div>\s*$", "", rendered)
|
||||
return rendered.strip()
|
||||
|
||||
|
||||
def _params_table(params: list) -> str:
|
||||
if not params:
|
||||
return ""
|
||||
rows = ["| Name | In | Type | Required | Description |",
|
||||
"|------|----|------|----------|-------------|"]
|
||||
for p in params:
|
||||
desc = (p.get("description", "") or "").replace("|", "\\|")
|
||||
if p["type"] == "enum" and p.get("options"):
|
||||
allowed = ", ".join(str(option) for option in p["options"]).replace("|", "\\|")
|
||||
desc = f"{desc} Allowed: {allowed}.".strip()
|
||||
rows.append(f"| `{p['name']}` | {p['location']} | {p['type']} | "
|
||||
f"{'yes' if p['required'] else 'no'} | {desc} |")
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def _endpoint_markdown(ep: dict) -> str:
|
||||
lines = [f"### `{ep['method']} {ep['path']}` - {ep['title']}", "", ep["summary"], "",
|
||||
f"*Minimal role:* {ep.get('min_role', ep['auth'])}"]
|
||||
if ep.get("params"):
|
||||
lines += ["", "**Parameters**", "", _params_table(ep["params"])]
|
||||
for note in ep.get("notes", []) or []:
|
||||
lines += ["", f"> {note}"]
|
||||
if ep.get("sample_response") is not None:
|
||||
lines += ["", "**Sample response**", "", "```json",
|
||||
json.dumps(ep["sample_response"], indent=2), "```"]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _group_markdown(group: dict) -> str:
|
||||
parts = [(group.get("intro", "") or "").strip(), ""]
|
||||
for ep in group.get("endpoints", []) or []:
|
||||
parts.append(_endpoint_markdown(ep))
|
||||
parts.append("")
|
||||
return "\n".join(parts).strip()
|
||||
|
||||
|
||||
def build_markdown(is_admin: bool, base: str, username: str = "", api_key: str = "") -> str:
|
||||
from devplacepy.routers.docs import DOCS_PAGES
|
||||
from devplacepy.services.manager import service_manager
|
||||
|
||||
user_ctx = {"role": "Admin"} if is_admin else None
|
||||
replacements = {
|
||||
"{{ base }}": base,
|
||||
"{{ username }}": username or "YOUR_USERNAME",
|
||||
"{{ api_key }}": api_key or "YOUR_API_KEY",
|
||||
}
|
||||
pages = [p for p in DOCS_PAGES if (not p.get("admin") or is_admin) and p["kind"] != "live"]
|
||||
slugs = {p["slug"] for p in pages}
|
||||
|
||||
out = [f"# DevPlace Documentation", "",
|
||||
f"Complete developer documentation for `{base}`.", "", "## Contents", ""]
|
||||
for page in pages:
|
||||
out.append(f"- [{page['title']}](#doc-{page['slug']})")
|
||||
out += ["", "---", ""]
|
||||
|
||||
for page in pages:
|
||||
if page["kind"] == "prose":
|
||||
body = _prose_markdown(page["slug"], {"base": base, "user": user_ctx})
|
||||
elif page.get("dynamic"):
|
||||
group = docs_api.build_services_group(service_manager.describe_all(), base)
|
||||
body = _group_markdown(docs_api._substitute(group, replacements))
|
||||
else:
|
||||
group = docs_api.render_group(page["slug"], base, username, api_key) or {}
|
||||
body = _group_markdown(group)
|
||||
out += [f'<a id="doc-{page["slug"]}"></a>', body, "", "---", ""]
|
||||
|
||||
text = "\n".join(out).strip() + "\n"
|
||||
return re.sub(
|
||||
r"\(/docs/([a-z0-9-]+)\.html(?:#[^)]*)?\)",
|
||||
lambda m: f"(#doc-{m.group(1)})" if m.group(1) in slugs else m.group(0),
|
||||
text,
|
||||
)
|
||||
|
||||
|
||||
_LAYOUT_CSS = """
|
||||
:root { color-scheme: light dark; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
||||
line-height: 1.65; color: #1f2328; background: #fff; }
|
||||
.doc-header { position: sticky; top: 0; z-index: 5; background: #0d1117; color: #fff;
|
||||
padding: 0.9rem 1.5rem; font-weight: 700; letter-spacing: 0.02em; box-shadow: 0 1px 0 rgba(0,0,0,0.15); }
|
||||
.doc-header span { opacity: 0.6; font-weight: 400; }
|
||||
main.doc { max-width: 860px; margin: 0 auto; padding: 2rem 1.5rem 4rem; }
|
||||
main.doc h1 { font-size: 1.9rem; margin: 2.4rem 0 1rem; padding-bottom: 0.3rem; border-bottom: 2px solid #e1e4e8; }
|
||||
main.doc h1:first-child { margin-top: 0; }
|
||||
main.doc h2 { font-size: 1.4rem; margin: 2rem 0 0.8rem; }
|
||||
main.doc h3 { font-size: 1.08rem; margin: 1.6rem 0 0.5rem; }
|
||||
main.doc h3 code { background: #eef1f4; padding: 0.1em 0.4em; border-radius: 5px; font-size: 0.95em; }
|
||||
main.doc a { color: #0969da; text-decoration: none; }
|
||||
main.doc a:hover { text-decoration: underline; }
|
||||
main.doc code { font-family: "SF Mono", Monaco, Consolas, monospace; font-size: 0.9em; }
|
||||
main.doc :not(pre) > code { background: #eef1f4; padding: 0.12em 0.4em; border-radius: 5px; }
|
||||
main.doc pre { background: #282c34; border-radius: 8px; padding: 1rem; overflow-x: auto; }
|
||||
main.doc pre code { color: #abb2bf; font-size: 0.85rem; line-height: 1.5; }
|
||||
main.doc table { border-collapse: collapse; width: 100%; margin: 1rem 0; font-size: 0.92rem; }
|
||||
main.doc th, main.doc td { border: 1px solid #d0d7de; padding: 0.45rem 0.7rem; text-align: left; }
|
||||
main.doc th { background: #f6f8fa; }
|
||||
main.doc blockquote { margin: 0.8rem 0; padding: 0.2rem 1rem; border-left: 4px solid #d0d7de; color: #57606a; }
|
||||
main.doc hr { border: none; border-top: 1px solid #e1e4e8; margin: 2.5rem 0; }
|
||||
@media (prefers-color-scheme: dark) {
|
||||
body { background: #0d1117; color: #c9d1d9; }
|
||||
main.doc h1, main.doc h2 { border-color: #30363d; }
|
||||
main.doc :not(pre) > code, main.doc h3 code { background: #21262d; }
|
||||
main.doc th { background: #161b22; }
|
||||
main.doc th, main.doc td, main.doc hr, main.doc blockquote { border-color: #30363d; }
|
||||
main.doc a { color: #58a6ff; }
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def build_html(markdown: str) -> str:
|
||||
marked = _vendor("marked.umd.js")
|
||||
hljs = _vendor("highlight.min.js")
|
||||
hl_css = _vendor("atom-one-dark.min.css")
|
||||
payload = base64.b64encode(markdown.encode("utf-8")).decode("ascii")
|
||||
return (
|
||||
"<!doctype html>\n<html lang=\"en\">\n<head>\n<meta charset=\"utf-8\">\n"
|
||||
"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n"
|
||||
"<title>DevPlace Documentation</title>\n"
|
||||
f"<style>{hl_css}</style>\n<style>{_LAYOUT_CSS}</style>\n"
|
||||
f"<script>{marked}</script>\n<script>{hljs}</script>\n"
|
||||
"</head>\n<body>\n"
|
||||
"<header class=\"doc-header\">DevPlace <span>Documentation</span></header>\n"
|
||||
"<main id=\"doc\" class=\"doc\"></main>\n"
|
||||
f"<script id=\"docmd\" type=\"application/x-markdown-base64\">{payload}</script>\n"
|
||||
"<script>\n(function(){\n"
|
||||
" var raw = atob(document.getElementById('docmd').textContent);\n"
|
||||
" var md = decodeURIComponent(escape(raw));\n"
|
||||
" marked.setOptions({ gfm: true, breaks: false });\n"
|
||||
" document.getElementById('doc').innerHTML = marked.parse(md);\n"
|
||||
" document.querySelectorAll('#doc pre code').forEach(function(el){ try { hljs.highlightElement(el); } catch (e) {} });\n"
|
||||
"})();\n</script>\n</body>\n</html>\n"
|
||||
)
|
||||
@@ -0,0 +1,147 @@
|
||||
from devplacepy.database import (
|
||||
db,
|
||||
get_table,
|
||||
get_setting,
|
||||
get_site_stats,
|
||||
get_top_authors,
|
||||
get_featured_news,
|
||||
get_gist_languages,
|
||||
get_daily_topic,
|
||||
get_streaks,
|
||||
get_user_stars,
|
||||
get_user_rank,
|
||||
)
|
||||
from devplacepy.utils import format_date
|
||||
|
||||
|
||||
def _count(table: str, **criteria) -> int:
|
||||
if table not in db.tables:
|
||||
return 0
|
||||
return get_table(table).count(**criteria)
|
||||
|
||||
|
||||
def _recent(table: str, user_uid: str, limit: int = 5) -> list:
|
||||
if table not in db.tables:
|
||||
return []
|
||||
return list(get_table(table).find(user_uid=user_uid, order_by=["-created_at"], _limit=limit))
|
||||
|
||||
|
||||
def _user_facts(user: dict) -> dict:
|
||||
uid = user["uid"]
|
||||
|
||||
posts = _recent("posts", uid)
|
||||
gists = _recent("gists", uid)
|
||||
projects = _recent("projects", uid)
|
||||
badges = [b.get("badge_name", "") for b in get_table("badges").find(user_uid=uid)] if "badges" in db.tables else []
|
||||
languages = sorted({
|
||||
g.get("language") for g in (get_table("gists").find(user_uid=uid) if "gists" in db.tables else [])
|
||||
if g.get("language")
|
||||
})
|
||||
|
||||
return {
|
||||
"username": user.get("username", ""),
|
||||
"role": user.get("role", "Member"),
|
||||
"member_since": format_date(user.get("created_at", "")) if user.get("created_at") else "",
|
||||
"level": user.get("level", 1),
|
||||
"xp": user.get("xp", 0),
|
||||
"stars": get_user_stars(uid),
|
||||
"rank": get_user_rank(uid),
|
||||
"streak": get_streaks(uid),
|
||||
"badges": badges,
|
||||
"languages": languages,
|
||||
"counts": {
|
||||
"posts": _count("posts", user_uid=uid),
|
||||
"gists": _count("gists", user_uid=uid),
|
||||
"projects": _count("projects", user_uid=uid),
|
||||
"comments": _count("comments", user_uid=uid),
|
||||
"followers": _count("follows", following_uid=uid),
|
||||
"following": _count("follows", follower_uid=uid),
|
||||
"bookmarks": _count("bookmarks", user_uid=uid),
|
||||
"unread_notifications": _count("notifications", user_uid=uid, read=False),
|
||||
"unread_messages": _count("messages", receiver_uid=uid, read=False),
|
||||
},
|
||||
"recent_posts": [
|
||||
{"title": (p.get("title") or (p.get("content") or "")[:60] or "Untitled"),
|
||||
"url": f"/posts/{p.get('slug') or p['uid']}"}
|
||||
for p in posts
|
||||
],
|
||||
"recent_gists": [
|
||||
{"title": g.get("title", "Untitled"), "language": g.get("language", ""),
|
||||
"url": f"/gists/{g.get('slug') or g['uid']}"}
|
||||
for g in gists
|
||||
],
|
||||
"recent_projects": [
|
||||
{"title": p.get("title", "Untitled"), "status": p.get("status", ""),
|
||||
"url": f"/projects/{p.get('slug') or p['uid']}"}
|
||||
for p in projects
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_live_facts(user: dict | None) -> dict:
|
||||
stats = get_site_stats()
|
||||
facts = {
|
||||
"site": {
|
||||
"name": get_setting("site_name", "DevPlace"),
|
||||
"tagline": get_setting("site_tagline", ""),
|
||||
"total_members": stats.get("total_members", 0),
|
||||
"posts_today": stats.get("posts_today", 0),
|
||||
"total_projects": stats.get("total_projects", 0),
|
||||
"total_gists": stats.get("total_gists", 0),
|
||||
},
|
||||
"top_authors": [
|
||||
{"username": a.get("username", ""), "stars": a.get("stars", 0)}
|
||||
for a in get_top_authors(10)
|
||||
],
|
||||
"languages": sorted(get_gist_languages()),
|
||||
"news": get_featured_news(5),
|
||||
"topic": get_daily_topic(),
|
||||
"user": _user_facts(user) if user else None,
|
||||
}
|
||||
return facts
|
||||
|
||||
|
||||
def live_text(facts: dict) -> str:
|
||||
parts: list[str] = []
|
||||
site = facts["site"]
|
||||
parts += [
|
||||
site["name"], site["tagline"],
|
||||
"members", str(site["total_members"]),
|
||||
"posts today", str(site["posts_today"]),
|
||||
"projects", str(site["total_projects"]),
|
||||
"gists", str(site["total_gists"]),
|
||||
"top authors leaderboard",
|
||||
]
|
||||
parts += [f"{a['username']} {a['stars']} stars" for a in facts["top_authors"]]
|
||||
parts += ["trending languages"] + facts["languages"]
|
||||
parts += ["developer news"] + [n.get("title", "") for n in facts["news"]]
|
||||
parts += [facts["topic"].get("title", "")]
|
||||
|
||||
user = facts.get("user")
|
||||
if user:
|
||||
counts = user["counts"]
|
||||
parts += [
|
||||
"your dashboard account stats",
|
||||
user["username"], user["role"], "member since", user["member_since"],
|
||||
"level", str(user["level"]), "xp", str(user["xp"]),
|
||||
"stars", str(user["stars"]), "rank", str(user["rank"]),
|
||||
"current streak", str(user["streak"].get("current", 0)),
|
||||
"longest streak", str(user["streak"].get("longest", 0)),
|
||||
"posts", str(counts["posts"]),
|
||||
"gists", str(counts["gists"]),
|
||||
"projects", str(counts["projects"]),
|
||||
"comments", str(counts["comments"]),
|
||||
"followers", str(counts["followers"]),
|
||||
"following", str(counts["following"]),
|
||||
"bookmarks saved", str(counts["bookmarks"]),
|
||||
"unread notifications", str(counts["unread_notifications"]),
|
||||
"unread messages", str(counts["unread_messages"]),
|
||||
"badges",
|
||||
]
|
||||
parts += user["badges"]
|
||||
parts += ["languages you use"] + user["languages"]
|
||||
parts += ["your recent posts"] + [p["title"] for p in user["recent_posts"]]
|
||||
parts += ["your gists"] + [g["title"] for g in user["recent_gists"]]
|
||||
parts += ["your projects"] + [p["title"] for p in user["recent_projects"]]
|
||||
|
||||
return " ".join(str(part) for part in parts if part)
|
||||
@@ -0,0 +1,171 @@
|
||||
import html
|
||||
import math
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.config import TEMPLATES_DIR
|
||||
from devplacepy import docs_api
|
||||
|
||||
K1 = 1.5
|
||||
B = 0.75
|
||||
_TOKEN = re.compile(r"[a-z0-9]+")
|
||||
_static_docs = None
|
||||
|
||||
|
||||
def _tokenize(text: str) -> list:
|
||||
return [t for t in _TOKEN.findall(text.lower()) if len(t) > 1]
|
||||
|
||||
|
||||
def _strip(raw: str) -> str:
|
||||
raw = re.sub(r"{#.*?#}", " ", raw, flags=re.S)
|
||||
raw = re.sub(r"{%.*?%}", " ", raw, flags=re.S)
|
||||
raw = re.sub(r"{{.*?}}", " ", raw, flags=re.S)
|
||||
raw = re.sub(r"<[^>]+>", " ", raw)
|
||||
return re.sub(r"\s+", " ", html.unescape(raw)).strip()
|
||||
|
||||
|
||||
def _prose_text(slug: str) -> str:
|
||||
path = Path(TEMPLATES_DIR) / "docs" / f"{slug}.html"
|
||||
try:
|
||||
return _strip(path.read_text(encoding="utf-8"))
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
|
||||
def _group_text(group: dict) -> str:
|
||||
parts = [group.get("intro", "")]
|
||||
for ep in group.get("endpoints", []) or []:
|
||||
parts += [ep.get("method", ""), ep.get("path", ""), ep.get("title", ""), ep.get("summary", "")]
|
||||
parts += [str(note) for note in ep.get("notes", []) or []]
|
||||
for param in ep.get("params", []) or []:
|
||||
parts += [param.get("name", ""), param.get("description", "")]
|
||||
return _strip("\n".join(str(p) for p in parts))
|
||||
|
||||
|
||||
def _collect_static_docs() -> list:
|
||||
global _static_docs
|
||||
if _static_docs is not None:
|
||||
return _static_docs
|
||||
from devplacepy.routers.docs import DOCS_PAGES
|
||||
from devplacepy.services.manager import service_manager
|
||||
|
||||
docs = []
|
||||
for page in DOCS_PAGES:
|
||||
if page["kind"] == "live":
|
||||
continue
|
||||
slug, title = page["slug"], page["title"]
|
||||
if page["kind"] == "prose":
|
||||
body = _prose_text(slug)
|
||||
elif page.get("dynamic"):
|
||||
body = _group_text(docs_api.build_services_group(service_manager.describe_all(), ""))
|
||||
else:
|
||||
body = _group_text(docs_api.get_group(slug) or {})
|
||||
docs.append({
|
||||
"slug": slug,
|
||||
"title": title,
|
||||
"admin": page.get("admin", False),
|
||||
"text": f"{title}\n{body}",
|
||||
})
|
||||
_static_docs = docs
|
||||
return docs
|
||||
|
||||
|
||||
def _live_docs(user) -> list:
|
||||
from devplacepy import docs_live
|
||||
|
||||
facts = docs_live.build_live_facts(user)
|
||||
return [{
|
||||
"slug": "dashboard",
|
||||
"title": "Your dashboard",
|
||||
"admin": False,
|
||||
"text": "Your dashboard live data stats\n" + docs_live.live_text(facts),
|
||||
}]
|
||||
|
||||
|
||||
def _build(docs: list) -> dict:
|
||||
postings: dict = {}
|
||||
doc_len: list = []
|
||||
doc_text: list = []
|
||||
df: dict = {}
|
||||
for i, doc in enumerate(docs):
|
||||
tokens = _tokenize(doc["text"])
|
||||
doc_len.append(len(tokens) or 1)
|
||||
doc_text.append(doc["text"])
|
||||
freqs: dict = {}
|
||||
for token in tokens:
|
||||
freqs[token] = freqs.get(token, 0) + 1
|
||||
for token, count in freqs.items():
|
||||
postings.setdefault(token, []).append((i, count))
|
||||
df[token] = df.get(token, 0) + 1
|
||||
n = len(docs) or 1
|
||||
avgdl = (sum(doc_len) / n) if doc_len else 1.0
|
||||
idf = {t: math.log(1 + (n - d + 0.5) / (d + 0.5)) for t, d in df.items()}
|
||||
return {
|
||||
"docs": docs,
|
||||
"postings": postings,
|
||||
"doc_len": doc_len,
|
||||
"doc_text": doc_text,
|
||||
"avgdl": avgdl or 1.0,
|
||||
"idf": idf,
|
||||
}
|
||||
|
||||
|
||||
def get_index(user=None) -> dict:
|
||||
return _build(list(_collect_static_docs()) + _live_docs(user))
|
||||
|
||||
|
||||
def reset_index() -> None:
|
||||
global _static_docs
|
||||
_static_docs = None
|
||||
|
||||
|
||||
def _snippet(text: str, terms: list, width: int = 280) -> str:
|
||||
low = text.lower()
|
||||
pos = -1
|
||||
for term in terms:
|
||||
found = low.find(term)
|
||||
if found != -1 and (pos == -1 or found < pos):
|
||||
pos = found
|
||||
if pos == -1:
|
||||
pos = 0
|
||||
start = max(0, pos - width // 3)
|
||||
end = min(len(text), start + width)
|
||||
fragment = html.escape(text[start:end])
|
||||
for term in sorted(set(terms), key=len, reverse=True):
|
||||
fragment = re.sub(r"(?i)(" + re.escape(html.escape(term)) + r")", r"<mark>\1</mark>", fragment)
|
||||
prefix = "… " if start > 0 else ""
|
||||
suffix = " …" if end < len(text) else ""
|
||||
return prefix + fragment + suffix
|
||||
|
||||
|
||||
def search(query: str, user=None, is_admin: bool = False, limit: int = 20) -> list:
|
||||
terms = _tokenize(query or "")
|
||||
if not terms:
|
||||
return []
|
||||
idx = get_index(user)
|
||||
scores: dict = {}
|
||||
for term in set(terms):
|
||||
plist = idx["postings"].get(term)
|
||||
if not plist:
|
||||
continue
|
||||
idf_t = idx["idf"][term]
|
||||
for i, tf in plist:
|
||||
dl = idx["doc_len"][i]
|
||||
denom = tf + K1 * (1 - B + B * dl / idx["avgdl"])
|
||||
scores[i] = scores.get(i, 0.0) + idf_t * (tf * (K1 + 1)) / denom
|
||||
ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
|
||||
results = []
|
||||
for i, score in ranked:
|
||||
doc = idx["docs"][i]
|
||||
if doc["admin"] and not is_admin:
|
||||
continue
|
||||
results.append({
|
||||
"slug": doc["slug"],
|
||||
"title": doc["title"],
|
||||
"score": round(score, 3),
|
||||
"url": f"/docs/{doc['slug']}.html",
|
||||
"snippet": _snippet(idx["doc_text"][i], terms),
|
||||
})
|
||||
if len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
+62
-14
@@ -1,21 +1,27 @@
|
||||
import asyncio
|
||||
import fcntl
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from contextlib import contextmanager
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from devplacepy.config import STATIC_DIR, PORT, SERVICE_LOCK_FILE
|
||||
from devplacepy.config import STATIC_DIR, PORT, SERVICE_LOCK_FILE, INIT_LOCK_FILE
|
||||
from devplacepy.database import init_db, get_table, db, refresh_snapshot, get_users_by_uids, get_comment_counts_by_post_uids, get_vote_counts, get_news_images_by_uids, get_setting, get_int_setting
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.responses import wants_json, json_error
|
||||
from devplacepy.schemas import ValidationErrorOut
|
||||
from fastapi.responses import JSONResponse
|
||||
from devplacepy.utils import get_current_user, time_ago
|
||||
from devplacepy.seo import base_seo_context, site_url, website_schema
|
||||
from devplacepy.routers import auth, feed, posts, comments, projects, profile, messages, notifications, votes, avatar, follow, admin, seo, bugs, news, gists, services as services_router, uploads, push, leaderboard, reactions, bookmarks, polls
|
||||
from devplacepy.routers import auth, feed, posts, comments, projects, profile, messages, notifications, votes, avatar, follow, admin, seo, bugs, news, gists, services as services_router, uploads, push, leaderboard, reactions, bookmarks, polls, docs, openai_gateway, devii
|
||||
from devplacepy.services.manager import service_manager
|
||||
from devplacepy.services.news import NewsService
|
||||
from devplacepy.services.bot import BotsService
|
||||
from devplacepy.services.openai_gateway import GatewayService
|
||||
from devplacepy.services.devii import DeviiService
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
@@ -26,10 +32,27 @@ logger = logging.getLogger(__name__)
|
||||
_rate_limit_store = defaultdict(list)
|
||||
RATE_LIMIT = int(os.environ.get("DEVPLACE_RATE_LIMIT", "60"))
|
||||
RATE_WINDOW = 60
|
||||
WEB_WORKERS = max(1, int(os.environ.get("DEVPLACE_WEB_WORKERS", "1")))
|
||||
|
||||
|
||||
def _worker_rate_limit(limit: int) -> int:
|
||||
return max(1, -(-limit // WEB_WORKERS))
|
||||
|
||||
|
||||
_service_lock_handle = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def init_lock():
|
||||
handle = open(INIT_LOCK_FILE, "w")
|
||||
try:
|
||||
fcntl.flock(handle, fcntl.LOCK_EX)
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(handle, fcntl.LOCK_UN)
|
||||
handle.close()
|
||||
|
||||
|
||||
def acquire_service_lock() -> bool:
|
||||
global _service_lock_handle
|
||||
handle = open(SERVICE_LOCK_FILE, "w")
|
||||
@@ -48,13 +71,15 @@ class UploadStaticFiles(StaticFiles):
|
||||
return response
|
||||
|
||||
|
||||
app = FastAPI(title="DevPlace")
|
||||
app = FastAPI(title="DevPlace", docs_url="/swagger", redoc_url="/redoc", openapi_url="/openapi.json")
|
||||
app.mount("/static/uploads", UploadStaticFiles(directory=str(STATIC_DIR / "uploads"), check_dir=False), name="uploads")
|
||||
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
||||
|
||||
|
||||
@app.exception_handler(404)
|
||||
async def not_found(request: Request, exc):
|
||||
if wants_json(request):
|
||||
return json_error(404, "Not found")
|
||||
seo_ctx = base_seo_context(request, title="Not Found - DevPlace", description="The page you requested does not exist.", robots="noindex")
|
||||
return templates.TemplateResponse(request, "error.html", {**seo_ctx, "request": request, "error_code": 404, "error_message": "Page not found"}, status_code=404)
|
||||
|
||||
@@ -62,6 +87,8 @@ async def not_found(request: Request, exc):
|
||||
@app.exception_handler(500)
|
||||
async def server_error(request: Request, exc):
|
||||
logger.exception("500 error on %s %s", request.method, request.url.path)
|
||||
if wants_json(request):
|
||||
return json_error(500, "Internal server error")
|
||||
seo_ctx = base_seo_context(request, title="Server Error - DevPlace", description="Something went wrong.", robots="noindex")
|
||||
return templates.TemplateResponse(request, "error.html", {**seo_ctx, "request": request, "error_code": 500, "error_message": "Internal server error"}, status_code=500)
|
||||
|
||||
@@ -92,6 +119,12 @@ def _friendly_error(err):
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def on_validation_error(request: Request, exc: RequestValidationError):
|
||||
errors = [_friendly_error(e) for e in exc.errors()]
|
||||
if wants_json(request):
|
||||
fields: dict = {}
|
||||
for raw, friendly in zip(exc.errors(), errors):
|
||||
name = raw["loc"][-1] if raw.get("loc") else "_"
|
||||
fields.setdefault(str(name), []).append(friendly)
|
||||
return JSONResponse(ValidationErrorOut(fields=fields, messages=errors).model_dump(mode="json"), status_code=422)
|
||||
path = request.url.path
|
||||
page = _AUTH_FORM_PAGES.get(path)
|
||||
if page is None and path.startswith("/auth/reset-password/"):
|
||||
@@ -128,6 +161,9 @@ app.include_router(leaderboard.router, prefix="/leaderboard")
|
||||
app.include_router(admin.router, prefix="/admin")
|
||||
app.include_router(seo.router)
|
||||
app.include_router(push.router)
|
||||
app.include_router(docs.router)
|
||||
app.include_router(openai_gateway.router, prefix="/openai")
|
||||
app.include_router(devii.router, prefix="/devii")
|
||||
app.include_router(bugs.router, prefix="/bugs")
|
||||
app.include_router(gists.router, prefix="/gists")
|
||||
app.include_router(news.router, prefix="/news")
|
||||
@@ -152,20 +188,25 @@ async def add_security_headers(request: Request, call_next):
|
||||
|
||||
@app.middleware("http")
|
||||
async def rate_limit_middleware(request: Request, call_next):
|
||||
if request.method in ("POST", "PUT", "DELETE", "PATCH"):
|
||||
limit = max(1, get_int_setting("rate_limit_per_minute", RATE_LIMIT))
|
||||
if request.method in ("POST", "PUT", "DELETE", "PATCH") and not request.url.path.startswith("/openai"):
|
||||
limit = _worker_rate_limit(max(1, get_int_setting("rate_limit_per_minute", RATE_LIMIT)))
|
||||
window = max(1, get_int_setting("rate_limit_window_seconds", RATE_WINDOW))
|
||||
ip = request.headers.get("X-Real-IP") or (request.client.host if request.client else "unknown")
|
||||
now = time.time()
|
||||
window_start = now - window
|
||||
_rate_limit_store[ip] = [t for t in _rate_limit_store[ip] if t > window_start]
|
||||
if len(_rate_limit_store[ip]) >= limit:
|
||||
return HTMLResponse("Rate limit exceeded. Try again later.", status_code=429)
|
||||
retry_after = {"Retry-After": str(window)}
|
||||
if wants_json(request):
|
||||
response = json_error(429, "Rate limit exceeded. Try again later.")
|
||||
response.headers["Retry-After"] = str(window)
|
||||
return response
|
||||
return HTMLResponse("Rate limit exceeded. Try again later.", status_code=429, headers=retry_after)
|
||||
_rate_limit_store[ip].append(now)
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
_MAINTENANCE_ALLOWED_PREFIXES = ("/static", "/avatar", "/auth", "/admin")
|
||||
_MAINTENANCE_ALLOWED_PREFIXES = ("/static", "/avatar", "/auth", "/admin", "/openai")
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
@@ -178,19 +219,26 @@ async def maintenance_middleware(request: Request, call_next):
|
||||
if user and user.get("role") == "Admin":
|
||||
return await call_next(request)
|
||||
message = get_setting("maintenance_message", "DevPlace is undergoing scheduled maintenance. Please check back shortly.")
|
||||
if wants_json(request):
|
||||
return json_error(503, message)
|
||||
seo_ctx = base_seo_context(request, title="Maintenance - DevPlace", description=message, robots="noindex")
|
||||
return templates.TemplateResponse(request, "error.html", {**seo_ctx, "request": request, "error_code": 503, "error_message": message}, status_code=503)
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
async def startup():
|
||||
init_db()
|
||||
from devplacepy.push import ensure_certificates
|
||||
ensure_certificates()
|
||||
with init_lock():
|
||||
init_db()
|
||||
from devplacepy.push import ensure_certificates
|
||||
ensure_certificates()
|
||||
service_manager.register(NewsService())
|
||||
service_manager.register(BotsService())
|
||||
service_manager.register(GatewayService())
|
||||
service_manager.register(DeviiService())
|
||||
if not os.environ.get("DEVPLACE_DISABLE_SERVICES"):
|
||||
if acquire_service_lock():
|
||||
service_manager.register(NewsService())
|
||||
asyncio.create_task(service_manager.start_all())
|
||||
service_manager.set_lock_owner(True)
|
||||
service_manager.supervise()
|
||||
logger.info(f"Background services started in worker pid {os.getpid()}")
|
||||
else:
|
||||
logger.info(f"Worker pid {os.getpid()} declined service lock; another worker owns background services")
|
||||
@@ -200,7 +248,7 @@ async def startup():
|
||||
@app.on_event("shutdown")
|
||||
async def shutdown():
|
||||
logger.info("Shutting down services...")
|
||||
await service_manager.stop_all()
|
||||
await service_manager.shutdown_all()
|
||||
|
||||
|
||||
@app.get("/")
|
||||
|
||||
+29
-8
@@ -3,6 +3,21 @@ from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
from devplacepy.constants import TOPICS, REACTION_EMOJI
|
||||
|
||||
|
||||
def normalize_poll_options(value):
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, str):
|
||||
value = [value]
|
||||
if not isinstance(value, list):
|
||||
return value
|
||||
if len(value) == 1 and isinstance(value[0], str):
|
||||
single = value[0]
|
||||
separator = "\n" if "\n" in single else ("," if "," in single else "")
|
||||
if separator:
|
||||
return [part.strip() for part in single.split(separator) if part.strip()]
|
||||
return value
|
||||
|
||||
|
||||
class SignupForm(BaseModel):
|
||||
username: str = Field(min_length=3, max_length=32)
|
||||
email: str = Field(min_length=1, max_length=255)
|
||||
@@ -73,17 +88,29 @@ class PostForm(BaseModel):
|
||||
def valid_topic(cls, value):
|
||||
return value if value in TOPICS else "random"
|
||||
|
||||
@field_validator("poll_options", mode="before")
|
||||
@classmethod
|
||||
def split_poll_options(cls, value):
|
||||
return normalize_poll_options(value)
|
||||
|
||||
|
||||
class PostEditForm(BaseModel):
|
||||
content: str = Field(min_length=10, max_length=2000)
|
||||
title: str = Field(default="", max_length=500)
|
||||
topic: str = "random"
|
||||
poll_question: str = Field(default="", max_length=200)
|
||||
poll_options: list[str] = []
|
||||
|
||||
@field_validator("topic")
|
||||
@classmethod
|
||||
def valid_topic(cls, value):
|
||||
return value if value in TOPICS else "random"
|
||||
|
||||
@field_validator("poll_options", mode="before")
|
||||
@classmethod
|
||||
def split_poll_options(cls, value):
|
||||
return normalize_poll_options(value)
|
||||
|
||||
|
||||
class CommentForm(BaseModel):
|
||||
content: str = Field(min_length=3, max_length=1000)
|
||||
@@ -127,7 +154,7 @@ class ProfileForm(BaseModel):
|
||||
class GistForm(BaseModel):
|
||||
title: str = Field(min_length=1, max_length=200)
|
||||
description: str = Field(default="", max_length=5000)
|
||||
source_code: str = Field(min_length=1, max_length=50000)
|
||||
source_code: str = Field(min_length=1, max_length=400000)
|
||||
language: str = Field(default="plaintext", max_length=50)
|
||||
attachment_uids: list[str] = []
|
||||
|
||||
@@ -135,7 +162,7 @@ class GistForm(BaseModel):
|
||||
class GistEditForm(BaseModel):
|
||||
title: str = Field(min_length=1, max_length=200)
|
||||
description: str = Field(default="", max_length=5000)
|
||||
source_code: str = Field(min_length=1, max_length=50000)
|
||||
source_code: str = Field(min_length=1, max_length=400000)
|
||||
language: str = Field(default="plaintext", max_length=50)
|
||||
|
||||
|
||||
@@ -183,17 +210,11 @@ class AdminSettingsForm(BaseModel):
|
||||
site_name: str = Field(default="", max_length=200)
|
||||
site_description: str = Field(default="", max_length=500)
|
||||
site_tagline: str = Field(default="", max_length=500)
|
||||
news_grade_threshold: str = Field(default="", max_length=10)
|
||||
news_api_url: str = Field(default="", max_length=500)
|
||||
news_ai_url: str = Field(default="", max_length=500)
|
||||
news_ai_model: str = Field(default="", max_length=200)
|
||||
news_ai_key: str = Field(default="", max_length=500)
|
||||
max_upload_size_mb: str = Field(default="", max_length=10)
|
||||
allowed_file_types: str = Field(default="", max_length=1000)
|
||||
max_attachments_per_resource: str = Field(default="", max_length=10)
|
||||
rate_limit_per_minute: str = Field(default="", max_length=10)
|
||||
rate_limit_window_seconds: str = Field(default="", max_length=10)
|
||||
news_service_interval: str = Field(default="", max_length=10)
|
||||
session_max_age_days: str = Field(default="", max_length=10)
|
||||
session_remember_days: str = Field(default="", max_length=10)
|
||||
registration_open: str = Field(default="", max_length=1)
|
||||
|
||||
+12
-3
@@ -1,6 +1,7 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import base64
|
||||
import fcntl
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -77,9 +78,17 @@ def generate_public_key() -> None:
|
||||
|
||||
|
||||
def ensure_certificates() -> None:
|
||||
generate_private_key()
|
||||
generate_pkcs8_private_key()
|
||||
generate_public_key()
|
||||
if VAPID_PRIVATE_KEY_FILE.exists() and VAPID_PRIVATE_KEY_PKCS8_FILE.exists() and VAPID_PUBLIC_KEY_FILE.exists():
|
||||
return
|
||||
lock_path = VAPID_PRIVATE_KEY_FILE.parent / ".vapid.lock"
|
||||
with open(lock_path, "w") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
try:
|
||||
generate_private_key()
|
||||
generate_pkcs8_private_key()
|
||||
generate_public_key()
|
||||
finally:
|
||||
fcntl.flock(lock, fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def hkdf(input_key: bytes, salt: bytes, info: bytes, length: int) -> bytes:
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import logging
|
||||
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse, RedirectResponse
|
||||
|
||||
from devplacepy.schemas import ActionResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def wants_json(request: Request) -> bool:
|
||||
# NOTE: `X-Requested-With: fetch` is intentionally NOT a trigger here. The browser
|
||||
# frontend sends it on form/engagement fetches and relies on the legacy HTML/redirect
|
||||
# behaviour (the four engagement endpoints handle that header themselves). JSON is
|
||||
# requested via the Accept or Content-Type header so existing flows are unchanged.
|
||||
content_type = request.headers.get("content-type", "")
|
||||
if content_type.startswith("application/json"):
|
||||
return True
|
||||
accept = request.headers.get("accept", "")
|
||||
return "application/json" in accept and "text/html" not in accept
|
||||
|
||||
|
||||
def respond(request: Request, template: str, context: dict, *, model):
|
||||
if wants_json(request):
|
||||
try:
|
||||
payload = model.model_validate(context).model_dump(mode="json")
|
||||
except Exception as exc:
|
||||
logger.warning("JSON serialization failed for %s: %s", template, exc)
|
||||
return json_error(500, "Could not serialize response")
|
||||
return JSONResponse(payload)
|
||||
from devplacepy.templating import templates
|
||||
return templates.TemplateResponse(request, template, context)
|
||||
|
||||
|
||||
def action_result(request: Request, redirect_url: str, *, data=None, status_code: int = 302):
|
||||
if wants_json(request):
|
||||
return JSONResponse(ActionResult(ok=True, redirect=redirect_url, data=data).model_dump(mode="json"))
|
||||
return RedirectResponse(url=redirect_url, status_code=status_code)
|
||||
|
||||
|
||||
def json_error(status_code: int, message: str, **extra) -> JSONResponse:
|
||||
return JSONResponse({"error": {"status": status_code, "message": message, **extra}}, status_code=status_code)
|
||||
+64
-20
@@ -2,11 +2,16 @@ import logging
|
||||
from typing import Annotated
|
||||
from fastapi import APIRouter, Request, Form
|
||||
from devplacepy.models import AdminRoleForm, AdminPasswordForm, AdminSettingsForm
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse
|
||||
from devplacepy.database import get_table, build_pagination, get_post_counts_by_user_uids, get_news_images_by_uids, clear_settings_cache
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
|
||||
from devplacepy.database import get_table, build_pagination, get_post_counts_by_user_uids, get_news_images_by_uids, clear_settings_cache, get_platform_analytics
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import require_admin, hash_password, generate_uid, time_ago, clear_user_cache
|
||||
from devplacepy.utils import require_admin, hash_password, generate_uid, time_ago, clear_user_cache, get_current_user, is_admin
|
||||
from devplacepy.seo import base_seo_context, site_url, website_schema
|
||||
from devplacepy.responses import respond, action_result
|
||||
from devplacepy.schemas import AdminUsersOut, AdminNewsOut, AdminSettingsOut, GatewayUsageOut
|
||||
from devplacepy.services.manager import service_manager
|
||||
from devplacepy.services.openai_gateway.analytics import build_analytics
|
||||
from devplacepy.services.openai_gateway.usage import pricing_from_cfg
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -15,7 +20,46 @@ router = APIRouter()
|
||||
@router.get("", response_class=HTMLResponse)
|
||||
async def admin_index(request: Request):
|
||||
require_admin(request)
|
||||
return RedirectResponse(url="/admin/users", status_code=302)
|
||||
return action_result(request, "/admin/users")
|
||||
|
||||
|
||||
@router.get("/analytics")
|
||||
async def admin_analytics(request: Request, top_n: int = 10):
|
||||
if not is_admin(get_current_user(request)):
|
||||
return JSONResponse({"error": "Admin access required"}, status_code=403)
|
||||
return JSONResponse(get_platform_analytics(top_n))
|
||||
|
||||
|
||||
@router.get("/ai-usage", response_class=HTMLResponse)
|
||||
async def admin_ai_usage(request: Request):
|
||||
admin = require_admin(request)
|
||||
base = site_url(request)
|
||||
seo_ctx = base_seo_context(
|
||||
request,
|
||||
title="AI usage - Admin",
|
||||
description="AI gateway token usage, cost, latency, and reliability metrics.",
|
||||
breadcrumbs=[
|
||||
{"name": "Home", "url": "/feed"},
|
||||
{"name": "Admin", "url": "/admin"},
|
||||
{"name": "AI usage", "url": "/admin/ai-usage"},
|
||||
],
|
||||
schemas=[website_schema(base)],
|
||||
)
|
||||
return respond(request, "ai_usage.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": admin,
|
||||
"admin_section": "ai-usage",
|
||||
}, model=GatewayUsageOut)
|
||||
|
||||
|
||||
@router.get("/ai-usage/data")
|
||||
async def admin_ai_usage_data(request: Request, hours: int = 48, top_n: int = 10):
|
||||
if not is_admin(get_current_user(request)):
|
||||
return JSONResponse({"error": "Admin access required"}, status_code=403)
|
||||
service = service_manager.get_service("openai")
|
||||
pricing = pricing_from_cfg(service.get_config()) if service is not None else None
|
||||
return JSONResponse(build_analytics(hours, top_n=top_n, pricing=pricing))
|
||||
|
||||
|
||||
@router.get("/users", response_class=HTMLResponse)
|
||||
@@ -41,14 +85,14 @@ async def admin_users(request: Request, page: int = 1):
|
||||
],
|
||||
schemas=[website_schema(base)],
|
||||
)
|
||||
return templates.TemplateResponse(request, "admin_users.html", {
|
||||
return respond(request, "admin_users.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": admin,
|
||||
"users": page_users,
|
||||
"pagination": pagination,
|
||||
"admin_section": "users",
|
||||
})
|
||||
}, model=AdminUsersOut)
|
||||
|
||||
|
||||
@router.post("/users/{uid}/role")
|
||||
@@ -56,12 +100,12 @@ async def admin_user_role(request: Request, uid: str, data: Annotated[AdminRoleF
|
||||
admin = require_admin(request)
|
||||
role = data.role.capitalize()
|
||||
if uid == admin["uid"]:
|
||||
return RedirectResponse(url="/admin/users", status_code=302)
|
||||
return action_result(request, "/admin/users")
|
||||
users = get_table("users")
|
||||
users.update({"uid": uid, "role": role}, ["uid"])
|
||||
clear_user_cache(uid)
|
||||
logger.info(f"Admin {admin['username']} set user {uid} role to {role}")
|
||||
return RedirectResponse(url="/admin/users", status_code=302)
|
||||
return action_result(request, "/admin/users")
|
||||
|
||||
|
||||
@router.post("/users/{uid}/password")
|
||||
@@ -70,14 +114,14 @@ async def admin_user_password(request: Request, uid: str, data: Annotated[AdminP
|
||||
users = get_table("users")
|
||||
users.update({"uid": uid, "password_hash": hash_password(data.password)}, ["uid"])
|
||||
logger.info(f"Admin {admin['username']} changed password for user {uid}")
|
||||
return RedirectResponse(url="/admin/users", status_code=302)
|
||||
return action_result(request, "/admin/users")
|
||||
|
||||
|
||||
@router.post("/users/{uid}/toggle")
|
||||
async def admin_user_toggle(request: Request, uid: str):
|
||||
admin = require_admin(request)
|
||||
if uid == admin["uid"]:
|
||||
return RedirectResponse(url="/admin/users", status_code=302)
|
||||
return action_result(request, "/admin/users")
|
||||
users = get_table("users")
|
||||
user = users.find_one(uid=uid)
|
||||
if user:
|
||||
@@ -85,7 +129,7 @@ async def admin_user_toggle(request: Request, uid: str):
|
||||
users.update({"uid": uid, "is_active": new_state}, ["uid"])
|
||||
clear_user_cache(uid)
|
||||
logger.info(f"Admin {admin['username']} {'disabled' if not new_state else 'enabled'} user {uid}")
|
||||
return RedirectResponse(url="/admin/users", status_code=302)
|
||||
return action_result(request, "/admin/users")
|
||||
|
||||
|
||||
@router.get("/settings", response_class=HTMLResponse)
|
||||
@@ -105,13 +149,13 @@ async def admin_settings(request: Request):
|
||||
],
|
||||
schemas=[website_schema(base)],
|
||||
)
|
||||
return templates.TemplateResponse(request, "admin_settings.html", {
|
||||
return respond(request, "admin_settings.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": admin,
|
||||
"settings": all_settings,
|
||||
"admin_section": "settings",
|
||||
})
|
||||
}, model=AdminSettingsOut)
|
||||
|
||||
|
||||
@router.get("/news", response_class=HTMLResponse)
|
||||
@@ -146,14 +190,14 @@ async def admin_news(request: Request, page: int = 1):
|
||||
],
|
||||
schemas=[website_schema(base)],
|
||||
)
|
||||
return templates.TemplateResponse(request, "admin_news.html", {
|
||||
return respond(request, "admin_news.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": admin,
|
||||
"articles": enriched,
|
||||
"pagination": pagination,
|
||||
"admin_section": "news",
|
||||
})
|
||||
}, model=AdminNewsOut)
|
||||
|
||||
|
||||
@router.post("/news/{uid}/toggle")
|
||||
@@ -165,7 +209,7 @@ async def admin_news_toggle(request: Request, uid: str):
|
||||
current = article.get("featured", 0)
|
||||
news_table.update({"uid": uid, "featured": 0 if current else 1}, ["uid"])
|
||||
logger.info(f"Admin {admin['username']} toggled news {uid} featured={'off' if current else 'on'}")
|
||||
return RedirectResponse(url="/admin/news", status_code=302)
|
||||
return action_result(request, "/admin/news")
|
||||
|
||||
|
||||
@router.post("/news/{uid}/publish")
|
||||
@@ -178,7 +222,7 @@ async def admin_news_publish(request: Request, uid: str):
|
||||
new_status = "draft" if current == "published" else "published"
|
||||
news_table.update({"uid": uid, "status": new_status}, ["uid"])
|
||||
logger.info(f"Admin {admin['username']} set news {uid} status={new_status}")
|
||||
return RedirectResponse(url="/admin/news", status_code=302)
|
||||
return action_result(request, "/admin/news")
|
||||
|
||||
|
||||
@router.post("/news/{uid}/landing")
|
||||
@@ -190,7 +234,7 @@ async def admin_news_landing(request: Request, uid: str):
|
||||
current = article.get("show_on_landing", 0)
|
||||
news_table.update({"uid": uid, "show_on_landing": 0 if current else 1}, ["uid"])
|
||||
logger.info(f"Admin {admin['username']} toggled news {uid} landing={'on' if not current else 'off'}")
|
||||
return RedirectResponse(url="/admin/news", status_code=302)
|
||||
return action_result(request, "/admin/news")
|
||||
|
||||
|
||||
@router.post("/news/{uid}/delete")
|
||||
@@ -204,7 +248,7 @@ async def admin_news_delete(request: Request, uid: str):
|
||||
images_table.delete(id=img["id"])
|
||||
news_table.delete(id=article["id"])
|
||||
logger.info(f"Admin {admin['username']} deleted news {uid}")
|
||||
return RedirectResponse(url="/admin/news", status_code=302)
|
||||
return action_result(request, "/admin/news")
|
||||
|
||||
|
||||
@router.post("/settings")
|
||||
@@ -221,4 +265,4 @@ async def admin_settings_save(request: Request, data: Annotated[AdminSettingsFor
|
||||
settings.insert({"uid": generate_uid(), "key": key, "value": value})
|
||||
clear_settings_cache()
|
||||
logger.info(f"Admin {admin['username']} updated settings")
|
||||
return RedirectResponse(url="/admin/settings", status_code=302)
|
||||
return action_result(request, "/admin/settings")
|
||||
|
||||
+34
-22
@@ -4,13 +4,15 @@ import logging
|
||||
from typing import Annotated
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from fastapi import APIRouter, Request, Form
|
||||
from fastapi.responses import RedirectResponse, HTMLResponse
|
||||
from fastapi.responses import RedirectResponse, HTMLResponse, JSONResponse
|
||||
from devplacepy.database import get_table, get_setting, get_int_setting
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import hash_password, verify_password, create_session, generate_uid, get_current_user, award_badge, clear_session_cache, safe_next
|
||||
from devplacepy.seo import base_seo_context
|
||||
from devplacepy.models import SignupForm, LoginForm, ForgotPasswordForm, ResetPasswordForm
|
||||
from devplacepy.config import SECONDS_PER_DAY
|
||||
from devplacepy.responses import respond, action_result, wants_json, json_error
|
||||
from devplacepy.schemas import AuthPageOut
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -20,7 +22,7 @@ router = APIRouter()
|
||||
async def signup_page(request: Request):
|
||||
user = get_current_user(request)
|
||||
if user:
|
||||
return RedirectResponse(url="/feed", status_code=302)
|
||||
return action_result(request, "/feed")
|
||||
seo_ctx = base_seo_context(
|
||||
request,
|
||||
title="Join DevPlace",
|
||||
@@ -28,21 +30,21 @@ async def signup_page(request: Request):
|
||||
robots="noindex,nofollow",
|
||||
)
|
||||
registration_closed = get_setting("registration_open", "1") != "1"
|
||||
return templates.TemplateResponse(request, "signup.html", {**seo_ctx, "request": request, "registration_closed": registration_closed})
|
||||
return respond(request, "signup.html", {**seo_ctx, "request": request, "page": "signup", "registration_closed": registration_closed}, model=AuthPageOut)
|
||||
|
||||
|
||||
@router.get("/login", response_class=HTMLResponse)
|
||||
async def login_page(request: Request, next: str | None = None):
|
||||
user = get_current_user(request)
|
||||
if user:
|
||||
return RedirectResponse(url=safe_next(next), status_code=302)
|
||||
return action_result(request, safe_next(next))
|
||||
seo_ctx = base_seo_context(
|
||||
request,
|
||||
title="Sign In",
|
||||
description="Sign in to DevPlace to connect with developers.",
|
||||
robots="noindex,nofollow",
|
||||
)
|
||||
return templates.TemplateResponse(request, "login.html", {**seo_ctx, "request": request, "next_url": safe_next(next, "")})
|
||||
return respond(request, "login.html", {**seo_ctx, "request": request, "page": "login", "next_url": safe_next(next, "")}, model=AuthPageOut)
|
||||
|
||||
|
||||
@router.post("/signup")
|
||||
@@ -52,6 +54,8 @@ async def signup(request: Request, data: Annotated[SignupForm, Form()]):
|
||||
password = data.password
|
||||
|
||||
if get_setting("registration_open", "1") != "1":
|
||||
if wants_json(request):
|
||||
return json_error(403, "Registration is closed")
|
||||
seo_ctx = base_seo_context(request, title="Join DevPlace", robots="noindex,nofollow")
|
||||
return templates.TemplateResponse(
|
||||
request, "signup.html",
|
||||
@@ -66,6 +70,8 @@ async def signup(request: Request, data: Annotated[SignupForm, Form()]):
|
||||
errors.append("Email already registered")
|
||||
|
||||
if errors:
|
||||
if wants_json(request):
|
||||
return json_error(400, "; ".join(errors), errors=errors)
|
||||
seo_ctx = base_seo_context(request, title="Join DevPlace", robots="noindex,nofollow")
|
||||
return templates.TemplateResponse(
|
||||
request, "signup.html",
|
||||
@@ -78,6 +84,7 @@ async def signup(request: Request, data: Annotated[SignupForm, Form()]):
|
||||
"uid": uid,
|
||||
"username": username,
|
||||
"email": email,
|
||||
"api_key": generate_uid(),
|
||||
"password_hash": hash_password(password),
|
||||
"bio": "",
|
||||
"location": "",
|
||||
@@ -95,7 +102,7 @@ async def signup(request: Request, data: Annotated[SignupForm, Form()]):
|
||||
|
||||
max_age = max(1, get_int_setting("session_max_age_days", 7)) * SECONDS_PER_DAY
|
||||
token = create_session(uid, max_age)
|
||||
response = RedirectResponse(url="/feed", status_code=302)
|
||||
response = action_result(request, "/feed", data={"username": username})
|
||||
response.set_cookie(key="session", value=token, max_age=max_age, httponly=True, samesite="lax")
|
||||
logger.info(f"User {username} signed up")
|
||||
return response
|
||||
@@ -115,6 +122,8 @@ async def login(request: Request, data: Annotated[LoginForm, Form()]):
|
||||
errors.append("Invalid email or password")
|
||||
|
||||
if errors:
|
||||
if wants_json(request):
|
||||
return json_error(401, "; ".join(errors), errors=errors)
|
||||
seo_ctx = base_seo_context(request, title="Sign In", robots="noindex,nofollow")
|
||||
return templates.TemplateResponse(
|
||||
request, "login.html",
|
||||
@@ -124,7 +133,7 @@ async def login(request: Request, data: Annotated[LoginForm, Form()]):
|
||||
days = get_int_setting("session_remember_days", 30) if remember_me else get_int_setting("session_max_age_days", 7)
|
||||
max_age = max(1, days) * SECONDS_PER_DAY
|
||||
token = create_session(user["uid"], max_age)
|
||||
response = RedirectResponse(url=safe_next(data.next), status_code=302)
|
||||
response = action_result(request, safe_next(data.next), data={"username": user["username"]})
|
||||
response.set_cookie(key="session", value=token, max_age=max_age, httponly=True, samesite="lax")
|
||||
logger.info(f"User {user['username']} logged in")
|
||||
return response
|
||||
@@ -137,7 +146,7 @@ async def forgot_password_page(request: Request):
|
||||
title="Reset Password",
|
||||
robots="noindex,nofollow",
|
||||
)
|
||||
return templates.TemplateResponse(request, "forgot_password.html", {**seo_ctx, "request": request})
|
||||
return respond(request, "forgot_password.html", {**seo_ctx, "request": request, "page": "forgot-password"}, model=AuthPageOut)
|
||||
|
||||
|
||||
@router.post("/forgot-password")
|
||||
@@ -161,6 +170,8 @@ async def forgot_password(request: Request, data: Annotated[ForgotPasswordForm,
|
||||
})
|
||||
logger.info(f"Password reset requested for {email}")
|
||||
|
||||
if wants_json(request):
|
||||
return JSONResponse({"ok": True, "sent": True})
|
||||
return templates.TemplateResponse(request, "forgot_password.html", {
|
||||
**seo_ctx, "request": request, "sent": True,
|
||||
})
|
||||
@@ -173,9 +184,9 @@ async def reset_password_page(request: Request, token: str):
|
||||
title="Set New Password",
|
||||
robots="noindex,nofollow",
|
||||
)
|
||||
return templates.TemplateResponse(request, "reset_password.html", {
|
||||
**seo_ctx, "request": request, "token": token,
|
||||
})
|
||||
return respond(request, "reset_password.html", {
|
||||
**seo_ctx, "request": request, "page": "reset-password", "token": token,
|
||||
}, model=AuthPageOut)
|
||||
|
||||
|
||||
@router.post("/reset-password/{token}")
|
||||
@@ -188,16 +199,17 @@ async def reset_password(request: Request, token: str, data: Annotated[ResetPass
|
||||
token_hash = hashlib.sha256(token.encode()).hexdigest()
|
||||
matched = resets.find_one(token=token_hash, used=False)
|
||||
|
||||
if not matched:
|
||||
errors.append("Invalid or expired reset token")
|
||||
return templates.TemplateResponse(request, "reset_password.html", {
|
||||
"request": request, "token": token, "errors": errors,
|
||||
})
|
||||
expires = datetime.fromisoformat(matched["expires_at"])
|
||||
if expires.tzinfo is None:
|
||||
expires = expires.replace(tzinfo=timezone.utc)
|
||||
if expires < datetime.now(timezone.utc):
|
||||
expired = False
|
||||
if matched:
|
||||
expires = datetime.fromisoformat(matched["expires_at"])
|
||||
if expires.tzinfo is None:
|
||||
expires = expires.replace(tzinfo=timezone.utc)
|
||||
expired = expires < datetime.now(timezone.utc)
|
||||
|
||||
if not matched or expired:
|
||||
errors.append("Invalid or expired reset token")
|
||||
if wants_json(request):
|
||||
return json_error(400, errors[0], errors=errors)
|
||||
return templates.TemplateResponse(request, "reset_password.html", {
|
||||
"request": request, "token": token, "errors": errors,
|
||||
})
|
||||
@@ -205,7 +217,7 @@ async def reset_password(request: Request, token: str, data: Annotated[ResetPass
|
||||
users.update({"uid": matched["user_uid"], "password_hash": hash_password(password)}, ["uid"])
|
||||
resets.update({"id": matched["id"], "used": True}, ["id"])
|
||||
logger.info(f"Password reset completed for user {matched['user_uid']}")
|
||||
return RedirectResponse(url="/auth/login", status_code=302)
|
||||
return action_result(request, "/auth/login")
|
||||
|
||||
|
||||
@router.get("/logout")
|
||||
@@ -217,6 +229,6 @@ async def logout(request: Request):
|
||||
if session:
|
||||
sessions.delete(id=session["id"])
|
||||
clear_session_cache(token)
|
||||
response = RedirectResponse(url="/", status_code=302)
|
||||
response = action_result(request, "/")
|
||||
response.delete_cookie("session")
|
||||
return response
|
||||
|
||||
@@ -7,6 +7,8 @@ from devplacepy.database import get_table, db, paginate, resolve_object_url
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import generate_uid, require_user, time_ago
|
||||
from devplacepy.seo import base_seo_context
|
||||
from devplacepy.responses import respond
|
||||
from devplacepy.schemas import SavedOut
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -52,13 +54,13 @@ async def saved_page(request: Request, before: str = None):
|
||||
})
|
||||
|
||||
seo_ctx = base_seo_context(request, title="Saved", description="Your saved content on DevPlace.", robots="noindex,nofollow")
|
||||
return templates.TemplateResponse(request, "saved.html", {
|
||||
return respond(request, "saved.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": user,
|
||||
"items": items,
|
||||
"next_cursor": next_cursor,
|
||||
})
|
||||
}, model=SavedOut)
|
||||
|
||||
|
||||
@router.post("/{target_type}/{target_uid}")
|
||||
|
||||
@@ -9,6 +9,8 @@ from devplacepy.attachments import get_attachments_batch, link_attachments
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import generate_uid, require_user, time_ago, get_current_user, create_mention_notifications
|
||||
from devplacepy.seo import base_seo_context
|
||||
from devplacepy.responses import respond, action_result
|
||||
from devplacepy.schemas import BugsOut
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -45,12 +47,12 @@ async def bugs_page(request: Request):
|
||||
{"name": "Bug Reports", "url": "/bugs"},
|
||||
],
|
||||
)
|
||||
return templates.TemplateResponse(request, "bugs.html", {
|
||||
return respond(request, "bugs.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": user,
|
||||
"bugs": bug_list,
|
||||
})
|
||||
}, model=BugsOut)
|
||||
|
||||
|
||||
@router.post("/create")
|
||||
@@ -75,4 +77,4 @@ async def create_bug(request: Request, data: Annotated[BugForm, Form()]):
|
||||
|
||||
create_mention_notifications(description, user["uid"], f"/bugs?highlight={bug_uid}")
|
||||
logger.info(f"Bug report created by {user['username']}: {title}")
|
||||
return RedirectResponse(url="/bugs", status_code=302)
|
||||
return action_result(request, "/bugs", data={"uid": bug_uid})
|
||||
|
||||
@@ -8,6 +8,7 @@ from devplacepy.attachments import link_attachments, delete_target_attachments
|
||||
from devplacepy.content import is_owner
|
||||
from devplacepy.utils import generate_uid, require_user, create_mention_notifications, create_notification, award_rewards, XP_COMMENT
|
||||
from devplacepy.models import CommentForm
|
||||
from devplacepy.responses import action_result, wants_json, json_error
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -59,7 +60,7 @@ async def create_comment(request: Request, data: Annotated[CommentForm, Form()])
|
||||
|
||||
create_mention_notifications(content, user["uid"], comment_url)
|
||||
logger.info(f"Comment by {user['username']} on {target_type} {target_uid}")
|
||||
return RedirectResponse(url=comment_url, status_code=302)
|
||||
return action_result(request, comment_url, data={"uid": comment_uid, "url": comment_url})
|
||||
|
||||
|
||||
@router.post("/delete/{comment_uid}")
|
||||
@@ -68,6 +69,8 @@ async def delete_comment(request: Request, comment_uid: str):
|
||||
comments = get_table("comments")
|
||||
comment = comments.find_one(uid=comment_uid)
|
||||
if not is_owner(comment, user):
|
||||
if wants_json(request):
|
||||
return json_error(403, "Not allowed")
|
||||
return RedirectResponse(url="/feed", status_code=302)
|
||||
target_type = comment.get("target_type", "post")
|
||||
target_uid = comment.get("target_uid") or comment.get("post_uid", "")
|
||||
@@ -77,4 +80,4 @@ async def delete_comment(request: Request, comment_uid: str):
|
||||
comments.delete(id=comment["id"])
|
||||
logger.info(f"Comment {comment_uid} deleted by {user['username']}")
|
||||
redirect_url = resolve_object_url(target_type, target_uid)
|
||||
return RedirectResponse(url=redirect_url, status_code=302)
|
||||
return action_result(request, redirect_url)
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
import uuid_utils
|
||||
from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect
|
||||
from fastapi.responses import JSONResponse, RedirectResponse
|
||||
|
||||
from devplacepy.database import get_int_setting
|
||||
from devplacepy.services.manager import service_manager
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import _user_from_api_key, _user_from_session, get_current_user
|
||||
|
||||
logger = logging.getLogger("devii.router")
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
GUEST_COOKIE = "devii_guest"
|
||||
GUEST_COOKIE_MAX_AGE = 31536000
|
||||
|
||||
|
||||
def _service():
|
||||
return service_manager.get_service("devii")
|
||||
|
||||
|
||||
def _bearer_key(headers) -> str:
|
||||
key = headers.get("x-api-key", "")
|
||||
if key:
|
||||
return key.strip()
|
||||
scheme, _, credentials = headers.get("authorization", "").partition(" ")
|
||||
if scheme.lower() == "bearer":
|
||||
return credentials.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _resolve_ws_owner(websocket: WebSocket):
|
||||
user = _user_from_session(websocket)
|
||||
if not user:
|
||||
key = _bearer_key(websocket.headers)
|
||||
if key:
|
||||
user = _user_from_api_key(key)
|
||||
if user:
|
||||
return "user", user["uid"], user.get("username", ""), user.get("api_key", "")
|
||||
guest_id = websocket.cookies.get(GUEST_COOKIE) or uuid_utils.uuid7().hex
|
||||
return "guest", guest_id, "guest", ""
|
||||
|
||||
|
||||
def _owner_from_request(request: Request):
|
||||
user = get_current_user(request)
|
||||
if user:
|
||||
return "user", user["uid"], user.get("username", "")
|
||||
guest_id = request.cookies.get(GUEST_COOKIE) or ""
|
||||
return "guest", guest_id, "guest"
|
||||
|
||||
|
||||
@router.get("/")
|
||||
async def devii_page(request: Request):
|
||||
user = get_current_user(request)
|
||||
response = templates.TemplateResponse(
|
||||
request, "devii.html", {"request": request, "user": user, "page_title": "Devii", "meta_robots": "noindex,nofollow"}
|
||||
)
|
||||
if not user and not request.cookies.get(GUEST_COOKIE):
|
||||
response.set_cookie(GUEST_COOKIE, uuid_utils.uuid7().hex, httponly=True, samesite="lax", max_age=GUEST_COOKIE_MAX_AGE)
|
||||
return response
|
||||
|
||||
|
||||
@router.get("/session")
|
||||
async def devii_session(request: Request):
|
||||
user = get_current_user(request)
|
||||
svc = _service()
|
||||
payload = {
|
||||
"loggedIn": bool(user),
|
||||
"username": user.get("username") if user else "guest",
|
||||
"enabled": bool(svc and svc.is_enabled()),
|
||||
"guestsEnabled": bool(svc and svc.guests_enabled()),
|
||||
}
|
||||
response = JSONResponse(payload)
|
||||
if not user and not request.cookies.get(GUEST_COOKIE):
|
||||
response.set_cookie(GUEST_COOKIE, uuid_utils.uuid7().hex, httponly=True, samesite="lax", max_age=GUEST_COOKIE_MAX_AGE)
|
||||
return response
|
||||
|
||||
|
||||
def _safe_next(value: str) -> str:
|
||||
if value.startswith("/") and not value.startswith("//"):
|
||||
return value
|
||||
return "/"
|
||||
|
||||
|
||||
@router.get("/adopt")
|
||||
async def devii_adopt(request: Request):
|
||||
# The terminal's agent logged in; adopt the real session it minted into the browser so both
|
||||
# share one session, then return to where the user was.
|
||||
target = _safe_next(request.query_params.get("next", "/"))
|
||||
response = RedirectResponse(target, status_code=303)
|
||||
svc = _service()
|
||||
if svc is None:
|
||||
return response
|
||||
owner_kind, owner_id, _ = _owner_from_request(request)
|
||||
session = svc.hub().find(owner_kind, owner_id) if owner_id else None
|
||||
token = session.take_pending_session() if session else None
|
||||
if token:
|
||||
max_age = max(1, get_int_setting("session_remember_days", 30)) * 86400
|
||||
response.set_cookie("session", token, max_age=max_age, httponly=True, samesite="lax")
|
||||
return response
|
||||
|
||||
|
||||
@router.get("/usage")
|
||||
async def devii_usage(request: Request):
|
||||
svc = _service()
|
||||
owner_kind, owner_id, _ = _owner_from_request(request)
|
||||
if svc is None:
|
||||
return JSONResponse({"spent_24h": 0.0, "limit": 0.0, "turns_today": 0})
|
||||
spent = svc.spent_24h(owner_kind, owner_id) if owner_id else 0.0
|
||||
turns = svc.hub().ledger.turns_24h(owner_kind, owner_id) if owner_id else 0
|
||||
return JSONResponse({
|
||||
"spent_24h": round(spent, 6),
|
||||
"limit": svc.daily_limit_for(owner_kind),
|
||||
"turns_today": turns,
|
||||
"owner_kind": owner_kind,
|
||||
})
|
||||
|
||||
|
||||
@router.post("/clippy/ai/chat")
|
||||
async def clippy_proxy(request: Request):
|
||||
svc = _service()
|
||||
if svc is None or not svc.is_enabled():
|
||||
return JSONResponse({"error": "Devii is unavailable"}, status_code=503)
|
||||
cfg = svc.effective_config()
|
||||
body = await request.body()
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if cfg.get("devii_ai_key"):
|
||||
headers["Authorization"] = f"Bearer {cfg['devii_ai_key']}"
|
||||
async with httpx.AsyncClient(timeout=45.0) as client:
|
||||
upstream = await client.post(cfg["devii_ai_url"], content=body, headers=headers)
|
||||
return JSONResponse(
|
||||
content=upstream.json() if upstream.headers.get("content-type", "").startswith("application/json") else {"raw": upstream.text},
|
||||
status_code=upstream.status_code,
|
||||
)
|
||||
|
||||
|
||||
@router.websocket("/ws")
|
||||
async def devii_ws(websocket: WebSocket):
|
||||
await websocket.accept()
|
||||
svc = _service()
|
||||
if svc is None or not svc.is_enabled():
|
||||
await websocket.close(code=1013)
|
||||
return
|
||||
if not service_manager.owns_lock():
|
||||
await websocket.close(code=1013)
|
||||
return
|
||||
owner_kind, owner_id, username, api_key = _resolve_ws_owner(websocket)
|
||||
if owner_kind == "guest" and not svc.guests_enabled():
|
||||
await websocket.send_json({"type": "error", "text": "Guest access to Devii is disabled."})
|
||||
await websocket.close(code=1008)
|
||||
return
|
||||
|
||||
session = svc.hub().get_or_create(owner_kind, owner_id, username, api_key, svc.instance_base_url())
|
||||
session.attach(websocket)
|
||||
try:
|
||||
await session.send_bootstrap(websocket)
|
||||
while True:
|
||||
data = await websocket.receive_json()
|
||||
kind = data.get("type")
|
||||
if kind == "input":
|
||||
text = str(data.get("text", "")).strip()
|
||||
if not text:
|
||||
continue
|
||||
limit = svc.daily_limit_for(owner_kind)
|
||||
spent = svc.spent_24h(owner_kind, owner_id)
|
||||
if spent >= limit:
|
||||
await websocket.send_json({
|
||||
"type": "error",
|
||||
"text": f"Daily limit reached (${spent:.4f} of ${limit:.2f}). Try again later.",
|
||||
})
|
||||
continue
|
||||
session.spawn_turn(text)
|
||||
elif kind == "reset":
|
||||
await session.reset_conversation()
|
||||
elif kind in ("avatar_result", "client_result"):
|
||||
session.resolve_query(str(data.get("id", "")), data.get("result"))
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
except Exception: # noqa: BLE001 - never let the socket loop crash the worker
|
||||
logger.exception("Devii websocket loop failed for %s/%s", owner_kind, owner_id)
|
||||
finally:
|
||||
session.detach(websocket)
|
||||
@@ -0,0 +1,170 @@
|
||||
import logging
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse, Response
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import get_current_user, not_found, is_admin as user_is_admin
|
||||
from devplacepy.seo import base_seo_context, site_url, website_schema
|
||||
from devplacepy.docs_api import api_doc_pages, render_group, build_services_group
|
||||
from devplacepy.services.manager import service_manager
|
||||
from devplacepy import docs_search, docs_export, docs_live
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
SECTION_GENERAL = "General"
|
||||
SECTION_API = "API"
|
||||
SECTION_ADMIN = "Administration"
|
||||
SECTION_DEVII = "Devii internals"
|
||||
SECTION_PROD = "Production"
|
||||
|
||||
DOCS_PAGES = [
|
||||
# General - how to use the site and Devii (everyone)
|
||||
{"slug": "index", "title": "Overview", "kind": "prose", "section": SECTION_GENERAL},
|
||||
{"slug": "devii", "title": "Devii Assistant", "kind": "prose", "section": SECTION_GENERAL},
|
||||
{"slug": "dashboard", "title": "Your dashboard", "kind": "live", "section": SECTION_GENERAL},
|
||||
# API - developer reference (everyone); admin-only groups are routed to Administration below
|
||||
{"slug": "authentication", "title": "Authentication", "kind": "prose", "section": SECTION_API},
|
||||
*[{**page, "section": (SECTION_ADMIN if page.get("admin") else SECTION_API)} for page in api_doc_pages()],
|
||||
# Devii internals - technical reference for the Devii assistant (admins only)
|
||||
{"slug": "devii-internals", "title": "Devii internals", "kind": "prose", "admin": True, "section": SECTION_DEVII},
|
||||
{"slug": "devii-architecture", "title": "Architecture and sessions", "kind": "prose", "admin": True, "section": SECTION_DEVII},
|
||||
{"slug": "devii-tools", "title": "Tools and scopes", "kind": "prose", "admin": True, "section": SECTION_DEVII},
|
||||
{"slug": "devii-data", "title": "Data and persistence", "kind": "prose", "admin": True, "section": SECTION_DEVII},
|
||||
{"slug": "devii-security", "title": "Security and limits", "kind": "prose", "admin": True, "section": SECTION_DEVII},
|
||||
{"slug": "devii-config", "title": "Configuration and CLI", "kind": "prose", "admin": True, "section": SECTION_DEVII},
|
||||
# Production - deployment and operations reference (admins only)
|
||||
{"slug": "production", "title": "Production overview", "kind": "prose", "admin": True, "section": SECTION_PROD},
|
||||
{"slug": "production-deploy", "title": "Deploy and update", "kind": "prose", "admin": True, "section": SECTION_PROD},
|
||||
{"slug": "production-nginx", "title": "nginx and networking", "kind": "prose", "admin": True, "section": SECTION_PROD},
|
||||
{"slug": "production-concurrency", "title": "Multi-worker and concurrency", "kind": "prose", "admin": True, "section": SECTION_PROD},
|
||||
]
|
||||
_PAGES_BY_SLUG = {page["slug"]: page for page in DOCS_PAGES}
|
||||
|
||||
|
||||
@router.get("/docs")
|
||||
async def docs_root(request: Request):
|
||||
return RedirectResponse(url="/docs/index.html", status_code=302)
|
||||
|
||||
|
||||
def _export_context(request: Request):
|
||||
user = get_current_user(request)
|
||||
is_admin = user_is_admin(user)
|
||||
return (
|
||||
is_admin,
|
||||
site_url(request),
|
||||
user.get("username") if user else "",
|
||||
user.get("api_key") if user else "",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/docs/download.md")
|
||||
async def docs_download_md(request: Request):
|
||||
is_admin, base, username, api_key = _export_context(request)
|
||||
markdown = docs_export.build_markdown(is_admin, base, username, api_key)
|
||||
return Response(
|
||||
content=markdown,
|
||||
media_type="text/markdown; charset=utf-8",
|
||||
headers={"Content-Disposition": 'attachment; filename="devplace-docs.md"'},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/docs/download.html")
|
||||
async def docs_download_html(request: Request):
|
||||
is_admin, base, username, api_key = _export_context(request)
|
||||
markdown = docs_export.build_markdown(is_admin, base, username, api_key)
|
||||
return HTMLResponse(
|
||||
content=docs_export.build_html(markdown),
|
||||
headers={"Content-Disposition": 'attachment; filename="devplace-docs.html"'},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/docs/{slug}.html", response_class=HTMLResponse)
|
||||
async def docs_page(request: Request, slug: str):
|
||||
user = get_current_user(request)
|
||||
is_admin = user_is_admin(user)
|
||||
base = site_url(request)
|
||||
visible_pages = [p for p in DOCS_PAGES if not p.get("admin") or is_admin]
|
||||
|
||||
if slug == "search":
|
||||
query = request.query_params.get("q", "")
|
||||
results = docs_search.search(query, user=user, is_admin=is_admin)
|
||||
seo_ctx = base_seo_context(
|
||||
request,
|
||||
title="Search - Documentation",
|
||||
description="Search the DevPlace developer documentation.",
|
||||
robots="noindex",
|
||||
breadcrumbs=[
|
||||
{"name": "Home", "url": "/feed"},
|
||||
{"name": "Docs", "url": "/docs/index.html"},
|
||||
{"name": "Search", "url": "/docs/search.html"},
|
||||
],
|
||||
schemas=[website_schema(base)],
|
||||
)
|
||||
return templates.TemplateResponse(request, "docs_base.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": user,
|
||||
"pages": visible_pages,
|
||||
"current": "search",
|
||||
"kind": "search",
|
||||
"page_title_doc": "Search",
|
||||
"search_query": query,
|
||||
"search_results": results,
|
||||
"base": base,
|
||||
})
|
||||
|
||||
page = _PAGES_BY_SLUG.get(slug)
|
||||
if not page:
|
||||
raise not_found("Documentation page not found")
|
||||
if page.get("admin") and not is_admin:
|
||||
raise not_found("Documentation page not found")
|
||||
if page["kind"] == "live":
|
||||
seo_ctx = base_seo_context(
|
||||
request,
|
||||
title=f"{page['title']} - Documentation",
|
||||
description="Your live DevPlace stats, activity, and site data.",
|
||||
robots="noindex",
|
||||
breadcrumbs=[
|
||||
{"name": "Home", "url": "/feed"},
|
||||
{"name": "Docs", "url": "/docs/index.html"},
|
||||
{"name": page["title"], "url": f"/docs/{slug}.html"},
|
||||
],
|
||||
schemas=[website_schema(base)],
|
||||
)
|
||||
return templates.TemplateResponse(request, "docs_base.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": user,
|
||||
"pages": visible_pages,
|
||||
"current": slug,
|
||||
"kind": "live",
|
||||
"facts": docs_live.build_live_facts(user),
|
||||
"page_title_doc": page["title"],
|
||||
"base": base,
|
||||
})
|
||||
if page.get("dynamic"):
|
||||
group = build_services_group(service_manager.describe_all(), base)
|
||||
else:
|
||||
group = render_group(slug, base, user.get("username") if user else "", user.get("api_key") if user else "")
|
||||
seo_ctx = base_seo_context(
|
||||
request,
|
||||
title=f"{page['title']} - Documentation",
|
||||
description="DevPlace developer documentation.",
|
||||
breadcrumbs=[
|
||||
{"name": "Home", "url": "/feed"},
|
||||
{"name": "Docs", "url": "/docs/index.html"},
|
||||
{"name": page["title"], "url": f"/docs/{slug}.html"},
|
||||
],
|
||||
schemas=[website_schema(base)],
|
||||
)
|
||||
return templates.TemplateResponse(request, "docs_base.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": user,
|
||||
"pages": visible_pages,
|
||||
"current": slug,
|
||||
"kind": page["kind"],
|
||||
"group": group,
|
||||
"page_title_doc": page["title"],
|
||||
"base": base,
|
||||
})
|
||||
@@ -7,6 +7,8 @@ from devplacepy.content import enrich_items
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import get_current_user
|
||||
from devplacepy.seo import list_page_seo, next_page_url
|
||||
from devplacepy.responses import respond
|
||||
from devplacepy.schemas import FeedOut
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -67,7 +69,7 @@ async def feed_page(request: Request, tab: str = "all", topic: str = None, befor
|
||||
breadcrumbs=[{"name": "Home", "url": "/feed"}, {"name": "Feed", "url": "/feed"}],
|
||||
next_url=next_page_url(request, next_cursor),
|
||||
)
|
||||
return templates.TemplateResponse(request, "feed.html", {
|
||||
return respond(request, "feed.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": user,
|
||||
@@ -81,4 +83,4 @@ async def feed_page(request: Request, tab: str = "all", topic: str = None, befor
|
||||
"top_authors": top_authors,
|
||||
"daily_topic": daily_topic,
|
||||
"next_cursor": next_cursor,
|
||||
})
|
||||
}, model=FeedOut)
|
||||
|
||||
@@ -4,6 +4,7 @@ from fastapi import APIRouter, Request
|
||||
from fastapi.responses import RedirectResponse
|
||||
from devplacepy.database import get_table
|
||||
from devplacepy.utils import generate_uid, require_user, create_notification, award_rewards, XP_FOLLOW
|
||||
from devplacepy.responses import action_result
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -15,12 +16,12 @@ async def follow_user(request: Request, username: str):
|
||||
users = get_table("users")
|
||||
target = users.find_one(username=username)
|
||||
if not target or target["uid"] == user["uid"]:
|
||||
return RedirectResponse(url=f"/profile/{username}", status_code=302)
|
||||
return action_result(request, f"/profile/{username}")
|
||||
|
||||
follows = get_table("follows")
|
||||
existing = follows.find_one(follower_uid=user["uid"], following_uid=target["uid"])
|
||||
if existing:
|
||||
return RedirectResponse(url=f"/profile/{username}", status_code=302)
|
||||
return action_result(request, f"/profile/{username}")
|
||||
|
||||
follows.insert({
|
||||
"uid": generate_uid(),
|
||||
@@ -33,7 +34,7 @@ async def follow_user(request: Request, username: str):
|
||||
award_rewards(target["uid"], XP_FOLLOW)
|
||||
|
||||
logger.info(f"{user['username']} followed {username}")
|
||||
return RedirectResponse(url=f"/profile/{username}", status_code=302)
|
||||
return action_result(request, f"/profile/{username}")
|
||||
|
||||
|
||||
@router.post("/unfollow/{username}")
|
||||
@@ -42,7 +43,7 @@ async def unfollow_user(request: Request, username: str):
|
||||
users = get_table("users")
|
||||
target = users.find_one(username=username)
|
||||
if not target:
|
||||
return RedirectResponse(url=f"/profile/{username}", status_code=302)
|
||||
return action_result(request, f"/profile/{username}")
|
||||
|
||||
follows = get_table("follows")
|
||||
existing = follows.find_one(follower_uid=user["uid"], following_uid=target["uid"])
|
||||
@@ -50,4 +51,4 @@ async def unfollow_user(request: Request, username: str):
|
||||
follows.delete(id=existing["id"])
|
||||
logger.info(f"{user['username']} unfollowed {username}")
|
||||
|
||||
return RedirectResponse(url=f"/profile/{username}", status_code=302)
|
||||
return action_result(request, f"/profile/{username}")
|
||||
|
||||
@@ -8,6 +8,8 @@ from devplacepy.content import load_detail, edit_content_item, delete_content_it
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import get_current_user, require_user, not_found, XP_GIST
|
||||
from devplacepy.seo import base_seo_context, site_url, website_schema, software_source_code_schema, list_page_seo, next_page_url
|
||||
from devplacepy.responses import respond, action_result
|
||||
from devplacepy.schemas import GistsOut, GistDetailOut
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -55,7 +57,7 @@ async def gists_page(request: Request, language: str = None, user_uid: str = Non
|
||||
],
|
||||
next_url=next_page_url(request, next_cursor),
|
||||
)
|
||||
return templates.TemplateResponse(request, "gists.html", {
|
||||
return respond(request, "gists.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": user,
|
||||
@@ -65,7 +67,7 @@ async def gists_page(request: Request, language: str = None, user_uid: str = Non
|
||||
"current_language": language,
|
||||
"languages": LANGUAGES,
|
||||
"gist_language_codes": get_gist_languages(),
|
||||
})
|
||||
}, model=GistsOut)
|
||||
|
||||
|
||||
@router.get("/{gist_slug}", response_class=HTMLResponse)
|
||||
@@ -93,9 +95,9 @@ async def gist_detail(request: Request, gist_slug: str):
|
||||
],
|
||||
schemas=[website_schema(base), software_source_code_schema(gist, base)],
|
||||
)
|
||||
return templates.TemplateResponse(request, "gist_detail.html", detail_context(request, user, detail, "gist", seo_ctx, {
|
||||
return respond(request, "gist_detail.html", detail_context(request, user, detail, "gist", seo_ctx, {
|
||||
"languages": LANGUAGES,
|
||||
}))
|
||||
}), model=GistDetailOut)
|
||||
|
||||
|
||||
@router.post("/create")
|
||||
@@ -116,7 +118,8 @@ async def create_gist(request: Request, data: Annotated[GistForm, Form()]):
|
||||
"source_code": source_code,
|
||||
"language": language,
|
||||
}, title, XP_GIST, "First Gist", description or "", data.attachment_uids)
|
||||
return RedirectResponse(url=f"/gists/{gist_slug}", status_code=302)
|
||||
url = f"/gists/{gist_slug}"
|
||||
return action_result(request, url, data={"uid": uid, "slug": gist_slug, "url": url})
|
||||
|
||||
|
||||
@router.post("/edit/{gist_slug}")
|
||||
@@ -125,7 +128,7 @@ async def edit_gist(request: Request, gist_slug: str, data: Annotated[GistEditFo
|
||||
language = data.language
|
||||
if language not in {l[0] for l in LANGUAGES}:
|
||||
language = "plaintext"
|
||||
return edit_content_item("gists", user, gist_slug, {
|
||||
return edit_content_item(request, "gists", user, gist_slug, {
|
||||
"title": data.title.strip(),
|
||||
"description": data.description.strip() or None,
|
||||
"source_code": data.source_code.strip(),
|
||||
@@ -136,4 +139,4 @@ async def edit_gist(request: Request, gist_slug: str, data: Annotated[GistEditFo
|
||||
@router.post("/delete/{gist_slug}")
|
||||
async def delete_gist(request: Request, gist_slug: str):
|
||||
user = require_user(request)
|
||||
return delete_content_item("gists", "gist", user, gist_slug, "/gists")
|
||||
return delete_content_item(request, "gists", "gist", user, gist_slug, "/gists")
|
||||
|
||||
@@ -5,6 +5,8 @@ from devplacepy.database import get_leaderboard, get_user_rank, get_site_stats,
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import get_current_user
|
||||
from devplacepy.seo import list_page_seo
|
||||
from devplacepy.responses import respond
|
||||
from devplacepy.schemas import LeaderboardOut
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -29,7 +31,7 @@ async def leaderboard_page(request: Request):
|
||||
description="Top contributors on DevPlace ranked by the stars their posts, projects, and gists have earned.",
|
||||
breadcrumbs=[{"name": "Home", "url": "/feed"}, {"name": "Leaderboard", "url": "/leaderboard"}],
|
||||
)
|
||||
return templates.TemplateResponse(request, "leaderboard.html", {
|
||||
return respond(request, "leaderboard.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": user,
|
||||
@@ -41,4 +43,4 @@ async def leaderboard_page(request: Request):
|
||||
"total_gists": stats["total_gists"],
|
||||
"top_authors": top_authors,
|
||||
"featured_news": featured_news,
|
||||
})
|
||||
}, model=LeaderboardOut)
|
||||
|
||||
@@ -9,6 +9,8 @@ from devplacepy.attachments import get_attachments_batch
|
||||
from devplacepy.templating import templates, clear_unread_cache, clear_messages_cache
|
||||
from devplacepy.utils import generate_uid, require_user, time_ago, create_mention_notifications
|
||||
from devplacepy.seo import base_seo_context
|
||||
from devplacepy.responses import respond, action_result
|
||||
from devplacepy.schemas import MessagesOut
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -119,7 +121,7 @@ async def messages_page(request: Request, with_uid: str = None, search: str = ""
|
||||
{"name": "Messages", "url": "/messages"},
|
||||
],
|
||||
)
|
||||
return templates.TemplateResponse(request, "messages.html", {
|
||||
return respond(request, "messages.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": user,
|
||||
@@ -128,7 +130,7 @@ async def messages_page(request: Request, with_uid: str = None, search: str = ""
|
||||
"other_user": other_user,
|
||||
"current_conversation": current_conversation,
|
||||
"search": search,
|
||||
})
|
||||
}, model=MessagesOut)
|
||||
|
||||
|
||||
@router.get("/search")
|
||||
@@ -154,7 +156,7 @@ async def send_message(request: Request, data: Annotated[MessageForm, Form()]):
|
||||
receiver_uid = data.receiver_uid
|
||||
|
||||
if not get_table("users").find_one(uid=receiver_uid):
|
||||
return RedirectResponse(url="/messages", status_code=302)
|
||||
return action_result(request, "/messages")
|
||||
|
||||
messages_table = get_table("messages")
|
||||
msg_uid = generate_uid()
|
||||
@@ -188,4 +190,4 @@ async def send_message(request: Request, data: Annotated[MessageForm, Form()]):
|
||||
create_mention_notifications(content, user["uid"], f"/messages?with_uid={receiver_uid}")
|
||||
|
||||
logger.info(f"Message {msg_uid} sent from {user['username']} to {receiver_uid}")
|
||||
return RedirectResponse(url=f"/messages?with_uid={receiver_uid}", status_code=302)
|
||||
return action_result(request, f"/messages?with_uid={receiver_uid}", data={"uid": msg_uid})
|
||||
|
||||
@@ -7,6 +7,8 @@ from devplacepy.templating import templates
|
||||
from devplacepy.utils import get_current_user, time_ago, not_found
|
||||
from devplacepy.content import canonical_redirect
|
||||
from devplacepy.seo import base_seo_context, website_schema, site_url, news_article_schema, list_page_seo, next_page_url
|
||||
from devplacepy.responses import respond
|
||||
from devplacepy.schemas import NewsListOut, NewsDetailOut
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -49,13 +51,13 @@ async def news_page(request: Request, before: str = None):
|
||||
next_url=next_page_url(request, next_cursor),
|
||||
)
|
||||
|
||||
return templates.TemplateResponse(request, "news.html", {
|
||||
return respond(request, "news.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": user,
|
||||
"articles": enriched,
|
||||
"next_cursor": next_cursor,
|
||||
})
|
||||
}, model=NewsListOut)
|
||||
|
||||
|
||||
@router.get("/{news_slug}", response_class=HTMLResponse)
|
||||
@@ -91,7 +93,7 @@ async def news_detail_page(request: Request, news_slug: str):
|
||||
schemas=[website_schema(base), news_article_schema(article, base, image_url)],
|
||||
)
|
||||
|
||||
return templates.TemplateResponse(request, "news_detail.html", {
|
||||
return respond(request, "news_detail.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": user,
|
||||
@@ -102,4 +104,4 @@ async def news_detail_page(request: Request, news_slug: str):
|
||||
"time_ago": time_ago(article["synced_at"]),
|
||||
"comments": comments,
|
||||
"bookmarked": bookmarked,
|
||||
})
|
||||
}, model=NewsDetailOut)
|
||||
|
||||
@@ -11,6 +11,8 @@ from devplacepy.templating import (
|
||||
)
|
||||
from devplacepy.utils import require_user, get_current_user, time_ago
|
||||
from devplacepy.seo import base_seo_context
|
||||
from devplacepy.responses import respond, action_result
|
||||
from devplacepy.schemas import NotificationsOut
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -94,13 +96,13 @@ async def notifications_page(request: Request, before: str = None):
|
||||
{"name": "Notifications", "url": "/notifications"},
|
||||
],
|
||||
)
|
||||
return templates.TemplateResponse(request, "notifications.html", {
|
||||
return respond(request, "notifications.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": user,
|
||||
"notification_groups": groups,
|
||||
"next_cursor": next_cursor,
|
||||
})
|
||||
}, model=NotificationsOut)
|
||||
|
||||
|
||||
@router.get("/counts")
|
||||
@@ -120,11 +122,11 @@ async def open_notification(request: Request, notification_uid: str):
|
||||
notifications_table = get_table("notifications")
|
||||
n = notifications_table.find_one(uid=notification_uid)
|
||||
if not n or n["user_uid"] != user["uid"]:
|
||||
return RedirectResponse(url="/notifications", status_code=302)
|
||||
return action_result(request, "/notifications")
|
||||
if not n["read"]:
|
||||
notifications_table.update({"id": n["id"], "read": True}, ["id"])
|
||||
clear_unread_cache(user["uid"])
|
||||
return RedirectResponse(url=n.get("target_url") or "/notifications", status_code=302)
|
||||
return action_result(request, n.get("target_url") or "/notifications")
|
||||
|
||||
|
||||
@router.post("/mark-read/{notification_uid}")
|
||||
@@ -135,7 +137,7 @@ async def mark_read(request: Request, notification_uid: str):
|
||||
if n and n["user_uid"] == user["uid"]:
|
||||
notifications_table.update({"id": n["id"], "read": True}, ["id"])
|
||||
clear_unread_cache(user["uid"])
|
||||
return RedirectResponse(url="/notifications", status_code=302)
|
||||
return action_result(request, "/notifications")
|
||||
|
||||
|
||||
@router.post("/mark-all-read")
|
||||
@@ -144,4 +146,4 @@ async def mark_all_read(request: Request):
|
||||
with db:
|
||||
db.query("UPDATE notifications SET read = 1 WHERE user_uid = :u AND read = 0", u=user["uid"])
|
||||
clear_unread_cache(user["uid"])
|
||||
return RedirectResponse(url="/notifications", status_code=302)
|
||||
return action_result(request, "/notifications")
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import logging
|
||||
from fastapi import APIRouter, Request, HTTPException
|
||||
from devplacepy.services.manager import service_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _service():
|
||||
svc = service_manager.get_service("openai")
|
||||
if svc is None:
|
||||
raise HTTPException(status_code=503, detail="Gateway is not available")
|
||||
return svc
|
||||
|
||||
|
||||
@router.post("/v1/chat/completions")
|
||||
async def chat_completions(request: Request):
|
||||
return await _service().handle(request, "chat/completions")
|
||||
|
||||
|
||||
@router.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"])
|
||||
async def passthrough(request: Request, path: str):
|
||||
return await _service().handle(request, path)
|
||||
@@ -4,10 +4,12 @@ from datetime import datetime, timezone
|
||||
from fastapi import APIRouter, Request, Form
|
||||
from fastapi.responses import RedirectResponse, HTMLResponse
|
||||
from devplacepy.constants import TOPICS
|
||||
from devplacepy.database import db, get_table
|
||||
from devplacepy.database import db, get_table, resolve_by_slug
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import get_current_user, require_user, time_ago, not_found, generate_uid, XP_POST
|
||||
from devplacepy.content import load_detail, edit_content_item, delete_content_item, create_content_item, detail_context, canonical_redirect, first_image_url
|
||||
from devplacepy.responses import respond, action_result
|
||||
from devplacepy.schemas import PostDetailOut
|
||||
from devplacepy.seo import base_seo_context, site_url, website_schema, discussion_forum_posting, truncate
|
||||
from devplacepy.attachments import save_inline_image
|
||||
from devplacepy.models import PostForm, PostEditForm
|
||||
@@ -42,7 +44,8 @@ async def create_post(request: Request, data: Annotated[PostForm, Form()]):
|
||||
}, slug_text, XP_POST, "First Post", content, data.attachment_uids)
|
||||
|
||||
create_poll(uid, user, data.poll_question, data.poll_options)
|
||||
return RedirectResponse(url=f"/posts/{post_slug}", status_code=302)
|
||||
url = f"/posts/{post_slug}"
|
||||
return action_result(request, url, data={"uid": uid, "slug": post_slug, "url": url})
|
||||
|
||||
|
||||
def create_poll(post_uid: str, user: dict, question: str, options: list[str]) -> None:
|
||||
@@ -116,24 +119,29 @@ async def view_post(request: Request, post_slug: str):
|
||||
"time_ago": time_ago(r["created_at"]),
|
||||
})
|
||||
|
||||
return templates.TemplateResponse(request, "post.html", detail_context(request, user, detail, "post", seo_ctx, {
|
||||
return respond(request, "post.html", detail_context(request, user, detail, "post", seo_ctx, {
|
||||
"comment_count": comment_count,
|
||||
"related_posts": related_posts,
|
||||
"topics": list(TOPICS),
|
||||
}))
|
||||
}), model=PostDetailOut)
|
||||
|
||||
|
||||
@router.post("/edit/{post_slug}")
|
||||
async def edit_post(request: Request, post_slug: str, data: Annotated[PostEditForm, Form()]):
|
||||
user = require_user(request)
|
||||
return edit_content_item("posts", user, post_slug, {
|
||||
result = edit_content_item(request, "posts", user, post_slug, {
|
||||
"content": data.content.strip(),
|
||||
"title": data.title.strip() or None,
|
||||
"topic": data.topic,
|
||||
}, "/feed")
|
||||
if data.poll_question.strip():
|
||||
post = resolve_by_slug(get_table("posts"), post_slug)
|
||||
if post and post["user_uid"] == user["uid"] and not get_table("polls").find_one(post_uid=post["uid"]):
|
||||
create_poll(post["uid"], user, data.poll_question, data.poll_options)
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/delete/{post_slug}")
|
||||
async def delete_post(request: Request, post_slug: str):
|
||||
user = require_user(request)
|
||||
return delete_content_item("posts", "post", user, post_slug, "/feed", inline_image_field="image")
|
||||
return delete_content_item(request, "posts", "post", user, post_slug, "/feed", inline_image_field="image")
|
||||
|
||||
@@ -3,10 +3,12 @@ from typing import Annotated
|
||||
from fastapi import APIRouter, Request, Form
|
||||
from devplacepy.models import ProfileForm
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse
|
||||
from devplacepy.database import get_table, db, get_user_stars, get_user_rank, get_comment_counts_by_post_uids, get_reactions_by_targets, get_user_bookmarks, get_polls_by_post_uids, get_activity_heatmap, get_streaks
|
||||
from devplacepy.database import get_table, db, get_user_stars, get_user_rank, get_comment_counts_by_post_uids, get_reactions_by_targets, get_user_bookmarks, get_polls_by_post_uids, get_activity_heatmap, get_activity_months, get_streaks, get_follow_counts, get_follow_list, get_following_among
|
||||
from devplacepy.content import enrich_items
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import get_current_user, require_user, require_user_api, time_ago, clear_user_cache, not_found
|
||||
from devplacepy.utils import get_current_user, require_user, require_user_api, time_ago, clear_user_cache, not_found, generate_uid
|
||||
from devplacepy.responses import respond, action_result
|
||||
from devplacepy.schemas import ProfileOut
|
||||
from devplacepy.avatar import avatar_url
|
||||
from devplacepy.seo import base_seo_context, site_url, website_schema, profile_page_schema
|
||||
|
||||
@@ -30,8 +32,43 @@ async def search_users(request: Request, q: str = ""):
|
||||
return JSONResponse({"results": results})
|
||||
|
||||
|
||||
def _follow_people(profile_uid: str, mode: str, current_user, page: int):
|
||||
people, pagination = get_follow_list(profile_uid, mode, page)
|
||||
mine = get_following_among(current_user["uid"], [p["uid"] for p in people]) if current_user else set()
|
||||
for person in people:
|
||||
person["is_following"] = person["uid"] in mine
|
||||
person["is_self"] = bool(current_user and current_user["uid"] == person["uid"])
|
||||
return people, pagination
|
||||
|
||||
|
||||
@router.get("/{username}/followers")
|
||||
async def profile_followers(request: Request, username: str, page: int = 1):
|
||||
return _follow_list_json(request, username, "followers", page)
|
||||
|
||||
|
||||
@router.get("/{username}/following")
|
||||
async def profile_following(request: Request, username: str, page: int = 1):
|
||||
return _follow_list_json(request, username, "following", page)
|
||||
|
||||
|
||||
def _follow_list_json(request: Request, username: str, mode: str, page: int):
|
||||
profile_user = get_table("users").find_one(username=username)
|
||||
if not profile_user:
|
||||
raise not_found("Profile not found")
|
||||
current_user = get_current_user(request)
|
||||
people, pagination = _follow_people(profile_user["uid"], mode, current_user, page)
|
||||
return JSONResponse({
|
||||
"username": profile_user["username"],
|
||||
"mode": mode,
|
||||
"count": pagination["total"],
|
||||
"page": pagination["page"],
|
||||
"total_pages": pagination["total_pages"],
|
||||
mode: [{"uid": p["uid"], "username": p["username"], "bio": p["bio"], "is_following": p["is_following"]} for p in people],
|
||||
})
|
||||
|
||||
|
||||
@router.get("/{username}", response_class=HTMLResponse)
|
||||
async def profile_page(request: Request, username: str, tab: str = "posts"):
|
||||
async def profile_page(request: Request, username: str, tab: str = "posts", page: int = 1):
|
||||
current_user = get_current_user(request)
|
||||
users = get_table("users")
|
||||
profile_user = users.find_one(username=username)
|
||||
@@ -39,6 +76,12 @@ async def profile_page(request: Request, username: str, tab: str = "posts"):
|
||||
raise not_found("Profile not found")
|
||||
profile_user["stars"] = get_user_stars(profile_user["uid"])
|
||||
rank = get_user_rank(profile_user["uid"])
|
||||
follow_counts = get_follow_counts(profile_user["uid"])
|
||||
|
||||
people = []
|
||||
follow_pagination = None
|
||||
if tab in ("followers", "following"):
|
||||
people, follow_pagination = _follow_people(profile_user["uid"], tab, current_user, page)
|
||||
|
||||
posts = []
|
||||
if tab == "posts":
|
||||
@@ -78,6 +121,7 @@ async def profile_page(request: Request, username: str, tab: str = "posts"):
|
||||
activities.sort(key=lambda a: a["created_at"], reverse=True)
|
||||
|
||||
heatmap = get_activity_heatmap(profile_user["uid"])
|
||||
heatmap_months = get_activity_months(heatmap)
|
||||
streak = get_streaks(profile_user["uid"])
|
||||
|
||||
is_following = False
|
||||
@@ -85,6 +129,10 @@ async def profile_page(request: Request, username: str, tab: str = "posts"):
|
||||
follows = get_table("follows")
|
||||
is_following = bool(follows.find_one(follower_uid=current_user["uid"], following_uid=profile_user["uid"]))
|
||||
|
||||
is_owner = bool(current_user and current_user["uid"] == profile_user["uid"])
|
||||
can_view_api_key = bool(current_user and (is_owner or current_user.get("role") == "Admin"))
|
||||
api_key = profile_user.get("api_key", "") if can_view_api_key else ""
|
||||
|
||||
base = site_url(request)
|
||||
robots = "noindex,follow" if posts_count < 2 else "index,follow"
|
||||
bio = profile_user.get("bio", "")
|
||||
@@ -106,7 +154,7 @@ async def profile_page(request: Request, username: str, tab: str = "posts"):
|
||||
],
|
||||
)
|
||||
|
||||
return templates.TemplateResponse(request, "profile.html", {
|
||||
return respond(request, "profile.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": current_user,
|
||||
@@ -118,11 +166,19 @@ async def profile_page(request: Request, username: str, tab: str = "posts"):
|
||||
"current_tab": tab,
|
||||
"posts_count": posts_count,
|
||||
"is_following": is_following,
|
||||
"is_owner": is_owner,
|
||||
"can_view_api_key": can_view_api_key,
|
||||
"api_key": api_key,
|
||||
"activities": activities,
|
||||
"rank": rank,
|
||||
"heatmap": heatmap,
|
||||
"heatmap_months": heatmap_months,
|
||||
"streak": streak,
|
||||
})
|
||||
"people": people,
|
||||
"follow_pagination": follow_pagination,
|
||||
"followers_count": follow_counts["followers"],
|
||||
"following_count": follow_counts["following"],
|
||||
}, model=ProfileOut)
|
||||
|
||||
|
||||
@router.post("/update")
|
||||
@@ -139,4 +195,14 @@ async def update_profile(request: Request, data: Annotated[ProfileForm, Form()])
|
||||
clear_user_cache(user["uid"])
|
||||
|
||||
logger.info(f"Profile updated for {user['username']}")
|
||||
return RedirectResponse(url=f"/profile/{user['username']}", status_code=302)
|
||||
return action_result(request, f"/profile/{user['username']}")
|
||||
|
||||
|
||||
@router.post("/regenerate-api-key")
|
||||
async def regenerate_api_key(request: Request):
|
||||
user = require_user(request)
|
||||
new_key = generate_uid()
|
||||
get_table("users").update({"uid": user["uid"], "api_key": new_key}, ["uid"])
|
||||
clear_user_cache(user["uid"])
|
||||
logger.info(f"API key regenerated for {user['username']}")
|
||||
return JSONResponse({"api_key": new_key})
|
||||
|
||||
@@ -9,6 +9,8 @@ from devplacepy.content import load_detail, delete_content_item, create_content_
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import get_current_user, require_user, not_found, XP_PROJECT
|
||||
from devplacepy.seo import base_seo_context, site_url, website_schema, software_application_schema, list_page_seo, next_page_url
|
||||
from devplacepy.responses import respond, action_result
|
||||
from devplacepy.schemas import ProjectsOut, ProjectDetailOut
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -68,7 +70,7 @@ async def projects_page(
|
||||
],
|
||||
next_url=next_page_url(request, next_cursor),
|
||||
)
|
||||
return templates.TemplateResponse(request, "projects.html", {
|
||||
return respond(request, "projects.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": user,
|
||||
@@ -79,7 +81,7 @@ async def projects_page(
|
||||
"total_count": total_count,
|
||||
"next_cursor": next_cursor,
|
||||
"total_members": total_members,
|
||||
})
|
||||
}, model=ProjectsOut)
|
||||
|
||||
|
||||
@router.get("/{project_slug}", response_class=HTMLResponse)
|
||||
@@ -106,15 +108,15 @@ async def project_detail(request: Request, project_slug: str):
|
||||
],
|
||||
schemas=[website_schema(base), software_application_schema(project, base)],
|
||||
)
|
||||
return templates.TemplateResponse(request, "project_detail.html", detail_context(request, user, detail, "project", seo_ctx, {
|
||||
return respond(request, "project_detail.html", detail_context(request, user, detail, "project", seo_ctx, {
|
||||
"platforms": project.get("platforms", "").split(",") if project.get("platforms") else [],
|
||||
}))
|
||||
}), model=ProjectDetailOut)
|
||||
|
||||
|
||||
@router.post("/delete/{project_slug}")
|
||||
async def delete_project(request: Request, project_slug: str):
|
||||
user = require_user(request)
|
||||
return delete_content_item("projects", "project", user, project_slug, "/projects")
|
||||
return delete_content_item(request, "projects", "project", user, project_slug, "/projects")
|
||||
|
||||
|
||||
@router.post("/create")
|
||||
@@ -132,4 +134,5 @@ async def create_project(request: Request, data: Annotated[ProjectForm, Form()])
|
||||
"platforms": data.platforms.strip(),
|
||||
"status": data.status,
|
||||
}, title, XP_PROJECT, "First Project", description, data.attachment_uids)
|
||||
return RedirectResponse(url=f"/projects/{project_slug}", status_code=302)
|
||||
url = f"/projects/{project_slug}"
|
||||
return action_result(request, url, data={"uid": uid, "slug": project_slug, "url": url})
|
||||
|
||||
@@ -2,7 +2,7 @@ import logging
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
from devplacepy.templating import templates
|
||||
from devplacepy.utils import require_admin
|
||||
from devplacepy.utils import require_admin, not_found
|
||||
from devplacepy.services.manager import service_manager
|
||||
from devplacepy.seo import base_seo_context, site_url, website_schema
|
||||
|
||||
@@ -13,12 +13,12 @@ router = APIRouter()
|
||||
@router.get("", response_class=HTMLResponse)
|
||||
async def services_page(request: Request):
|
||||
admin = require_admin(request)
|
||||
services = service_manager.list_services()
|
||||
services = service_manager.describe_all()
|
||||
base = site_url(request)
|
||||
seo_ctx = base_seo_context(
|
||||
request,
|
||||
title="Services - Admin",
|
||||
description="Monitor background services on DevPlace.",
|
||||
description="Monitor and configure background services on DevPlace.",
|
||||
breadcrumbs=[
|
||||
{"name": "Home", "url": "/feed"},
|
||||
{"name": "Admin", "url": "/admin"},
|
||||
@@ -38,5 +38,87 @@ async def services_page(request: Request):
|
||||
@router.get("/data")
|
||||
async def services_data(request: Request):
|
||||
require_admin(request)
|
||||
services = service_manager.list_services()
|
||||
return JSONResponse({"services": services})
|
||||
return JSONResponse({"services": service_manager.describe_all()})
|
||||
|
||||
|
||||
@router.get("/{name}", response_class=HTMLResponse)
|
||||
async def service_detail(request: Request, name: str):
|
||||
admin = require_admin(request)
|
||||
svc = service_manager.get_service(name)
|
||||
if svc is None:
|
||||
raise not_found("Service not found")
|
||||
info = svc.describe()
|
||||
base = site_url(request)
|
||||
seo_ctx = base_seo_context(
|
||||
request,
|
||||
title=f"{info['title']} - Services",
|
||||
description=info["description"] or f"Configure the {info['title']} service.",
|
||||
robots="noindex",
|
||||
breadcrumbs=[
|
||||
{"name": "Home", "url": "/feed"},
|
||||
{"name": "Admin", "url": "/admin"},
|
||||
{"name": "Services", "url": "/admin/services"},
|
||||
{"name": info["title"], "url": f"/admin/services/{name}"},
|
||||
],
|
||||
schemas=[website_schema(base)],
|
||||
)
|
||||
return templates.TemplateResponse(request, "service_detail.html", {
|
||||
**seo_ctx,
|
||||
"request": request,
|
||||
"user": admin,
|
||||
"service": info,
|
||||
"admin_section": "services",
|
||||
})
|
||||
|
||||
|
||||
@router.get("/{name}/data")
|
||||
async def service_detail_data(request: Request, name: str):
|
||||
require_admin(request)
|
||||
svc = service_manager.get_service(name)
|
||||
if svc is None:
|
||||
return JSONResponse({"error": "unknown service"}, status_code=404)
|
||||
return JSONResponse({"service": svc.describe()})
|
||||
|
||||
|
||||
@router.post("/{name}/start")
|
||||
async def service_start(request: Request, name: str):
|
||||
require_admin(request)
|
||||
if not service_manager.set_enabled(name, True):
|
||||
return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
|
||||
return JSONResponse({"ok": True})
|
||||
|
||||
|
||||
@router.post("/{name}/stop")
|
||||
async def service_stop(request: Request, name: str):
|
||||
require_admin(request)
|
||||
if not service_manager.set_enabled(name, False):
|
||||
return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
|
||||
return JSONResponse({"ok": True})
|
||||
|
||||
|
||||
@router.post("/{name}/run")
|
||||
async def service_run(request: Request, name: str):
|
||||
require_admin(request)
|
||||
if not service_manager.send_command(name, "run"):
|
||||
return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
|
||||
return JSONResponse({"ok": True})
|
||||
|
||||
|
||||
@router.post("/{name}/clear-logs")
|
||||
async def service_clear_logs(request: Request, name: str):
|
||||
require_admin(request)
|
||||
if not service_manager.send_command(name, "clear"):
|
||||
return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
|
||||
return JSONResponse({"ok": True})
|
||||
|
||||
|
||||
@router.post("/{name}/config")
|
||||
async def service_config(request: Request, name: str):
|
||||
require_admin(request)
|
||||
form = await request.form()
|
||||
result = service_manager.save_config(name, dict(form))
|
||||
if result is None:
|
||||
return JSONResponse({"ok": False, "error": "unknown service"}, status_code=404)
|
||||
if not result["ok"]:
|
||||
return JSONResponse(result, status_code=400)
|
||||
return JSONResponse(result)
|
||||
|
||||
@@ -0,0 +1,494 @@
|
||||
"""Pydantic response models for JSON content negotiation.
|
||||
|
||||
Every model uses ``extra="ignore"`` so it can be validated directly against the
|
||||
dict context a template already receives, dropping internal keys (request, SEO)
|
||||
and - importantly - sensitive user fields (email, api_key, password_hash) that
|
||||
are never part of ``UserOut``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
|
||||
class _Out(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
# ---------- envelopes ----------
|
||||
|
||||
class ActionResult(_Out):
|
||||
ok: bool = True
|
||||
redirect: Optional[str] = None
|
||||
data: Optional[Any] = None
|
||||
|
||||
|
||||
class ErrorOut(_Out):
|
||||
error: dict
|
||||
|
||||
|
||||
class ValidationErrorOut(_Out):
|
||||
error: str = "validation"
|
||||
fields: dict = {}
|
||||
messages: list = []
|
||||
|
||||
|
||||
# ---------- shared resource projections ----------
|
||||
|
||||
class UserOut(_Out):
|
||||
uid: str = ""
|
||||
username: str = ""
|
||||
avatar_seed: Optional[str] = None
|
||||
role: Optional[str] = None
|
||||
bio: Optional[str] = None
|
||||
location: Optional[str] = None
|
||||
git_link: Optional[str] = None
|
||||
website: Optional[str] = None
|
||||
level: Optional[int] = None
|
||||
xp: Optional[int] = None
|
||||
stars: Optional[int] = None
|
||||
created_at: Optional[str] = None
|
||||
|
||||
|
||||
class AttachmentOut(_Out):
|
||||
uid: str = ""
|
||||
filename: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
size: Optional[int] = None
|
||||
is_image: Optional[bool] = None
|
||||
mime_type: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
|
||||
|
||||
class VotesOut(_Out):
|
||||
up: int = 0
|
||||
down: int = 0
|
||||
|
||||
|
||||
class ReactionsOut(_Out):
|
||||
counts: dict = {}
|
||||
mine: list = []
|
||||
|
||||
|
||||
class PollOptionOut(_Out):
|
||||
uid: str = ""
|
||||
label: Optional[str] = None
|
||||
count: Optional[int] = None
|
||||
votes: Optional[int] = None
|
||||
pct: Optional[int] = None
|
||||
|
||||
|
||||
class PollOut(_Out):
|
||||
uid: str = ""
|
||||
question: Optional[str] = None
|
||||
options: list[PollOptionOut] = []
|
||||
total: int = 0
|
||||
my_choice: Optional[str] = None
|
||||
voted: Optional[str] = None
|
||||
|
||||
|
||||
class BadgeOut(_Out):
|
||||
name: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
|
||||
|
||||
class PostOut(_Out):
|
||||
uid: str = ""
|
||||
slug: Optional[str] = None
|
||||
user_uid: Optional[str] = None
|
||||
title: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
topic: Optional[str] = None
|
||||
stars: Optional[int] = None
|
||||
image: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
updated_at: Optional[str] = None
|
||||
|
||||
|
||||
class GistOut(_Out):
|
||||
uid: str = ""
|
||||
slug: Optional[str] = None
|
||||
user_uid: Optional[str] = None
|
||||
title: Optional[str] = None
|
||||
description: Optional[str] = None
|
||||
source_code: Optional[str] = None
|
||||
language: Optional[str] = None
|
||||
stars: Optional[int] = None
|
||||
created_at: Optional[str] = None
|
||||
updated_at: Optional[str] = None
|
||||
|
||||
|
||||
class ProjectOut(_Out):
|
||||
uid: str = ""
|
||||
slug: Optional[str] = None
|
||||
user_uid: Optional[str] = None
|
||||
title: Optional[str] = None
|
||||
description: Optional[str] = None
|
||||
project_type: Optional[str] = None
|
||||
status: Optional[str] = None
|
||||
platforms: Optional[Any] = None
|
||||
stars: Optional[int] = None
|
||||
release_date: Optional[str] = None
|
||||
demo_date: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
updated_at: Optional[str] = None
|
||||
|
||||
|
||||
class NewsOut(_Out):
|
||||
uid: str = ""
|
||||
slug: Optional[str] = None
|
||||
title: Optional[str] = None
|
||||
description: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
source_name: Optional[str] = None
|
||||
author: Optional[str] = None
|
||||
grade: Optional[int] = None
|
||||
status: Optional[str] = None
|
||||
image_url: Optional[str] = None
|
||||
article_published: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
synced_at: Optional[str] = None
|
||||
|
||||
|
||||
class CommentOut(_Out):
|
||||
uid: str = ""
|
||||
user_uid: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
parent_uid: Optional[str] = None
|
||||
target_type: Optional[str] = None
|
||||
target_uid: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
|
||||
|
||||
class CommentItemOut(_Out):
|
||||
comment: CommentOut
|
||||
author: Optional[UserOut] = None
|
||||
time_ago: Optional[str] = None
|
||||
votes: VotesOut = VotesOut()
|
||||
my_vote: int = 0
|
||||
children: list["CommentItemOut"] = []
|
||||
attachments: list[AttachmentOut] = []
|
||||
reactions: ReactionsOut = ReactionsOut()
|
||||
|
||||
|
||||
class NotificationOut(_Out):
|
||||
uid: str = ""
|
||||
type: Optional[str] = None
|
||||
message: Optional[str] = None
|
||||
read: Optional[bool] = None
|
||||
related_uid: Optional[str] = None
|
||||
target_url: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
|
||||
|
||||
class MessageOut(_Out):
|
||||
uid: str = ""
|
||||
sender_uid: Optional[str] = None
|
||||
receiver_uid: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
read: Optional[bool] = None
|
||||
created_at: Optional[str] = None
|
||||
|
||||
|
||||
class BugOut(_Out):
|
||||
uid: str = ""
|
||||
user_uid: Optional[str] = None
|
||||
title: Optional[str] = None
|
||||
description: Optional[str] = None
|
||||
status: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
|
||||
|
||||
# ---------- list-item wrappers ----------
|
||||
|
||||
class FeedItemOut(_Out):
|
||||
post: PostOut
|
||||
author: Optional[UserOut] = None
|
||||
time_ago: Optional[str] = None
|
||||
my_vote: int = 0
|
||||
comment_count: int = 0
|
||||
attachments: list[AttachmentOut] = []
|
||||
recent_comments: list[CommentItemOut] = []
|
||||
reactions: ReactionsOut = ReactionsOut()
|
||||
bookmarked: bool = False
|
||||
poll: Optional[PollOut] = None
|
||||
|
||||
|
||||
class GistItemOut(_Out):
|
||||
gist: GistOut
|
||||
author: Optional[UserOut] = None
|
||||
time_ago: Optional[str] = None
|
||||
my_vote: int = 0
|
||||
comment_count: int = 0
|
||||
|
||||
|
||||
class ProjectListItemOut(ProjectOut):
|
||||
author_name: Optional[str] = None
|
||||
my_vote: int = 0
|
||||
|
||||
|
||||
class NewsListItemOut(_Out):
|
||||
article: NewsOut
|
||||
time_ago: Optional[str] = None
|
||||
image_url: Optional[str] = None
|
||||
grade: Optional[int] = None
|
||||
|
||||
|
||||
class ConversationOut(_Out):
|
||||
other_user: Optional[UserOut] = None
|
||||
last_message: Optional[str] = None
|
||||
last_message_at: Optional[str] = None
|
||||
unread: bool = False
|
||||
|
||||
|
||||
class MessageItemOut(_Out):
|
||||
message: MessageOut
|
||||
sender: Optional[UserOut] = None
|
||||
is_mine: bool = False
|
||||
time_ago: Optional[str] = None
|
||||
attachments: list[AttachmentOut] = []
|
||||
|
||||
|
||||
class NotificationItemOut(_Out):
|
||||
notification: NotificationOut
|
||||
actor: Optional[UserOut] = None
|
||||
time_ago: Optional[str] = None
|
||||
|
||||
|
||||
class NotificationGroupOut(_Out):
|
||||
label: Optional[str] = None
|
||||
entries: list[NotificationItemOut] = []
|
||||
|
||||
|
||||
class BugItemOut(_Out):
|
||||
bug: BugOut
|
||||
author: Optional[UserOut] = None
|
||||
time_ago: Optional[str] = None
|
||||
comments: list[CommentItemOut] = []
|
||||
attachments: list[AttachmentOut] = []
|
||||
|
||||
|
||||
class LeaderboardEntryOut(_Out):
|
||||
uid: Optional[str] = None
|
||||
username: Optional[str] = None
|
||||
avatar_seed: Optional[str] = None
|
||||
stars: Optional[int] = None
|
||||
level: Optional[int] = None
|
||||
rank: Optional[int] = None
|
||||
|
||||
|
||||
class SavedItemOut(_Out):
|
||||
target_type: Optional[str] = None
|
||||
target_uid: Optional[str] = None
|
||||
type_label: Optional[str] = None
|
||||
title: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
time_ago: Optional[str] = None
|
||||
|
||||
|
||||
class AdminUserOut(UserOut):
|
||||
email: Optional[str] = None
|
||||
is_active: Optional[bool] = None
|
||||
posts_count: Optional[int] = None
|
||||
|
||||
|
||||
class AdminNewsItemOut(_Out):
|
||||
article: NewsOut
|
||||
time_ago: Optional[str] = None
|
||||
synced_at: Optional[str] = None
|
||||
grade: Optional[int] = None
|
||||
has_image: Optional[bool] = None
|
||||
|
||||
|
||||
# ---------- page envelopes ----------
|
||||
|
||||
class FeedOut(_Out):
|
||||
posts: list[FeedItemOut] = []
|
||||
current_tab: Optional[str] = None
|
||||
current_topic: Optional[str] = None
|
||||
next_cursor: Optional[str] = None
|
||||
total_members: Optional[int] = None
|
||||
posts_today: Optional[int] = None
|
||||
total_projects: Optional[int] = None
|
||||
total_gists: Optional[int] = None
|
||||
top_authors: list[UserOut] = []
|
||||
daily_topic: Optional[Any] = None
|
||||
|
||||
|
||||
class PostDetailOut(_Out):
|
||||
post: PostOut
|
||||
author: Optional[UserOut] = None
|
||||
is_owner: bool = False
|
||||
star_count: int = 0
|
||||
my_vote: int = 0
|
||||
time_ago: Optional[str] = None
|
||||
comments: list[CommentItemOut] = []
|
||||
attachments: list[AttachmentOut] = []
|
||||
reactions: ReactionsOut = ReactionsOut()
|
||||
bookmarked: bool = False
|
||||
poll: Optional[PollOut] = None
|
||||
comment_count: Optional[int] = None
|
||||
related_posts: list[FeedItemOut] = []
|
||||
|
||||
|
||||
class ProjectsOut(_Out):
|
||||
projects: list[ProjectListItemOut] = []
|
||||
current_tab: Optional[str] = None
|
||||
search: Optional[str] = None
|
||||
project_type: Optional[str] = None
|
||||
total_count: Optional[int] = None
|
||||
next_cursor: Optional[str] = None
|
||||
total_members: Optional[int] = None
|
||||
|
||||
|
||||
class ProjectDetailOut(_Out):
|
||||
project: ProjectOut
|
||||
author: Optional[UserOut] = None
|
||||
is_owner: bool = False
|
||||
star_count: int = 0
|
||||
my_vote: int = 0
|
||||
time_ago: Optional[str] = None
|
||||
comments: list[CommentItemOut] = []
|
||||
attachments: list[AttachmentOut] = []
|
||||
reactions: ReactionsOut = ReactionsOut()
|
||||
bookmarked: bool = False
|
||||
platforms: Optional[Any] = None
|
||||
|
||||
|
||||
class GistsOut(_Out):
|
||||
gists: list[GistItemOut] = []
|
||||
total_count: Optional[int] = None
|
||||
next_cursor: Optional[str] = None
|
||||
current_language: Optional[str] = None
|
||||
|
||||
|
||||
class GistDetailOut(_Out):
|
||||
gist: GistOut
|
||||
author: Optional[UserOut] = None
|
||||
is_owner: bool = False
|
||||
star_count: int = 0
|
||||
my_vote: int = 0
|
||||
time_ago: Optional[str] = None
|
||||
comments: list[CommentItemOut] = []
|
||||
attachments: list[AttachmentOut] = []
|
||||
reactions: ReactionsOut = ReactionsOut()
|
||||
bookmarked: bool = False
|
||||
|
||||
|
||||
class NewsListOut(_Out):
|
||||
articles: list[NewsListItemOut] = []
|
||||
next_cursor: Optional[str] = None
|
||||
|
||||
|
||||
class NewsDetailOut(_Out):
|
||||
article: NewsOut
|
||||
canonical_slug: Optional[str] = None
|
||||
image_url: Optional[str] = None
|
||||
grade: Optional[int] = None
|
||||
time_ago: Optional[str] = None
|
||||
comments: list[CommentItemOut] = []
|
||||
bookmarked: bool = False
|
||||
|
||||
|
||||
class ProfileOut(_Out):
|
||||
profile_user: UserOut
|
||||
posts: list[FeedItemOut] = []
|
||||
badges: list[BadgeOut] = []
|
||||
projects: list[ProjectOut] = []
|
||||
gists: list[GistItemOut] = []
|
||||
current_tab: Optional[str] = None
|
||||
posts_count: Optional[int] = None
|
||||
is_following: bool = False
|
||||
is_owner: bool = False
|
||||
can_view_api_key: bool = False
|
||||
api_key: Optional[str] = None
|
||||
activities: list[Any] = []
|
||||
rank: Optional[int] = None
|
||||
heatmap: Optional[Any] = None
|
||||
heatmap_months: Optional[Any] = None
|
||||
streak: Optional[Any] = None
|
||||
people: list[Any] = []
|
||||
follow_pagination: Optional[Any] = None
|
||||
followers_count: Optional[int] = None
|
||||
following_count: Optional[int] = None
|
||||
|
||||
|
||||
class MessagesOut(_Out):
|
||||
conversations: list[ConversationOut] = []
|
||||
messages: list[MessageItemOut] = []
|
||||
other_user: Optional[UserOut] = None
|
||||
current_conversation: Optional[str] = None
|
||||
search: Optional[str] = None
|
||||
|
||||
|
||||
class NotificationsOut(_Out):
|
||||
notification_groups: list[NotificationGroupOut] = []
|
||||
next_cursor: Optional[str] = None
|
||||
|
||||
|
||||
class LeaderboardOut(_Out):
|
||||
entries: list[LeaderboardEntryOut] = []
|
||||
user_rank: Optional[int] = None
|
||||
total_members: Optional[int] = None
|
||||
posts_today: Optional[int] = None
|
||||
total_projects: Optional[int] = None
|
||||
total_gists: Optional[int] = None
|
||||
top_authors: list[UserOut] = []
|
||||
featured_news: list[NewsOut] = []
|
||||
|
||||
|
||||
class BugsOut(_Out):
|
||||
bugs: list[BugItemOut] = []
|
||||
|
||||
|
||||
class SavedOut(_Out):
|
||||
items: list[SavedItemOut] = []
|
||||
next_cursor: Optional[str] = None
|
||||
|
||||
|
||||
class AdminUsersOut(_Out):
|
||||
users: list[AdminUserOut] = []
|
||||
pagination: Optional[Any] = None
|
||||
admin_section: Optional[str] = None
|
||||
|
||||
|
||||
class AdminNewsOut(_Out):
|
||||
articles: list[AdminNewsItemOut] = []
|
||||
pagination: Optional[Any] = None
|
||||
admin_section: Optional[str] = None
|
||||
|
||||
|
||||
class AdminSettingsOut(_Out):
|
||||
settings: dict = {}
|
||||
admin_section: Optional[str] = None
|
||||
|
||||
|
||||
class GatewayUsageOut(_Out):
|
||||
window_hours: int = 48
|
||||
generated_at: Optional[str] = None
|
||||
requests: int = 0
|
||||
volume: dict = {}
|
||||
tokens: dict = {}
|
||||
latency: dict = {}
|
||||
errors: dict = {}
|
||||
cost: dict = {}
|
||||
behavior: dict = {}
|
||||
hourly: list = []
|
||||
notes: dict = {}
|
||||
admin_section: Optional[str] = None
|
||||
|
||||
|
||||
# ---------- auth pages ----------
|
||||
|
||||
class AuthPageOut(_Out):
|
||||
page: str = ""
|
||||
next_url: Optional[str] = None
|
||||
registration_closed: Optional[bool] = None
|
||||
sent: Optional[bool] = None
|
||||
token: Optional[str] = None
|
||||
errors: list = []
|
||||
|
||||
|
||||
CommentItemOut.model_rebuild()
|
||||
+354
-56
@@ -1,93 +1,391 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import deque
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from devplacepy.database import db, get_table, get_setting, get_int_setting
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ConfigField:
|
||||
def __init__(self, key, label, type="str", default="", help="",
|
||||
options=None, secret=False, minimum=None, maximum=None, group="General"):
|
||||
self.key = key
|
||||
self.label = label
|
||||
self.type = type
|
||||
self.default = default
|
||||
self.help = help
|
||||
self.options = options or []
|
||||
self.secret = secret
|
||||
self.minimum = minimum
|
||||
self.maximum = maximum
|
||||
self.group = group
|
||||
|
||||
def coerce(self, raw):
|
||||
if self.type == "int":
|
||||
try:
|
||||
value = int(str(raw).strip())
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError(f"{self.label} must be a whole number")
|
||||
if self.minimum is not None and value < self.minimum:
|
||||
raise ValueError(f"{self.label} must be at least {self.minimum}")
|
||||
if self.maximum is not None and value > self.maximum:
|
||||
raise ValueError(f"{self.label} must be at most {self.maximum}")
|
||||
return value
|
||||
if self.type == "float":
|
||||
try:
|
||||
value = float(str(raw).strip())
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError(f"{self.label} must be a number")
|
||||
if self.minimum is not None and value < self.minimum:
|
||||
raise ValueError(f"{self.label} must be at least {self.minimum}")
|
||||
if self.maximum is not None and value > self.maximum:
|
||||
raise ValueError(f"{self.label} must be at most {self.maximum}")
|
||||
return value
|
||||
if self.type == "bool":
|
||||
text = str(raw).strip()
|
||||
if text in ("0", "1"):
|
||||
return text == "1"
|
||||
raise ValueError(f"{self.label} must be enabled or disabled")
|
||||
if self.type == "select":
|
||||
allowed = [option["value"] for option in self.options]
|
||||
if raw not in allowed:
|
||||
raise ValueError(f"{self.label} has an invalid selection")
|
||||
return raw
|
||||
if self.type == "url":
|
||||
text = str(raw).strip()
|
||||
if text and not (text.startswith("http://") or text.startswith("https://")):
|
||||
raise ValueError(f"{self.label} must be a http(s) URL")
|
||||
return text
|
||||
return str(raw)
|
||||
|
||||
def to_storage(self, value):
|
||||
if isinstance(value, bool):
|
||||
return "1" if value else "0"
|
||||
return str(value)
|
||||
|
||||
def read(self):
|
||||
raw = get_setting(self.key, "")
|
||||
if raw == "":
|
||||
return self.default
|
||||
try:
|
||||
return self.coerce(raw)
|
||||
except ValueError:
|
||||
return self.default
|
||||
|
||||
def display_value(self):
|
||||
if self.secret:
|
||||
return ""
|
||||
raw = get_setting(self.key, "")
|
||||
if raw == "":
|
||||
return self.to_storage(self.default)
|
||||
return raw
|
||||
|
||||
def spec(self):
|
||||
return {
|
||||
"key": self.key,
|
||||
"label": self.label,
|
||||
"type": self.type,
|
||||
"help": self.help,
|
||||
"options": self.options,
|
||||
"secret": self.secret,
|
||||
"minimum": self.minimum,
|
||||
"maximum": self.maximum,
|
||||
"value": self.display_value(),
|
||||
"is_set": bool(get_setting(self.key, "")) if self.secret else None,
|
||||
"group": self.group,
|
||||
}
|
||||
|
||||
|
||||
class BaseService(ABC):
|
||||
config_fields = []
|
||||
interval_key = None
|
||||
min_interval = 1
|
||||
default_enabled = True
|
||||
title = ""
|
||||
description = ""
|
||||
DEFAULT_LOG_SIZE = 20
|
||||
TICK_SECONDS = 1
|
||||
PERSIST_SECONDS = 3
|
||||
STALE_SECONDS = 15
|
||||
|
||||
def __init__(self, name: str, interval_seconds: int = 3600):
|
||||
self.name = name
|
||||
self.interval_seconds = interval_seconds
|
||||
self.log_buffer = deque(maxlen=20)
|
||||
self.interval_key = self.interval_key or f"service_{name}_interval"
|
||||
self.enabled_key = f"service_{name}_enabled"
|
||||
self.command_key = f"service_{name}_command"
|
||||
self.log_size_key = f"service_{name}_log_size"
|
||||
self.log_buffer = deque(maxlen=self.DEFAULT_LOG_SIZE)
|
||||
self._task = None
|
||||
self._running = False
|
||||
self._shutdown = False
|
||||
self._started_at = None
|
||||
self._last_run = None
|
||||
self._next_run = None
|
||||
self._started_at = None
|
||||
self._next_due = None
|
||||
self._run_pending = False
|
||||
self._last_command = None
|
||||
self._last_persist = None
|
||||
self.enabled_field = ConfigField(
|
||||
self.enabled_key, "Enabled", type="bool", default=self.default_enabled,
|
||||
help="When enabled the service runs on its interval and starts on boot.",
|
||||
group="General")
|
||||
self.interval_field = ConfigField(
|
||||
self.interval_key, "Run interval (seconds)", type="int",
|
||||
default=interval_seconds, minimum=self.min_interval,
|
||||
help=f"Delay between runs. Minimum {self.min_interval} seconds.",
|
||||
group="General")
|
||||
self.log_size_field = ConfigField(
|
||||
self.log_size_key, "Log buffer size", type="int",
|
||||
default=self.DEFAULT_LOG_SIZE, minimum=1, maximum=200,
|
||||
help="Number of recent log lines to keep.",
|
||||
group="Advanced")
|
||||
|
||||
def all_fields(self) -> list:
|
||||
return [self.enabled_field, self.interval_field, *self.config_fields, self.log_size_field]
|
||||
|
||||
def _grouped_fields(self) -> list:
|
||||
groups: dict = {}
|
||||
order: list = []
|
||||
for field in self.all_fields():
|
||||
spec = field.spec()
|
||||
group = spec["group"]
|
||||
if group not in groups:
|
||||
groups[group] = []
|
||||
order.append(group)
|
||||
groups[group].append(spec)
|
||||
return [{"name": name, "fields": groups[name]} for name in order]
|
||||
|
||||
@property
|
||||
def status(self) -> str:
|
||||
return "running" if self._running else "stopped"
|
||||
|
||||
@property
|
||||
def last_run(self) -> str | None:
|
||||
return self._last_run
|
||||
def is_enabled(self) -> bool:
|
||||
return get_setting(self.enabled_key, "1" if self.default_enabled else "0") == "1"
|
||||
|
||||
@property
|
||||
def next_run(self) -> str | None:
|
||||
return self._next_run
|
||||
def current_interval(self) -> int:
|
||||
return max(self.min_interval, get_int_setting(self.interval_key, self.interval_seconds))
|
||||
|
||||
@property
|
||||
def uptime(self) -> str | None:
|
||||
if self._started_at is None:
|
||||
return None
|
||||
delta = datetime.now(timezone.utc) - self._started_at
|
||||
return str(delta).split(".")[0]
|
||||
def get_config(self) -> dict:
|
||||
return {field.key: field.read() for field in self.all_fields()}
|
||||
|
||||
def log(self, message: str) -> None:
|
||||
stamp = datetime.now(timezone.utc).strftime("%H:%M:%S")
|
||||
entry = f"[{stamp}] {message}"
|
||||
self.log_buffer.append(entry)
|
||||
self.log_buffer.append(f"[{stamp}] {message}")
|
||||
logger.info(f"[{self.name}] {message}")
|
||||
|
||||
@abstractmethod
|
||||
async def run_once(self) -> None:
|
||||
pass
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
self._running = True
|
||||
self._started_at = datetime.now(timezone.utc)
|
||||
self.log(f"Service started (interval={self.interval_seconds}s)")
|
||||
while self._running:
|
||||
try:
|
||||
self._last_run = datetime.now(timezone.utc).isoformat()
|
||||
self._next_run = None
|
||||
await self.run_once()
|
||||
except asyncio.CancelledError:
|
||||
self.log("Service cancelled")
|
||||
break
|
||||
except Exception as e:
|
||||
self.log(f"Error in run_once: {e}")
|
||||
if not self._running:
|
||||
break
|
||||
self._next_run = datetime.now(timezone.utc).isoformat()
|
||||
self.log(f"Sleeping for {self.interval_seconds}s")
|
||||
try:
|
||||
await asyncio.sleep(self.interval_seconds)
|
||||
except asyncio.CancelledError:
|
||||
self.log("Service cancelled during sleep")
|
||||
break
|
||||
self._running = False
|
||||
self.log("Service stopped")
|
||||
async def on_enable(self) -> None:
|
||||
pass
|
||||
|
||||
async def start(self) -> None:
|
||||
if self._running:
|
||||
self.log("Already running")
|
||||
async def on_disable(self) -> None:
|
||||
pass
|
||||
|
||||
def collect_metrics(self) -> dict:
|
||||
return {}
|
||||
|
||||
def _safe_metrics(self) -> dict:
|
||||
try:
|
||||
return self.collect_metrics()
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not collect metrics for {self.name}: {e}")
|
||||
return {}
|
||||
|
||||
def start_supervisor(self) -> None:
|
||||
if self._task is not None:
|
||||
return
|
||||
self._running = True
|
||||
self._shutdown = False
|
||||
self._task = asyncio.create_task(self._run_loop())
|
||||
|
||||
async def stop(self) -> None:
|
||||
if not self._running:
|
||||
return
|
||||
self._running = False
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
def request_shutdown(self) -> None:
|
||||
self._shutdown = True
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
self._last_command = get_setting(self.command_key, "")
|
||||
self.log("Supervisor started")
|
||||
self._persist_state(force=True)
|
||||
while not self._shutdown:
|
||||
try:
|
||||
await asyncio.wait_for(self._task, timeout=10.0)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError):
|
||||
pass
|
||||
self._task = None
|
||||
await self._tick()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
self.log(f"Loop error: {e}")
|
||||
try:
|
||||
await asyncio.sleep(self.TICK_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
if self._started_at is not None:
|
||||
await self._safe_disable()
|
||||
self._running = False
|
||||
self._started_at = None
|
||||
self._persist_state(force=True)
|
||||
self.log("Supervisor stopped")
|
||||
|
||||
async def _tick(self) -> None:
|
||||
self._sync_log_size()
|
||||
self._handle_commands()
|
||||
if self.is_enabled():
|
||||
if self._started_at is None:
|
||||
self._started_at = datetime.now(timezone.utc)
|
||||
self._running = True
|
||||
self.log("Service enabled")
|
||||
await self.on_enable()
|
||||
due = self._next_due is None or datetime.now(timezone.utc) >= self._next_due
|
||||
if self._run_pending or due:
|
||||
self._run_pending = False
|
||||
await self._execute_run()
|
||||
elif self._started_at is not None:
|
||||
await self._safe_disable()
|
||||
self._started_at = None
|
||||
self._running = False
|
||||
self._next_due = None
|
||||
self._next_run = None
|
||||
self.log("Service disabled")
|
||||
self._persist_state()
|
||||
|
||||
async def _safe_disable(self) -> None:
|
||||
try:
|
||||
await self.on_disable()
|
||||
except Exception as e:
|
||||
self.log(f"on_disable error: {e}")
|
||||
|
||||
async def _execute_run(self) -> None:
|
||||
self.interval_seconds = self.current_interval()
|
||||
self._last_run = datetime.now(timezone.utc).isoformat()
|
||||
self._next_run = None
|
||||
self._persist_state(force=True)
|
||||
try:
|
||||
await self.run_once()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.log(f"Error in run_once: {e}")
|
||||
self.interval_seconds = self.current_interval()
|
||||
self._next_due = datetime.now(timezone.utc) + timedelta(seconds=self.interval_seconds)
|
||||
self._next_run = self._next_due.isoformat()
|
||||
self.log(f"Next run in {self.interval_seconds}s")
|
||||
self._persist_state(force=True)
|
||||
|
||||
def _handle_commands(self) -> None:
|
||||
raw = get_setting(self.command_key, "")
|
||||
if not raw or raw == self._last_command:
|
||||
return
|
||||
self._last_command = raw
|
||||
verb = raw.split(":", 1)[0]
|
||||
if verb == "run":
|
||||
self._run_pending = True
|
||||
self.log("Run requested")
|
||||
elif verb == "clear":
|
||||
self.log_buffer.clear()
|
||||
self.log("Logs cleared")
|
||||
|
||||
def _sync_log_size(self) -> None:
|
||||
size = max(1, get_int_setting(self.log_size_key, self.DEFAULT_LOG_SIZE))
|
||||
if size != self.log_buffer.maxlen:
|
||||
self.log_buffer = deque(self.log_buffer, maxlen=size)
|
||||
|
||||
def _persist_state(self, force: bool = False) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
if not force and self._last_persist is not None:
|
||||
if (now - self._last_persist).total_seconds() < self.PERSIST_SECONDS:
|
||||
return
|
||||
self._last_persist = now
|
||||
record = {
|
||||
"name": self.name,
|
||||
"status": self.status,
|
||||
"last_run": self._last_run or "",
|
||||
"next_run": self._next_run or "",
|
||||
"started_at": self._started_at.isoformat() if self._started_at else "",
|
||||
"heartbeat": now.isoformat(),
|
||||
"logs": json.dumps(list(self.log_buffer)),
|
||||
"metrics": json.dumps(self._safe_metrics()),
|
||||
"updated_at": now.isoformat(),
|
||||
}
|
||||
try:
|
||||
table = get_table("service_state")
|
||||
existing = table.find_one(name=self.name)
|
||||
if existing:
|
||||
table.update({**record, "id": existing["id"]}, ["id"])
|
||||
else:
|
||||
table.insert(record)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not persist state for {self.name}: {e}")
|
||||
|
||||
def _read_state(self) -> dict:
|
||||
if "service_state" not in db.tables:
|
||||
return {}
|
||||
row = db["service_state"].find_one(name=self.name)
|
||||
return row or {}
|
||||
|
||||
def _display_status(self, enabled: bool, state: dict) -> str:
|
||||
if not enabled:
|
||||
return "stopped"
|
||||
heartbeat = state.get("heartbeat")
|
||||
if not heartbeat:
|
||||
return "stalled"
|
||||
try:
|
||||
beat = datetime.fromisoformat(heartbeat)
|
||||
except ValueError:
|
||||
return "stalled"
|
||||
if (datetime.now(timezone.utc) - beat).total_seconds() > self.STALE_SECONDS:
|
||||
return "stalled"
|
||||
return "running"
|
||||
|
||||
def _uptime(self, state: dict, status: str) -> str | None:
|
||||
if status != "running":
|
||||
return None
|
||||
started = state.get("started_at")
|
||||
if not started:
|
||||
return None
|
||||
try:
|
||||
since = datetime.fromisoformat(started)
|
||||
except ValueError:
|
||||
return None
|
||||
return str(datetime.now(timezone.utc) - since).split(".")[0]
|
||||
|
||||
def _logs(self, state: dict) -> list:
|
||||
raw = state.get("logs")
|
||||
if not raw:
|
||||
return []
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except (ValueError, TypeError):
|
||||
return []
|
||||
|
||||
def _metrics(self, state: dict) -> dict:
|
||||
raw = state.get("metrics")
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except (ValueError, TypeError):
|
||||
return {}
|
||||
|
||||
def describe(self) -> dict:
|
||||
state = self._read_state()
|
||||
enabled = self.is_enabled()
|
||||
status = self._display_status(enabled, state)
|
||||
return {
|
||||
"name": self.name,
|
||||
"title": self.title or self.name.capitalize(),
|
||||
"description": self.description,
|
||||
"enabled": enabled,
|
||||
"status": status,
|
||||
"interval_seconds": self.current_interval(),
|
||||
"last_run": state.get("last_run") or None,
|
||||
"next_run": state.get("next_run") or None,
|
||||
"heartbeat": state.get("heartbeat") or None,
|
||||
"uptime": self._uptime(state, status),
|
||||
"log_buffer": self._logs(state),
|
||||
"metrics": self._metrics(state),
|
||||
"fields": [field.spec() for field in self.all_fields()],
|
||||
"field_groups": self._grouped_fields(),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from devplacepy.services.bot.service import BotsService
|
||||
|
||||
__all__ = ["BotsService"]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,162 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
from typing import Optional
|
||||
|
||||
from playwright.async_api import Page, Playwright, async_playwright
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BotBrowser:
|
||||
def __init__(self, headless: bool = True):
|
||||
self._headless = headless
|
||||
self._playwright: Optional[Playwright] = None
|
||||
self._browser = None
|
||||
self._context = None
|
||||
self._page: Optional[Page] = None
|
||||
|
||||
async def launch(self) -> None:
|
||||
self._playwright = await async_playwright().start()
|
||||
self._browser = await self._playwright.chromium.launch(
|
||||
headless=self._headless,
|
||||
args=[
|
||||
"--disable-blink-features=AutomationControlled", "--no-first-run",
|
||||
"--disable-dev-shm-usage", "--window-size=1280,900",
|
||||
],
|
||||
)
|
||||
self._context = await self._browser.new_context(
|
||||
user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36",
|
||||
viewport={"width": 1280, "height": 900},
|
||||
locale="en-US",
|
||||
)
|
||||
self._page = await self._context.new_page()
|
||||
|
||||
async def close(self) -> None:
|
||||
for label, resource, shutdown in (
|
||||
("context", self._context, lambda r: r.close()),
|
||||
("browser", self._browser, lambda r: r.close()),
|
||||
("playwright", self._playwright, lambda r: r.stop()),
|
||||
):
|
||||
if resource is None:
|
||||
continue
|
||||
try:
|
||||
await shutdown(resource)
|
||||
except Exception as e:
|
||||
logger.debug("Browser %s close error: %s", label, e)
|
||||
self._page = None
|
||||
self._context = None
|
||||
self._browser = None
|
||||
self._playwright = None
|
||||
|
||||
@property
|
||||
def page(self) -> Page:
|
||||
return self._page
|
||||
|
||||
async def goto(self, url: str) -> None:
|
||||
try:
|
||||
await self._page.goto(url, wait_until="domcontentloaded", timeout=15000)
|
||||
except Exception as e:
|
||||
logger.debug("goto (domcontentloaded) failed for %s: %s", url[:80], e)
|
||||
try:
|
||||
await self._page.goto(url, timeout=20000)
|
||||
except Exception as e2:
|
||||
logger.warning("goto failed for %s: %s", url[:80], e2)
|
||||
await self._idle(0.5, 1.5)
|
||||
|
||||
async def _idle(self, lo: float = 0.5, hi: float = 2.0) -> None:
|
||||
await asyncio.sleep(random.uniform(lo, hi))
|
||||
|
||||
async def scroll(self, amount: Optional[int] = None) -> None:
|
||||
if amount is None:
|
||||
amount = random.randint(300, 800)
|
||||
try:
|
||||
await self._page.evaluate(f"window.scrollBy(0, {amount})")
|
||||
except Exception as e:
|
||||
logger.debug("scroll failed: %s", e)
|
||||
await self._idle(0.2, 0.6)
|
||||
|
||||
async def scroll_to_bottom(self) -> None:
|
||||
try:
|
||||
await self._page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
||||
except Exception as e:
|
||||
logger.debug("scroll_to_bottom failed: %s", e)
|
||||
|
||||
async def fill(self, sel: str, val: str) -> bool:
|
||||
try:
|
||||
el = self._page.locator(sel).first
|
||||
if await el.count() == 0:
|
||||
logger.debug("fill: selector '%s' not found", sel)
|
||||
return False
|
||||
await el.click(timeout=3000)
|
||||
await self._idle(0.05, 0.15)
|
||||
await el.fill("")
|
||||
for ch in val:
|
||||
await self._page.keyboard.type(ch, delay=random.randint(20, 60))
|
||||
if ch == " ":
|
||||
await asyncio.sleep(random.uniform(0.02, 0.08))
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug("fill failed for '%s': %s", sel, e)
|
||||
return False
|
||||
|
||||
async def click(self, sel: str) -> bool:
|
||||
try:
|
||||
el = self._page.locator(sel).first
|
||||
if await el.count() == 0:
|
||||
logger.debug("click: selector '%s' not found", sel)
|
||||
return False
|
||||
box = await el.bounding_box()
|
||||
if box:
|
||||
tx = box["x"] + box["width"] / 2 + random.uniform(-3, 3)
|
||||
ty = box["y"] + box["height"] / 2 + random.uniform(-3, 3)
|
||||
await self._page.mouse.move(tx, ty, steps=random.randint(8, 18))
|
||||
await self._idle(0.02, 0.06)
|
||||
await el.click(timeout=5000)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug("click failed for '%s': %s", sel, e)
|
||||
return False
|
||||
|
||||
async def text(self, sel: str) -> str:
|
||||
try:
|
||||
return (await self._page.locator(sel).first.inner_text()) or ""
|
||||
except Exception as e:
|
||||
logger.debug("text failed for '%s': %s", sel, e)
|
||||
return ""
|
||||
|
||||
async def attr(self, sel: str, attr: str) -> str:
|
||||
try:
|
||||
val = await self._page.locator(sel).first.get_attribute(attr)
|
||||
return val or ""
|
||||
except Exception as e:
|
||||
logger.debug("attr '%s' on '%s' failed: %s", attr, sel, e)
|
||||
return ""
|
||||
|
||||
async def html(self) -> str:
|
||||
try:
|
||||
return await self._page.content()
|
||||
except Exception as e:
|
||||
logger.debug("html() failed: %s", e)
|
||||
return ""
|
||||
|
||||
async def url(self) -> str:
|
||||
try:
|
||||
return self._page.url
|
||||
except Exception as e:
|
||||
logger.debug("url() failed: %s", e)
|
||||
return ""
|
||||
|
||||
async def title(self) -> str:
|
||||
try:
|
||||
return await self._page.title()
|
||||
except Exception as e:
|
||||
logger.debug("title() failed: %s", e)
|
||||
return ""
|
||||
|
||||
async def reload(self) -> None:
|
||||
try:
|
||||
await self._page.reload(timeout=15000)
|
||||
await self._idle()
|
||||
except Exception as e:
|
||||
logger.debug("reload failed: %s", e)
|
||||
@@ -0,0 +1,58 @@
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
|
||||
|
||||
BASE_URL_DEFAULT = "https://pravda.education"
|
||||
API_URL_DEFAULT = INTERNAL_GATEWAY_URL
|
||||
NEWS_API_DEFAULT = "https://news.app.molodetz.nl/api"
|
||||
MODEL_DEFAULT = INTERNAL_MODEL
|
||||
INPUT_COST_PER_1M_DEFAULT = 0.27
|
||||
OUTPUT_COST_PER_1M_DEFAULT = 1.10
|
||||
|
||||
STATE_DIR = Path.home() / ".devplace_bots"
|
||||
ARTICLE_REGISTRY_PATH = Path.home() / ".dpbot_article_registry.json"
|
||||
|
||||
PROJECT_STATUSES = ["In Development", "Released"]
|
||||
|
||||
PERSONAS = [
|
||||
"enthusiastic_junior",
|
||||
"grumpy_senior",
|
||||
"hobbyist_maker",
|
||||
"academic_type",
|
||||
"minimalist",
|
||||
"storyteller",
|
||||
"rebel",
|
||||
"mentor",
|
||||
]
|
||||
|
||||
CATEGORIES = ["devlog", "showcase", "question", "rant", "fun", "random"]
|
||||
|
||||
GIST_LANGUAGES = [
|
||||
"python", "javascript", "typescript", "bash", "go", "rust",
|
||||
"sql", "html", "css", "java", "c", "cpp", "ruby", "php", "lua",
|
||||
]
|
||||
|
||||
FEED_TOPICS = ["devlog", "showcase", "question", "rant", "fun"]
|
||||
PROJECT_TYPES = ["game", "game_asset", "software", "mobile_app", "website"]
|
||||
|
||||
SEARCH_TERMS = {
|
||||
"enthusiastic_junior": ["vibe coding", "first app", "react", "python", "ai tools"],
|
||||
"grumpy_senior": ["rust", "perl", "legacy", "tech debt", "kubernetes"],
|
||||
"hobbyist_maker": ["arduino", "raspberry pi", "side project", "3d printing", "game"],
|
||||
"academic_type": ["algorithms", "type theory", "distributed systems", "compilers"],
|
||||
"minimalist": ["cli", "vim", "golang", "suckless"],
|
||||
"storyteller": ["postmortem", "war story", "migration", "outage"],
|
||||
"rebel": ["hot take", "overrated", "monorepo", "microservices"],
|
||||
"mentor": ["best practices", "code review", "testing", "career"],
|
||||
}
|
||||
|
||||
MIN_COMMENT_LEN = 40
|
||||
MIN_POST_LEN = 120
|
||||
RESTATEMENT_OVERLAP_THRESHOLD = 0.6
|
||||
|
||||
GENERIC_COMMENT_PHRASES = [
|
||||
"great point", "good point", "i agree", "totally agree", "well said",
|
||||
"nice post", "great post", "thanks for sharing", "interesting read",
|
||||
"this is interesting", "so true", "couldn't agree more", "spot on",
|
||||
"great article", "love this", "this is great", "good read",
|
||||
]
|
||||
@@ -0,0 +1,274 @@
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from devplacepy.services.bot.config import CATEGORIES, GIST_LANGUAGES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LLMClient:
|
||||
def __init__(self, api_key: str, api_url: str, model: str,
|
||||
input_cost_per_1m: float, output_cost_per_1m: float):
|
||||
if not api_key:
|
||||
raise RuntimeError("LLM API key not set")
|
||||
self.api_key = api_key
|
||||
self.api_url = api_url
|
||||
self.model = model
|
||||
self.input_cost_per_1m = input_cost_per_1m
|
||||
self.output_cost_per_1m = output_cost_per_1m
|
||||
self.total_cost = 0.0
|
||||
self.total_calls = 0
|
||||
self.total_in_tokens = 0
|
||||
self.total_out_tokens = 0
|
||||
|
||||
def _call(self, system: str, prompt: str, temperature: float = 0.7) -> str:
|
||||
for attempt in range(3):
|
||||
try:
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"messages": [{"role": "system", "content": system}, {"role": "user", "content": prompt}],
|
||||
"temperature": temperature,
|
||||
}
|
||||
logger.info("LLM >>> model=%s system=%s prompt=%s",
|
||||
self.model, json.dumps(system[:500]), json.dumps(prompt[:500]))
|
||||
data = json.dumps(payload).encode()
|
||||
req = urllib.request.Request(
|
||||
self.api_url, data=data,
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
logger.info("LLM <<< %s", raw[:2000].decode(errors="replace"))
|
||||
result = json.loads(raw)
|
||||
usage = result.get("usage", {})
|
||||
in_tokens = usage.get("prompt_tokens", 0)
|
||||
out_tokens = usage.get("completion_tokens", 0)
|
||||
cost = (in_tokens * self.input_cost_per_1m / 1_000_000) + \
|
||||
(out_tokens * self.output_cost_per_1m / 1_000_000)
|
||||
self.total_cost += cost
|
||||
self.total_calls += 1
|
||||
self.total_in_tokens += in_tokens
|
||||
self.total_out_tokens += out_tokens
|
||||
return self.clean(result["choices"][0]["message"]["content"])
|
||||
except (urllib.error.HTTPError, urllib.error.URLError, json.JSONDecodeError, KeyError) as e:
|
||||
logger.warning("LLM call attempt %d/3 failed: %s", attempt + 1, e)
|
||||
if attempt == 2:
|
||||
raise
|
||||
time.sleep(2 ** attempt)
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def clean(text: str, preserve_md: bool = False) -> str:
|
||||
if not preserve_md:
|
||||
text = re.sub(r"\*\*|__|\*|_", "", text)
|
||||
text = text.replace("-", "-").replace("–", "-")
|
||||
text = text.replace("‘", "'").replace("’", "'")
|
||||
text = text.replace("“", '"').replace("”", '"')
|
||||
text = re.sub(r"\s{2,}", " ", text)
|
||||
return text.strip()
|
||||
|
||||
@staticmethod
|
||||
def strip_md(text: str) -> str:
|
||||
return LLMClient.clean(text)[:200]
|
||||
|
||||
@staticmethod
|
||||
def strip_code_fences(text: str) -> str:
|
||||
text = text.strip()
|
||||
if text.startswith("```"):
|
||||
lines = text.split("\n")
|
||||
if lines and lines[0].lstrip().startswith("```"):
|
||||
lines = lines[1:]
|
||||
if lines and lines[-1].strip().startswith("```"):
|
||||
lines = lines[:-1]
|
||||
text = "\n".join(lines)
|
||||
return text.strip()
|
||||
|
||||
def generate_post(self, title: str, desc: str, persona: str = "", category: str = "") -> str:
|
||||
persona_extras = {
|
||||
"enthusiastic_junior": "Be excited. Use **bold** for emphasis. Short excited sentences. End with a question sometimes.",
|
||||
"grumpy_senior": "Be slightly cynical but helpful. Short blunt sentences. No fluff. Call out bad practices.",
|
||||
"hobbyist_maker": "Be casual and friendly. Mention side projects and tinkering. Use *italics* for tools/libraries.",
|
||||
"academic_type": "Be precise and structured. Use *italics* for terminology. Well thought out arguments.",
|
||||
"storyteller": "Use **bold** for key points. Write anecdotal, narrative style. Longer flowing sentences.",
|
||||
"rebel": "Be informal. Skip punctuation sometimes. Use slang. Type like you're in a hurry.",
|
||||
"mentor": "Be helpful and explanatory. Use **bold** for key takeaways. Include practical advice.",
|
||||
"minimalist": "One paragraph. Short blunt declarative sentences. No markdown. No fluff.",
|
||||
}
|
||||
category_extras = {
|
||||
"devlog": "Write it as a personal devlog entry - share your learning process and insights about this topic.",
|
||||
"showcase": "Write it as a showcase - express genuine excitement about what makes this impressive.",
|
||||
"question": "Write it as a discussion starter - ask thoughtful questions, invite others to share perspectives.",
|
||||
"rant": "Write it as an opinionated rant - strong viewpoint, passionate criticism, but keep it substantive.",
|
||||
"fun": "Write it lighthearted and playful, but stay tied to the actual tech topic. Humor about the technology itself, not off-topic jokes or lyrics.",
|
||||
"random": "Be natural and conversational - share your thoughts like any casual discussion.",
|
||||
}
|
||||
persona_extra = f" {persona_extras.get(persona, 'Be casual. No markdown.')}" if persona else " Be casual. No markdown."
|
||||
category_extra = f" {category_extras.get(category, '')}" if category else ""
|
||||
preserve = persona in ("enthusiastic_junior", "hobbyist_maker", "academic_type", "storyteller", "mentor")
|
||||
text = self._call(
|
||||
f"You are a dev writing a social media post reacting to tech news. Write 2-4 short paragraphs."
|
||||
f"{persona_extra}{category_extra} Do not summarize the article; assume the reader already saw it. "
|
||||
f"Add your own value: a concrete opinion, an implication, a personal experience, or a pointed question. "
|
||||
f"Reference a specific detail rather than restating the headline. No em dashes. Under 300 words.",
|
||||
f"News: {title}\n\n{desc[:800]}",
|
||||
)
|
||||
return self.clean(text, preserve_md=preserve)
|
||||
|
||||
def select_category(self, title: str, desc: str) -> str:
|
||||
text = self._call(
|
||||
"You are a classifier. Given tech news, choose the single best post category. Reply with ONLY the category name.",
|
||||
f"Categories:\n- devlog: learning journey, building something, personal dev experience\n- showcase: impressive release, new tool, achievement worth highlighting\n- question: uncertainty, asking for advice, curiosity about implications\n- rant: frustration, bad practices, criticism of industry trends\n- fun: amusing, surprising, entertaining but not deeply serious\n- random: anything else that doesn't clearly fit above\n\nNews: {title}\n\n{desc[:600]}",
|
||||
temperature=0.2,
|
||||
)
|
||||
for c in CATEGORIES:
|
||||
if c in text.lower():
|
||||
return c
|
||||
return "random"
|
||||
|
||||
def generate_comment(self, post_snippet: str, persona: str = "",
|
||||
mention_target: str = "", parent_context: str = "") -> str:
|
||||
extras = {
|
||||
"enthusiastic_junior": "Be excited. Use **bold** for agreement. Short replies.",
|
||||
"grumpy_senior": "Be blunt and short. No markdown. One sarcastic remark or actual advice.",
|
||||
"rebel": "Super casual. Skip caps sometimes. Short.",
|
||||
"mentor": "Be helpful. Use **bold** for key point.",
|
||||
"storyteller": "Share a quick related story. Use *italics* for emphasis.",
|
||||
"minimalist": "Shortest possible reply. One sentence max.",
|
||||
}
|
||||
extra = f" {extras.get(persona, 'Be casual. No markdown.')}" if persona else " Be casual. No markdown."
|
||||
preserve = persona in ("enthusiastic_junior", "mentor", "storyteller")
|
||||
mention_rule = ""
|
||||
if mention_target:
|
||||
mention_rule = (
|
||||
f" Address @{mention_target} naturally in the first sentence "
|
||||
f"(weave the @{mention_target} mention inline, do not append it at the end)."
|
||||
)
|
||||
ctx = ""
|
||||
if parent_context:
|
||||
ctx = f"\n\nReplying to this comment:\n{parent_context[:800]}"
|
||||
system = (
|
||||
f"You are a dev replying to a post. Write 1-3 short sentences.{extra}{mention_rule} "
|
||||
"No em dashes. Reference a specific detail from the post. "
|
||||
"Do not open with or rely on generic filler like 'great point', 'I agree', 'nice', "
|
||||
"'thanks for sharing', or 'interesting'. Add something real: a concrete experience, "
|
||||
"a caveat, a counterexample, respectful disagreement, or a pointed question. "
|
||||
"Never just paraphrase or agree blandly."
|
||||
)
|
||||
reply = self.clean(
|
||||
self._call(system, f"Post:\n{post_snippet[:1500]}{ctx}").strip(),
|
||||
preserve_md=preserve,
|
||||
)
|
||||
return reply[:2000]
|
||||
|
||||
@staticmethod
|
||||
def _overlap_ratio(text: str, context: str) -> float:
|
||||
def tokens(s: str) -> list[str]:
|
||||
return [w for w in re.findall(r"[a-z0-9]+", s.lower()) if len(w) > 3]
|
||||
|
||||
candidate = tokens(text)
|
||||
source = set(tokens(context))
|
||||
if not candidate or not source:
|
||||
return 0.0
|
||||
hits = sum(1 for w in candidate if w in source)
|
||||
return hits / len(candidate)
|
||||
|
||||
def quality_check(self, kind: str, text: str, context: str = "") -> tuple[bool, str]:
|
||||
from devplacepy.services.bot.config import (
|
||||
MIN_COMMENT_LEN, MIN_POST_LEN, RESTATEMENT_OVERLAP_THRESHOLD, GENERIC_COMMENT_PHRASES,
|
||||
)
|
||||
stripped = self.clean(text or "").strip()
|
||||
min_len = MIN_COMMENT_LEN if kind == "comment" else MIN_POST_LEN
|
||||
if len(stripped) < min_len:
|
||||
return False, f"too short ({len(stripped)} < {min_len})"
|
||||
lowered = stripped.lower()
|
||||
if kind == "comment":
|
||||
for phrase in GENERIC_COMMENT_PHRASES:
|
||||
if phrase in lowered:
|
||||
return False, f"generic phrase '{phrase}'"
|
||||
if context and self._overlap_ratio(stripped, context) > RESTATEMENT_OVERLAP_THRESHOLD:
|
||||
return False, "restates the source"
|
||||
verdict = self._call(
|
||||
"You are a strict content quality reviewer for a developer community. "
|
||||
"Reject text that is generic, low-effort, pure filler, or merely summarizes its "
|
||||
"source without adding an opinion, experience, or insight. "
|
||||
"Reply with exactly PASS or 'FAIL: <short reason>'.",
|
||||
f"Type: {kind}\nText:\n{stripped[:1200]}",
|
||||
temperature=0.0,
|
||||
)
|
||||
if verdict.strip().upper().startswith("PASS"):
|
||||
return True, "ok"
|
||||
return False, verdict.strip()[:120] or "judge rejected"
|
||||
|
||||
def generate_bug(self, topic: str) -> tuple[str, str]:
|
||||
text = self._call(
|
||||
"Write a bug report. One line title. Then 2-3 sentences describing what happened and what should have happened. Like a real dev reporting a bug. No em dashes.",
|
||||
f"Bug: {topic}",
|
||||
)
|
||||
lines = text.strip().split("\n")
|
||||
title = lines[0].strip().lstrip("#").strip()[:120] or f"Bug report: {topic}"
|
||||
desc = " ".join(line.lstrip("-* ").strip() for line in lines[1:] if line.strip())
|
||||
return title, desc[:2000] or text[:500]
|
||||
|
||||
def generate_bio(self) -> str:
|
||||
return self._call(
|
||||
"Write a short bio for a developer profile. 1-2 sentences. What they work on, what they like. Sound human. No em dashes.",
|
||||
"Bio:",
|
||||
)
|
||||
|
||||
def generate_profile_fields(self, handle: str) -> tuple[str, str, str]:
|
||||
location = self.clean(self._call(
|
||||
"Name one plausible city and country for a developer. Reply with ONLY 'City, Country'. No extra words.",
|
||||
"Location:", temperature=0.9,
|
||||
))[:80]
|
||||
slug = re.sub(r"[^a-z0-9_-]", "", handle.lower()) or "dev"
|
||||
git_link = f"https://github.com/{slug}"
|
||||
website = f"https://{slug}.dev"
|
||||
return location, git_link, website
|
||||
|
||||
def generate_dm(self, persona: str = "", context: str = "") -> str:
|
||||
extra = {
|
||||
"enthusiastic_junior": "Be friendly and excited.",
|
||||
"grumpy_senior": "Be blunt but not rude.",
|
||||
"rebel": "Be super casual.",
|
||||
"mentor": "Be warm and encouraging.",
|
||||
"storyteller": "Open with a small hook.",
|
||||
"minimalist": "One short sentence.",
|
||||
}.get(persona, "Be casual and friendly.")
|
||||
ctx = f"\n\nEarlier message:\n{context[:400]}" if context else ""
|
||||
return self.clean(self._call(
|
||||
f"Write one short, friendly direct message to another developer. 1-2 sentences. {extra} No em dashes.",
|
||||
f"Write a DM to start or continue a chat.{ctx}",
|
||||
))[:500]
|
||||
|
||||
def generate_project_title(self) -> str:
|
||||
return self._call("Come up with a project name. 2-4 words. A tool, game, or app idea. No em dashes.", "Name:", temperature=0.9)
|
||||
|
||||
def generate_project_desc(self, title: str) -> str:
|
||||
return self._call(
|
||||
"You are a developer. Write a compelling 2-3 sentence project description including tech stack and purpose.",
|
||||
f"Project: {title}",
|
||||
)
|
||||
|
||||
def generate_gist(self, persona: str = "") -> tuple[str, str, str, str]:
|
||||
language = random.choice(GIST_LANGUAGES)
|
||||
title = self.clean(self._call(
|
||||
"Name a short, useful code snippet. 2-5 words. No quotes. No markdown.",
|
||||
f"Language: {language}. Snippet name:",
|
||||
temperature=0.9,
|
||||
))[:120]
|
||||
description = self.clean(self._call(
|
||||
"Write a one-sentence description of what a code snippet does, like a dev sharing something handy. No em dashes.",
|
||||
f"Snippet: {title}\nLanguage: {language}",
|
||||
))[:400]
|
||||
code = self.strip_code_fences(self._call(
|
||||
f"Write a short, correct, self-contained {language} snippet of 5 to 20 lines for the description. "
|
||||
"Output ONLY raw code. No markdown fences. No commentary.",
|
||||
f"Title: {title}\nDescription: {description}\nLanguage: {language}",
|
||||
temperature=0.4,
|
||||
))[:4000]
|
||||
return title, description, language, code
|
||||
@@ -0,0 +1,31 @@
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NewsFetcher:
|
||||
TTL_SECONDS = 1800
|
||||
|
||||
def __init__(self, news_api: str):
|
||||
self.news_api = news_api
|
||||
self.articles: list[dict] = []
|
||||
self.fetched_at: float = 0.0
|
||||
|
||||
def fetch(self) -> list[dict]:
|
||||
fresh = self.articles and (time.monotonic() - self.fetched_at) < self.TTL_SECONDS
|
||||
if fresh:
|
||||
return self.articles
|
||||
try:
|
||||
req = urllib.request.Request(self.news_api, headers={"User-Agent": "dpbot/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
articles = json.loads(resp.read()).get("articles", [])
|
||||
if articles:
|
||||
self.articles = articles
|
||||
self.fetched_at = time.monotonic()
|
||||
logger.info("Fetched %d news articles", len(self.articles))
|
||||
except Exception as e:
|
||||
logger.warning("News fetch failed: %s", e)
|
||||
return self.articles
|
||||
@@ -0,0 +1,94 @@
|
||||
import fcntl
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ArticleRegistry:
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
|
||||
def _lock(self) -> tuple:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
fd = os.open(str(self.path), os.O_RDWR | os.O_CREAT)
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
data = {}
|
||||
if os.path.getsize(str(self.path)) > 0:
|
||||
chunks = []
|
||||
while True:
|
||||
chunk = os.read(fd, 65536)
|
||||
if not chunk:
|
||||
break
|
||||
chunks.append(chunk)
|
||||
raw = b"".join(chunks).decode(errors="replace")
|
||||
if raw.strip():
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
data = {}
|
||||
if not isinstance(data, dict):
|
||||
data = {}
|
||||
now = datetime.now()
|
||||
kept: dict = {}
|
||||
for title, meta in data.items():
|
||||
if not isinstance(meta, dict):
|
||||
continue
|
||||
ts = meta.get("time", "")
|
||||
try:
|
||||
age_days = (now - datetime.fromisoformat(ts)).total_seconds() / 86400
|
||||
except (ValueError, TypeError):
|
||||
age_days = 0
|
||||
if age_days < 7:
|
||||
kept[title] = meta
|
||||
return fd, kept
|
||||
except Exception as e:
|
||||
logger.warning("Failed to lock article registry: %s", e)
|
||||
return None, {}
|
||||
|
||||
@staticmethod
|
||||
def _unlock(fd) -> None:
|
||||
if fd is not None:
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
os.close(fd)
|
||||
except Exception as e:
|
||||
logger.debug("Failed to unlock article registry: %s", e)
|
||||
|
||||
def reserve(self, title: str, owner: str) -> bool:
|
||||
fd, registry = self._lock()
|
||||
if fd is None:
|
||||
return True
|
||||
try:
|
||||
if title in registry:
|
||||
return False
|
||||
registry[title] = {"bot": owner, "time": datetime.now().isoformat()}
|
||||
self.path.write_text(json.dumps(registry, indent=2, default=str))
|
||||
return True
|
||||
finally:
|
||||
self._unlock(fd)
|
||||
|
||||
def reserve_unused(self, articles: list[dict], owner: str, clean) -> Optional[dict]:
|
||||
if not articles:
|
||||
return None
|
||||
fd, registry = self._lock()
|
||||
if fd is None:
|
||||
return random.choice(articles) if articles else None
|
||||
try:
|
||||
used = set(registry.keys())
|
||||
random.shuffle(articles)
|
||||
for a in articles:
|
||||
title = clean(a.get("title", "")[:200])
|
||||
if title and title not in used:
|
||||
registry[title] = {"bot": owner, "time": datetime.now().isoformat()}
|
||||
self.path.write_text(json.dumps(registry, indent=2, default=str))
|
||||
return a
|
||||
return None
|
||||
finally:
|
||||
self._unlock(fd)
|
||||
@@ -0,0 +1,18 @@
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from devplacepy.services.bot import config
|
||||
|
||||
|
||||
@dataclass
|
||||
class BotRuntimeConfig:
|
||||
state_path: Path
|
||||
base_url: str = config.BASE_URL_DEFAULT
|
||||
api_url: str = config.API_URL_DEFAULT
|
||||
news_api: str = config.NEWS_API_DEFAULT
|
||||
model: str = config.MODEL_DEFAULT
|
||||
api_key: str = ""
|
||||
input_cost_per_1m: float = config.INPUT_COST_PER_1M_DEFAULT
|
||||
output_cost_per_1m: float = config.OUTPUT_COST_PER_1M_DEFAULT
|
||||
headless: bool = True
|
||||
registry_path: Path = config.ARTICLE_REGISTRY_PATH
|
||||
@@ -0,0 +1,215 @@
|
||||
import asyncio
|
||||
import importlib
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from devplacepy.services.base import BaseService, ConfigField
|
||||
from devplacepy.services.bot import config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BotsService(BaseService):
|
||||
default_enabled = False
|
||||
min_interval = 5
|
||||
title = "Bots"
|
||||
description = (
|
||||
"Runs a fleet of Playwright-driven AI personas that browse a DevPlace "
|
||||
"instance and post, comment, vote, and react like real users. Fleet size, "
|
||||
"target site, and LLM settings are configurable, with live cost and usage "
|
||||
"metrics per bot."
|
||||
)
|
||||
config_fields = [
|
||||
ConfigField("bot_fleet_size", "Fleet size (bots)", type="int", default=1, minimum=0, maximum=20,
|
||||
help="Number of concurrent bot accounts to run.", group="Fleet"),
|
||||
ConfigField("bot_headless", "Headless browser", type="bool", default=True,
|
||||
help="Run Chromium headless. Disable only for local debugging on a desktop.",
|
||||
group="Fleet"),
|
||||
ConfigField("bot_max_actions", "Max actions per bot", type="int", default=0, minimum=0,
|
||||
help="Stop a bot after this many actions (0 = run indefinitely).", group="Fleet"),
|
||||
ConfigField("bot_base_url", "Target site URL", type="url", default=config.BASE_URL_DEFAULT,
|
||||
help="DevPlace instance the bots browse and post to.", group="Target"),
|
||||
ConfigField("bot_news_api", "News API URL", type="url", default=config.NEWS_API_DEFAULT,
|
||||
help="Source of articles the bots react to.", group="Target"),
|
||||
ConfigField("bot_api_url", "LLM API URL", type="url", default=config.API_URL_DEFAULT,
|
||||
help="OpenAI-compatible chat-completions endpoint.", group="LLM"),
|
||||
ConfigField("bot_model", "LLM model", type="str", default=config.MODEL_DEFAULT,
|
||||
help="Model name sent to the LLM API.", group="LLM"),
|
||||
ConfigField("bot_api_key", "LLM API key", type="password", default="", secret=True,
|
||||
help="Defaults to the gateway's internal key (the bots use the local AI gateway).",
|
||||
group="LLM"),
|
||||
ConfigField("bot_input_cost_per_1m", "Input cost per 1M tokens ($)", type="float",
|
||||
default=config.INPUT_COST_PER_1M_DEFAULT, minimum=0,
|
||||
help="Used to compute live spend per bot and across the fleet.", group="LLM"),
|
||||
ConfigField("bot_output_cost_per_1m", "Output cost per 1M tokens ($)", type="float",
|
||||
default=config.OUTPUT_COST_PER_1M_DEFAULT, minimum=0,
|
||||
help="Used to compute live spend per bot and across the fleet.", group="LLM"),
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="bots", interval_seconds=15)
|
||||
self._fleet = {}
|
||||
self._state_dir = config.STATE_DIR
|
||||
self._cost_baseline = None
|
||||
self._cost_baseline_at = None
|
||||
|
||||
def _resolve_api_key(self, cfg: dict) -> str:
|
||||
from devplacepy.database import internal_gateway_key
|
||||
return cfg["bot_api_key"] or internal_gateway_key()
|
||||
|
||||
@staticmethod
|
||||
def _ensure_importable() -> None:
|
||||
importlib.import_module("devplacepy.services.bot.bot")
|
||||
|
||||
async def run_once(self) -> None:
|
||||
cfg = self.get_config()
|
||||
desired = cfg["bot_fleet_size"]
|
||||
api_key = self._resolve_api_key(cfg)
|
||||
if not api_key:
|
||||
if self._fleet:
|
||||
await self._stop_fleet()
|
||||
self.log("No LLM API key available (bot_api_key / gateway internal key); fleet idle")
|
||||
return
|
||||
try:
|
||||
self._ensure_importable()
|
||||
except ImportError as e:
|
||||
if self._fleet:
|
||||
await self._stop_fleet()
|
||||
self.log(f"Bot dependencies unavailable ({e}); install the 'bots' extra")
|
||||
return
|
||||
|
||||
for slot in list(self._fleet):
|
||||
task = self._fleet[slot]["task"]
|
||||
if task.done():
|
||||
self._report_exit(slot, task)
|
||||
del self._fleet[slot]
|
||||
|
||||
while len(self._fleet) > desired:
|
||||
await self._stop_slot(max(self._fleet))
|
||||
|
||||
for slot in range(desired):
|
||||
if slot not in self._fleet:
|
||||
self._launch_slot(slot, cfg, api_key)
|
||||
|
||||
self.log(f"Fleet: {len(self._fleet)}/{desired} bots active")
|
||||
|
||||
def _report_exit(self, slot: int, task) -> None:
|
||||
if task.cancelled():
|
||||
return
|
||||
exc = task.exception()
|
||||
if exc:
|
||||
self.log(f"bot{slot} exited with error: {exc}; relaunching")
|
||||
else:
|
||||
self.log(f"bot{slot} finished; relaunching")
|
||||
|
||||
def _launch_slot(self, slot: int, cfg: dict, api_key: str) -> None:
|
||||
from devplacepy.services.bot.bot import DevPlaceBot
|
||||
from devplacepy.services.bot.runtime import BotRuntimeConfig
|
||||
rc = BotRuntimeConfig(
|
||||
state_path=self._state_dir / f"state_slot{slot}.json",
|
||||
base_url=cfg["bot_base_url"],
|
||||
api_url=cfg["bot_api_url"],
|
||||
news_api=cfg["bot_news_api"],
|
||||
model=cfg["bot_model"],
|
||||
api_key=api_key,
|
||||
input_cost_per_1m=cfg["bot_input_cost_per_1m"],
|
||||
output_cost_per_1m=cfg["bot_output_cost_per_1m"],
|
||||
headless=cfg["bot_headless"],
|
||||
registry_path=config.ARTICLE_REGISTRY_PATH,
|
||||
)
|
||||
bot = DevPlaceBot(rc, on_event=self.log)
|
||||
task = asyncio.create_task(self._run_slot(slot, bot, cfg["bot_max_actions"]))
|
||||
self._fleet[slot] = {"bot": bot, "task": task}
|
||||
self.log(f"Launched bot{slot}")
|
||||
|
||||
async def _run_slot(self, slot: int, bot, max_actions: int) -> None:
|
||||
try:
|
||||
await bot.run_forever(action_limit=max_actions)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.log(f"bot{slot} crashed: {e}")
|
||||
|
||||
async def on_disable(self) -> None:
|
||||
await self._stop_fleet()
|
||||
|
||||
async def _stop_fleet(self) -> None:
|
||||
for slot in list(self._fleet):
|
||||
await self._stop_slot(slot)
|
||||
|
||||
async def _stop_slot(self, slot: int) -> None:
|
||||
entry = self._fleet.pop(slot, None)
|
||||
if not entry:
|
||||
return
|
||||
task = entry["task"]
|
||||
task.cancel()
|
||||
try:
|
||||
await asyncio.wait_for(task, timeout=30)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError):
|
||||
pass
|
||||
except Exception as e:
|
||||
self.log(f"bot{slot} stop error: {e}")
|
||||
self.log(f"Stopped bot{slot}")
|
||||
|
||||
def collect_metrics(self) -> dict:
|
||||
rows = []
|
||||
total_cost = 0.0
|
||||
total_calls = total_in = total_out = 0
|
||||
posts = comments = votes = running = 0
|
||||
for slot in sorted(self._fleet):
|
||||
entry = self._fleet[slot]
|
||||
bot = entry["bot"]
|
||||
alive = not entry["task"].done()
|
||||
running += 1 if alive else 0
|
||||
llm = getattr(bot, "llm", None)
|
||||
st = getattr(bot, "state", None)
|
||||
cost = llm.total_cost if llm else 0.0
|
||||
calls = llm.total_calls if llm else 0
|
||||
total_cost += cost
|
||||
total_calls += calls
|
||||
total_in += llm.total_in_tokens if llm else 0
|
||||
total_out += llm.total_out_tokens if llm else 0
|
||||
if st:
|
||||
posts += st.created_posts
|
||||
comments += st.comments_posted
|
||||
votes += st.votes_cast
|
||||
rows.append([
|
||||
f"bot{slot}",
|
||||
(st.username if st else "") or "-",
|
||||
(st.persona if st else "") or "-",
|
||||
"running" if alive else "stopped",
|
||||
st.created_posts if st else 0,
|
||||
st.comments_posted if st else 0,
|
||||
st.votes_cast if st else 0,
|
||||
calls,
|
||||
f"${cost:.4f}",
|
||||
])
|
||||
now = datetime.now(timezone.utc)
|
||||
if self._started_at is None:
|
||||
self._cost_baseline = None
|
||||
self._cost_baseline_at = None
|
||||
elif self._cost_baseline is None or self._cost_baseline[0] != self._started_at:
|
||||
self._cost_baseline = (self._started_at, total_cost)
|
||||
self._cost_baseline_at = now
|
||||
window = (now - self._cost_baseline_at).total_seconds() if self._cost_baseline_at else 0.0
|
||||
observed_cost = max(0.0, total_cost - self._cost_baseline[1]) if self._cost_baseline else 0.0
|
||||
cost_per_hour = observed_cost / window * 3600 if window > 0 else 0.0
|
||||
projected_24h = observed_cost / window * 86400 if window > 0 else 0.0
|
||||
stats = [
|
||||
{"label": "Bots running", "value": running},
|
||||
{"label": "Fleet cost", "value": f"${total_cost:.4f}"},
|
||||
{"label": "Observed window", "value": f"{window:.0f}s"},
|
||||
{"label": "Cost rate", "value": f"${cost_per_hour:.4f}/h"},
|
||||
{"label": "Projected 24h cost", "value": f"${projected_24h:.2f}"},
|
||||
{"label": "LLM calls", "value": total_calls},
|
||||
{"label": "Tokens in", "value": total_in},
|
||||
{"label": "Tokens out", "value": total_out},
|
||||
{"label": "Posts", "value": posts},
|
||||
{"label": "Comments", "value": comments},
|
||||
{"label": "Votes", "value": votes},
|
||||
]
|
||||
table = {
|
||||
"columns": ["Bot", "User", "Persona", "Status", "Posts", "Comments", "Votes", "Calls", "Cost"],
|
||||
"rows": rows,
|
||||
}
|
||||
return {"stats": stats, "table": table}
|
||||
@@ -0,0 +1,65 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BotState:
|
||||
username: str = ""
|
||||
email: str = ""
|
||||
password: str = ""
|
||||
uid: str = ""
|
||||
persona: str = ""
|
||||
log: list[str] = field(default_factory=list)
|
||||
known_users: list[str] = field(default_factory=list)
|
||||
known_posts: list[str] = field(default_factory=list)
|
||||
created_posts: int = 0
|
||||
posted_content_hashes: list[str] = field(default_factory=list)
|
||||
comments_posted: int = 0
|
||||
quality_rejections: int = 0
|
||||
votes_cast: int = 0
|
||||
bugs_filed: int = 0
|
||||
projects_created: int = 0
|
||||
gists_created: int = 0
|
||||
profiles_viewed: int = 0
|
||||
started_at: str = ""
|
||||
own_post_urls: list[str] = field(default_factory=list)
|
||||
voted_post_ids: list[str] = field(default_factory=list)
|
||||
commented_post_ids: list[str] = field(default_factory=list)
|
||||
notifications_seen: list[str] = field(default_factory=list)
|
||||
mentions_received: int = 0
|
||||
mentions_replied: int = 0
|
||||
replies_to_own_posts: int = 0
|
||||
news_comments: int = 0
|
||||
projects_starred: int = 0
|
||||
messages_sent: int = 0
|
||||
searches_run: int = 0
|
||||
poll_votes_cast: int = 0
|
||||
reactions_cast: int = 0
|
||||
polls_voted: list[str] = field(default_factory=list)
|
||||
reacted_targets: list[str] = field(default_factory=list)
|
||||
reading_speed_wpm: int = 0
|
||||
total_cost: float = 0.0
|
||||
total_calls: int = 0
|
||||
total_in_tokens: int = 0
|
||||
total_out_tokens: int = 0
|
||||
|
||||
def save(self, path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(asdict(self), indent=2, default=str))
|
||||
os.replace(str(tmp), str(path))
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> "BotState":
|
||||
if path.exists():
|
||||
try:
|
||||
data = json.loads(path.read_text())
|
||||
return cls(**data)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load state: %s", e)
|
||||
return cls()
|
||||
@@ -0,0 +1,5 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .service import DeviiService
|
||||
|
||||
__all__ = ["DeviiService"]
|
||||
@@ -0,0 +1,6 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,7 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .catalog import ACTIONS, PLATFORM_CATALOG
|
||||
from .dispatcher import Dispatcher
|
||||
from .spec import Action, Catalog, Param
|
||||
|
||||
__all__ = ["ACTIONS", "PLATFORM_CATALOG", "Dispatcher", "Action", "Catalog", "Param"]
|
||||
@@ -0,0 +1,149 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .spec import Action, Param
|
||||
|
||||
|
||||
def arg(name: str, description: str, required: bool = False, kind: str = "string") -> Param:
|
||||
return Param(name=name, location="body", description=description, required=required, type=kind)
|
||||
|
||||
|
||||
DEVII = (
|
||||
"Controls devii, the on-screen avatar that is you rendered as a classic animated "
|
||||
"desktop character. Only effective in the web interface; in other interfaces it reports "
|
||||
"that no avatar is attached."
|
||||
)
|
||||
|
||||
AVATAR_ACTIONS: tuple[Action, ...] = (
|
||||
Action(
|
||||
name="avatar_show",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Show devii (yourself) on screen",
|
||||
description=DEVII,
|
||||
handler="avatar",
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="avatar_hide",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Hide devii from the screen (plays a goodbye/hide animation)",
|
||||
description=(
|
||||
DEVII + " This already plays a goodbye animation, so do not queue a separate "
|
||||
"wave/goodbye animation immediately before calling it - hiding interrupts a still-"
|
||||
"queued animation."
|
||||
),
|
||||
handler="avatar",
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="avatar_speak",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Show a line of text in devii's speech balloon",
|
||||
description=DEVII,
|
||||
handler="avatar",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
arg("text", "The text devii should say.", required=True),
|
||||
arg("hold", "Keep the balloon open until the next action.", kind="boolean"),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="avatar_list_animations",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="List every animation the current character supports",
|
||||
description=DEVII,
|
||||
handler="avatar",
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="avatar_play_animation",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Play a specific named animation (use avatar_list_animations first)",
|
||||
description=(
|
||||
DEVII + " A following avatar_hide or avatar_stop interrupts an animation that is "
|
||||
"still playing, so do not hide immediately after if you want it to finish."
|
||||
),
|
||||
handler="avatar",
|
||||
requires_auth=False,
|
||||
params=(arg("name", "Exact animation name.", required=True),),
|
||||
),
|
||||
Action(
|
||||
name="avatar_random_animation",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Play a random non-idle animation to react expressively",
|
||||
description=DEVII,
|
||||
handler="avatar",
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="avatar_move_to",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Walk devii to a pixel position on screen",
|
||||
description=DEVII,
|
||||
handler="avatar",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
arg("x", "Target x in pixels.", required=True, kind="integer"),
|
||||
arg("y", "Target y in pixels.", required=True, kind="integer"),
|
||||
arg("duration", "Movement duration in milliseconds.", kind="integer"),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="avatar_gesture_at",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Make devii gesture toward a pixel position",
|
||||
description=DEVII,
|
||||
handler="avatar",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
arg("x", "Target x in pixels.", required=True, kind="integer"),
|
||||
arg("y", "Target y in pixels.", required=True, kind="integer"),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="avatar_get_viewport",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Get the screen size and devii's current position",
|
||||
description=DEVII,
|
||||
handler="avatar",
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="avatar_stop",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Stop all of devii's queued animations and speech immediately",
|
||||
description=DEVII,
|
||||
handler="avatar",
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="avatar_list_characters",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="List every character devii can appear as",
|
||||
description=DEVII,
|
||||
handler="avatar",
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="avatar_switch_character",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Change which character devii is rendered as",
|
||||
description=DEVII,
|
||||
handler="avatar",
|
||||
requires_auth=False,
|
||||
params=(arg("name", "Character name (e.g. Clippy, Merlin, Bonzi, Genie, Rover).", required=True),),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,692 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .spec import Action, Catalog, Param
|
||||
|
||||
|
||||
def path(name: str, description: str, required: bool = True) -> Param:
|
||||
return Param(name=name, location="path", description=description, required=required)
|
||||
|
||||
|
||||
def query(name: str, description: str, required: bool = False) -> Param:
|
||||
return Param(name=name, location="query", description=description, required=required)
|
||||
|
||||
|
||||
def body(name: str, description: str, required: bool = False) -> Param:
|
||||
return Param(name=name, location="body", description=description, required=required)
|
||||
|
||||
|
||||
def upload(name: str, description: str) -> Param:
|
||||
return Param(name=name, location="file", description=description, required=True)
|
||||
|
||||
|
||||
ATTACHMENTS = "Comma separated attachment uids returned by upload_file."
|
||||
TARGET_TYPE = "Target type, e.g. post, comment, project, gist."
|
||||
|
||||
ACTIONS: tuple[Action, ...] = (
|
||||
Action(
|
||||
name="auth_status",
|
||||
method="GET",
|
||||
path="",
|
||||
summary="Report whether the current session is authenticated",
|
||||
handler="status",
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="login",
|
||||
method="POST",
|
||||
path="/auth/login",
|
||||
summary="Authenticate with email and password and start a session",
|
||||
handler="login",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
body("email", "Account email address.", required=True),
|
||||
body("password", "Account password.", required=True),
|
||||
body("remember_me", "Keep the session alive longer ('on' or '')."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="logout",
|
||||
method="GET",
|
||||
path="/auth/logout",
|
||||
summary="End the current session",
|
||||
handler="logout",
|
||||
requires_auth=True,
|
||||
),
|
||||
Action(
|
||||
name="signup",
|
||||
method="POST",
|
||||
path="/auth/signup",
|
||||
summary="Create a new account",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
body("username", "Desired username.", required=True),
|
||||
body("email", "Email address.", required=True),
|
||||
body("password", "Password (minimum six characters).", required=True),
|
||||
body("confirm_password", "Password confirmation.", required=True),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="forgot_password",
|
||||
method="POST",
|
||||
path="/auth/forgot-password",
|
||||
summary="Request a password reset email",
|
||||
requires_auth=False,
|
||||
params=(body("email", "Account email address.", required=True),),
|
||||
),
|
||||
Action(
|
||||
name="reset_password",
|
||||
method="POST",
|
||||
path="/auth/reset-password/{token}",
|
||||
summary="Reset a password using a reset token",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
path("token", "Reset token from the email link."),
|
||||
body("password", "New password.", required=True),
|
||||
body("confirm_password", "New password confirmation.", required=True),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="view_feed",
|
||||
method="GET",
|
||||
path="/feed",
|
||||
summary="View the activity feed",
|
||||
params=(
|
||||
query("tab", "Feed tab to view."),
|
||||
query("topic", "Filter by topic."),
|
||||
query("before", "Pagination cursor."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="create_post",
|
||||
method="POST",
|
||||
path="/posts/create",
|
||||
summary="Create a new post",
|
||||
params=(
|
||||
body("content", "Post body text.", required=True),
|
||||
body("title", "Optional post title."),
|
||||
body("topic", "Optional topic."),
|
||||
body("project_uid", "Attach the post to a project uid."),
|
||||
body("attachment_uids", ATTACHMENTS),
|
||||
body("poll_question", "Optional poll question."),
|
||||
body("poll_options", "Poll options as a JSON array of strings, or one option per line, or comma separated. At least two are required for the poll to be created."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="view_post",
|
||||
method="GET",
|
||||
path="/posts/{post_slug}",
|
||||
summary="View a single post by slug",
|
||||
params=(path("post_slug", "Exact post slug copied from a /posts/... link in a feed or listing response; do not build it from the title."),),
|
||||
),
|
||||
Action(
|
||||
name="edit_post",
|
||||
method="POST",
|
||||
path="/posts/edit/{post_slug}",
|
||||
summary="Edit an existing post",
|
||||
params=(
|
||||
path("post_slug", "Exact post slug copied from a /posts/... link in a feed or listing response; do not build it from the title."),
|
||||
body("content", "Updated post body.", required=True),
|
||||
body("title", "Updated title."),
|
||||
body("topic", "Updated topic."),
|
||||
body("poll_question", "Optional poll question. Adds a poll to a post that does not already have one."),
|
||||
body("poll_options", "Poll options as a JSON array of strings, or one option per line, or comma separated. At least two are required for the poll to be created."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="delete_post",
|
||||
method="POST",
|
||||
path="/posts/delete/{post_slug}",
|
||||
summary="Delete a post",
|
||||
params=(path("post_slug", "Exact post slug copied from a /posts/... link in a feed or listing response; do not build it from the title."),),
|
||||
),
|
||||
Action(
|
||||
name="create_comment",
|
||||
method="POST",
|
||||
path="/comments/create",
|
||||
summary="Create a comment on a post or other target",
|
||||
params=(
|
||||
body("content", "Comment body.", required=True),
|
||||
body("post_uid", "Uid of the post being commented on."),
|
||||
body("target_uid", "Uid of the target when not a post."),
|
||||
body("target_type", TARGET_TYPE),
|
||||
body("parent_uid", "Parent comment uid for replies."),
|
||||
body("attachment_uids", ATTACHMENTS),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="delete_comment",
|
||||
method="POST",
|
||||
path="/comments/delete/{comment_uid}",
|
||||
summary="Delete a comment",
|
||||
params=(path("comment_uid", "Uid of the comment."),),
|
||||
),
|
||||
Action(
|
||||
name="list_projects",
|
||||
method="GET",
|
||||
path="/projects",
|
||||
summary="List projects",
|
||||
params=(
|
||||
query("tab", "Projects tab."),
|
||||
query("search", "Search query."),
|
||||
query("user_uid", "Filter by owner uid."),
|
||||
query("project_type", "Filter by project type."),
|
||||
query("before", "Pagination cursor."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="view_project",
|
||||
method="GET",
|
||||
path="/projects/{project_slug}",
|
||||
summary="View a project by slug",
|
||||
params=(path("project_slug", "Exact project slug copied from a /projects/... link in a listing response; do not build it from the title."),),
|
||||
),
|
||||
Action(
|
||||
name="create_project",
|
||||
method="POST",
|
||||
path="/projects/create",
|
||||
summary="Create a new project",
|
||||
params=(
|
||||
body("title", "Project title.", required=True),
|
||||
body("description", "Project description.", required=True),
|
||||
body("release_date", "Release date."),
|
||||
body("demo_date", "Demo date."),
|
||||
body("project_type", "Project type."),
|
||||
body("platforms", "Supported platforms."),
|
||||
body("status", "Project status."),
|
||||
body("attachment_uids", ATTACHMENTS),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="delete_project",
|
||||
method="POST",
|
||||
path="/projects/delete/{project_slug}",
|
||||
summary="Delete a project",
|
||||
params=(path("project_slug", "Exact project slug copied from a /projects/... link in a listing response; do not build it from the title."),),
|
||||
),
|
||||
Action(
|
||||
name="search_users",
|
||||
method="GET",
|
||||
path="/profile/search",
|
||||
summary="Search for users by name",
|
||||
params=(query("q", "Search query."),),
|
||||
),
|
||||
Action(
|
||||
name="view_profile",
|
||||
method="GET",
|
||||
path="/profile/{username}",
|
||||
summary="View a user profile",
|
||||
params=(
|
||||
path("username", "Username to view."),
|
||||
query("tab", "Profile tab."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="update_profile",
|
||||
method="POST",
|
||||
path="/profile/update",
|
||||
summary="Update the current user's profile",
|
||||
params=(
|
||||
body("bio", "Profile biography."),
|
||||
body("location", "Location."),
|
||||
body("git_link", "Git profile link."),
|
||||
body("website", "Personal website."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="regenerate_api_key",
|
||||
method="POST",
|
||||
path="/profile/regenerate-api-key",
|
||||
summary="Issue a new API key and invalidate the current one",
|
||||
description=(
|
||||
"Returns the new api_key. Warning: this immediately invalidates any key currently "
|
||||
"used for authentication, so confirm with the user before calling it."
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="list_messages",
|
||||
method="GET",
|
||||
path="/messages",
|
||||
summary="View direct message conversations",
|
||||
params=(
|
||||
query("with_uid", "Open a conversation with a user uid."),
|
||||
query("search", "Search conversations."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="search_message_users",
|
||||
method="GET",
|
||||
path="/messages/search",
|
||||
summary="Search users to message",
|
||||
params=(query("q", "Search query."),),
|
||||
),
|
||||
Action(
|
||||
name="send_message",
|
||||
method="POST",
|
||||
path="/messages/send",
|
||||
summary="Send a direct message",
|
||||
params=(
|
||||
body("content", "Message body.", required=True),
|
||||
body("receiver_uid", "Recipient user uid.", required=True),
|
||||
body("attachment_uids", ATTACHMENTS),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="list_notifications",
|
||||
method="GET",
|
||||
path="/notifications",
|
||||
summary="List notifications",
|
||||
params=(query("before", "Pagination cursor."),),
|
||||
),
|
||||
Action(
|
||||
name="notification_counts",
|
||||
method="GET",
|
||||
path="/notifications/counts",
|
||||
summary="Get unread notification and message counts",
|
||||
),
|
||||
Action(
|
||||
name="open_notification",
|
||||
method="GET",
|
||||
path="/notifications/open/{notification_uid}",
|
||||
summary="Open a notification and follow it",
|
||||
params=(path("notification_uid", "Notification uid."),),
|
||||
),
|
||||
Action(
|
||||
name="mark_notification_read",
|
||||
method="POST",
|
||||
path="/notifications/mark-read/{notification_uid}",
|
||||
summary="Mark a single notification read",
|
||||
params=(path("notification_uid", "Notification uid."),),
|
||||
),
|
||||
Action(
|
||||
name="mark_all_notifications_read",
|
||||
method="POST",
|
||||
path="/notifications/mark-all-read",
|
||||
summary="Mark all notifications read",
|
||||
),
|
||||
Action(
|
||||
name="vote",
|
||||
method="POST",
|
||||
path="/votes/{target_type}/{target_uid}",
|
||||
summary="Cast or toggle a vote on a target",
|
||||
description="Re-sending the same value removes the vote. Returns the net/up/down tally.",
|
||||
ajax=True,
|
||||
params=(
|
||||
path("target_type", TARGET_TYPE),
|
||||
path("target_uid", "Uid of the target, copied from a listing response; do not invent it."),
|
||||
body("value", "Vote value: 1 to upvote, -1 to downvote (re-send to remove).", required=True),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="react",
|
||||
method="POST",
|
||||
path="/reactions/{target_type}/{target_uid}",
|
||||
summary="Toggle an emoji reaction on a target",
|
||||
description="Re-sending the same emoji removes it. Returns reaction counts.",
|
||||
ajax=True,
|
||||
params=(
|
||||
path("target_type", TARGET_TYPE),
|
||||
path("target_uid", "Uid of the target, copied from a listing response; do not invent it."),
|
||||
body("emoji", "One of the allowed reaction emoji.", required=True),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="list_bookmarks",
|
||||
method="GET",
|
||||
path="/bookmarks/saved",
|
||||
summary="List saved bookmarks",
|
||||
params=(query("before", "Pagination cursor."),),
|
||||
),
|
||||
Action(
|
||||
name="toggle_bookmark",
|
||||
method="POST",
|
||||
path="/bookmarks/{target_type}/{target_uid}",
|
||||
summary="Toggle a bookmark on a target",
|
||||
description="Re-sending removes the bookmark. Returns {saved: true|false}.",
|
||||
ajax=True,
|
||||
params=(
|
||||
path("target_type", TARGET_TYPE),
|
||||
path("target_uid", "Uid of the target, copied from a listing response; do not invent it."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="vote_poll",
|
||||
method="POST",
|
||||
path="/polls/{poll_uid}/vote",
|
||||
summary="Vote in a poll",
|
||||
description="Casts or changes your vote on a poll option. Returns the option tallies.",
|
||||
ajax=True,
|
||||
params=(
|
||||
path("poll_uid", "Poll uid."),
|
||||
body("option_uid", "Chosen option uid.", required=True),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="follow_user",
|
||||
method="POST",
|
||||
path="/follow/{username}",
|
||||
summary="Follow a user",
|
||||
params=(path("username", "Username to follow."),),
|
||||
),
|
||||
Action(
|
||||
name="unfollow_user",
|
||||
method="POST",
|
||||
path="/follow/unfollow/{username}",
|
||||
summary="Unfollow a user",
|
||||
params=(path("username", "Username to unfollow."),),
|
||||
),
|
||||
Action(
|
||||
name="list_followers",
|
||||
method="GET",
|
||||
path="/profile/{username}/followers",
|
||||
summary="List the users who follow a profile",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
path("username", "Username whose followers to list."),
|
||||
query("page", "Page number, 25 per page."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="list_following",
|
||||
method="GET",
|
||||
path="/profile/{username}/following",
|
||||
summary="List the users a profile follows",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
path("username", "Username whose following to list."),
|
||||
query("page", "Page number, 25 per page."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="view_leaderboard",
|
||||
method="GET",
|
||||
path="/leaderboard",
|
||||
summary="View the leaderboard",
|
||||
),
|
||||
Action(
|
||||
name="list_bugs",
|
||||
method="GET",
|
||||
path="/bugs",
|
||||
summary="List reported bugs",
|
||||
),
|
||||
Action(
|
||||
name="create_bug",
|
||||
method="POST",
|
||||
path="/bugs/create",
|
||||
summary="Report a bug",
|
||||
params=(
|
||||
body("title", "Bug title.", required=True),
|
||||
body("description", "Bug description.", required=True),
|
||||
body("attachment_uids", ATTACHMENTS),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="list_gists",
|
||||
method="GET",
|
||||
path="/gists",
|
||||
summary="List gists",
|
||||
params=(
|
||||
query("language", "Filter by language."),
|
||||
query("user_uid", "Filter by owner uid."),
|
||||
query("before", "Pagination cursor."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="view_gist",
|
||||
method="GET",
|
||||
path="/gists/{gist_slug}",
|
||||
summary="View a gist by slug",
|
||||
params=(path("gist_slug", "Exact gist slug copied from a /gists/... link in a listing response; do not build it from the title."),),
|
||||
),
|
||||
Action(
|
||||
name="create_gist",
|
||||
method="POST",
|
||||
path="/gists/create",
|
||||
summary="Create a gist",
|
||||
params=(
|
||||
body("title", "Gist title.", required=True),
|
||||
body("source_code", "Gist source code.", required=True),
|
||||
body("description", "Gist description."),
|
||||
body("language", "Programming language."),
|
||||
body("attachment_uids", ATTACHMENTS),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="edit_gist",
|
||||
method="POST",
|
||||
path="/gists/edit/{gist_slug}",
|
||||
summary="Edit a gist",
|
||||
params=(
|
||||
path("gist_slug", "Exact gist slug copied from a /gists/... link in a listing response; do not build it from the title."),
|
||||
body("title", "Gist title.", required=True),
|
||||
body("source_code", "Gist source code.", required=True),
|
||||
body("description", "Gist description."),
|
||||
body("language", "Programming language."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="delete_gist",
|
||||
method="POST",
|
||||
path="/gists/delete/{gist_slug}",
|
||||
summary="Delete a gist",
|
||||
params=(path("gist_slug", "Exact gist slug copied from a /gists/... link in a listing response; do not build it from the title."),),
|
||||
),
|
||||
Action(
|
||||
name="list_news",
|
||||
method="GET",
|
||||
path="/news",
|
||||
summary="List news articles",
|
||||
params=(query("before", "Pagination cursor."),),
|
||||
),
|
||||
Action(
|
||||
name="view_news",
|
||||
method="GET",
|
||||
path="/news/{news_slug}",
|
||||
summary="View a news article by slug",
|
||||
params=(path("news_slug", "Exact news slug copied from a /news/... link in a listing response; do not build it from the title."),),
|
||||
),
|
||||
Action(
|
||||
name="upload_file",
|
||||
method="POST",
|
||||
path="/uploads/upload",
|
||||
summary="Upload a local file and obtain its attachment uid",
|
||||
params=(upload("file", "Local filesystem path of the file to upload."),),
|
||||
),
|
||||
Action(
|
||||
name="delete_attachment",
|
||||
method="DELETE",
|
||||
path="/uploads/delete/{attachment_uid}",
|
||||
summary="Delete an uploaded attachment",
|
||||
params=(path("attachment_uid", "Uid of the attachment."),),
|
||||
),
|
||||
Action(
|
||||
name="admin_overview",
|
||||
method="GET",
|
||||
path="/admin",
|
||||
summary="View the admin overview",
|
||||
),
|
||||
Action(
|
||||
name="site_analytics",
|
||||
method="GET",
|
||||
path="/admin/analytics",
|
||||
summary="Site-wide aggregate analytics in one call (admin only)",
|
||||
description=(
|
||||
"Returns JSON: total members, active users in the last 24h/7d/30d, users signed in "
|
||||
"now, new signups (24h/7d/30d), content totals (posts, comments, gists, projects, "
|
||||
"news), and top authors. Use this for any 'how many'/'how active'/count question "
|
||||
"instead of paging through admin_list_users."
|
||||
),
|
||||
params=(query("top_n", "How many top authors to include (1-50)."),),
|
||||
),
|
||||
Action(
|
||||
name="ai_usage",
|
||||
method="GET",
|
||||
path="/admin/ai-usage/data",
|
||||
summary="AI gateway usage, cost, latency, and reliability metrics (admin only)",
|
||||
description=(
|
||||
"Returns JSON for a bounded window: request volume and throughput, token usage with "
|
||||
"averages and percentiles, latency (upstream, gateway overhead, queue wait, connect), "
|
||||
"error rates by category, cost in USD (per model, per caller, input vs output, projected "
|
||||
"monthly burn, caching savings), caller behavior, and an hourly breakdown. Use this for "
|
||||
"any question about AI spend, token consumption, gateway performance, or errors."
|
||||
),
|
||||
params=(
|
||||
query("hours", "Lookback window in hours (1-168, default 48)."),
|
||||
query("top_n", "How many rows in each top-N breakdown (default 10)."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="admin_list_users",
|
||||
method="GET",
|
||||
path="/admin/users",
|
||||
summary="List users for administration",
|
||||
params=(query("page", "Page number."),),
|
||||
),
|
||||
Action(
|
||||
name="admin_set_user_role",
|
||||
method="POST",
|
||||
path="/admin/users/{uid}/role",
|
||||
summary="Set a user's role",
|
||||
params=(
|
||||
path("uid", "User uid."),
|
||||
body("role", "New role.", required=True),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="admin_set_user_password",
|
||||
method="POST",
|
||||
path="/admin/users/{uid}/password",
|
||||
summary="Set a user's password",
|
||||
params=(
|
||||
path("uid", "User uid."),
|
||||
body("password", "New password.", required=True),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="admin_toggle_user",
|
||||
method="POST",
|
||||
path="/admin/users/{uid}/toggle",
|
||||
summary="Toggle a user's active state",
|
||||
params=(path("uid", "User uid."),),
|
||||
),
|
||||
Action(
|
||||
name="admin_get_settings",
|
||||
method="GET",
|
||||
path="/admin/settings",
|
||||
summary="View site settings",
|
||||
),
|
||||
Action(
|
||||
name="admin_save_settings",
|
||||
method="POST",
|
||||
path="/admin/settings",
|
||||
summary="Save site settings",
|
||||
params=(
|
||||
body("site_name", "Site name."),
|
||||
body("site_description", "Site description."),
|
||||
body("site_tagline", "Site tagline."),
|
||||
body("max_upload_size_mb", "Maximum upload size in megabytes."),
|
||||
body("allowed_file_types", "Allowed file types."),
|
||||
body("max_attachments_per_resource", "Maximum attachments per resource."),
|
||||
body("rate_limit_per_minute", "Rate limit per minute."),
|
||||
body("rate_limit_window_seconds", "Rate limit window in seconds."),
|
||||
body("session_max_age_days", "Session maximum age in days."),
|
||||
body("session_remember_days", "Remember-me duration in days."),
|
||||
body("registration_open", "Whether registration is open."),
|
||||
body("maintenance_mode", "Whether maintenance mode is on."),
|
||||
body("maintenance_message", "Maintenance message."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="admin_list_news",
|
||||
method="GET",
|
||||
path="/admin/news",
|
||||
summary="List news for administration",
|
||||
params=(query("page", "Page number."),),
|
||||
),
|
||||
Action(
|
||||
name="admin_toggle_news",
|
||||
method="POST",
|
||||
path="/admin/news/{uid}/toggle",
|
||||
summary="Toggle a news article",
|
||||
params=(path("uid", "News uid."),),
|
||||
),
|
||||
Action(
|
||||
name="admin_publish_news",
|
||||
method="POST",
|
||||
path="/admin/news/{uid}/publish",
|
||||
summary="Publish a news article",
|
||||
params=(path("uid", "News uid."),),
|
||||
),
|
||||
Action(
|
||||
name="admin_landing_news",
|
||||
method="POST",
|
||||
path="/admin/news/{uid}/landing",
|
||||
summary="Set a news article as landing content",
|
||||
params=(path("uid", "News uid."),),
|
||||
),
|
||||
Action(
|
||||
name="admin_delete_news",
|
||||
method="POST",
|
||||
path="/admin/news/{uid}/delete",
|
||||
summary="Delete a news article",
|
||||
params=(path("uid", "News uid."),),
|
||||
),
|
||||
Action(
|
||||
name="admin_list_services",
|
||||
method="GET",
|
||||
path="/admin/services",
|
||||
summary="View managed services",
|
||||
),
|
||||
Action(
|
||||
name="admin_services_data",
|
||||
method="GET",
|
||||
path="/admin/services/data",
|
||||
summary="Get live status, metrics, and log tail for every background service",
|
||||
),
|
||||
Action(
|
||||
name="admin_service_status",
|
||||
method="GET",
|
||||
path="/admin/services/{name}/data",
|
||||
summary="Get live status, metrics, and log tail for one background service",
|
||||
params=(path("name", "Service name (e.g. news, bots, openai)."),),
|
||||
),
|
||||
Action(
|
||||
name="admin_start_service",
|
||||
method="POST",
|
||||
path="/admin/services/{name}/start",
|
||||
summary="Start a managed service",
|
||||
params=(path("name", "Service name."),),
|
||||
),
|
||||
Action(
|
||||
name="admin_stop_service",
|
||||
method="POST",
|
||||
path="/admin/services/{name}/stop",
|
||||
summary="Stop a managed service",
|
||||
params=(path("name", "Service name."),),
|
||||
),
|
||||
Action(
|
||||
name="admin_run_service",
|
||||
method="POST",
|
||||
path="/admin/services/{name}/run",
|
||||
summary="Run a managed service once",
|
||||
params=(path("name", "Service name."),),
|
||||
),
|
||||
Action(
|
||||
name="admin_clear_service_logs",
|
||||
method="POST",
|
||||
path="/admin/services/{name}/clear-logs",
|
||||
summary="Clear a managed service's logs",
|
||||
params=(path("name", "Service name."),),
|
||||
),
|
||||
Action(
|
||||
name="admin_config_service",
|
||||
method="POST",
|
||||
path="/admin/services/{name}/config",
|
||||
summary="Update a managed service's configuration",
|
||||
params=(path("name", "Service name."),),
|
||||
freeform_body=True,
|
||||
),
|
||||
)
|
||||
|
||||
PLATFORM_CATALOG = Catalog(actions=ACTIONS)
|
||||
@@ -0,0 +1,41 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .spec import Action, Param
|
||||
|
||||
CHUNK_ACTIONS: tuple[Action, ...] = (
|
||||
Action(
|
||||
name="read_more",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Read the next part of a previously truncated tool result",
|
||||
description=(
|
||||
"When any tool result is truncated it includes a chunk_id, total_chars, "
|
||||
"remaining_chars, and next_offset. Call read_more with that chunk_id and offset to "
|
||||
"page through the rest until remaining_chars is 0, so you can read the whole content."
|
||||
),
|
||||
handler="chunks",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
Param(
|
||||
name="chunk_id",
|
||||
location="body",
|
||||
description="The chunk_id from a truncated result.",
|
||||
required=True,
|
||||
),
|
||||
Param(
|
||||
name="offset",
|
||||
location="body",
|
||||
description="Character offset to read from (use next_offset from the prior part).",
|
||||
type="integer",
|
||||
),
|
||||
Param(
|
||||
name="length",
|
||||
location="body",
|
||||
description="Optional number of characters to return (capped to the response limit).",
|
||||
type="integer",
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,112 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .spec import Action, Param
|
||||
|
||||
|
||||
def arg(name: str, description: str, required: bool = False, kind: str = "string") -> Param:
|
||||
return Param(name=name, location="body", description=description, required=required, type=kind)
|
||||
|
||||
|
||||
CLIENT = (
|
||||
"Runs in the user's own browser through the web terminal. Only effective in the web "
|
||||
"interface with a live connection; elsewhere it reports that no browser is attached."
|
||||
)
|
||||
|
||||
CLIENT_ACTIONS: tuple[Action, ...] = (
|
||||
Action(
|
||||
name="get_page_context",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Read the user's current page: URL, title, viewport, scroll, selected text, visible headings, and whether they are signed in",
|
||||
description=CLIENT + " Use this to understand where the user is and what they are looking at before acting or guiding them.",
|
||||
handler="client",
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="run_js",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Execute JavaScript in the user's browser and return its result",
|
||||
description=(
|
||||
CLIENT + " The code is the body of an async function; use 'return value' to return a "
|
||||
"JSON-serializable result. You have full access to window and document. Use this for "
|
||||
"anything not covered by the dedicated tools: read or change the DOM, drive a live "
|
||||
"demo, inspect state, or update the screen. Prefer the dedicated tools "
|
||||
"(highlight_element, show_toast, scroll_to_element, navigate_to, reload_page) when they fit."
|
||||
),
|
||||
handler="client",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
arg("code", "JavaScript to run as an async function body. Return a JSON-serializable value.", required=True),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="highlight_element",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Highlight an element on screen with an outline and an optional callout label, for live tutorials",
|
||||
description=CLIENT + " Scrolls the element into view and draws an attention outline. Call clear_highlights to remove it.",
|
||||
handler="client",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
arg("selector", "CSS selector, or the element's exact visible text (e.g. a heading or link label).", required=True),
|
||||
arg("label", "Optional callout text shown next to the element."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="clear_highlights",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Remove all highlights and callouts placed by highlight_element",
|
||||
description=CLIENT,
|
||||
handler="client",
|
||||
requires_auth=False,
|
||||
),
|
||||
Action(
|
||||
name="show_toast",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Show a brief on-screen message (toast) to the user",
|
||||
description=CLIENT,
|
||||
handler="client",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
arg("text", "Message to display.", required=True),
|
||||
arg("duration_ms", "How long to show it, in milliseconds (default 4000).", kind="integer"),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="scroll_to_element",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Smoothly scroll an element into view",
|
||||
description=CLIENT,
|
||||
handler="client",
|
||||
requires_auth=False,
|
||||
params=(arg("selector", "CSS selector, or the element's exact visible text.", required=True),),
|
||||
),
|
||||
Action(
|
||||
name="navigate_to",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Send the user's browser to a URL",
|
||||
description=(
|
||||
CLIENT + " Use a same-origin path like /feed or /docs/index.html, or a full URL. The "
|
||||
"page reloads; the user's Devii session and conversation persist and reconnect automatically."
|
||||
),
|
||||
handler="client",
|
||||
requires_auth=False,
|
||||
params=(arg("url", "Path or URL to navigate to.", required=True),),
|
||||
),
|
||||
Action(
|
||||
name="reload_page",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Refresh the user's current page, e.g. after something changed",
|
||||
description=CLIENT + " The Devii session and conversation persist and reconnect automatically.",
|
||||
handler="client",
|
||||
requires_auth=False,
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,22 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .spec import Action
|
||||
|
||||
COST_ACTIONS: tuple[Action, ...] = (
|
||||
Action(
|
||||
name="cost_stats",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Report token usage and cost statistics for the current session",
|
||||
description=(
|
||||
"Returns this session's LLM token counts (prompt, completion, total, cache hit/miss, "
|
||||
"reasoning), the cost in USD broken down by cache-hit input, cache-miss input, and "
|
||||
"output, per-request averages, cache hit rate, and session timing. Costs are priced "
|
||||
"as DeepSeek V4 Flash, the cheapest official DeepSeek model."
|
||||
),
|
||||
handler="cost",
|
||||
requires_auth=False,
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,223 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
from urllib.parse import quote, urlencode
|
||||
|
||||
from ..config import Settings
|
||||
from ..errors import (
|
||||
AuthRequiredError,
|
||||
DeviiError,
|
||||
ToolInputError,
|
||||
error_result,
|
||||
unexpected_result,
|
||||
)
|
||||
from ..agentic.controller import AgenticController
|
||||
from ..agentic.state import record_mutation
|
||||
from ..avatar import AvatarController
|
||||
from ..chunks import ChunkController, get_store, serve_resource, wrap_if_large
|
||||
from ..cost import CostController
|
||||
from ..docs import DocsController
|
||||
from ..fetch import FetchController
|
||||
from ..http_client import PlatformClient
|
||||
from ..rsearch import RsearchController
|
||||
from ..tasks.controller import TaskController
|
||||
from ..text import format_response
|
||||
from .spec import Action, Catalog
|
||||
|
||||
MUTATING_METHODS = ("POST", "DELETE", "PUT", "PATCH")
|
||||
|
||||
logger = logging.getLogger("devii.dispatch")
|
||||
|
||||
|
||||
class Dispatcher:
|
||||
def __init__(
|
||||
self,
|
||||
catalog: Catalog,
|
||||
client: PlatformClient,
|
||||
settings: Settings,
|
||||
tasks: TaskController,
|
||||
agentic: AgenticController,
|
||||
avatar: AvatarController | None = None,
|
||||
browser: Any = None,
|
||||
) -> None:
|
||||
self._actions = catalog.by_name()
|
||||
self._client = client
|
||||
self._settings = settings
|
||||
self._tasks = tasks
|
||||
self._agentic = agentic
|
||||
self._avatar = avatar
|
||||
self._browser = browser
|
||||
self._fetch = FetchController(settings)
|
||||
self._docs = DocsController(settings)
|
||||
self._cost = CostController()
|
||||
self._chunks = ChunkController(settings)
|
||||
self._rsearch = RsearchController(settings)
|
||||
|
||||
async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
|
||||
action = self._actions.get(name)
|
||||
if action is None:
|
||||
return error_result(ToolInputError(f"Unknown tool: {name}"))
|
||||
|
||||
logger.info("Dispatch %s args=%s", name, list(arguments))
|
||||
try:
|
||||
if action.requires_auth and not self._client.authenticated:
|
||||
raise AuthRequiredError(
|
||||
"Not authenticated. Ask the user for credentials and call the login tool first.",
|
||||
tool=name,
|
||||
)
|
||||
resource_key = self._resource_key(action, arguments)
|
||||
if resource_key:
|
||||
cached = serve_resource(resource_key, self._settings.max_response_chars)
|
||||
if cached is not None:
|
||||
logger.info("Resource cache hit for %s (%s)", name, resource_key)
|
||||
return cached
|
||||
result = await self._run(action, arguments)
|
||||
if action.handler == "chunks":
|
||||
return result
|
||||
return wrap_if_large(result, self._settings.max_response_chars, resource_key)
|
||||
except DeviiError as exc:
|
||||
logger.info("Dispatch %s failed: %s", name, exc.message)
|
||||
return error_result(exc)
|
||||
except Exception as exc: # noqa: BLE001 - surfaced to the model as data
|
||||
logger.exception("Dispatch %s crashed", name)
|
||||
return unexpected_result(exc)
|
||||
|
||||
async def _run(self, action: Action, arguments: dict[str, Any]) -> str:
|
||||
if action.handler == "status":
|
||||
return json.dumps(
|
||||
{
|
||||
"authenticated": self._client.authenticated,
|
||||
"user": self._client.username,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
if action.handler == "login":
|
||||
result = await self._client.login(
|
||||
email=self._require(arguments, "email"),
|
||||
password=self._require(arguments, "password"),
|
||||
remember_me=str(arguments.get("remember_me", "on")).lower() not in ("", "false", "off", "no"),
|
||||
)
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
if action.handler == "logout":
|
||||
return json.dumps(await self._client.logout(), ensure_ascii=False)
|
||||
|
||||
if action.handler == "task":
|
||||
return await self._tasks.dispatch(action.name, arguments)
|
||||
|
||||
if action.handler == "agentic":
|
||||
return await self._agentic.dispatch(action.name, arguments)
|
||||
|
||||
if action.handler == "fetch":
|
||||
return await self._fetch.dispatch(action.name, arguments)
|
||||
|
||||
if action.handler == "docs":
|
||||
return await self._docs.dispatch(action.name, arguments)
|
||||
|
||||
if action.handler == "cost":
|
||||
return await self._cost.dispatch(action.name, arguments)
|
||||
|
||||
if action.handler == "chunks":
|
||||
return await self._chunks.dispatch(action.name, arguments)
|
||||
|
||||
if action.handler == "rsearch":
|
||||
return await self._rsearch.dispatch(action.name, arguments)
|
||||
|
||||
if action.handler == "avatar":
|
||||
if self._avatar is None:
|
||||
return error_result(
|
||||
ToolInputError(
|
||||
"No avatar is attached; devii's on-screen actions need the web interface."
|
||||
)
|
||||
)
|
||||
return await self._avatar.dispatch(action.name, arguments)
|
||||
|
||||
if action.handler == "client":
|
||||
if self._browser is None:
|
||||
return error_result(
|
||||
ToolInputError(
|
||||
"No browser is attached; client-side actions need the web interface."
|
||||
)
|
||||
)
|
||||
return await self._browser.dispatch(action.name, arguments)
|
||||
|
||||
return await self._run_http(action, arguments)
|
||||
|
||||
def _build_request(
|
||||
self, action: Action, arguments: dict[str, Any]
|
||||
) -> tuple[str, dict[str, Any], dict[str, Any], tuple[str, str] | None]:
|
||||
url_path = action.path
|
||||
params: dict[str, Any] = {}
|
||||
data: dict[str, Any] = {}
|
||||
file_field: tuple[str, str] | None = None
|
||||
|
||||
for param in action.params:
|
||||
if param.name not in arguments or arguments[param.name] is None:
|
||||
if param.required:
|
||||
raise ToolInputError(
|
||||
f"Missing required parameter '{param.name}' for {action.name}.")
|
||||
continue
|
||||
value = arguments[param.name]
|
||||
if param.location == "path":
|
||||
url_path = url_path.replace("{" + param.name + "}", quote(str(value), safe=""))
|
||||
elif param.location == "query":
|
||||
params[param.name] = value
|
||||
elif param.location == "body":
|
||||
data[param.name] = value
|
||||
elif param.location == "file":
|
||||
file_field = (param.name, str(value))
|
||||
|
||||
if action.freeform_body:
|
||||
extra = arguments.get("form_fields") or {}
|
||||
if isinstance(extra, dict):
|
||||
data.update({str(k): v for k, v in extra.items()})
|
||||
|
||||
return url_path, params, data, file_field
|
||||
|
||||
def _resource_key(self, action: Action, arguments: dict[str, Any]) -> str | None:
|
||||
try:
|
||||
if action.handler == "fetch" and action.name == "fetch_url":
|
||||
url = str(arguments.get("url", "")).strip()
|
||||
if not url:
|
||||
return None
|
||||
if "://" not in url:
|
||||
url = "https://" + url
|
||||
return f"fetch:{url}"
|
||||
if action.handler == "http" and action.method == "GET":
|
||||
url_path, params, _, _ = self._build_request(action, arguments)
|
||||
query = urlencode(sorted((str(k), str(v)) for k, v in params.items()))
|
||||
return f"http:GET {url_path}?{query}"
|
||||
except ToolInputError:
|
||||
return None
|
||||
return None
|
||||
|
||||
async def _run_http(self, action: Action, arguments: dict[str, Any]) -> str:
|
||||
url_path, params, data, file_field = self._build_request(action, arguments)
|
||||
|
||||
headers = {"X-Requested-With": "fetch"} if action.ajax else None
|
||||
response = await self._client.call(
|
||||
method=action.method,
|
||||
path=url_path,
|
||||
params=params or None,
|
||||
data=data or None,
|
||||
file_field=file_field,
|
||||
headers=headers,
|
||||
)
|
||||
if action.method in MUTATING_METHODS:
|
||||
record_mutation(action.name)
|
||||
store = get_store()
|
||||
if store is not None:
|
||||
store.invalidate_resources(prefix="http:")
|
||||
return format_response(response)
|
||||
|
||||
@staticmethod
|
||||
def _require(arguments: dict[str, Any], key: str) -> str:
|
||||
value = arguments.get(key)
|
||||
if value is None or str(value).strip() == "":
|
||||
raise ToolInputError(f"Missing required field '{key}'.")
|
||||
return str(value)
|
||||
@@ -0,0 +1,35 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .spec import Action, Param
|
||||
|
||||
DOCS_ACTIONS: tuple[Action, ...] = (
|
||||
Action(
|
||||
name="search_docs",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Full-text search the DevPlace API documentation",
|
||||
description=(
|
||||
"Searches the platform's developer documentation and returns the most relevant "
|
||||
"sections (title and content). Use it to confirm how an endpoint, parameter, or "
|
||||
"feature works before acting."
|
||||
),
|
||||
handler="docs",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
Param(
|
||||
name="query",
|
||||
location="body",
|
||||
description="What to look for in the documentation.",
|
||||
required=True,
|
||||
),
|
||||
Param(
|
||||
name="max_results",
|
||||
location="body",
|
||||
description="Maximum number of documentation sections to return (1-10).",
|
||||
type="integer",
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,36 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .spec import Action, Param
|
||||
|
||||
FETCH_ACTIONS: tuple[Action, ...] = (
|
||||
Action(
|
||||
name="fetch_url",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Fetch a web page and return its readable text content",
|
||||
description=(
|
||||
"Retrieves any http(s) URL as a real browser would (stealth headers) and returns "
|
||||
"the page title and readable text with links preserved, safely truncated to fit "
|
||||
"the context window. Use it to read external pages, then summarize or create posts, "
|
||||
"gists, or articles from the content. Private and loopback addresses are refused."
|
||||
),
|
||||
handler="fetch",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
Param(
|
||||
name="url",
|
||||
location="body",
|
||||
description="The page URL to fetch (https is assumed if no scheme is given).",
|
||||
required=True,
|
||||
),
|
||||
Param(
|
||||
name="max_chars",
|
||||
location="body",
|
||||
description="Optional cap on returned characters; clamped to a context-safe maximum.",
|
||||
type="integer",
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,85 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .spec import Action, Param
|
||||
|
||||
REMOTE_NOTE = (
|
||||
"This is an EXTERNAL public service, not this DevPlace platform. Only call it when the user "
|
||||
"explicitly asks to search the web or an outside source; prefer platform tools otherwise."
|
||||
)
|
||||
|
||||
RSEARCH_ACTIONS: tuple[Action, ...] = (
|
||||
Action(
|
||||
name="rsearch",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Search the public web (and images) via the external rsearch aggregator",
|
||||
description=(
|
||||
"Queries multiple independent web search providers and returns ranked results "
|
||||
"(title, URL, description, source). Set content=true to also fetch each page's "
|
||||
"readable text, deep=true for a deeper research pass, and type=images for image "
|
||||
"results. " + REMOTE_NOTE
|
||||
),
|
||||
handler="rsearch",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
Param(name="query", location="body", description="The web search query.", required=True),
|
||||
Param(name="count", location="body", description="Number of results (1-100, default 10).", type="integer"),
|
||||
Param(name="content", location="body", description="Fetch full page content for each result.", type="boolean"),
|
||||
Param(name="deep", location="body", description="Run a deeper research pass.", type="boolean"),
|
||||
Param(name="type", location="body", description="Result type: 'web' (default) or 'images'."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="rsearch_answer",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Get an AI answer to a question, grounded in fresh public web search",
|
||||
description=(
|
||||
"Sends the question to an external AI that autonomously searches the public web and "
|
||||
"returns a written answer plus the source links it used. Use for current, real-world "
|
||||
"questions the platform cannot answer. " + REMOTE_NOTE
|
||||
),
|
||||
handler="rsearch",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
Param(name="query", location="body", description="The question or prompt to answer.", required=True),
|
||||
Param(name="content", location="body", description="Let the AI read full page content while answering.", type="boolean"),
|
||||
Param(name="count", location="body", description="Max sources to consider (1-100, default 10).", type="integer"),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="rsearch_chat",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Send a prompt to the external rsearch AI for a direct answer (no web search)",
|
||||
description=(
|
||||
"Direct chat completion from the external rsearch AI model, with no web search or "
|
||||
"platform context. Set json=true to force a JSON-only answer, or pass a system message "
|
||||
"to steer the persona. " + REMOTE_NOTE
|
||||
),
|
||||
handler="rsearch",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
Param(name="prompt", location="body", description="The prompt to send.", required=True),
|
||||
Param(name="json", location="body", description="Force a valid-JSON-only response.", type="boolean"),
|
||||
Param(name="system", location="body", description="Optional system message to steer the answer."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="rsearch_describe_image",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Describe a public image URL using the external rsearch vision model",
|
||||
description=(
|
||||
"Sends an image URL to the external rsearch vision model and returns a written "
|
||||
"description. " + REMOTE_NOTE
|
||||
),
|
||||
handler="rsearch",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
Param(name="url", location="body", description="Public URL of the image to describe.", required=True),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,90 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Literal
|
||||
|
||||
ParamLocation = Literal["path", "query", "body", "file"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Param:
|
||||
name: str
|
||||
location: ParamLocation
|
||||
description: str
|
||||
required: bool = False
|
||||
type: str = "string"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Action:
|
||||
name: str
|
||||
method: str
|
||||
path: str
|
||||
summary: str
|
||||
description: str = ""
|
||||
params: tuple[Param, ...] = ()
|
||||
requires_auth: bool = True
|
||||
handler: Literal[
|
||||
"http", "login", "logout", "status", "task", "agentic", "avatar", "client", "fetch",
|
||||
"docs", "cost", "chunks", "rsearch"
|
||||
] = "http"
|
||||
freeform_body: bool = False
|
||||
ajax: bool = False
|
||||
|
||||
def tool_schema(self) -> dict[str, Any]:
|
||||
properties: dict[str, Any] = {}
|
||||
required: list[str] = []
|
||||
for param in self.params:
|
||||
if param.type == "array":
|
||||
properties[param.name] = {
|
||||
"type": "array",
|
||||
"items": {"type": "object"},
|
||||
"description": param.description,
|
||||
}
|
||||
else:
|
||||
properties[param.name] = {
|
||||
"type": param.type,
|
||||
"description": param.description,
|
||||
}
|
||||
if param.required:
|
||||
required.append(param.name)
|
||||
if self.freeform_body:
|
||||
properties["form_fields"] = {
|
||||
"type": "object",
|
||||
"description": "Additional form fields as key/value string pairs.",
|
||||
"additionalProperties": {"type": "string"},
|
||||
}
|
||||
text = self.summary if not self.description else f"{self.summary}. {self.description}"
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": self.name,
|
||||
"description": text,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": properties,
|
||||
"required": required,
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Catalog:
|
||||
actions: tuple[Action, ...] = field(default_factory=tuple)
|
||||
|
||||
def by_name(self) -> dict[str, Action]:
|
||||
return {action.name: action for action in self.actions}
|
||||
|
||||
def tool_schemas(self) -> list[dict[str, Any]]:
|
||||
return [action.tool_schema() for action in self.actions]
|
||||
|
||||
def tool_schemas_for(self, authenticated: bool) -> list[dict[str, Any]]:
|
||||
return [
|
||||
action.tool_schema()
|
||||
for action in self.actions
|
||||
if authenticated or not action.requires_auth
|
||||
]
|
||||
@@ -0,0 +1,165 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
from .actions import Dispatcher
|
||||
from .agentic import AgentState, LessonStore, react_loop
|
||||
from .agentic.loop import TraceCallback
|
||||
from .config import Settings
|
||||
from .llm import LLMClient
|
||||
|
||||
logger = logging.getLogger("devii.agent")
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
"You are Devii, an agentic assistant that manages a user's account on this "
|
||||
"DevPlace developer social network entirely through the provided tools, running a "
|
||||
"ReAct loop with structured planning, reflection, a verification gate, and persistent "
|
||||
"self-learning memory.\n\n"
|
||||
"OPERATING PROTOCOL\n"
|
||||
"1. PLAN FIRST. When a request needs any tool use, your very first tool call must be "
|
||||
"plan() with goal, ordered steps, success_criteria, and a confidence estimate.\n"
|
||||
"2. PERMISSIONS. You only have the tools your current access allows; account actions are "
|
||||
"present only when the user is already signed in. Never pressure the user to log in and "
|
||||
"never ask for credentials unprompted - just work with the tools you have. If the user "
|
||||
"explicitly asks for something that needs an account you cannot reach, note briefly that it "
|
||||
"requires signing in, then continue with whatever you can do. Never invent credentials.\n"
|
||||
"3. RECALL AND ACT. Use recall() to consult lessons from past requests when unsure. "
|
||||
"Investigate with read actions before changing anything. Independent reads may be issued "
|
||||
"together in one turn; they run in parallel.\n"
|
||||
"4. VERIFY. After mutating actions (create, edit, delete, vote, react, follow, admin "
|
||||
"changes), confirm the result and call verify() before your final answer.\n"
|
||||
"5. REFLECT. After any tool error, the harness asks you to call reflect(); diagnose the "
|
||||
"cause and record the lesson - a reusable rule or procedure - rather than blindly retrying. "
|
||||
"Reflect at the end of non-trivial tasks too, so you learn. Your lesson memory is PRIVATE to "
|
||||
"this account (or, for a guest, this web session only) and is never shared with other users. "
|
||||
"Never store credentials, passwords, API keys, tokens, or other secrets in a lesson. When the "
|
||||
"user asks you to forget something, call forget_lessons (optionally with a query).\n"
|
||||
"6. DELEGATE. For a self-contained sub-task, call delegate() to run it in an isolated "
|
||||
"sub-agent that returns a concise result.\n\n"
|
||||
"Tool results are JSON; an 'error' field means failure - read the message and recover. "
|
||||
"If a result has 'truncated': true it includes chunk_id, remaining_chars, and next_offset; "
|
||||
"call read_more with that chunk_id and offset to page through the rest until remaining_chars "
|
||||
"is 0 whenever you need the full content. Use read_more to continue - never re-run the "
|
||||
"original tool to get more of the same resource; its full content is already cached. "
|
||||
"Platform calls return structured JSON: reads give objects with fields like uid, slug, "
|
||||
"next_cursor, and nested authors/comments; write actions return {ok, redirect, data} where "
|
||||
"data has the created resource's uid/slug/url. Always reuse those exact slugs/uids/urls for "
|
||||
"follow-up calls instead of constructing them. "
|
||||
"In the web terminal you can act on the user's own screen: get_page_context tells you where "
|
||||
"they are and what they see; run_js executes JavaScript in their browser and returns a value; "
|
||||
"highlight_element, show_toast, scroll_to_element and clear_highlights let you guide them with "
|
||||
"live, on-screen tutorials; navigate_to and reload_page move or refresh their page (their "
|
||||
"session and this conversation persist and reconnect automatically). Read the page context "
|
||||
"before guiding, prefer the dedicated tools over raw run_js, and clear highlights when done. "
|
||||
"Confirm destructive actions with the user first. Schedule autonomous work with "
|
||||
"create_task and related tools. All times are UTC.\n\n"
|
||||
"AGGREGATES AND LARGE DATA\n"
|
||||
"For any count, total, or 'how many' / 'how active' question, call site_analytics - it returns "
|
||||
"member totals, active users over 24h/7d/30d, signups, content totals, and top authors in a "
|
||||
"single call. Never page through admin_list_users or any list_* endpoint to count records: "
|
||||
"fanning out paginated calls wastes the context window and is forbidden. When the user actually "
|
||||
"wants items (not a count), page with the cursor and stop as soon as you have enough.\n\n"
|
||||
"NEVER GUESS - CHECK THE DOCS FIRST\n"
|
||||
"When you are unsure about a route, endpoint, parameter, capability, or whether a page or "
|
||||
"feature exists, call search_docs first (the documentation lists every route and endpoint), and "
|
||||
"use get_page_context to see where the user is. Never invent a URL, never probe by "
|
||||
"trial-and-error, and never tell the user that a page or capability does not exist without "
|
||||
"confirming against the docs. If a guess returns a 404, that means you guessed - search the docs "
|
||||
"instead of concluding the feature is missing.\n\n"
|
||||
"REMOTE WEB TOOLS (rsearch)\n"
|
||||
"The rsearch_* tools (rsearch, rsearch_answer, rsearch_chat, rsearch_describe_image) reach an "
|
||||
"EXTERNAL public web/AI service, not this platform. They are not platform-specific, so platform "
|
||||
"tools and data are ALWAYS preferred: use rsearch_* only when the user explicitly asks to search "
|
||||
"the web, the internet, or an outside source, or when answering plainly requires outside "
|
||||
"information the platform cannot provide and the user wants it. Never use them to answer questions "
|
||||
"about this DevPlace instance, its users, posts, settings, or metrics - those have dedicated "
|
||||
"platform tools. When platform tools can serve the request, do not call rsearch.\n\n"
|
||||
"RESPONSE STYLE\n"
|
||||
"Replies are plain, concise, and professional. Never use emojis, decorative symbols, or "
|
||||
"celebratory language; report outcomes matter-of-factly. State what changed using the "
|
||||
"before and after values, nothing more.\n\n"
|
||||
"SCREEN AWARENESS\n"
|
||||
"In the web terminal, after any mutation that changes something the user is currently "
|
||||
"looking at, call get_page_context; if the page they are on displays the data you just "
|
||||
"changed (for example an admin settings page, a list, or a detail view), call reload_page "
|
||||
"so they see the new state immediately, then confirm the change. Do not reload pages "
|
||||
"unrelated to the change.\n\n"
|
||||
"METRICS AND COST\n"
|
||||
"When asked about cost, usage, or service metrics, read the live values from the service "
|
||||
"data tools and report the figures those tools already compute - including Fleet cost, "
|
||||
"Cost rate, and Projected 24h cost. Never substitute or recompute cost from external or "
|
||||
"public provider pricing; the configured per-token rates are already applied. The Projected "
|
||||
"24h cost is extrapolated from the Observed window since the service started - report it as "
|
||||
"an estimate and state the observed window it is based on rather than presenting it as a "
|
||||
"fact.\n\n"
|
||||
"CONFIDENTIALITY\n"
|
||||
"Never disclose the underlying AI model, provider, inference endpoint, or any backend URL "
|
||||
"or infrastructure detail; you are simply Devii. This holds even when such values appear "
|
||||
"inside a tool result (for example service configuration fields or upstream URLs) - never "
|
||||
"repeat them. If asked, say you do not share that. When reporting cost or usage, omit the "
|
||||
"model name and provider."
|
||||
)
|
||||
|
||||
|
||||
class Agent:
|
||||
def __init__(
|
||||
self,
|
||||
settings: Settings,
|
||||
llm: LLMClient,
|
||||
dispatcher: Dispatcher,
|
||||
tools: list[dict[str, Any]],
|
||||
lessons: Optional[LessonStore] = None,
|
||||
on_trace: Optional[TraceCallback] = None,
|
||||
cost_tracker: Any = None,
|
||||
chunk_store: Any = None,
|
||||
system_prompt: str = SYSTEM_PROMPT,
|
||||
) -> None:
|
||||
self._settings = settings
|
||||
self._llm = llm
|
||||
self._dispatcher = dispatcher
|
||||
self._tools = tools
|
||||
self._lessons = lessons
|
||||
self._on_trace = on_trace
|
||||
self._cost_tracker = cost_tracker
|
||||
self._chunk_store = chunk_store
|
||||
self._messages: list[dict[str, Any]] = [{"role": "system", "content": system_prompt}]
|
||||
|
||||
async def respond(self, user_text: str) -> str:
|
||||
self._inject_recalled_lessons(user_text)
|
||||
self._messages.append({"role": "user", "content": user_text})
|
||||
state = AgentState()
|
||||
return await react_loop(
|
||||
llm=self._llm,
|
||||
dispatcher=self._dispatcher,
|
||||
messages=self._messages,
|
||||
tools=self._tools,
|
||||
state=state,
|
||||
settings=self._settings,
|
||||
max_iterations=self._settings.max_tool_iterations,
|
||||
plan_required=self._settings.plan_required,
|
||||
verify_required=self._settings.verify_required,
|
||||
on_trace=self._on_trace,
|
||||
cost_tracker=self._cost_tracker,
|
||||
chunk_store=self._chunk_store,
|
||||
)
|
||||
|
||||
def _inject_recalled_lessons(self, user_text: str) -> None:
|
||||
if self._lessons is None or self._lessons.count() == 0:
|
||||
return
|
||||
hits = self._lessons.search(user_text, k=self._settings.recall_top_k)
|
||||
if not hits:
|
||||
return
|
||||
lines = [
|
||||
f"- {hit['conclusion']} -> {hit['next_action']}"
|
||||
for hit in hits
|
||||
if hit.get("conclusion") and hit.get("next_action")
|
||||
]
|
||||
if not lines:
|
||||
return
|
||||
note = "[memory] Relevant lessons from past sessions:\n" + "\n".join(lines)
|
||||
self._messages.append({"role": "user", "content": note})
|
||||
logger.debug("Injected %d recalled lessons", len(lines))
|
||||
@@ -0,0 +1,9 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .actions import AGENTIC_ACTIONS
|
||||
from .controller import AgenticController
|
||||
from .lessons import LessonStore
|
||||
from .loop import react_loop
|
||||
from .state import AgentState
|
||||
|
||||
__all__ = ["AGENTIC_ACTIONS", "AgenticController", "LessonStore", "react_loop", "AgentState"]
|
||||
@@ -0,0 +1,101 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..actions.spec import Action, Param
|
||||
|
||||
|
||||
def arg(name: str, description: str, required: bool = False, kind: str = "string") -> Param:
|
||||
return Param(name=name, location="body", description=description, required=required, type=kind)
|
||||
|
||||
|
||||
AGENTIC_ACTIONS: tuple[Action, ...] = (
|
||||
Action(
|
||||
name="plan",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Record the structured execution plan for the current request",
|
||||
description="Must be the very first tool call when a request requires any tool use.",
|
||||
handler="agentic",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
arg("goal", "One-line restatement of the user's goal.", required=True),
|
||||
arg("steps", "Ordered list of step objects, each with id, action, depends_on.", required=True, kind="array"),
|
||||
arg("success_criteria", "Concrete criteria for declaring the task complete.", required=True),
|
||||
arg("confidence", "Self-estimate of plan correctness from 0.0 to 1.0.", kind="number"),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="reflect",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Record a reflection and persist it as a reusable lesson for future requests",
|
||||
description="Call after any tool error and at the end of a non-trivial task to learn from it.",
|
||||
handler="agentic",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
arg("observation", "What was observed: the failure mode or notable outcome.", required=True),
|
||||
arg("conclusion", "Diagnosis or interpretation of the observation.", required=True),
|
||||
arg("next_action", "The chosen next step or the rule to apply in future.", required=True),
|
||||
arg("tags", "Optional comma-separated keywords to aid later recall."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="recall",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Retrieve relevant lessons learned from past requests",
|
||||
description="Search the persistent lesson memory for guidance before acting on unfamiliar work.",
|
||||
handler="agentic",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
arg("query", "Natural-language or keyword query describing the current situation.", required=True),
|
||||
arg("k", "Number of lessons to return.", kind="integer"),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="forget_lessons",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Permanently delete learned lessons from your private memory",
|
||||
description=(
|
||||
"Removes lessons you previously stored with reflect(). With a query, only matching "
|
||||
"lessons are deleted; with no query, all of them are. Use this when the user asks you "
|
||||
"to forget something. Your lesson memory is private to this account or guest session."
|
||||
),
|
||||
handler="agentic",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
arg("query", "Optional topic/keywords; only matching lessons are forgotten. Omit to forget everything."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="verify",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Confirm that a change took effect, satisfying the verification gate",
|
||||
description=(
|
||||
"After mutating actions (create, edit, delete, vote, follow, admin changes), confirm "
|
||||
"the result, then call this with a summary of what you confirmed."
|
||||
),
|
||||
handler="agentic",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
arg("summary", "What was confirmed and how.", required=True),
|
||||
arg("confirmed", "Whether the change was confirmed successful.", kind="boolean"),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="delegate",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Spawn a focused sub-agent to complete a self-contained sub-task in isolation",
|
||||
description="The sub-agent runs its own plan/act/verify loop and returns a concise result.",
|
||||
handler="agentic",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
arg("task", "Clear, scoped task description for the sub-agent.", required=True),
|
||||
arg("allowed_tools", "Optional list of tool names the sub-agent may use.", kind="array"),
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,59 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger("devii.agentic.compaction")
|
||||
|
||||
SUMMARY_INPUT_CAP = 120_000
|
||||
SUMMARY_PROMPT = (
|
||||
"Summarize the following assistant conversation segment as a concise factual log of "
|
||||
"actions taken, tools called, entities created or changed, conclusions reached, and "
|
||||
"outstanding work. Keep identifiers, slugs, uids, and decisions verbatim. Maximum 800 words.\n\n"
|
||||
"---\n\n"
|
||||
)
|
||||
|
||||
|
||||
def context_size(messages: list[dict[str, Any]]) -> int:
|
||||
return len(json.dumps(messages, default=str))
|
||||
|
||||
|
||||
def find_compaction_split(messages: list[dict[str, Any]], keep_tail: int) -> int:
|
||||
if len(messages) <= keep_tail:
|
||||
return 1
|
||||
candidate = len(messages) - keep_tail
|
||||
while candidate > 1:
|
||||
if messages[candidate].get("role") == "user":
|
||||
return candidate
|
||||
candidate -= 1
|
||||
return 1
|
||||
|
||||
|
||||
async def compact_messages(llm: Any, messages: list[dict[str, Any]], keep_tail: int) -> list[dict[str, Any]]:
|
||||
if len(messages) < keep_tail + 3:
|
||||
return messages
|
||||
split = find_compaction_split(messages, keep_tail)
|
||||
if split <= 1:
|
||||
return messages
|
||||
system_message = messages[0]
|
||||
middle = messages[1:split]
|
||||
tail = messages[split:]
|
||||
if not middle:
|
||||
return messages
|
||||
|
||||
segment = json.dumps(middle, default=str)[:SUMMARY_INPUT_CAP]
|
||||
try:
|
||||
summary = await llm.summarize(SUMMARY_PROMPT + segment)
|
||||
except Exception: # noqa: BLE001 - compaction must never break the loop
|
||||
logger.exception("Compaction summary failed; keeping full context")
|
||||
return messages
|
||||
|
||||
logger.info("Compacted %d messages into a summary", len(middle))
|
||||
return [
|
||||
system_message,
|
||||
{"role": "assistant", "content": f"[compacted earlier turns]\n\n{summary}"},
|
||||
*tail,
|
||||
]
|
||||
@@ -0,0 +1,195 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
from ..config import Settings
|
||||
from ..errors import ToolInputError
|
||||
from .lessons import LessonStore
|
||||
from .loop import TraceCallback, react_loop
|
||||
from .state import AgentState, get_state
|
||||
|
||||
logger = logging.getLogger("devii.agentic.controller")
|
||||
|
||||
SUB_AGENT_SYSTEM_PROMPT = (
|
||||
"You are a focused sub-agent spawned to complete a single scoped task on this "
|
||||
"DevPlace platform. Begin with a plan() call. Investigate, then act with the most specific "
|
||||
"tools available. If you change anything, confirm it and call verify() before returning. "
|
||||
"Return a concise factual result string under 1500 characters."
|
||||
)
|
||||
NO_DELEGATE = "delegate"
|
||||
|
||||
|
||||
class AgenticController:
|
||||
def __init__(self, lessons: LessonStore, settings: Settings) -> None:
|
||||
self._lessons = lessons
|
||||
self._settings = settings
|
||||
self._llm: Any = None
|
||||
self._dispatcher: Any = None
|
||||
self._tools: list[dict[str, Any]] = []
|
||||
self._on_trace: Optional[TraceCallback] = None
|
||||
self._cost_tracker: Any = None
|
||||
self._chunk_store: Any = None
|
||||
|
||||
def bind(
|
||||
self,
|
||||
llm: Any,
|
||||
dispatcher: Any,
|
||||
tools: list[dict[str, Any]],
|
||||
on_trace: Optional[TraceCallback],
|
||||
cost_tracker: Any = None,
|
||||
chunk_store: Any = None,
|
||||
) -> None:
|
||||
self._llm = llm
|
||||
self._dispatcher = dispatcher
|
||||
self._tools = tools
|
||||
self._on_trace = on_trace
|
||||
self._cost_tracker = cost_tracker
|
||||
self._chunk_store = chunk_store
|
||||
|
||||
async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
|
||||
handlers = {
|
||||
"plan": self._plan,
|
||||
"reflect": self._reflect,
|
||||
"recall": self._recall,
|
||||
"forget_lessons": self._forget,
|
||||
"verify": self._verify,
|
||||
"delegate": self._delegate,
|
||||
}
|
||||
handler = handlers.get(name)
|
||||
if handler is None:
|
||||
raise ToolInputError(f"Unknown agentic tool: {name}")
|
||||
return await handler(arguments)
|
||||
|
||||
async def _plan(self, arguments: dict[str, Any]) -> str:
|
||||
goal = str(arguments.get("goal", "")).strip()
|
||||
steps = arguments.get("steps") or []
|
||||
if not goal:
|
||||
raise ToolInputError("plan requires a goal.")
|
||||
if not isinstance(steps, list) or not steps:
|
||||
raise ToolInputError("plan requires a non-empty steps list.")
|
||||
confidence = float(arguments.get("confidence", 0.8) or 0.8)
|
||||
state = get_state()
|
||||
if state is not None:
|
||||
state.plan = {
|
||||
"goal": goal,
|
||||
"steps": steps,
|
||||
"success_criteria": arguments.get("success_criteria", ""),
|
||||
"confidence": confidence,
|
||||
}
|
||||
advice = ""
|
||||
if confidence < 0.6:
|
||||
advice = "Confidence is below 0.6 - gather more context or recall() past lessons before executing."
|
||||
return json.dumps(
|
||||
{"status": "success", "plan_recorded": True, "step_count": len(steps), "advice": advice},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
async def _reflect(self, arguments: dict[str, Any]) -> str:
|
||||
observation = str(arguments.get("observation", "")).strip()
|
||||
conclusion = str(arguments.get("conclusion", "")).strip()
|
||||
next_action = str(arguments.get("next_action", "")).strip()
|
||||
if not (observation and conclusion and next_action):
|
||||
raise ToolInputError("reflect requires observation, conclusion, and next_action.")
|
||||
tags = str(arguments.get("tags", "") or "").strip()
|
||||
record = self._lessons.add(observation, conclusion, next_action, tags)
|
||||
state = get_state()
|
||||
if state is not None:
|
||||
state.reflections.append(
|
||||
{"observation": observation, "conclusion": conclusion, "next_action": next_action}
|
||||
)
|
||||
return json.dumps(
|
||||
{
|
||||
"status": "success",
|
||||
"lesson_uid": record["uid"],
|
||||
"total_lessons": self._lessons.count(),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
async def _recall(self, arguments: dict[str, Any]) -> str:
|
||||
query = str(arguments.get("query", "")).strip()
|
||||
if not query:
|
||||
raise ToolInputError("recall requires a query.")
|
||||
k = int(arguments.get("k", self._settings.recall_top_k) or self._settings.recall_top_k)
|
||||
hits = self._lessons.search(query, k=k)
|
||||
return json.dumps({"status": "success", "count": len(hits), "lessons": hits}, ensure_ascii=False)
|
||||
|
||||
async def _forget(self, arguments: dict[str, Any]) -> str:
|
||||
query = str(arguments.get("query", "") or "").strip()
|
||||
if query:
|
||||
removed = 0
|
||||
for hit in self._lessons.search(query, k=50):
|
||||
uid = hit.get("uid")
|
||||
if uid and self._lessons.delete(uid):
|
||||
removed += 1
|
||||
else:
|
||||
removed = self._lessons.clear()
|
||||
return json.dumps(
|
||||
{"status": "success", "forgotten": removed, "remaining": self._lessons.count()},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
async def _verify(self, arguments: dict[str, Any]) -> str:
|
||||
summary = str(arguments.get("summary", "")).strip()
|
||||
if not summary:
|
||||
raise ToolInputError("verify requires a summary of what was confirmed.")
|
||||
confirmed = arguments.get("confirmed", True)
|
||||
if isinstance(confirmed, str):
|
||||
confirmed = confirmed.strip().lower() not in ("", "false", "no", "0")
|
||||
state = get_state()
|
||||
if state is not None and confirmed:
|
||||
state.verified = True
|
||||
return json.dumps(
|
||||
{"status": "success", "verified": bool(confirmed), "summary": summary}, ensure_ascii=False
|
||||
)
|
||||
|
||||
async def _delegate(self, arguments: dict[str, Any]) -> str:
|
||||
task = str(arguments.get("task", "")).strip()
|
||||
if not task:
|
||||
raise ToolInputError("delegate requires a task description.")
|
||||
if self._llm is None or self._dispatcher is None:
|
||||
raise ToolInputError("Delegation is not available in this context.")
|
||||
allowed = arguments.get("allowed_tools")
|
||||
if allowed:
|
||||
allowed_set = {str(name) for name in allowed}
|
||||
tools = [
|
||||
tool
|
||||
for tool in self._tools
|
||||
if tool["function"]["name"] in allowed_set and tool["function"]["name"] != NO_DELEGATE
|
||||
]
|
||||
else:
|
||||
tools = [tool for tool in self._tools if tool["function"]["name"] != NO_DELEGATE]
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": SUB_AGENT_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": task},
|
||||
]
|
||||
sub_state = AgentState()
|
||||
result = await react_loop(
|
||||
llm=self._llm,
|
||||
dispatcher=self._dispatcher,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
state=sub_state,
|
||||
settings=self._settings,
|
||||
max_iterations=self._settings.delegate_max_iterations,
|
||||
plan_required=self._settings.plan_required,
|
||||
verify_required=self._settings.verify_required,
|
||||
on_trace=self._on_trace,
|
||||
cost_tracker=self._cost_tracker,
|
||||
chunk_store=self._chunk_store,
|
||||
)
|
||||
return json.dumps(
|
||||
{
|
||||
"status": "success",
|
||||
"iterations": sub_state.iteration,
|
||||
"verified": sub_state.verified,
|
||||
"reflections": len(sub_state.reflections),
|
||||
"result": result,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
@@ -0,0 +1,161 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from ..tasks.schedule import now_utc, to_iso
|
||||
|
||||
logger = logging.getLogger("devii.agentic.lessons")
|
||||
|
||||
TABLE = "devii_lessons"
|
||||
TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*|\d+")
|
||||
BM25_K1 = 1.5
|
||||
BM25_B = 0.75
|
||||
|
||||
|
||||
def tokenize(text: str) -> list[str]:
|
||||
tokens = TOKEN_RE.findall((text or "").lower())
|
||||
extra: list[str] = []
|
||||
for token in tokens:
|
||||
extra.extend(re.findall(r"[a-z]+", token))
|
||||
return list(dict.fromkeys(tokens + extra))
|
||||
|
||||
|
||||
class LessonStore:
|
||||
def __init__(self, db: Any, owner_kind: str, owner_id: str) -> None:
|
||||
self._db = db
|
||||
self._owner_kind = owner_kind
|
||||
self._owner_id = owner_id
|
||||
self._dirty = True
|
||||
self._docs: list[dict[str, Any]] = []
|
||||
self._tf: list[collections.Counter] = []
|
||||
self._dl: list[int] = []
|
||||
self._idf: dict[str, float] = {}
|
||||
self._avgdl = 0.0
|
||||
self._n = 0
|
||||
self._ensure_indexes()
|
||||
|
||||
def _ensure_indexes(self) -> None:
|
||||
if TABLE not in self._db.tables:
|
||||
return
|
||||
self._db[TABLE].create_index(["owner_kind", "owner_id"])
|
||||
|
||||
@property
|
||||
def _table(self) -> Any:
|
||||
return self._db[TABLE]
|
||||
|
||||
@property
|
||||
def _scope(self) -> dict[str, str]:
|
||||
return {"owner_kind": self._owner_kind, "owner_id": self._owner_id}
|
||||
|
||||
def count(self) -> int:
|
||||
if TABLE not in self._db.tables:
|
||||
return 0
|
||||
return self._table.count(**self._scope)
|
||||
|
||||
def add(self, observation: str, conclusion: str, next_action: str, tags: str = "") -> dict[str, Any]:
|
||||
record = {
|
||||
"uid": uuid.uuid4().hex,
|
||||
"observation": observation,
|
||||
"conclusion": conclusion,
|
||||
"next_action": next_action,
|
||||
"tags": tags,
|
||||
"created_at": to_iso(now_utc()),
|
||||
"hits": 0,
|
||||
**self._scope,
|
||||
}
|
||||
self._table.insert(record)
|
||||
self._dirty = True
|
||||
logger.info("Lesson stored owner=%s/%s tags=%s", self._owner_kind, self._owner_id, tags)
|
||||
return record
|
||||
|
||||
def all(self) -> list[dict[str, Any]]:
|
||||
if TABLE not in self._db.tables:
|
||||
return []
|
||||
return list(self._table.find(**self._scope))
|
||||
|
||||
def delete(self, uid: str) -> bool:
|
||||
if TABLE not in self._db.tables:
|
||||
return False
|
||||
deleted = self._table.delete(uid=uid, **self._scope)
|
||||
self._dirty = True
|
||||
return bool(deleted)
|
||||
|
||||
def clear(self) -> int:
|
||||
n = self.count()
|
||||
if TABLE in self._db.tables:
|
||||
self._table.delete(**self._scope)
|
||||
self._dirty = True
|
||||
logger.info("Cleared %d lesson(s) for owner=%s/%s", n, self._owner_kind, self._owner_id)
|
||||
return n
|
||||
|
||||
def _rebuild(self) -> None:
|
||||
rows = self.all()
|
||||
df: collections.Counter = collections.Counter()
|
||||
docs: list[dict[str, Any]] = []
|
||||
tf_list: list[collections.Counter] = []
|
||||
dl: list[int] = []
|
||||
for row in rows:
|
||||
text = " ".join(
|
||||
str(row.get(field) or "")
|
||||
for field in ("observation", "conclusion", "next_action", "tags")
|
||||
)
|
||||
tokens = tokenize(text)
|
||||
if not tokens:
|
||||
continue
|
||||
tf = collections.Counter(tokens)
|
||||
for term in tf:
|
||||
df[term] += 1
|
||||
docs.append(row)
|
||||
tf_list.append(tf)
|
||||
dl.append(len(tokens))
|
||||
self._docs = docs
|
||||
self._tf = tf_list
|
||||
self._dl = dl
|
||||
self._n = len(docs)
|
||||
self._avgdl = sum(dl) / max(self._n, 1)
|
||||
self._idf = {
|
||||
term: math.log((self._n - freq + 0.5) / (freq + 0.5) + 1)
|
||||
for term, freq in df.items()
|
||||
}
|
||||
self._dirty = False
|
||||
|
||||
def search(self, query: str, k: int = 3) -> list[dict[str, Any]]:
|
||||
if self._dirty:
|
||||
self._rebuild()
|
||||
tokens = tokenize(query)
|
||||
if not tokens or not self._docs:
|
||||
return []
|
||||
scored: list[tuple[float, int]] = []
|
||||
for index, tf in enumerate(self._tf):
|
||||
score = 0.0
|
||||
for token in tokens:
|
||||
freq = tf.get(token, 0)
|
||||
if freq == 0:
|
||||
continue
|
||||
idf = self._idf.get(token, 0.0)
|
||||
norm = 1 - BM25_B + BM25_B * (self._dl[index] / max(self._avgdl, 1))
|
||||
score += idf * (freq * (BM25_K1 + 1)) / (freq + BM25_K1 * norm)
|
||||
if score > 0:
|
||||
scored.append((score, index))
|
||||
scored.sort(reverse=True)
|
||||
results: list[dict[str, Any]] = []
|
||||
for score, index in scored[:k]:
|
||||
row = self._docs[index]
|
||||
results.append(
|
||||
{
|
||||
"uid": row.get("uid"),
|
||||
"observation": row.get("observation"),
|
||||
"conclusion": row.get("conclusion"),
|
||||
"next_action": row.get("next_action"),
|
||||
"tags": row.get("tags"),
|
||||
"score": round(score, 3),
|
||||
}
|
||||
)
|
||||
return results
|
||||
@@ -0,0 +1,242 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
from ..config import Settings
|
||||
from ..errors import LLMError
|
||||
from ..chunks import reset_store, set_store
|
||||
from ..cost import reset_tracker, set_tracker
|
||||
from .compaction import compact_messages, context_size
|
||||
from .state import AgentState, reset_state, set_state
|
||||
|
||||
logger = logging.getLogger("devii.agentic.loop")
|
||||
|
||||
TraceCallback = Callable[[str, str, str], None]
|
||||
OUTPUT_CAP_CHARS = 200_000
|
||||
|
||||
REFLECTION_TRIGGER = (
|
||||
"[reflection-trigger] One or more tool calls returned an error. Call reflect() with the "
|
||||
"observation, root-cause conclusion, and next action before retrying. Do not repeat the "
|
||||
"same call without diagnosis."
|
||||
)
|
||||
PLAN_VIOLATION = (
|
||||
"Protocol violation: your first tool call must be plan(). Restart with a structured plan "
|
||||
"before any other action."
|
||||
)
|
||||
VERIFICATION_GATE = (
|
||||
"[verification-gate] You produced a final answer after performing changes without calling "
|
||||
"verify(). Confirm the change took effect and call verify() with a summary. If verification "
|
||||
"truly does not apply, reply explicitly starting with: 'No verification applicable: <reason>'."
|
||||
)
|
||||
ITERATION_LIMIT_MESSAGE = "[stopped] Maximum iterations reached without a final answer."
|
||||
|
||||
|
||||
LABEL_KEYS = (
|
||||
"post_slug",
|
||||
"project_slug",
|
||||
"gist_slug",
|
||||
"news_slug",
|
||||
"username",
|
||||
"comment_uid",
|
||||
"notification_uid",
|
||||
"attachment_uid",
|
||||
"poll_uid",
|
||||
"receiver_uid",
|
||||
"uid",
|
||||
"name",
|
||||
"q",
|
||||
"query",
|
||||
"title",
|
||||
"goal",
|
||||
"task",
|
||||
"email",
|
||||
)
|
||||
LABEL_MAX_CHARS = 60
|
||||
|
||||
|
||||
def call_label(call: dict[str, Any]) -> str:
|
||||
raw = call.get("function", {}).get("arguments") or "{}"
|
||||
try:
|
||||
arguments = json.loads(raw) if isinstance(raw, str) else raw
|
||||
except (ValueError, TypeError):
|
||||
return ""
|
||||
if not isinstance(arguments, dict):
|
||||
return ""
|
||||
|
||||
def clean(value: Any) -> str:
|
||||
return str(value).replace("\n", " ").strip()[:LABEL_MAX_CHARS]
|
||||
|
||||
if arguments.get("target_uid"):
|
||||
target_type = arguments.get("target_type")
|
||||
target = clean(arguments["target_uid"])
|
||||
return f"{target_type}/{target}" if target_type else target
|
||||
for key in LABEL_KEYS:
|
||||
value = arguments.get(key)
|
||||
if value not in (None, ""):
|
||||
return clean(value)
|
||||
for value in arguments.values():
|
||||
if isinstance(value, (str, int, float, bool)) and clean(value):
|
||||
return clean(value)
|
||||
return ""
|
||||
|
||||
|
||||
def normalize(message: dict[str, Any]) -> dict[str, Any]:
|
||||
normalized: dict[str, Any] = {
|
||||
"role": message.get("role", "assistant"),
|
||||
"content": message.get("content") or "",
|
||||
}
|
||||
if message.get("tool_calls"):
|
||||
normalized["tool_calls"] = message["tool_calls"]
|
||||
return normalized
|
||||
|
||||
|
||||
def _is_error(result: str) -> bool:
|
||||
try:
|
||||
return isinstance(json.loads(result), dict) and json.loads(result).get("error") is not None
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def _summary(result: str) -> str:
|
||||
try:
|
||||
parsed = json.loads(result)
|
||||
except (ValueError, TypeError):
|
||||
return ""
|
||||
if not isinstance(parsed, dict):
|
||||
return ""
|
||||
if parsed.get("error"):
|
||||
return str(parsed.get("message") or parsed.get("error"))
|
||||
if "status" in parsed:
|
||||
return str(parsed["status"])
|
||||
return ""
|
||||
|
||||
|
||||
async def _run_tool_call(dispatcher: Any, call: dict[str, Any]) -> str:
|
||||
function = call.get("function", {})
|
||||
name = function.get("name", "")
|
||||
raw_arguments = function.get("arguments") or "{}"
|
||||
try:
|
||||
arguments = json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments
|
||||
if not isinstance(arguments, dict):
|
||||
raise ValueError("arguments must be an object")
|
||||
except ValueError as exc:
|
||||
return json.dumps({"error": "tool_input_error", "message": f"Invalid arguments: {exc}"})
|
||||
return await dispatcher.dispatch(name, arguments)
|
||||
|
||||
|
||||
async def react_loop(
|
||||
llm: Any,
|
||||
dispatcher: Any,
|
||||
messages: list[dict[str, Any]],
|
||||
tools: list[dict[str, Any]],
|
||||
state: AgentState,
|
||||
settings: Settings,
|
||||
max_iterations: int,
|
||||
plan_required: bool,
|
||||
verify_required: bool,
|
||||
on_trace: Optional[TraceCallback] = None,
|
||||
cost_tracker: Any = None,
|
||||
chunk_store: Any = None,
|
||||
) -> str:
|
||||
def trace(event: str, name: str = "", detail: str = "") -> None:
|
||||
if on_trace is not None:
|
||||
on_trace(event, name, detail)
|
||||
|
||||
token = set_state(state)
|
||||
cost_token = set_tracker(cost_tracker) if cost_tracker is not None else None
|
||||
chunk_token = set_store(chunk_store) if chunk_store is not None else None
|
||||
final_content = ""
|
||||
try:
|
||||
while state.iteration < max_iterations:
|
||||
state.iteration += 1
|
||||
|
||||
if context_size(messages) > settings.context_compact_threshold:
|
||||
trace("compact")
|
||||
messages[:] = await compact_messages(llm, messages, settings.context_keep_tail)
|
||||
|
||||
try:
|
||||
message = await llm.complete(messages, tools)
|
||||
except LLMError as exc:
|
||||
logger.info("LLM error: %s", exc.message)
|
||||
return f"[model error] {exc.message}"
|
||||
|
||||
messages.append(normalize(message))
|
||||
tool_calls = message.get("tool_calls") or []
|
||||
|
||||
if tool_calls:
|
||||
if plan_required and state.plan is None and tool_calls[0]["function"]["name"] != "plan":
|
||||
trace("plan-gate")
|
||||
for call in tool_calls:
|
||||
messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": call.get("id", ""),
|
||||
"name": call.get("function", {}).get("name", ""),
|
||||
"content": json.dumps({"error": "protocol", "message": PLAN_VIOLATION}),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
for call in tool_calls:
|
||||
trace("call", call.get("function", {}).get("name", ""), call_label(call))
|
||||
|
||||
results = await asyncio.gather(
|
||||
*[_run_tool_call(dispatcher, call) for call in tool_calls]
|
||||
)
|
||||
|
||||
any_error = False
|
||||
for call, result in zip(tool_calls, results):
|
||||
if len(result) > OUTPUT_CAP_CHARS:
|
||||
result = result[:OUTPUT_CAP_CHARS] + f"\n...[truncated {len(result)} chars]"
|
||||
name = call.get("function", {}).get("name", "")
|
||||
messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": call.get("id", ""),
|
||||
"name": name,
|
||||
"content": result,
|
||||
}
|
||||
)
|
||||
label = call_label(call)
|
||||
summary = _summary(result)
|
||||
detail = " ".join(part for part in (label, f"({summary})" if summary else "") if part)
|
||||
if _is_error(result):
|
||||
any_error = True
|
||||
trace("err", name, detail)
|
||||
else:
|
||||
trace("ok", name, detail)
|
||||
|
||||
if any_error:
|
||||
trace("reflect-trigger")
|
||||
messages.append({"role": "user", "content": REFLECTION_TRIGGER})
|
||||
continue
|
||||
|
||||
content = message.get("content")
|
||||
if content:
|
||||
if (
|
||||
verify_required
|
||||
and not state.verified
|
||||
and not state.gate_triggered
|
||||
and state.mutations
|
||||
):
|
||||
state.gate_triggered = True
|
||||
trace("verify-gate")
|
||||
messages.append({"role": "user", "content": VERIFICATION_GATE})
|
||||
continue
|
||||
final_content = content
|
||||
break
|
||||
|
||||
if state.iteration >= max_iterations and not final_content:
|
||||
return ITERATION_LIMIT_MESSAGE
|
||||
return final_content
|
||||
finally:
|
||||
if chunk_token is not None:
|
||||
reset_store(chunk_token)
|
||||
if cost_token is not None:
|
||||
reset_tracker(cost_token)
|
||||
reset_state(token)
|
||||
@@ -0,0 +1,37 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
|
||||
_active_state: contextvars.ContextVar = contextvars.ContextVar("devii_agent_state", default=None)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgentState:
|
||||
plan: Optional[dict[str, Any]] = None
|
||||
reflections: list[dict[str, str]] = field(default_factory=list)
|
||||
iteration: int = 0
|
||||
verified: bool = False
|
||||
gate_triggered: bool = False
|
||||
mutations: set[str] = field(default_factory=set)
|
||||
|
||||
|
||||
def get_state() -> Optional[AgentState]:
|
||||
return _active_state.get()
|
||||
|
||||
|
||||
def set_state(state: AgentState) -> contextvars.Token:
|
||||
return _active_state.set(state)
|
||||
|
||||
|
||||
def reset_state(token: contextvars.Token) -> None:
|
||||
_active_state.reset(token)
|
||||
|
||||
|
||||
def record_mutation(name: str) -> None:
|
||||
state = _active_state.get()
|
||||
if state is not None:
|
||||
state.mutations.add(name)
|
||||
@@ -0,0 +1,5 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .controller import AvatarController
|
||||
|
||||
__all__ = ["AvatarController"]
|
||||
@@ -0,0 +1,56 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Awaitable, Callable, Optional
|
||||
|
||||
logger = logging.getLogger("devii.avatar")
|
||||
|
||||
RequestSink = Callable[[str, dict[str, Any]], Awaitable[Any]]
|
||||
|
||||
PREFIX = "avatar_"
|
||||
UNAVAILABLE = {
|
||||
"status": "unavailable",
|
||||
"message": (
|
||||
"No avatar is attached to this session. devii's on-screen actions are only available "
|
||||
"in the web interface (devii-terminal with a devii-avatar)."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class AvatarController:
|
||||
def __init__(self) -> None:
|
||||
self._request: Optional[RequestSink] = None
|
||||
|
||||
def bind(self, request: RequestSink) -> None:
|
||||
self._request = request
|
||||
logger.debug("Avatar controller bound to a session")
|
||||
|
||||
def unbind(self) -> None:
|
||||
self._request = None
|
||||
logger.debug("Avatar controller unbound")
|
||||
|
||||
@property
|
||||
def attached(self) -> bool:
|
||||
return self._request is not None
|
||||
|
||||
async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
|
||||
action = name[len(PREFIX):] if name.startswith(PREFIX) else name
|
||||
if self._request is None:
|
||||
return json.dumps(UNAVAILABLE, ensure_ascii=False)
|
||||
try:
|
||||
result = await self._request(action, arguments)
|
||||
except Exception as exc: # noqa: BLE001 - surfaced to the model as data
|
||||
logger.exception("Avatar action %s failed", action)
|
||||
return json.dumps({"status": "error", "action": action, "message": str(exc)})
|
||||
|
||||
if isinstance(result, dict) and result.get("error"):
|
||||
return json.dumps(
|
||||
{"status": "error", "action": action, "message": str(result["error"])},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
return json.dumps(
|
||||
{"status": "success", "action": action, "result": result}, ensure_ascii=False
|
||||
)
|
||||
@@ -0,0 +1,23 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .controller import ChunkController
|
||||
from .store import (
|
||||
ChunkStore,
|
||||
chunk_envelope,
|
||||
get_store,
|
||||
reset_store,
|
||||
serve_resource,
|
||||
set_store,
|
||||
wrap_if_large,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ChunkController",
|
||||
"ChunkStore",
|
||||
"chunk_envelope",
|
||||
"get_store",
|
||||
"set_store",
|
||||
"reset_store",
|
||||
"serve_resource",
|
||||
"wrap_if_large",
|
||||
]
|
||||
@@ -0,0 +1,61 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from ..config import Settings
|
||||
from ..errors import ToolInputError
|
||||
from .store import get_store
|
||||
|
||||
|
||||
class ChunkController:
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self._settings = settings
|
||||
|
||||
async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
|
||||
if name != "read_more":
|
||||
raise ToolInputError(f"Unknown chunk tool: {name}")
|
||||
store = get_store()
|
||||
if store is None:
|
||||
return json.dumps(
|
||||
{"status": "unavailable", "message": "No chunk store is active for this session."}
|
||||
)
|
||||
chunk_id = str(arguments.get("chunk_id", "")).strip()
|
||||
if not chunk_id:
|
||||
raise ToolInputError("read_more requires a chunk_id.")
|
||||
text = store.get(chunk_id)
|
||||
if text is None:
|
||||
return json.dumps(
|
||||
{
|
||||
"status": "not_found",
|
||||
"message": (
|
||||
f"No stored content for chunk_id '{chunk_id}'. It may have expired; "
|
||||
"re-run the original tool to get a fresh chunk_id."
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
offset = max(0, int(arguments.get("offset", 0) or 0))
|
||||
length = int(arguments.get("length", self._settings.max_response_chars) or self._settings.max_response_chars)
|
||||
length = max(1, min(length, self._settings.max_response_chars))
|
||||
|
||||
total = len(text)
|
||||
piece = text[offset : offset + length]
|
||||
next_offset = offset + len(piece)
|
||||
has_more = next_offset < total
|
||||
return json.dumps(
|
||||
{
|
||||
"status": "success",
|
||||
"chunk_id": chunk_id,
|
||||
"offset": offset,
|
||||
"shown_chars": len(piece),
|
||||
"total_chars": total,
|
||||
"remaining_chars": max(0, total - next_offset),
|
||||
"next_offset": next_offset if has_more else None,
|
||||
"has_more": has_more,
|
||||
"content": piece,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
@@ -0,0 +1,130 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from collections import OrderedDict
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger("devii.chunks")
|
||||
|
||||
STORE_MAX_ENTRIES = 24
|
||||
STORE_MAX_CHARS = 2_000_000
|
||||
|
||||
_active_store: contextvars.ContextVar = contextvars.ContextVar("devii_chunk_store", default=None)
|
||||
|
||||
|
||||
class ChunkStore:
|
||||
def __init__(self) -> None:
|
||||
self._items: "OrderedDict[str, str]" = OrderedDict()
|
||||
self._resources: dict[str, str] = {}
|
||||
|
||||
def put(self, text: str, resource_key: Optional[str] = None) -> str:
|
||||
chunk_id = uuid.uuid4().hex[:12]
|
||||
self._items[chunk_id] = text[:STORE_MAX_CHARS]
|
||||
self._items.move_to_end(chunk_id)
|
||||
if resource_key:
|
||||
self._resources[resource_key] = chunk_id
|
||||
while len(self._items) > STORE_MAX_ENTRIES:
|
||||
evicted, _ = self._items.popitem(last=False)
|
||||
self._drop_resource_for(evicted)
|
||||
logger.debug("Stored chunkable result %s (%d chars)", chunk_id, len(text))
|
||||
return chunk_id
|
||||
|
||||
def get(self, chunk_id: str) -> Optional[str]:
|
||||
text = self._items.get(chunk_id)
|
||||
if text is not None:
|
||||
self._items.move_to_end(chunk_id)
|
||||
return text
|
||||
|
||||
def get_by_resource(self, resource_key: str) -> Optional[tuple[str, str]]:
|
||||
chunk_id = self._resources.get(resource_key)
|
||||
if chunk_id is None:
|
||||
return None
|
||||
text = self._items.get(chunk_id)
|
||||
if text is None:
|
||||
del self._resources[resource_key]
|
||||
return None
|
||||
self._items.move_to_end(chunk_id)
|
||||
return chunk_id, text
|
||||
|
||||
def invalidate_resources(self, prefix: str = "") -> None:
|
||||
if not prefix:
|
||||
self._resources.clear()
|
||||
return
|
||||
for key in [key for key in self._resources if key.startswith(prefix)]:
|
||||
del self._resources[key]
|
||||
|
||||
def _drop_resource_for(self, chunk_id: str) -> None:
|
||||
for key in [key for key, value in self._resources.items() if value == chunk_id]:
|
||||
del self._resources[key]
|
||||
|
||||
|
||||
def get_store() -> Optional[ChunkStore]:
|
||||
return _active_store.get()
|
||||
|
||||
|
||||
def set_store(store: ChunkStore) -> contextvars.Token:
|
||||
return _active_store.set(store)
|
||||
|
||||
|
||||
def reset_store(token: contextvars.Token) -> None:
|
||||
_active_store.reset(token)
|
||||
|
||||
|
||||
def chunk_envelope(text: str, chunk_id: str, max_chars: int) -> str:
|
||||
total = len(text)
|
||||
shown = text[:max_chars]
|
||||
return json.dumps(
|
||||
{
|
||||
"truncated": True,
|
||||
"chunk_id": chunk_id,
|
||||
"total_chars": total,
|
||||
"shown_chars": len(shown),
|
||||
"remaining_chars": total - len(shown),
|
||||
"next_offset": len(shown),
|
||||
"hint": (
|
||||
f"Output truncated. Call read_more with chunk_id='{chunk_id}' and "
|
||||
f"offset={len(shown)} to read the next part (repeat until remaining_chars is 0). "
|
||||
"Do not re-run the original tool; the full content is already cached."
|
||||
),
|
||||
"content": shown,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
|
||||
def wrap_if_large(result: str, max_chars: int, resource_key: Optional[str] = None) -> str:
|
||||
if len(result) <= max_chars:
|
||||
return result
|
||||
store = get_store()
|
||||
if store is None:
|
||||
total = len(result)
|
||||
shown = result[:max_chars]
|
||||
return json.dumps(
|
||||
{
|
||||
"truncated": True,
|
||||
"total_chars": total,
|
||||
"shown_chars": len(shown),
|
||||
"remaining_chars": total - len(shown),
|
||||
"hint": "Output truncated; no chunk store is available to fetch the rest.",
|
||||
"content": shown,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
chunk_id = store.put(result, resource_key)
|
||||
return chunk_envelope(result, chunk_id, max_chars)
|
||||
|
||||
|
||||
def serve_resource(resource_key: str, max_chars: int) -> Optional[str]:
|
||||
store = get_store()
|
||||
if store is None:
|
||||
return None
|
||||
hit = store.get_by_resource(resource_key)
|
||||
if hit is None:
|
||||
return None
|
||||
chunk_id, text = hit
|
||||
return chunk_envelope(text, chunk_id, max_chars)
|
||||
@@ -0,0 +1,271 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import replace
|
||||
from typing import Any, Optional
|
||||
|
||||
import dataset
|
||||
|
||||
from . import console
|
||||
from .actions import Dispatcher
|
||||
from .agent import Agent
|
||||
from .agentic import AgenticController, LessonStore
|
||||
from .chunks import ChunkStore
|
||||
from .config import Settings, load_settings
|
||||
from .cost import CostTracker
|
||||
from .errors import AuthRequiredError
|
||||
from .http_client import PlatformClient
|
||||
from .llm import LLMClient
|
||||
from .registry import CATALOG
|
||||
from .tasks import Scheduler, TaskController, TaskStore
|
||||
|
||||
logger = logging.getLogger("devii.cli")
|
||||
|
||||
EXIT_COMMANDS = {"/exit", "/quit", "/q"}
|
||||
WELCOME = "Devii - DevPlace assistant"
|
||||
LOGIN_REQUEST = (
|
||||
"Welcome. I manage your DevPlace account, but you must log in first. "
|
||||
"Please tell me your email and password to continue."
|
||||
)
|
||||
GOODBYE = "Session ended."
|
||||
RESULT_NOTICE_CHARS = 20000
|
||||
CLI_TASKS_DB = "devii_tasks.db"
|
||||
|
||||
|
||||
def _label(row: dict[str, Any]) -> str:
|
||||
return row.get("label") or row.get("uid", "task")
|
||||
|
||||
|
||||
def _make_event_handler() -> Any:
|
||||
def handle(kind: str, row: dict[str, Any], payload: str) -> None:
|
||||
name = _label(row)
|
||||
if kind == "start":
|
||||
console.notice(f"\n[task {name}] running...")
|
||||
elif kind in ("done", "finished"):
|
||||
suffix = " (completed, no further runs)" if kind == "finished" else ""
|
||||
console.notice(f"[task {name}] finished{suffix}:")
|
||||
console.assistant(payload[:RESULT_NOTICE_CHARS])
|
||||
elif kind == "error":
|
||||
console.warn(f"[task {name}] failed: {payload}")
|
||||
|
||||
return handle
|
||||
|
||||
|
||||
TRACE_GLYPHS = {
|
||||
"call": "↳",
|
||||
"ok": "✓",
|
||||
"err": "✗",
|
||||
"plan-gate": "plan required first",
|
||||
"reflect-trigger": "reflecting on error",
|
||||
"verify-gate": "verification gate",
|
||||
"compact": "context compaction",
|
||||
}
|
||||
|
||||
|
||||
def _trace(event: str, name: str = "", detail: str = "") -> None:
|
||||
glyph = TRACE_GLYPHS.get(event, event)
|
||||
if event in ("call", "ok", "err"):
|
||||
suffix = f" {detail}" if detail else ""
|
||||
console.notice(f" {glyph} {name}{suffix}")
|
||||
else:
|
||||
console.notice(f" ! {glyph}")
|
||||
|
||||
|
||||
def _make_executor(
|
||||
settings: Settings,
|
||||
llm: LLMClient,
|
||||
dispatcher: Dispatcher,
|
||||
lessons: LessonStore,
|
||||
cost_tracker: CostTracker,
|
||||
chunk_store: ChunkStore,
|
||||
) -> Any:
|
||||
tools = CATALOG.tool_schemas()
|
||||
|
||||
async def execute(prompt: str) -> str:
|
||||
worker = Agent(
|
||||
settings, llm, dispatcher, tools,
|
||||
lessons=lessons, cost_tracker=cost_tracker, chunk_store=chunk_store,
|
||||
)
|
||||
return await worker.respond(prompt)
|
||||
|
||||
return execute
|
||||
|
||||
|
||||
async def _bootstrap_auth(client: PlatformClient, settings: Settings) -> str:
|
||||
if client.authenticated:
|
||||
logger.info("Authenticated via API key")
|
||||
return "Authenticated via API key. How can I help?"
|
||||
if settings.login_email and settings.login_password:
|
||||
try:
|
||||
await client.login(settings.login_email, settings.login_password)
|
||||
return f"Logged in as {client.username}. How can I help?"
|
||||
except AuthRequiredError as exc:
|
||||
console.warn(f"[auth] automatic login failed: {exc.message}")
|
||||
return LOGIN_REQUEST
|
||||
|
||||
|
||||
def _resolve_owner(settings: Settings) -> tuple[str, str]:
|
||||
"""Resolve the local DevPlace user behind the CLI's credentials so the CLI shares the
|
||||
same per-user memory, tasks, and admin analytics as the web session for that account."""
|
||||
try:
|
||||
from devplacepy.database import db, get_table
|
||||
if "users" not in db.tables:
|
||||
return "user", "cli"
|
||||
users = get_table("users")
|
||||
user = None
|
||||
if settings.platform_api_key:
|
||||
user = users.find_one(api_key=settings.platform_api_key)
|
||||
if user is None and settings.login_email:
|
||||
user = users.find_one(email=settings.login_email.lower())
|
||||
if user:
|
||||
return "user", user["uid"]
|
||||
except Exception: # noqa: BLE001 - no local DB / remote-only: fall back to a standalone store
|
||||
return "user", "cli"
|
||||
return "user", "cli"
|
||||
|
||||
|
||||
def _build_stores(owner_kind: str, owner_id: str):
|
||||
if owner_id == "cli":
|
||||
owned_db = dataset.connect(f"sqlite:///{CLI_TASKS_DB}")
|
||||
else:
|
||||
from devplacepy.database import db as owned_db # share the platform DB for this account
|
||||
return TaskStore(owned_db, owner_kind, owner_id), LessonStore(owned_db, owner_kind, owner_id)
|
||||
|
||||
|
||||
async def run(settings: Settings, prompt: Optional[str] = None) -> None:
|
||||
client = PlatformClient(settings.base_url, settings.timeout_seconds, settings.platform_api_key)
|
||||
llm = LLMClient(settings)
|
||||
greeting = await _bootstrap_auth(client, settings)
|
||||
|
||||
owner_kind, owner_id = _resolve_owner(settings)
|
||||
store, lessons = _build_stores(owner_kind, owner_id)
|
||||
controller = TaskController(store)
|
||||
cost_tracker = CostTracker()
|
||||
chunk_store = ChunkStore()
|
||||
agentic = AgenticController(lessons, settings)
|
||||
dispatcher = Dispatcher(CATALOG, client, settings, controller, agentic)
|
||||
tools = CATALOG.tool_schemas_for(client.authenticated)
|
||||
agentic.bind(
|
||||
llm=llm, dispatcher=dispatcher, tools=tools,
|
||||
on_trace=_trace, cost_tracker=cost_tracker, chunk_store=chunk_store,
|
||||
)
|
||||
agent = Agent(
|
||||
settings, llm, dispatcher, tools,
|
||||
lessons=lessons, on_trace=_trace, cost_tracker=cost_tracker, chunk_store=chunk_store,
|
||||
)
|
||||
|
||||
if prompt is not None:
|
||||
try:
|
||||
reply = await agent.respond(prompt)
|
||||
print(reply)
|
||||
finally:
|
||||
await client.aclose()
|
||||
await llm.aclose()
|
||||
return
|
||||
|
||||
scheduler = Scheduler(
|
||||
store,
|
||||
_make_executor(settings, llm, dispatcher, lessons, cost_tracker, chunk_store),
|
||||
_make_event_handler(),
|
||||
settings.scheduler_tick_seconds,
|
||||
)
|
||||
|
||||
console.banner(WELCOME)
|
||||
console.notice(f"Endpoint {settings.base_url} | model {settings.ai_model}")
|
||||
console.notice(
|
||||
"Type /exit to quit. Agentic ReAct loop with planning, self-learning memory, and "
|
||||
"background tasks is active.\n"
|
||||
)
|
||||
console.assistant(greeting)
|
||||
|
||||
scheduler.start()
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
user_text = (await console.prompt("you> ")).strip()
|
||||
except (EOFError, KeyboardInterrupt):
|
||||
print()
|
||||
break
|
||||
|
||||
if not user_text:
|
||||
continue
|
||||
if user_text.lower() in EXIT_COMMANDS:
|
||||
break
|
||||
|
||||
try:
|
||||
reply = await agent.respond(user_text)
|
||||
except Exception as exc: # noqa: BLE001 - keep the chat alive
|
||||
logger.exception("Unhandled error while responding")
|
||||
console.warn(f"[error] {exc}")
|
||||
continue
|
||||
|
||||
console.assistant(reply)
|
||||
finally:
|
||||
await scheduler.stop()
|
||||
await client.aclose()
|
||||
await llm.aclose()
|
||||
console.notice(GOODBYE)
|
||||
|
||||
|
||||
def _parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="devii",
|
||||
description="Agentic chat assistant that manages a DevPlace account.",
|
||||
)
|
||||
parser.add_argument("-p", "--prompt", help="Run a single prompt non-interactively and exit.")
|
||||
parser.add_argument("--api-key", help="DevPlace API key for automatic authentication (Bearer).")
|
||||
parser.add_argument("--email", help="Login email for automatic basic-auth login.")
|
||||
parser.add_argument("--password", help="Login password for automatic basic-auth login.")
|
||||
parser.add_argument("--basic-auth", help="Shorthand credentials in the form email:password.")
|
||||
parser.add_argument("--base-url", help="Override the platform base URL.")
|
||||
parser.add_argument("--ai-url", help="Override the model endpoint URL.")
|
||||
parser.add_argument("--ai-key", help="Override the model API key.")
|
||||
parser.add_argument("--model", help="Override the model name.")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def _settings_from_args(args: argparse.Namespace) -> Settings:
|
||||
settings = load_settings()
|
||||
email = args.email or settings.login_email
|
||||
password = args.password or settings.login_password
|
||||
if args.basic_auth and ":" in args.basic_auth:
|
||||
email, password = args.basic_auth.split(":", 1)
|
||||
overrides: dict[str, Any] = {"login_email": email, "login_password": password}
|
||||
if args.api_key:
|
||||
overrides["platform_api_key"] = args.api_key
|
||||
if args.base_url:
|
||||
overrides["base_url"] = args.base_url.rstrip("/")
|
||||
if args.ai_url:
|
||||
overrides["ai_url"] = args.ai_url
|
||||
if args.ai_key:
|
||||
overrides["ai_key"] = args.ai_key
|
||||
if args.model:
|
||||
overrides["ai_model"] = args.model
|
||||
return replace(settings, **overrides)
|
||||
|
||||
|
||||
NOISY_LOGGERS = ("httpcore", "httpx", "httpcore.http11", "httpcore.connection", "asyncio", "urllib3")
|
||||
|
||||
|
||||
def main(argv: Optional[list[str]] = None) -> None:
|
||||
args = _parse_args(argv)
|
||||
settings = _settings_from_args(args)
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, settings.log_level, logging.WARNING),
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
)
|
||||
for name in NOISY_LOGGERS:
|
||||
logging.getLogger(name).setLevel(logging.WARNING)
|
||||
logger.info("Starting Devii")
|
||||
try:
|
||||
asyncio.run(run(settings, prompt=args.prompt))
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,5 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .controller import ClientController
|
||||
|
||||
__all__ = ["ClientController"]
|
||||
@@ -0,0 +1,58 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Awaitable, Callable, Optional
|
||||
|
||||
logger = logging.getLogger("devii.client")
|
||||
|
||||
RequestSink = Callable[[str, dict[str, Any]], Awaitable[Any]]
|
||||
|
||||
UNAVAILABLE = {
|
||||
"status": "unavailable",
|
||||
"message": (
|
||||
"No browser is attached to this session. Client-side actions only work in the web "
|
||||
"terminal, with a live connection open."
|
||||
),
|
||||
}
|
||||
EVAL_DISABLED = {
|
||||
"status": "disabled",
|
||||
"message": "JavaScript execution is disabled for Devii by the administrator.",
|
||||
}
|
||||
|
||||
|
||||
class ClientController:
|
||||
def __init__(self, allow_eval: bool = True) -> None:
|
||||
self._request: Optional[RequestSink] = None
|
||||
self._allow_eval = allow_eval
|
||||
|
||||
def bind(self, request: RequestSink) -> None:
|
||||
self._request = request
|
||||
logger.debug("Client controller bound to a session")
|
||||
|
||||
def unbind(self) -> None:
|
||||
self._request = None
|
||||
logger.debug("Client controller unbound")
|
||||
|
||||
@property
|
||||
def attached(self) -> bool:
|
||||
return self._request is not None
|
||||
|
||||
async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
|
||||
if name == "run_js" and not self._allow_eval:
|
||||
return json.dumps(EVAL_DISABLED, ensure_ascii=False)
|
||||
if self._request is None:
|
||||
return json.dumps(UNAVAILABLE, ensure_ascii=False)
|
||||
try:
|
||||
result = await self._request(name, arguments)
|
||||
except Exception as exc: # noqa: BLE001 - surfaced to the model as data
|
||||
logger.exception("Client action %s failed", name)
|
||||
return json.dumps({"status": "error", "action": name, "message": str(exc)}, ensure_ascii=False)
|
||||
|
||||
if isinstance(result, dict) and result.get("error"):
|
||||
return json.dumps(
|
||||
{"status": "error", "action": name, "message": str(result["error"])}, ensure_ascii=False
|
||||
)
|
||||
return json.dumps({"status": "success", "action": name, "result": result}, ensure_ascii=False)
|
||||
@@ -0,0 +1,149 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
from devplacepy.config import INTERNAL_GATEWAY_URL, INTERNAL_MODEL
|
||||
|
||||
|
||||
DEFAULT_AI_URL = INTERNAL_GATEWAY_URL
|
||||
DEFAULT_AI_MODEL = INTERNAL_MODEL
|
||||
DEFAULT_BASE_URL = "http://127.0.0.1:10500"
|
||||
DEFAULT_TIMEOUT_SECONDS = 45.0
|
||||
DEFAULT_MAX_RESPONSE_CHARS = 12000
|
||||
DEFAULT_MAX_TOOL_ITERATIONS = 40
|
||||
DEFAULT_DELEGATE_MAX_ITERATIONS = 25
|
||||
DEFAULT_CONTEXT_COMPACT_THRESHOLD = 220_000
|
||||
DEFAULT_CONTEXT_KEEP_TAIL = 12
|
||||
DEFAULT_RECALL_TOP_K = 3
|
||||
DEFAULT_FETCH_MAX_CHARS = 24_000
|
||||
DEFAULT_FETCH_TIMEOUT_SECONDS = 30.0
|
||||
DEFAULT_FETCH_MAX_BYTES = 5_000_000
|
||||
DEFAULT_RSEARCH_URL = "https://rsearch.app.molodetz.nl"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
ai_url: str
|
||||
ai_key: str
|
||||
ai_model: str
|
||||
base_url: str
|
||||
timeout_seconds: float
|
||||
max_response_chars: int
|
||||
max_tool_iterations: int
|
||||
log_level: str
|
||||
log_path: str
|
||||
tasks_db_path: str
|
||||
scheduler_tick_seconds: float
|
||||
lessons_db_path: str
|
||||
delegate_max_iterations: int
|
||||
context_compact_threshold: int
|
||||
context_keep_tail: int
|
||||
recall_top_k: int
|
||||
plan_required: bool
|
||||
verify_required: bool
|
||||
platform_api_key: str
|
||||
login_email: str
|
||||
login_password: str
|
||||
fetch_max_chars: int
|
||||
fetch_timeout_seconds: float
|
||||
fetch_max_bytes: int
|
||||
fetch_allow_private: bool
|
||||
allow_eval: bool
|
||||
rsearch_enabled: bool
|
||||
rsearch_url: str
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
return Settings(
|
||||
ai_url=os.environ.get("DEVII_AI_URL", DEFAULT_AI_URL),
|
||||
ai_key=os.environ.get("DEVII_AI_KEY", str(uuid.uuid4())),
|
||||
ai_model=os.environ.get("DEVII_AI_MODEL", DEFAULT_AI_MODEL),
|
||||
base_url=os.environ.get("DEVII_BASE_URL", DEFAULT_BASE_URL).rstrip("/"),
|
||||
timeout_seconds=float(os.environ.get("DEVII_TIMEOUT", DEFAULT_TIMEOUT_SECONDS)),
|
||||
max_response_chars=int(os.environ.get("DEVII_MAX_RESPONSE", DEFAULT_MAX_RESPONSE_CHARS)),
|
||||
max_tool_iterations=int(os.environ.get("DEVII_MAX_TOOL_ITERATIONS", DEFAULT_MAX_TOOL_ITERATIONS)),
|
||||
log_level=os.environ.get("DEVII_LOG_LEVEL", "WARNING").upper(),
|
||||
log_path=os.environ.get("DEVII_LOG_PATH", "devii.log"),
|
||||
tasks_db_path=os.environ.get("DEVII_TASKS_DB", "devii_tasks.db"),
|
||||
scheduler_tick_seconds=float(os.environ.get("DEVII_SCHEDULER_TICK", "1.0")),
|
||||
lessons_db_path=os.environ.get("DEVII_LESSONS_DB", "devii_lessons.db"),
|
||||
delegate_max_iterations=int(
|
||||
os.environ.get("DEVII_DELEGATE_MAX_ITERATIONS", DEFAULT_DELEGATE_MAX_ITERATIONS)
|
||||
),
|
||||
context_compact_threshold=int(
|
||||
os.environ.get("DEVII_CONTEXT_COMPACT_THRESHOLD", DEFAULT_CONTEXT_COMPACT_THRESHOLD)
|
||||
),
|
||||
context_keep_tail=int(os.environ.get("DEVII_CONTEXT_KEEP_TAIL", DEFAULT_CONTEXT_KEEP_TAIL)),
|
||||
recall_top_k=int(os.environ.get("DEVII_RECALL_TOP_K", DEFAULT_RECALL_TOP_K)),
|
||||
plan_required=os.environ.get("DEVII_PLAN_REQUIRED", "1").lower() not in ("0", "false", "no"),
|
||||
verify_required=os.environ.get("DEVII_VERIFY_REQUIRED", "1").lower() not in ("0", "false", "no"),
|
||||
platform_api_key=os.environ.get("DEVII_PLATFORM_API_KEY", ""),
|
||||
login_email=os.environ.get("DEVII_LOGIN_EMAIL", ""),
|
||||
login_password=os.environ.get("DEVII_LOGIN_PASSWORD", ""),
|
||||
fetch_max_chars=int(os.environ.get("DEVII_FETCH_MAX_CHARS", DEFAULT_FETCH_MAX_CHARS)),
|
||||
fetch_timeout_seconds=float(
|
||||
os.environ.get("DEVII_FETCH_TIMEOUT", DEFAULT_FETCH_TIMEOUT_SECONDS)
|
||||
),
|
||||
fetch_max_bytes=int(os.environ.get("DEVII_FETCH_MAX_BYTES", DEFAULT_FETCH_MAX_BYTES)),
|
||||
fetch_allow_private=os.environ.get("DEVII_FETCH_ALLOW_PRIVATE", "0").lower()
|
||||
in ("1", "true", "yes", "on"),
|
||||
allow_eval=os.environ.get("DEVII_ALLOW_EVAL", "1").lower() in ("1", "true", "yes", "on"),
|
||||
rsearch_enabled=os.environ.get("DEVII_RSEARCH_ENABLED", "1").lower() in ("1", "true", "yes", "on"),
|
||||
rsearch_url=os.environ.get("DEVII_RSEARCH_URL", DEFAULT_RSEARCH_URL).rstrip("/"),
|
||||
)
|
||||
|
||||
|
||||
FIELD_AI_URL = "devii_ai_url"
|
||||
FIELD_AI_MODEL = "devii_ai_model"
|
||||
FIELD_AI_KEY = "devii_ai_key"
|
||||
FIELD_PLAN_REQUIRED = "devii_plan_required"
|
||||
FIELD_VERIFY_REQUIRED = "devii_verify_required"
|
||||
FIELD_MAX_ITERATIONS = "devii_max_iterations"
|
||||
FIELD_USER_DAILY_USD = "devii_user_daily_usd"
|
||||
FIELD_GUEST_DAILY_USD = "devii_guest_daily_usd"
|
||||
FIELD_GUESTS_ENABLED = "devii_guests_enabled"
|
||||
FIELD_PRICE_CACHE_HIT = "devii_price_cache_hit"
|
||||
FIELD_PRICE_CACHE_MISS = "devii_price_cache_miss"
|
||||
FIELD_PRICE_OUTPUT = "devii_price_output"
|
||||
FIELD_ALLOW_EVAL = "devii_allow_eval"
|
||||
FIELD_RSEARCH_ENABLED = "devii_rsearch_enabled"
|
||||
FIELD_RSEARCH_URL = "devii_rsearch_url"
|
||||
|
||||
LESSONS_DB_PATH = os.environ.get("DEVII_LESSONS_DB", "devii_lessons.db")
|
||||
|
||||
|
||||
def build_settings(config: dict, base_url: str, api_key: str) -> Settings:
|
||||
return Settings(
|
||||
ai_url=config[FIELD_AI_URL] or DEFAULT_AI_URL,
|
||||
ai_key=config[FIELD_AI_KEY] or str(uuid.uuid4()),
|
||||
ai_model=config[FIELD_AI_MODEL] or DEFAULT_AI_MODEL,
|
||||
base_url=(base_url or DEFAULT_BASE_URL).rstrip("/"),
|
||||
timeout_seconds=DEFAULT_TIMEOUT_SECONDS,
|
||||
max_response_chars=DEFAULT_MAX_RESPONSE_CHARS,
|
||||
max_tool_iterations=int(config[FIELD_MAX_ITERATIONS]),
|
||||
log_level="INFO",
|
||||
log_path="devii.log",
|
||||
tasks_db_path="",
|
||||
scheduler_tick_seconds=1.0,
|
||||
lessons_db_path=LESSONS_DB_PATH,
|
||||
delegate_max_iterations=DEFAULT_DELEGATE_MAX_ITERATIONS,
|
||||
context_compact_threshold=DEFAULT_CONTEXT_COMPACT_THRESHOLD,
|
||||
context_keep_tail=DEFAULT_CONTEXT_KEEP_TAIL,
|
||||
recall_top_k=DEFAULT_RECALL_TOP_K,
|
||||
plan_required=bool(config[FIELD_PLAN_REQUIRED]),
|
||||
verify_required=bool(config[FIELD_VERIFY_REQUIRED]),
|
||||
platform_api_key=api_key,
|
||||
login_email="",
|
||||
login_password="",
|
||||
fetch_max_chars=DEFAULT_FETCH_MAX_CHARS,
|
||||
fetch_timeout_seconds=DEFAULT_FETCH_TIMEOUT_SECONDS,
|
||||
fetch_max_bytes=DEFAULT_FETCH_MAX_BYTES,
|
||||
fetch_allow_private=False,
|
||||
allow_eval=bool(config.get(FIELD_ALLOW_EVAL, True)),
|
||||
rsearch_enabled=bool(config.get(FIELD_RSEARCH_ENABLED, True)),
|
||||
rsearch_url=(config.get(FIELD_RSEARCH_URL) or DEFAULT_RSEARCH_URL).rstrip("/"),
|
||||
)
|
||||
@@ -0,0 +1,44 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
RESET = "\033[0m"
|
||||
BOLD = "\033[1m"
|
||||
CYAN = "\033[36m"
|
||||
GREEN = "\033[32m"
|
||||
YELLOW = "\033[33m"
|
||||
GREY = "\033[90m"
|
||||
|
||||
|
||||
def _supports_color() -> bool:
|
||||
return sys.stdout.isatty()
|
||||
|
||||
|
||||
def paint(text: str, color: str) -> str:
|
||||
if not _supports_color():
|
||||
return text
|
||||
return f"{color}{text}{RESET}"
|
||||
|
||||
|
||||
def banner(text: str) -> None:
|
||||
print(paint(text, BOLD + CYAN))
|
||||
|
||||
|
||||
def assistant(text: str) -> None:
|
||||
print(paint("agent> ", GREEN) + text)
|
||||
|
||||
|
||||
def notice(text: str) -> None:
|
||||
print(paint(text, GREY))
|
||||
|
||||
|
||||
def warn(text: str) -> None:
|
||||
print(paint(text, YELLOW))
|
||||
|
||||
|
||||
async def prompt(label: str) -> str:
|
||||
rendered = paint(label, BOLD + CYAN)
|
||||
return await asyncio.to_thread(input, rendered)
|
||||
@@ -0,0 +1,6 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .controller import CostController
|
||||
from .tracker import CostTracker, Pricing, record_usage, reset_tracker, set_tracker
|
||||
|
||||
__all__ = ["CostTracker", "Pricing", "CostController", "record_usage", "set_tracker", "reset_tracker"]
|
||||
@@ -0,0 +1,21 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from ..errors import ToolInputError
|
||||
from .tracker import get_tracker
|
||||
|
||||
|
||||
class CostController:
|
||||
async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
|
||||
if name != "cost_stats":
|
||||
raise ToolInputError(f"Unknown cost tool: {name}")
|
||||
tracker = get_tracker()
|
||||
if tracker is None:
|
||||
return json.dumps(
|
||||
{"status": "unavailable", "message": "No cost tracker is active for this session."}
|
||||
)
|
||||
return json.dumps({"status": "success", **tracker.stats()}, ensure_ascii=False)
|
||||
@@ -0,0 +1,153 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Optional
|
||||
|
||||
logger = logging.getLogger("devii.cost")
|
||||
|
||||
PRICE_MODEL = os.environ.get("DEVII_PRICE_MODEL", "deepseek-v4-flash")
|
||||
PRICE_CACHE_HIT_PER_M = float(os.environ.get("DEVII_PRICE_CACHE_HIT", "0.0028"))
|
||||
PRICE_CACHE_MISS_PER_M = float(os.environ.get("DEVII_PRICE_CACHE_MISS", "0.14"))
|
||||
PRICE_OUTPUT_PER_M = float(os.environ.get("DEVII_PRICE_OUTPUT", "0.28"))
|
||||
PER_MILLION = 1_000_000
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Pricing:
|
||||
model: str = PRICE_MODEL
|
||||
cache_hit_per_m: float = PRICE_CACHE_HIT_PER_M
|
||||
cache_miss_per_m: float = PRICE_CACHE_MISS_PER_M
|
||||
output_per_m: float = PRICE_OUTPUT_PER_M
|
||||
|
||||
|
||||
DEFAULT_PRICING = Pricing()
|
||||
|
||||
_active_tracker: contextvars.ContextVar = contextvars.ContextVar("devii_cost_tracker", default=None)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CostTracker:
|
||||
pricing: Pricing = field(default_factory=lambda: DEFAULT_PRICING)
|
||||
started_at: datetime = field(default_factory=_now)
|
||||
requests: int = 0
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
total_tokens: int = 0
|
||||
cache_hit_tokens: int = 0
|
||||
cache_miss_tokens: int = 0
|
||||
reasoning_tokens: int = 0
|
||||
|
||||
def record(self, usage: Optional[dict[str, Any]]) -> None:
|
||||
if not usage:
|
||||
return
|
||||
prompt = int(usage.get("prompt_tokens", 0) or 0)
|
||||
completion = int(usage.get("completion_tokens", 0) or 0)
|
||||
total = int(usage.get("total_tokens", prompt + completion) or 0)
|
||||
|
||||
hit = usage.get("prompt_cache_hit_tokens")
|
||||
if hit is None:
|
||||
details = usage.get("prompt_tokens_details") or {}
|
||||
hit = details.get("cached_tokens", 0)
|
||||
hit = int(hit or 0)
|
||||
|
||||
miss = usage.get("prompt_cache_miss_tokens")
|
||||
if miss is None:
|
||||
miss = max(prompt - hit, 0)
|
||||
miss = int(miss or 0)
|
||||
|
||||
completion_details = usage.get("completion_tokens_details") or {}
|
||||
reasoning = int(completion_details.get("reasoning_tokens", 0) or 0)
|
||||
|
||||
self.requests += 1
|
||||
self.prompt_tokens += prompt
|
||||
self.completion_tokens += completion
|
||||
self.total_tokens += total
|
||||
self.cache_hit_tokens += hit
|
||||
self.cache_miss_tokens += miss
|
||||
self.reasoning_tokens += reasoning
|
||||
logger.debug(
|
||||
"Cost record: +%d prompt (+%d hit/+%d miss), +%d completion", prompt, hit, miss, completion
|
||||
)
|
||||
|
||||
def cost_usd(self) -> dict[str, float]:
|
||||
cache_hit = self.cache_hit_tokens / PER_MILLION * self.pricing.cache_hit_per_m
|
||||
cache_miss = self.cache_miss_tokens / PER_MILLION * self.pricing.cache_miss_per_m
|
||||
output = self.completion_tokens / PER_MILLION * self.pricing.output_per_m
|
||||
total = cache_hit + cache_miss + output
|
||||
return {
|
||||
"cache_hit_input": round(cache_hit, 8),
|
||||
"cache_miss_input": round(cache_miss, 8),
|
||||
"output": round(output, 8),
|
||||
"total": round(total, 8),
|
||||
}
|
||||
|
||||
def stats(self) -> dict[str, Any]:
|
||||
elapsed = max((_now() - self.started_at).total_seconds(), 0.0)
|
||||
costs = self.cost_usd()
|
||||
requests = self.requests or 1
|
||||
cache_hit_rate = (self.cache_hit_tokens / self.prompt_tokens) if self.prompt_tokens else 0.0
|
||||
per_minute = (self.requests / elapsed * 60.0) if elapsed > 0 else 0.0
|
||||
return {
|
||||
"model": self.pricing.model,
|
||||
"pricing_usd_per_million": {
|
||||
"cache_hit_input": self.pricing.cache_hit_per_m,
|
||||
"cache_miss_input": self.pricing.cache_miss_per_m,
|
||||
"output": self.pricing.output_per_m,
|
||||
},
|
||||
"requests": self.requests,
|
||||
"tokens": {
|
||||
"prompt": self.prompt_tokens,
|
||||
"completion": self.completion_tokens,
|
||||
"total": self.total_tokens,
|
||||
"cache_hit": self.cache_hit_tokens,
|
||||
"cache_miss": self.cache_miss_tokens,
|
||||
"reasoning": self.reasoning_tokens,
|
||||
},
|
||||
"cache_hit_rate": round(cache_hit_rate, 4),
|
||||
"cost_usd": costs,
|
||||
"cost_usd_total": costs["total"],
|
||||
"cost_cents_total": round(costs["total"] * 100, 6),
|
||||
"averages": {
|
||||
"tokens_per_request": round(self.total_tokens / requests, 2),
|
||||
"prompt_tokens_per_request": round(self.prompt_tokens / requests, 2),
|
||||
"completion_tokens_per_request": round(self.completion_tokens / requests, 2),
|
||||
"cost_usd_per_request": round(costs["total"] / requests, 8),
|
||||
},
|
||||
"session": {
|
||||
"started_at": self.started_at.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"elapsed_seconds": round(elapsed, 1),
|
||||
"requests_per_minute": round(per_minute, 2),
|
||||
},
|
||||
"note": (
|
||||
"Priced as DeepSeek V4 Flash (the cheapest official DeepSeek model). "
|
||||
"Token counts are reported by the model endpoint."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def get_tracker() -> Optional[CostTracker]:
|
||||
return _active_tracker.get()
|
||||
|
||||
|
||||
def set_tracker(tracker: CostTracker) -> contextvars.Token:
|
||||
return _active_tracker.set(tracker)
|
||||
|
||||
|
||||
def reset_tracker(token: contextvars.Token) -> None:
|
||||
_active_tracker.reset(token)
|
||||
|
||||
|
||||
def record_usage(usage: Optional[dict[str, Any]]) -> None:
|
||||
tracker = _active_tracker.get()
|
||||
if tracker is not None:
|
||||
tracker.record(usage)
|
||||
@@ -0,0 +1,5 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .controller import DocsController
|
||||
|
||||
__all__ = ["DocsController"]
|
||||
@@ -0,0 +1,109 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from ..agentic.lessons import tokenize
|
||||
from ..config import Settings
|
||||
from ..errors import NetworkError, ToolInputError
|
||||
from ..fetch.controller import STEALTH_HEADERS
|
||||
from ..text import truncate
|
||||
|
||||
logger = logging.getLogger("devii.docs")
|
||||
|
||||
DOCS_PATH = "/docs/download.md"
|
||||
HEADING = re.compile(r"^(#{1,3})\s+(.*)$")
|
||||
DEFAULT_MAX_RESULTS = 5
|
||||
SECTION_CHARS = 1800
|
||||
HEADING_WEIGHT = 3
|
||||
|
||||
|
||||
class DocsController:
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self._settings = settings
|
||||
self._sections: list[tuple[str, str]] | None = None
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
|
||||
if name != "search_docs":
|
||||
raise ToolInputError(f"Unknown docs tool: {name}")
|
||||
return await self._search(arguments)
|
||||
|
||||
async def _load(self) -> list[tuple[str, str]]:
|
||||
async with self._lock:
|
||||
if self._sections is not None:
|
||||
return self._sections
|
||||
url = self._settings.base_url + DOCS_PATH
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
headers=STEALTH_HEADERS,
|
||||
follow_redirects=True,
|
||||
timeout=self._settings.fetch_timeout_seconds,
|
||||
) as client:
|
||||
response = await client.get(url)
|
||||
response.raise_for_status()
|
||||
markdown = response.text
|
||||
except httpx.HTTPError as exc:
|
||||
raise NetworkError(f"Could not load documentation: {exc}", url=url) from exc
|
||||
self._sections = self._split(markdown)
|
||||
logger.info("Loaded %d documentation sections", len(self._sections))
|
||||
return self._sections
|
||||
|
||||
@staticmethod
|
||||
def _split(markdown: str) -> list[tuple[str, str]]:
|
||||
sections: list[tuple[str, str]] = []
|
||||
heading = "Overview"
|
||||
body: list[str] = []
|
||||
for line in markdown.splitlines():
|
||||
match = HEADING.match(line)
|
||||
if match:
|
||||
if body:
|
||||
sections.append((heading, "\n".join(body).strip()))
|
||||
body = []
|
||||
heading = match.group(2).strip()
|
||||
else:
|
||||
body.append(line)
|
||||
if body:
|
||||
sections.append((heading, "\n".join(body).strip()))
|
||||
return [(title, text) for title, text in sections if text]
|
||||
|
||||
async def _search(self, arguments: dict[str, Any]) -> str:
|
||||
query = str(arguments.get("query", "")).strip()
|
||||
if not query:
|
||||
raise ToolInputError("search_docs requires a query.")
|
||||
max_results = int(arguments.get("max_results", DEFAULT_MAX_RESULTS) or DEFAULT_MAX_RESULTS)
|
||||
max_results = max(1, min(max_results, 10))
|
||||
|
||||
sections = await self._load()
|
||||
terms = tokenize(query)
|
||||
scored: list[tuple[float, str, str]] = []
|
||||
for heading, text in sections:
|
||||
heading_tokens = set(tokenize(heading))
|
||||
body_tokens = tokenize(text)
|
||||
body_counts: dict[str, int] = {}
|
||||
for token in body_tokens:
|
||||
body_counts[token] = body_counts.get(token, 0) + 1
|
||||
score = 0.0
|
||||
for term in terms:
|
||||
score += body_counts.get(term, 0)
|
||||
if term in heading_tokens:
|
||||
score += HEADING_WEIGHT
|
||||
if score > 0:
|
||||
scored.append((score, heading, text))
|
||||
scored.sort(key=lambda item: item[0], reverse=True)
|
||||
|
||||
results = [
|
||||
{"title": heading, "score": round(score, 2), "content": truncate(text, SECTION_CHARS)}
|
||||
for score, heading, text in scored[:max_results]
|
||||
]
|
||||
return json.dumps(
|
||||
{"status": "success", "query": query, "count": len(results), "results": results},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
@@ -0,0 +1,51 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
|
||||
class DeviiError(Exception):
|
||||
code: str = "error"
|
||||
|
||||
def __init__(self, message: str, **details: Any) -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.details = details
|
||||
|
||||
def as_payload(self) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {"error": self.code, "message": self.message}
|
||||
payload.update(self.details)
|
||||
return payload
|
||||
|
||||
|
||||
class AuthRequiredError(DeviiError):
|
||||
code = "auth_required"
|
||||
|
||||
|
||||
class NetworkError(DeviiError):
|
||||
code = "network_error"
|
||||
|
||||
|
||||
class UpstreamError(DeviiError):
|
||||
code = "upstream_error"
|
||||
|
||||
|
||||
class ToolInputError(DeviiError):
|
||||
code = "tool_input_error"
|
||||
|
||||
|
||||
class LLMError(DeviiError):
|
||||
code = "llm_error"
|
||||
|
||||
|
||||
def error_result(error: DeviiError) -> str:
|
||||
return json.dumps(error.as_payload(), ensure_ascii=False)
|
||||
|
||||
|
||||
def unexpected_result(exc: Exception) -> str:
|
||||
return json.dumps(
|
||||
{"error": "unexpected", "message": str(exc), "type": type(exc).__name__},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
@@ -0,0 +1,5 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .controller import FetchController
|
||||
|
||||
__all__ = ["FetchController"]
|
||||
@@ -0,0 +1,181 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from ..config import Settings
|
||||
from ..errors import NetworkError, ToolInputError, UpstreamError
|
||||
from ..text import html_to_text
|
||||
|
||||
logger = logging.getLogger("devii.fetch")
|
||||
|
||||
CHROME_VERSION = "131"
|
||||
CHROME_VERSION_FULL = "131.0.0.0"
|
||||
STEALTH_HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
f"(KHTML, like Gecko) Chrome/{CHROME_VERSION_FULL} Safari/537.36"
|
||||
),
|
||||
"Accept": (
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,"
|
||||
"image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
|
||||
),
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"sec-ch-ua": (
|
||||
f'"Google Chrome";v="{CHROME_VERSION}", "Chromium";v="{CHROME_VERSION}", '
|
||||
'"Not_A Brand";v="24"'
|
||||
),
|
||||
"sec-ch-ua-mobile": "?0",
|
||||
"sec-ch-ua-platform": '"Linux"',
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Cache-Control": "max-age=0",
|
||||
"DNT": "1",
|
||||
}
|
||||
TITLE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
|
||||
MIN_FETCH_CHARS = 1000
|
||||
|
||||
NAT64_PREFIXES = (
|
||||
ipaddress.ip_network("64:ff9b::/96"),
|
||||
ipaddress.ip_network("64:ff9b:1::/48"),
|
||||
)
|
||||
|
||||
|
||||
def _effective_address(address: Any) -> Any:
|
||||
"""Resolve NAT64 / IPv4-mapped IPv6 to the embedded IPv4 so DNS64 networks
|
||||
don't make public sites look like reserved addresses."""
|
||||
if isinstance(address, ipaddress.IPv6Address):
|
||||
if address.ipv4_mapped is not None:
|
||||
return address.ipv4_mapped
|
||||
for prefix in NAT64_PREFIXES:
|
||||
if address in prefix:
|
||||
return ipaddress.IPv4Address(int(address) & 0xFFFFFFFF)
|
||||
return address
|
||||
|
||||
|
||||
class FetchController:
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self._settings = settings
|
||||
|
||||
async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
|
||||
if name != "fetch_url":
|
||||
raise ToolInputError(f"Unknown fetch tool: {name}")
|
||||
return await self._fetch_url(arguments)
|
||||
|
||||
def _optional_cap(self, requested: Any, content: str) -> str:
|
||||
if requested is None:
|
||||
return content
|
||||
try:
|
||||
cap = max(MIN_FETCH_CHARS, int(requested))
|
||||
except (TypeError, ValueError):
|
||||
return content
|
||||
return content[:cap]
|
||||
|
||||
async def _fetch_url(self, arguments: dict[str, Any]) -> str:
|
||||
url = str(arguments.get("url", "")).strip()
|
||||
if not url:
|
||||
raise ToolInputError("fetch_url requires a url.")
|
||||
if "://" not in url:
|
||||
url = "https://" + url
|
||||
await self._guard(url)
|
||||
|
||||
body, final_url, content_type, status = await self._download(url)
|
||||
|
||||
if "html" in content_type or "xml" in content_type or content_type.startswith("text/"):
|
||||
title_match = TITLE.search(body)
|
||||
title = html_to_text(title_match.group(1)) if title_match else ""
|
||||
content = html_to_text(body)
|
||||
elif "json" in content_type:
|
||||
title = ""
|
||||
content = body
|
||||
else:
|
||||
title = ""
|
||||
content = f"(non-text response: {content_type})"
|
||||
|
||||
return json.dumps(
|
||||
{
|
||||
"status": "success",
|
||||
"url": final_url,
|
||||
"http_status": status,
|
||||
"content_type": content_type,
|
||||
"title": title,
|
||||
"content": self._optional_cap(arguments.get("max_chars"), content),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
async def _guard(self, url: str) -> None:
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
raise ToolInputError("Only http and https URLs can be fetched.")
|
||||
host = parsed.hostname
|
||||
if not host:
|
||||
raise ToolInputError("URL has no host.")
|
||||
if self._settings.fetch_allow_private:
|
||||
return
|
||||
try:
|
||||
infos = await asyncio.to_thread(socket.getaddrinfo, host, None)
|
||||
except socket.gaierror as exc:
|
||||
raise NetworkError(f"Could not resolve host: {host}", host=host) from exc
|
||||
for info in infos:
|
||||
address = _effective_address(ipaddress.ip_address(info[4][0]))
|
||||
if (
|
||||
address.is_private
|
||||
or address.is_loopback
|
||||
or address.is_link_local
|
||||
or address.is_reserved
|
||||
or address.is_multicast
|
||||
or address.is_unspecified
|
||||
):
|
||||
raise ToolInputError(
|
||||
f"Refusing to fetch a private or local address ({address}). "
|
||||
"Set DEVII_FETCH_ALLOW_PRIVATE=1 to override."
|
||||
)
|
||||
|
||||
async def _download(self, url: str) -> tuple[str, str, str, int]:
|
||||
limit = self._settings.fetch_max_bytes
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
headers=STEALTH_HEADERS,
|
||||
follow_redirects=True,
|
||||
timeout=self._settings.fetch_timeout_seconds,
|
||||
) as client:
|
||||
async with client.stream("GET", url) as response:
|
||||
if response.status_code >= 500:
|
||||
raise UpstreamError(
|
||||
f"Server returned {response.status_code}.",
|
||||
status=response.status_code,
|
||||
url=url,
|
||||
)
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
async for chunk in response.aiter_bytes():
|
||||
chunks.append(chunk)
|
||||
total += len(chunk)
|
||||
if total >= limit:
|
||||
break
|
||||
raw = b"".join(chunks)[:limit]
|
||||
encoding = response.encoding or "utf-8"
|
||||
try:
|
||||
body = raw.decode(encoding, errors="replace")
|
||||
except LookupError:
|
||||
body = raw.decode("utf-8", errors="replace")
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
return body, str(response.url), content_type, response.status_code
|
||||
except httpx.TimeoutException as exc:
|
||||
raise NetworkError(f"Request timed out fetching {url}", url=url) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise NetworkError(f"Could not fetch {url}: {exc}", url=url) from exc
|
||||
@@ -0,0 +1,143 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from .errors import AuthRequiredError, NetworkError, ToolInputError, UpstreamError
|
||||
|
||||
logger = logging.getLogger("devii.http")
|
||||
|
||||
LOGIN_FAILURE_MARKERS = ("Invalid email or password", "name=\"password\"")
|
||||
LOGIN_PATH = "/auth/login"
|
||||
|
||||
|
||||
class PlatformClient:
|
||||
def __init__(self, base_url: str, timeout_seconds: float, api_key: str = "") -> None:
|
||||
headers = {"User-Agent": "devii/0.1", "Accept": "application/json"}
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
headers["X-API-Key"] = api_key
|
||||
self._client = httpx.AsyncClient(
|
||||
base_url=base_url,
|
||||
timeout=timeout_seconds,
|
||||
follow_redirects=True,
|
||||
headers=headers,
|
||||
)
|
||||
self.authenticated: bool = bool(api_key)
|
||||
self.username: str | None = "api-key" if api_key else None
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._client.aclose()
|
||||
|
||||
def session_cookie(self) -> str:
|
||||
return self._client.cookies.get("session") or ""
|
||||
|
||||
async def login(self, email: str, password: str, remember_me: bool = True) -> dict[str, Any]:
|
||||
form = {
|
||||
"email": email,
|
||||
"password": password,
|
||||
"remember_me": "on" if remember_me else "",
|
||||
"next": "/feed",
|
||||
}
|
||||
response = await self._send("POST", LOGIN_PATH, data=form)
|
||||
has_session = len(self._client.cookies) > 0
|
||||
failed = response.status_code >= 400 or not has_session
|
||||
try:
|
||||
data = response.json()
|
||||
if isinstance(data, dict) and (
|
||||
data.get("ok") is False or data.get("error") or data.get("detail")
|
||||
):
|
||||
failed = True
|
||||
except ValueError:
|
||||
if any(marker in response.text for marker in LOGIN_FAILURE_MARKERS):
|
||||
failed = True
|
||||
|
||||
if failed:
|
||||
self.authenticated = False
|
||||
self.username = None
|
||||
logger.info("Login failed for %s", email)
|
||||
raise AuthRequiredError(
|
||||
"Login failed: invalid email or password.",
|
||||
email=email,
|
||||
)
|
||||
|
||||
self.authenticated = True
|
||||
self.username = email
|
||||
logger.info("Login succeeded for %s", email)
|
||||
return {"status": "logged_in", "user": email}
|
||||
|
||||
async def logout(self) -> dict[str, Any]:
|
||||
await self._send("GET", "/auth/logout")
|
||||
self._client.cookies.clear()
|
||||
self.authenticated = False
|
||||
previous = self.username
|
||||
self.username = None
|
||||
logger.info("Logged out %s", previous)
|
||||
return {"status": "logged_out", "previous_user": previous}
|
||||
|
||||
async def call(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
params: dict[str, Any] | None = None,
|
||||
data: dict[str, Any] | None = None,
|
||||
file_field: tuple[str, str] | None = None,
|
||||
headers: dict[str, str] | None = None,
|
||||
) -> httpx.Response:
|
||||
files = None
|
||||
if file_field is not None:
|
||||
field_name, raw_path = file_field
|
||||
files = {field_name: self._open_upload(raw_path)}
|
||||
|
||||
response = await self._send(
|
||||
method, path, params=params, data=data, files=files, headers=headers
|
||||
)
|
||||
|
||||
if path != LOGIN_PATH and LOGIN_PATH in str(response.url):
|
||||
self.authenticated = False
|
||||
self.username = None
|
||||
logger.info("Session expired during %s %s", method, path)
|
||||
raise AuthRequiredError(
|
||||
"Session expired or not authenticated. Use the login tool again.",
|
||||
path=path,
|
||||
)
|
||||
|
||||
if response.status_code == 401:
|
||||
self.authenticated = False
|
||||
logger.info("Unauthorized (401) during %s %s", method, path)
|
||||
raise AuthRequiredError(
|
||||
"Credentials are missing or invalid (401). Log in again or check the API key.",
|
||||
path=path,
|
||||
)
|
||||
|
||||
if response.status_code >= 500:
|
||||
raise UpstreamError(
|
||||
f"Server returned {response.status_code}.",
|
||||
status=response.status_code,
|
||||
path=path,
|
||||
)
|
||||
return response
|
||||
|
||||
def _open_upload(self, raw_path: str) -> tuple[str, bytes]:
|
||||
candidate = Path(raw_path).expanduser()
|
||||
if not candidate.is_file():
|
||||
raise ToolInputError(f"File not found: {raw_path}", path=raw_path)
|
||||
return candidate.name, candidate.read_bytes()
|
||||
|
||||
async def _send(self, method: str, path: str, **kwargs: Any) -> httpx.Response:
|
||||
try:
|
||||
logger.debug("Request %s %s kwargs=%s", method, path, list(kwargs))
|
||||
response = await self._client.request(method, path, **kwargs)
|
||||
logger.debug("Response %s %s -> %s", method, path, response.status_code)
|
||||
return response
|
||||
except httpx.TimeoutException as exc:
|
||||
raise NetworkError(f"Request timed out: {method} {path}", path=path) from exc
|
||||
except httpx.ConnectError as exc:
|
||||
raise NetworkError(f"Could not connect: {method} {path}", path=path) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise NetworkError(f"HTTP error: {exc}", path=path) from exc
|
||||
@@ -0,0 +1,84 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Callable
|
||||
|
||||
from devplacepy.database import db
|
||||
|
||||
from .agentic import LessonStore
|
||||
from .config import Settings
|
||||
from .cost.tracker import Pricing
|
||||
from .llm import LLMClient
|
||||
from .session import DeviiSession
|
||||
from .store import ConversationStore, TurnAudit, UsageLedger
|
||||
from .tasks.store import TaskStore, memory_db
|
||||
|
||||
logger = logging.getLogger("devii.hub")
|
||||
|
||||
SettingsBuilder = Callable[[str, str], "tuple[Settings, Pricing]"]
|
||||
|
||||
|
||||
class DeviiHub:
|
||||
def __init__(self, settings_builder: SettingsBuilder) -> None:
|
||||
self._build = settings_builder
|
||||
self._stores: dict[str, Any] = {
|
||||
"conversations": ConversationStore(),
|
||||
"ledger": UsageLedger(),
|
||||
"turns": TurnAudit(),
|
||||
}
|
||||
self._sessions: dict[tuple[str, str], DeviiSession] = {}
|
||||
|
||||
@property
|
||||
def ledger(self) -> UsageLedger:
|
||||
return self._stores["ledger"]
|
||||
|
||||
def get_or_create(
|
||||
self, owner_kind: str, owner_id: str, username: str, api_key: str, base_url: str
|
||||
) -> DeviiSession:
|
||||
key = (owner_kind, owner_id)
|
||||
session = self._sessions.get(key)
|
||||
if session is not None:
|
||||
return session
|
||||
settings, pricing = self._build(api_key, base_url)
|
||||
llm = LLMClient(settings)
|
||||
# Persistent, owner-isolated stores for signed-in users; ephemeral in-memory for guests.
|
||||
owned_db = db if owner_kind == "user" else memory_db()
|
||||
task_store = TaskStore(owned_db, owner_kind, owner_id)
|
||||
lessons = LessonStore(owned_db, owner_kind, owner_id)
|
||||
session = DeviiSession(
|
||||
owner_kind, owner_id, username, settings, llm, lessons, pricing, task_store, self._stores
|
||||
)
|
||||
if owner_kind == "user":
|
||||
saved = self._stores["conversations"].load(owner_kind, owner_id)
|
||||
if saved:
|
||||
session.restore_history(saved)
|
||||
self._sessions[key] = session
|
||||
logger.info("Created session %s/%s (total %d)", owner_kind, owner_id, len(self._sessions))
|
||||
return session
|
||||
|
||||
def find(self, owner_kind: str, owner_id: str) -> DeviiSession | None:
|
||||
return self._sessions.get((owner_kind, owner_id))
|
||||
|
||||
def active_sessions(self) -> int:
|
||||
return len(self._sessions)
|
||||
|
||||
def connection_count(self) -> int:
|
||||
return sum(s.connection_count for s in self._sessions.values())
|
||||
|
||||
async def gc_idle(self) -> int:
|
||||
removed = 0
|
||||
for key, session in list(self._sessions.items()):
|
||||
if session.connection_count == 0:
|
||||
await session.aclose()
|
||||
del self._sessions[key]
|
||||
removed += 1
|
||||
if removed:
|
||||
logger.info("GC removed %d idle session(s)", removed)
|
||||
return removed
|
||||
|
||||
async def aclose(self) -> None:
|
||||
for session in list(self._sessions.values()):
|
||||
await session.aclose()
|
||||
self._sessions.clear()
|
||||
@@ -0,0 +1,105 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from .config import Settings
|
||||
from .cost import record_usage
|
||||
from .errors import LLMError
|
||||
|
||||
logger = logging.getLogger("devii.llm")
|
||||
|
||||
|
||||
class LLMClient:
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self._settings = settings
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=settings.timeout_seconds,
|
||||
headers={
|
||||
"Authorization": f"Bearer {settings.ai_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._client.aclose()
|
||||
|
||||
async def complete(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
tools: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
payload = {
|
||||
"model": self._settings.ai_model,
|
||||
"messages": messages,
|
||||
"tools": tools,
|
||||
"tool_choice": "auto",
|
||||
}
|
||||
try:
|
||||
logger.debug("LLM request with %d messages", len(messages))
|
||||
response = await self._client.post(self._settings.ai_url, json=payload)
|
||||
except httpx.HTTPError as exc:
|
||||
raise LLMError(f"Could not reach the model endpoint: {exc}") from exc
|
||||
|
||||
if response.status_code >= 400:
|
||||
raise LLMError(
|
||||
f"Model endpoint returned {response.status_code}: {self._reason(response)}",
|
||||
status=response.status_code,
|
||||
body=response.text[:500],
|
||||
)
|
||||
|
||||
try:
|
||||
data = response.json()
|
||||
except ValueError as exc:
|
||||
raise LLMError("Model endpoint returned invalid JSON.") from exc
|
||||
|
||||
choices = data.get("choices")
|
||||
if not choices:
|
||||
raise LLMError("Model response contained no choices.", body=str(data)[:500])
|
||||
|
||||
message = choices[0].get("message")
|
||||
if message is None:
|
||||
raise LLMError("Model response contained no message.", body=str(data)[:500])
|
||||
|
||||
record_usage(data.get("usage"))
|
||||
logger.debug("LLM response received (tool_calls=%s)", bool(message.get("tool_calls")))
|
||||
return message
|
||||
|
||||
async def summarize(self, text: str) -> str:
|
||||
payload = {
|
||||
"model": self._settings.ai_model,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a precise technical summarizer."},
|
||||
{"role": "user", "content": text},
|
||||
],
|
||||
"temperature": 0.0,
|
||||
}
|
||||
try:
|
||||
response = await self._client.post(self._settings.ai_url, json=payload)
|
||||
except httpx.HTTPError as exc:
|
||||
raise LLMError(f"Could not reach the model endpoint: {exc}") from exc
|
||||
if response.status_code >= 400:
|
||||
raise LLMError(f"Model endpoint returned {response.status_code}: {self._reason(response)}")
|
||||
try:
|
||||
data = response.json()
|
||||
content = data["choices"][0]["message"]["content"] or ""
|
||||
except (ValueError, KeyError, IndexError) as exc:
|
||||
raise LLMError("Model summarization returned an unexpected response.") from exc
|
||||
record_usage(data.get("usage"))
|
||||
return content
|
||||
|
||||
@staticmethod
|
||||
def _reason(response: httpx.Response) -> str:
|
||||
try:
|
||||
error = response.json().get("error")
|
||||
except ValueError:
|
||||
return response.text[:200] or response.reason_phrase
|
||||
if isinstance(error, dict):
|
||||
return str(error.get("message", error))
|
||||
if error:
|
||||
return str(error)
|
||||
return response.reason_phrase
|
||||
@@ -0,0 +1,28 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .actions.avatar_actions import AVATAR_ACTIONS
|
||||
from .actions.catalog import ACTIONS
|
||||
from .actions.chunk_actions import CHUNK_ACTIONS
|
||||
from .actions.client_actions import CLIENT_ACTIONS
|
||||
from .actions.cost_actions import COST_ACTIONS
|
||||
from .actions.docs_actions import DOCS_ACTIONS
|
||||
from .actions.fetch_actions import FETCH_ACTIONS
|
||||
from .actions.rsearch_actions import RSEARCH_ACTIONS
|
||||
from .actions.spec import Catalog
|
||||
from .agentic.actions import AGENTIC_ACTIONS
|
||||
from .tasks.actions import TASK_ACTIONS
|
||||
|
||||
CATALOG = Catalog(
|
||||
actions=ACTIONS
|
||||
+ TASK_ACTIONS
|
||||
+ AGENTIC_ACTIONS
|
||||
+ AVATAR_ACTIONS
|
||||
+ CLIENT_ACTIONS
|
||||
+ FETCH_ACTIONS
|
||||
+ DOCS_ACTIONS
|
||||
+ COST_ACTIONS
|
||||
+ CHUNK_ACTIONS
|
||||
+ RSEARCH_ACTIONS
|
||||
)
|
||||
@@ -0,0 +1,5 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .controller import RsearchController
|
||||
|
||||
__all__ = ["RsearchController"]
|
||||
@@ -0,0 +1,158 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from ..config import Settings
|
||||
from ..errors import NetworkError, ToolInputError, UpstreamError
|
||||
|
||||
logger = logging.getLogger("devii.rsearch")
|
||||
|
||||
USER_AGENT = "devii/0.1"
|
||||
|
||||
|
||||
def _flag(value: Any) -> str:
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
return "true" if str(value).strip().lower() in ("1", "true", "yes", "on") else "false"
|
||||
|
||||
|
||||
class RsearchController:
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self._settings = settings
|
||||
|
||||
async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
|
||||
if not self._settings.rsearch_enabled:
|
||||
raise ToolInputError(
|
||||
"Web search tools are disabled by the administrator; use platform tools instead."
|
||||
)
|
||||
if name == "rsearch":
|
||||
return await self._search(arguments)
|
||||
if name == "rsearch_answer":
|
||||
return await self._answer(arguments)
|
||||
if name == "rsearch_chat":
|
||||
return await self._chat(arguments)
|
||||
if name == "rsearch_describe_image":
|
||||
return await self._describe(arguments)
|
||||
raise ToolInputError(f"Unknown rsearch tool: {name}")
|
||||
|
||||
async def _request(self, path: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
base_url=self._settings.rsearch_url,
|
||||
headers=headers,
|
||||
follow_redirects=True,
|
||||
timeout=self._settings.fetch_timeout_seconds,
|
||||
) as client:
|
||||
response = await client.get(path, params=params)
|
||||
except httpx.TimeoutException as exc:
|
||||
raise NetworkError(f"rsearch timed out calling {path}", path=path) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise NetworkError(f"rsearch request failed: {exc}", path=path) from exc
|
||||
if response.status_code >= 400:
|
||||
raise UpstreamError(
|
||||
f"rsearch returned {response.status_code} for {path}",
|
||||
status=response.status_code,
|
||||
path=path,
|
||||
)
|
||||
try:
|
||||
return response.json()
|
||||
except ValueError as exc:
|
||||
raise UpstreamError("rsearch returned a non-JSON response", path=path) from exc
|
||||
|
||||
def _require_text(self, arguments: dict[str, Any], key: str) -> str:
|
||||
value = str(arguments.get(key, "")).strip()
|
||||
if not value:
|
||||
raise ToolInputError(f"rsearch requires a non-empty '{key}'.")
|
||||
return value
|
||||
|
||||
def _count(self, arguments: dict[str, Any]) -> int:
|
||||
try:
|
||||
count = int(arguments.get("count", 10))
|
||||
except (TypeError, ValueError):
|
||||
count = 10
|
||||
return max(1, min(count, 100))
|
||||
|
||||
async def _search(self, arguments: dict[str, Any]) -> str:
|
||||
query = self._require_text(arguments, "query")
|
||||
result_type = str(arguments.get("type", "web")).strip().lower()
|
||||
if result_type not in ("web", "images"):
|
||||
result_type = "web"
|
||||
params: dict[str, Any] = {
|
||||
"query": query,
|
||||
"count": self._count(arguments),
|
||||
"content": _flag(arguments.get("content")),
|
||||
"deep": _flag(arguments.get("deep")),
|
||||
}
|
||||
if result_type == "images":
|
||||
params["type"] = "images"
|
||||
data = await self._request("/search", params)
|
||||
results = data.get("results") or []
|
||||
compact = [
|
||||
{
|
||||
"title": item.get("title"),
|
||||
"url": item.get("url"),
|
||||
"description": item.get("description"),
|
||||
"source": item.get("source"),
|
||||
"content": item.get("content"),
|
||||
}
|
||||
for item in results
|
||||
]
|
||||
return json.dumps({
|
||||
"status": "success",
|
||||
"query": data.get("query", query),
|
||||
"type": result_type,
|
||||
"source": data.get("source"),
|
||||
"count": len(compact),
|
||||
"results": compact,
|
||||
}, ensure_ascii=False)
|
||||
|
||||
async def _answer(self, arguments: dict[str, Any]) -> str:
|
||||
query = self._require_text(arguments, "query")
|
||||
params = {
|
||||
"query": query,
|
||||
"ai": "true",
|
||||
"content": _flag(arguments.get("content")),
|
||||
"count": self._count(arguments),
|
||||
}
|
||||
data = await self._request("/search", params)
|
||||
results = data.get("results") or []
|
||||
return json.dumps({
|
||||
"status": "success",
|
||||
"query": query,
|
||||
"answer": data.get("ai_response"),
|
||||
"answer_error": data.get("ai_error"),
|
||||
"sources": [
|
||||
{"title": item.get("title"), "url": item.get("url")}
|
||||
for item in results
|
||||
],
|
||||
}, ensure_ascii=False)
|
||||
|
||||
async def _chat(self, arguments: dict[str, Any]) -> str:
|
||||
prompt = self._require_text(arguments, "prompt")
|
||||
params: dict[str, Any] = {"prompt": prompt, "json": _flag(arguments.get("json"))}
|
||||
system = str(arguments.get("system", "")).strip()
|
||||
if system:
|
||||
params["system"] = system
|
||||
data = await self._request("/chat", params)
|
||||
return json.dumps({
|
||||
"status": "success",
|
||||
"response": data.get("response"),
|
||||
"json_mode": data.get("json_mode", False),
|
||||
}, ensure_ascii=False)
|
||||
|
||||
async def _describe(self, arguments: dict[str, Any]) -> str:
|
||||
url = self._require_text(arguments, "url")
|
||||
data = await self._request("/describe", {"url": url})
|
||||
return json.dumps({
|
||||
"status": "success",
|
||||
"url": url,
|
||||
"mime_type": data.get("mime_type"),
|
||||
"description": data.get("description"),
|
||||
}, ensure_ascii=False)
|
||||
@@ -0,0 +1,154 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from devplacepy.services.base import BaseService, ConfigField
|
||||
|
||||
from . import config
|
||||
from .config import (
|
||||
DEFAULT_AI_MODEL,
|
||||
DEFAULT_AI_URL,
|
||||
build_settings,
|
||||
)
|
||||
from .cost.tracker import Pricing
|
||||
|
||||
logger = logging.getLogger("devii.service")
|
||||
|
||||
INSTANCE_ORIGIN_DEFAULT = "http://127.0.0.1:10500"
|
||||
|
||||
|
||||
class DeviiService(BaseService):
|
||||
default_enabled = False
|
||||
min_interval = 10
|
||||
title = "Devii"
|
||||
description = (
|
||||
"An in-platform agentic assistant exposed as a WebSocket terminal at /devii/ws. "
|
||||
"Each signed-in user's Devii operates their own DevPlace account; guests get a "
|
||||
"sandbox session. Spend is capped per user and per guest over a rolling 24 hours, "
|
||||
"with every turn recorded to a usage ledger and an audit log."
|
||||
)
|
||||
config_fields = [
|
||||
ConfigField(config.FIELD_AI_URL, "AI URL", type="url", default=DEFAULT_AI_URL,
|
||||
help="OpenAI-compatible chat-completions endpoint Devii reasons with.", group="AI"),
|
||||
ConfigField(config.FIELD_AI_MODEL, "AI model", type="str", default=DEFAULT_AI_MODEL,
|
||||
help="Model name sent to the AI endpoint.", group="AI"),
|
||||
ConfigField(config.FIELD_AI_KEY, "AI API key", type="password", default="", secret=True,
|
||||
help="Defaults to the DEVII_AI_KEY env var, then the gateway's internal key.", group="AI"),
|
||||
ConfigField("devii_base_url", "Platform base URL", type="url", default="",
|
||||
help="Origin Devii drives via each user's API key. Blank uses this instance.",
|
||||
group="AI"),
|
||||
ConfigField(config.FIELD_PLAN_REQUIRED, "Require plan step", type="bool", default=True,
|
||||
help="Force a plan() call before any tool use.", group="Agent"),
|
||||
ConfigField(config.FIELD_VERIFY_REQUIRED, "Require verify step", type="bool", default=True,
|
||||
help="Force a verify() call after mutating actions.", group="Agent"),
|
||||
ConfigField(config.FIELD_MAX_ITERATIONS, "Max tool iterations", type="int", default=40,
|
||||
minimum=1, maximum=200, help="Upper bound on tool-loop iterations per turn.",
|
||||
group="Agent"),
|
||||
ConfigField(config.FIELD_ALLOW_EVAL, "Allow JavaScript execution", type="bool", default=True,
|
||||
help="Let Devii run JavaScript in the user's own browser (run_js tool). "
|
||||
"Other client tools (navigate, reload, highlight, toast, context) are unaffected.",
|
||||
group="Agent"),
|
||||
ConfigField(config.FIELD_USER_DAILY_USD, "Max USD per user / 24h", type="float", default=1.0,
|
||||
minimum=0, help="Rolling 24-hour spend cap for a signed-in user.", group="Limits"),
|
||||
ConfigField(config.FIELD_GUEST_DAILY_USD, "Max USD per guest / 24h", type="float", default=0.05,
|
||||
minimum=0, help="Rolling 24-hour spend cap for an anonymous guest.", group="Limits"),
|
||||
ConfigField(config.FIELD_GUESTS_ENABLED, "Allow guests", type="bool", default=True,
|
||||
help="When off, only signed-in users may use Devii.", group="Limits"),
|
||||
ConfigField(config.FIELD_PRICE_CACHE_HIT, "Price cache-hit / 1M", type="float", default=0.0028,
|
||||
minimum=0, help="USD per million cache-hit input tokens.", group="Pricing"),
|
||||
ConfigField(config.FIELD_PRICE_CACHE_MISS, "Price cache-miss / 1M", type="float", default=0.14,
|
||||
minimum=0, help="USD per million cache-miss input tokens.", group="Pricing"),
|
||||
ConfigField(config.FIELD_PRICE_OUTPUT, "Price output / 1M", type="float", default=0.28,
|
||||
minimum=0, help="USD per million output tokens.", group="Pricing"),
|
||||
ConfigField(config.FIELD_RSEARCH_ENABLED, "Enable web search tools", type="bool", default=True,
|
||||
help="Allow the external rsearch_* tools (web/image search, web-grounded AI answer, "
|
||||
"chat, image description). When off, those tools are refused. These reach an external "
|
||||
"public service, not this platform.", group="Web search"),
|
||||
ConfigField(config.FIELD_RSEARCH_URL, "Web search service URL", type="url",
|
||||
default=config.DEFAULT_RSEARCH_URL,
|
||||
help="Base URL of the rsearch-compatible service the rsearch_* tools call.",
|
||||
group="Web search"),
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(name="devii", interval_seconds=60)
|
||||
self._hub = None
|
||||
|
||||
def hub(self):
|
||||
if self._hub is None:
|
||||
from .hub import DeviiHub
|
||||
self._hub = DeviiHub(self._build_settings)
|
||||
return self._hub
|
||||
|
||||
def effective_config(self) -> dict:
|
||||
from devplacepy.database import internal_gateway_key
|
||||
cfg = self.get_config()
|
||||
cfg[config.FIELD_AI_KEY] = (cfg[config.FIELD_AI_KEY]
|
||||
or os.environ.get("DEVII_AI_KEY", "")
|
||||
or internal_gateway_key())
|
||||
return cfg
|
||||
|
||||
def instance_base_url(self) -> str:
|
||||
cfg = self.get_config()
|
||||
configured = cfg.get("devii_base_url", "")
|
||||
if configured:
|
||||
return configured
|
||||
return os.environ.get("DEVII_BASE_URL", INSTANCE_ORIGIN_DEFAULT)
|
||||
|
||||
def _pricing(self, cfg: dict) -> Pricing:
|
||||
return Pricing(
|
||||
model=cfg[config.FIELD_AI_MODEL] or DEFAULT_AI_MODEL,
|
||||
cache_hit_per_m=float(cfg[config.FIELD_PRICE_CACHE_HIT]),
|
||||
cache_miss_per_m=float(cfg[config.FIELD_PRICE_CACHE_MISS]),
|
||||
output_per_m=float(cfg[config.FIELD_PRICE_OUTPUT]),
|
||||
)
|
||||
|
||||
def _build_settings(self, api_key: str, base_url: str):
|
||||
cfg = self.effective_config()
|
||||
settings = build_settings(cfg, base_url, api_key)
|
||||
return settings, self._pricing(cfg)
|
||||
|
||||
def daily_limit_for(self, owner_kind: str) -> float:
|
||||
cfg = self.get_config()
|
||||
if owner_kind == "guest":
|
||||
return float(cfg[config.FIELD_GUEST_DAILY_USD])
|
||||
return float(cfg[config.FIELD_USER_DAILY_USD])
|
||||
|
||||
def guests_enabled(self) -> bool:
|
||||
return bool(self.get_config()[config.FIELD_GUESTS_ENABLED])
|
||||
|
||||
def spent_24h(self, owner_kind: str, owner_id: str) -> float:
|
||||
return self.hub().ledger.spent_24h(owner_kind, owner_id)
|
||||
|
||||
async def run_once(self) -> None:
|
||||
if not self.is_enabled():
|
||||
return
|
||||
hub = self.hub()
|
||||
pruned = hub.ledger.prune(48)
|
||||
removed = await hub.gc_idle()
|
||||
if pruned or removed:
|
||||
self.log(f"Housekeeping: pruned {pruned} ledger rows, closed {removed} idle sessions")
|
||||
|
||||
async def on_disable(self) -> None:
|
||||
if self._hub is not None:
|
||||
await self._hub.aclose()
|
||||
self._hub = None
|
||||
|
||||
def collect_metrics(self) -> dict:
|
||||
cfg = self.get_config()
|
||||
hub = self._hub
|
||||
active = hub.active_sessions() if hub is not None else 0
|
||||
conns = hub.connection_count() if hub is not None else 0
|
||||
spent = hub.ledger.total_spent_24h() if hub is not None else 0.0
|
||||
return {
|
||||
"stats": [
|
||||
{"label": "Active sessions", "value": active},
|
||||
{"label": "Connections", "value": conns},
|
||||
{"label": "Spend 24h", "value": f"${spent:.4f}"},
|
||||
{"label": "User cap 24h", "value": f"${float(cfg[config.FIELD_USER_DAILY_USD]):.2f}"},
|
||||
{"label": "Guest cap 24h", "value": f"${float(cfg[config.FIELD_GUEST_DAILY_USD]):.2f}"},
|
||||
{"label": "Guests", "value": "on" if cfg[config.FIELD_GUESTS_ENABLED] else "off"},
|
||||
{"label": "Model", "value": cfg[config.FIELD_AI_MODEL]},
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import uuid_utils
|
||||
|
||||
from .actions import Dispatcher
|
||||
from .agent import Agent, SYSTEM_PROMPT
|
||||
from .agentic import AgenticController, LessonStore
|
||||
from .avatar import AvatarController
|
||||
from .chunks import ChunkStore
|
||||
from .client import ClientController
|
||||
from .config import Settings
|
||||
from .cost import CostTracker
|
||||
from .cost.tracker import Pricing
|
||||
from .errors import AuthRequiredError
|
||||
from .http_client import PlatformClient
|
||||
from .llm import LLMClient
|
||||
from .registry import CATALOG
|
||||
from .tasks import Scheduler, TaskController, TaskStore
|
||||
|
||||
logger = logging.getLogger("devii.session")
|
||||
|
||||
AVATAR_QUERY_TIMEOUT_SECONDS = 30.0
|
||||
BROWSER_REQUEST_DEADLINE_SECONDS = 60.0
|
||||
RESULT_NOTICE_CHARS = 20000
|
||||
INTERNAL_PREFIXES = (
|
||||
"[memory]",
|
||||
"[reflection-trigger]",
|
||||
"[verification-gate]",
|
||||
"[stopped]",
|
||||
"Protocol violation:",
|
||||
)
|
||||
BUFFERED_TYPES = ("task", "error", "reply", "status")
|
||||
LOGIN_REQUEST = (
|
||||
"Welcome to Devii. Ask me anything - I can generate clients and bots, search the docs, "
|
||||
"fetch pages, and more. Sign in to DevPlace whenever you want me to work on your own account."
|
||||
)
|
||||
|
||||
|
||||
class DeviiSession:
|
||||
def __init__(
|
||||
self,
|
||||
owner_kind: str,
|
||||
owner_id: str,
|
||||
username: str,
|
||||
settings: Settings,
|
||||
llm: LLMClient,
|
||||
lessons: LessonStore,
|
||||
pricing: Pricing,
|
||||
task_store: TaskStore,
|
||||
stores: dict[str, Any],
|
||||
) -> None:
|
||||
self.owner_kind = owner_kind
|
||||
self.owner_id = owner_id
|
||||
self.username = username
|
||||
self.settings = settings
|
||||
self.persist_conversation = owner_kind == "user"
|
||||
self._llm = llm
|
||||
self._lessons = lessons
|
||||
self.client = PlatformClient(settings.base_url, settings.timeout_seconds, settings.platform_api_key)
|
||||
self.avatar = AvatarController()
|
||||
self.browser = ClientController(allow_eval=settings.allow_eval)
|
||||
self.store = task_store
|
||||
self.task_controller = TaskController(self.store)
|
||||
self.agentic = AgenticController(lessons, settings)
|
||||
self.cost = CostTracker(pricing=pricing)
|
||||
self.chunks = ChunkStore()
|
||||
self.dispatcher = Dispatcher(
|
||||
CATALOG, self.client, settings, self.task_controller, self.agentic,
|
||||
avatar=self.avatar, browser=self.browser,
|
||||
)
|
||||
self.tools = CATALOG.tool_schemas_for(self.client.authenticated)
|
||||
self.agentic.bind(
|
||||
llm=llm, dispatcher=self.dispatcher, tools=self.tools,
|
||||
on_trace=self._trace, cost_tracker=self.cost, chunk_store=self.chunks,
|
||||
)
|
||||
self.agent = Agent(
|
||||
settings, llm, self.dispatcher, self.tools, lessons=lessons,
|
||||
on_trace=self._trace, cost_tracker=self.cost, chunk_store=self.chunks,
|
||||
)
|
||||
self.scheduler = Scheduler(
|
||||
self.store, self._make_executor(), self._task_event, settings.scheduler_tick_seconds
|
||||
)
|
||||
self._conv = stores["conversations"]
|
||||
self._ledger = stores["ledger"]
|
||||
self._audit = stores["turns"]
|
||||
self._conns: set[Any] = set()
|
||||
self._primary: Any = None
|
||||
self._lock = asyncio.Lock()
|
||||
self._send_lock = asyncio.Lock()
|
||||
self._pending: dict[str, asyncio.Future] = {}
|
||||
self._query_seq = 0
|
||||
self._connected = asyncio.Event()
|
||||
self._disconnected = asyncio.Event()
|
||||
self._disconnected.set()
|
||||
self._started = False
|
||||
self._buffer: list[dict[str, Any]] = []
|
||||
self._turn_tool_calls = 0
|
||||
self._pending_session: str | None = None
|
||||
|
||||
def restore_history(self, messages: list[dict[str, Any]]) -> None:
|
||||
if messages:
|
||||
self.agent._messages = messages
|
||||
|
||||
def history(self) -> list[dict[str, Any]]:
|
||||
visible: list[dict[str, Any]] = []
|
||||
for message in self.agent._messages:
|
||||
role = message.get("role")
|
||||
content = message.get("content")
|
||||
if role not in ("user", "assistant") or not content:
|
||||
continue
|
||||
if role == "user" and isinstance(content, str) and content.startswith(INTERNAL_PREFIXES):
|
||||
continue
|
||||
visible.append({"role": role, "content": content})
|
||||
return visible
|
||||
|
||||
def _make_executor(self) -> Any:
|
||||
async def execute(prompt: str) -> str:
|
||||
async with self._lock:
|
||||
worker = Agent(
|
||||
self.settings, self._llm, self.dispatcher, self.tools, lessons=self._lessons,
|
||||
on_trace=self._trace, cost_tracker=self.cost, chunk_store=self.chunks,
|
||||
)
|
||||
return await worker.respond(prompt)
|
||||
|
||||
return execute
|
||||
|
||||
def attach(self, ws: Any) -> None:
|
||||
self._conns.add(ws)
|
||||
self._primary = ws
|
||||
self._connected.set()
|
||||
self._disconnected.clear()
|
||||
self.avatar.bind(self._avatar_request)
|
||||
self.browser.bind(self._client_request)
|
||||
if not self._started:
|
||||
self.scheduler.start()
|
||||
self._started = True
|
||||
if self._buffer:
|
||||
pending = self._buffer
|
||||
self._buffer = []
|
||||
asyncio.create_task(self._flush(pending))
|
||||
logger.info("Session %s/%s attached (%d conns)", self.owner_kind, self.owner_id, len(self._conns))
|
||||
|
||||
def detach(self, ws: Any) -> None:
|
||||
self._conns.discard(ws)
|
||||
if self._primary is ws:
|
||||
self._primary = next(iter(self._conns), None)
|
||||
if not self._conns:
|
||||
self._connected.clear()
|
||||
self._disconnected.set()
|
||||
self.avatar.unbind()
|
||||
self.browser.unbind()
|
||||
logger.info("Session %s/%s detached (%d conns)", self.owner_kind, self.owner_id, len(self._conns))
|
||||
|
||||
@property
|
||||
def connection_count(self) -> int:
|
||||
return len(self._conns)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self.scheduler.stop()
|
||||
await self.client.aclose()
|
||||
await self._llm.aclose()
|
||||
|
||||
async def bootstrap_greeting(self) -> str:
|
||||
if self.client.authenticated:
|
||||
return f"Signed in as {self.username}. Devii is operating your DevPlace account. How can I help?"
|
||||
return LOGIN_REQUEST
|
||||
|
||||
async def send_bootstrap(self, ws: Any) -> None:
|
||||
await self._send_to(ws, {"type": "status", "text": "Devii ready"})
|
||||
history = self.history()
|
||||
if history:
|
||||
await self._send_to(ws, {"type": "history", "messages": history})
|
||||
else:
|
||||
await self._send_to(ws, {"type": "reply", "text": await self.bootstrap_greeting()})
|
||||
|
||||
async def reset_conversation(self) -> None:
|
||||
async with self._lock:
|
||||
messages = self.agent._messages
|
||||
if messages and messages[0].get("role") == "system":
|
||||
system = messages[0]
|
||||
else:
|
||||
system = {"role": "system", "content": SYSTEM_PROMPT}
|
||||
self.agent._messages = [system]
|
||||
self._buffer = []
|
||||
if self.persist_conversation:
|
||||
try:
|
||||
self._conv.clear(self.owner_kind, self.owner_id)
|
||||
except Exception: # noqa: BLE001 - clearing storage must not break the socket
|
||||
logger.exception("Failed to clear conversation for %s/%s", self.owner_kind, self.owner_id)
|
||||
await self._emit({"type": "clear"}, buffer=False)
|
||||
|
||||
def spawn_turn(self, text: str) -> None:
|
||||
asyncio.create_task(self._run_turn(text))
|
||||
|
||||
async def _run_turn(self, text: str) -> None:
|
||||
turn_id = uuid_utils.uuid7().hex
|
||||
started_at = _now_iso()
|
||||
before = self._cost_snapshot()
|
||||
self._turn_tool_calls = 0
|
||||
reply = ""
|
||||
error = ""
|
||||
try:
|
||||
async with self._lock:
|
||||
reply = await self.agent.respond(text)
|
||||
await self._emit({"type": "reply", "text": reply}, buffer=True)
|
||||
except Exception as exc: # noqa: BLE001 - reported to the browser, recorded for audit
|
||||
error = str(exc)
|
||||
logger.exception("Turn failed for %s/%s", self.owner_kind, self.owner_id)
|
||||
await self._emit({"type": "error", "text": error}, buffer=True)
|
||||
finally:
|
||||
self._record_turn(turn_id, started_at, text, reply, error, before)
|
||||
|
||||
def _cost_snapshot(self) -> dict[str, Any]:
|
||||
return {
|
||||
"requests": self.cost.requests,
|
||||
"prompt_tokens": self.cost.prompt_tokens,
|
||||
"completion_tokens": self.cost.completion_tokens,
|
||||
"cache_hit_tokens": self.cost.cache_hit_tokens,
|
||||
"cache_miss_tokens": self.cost.cache_miss_tokens,
|
||||
"cost_usd": self.cost.cost_usd()["total"],
|
||||
}
|
||||
|
||||
def _record_turn(self, turn_id, started_at, prompt, reply, error, before) -> None:
|
||||
after = self._cost_snapshot()
|
||||
usage = {
|
||||
"prompt_tokens": after["prompt_tokens"] - before["prompt_tokens"],
|
||||
"completion_tokens": after["completion_tokens"] - before["completion_tokens"],
|
||||
"cache_hit_tokens": after["cache_hit_tokens"] - before["cache_hit_tokens"],
|
||||
"cache_miss_tokens": after["cache_miss_tokens"] - before["cache_miss_tokens"],
|
||||
}
|
||||
cost_delta = round(after["cost_usd"] - before["cost_usd"], 8)
|
||||
try:
|
||||
if self.persist_conversation:
|
||||
self._conv.save(self.owner_kind, self.owner_id, self.agent._messages)
|
||||
self._ledger.record(self.owner_kind, self.owner_id, turn_id, usage, cost_delta, self.settings.ai_model)
|
||||
self._audit.record({
|
||||
"turn_id": turn_id,
|
||||
"owner_kind": self.owner_kind,
|
||||
"owner_id": self.owner_id,
|
||||
"username": self.username,
|
||||
"started_at": started_at,
|
||||
"ended_at": _now_iso(),
|
||||
"prompt": prompt,
|
||||
"reply": reply,
|
||||
"iterations": after["requests"] - before["requests"],
|
||||
"tool_calls": self._turn_tool_calls,
|
||||
"cost_usd": cost_delta,
|
||||
"error": error,
|
||||
})
|
||||
except Exception: # noqa: BLE001 - persistence must never break a turn
|
||||
logger.exception("Failed to persist turn for %s/%s", self.owner_kind, self.owner_id)
|
||||
|
||||
def resolve_query(self, query_id: str, payload: Any) -> None:
|
||||
future = self._pending.pop(query_id, None)
|
||||
if future is not None and not future.done():
|
||||
future.set_result(payload)
|
||||
|
||||
async def _flush(self, frames: list[dict[str, Any]]) -> None:
|
||||
for frame in frames:
|
||||
await self._emit(frame, buffer=True)
|
||||
|
||||
def _trace(self, event: str, name: str = "", detail: str = "") -> None:
|
||||
if event == "call":
|
||||
self._turn_tool_calls += 1
|
||||
elif event == "ok":
|
||||
self._sync_auth(name)
|
||||
payload = {"type": "trace", "event": event, "name": name, "detail": detail}
|
||||
asyncio.create_task(self._emit(payload, buffer=False))
|
||||
|
||||
def _sync_auth(self, name: str) -> None:
|
||||
# When the agent changes who is logged in, propagate it to the browser so the terminal and
|
||||
# the browser share one session. Login mints a real platform session in the client's cookie
|
||||
# jar; hand that token to the browser via /devii/adopt. Logout reuses /auth/logout.
|
||||
if name in ("login", "signup"):
|
||||
token = self.client.session_cookie()
|
||||
if token:
|
||||
self._pending_session = token
|
||||
asyncio.create_task(self._emit({"type": "auth", "action": "adopt"}, buffer=False))
|
||||
elif name == "logout":
|
||||
self._pending_session = None
|
||||
asyncio.create_task(self._emit({"type": "auth", "action": "logout"}, buffer=False))
|
||||
|
||||
def take_pending_session(self) -> str | None:
|
||||
token = self._pending_session
|
||||
self._pending_session = None
|
||||
return token
|
||||
|
||||
async def _send_to(self, ws: Any, payload: dict[str, Any]) -> None:
|
||||
async with self._send_lock:
|
||||
await ws.send_json(payload)
|
||||
|
||||
async def _emit(self, payload: dict[str, Any], buffer: bool = True) -> None:
|
||||
delivered = False
|
||||
for ws in list(self._conns):
|
||||
try:
|
||||
async with self._send_lock:
|
||||
await ws.send_json(payload)
|
||||
delivered = True
|
||||
except Exception: # noqa: BLE001 - drop dead sockets, keep serving the rest
|
||||
self._conns.discard(ws)
|
||||
if not delivered and buffer and payload.get("type") in BUFFERED_TYPES:
|
||||
self._buffer.append(payload)
|
||||
if len(self._buffer) > 100:
|
||||
self._buffer = self._buffer[-100:]
|
||||
|
||||
async def _avatar_request(self, action: str, args: dict[str, Any]) -> Any:
|
||||
return await self._browser_request("avatar", action, args)
|
||||
|
||||
async def _client_request(self, action: str, args: dict[str, Any]) -> Any:
|
||||
return await self._browser_request("client", action, args)
|
||||
|
||||
async def _browser_request(self, channel: str, action: str, args: dict[str, Any]) -> Any:
|
||||
loop = asyncio.get_event_loop()
|
||||
deadline = loop.time() + BROWSER_REQUEST_DEADLINE_SECONDS
|
||||
self._query_seq += 1
|
||||
request_id = f"{self.owner_id}:{self._query_seq}"
|
||||
future: asyncio.Future = loop.create_future()
|
||||
self._pending[request_id] = future
|
||||
frame = {"type": channel, "id": request_id, "action": action, "args": args}
|
||||
try:
|
||||
while not future.done():
|
||||
remaining = deadline - loop.time()
|
||||
if remaining <= 0:
|
||||
raise RuntimeError(f"Browser did not respond to '{action}' in time")
|
||||
try:
|
||||
await asyncio.wait_for(self._connected.wait(), timeout=remaining)
|
||||
except asyncio.TimeoutError as exc:
|
||||
raise RuntimeError(f"No browser connected to handle '{action}'") from exc
|
||||
ws = self._primary
|
||||
if ws is None:
|
||||
continue
|
||||
try:
|
||||
await self._send_to(ws, frame)
|
||||
except Exception: # noqa: BLE001 - socket dying mid-send; retry on reconnect
|
||||
await asyncio.sleep(0.1)
|
||||
continue
|
||||
drop = asyncio.create_task(self._disconnected.wait())
|
||||
try:
|
||||
await asyncio.wait(
|
||||
{future, drop},
|
||||
timeout=max(0.0, deadline - loop.time()),
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
finally:
|
||||
drop.cancel()
|
||||
# future done -> loop exits and returns; disconnected -> re-send after
|
||||
# reconnect; otherwise the deadline check at the top raises.
|
||||
return future.result()
|
||||
finally:
|
||||
self._pending.pop(request_id, None)
|
||||
|
||||
def _task_event(self, kind: str, row: dict[str, Any], payload: str) -> None:
|
||||
label = row.get("label") or row.get("uid", "task")
|
||||
message = {"type": "task", "kind": kind, "label": label, "payload": payload[:RESULT_NOTICE_CHARS]}
|
||||
asyncio.create_task(self._emit(message, buffer=True))
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
from datetime import datetime, timezone
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
@@ -0,0 +1,137 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from devplacepy.database import db, get_table
|
||||
|
||||
logger = logging.getLogger("devii.store")
|
||||
|
||||
CONVERSATIONS = "devii_conversations"
|
||||
LEDGER = "devii_usage_ledger"
|
||||
TURNS = "devii_turns"
|
||||
|
||||
PROMPT_AUDIT_CHARS = 4000
|
||||
REPLY_AUDIT_CHARS = 4000
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _iso(moment: datetime) -> str:
|
||||
return moment.isoformat()
|
||||
|
||||
|
||||
class ConversationStore:
|
||||
def load(self, owner_kind: str, owner_id: str) -> list[dict[str, Any]] | None:
|
||||
if CONVERSATIONS not in db.tables:
|
||||
return None
|
||||
row = get_table(CONVERSATIONS).find_one(owner_kind=owner_kind, owner_id=owner_id)
|
||||
if not row or not row.get("messages"):
|
||||
return None
|
||||
try:
|
||||
return json.loads(row["messages"])
|
||||
except (ValueError, TypeError):
|
||||
logger.warning("Corrupt conversation for %s/%s", owner_kind, owner_id)
|
||||
return None
|
||||
|
||||
def save(self, owner_kind: str, owner_id: str, messages: list[dict[str, Any]]) -> None:
|
||||
now = _iso(_now())
|
||||
record = {
|
||||
"owner_kind": owner_kind,
|
||||
"owner_id": owner_id,
|
||||
"messages": json.dumps(messages),
|
||||
"updated_at": now,
|
||||
}
|
||||
table = get_table(CONVERSATIONS)
|
||||
existing = table.find_one(owner_kind=owner_kind, owner_id=owner_id) if CONVERSATIONS in db.tables else None
|
||||
if existing:
|
||||
table.update({**record, "id": existing["id"]}, ["id"])
|
||||
else:
|
||||
record["created_at"] = now
|
||||
table.insert(record)
|
||||
|
||||
def clear(self, owner_kind: str, owner_id: str) -> None:
|
||||
if CONVERSATIONS in db.tables:
|
||||
get_table(CONVERSATIONS).delete(owner_kind=owner_kind, owner_id=owner_id)
|
||||
|
||||
|
||||
class UsageLedger:
|
||||
def record(
|
||||
self,
|
||||
owner_kind: str,
|
||||
owner_id: str,
|
||||
turn_id: str,
|
||||
usage: dict[str, int],
|
||||
cost_usd: float,
|
||||
model: str,
|
||||
) -> None:
|
||||
get_table(LEDGER).insert({
|
||||
"owner_kind": owner_kind,
|
||||
"owner_id": owner_id,
|
||||
"turn_id": turn_id,
|
||||
"created_at": _iso(_now()),
|
||||
"prompt_tokens": int(usage.get("prompt_tokens", 0)),
|
||||
"completion_tokens": int(usage.get("completion_tokens", 0)),
|
||||
"cache_hit_tokens": int(usage.get("cache_hit_tokens", 0)),
|
||||
"cache_miss_tokens": int(usage.get("cache_miss_tokens", 0)),
|
||||
"cost_usd": float(cost_usd),
|
||||
"model": model,
|
||||
})
|
||||
|
||||
def spent_24h(self, owner_kind: str, owner_id: str) -> float:
|
||||
if LEDGER not in db.tables:
|
||||
return 0.0
|
||||
cutoff = _iso(_now() - timedelta(hours=24))
|
||||
rows = db.query(
|
||||
f"SELECT COALESCE(SUM(cost_usd), 0) AS total FROM {LEDGER} "
|
||||
"WHERE owner_kind = :owner_kind AND owner_id = :owner_id AND created_at >= :cutoff",
|
||||
owner_kind=owner_kind, owner_id=owner_id, cutoff=cutoff,
|
||||
)
|
||||
for row in rows:
|
||||
return float(row["total"] or 0.0)
|
||||
return 0.0
|
||||
|
||||
def turns_24h(self, owner_kind: str, owner_id: str) -> int:
|
||||
if LEDGER not in db.tables:
|
||||
return 0
|
||||
cutoff = _iso(_now() - timedelta(hours=24))
|
||||
rows = db.query(
|
||||
f"SELECT COUNT(*) AS n FROM {LEDGER} "
|
||||
"WHERE owner_kind = :owner_kind AND owner_id = :owner_id AND created_at >= :cutoff",
|
||||
owner_kind=owner_kind, owner_id=owner_id, cutoff=cutoff,
|
||||
)
|
||||
for row in rows:
|
||||
return int(row["n"] or 0)
|
||||
return 0
|
||||
|
||||
def total_spent_24h(self) -> float:
|
||||
if LEDGER not in db.tables:
|
||||
return 0.0
|
||||
cutoff = _iso(_now() - timedelta(hours=24))
|
||||
rows = db.query(
|
||||
f"SELECT COALESCE(SUM(cost_usd), 0) AS total FROM {LEDGER} WHERE created_at >= :cutoff",
|
||||
cutoff=cutoff,
|
||||
)
|
||||
for row in rows:
|
||||
return float(row["total"] or 0.0)
|
||||
return 0.0
|
||||
|
||||
def prune(self, older_than_hours: int = 48) -> int:
|
||||
if LEDGER not in db.tables:
|
||||
return 0
|
||||
cutoff = _iso(_now() - timedelta(hours=older_than_hours))
|
||||
return int(get_table(LEDGER).delete(created_at={"<": cutoff}))
|
||||
|
||||
|
||||
class TurnAudit:
|
||||
def record(self, record: dict[str, Any]) -> None:
|
||||
record = dict(record)
|
||||
record["prompt"] = (record.get("prompt") or "")[:PROMPT_AUDIT_CHARS]
|
||||
record["reply"] = (record.get("reply") or "")[:REPLY_AUDIT_CHARS]
|
||||
get_table(TURNS).insert(record)
|
||||
@@ -0,0 +1,8 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from .actions import TASK_ACTIONS
|
||||
from .controller import TaskController
|
||||
from .scheduler import Scheduler
|
||||
from .store import TaskStore
|
||||
|
||||
__all__ = ["TASK_ACTIONS", "TaskController", "Scheduler", "TaskStore"]
|
||||
@@ -0,0 +1,128 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..actions.spec import Action, Param
|
||||
|
||||
|
||||
def field(name: str, description: str, required: bool = False, kind: str = "string") -> Param:
|
||||
return Param(name=name, location="body", description=description, required=required, type=kind)
|
||||
|
||||
|
||||
SCHEDULE_FIELDS: tuple[Param, ...] = (
|
||||
field(
|
||||
"kind",
|
||||
"Schedule type: 'once' (run a single time), 'interval' (repeat every N seconds), "
|
||||
"or 'cron' (5-field cron expression).",
|
||||
required=True,
|
||||
),
|
||||
field(
|
||||
"run_at",
|
||||
"For kind=once: absolute UTC time in ISO 8601, e.g. 2026-06-09T14:30:00.",
|
||||
),
|
||||
field("delay_seconds", "For kind=once: run this many seconds from now.", kind="integer"),
|
||||
field("every_seconds", "For kind=interval: number of seconds between runs.", kind="integer"),
|
||||
field(
|
||||
"start_at",
|
||||
"For kind=interval: optional ISO 8601 UTC time of the first run. "
|
||||
"Defaults to one interval from now.",
|
||||
),
|
||||
field(
|
||||
"cron",
|
||||
"For kind=cron: a 5-field cron expression 'minute hour day-of-month month day-of-week'. "
|
||||
"Supports *, ranges (1-5), lists (1,3,5) and steps (*/15).",
|
||||
),
|
||||
field(
|
||||
"max_runs",
|
||||
"Optional maximum number of executions; the task disables itself afterwards.",
|
||||
kind="integer",
|
||||
),
|
||||
)
|
||||
|
||||
TASK_ACTIONS: tuple[Action, ...] = (
|
||||
Action(
|
||||
name="create_task",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Queue a prompt to run autonomously on a schedule with full tool access",
|
||||
description=(
|
||||
"The prompt is executed later by a fresh agent that has every tool available, "
|
||||
"running inside the current authenticated session."
|
||||
),
|
||||
handler="task",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
field(
|
||||
"prompt",
|
||||
"A complete, self-contained instruction the future agent will execute when the "
|
||||
"task fires (it has no other context). Write an imperative command, not a "
|
||||
"description: e.g. 'Create a post in the fun topic about raccoons with a short "
|
||||
"funny text, then verify it appears.'",
|
||||
required=True,
|
||||
),
|
||||
field("label", "Optional short human-readable label for the task."),
|
||||
*SCHEDULE_FIELDS,
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="list_tasks",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="List queued tasks and their schedules and last results",
|
||||
handler="task",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
field("enabled_only", "Only return enabled tasks.", kind="boolean"),
|
||||
field("status", "Filter by status: pending, running, done, error, disabled."),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="get_task",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Get a single task including its full last result",
|
||||
handler="task",
|
||||
requires_auth=False,
|
||||
params=(field("uid", "Uid of the task.", required=True),),
|
||||
),
|
||||
Action(
|
||||
name="update_task",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Update a task's prompt, label, enabled state, or schedule",
|
||||
description="Provide schedule fields together with kind to reschedule the task.",
|
||||
handler="task",
|
||||
requires_auth=False,
|
||||
params=(
|
||||
field("uid", "Uid of the task.", required=True),
|
||||
field("prompt", "New prompt."),
|
||||
field("label", "New label."),
|
||||
field("enabled", "Enable or disable the task.", kind="boolean"),
|
||||
field("kind", "New schedule type when rescheduling."),
|
||||
field("run_at", "New absolute run time for kind=once."),
|
||||
field("delay_seconds", "New relative delay for kind=once.", kind="integer"),
|
||||
field("every_seconds", "New interval for kind=interval.", kind="integer"),
|
||||
field("start_at", "New first-run time for kind=interval."),
|
||||
field("cron", "New cron expression for kind=cron."),
|
||||
field("max_runs", "New maximum number of executions.", kind="integer"),
|
||||
),
|
||||
),
|
||||
Action(
|
||||
name="delete_task",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Delete a queued task",
|
||||
handler="task",
|
||||
requires_auth=False,
|
||||
params=(field("uid", "Uid of the task.", required=True),),
|
||||
),
|
||||
Action(
|
||||
name="run_task_now",
|
||||
method="LOCAL",
|
||||
path="",
|
||||
summary="Trigger a task to execute immediately on the next scheduler tick",
|
||||
handler="task",
|
||||
requires_auth=False,
|
||||
params=(field("uid", "Uid of the task.", required=True),),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,211 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from ..errors import ToolInputError
|
||||
from .schedule import Schedule, next_run, now_utc, to_iso
|
||||
from .store import TaskStore
|
||||
|
||||
logger = logging.getLogger("devii.tasks.controller")
|
||||
|
||||
SCHEDULE_KEYS = ("kind", "run_at", "delay_seconds", "every_seconds", "start_at", "cron", "max_runs")
|
||||
RESULT_PREVIEW_CHARS = 500
|
||||
TRUTHY = {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _as_bool(value: Any, default: bool = True) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if value is None:
|
||||
return default
|
||||
return str(value).strip().lower() in TRUTHY
|
||||
|
||||
|
||||
def _serialize(row: dict[str, Any], preview: bool) -> dict[str, Any]:
|
||||
view = {
|
||||
"uid": row.get("uid"),
|
||||
"label": row.get("label"),
|
||||
"prompt": row.get("prompt"),
|
||||
"enabled": bool(row.get("enabled")),
|
||||
"status": row.get("status"),
|
||||
"kind": row.get("kind"),
|
||||
"next_run_at": row.get("next_run_at"),
|
||||
"last_run_at": row.get("last_run_at"),
|
||||
"run_count": row.get("run_count"),
|
||||
"max_runs": row.get("max_runs"),
|
||||
"every_seconds": row.get("every_seconds"),
|
||||
"cron": row.get("cron"),
|
||||
"run_at": row.get("run_at"),
|
||||
}
|
||||
last_error = row.get("last_error")
|
||||
if last_error:
|
||||
view["last_error"] = last_error
|
||||
result = row.get("last_result")
|
||||
if result:
|
||||
view["last_result"] = result[:RESULT_PREVIEW_CHARS] if preview else result
|
||||
return view
|
||||
|
||||
|
||||
class TaskController:
|
||||
def __init__(self, store: TaskStore) -> None:
|
||||
self._store = store
|
||||
|
||||
async def dispatch(self, name: str, arguments: dict[str, Any]) -> str:
|
||||
handlers = {
|
||||
"create_task": self.create_task,
|
||||
"list_tasks": self.list_tasks,
|
||||
"get_task": self.get_task,
|
||||
"update_task": self.update_task,
|
||||
"delete_task": self.delete_task,
|
||||
"run_task_now": self.run_task_now,
|
||||
}
|
||||
handler = handlers.get(name)
|
||||
if handler is None:
|
||||
raise ToolInputError(f"Unknown task tool: {name}")
|
||||
return handler(arguments)
|
||||
|
||||
def create_task(self, arguments: dict[str, Any]) -> str:
|
||||
prompt = str(arguments.get("prompt", "")).strip()
|
||||
if not prompt:
|
||||
raise ToolInputError("create_task requires a non-empty prompt.")
|
||||
schedule = self._build_schedule(arguments)
|
||||
reference = now_utc()
|
||||
first = schedule.first_run(reference)
|
||||
record: dict[str, Any] = {
|
||||
"uid": uuid.uuid4().hex,
|
||||
"label": (arguments.get("label") or "").strip() or None,
|
||||
"prompt": prompt,
|
||||
"enabled": True,
|
||||
"status": "pending",
|
||||
"created_at": to_iso(reference),
|
||||
"next_run_at": to_iso(first),
|
||||
"last_run_at": None,
|
||||
"run_count": 0,
|
||||
"last_result": None,
|
||||
"last_error": None,
|
||||
**schedule.columns(),
|
||||
}
|
||||
self._store.create(record)
|
||||
return json.dumps(
|
||||
{"status": "created", "task": _serialize(record, preview=True)},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
def list_tasks(self, arguments: dict[str, Any]) -> str:
|
||||
rows = self._store.list(
|
||||
enabled_only=_as_bool(arguments.get("enabled_only"), default=False),
|
||||
status=(arguments.get("status") or None),
|
||||
)
|
||||
rows.sort(key=lambda row: row.get("next_run_at") or "")
|
||||
return json.dumps(
|
||||
{"count": len(rows), "tasks": [_serialize(row, preview=True) for row in rows]},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
def get_task(self, arguments: dict[str, Any]) -> str:
|
||||
row = self._require_task(arguments)
|
||||
return json.dumps({"task": _serialize(row, preview=False)}, ensure_ascii=False)
|
||||
|
||||
def update_task(self, arguments: dict[str, Any]) -> str:
|
||||
row = self._require_task(arguments)
|
||||
changes: dict[str, Any] = {}
|
||||
|
||||
if "prompt" in arguments and arguments["prompt"] is not None:
|
||||
new_prompt = str(arguments["prompt"]).strip()
|
||||
if not new_prompt:
|
||||
raise ToolInputError("prompt cannot be empty.")
|
||||
changes["prompt"] = new_prompt
|
||||
if "label" in arguments:
|
||||
changes["label"] = (arguments.get("label") or "").strip() or None
|
||||
if "enabled" in arguments:
|
||||
changes["enabled"] = _as_bool(arguments.get("enabled"))
|
||||
|
||||
if any(key in arguments and arguments[key] is not None for key in SCHEDULE_KEYS):
|
||||
merged = {key: row.get(key) for key in SCHEDULE_KEYS}
|
||||
for key in SCHEDULE_KEYS:
|
||||
if key in arguments and arguments[key] is not None:
|
||||
merged[key] = arguments[key]
|
||||
schedule = self._build_schedule(merged)
|
||||
changes.update(schedule.columns())
|
||||
changes["next_run_at"] = to_iso(schedule.first_run(now_utc()))
|
||||
changes["status"] = "pending"
|
||||
|
||||
if changes.get("enabled") and row.get("status") in ("done", "disabled", "error"):
|
||||
changes.setdefault("status", "pending")
|
||||
if not changes.get("next_run_at") and not row.get("next_run_at"):
|
||||
schedule = self._build_schedule({key: row.get(key) for key in SCHEDULE_KEYS})
|
||||
changes["next_run_at"] = to_iso(schedule.first_run(now_utc()))
|
||||
if changes.get("enabled") is False:
|
||||
changes["status"] = "disabled"
|
||||
|
||||
if not changes:
|
||||
raise ToolInputError("No updatable fields supplied.")
|
||||
|
||||
self._store.update(row["uid"], changes)
|
||||
return json.dumps(
|
||||
{"status": "updated", "task": _serialize(self._store.get(row["uid"]), preview=True)},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
def delete_task(self, arguments: dict[str, Any]) -> str:
|
||||
uid = self._require_uid(arguments)
|
||||
deleted = self._store.delete(uid)
|
||||
if not deleted:
|
||||
raise ToolInputError(f"No task found with uid {uid}.")
|
||||
return json.dumps({"status": "deleted", "uid": uid}, ensure_ascii=False)
|
||||
|
||||
def run_task_now(self, arguments: dict[str, Any]) -> str:
|
||||
row = self._require_task(arguments)
|
||||
self._store.update(
|
||||
row["uid"],
|
||||
{"enabled": True, "status": "pending", "next_run_at": to_iso(now_utc())},
|
||||
)
|
||||
return json.dumps(
|
||||
{"status": "queued", "uid": row["uid"], "note": "Will execute on the next scheduler tick."},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
def _build_schedule(self, source: dict[str, Any]) -> Schedule:
|
||||
payload = {key: source.get(key) for key in SCHEDULE_KEYS if source.get(key) is not None}
|
||||
try:
|
||||
return Schedule(**payload)
|
||||
except ValidationError as exc:
|
||||
raise ToolInputError(f"Invalid schedule: {exc.errors()[0]['msg']}") from exc
|
||||
except ValueError as exc:
|
||||
raise ToolInputError(f"Invalid schedule: {exc}") from exc
|
||||
|
||||
def _require_uid(self, arguments: dict[str, Any]) -> str:
|
||||
uid = str(arguments.get("uid", "")).strip()
|
||||
if not uid:
|
||||
raise ToolInputError("This task tool requires a uid.")
|
||||
return uid
|
||||
|
||||
def _require_task(self, arguments: dict[str, Any]) -> dict[str, Any]:
|
||||
uid = self._require_uid(arguments)
|
||||
row = self._store.get(uid)
|
||||
if row is None:
|
||||
raise ToolInputError(f"No task found with uid {uid}.")
|
||||
return row
|
||||
|
||||
|
||||
def compute_followup(row: dict[str, Any], reference: Any) -> tuple[dict[str, Any], bool]:
|
||||
run_count = int(row.get("run_count") or 0) + 1
|
||||
max_runs = row.get("max_runs")
|
||||
upcoming = next_run(row.get("kind"), row.get("every_seconds"), row.get("cron"), reference)
|
||||
|
||||
changes: dict[str, Any] = {"run_count": run_count, "last_run_at": to_iso(reference)}
|
||||
if upcoming is None or (max_runs is not None and run_count >= int(max_runs)):
|
||||
changes["status"] = "done"
|
||||
changes["enabled"] = False
|
||||
changes["next_run_at"] = None
|
||||
else:
|
||||
changes["status"] = "pending"
|
||||
changes["next_run_at"] = to_iso(upcoming)
|
||||
return changes, changes["status"] == "done"
|
||||
@@ -0,0 +1,147 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Literal, Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
ISO_FORMAT = "%Y-%m-%dT%H:%M:%S"
|
||||
CRON_MINUTES_LIMIT = 525600 * 4
|
||||
|
||||
ScheduleKind = Literal["once", "interval", "cron"]
|
||||
|
||||
|
||||
def now_utc() -> datetime:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0)
|
||||
|
||||
|
||||
def to_iso(moment: datetime) -> str:
|
||||
if moment.tzinfo is None:
|
||||
moment = moment.replace(tzinfo=timezone.utc)
|
||||
return moment.astimezone(timezone.utc).strftime(ISO_FORMAT)
|
||||
|
||||
|
||||
def from_iso(value: str) -> datetime:
|
||||
return datetime.strptime(value, ISO_FORMAT).replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _parse_field(field: str, low: int, high: int) -> set[int]:
|
||||
values: set[int] = set()
|
||||
for part in field.split(","):
|
||||
token, _, step_text = part.partition("/")
|
||||
step = int(step_text) if step_text else 1
|
||||
if step < 1:
|
||||
raise ValueError(f"Invalid step in cron field: {part}")
|
||||
if token == "*":
|
||||
start, end = low, high
|
||||
elif "-" in token:
|
||||
start_text, end_text = token.split("-", 1)
|
||||
start, end = int(start_text), int(end_text)
|
||||
else:
|
||||
start = end = int(token)
|
||||
if start < low or end > high or start > end:
|
||||
raise ValueError(f"Cron value out of range: {part}")
|
||||
values.update(range(start, end + 1, step))
|
||||
return values
|
||||
|
||||
|
||||
def cron_next(expr: str, after: datetime) -> datetime:
|
||||
fields = expr.split()
|
||||
if len(fields) != 5:
|
||||
raise ValueError("Cron expression must have 5 fields: minute hour dom month dow")
|
||||
minutes = _parse_field(fields[0], 0, 59)
|
||||
hours = _parse_field(fields[1], 0, 23)
|
||||
days = _parse_field(fields[2], 1, 31)
|
||||
months = _parse_field(fields[3], 1, 12)
|
||||
weekdays_raw = _parse_field(fields[4], 0, 7)
|
||||
weekdays = {0 if value == 7 else value for value in weekdays_raw}
|
||||
day_restricted = fields[2] != "*"
|
||||
weekday_restricted = fields[4] != "*"
|
||||
|
||||
candidate = after.replace(second=0, microsecond=0) + timedelta(minutes=1)
|
||||
for _ in range(CRON_MINUTES_LIMIT):
|
||||
if (
|
||||
candidate.minute in minutes
|
||||
and candidate.hour in hours
|
||||
and candidate.month in months
|
||||
):
|
||||
cron_weekday = (candidate.weekday() + 1) % 7
|
||||
day_ok = candidate.day in days
|
||||
weekday_ok = cron_weekday in weekdays
|
||||
if day_restricted and weekday_restricted:
|
||||
match_day = day_ok or weekday_ok
|
||||
elif day_restricted:
|
||||
match_day = day_ok
|
||||
elif weekday_restricted:
|
||||
match_day = weekday_ok
|
||||
else:
|
||||
match_day = True
|
||||
if match_day:
|
||||
return candidate
|
||||
candidate += timedelta(minutes=1)
|
||||
raise ValueError("No cron match found within four years")
|
||||
|
||||
|
||||
class Schedule(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
kind: ScheduleKind
|
||||
run_at: Optional[datetime] = None
|
||||
delay_seconds: Optional[int] = Field(default=None, ge=1)
|
||||
every_seconds: Optional[int] = Field(default=None, ge=1)
|
||||
start_at: Optional[datetime] = None
|
||||
cron: Optional[str] = None
|
||||
max_runs: Optional[int] = Field(default=None, ge=1)
|
||||
|
||||
@field_validator("run_at", "start_at")
|
||||
@classmethod
|
||||
def _ensure_utc(cls, value: Optional[datetime]) -> Optional[datetime]:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_kind(self) -> "Schedule":
|
||||
if self.kind == "once" and self.run_at is None and self.delay_seconds is None:
|
||||
raise ValueError("kind=once requires run_at or delay_seconds")
|
||||
if self.kind == "interval" and self.every_seconds is None:
|
||||
raise ValueError("kind=interval requires every_seconds")
|
||||
if self.kind == "cron":
|
||||
if not self.cron:
|
||||
raise ValueError("kind=cron requires a cron expression")
|
||||
cron_next(self.cron, now_utc())
|
||||
return self
|
||||
|
||||
def first_run(self, reference: datetime) -> datetime:
|
||||
if self.kind == "once":
|
||||
if self.run_at is not None:
|
||||
return self.run_at
|
||||
return reference + timedelta(seconds=self.delay_seconds or 0)
|
||||
if self.kind == "interval":
|
||||
if self.start_at is not None:
|
||||
return self.start_at
|
||||
return reference + timedelta(seconds=self.every_seconds or 0)
|
||||
return cron_next(self.cron or "", reference)
|
||||
|
||||
def columns(self) -> dict[str, object]:
|
||||
return {
|
||||
"kind": self.kind,
|
||||
"run_at": to_iso(self.run_at) if self.run_at else None,
|
||||
"delay_seconds": self.delay_seconds,
|
||||
"every_seconds": self.every_seconds,
|
||||
"start_at": to_iso(self.start_at) if self.start_at else None,
|
||||
"cron": self.cron,
|
||||
"max_runs": self.max_runs,
|
||||
}
|
||||
|
||||
|
||||
def next_run(kind: str, every_seconds: Optional[int], cron: Optional[str], reference: datetime) -> Optional[datetime]:
|
||||
if kind == "interval" and every_seconds:
|
||||
return reference + timedelta(seconds=int(every_seconds))
|
||||
if kind == "cron" and cron:
|
||||
return cron_next(str(cron), reference)
|
||||
return None
|
||||
@@ -0,0 +1,86 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any, Awaitable, Callable
|
||||
|
||||
from .controller import compute_followup
|
||||
from .schedule import now_utc, to_iso
|
||||
from .store import TaskStore
|
||||
|
||||
logger = logging.getLogger("devii.tasks.scheduler")
|
||||
|
||||
PromptExecutor = Callable[[str], Awaitable[str]]
|
||||
EventCallback = Callable[[str, dict[str, Any], str], None]
|
||||
|
||||
DEFAULT_TICK_SECONDS = 1.0
|
||||
|
||||
|
||||
class Scheduler:
|
||||
def __init__(
|
||||
self,
|
||||
store: TaskStore,
|
||||
executor: PromptExecutor,
|
||||
on_event: EventCallback,
|
||||
tick_seconds: float = DEFAULT_TICK_SECONDS,
|
||||
) -> None:
|
||||
self._store = store
|
||||
self._executor = executor
|
||||
self._on_event = on_event
|
||||
self._tick_seconds = tick_seconds
|
||||
self._loop_task: asyncio.Task[None] | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
if self._loop_task is None:
|
||||
self._store.recover_running()
|
||||
self._loop_task = asyncio.create_task(self._loop())
|
||||
logger.info("Scheduler started")
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._loop_task is None:
|
||||
return
|
||||
self._loop_task.cancel()
|
||||
try:
|
||||
await self._loop_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._loop_task = None
|
||||
logger.info("Scheduler stopped")
|
||||
|
||||
async def _loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self._tick()
|
||||
except Exception: # noqa: BLE001 - the scheduler must never die
|
||||
logger.exception("Scheduler tick failed")
|
||||
await asyncio.sleep(self._tick_seconds)
|
||||
|
||||
async def _tick(self) -> None:
|
||||
now = now_utc()
|
||||
for row in self._store.due(to_iso(now)):
|
||||
await self._execute(row)
|
||||
|
||||
async def _execute(self, row: dict[str, Any]) -> None:
|
||||
uid = row["uid"]
|
||||
self._store.update(uid, {"status": "running"})
|
||||
self._on_event("start", row, "")
|
||||
logger.info("Executing task uid=%s", uid)
|
||||
|
||||
try:
|
||||
result = await self._executor(row["prompt"])
|
||||
changes, finished = compute_followup(row, now_utc())
|
||||
changes["last_result"] = result
|
||||
changes["last_error"] = None
|
||||
except Exception as exc: # noqa: BLE001 - surfaced into the task record
|
||||
logger.exception("Task uid=%s crashed", uid)
|
||||
self._store.update(
|
||||
uid,
|
||||
{"status": "error", "last_error": str(exc), "last_run_at": to_iso(now_utc())},
|
||||
)
|
||||
self._on_event("error", row, str(exc))
|
||||
return
|
||||
|
||||
self._store.update(uid, changes)
|
||||
self._on_event("done" if not finished else "finished", row, result)
|
||||
@@ -0,0 +1,80 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import dataset
|
||||
|
||||
logger = logging.getLogger("devii.tasks.store")
|
||||
|
||||
TABLE = "devii_tasks"
|
||||
INDEXED_COLUMNS = (["owner_kind", "owner_id"], ["uid"], ["enabled"], ["next_run_at"], ["status"])
|
||||
|
||||
|
||||
def memory_db() -> Any:
|
||||
return dataset.connect("sqlite:///:memory:")
|
||||
|
||||
|
||||
class TaskStore:
|
||||
def __init__(self, db: Any, owner_kind: str, owner_id: str) -> None:
|
||||
self._db = db
|
||||
self._owner_kind = owner_kind
|
||||
self._owner_id = owner_id
|
||||
self._ensure_indexes()
|
||||
|
||||
def _ensure_indexes(self) -> None:
|
||||
if TABLE not in self._db.tables:
|
||||
return
|
||||
table = self._db[TABLE]
|
||||
for columns in INDEXED_COLUMNS:
|
||||
table.create_index(columns)
|
||||
|
||||
@property
|
||||
def _table(self) -> Any:
|
||||
return self._db[TABLE]
|
||||
|
||||
@property
|
||||
def _scope(self) -> dict[str, str]:
|
||||
return {"owner_kind": self._owner_kind, "owner_id": self._owner_id}
|
||||
|
||||
def create(self, record: dict[str, Any]) -> None:
|
||||
self._table.insert({**record, **self._scope})
|
||||
logger.info("Task created uid=%s owner=%s/%s", record.get("uid"), self._owner_kind, self._owner_id)
|
||||
|
||||
def get(self, uid: str) -> dict[str, Any] | None:
|
||||
return self._table.find_one(uid=uid, **self._scope)
|
||||
|
||||
def list(self, enabled_only: bool = False, status: str | None = None) -> list[dict[str, Any]]:
|
||||
criteria: dict[str, Any] = dict(self._scope)
|
||||
if enabled_only:
|
||||
criteria["enabled"] = True
|
||||
if status:
|
||||
criteria["status"] = status
|
||||
return list(self._table.find(**criteria))
|
||||
|
||||
def update(self, uid: str, changes: dict[str, Any]) -> None:
|
||||
changes = {**changes, "uid": uid, **self._scope}
|
||||
self._table.update(changes, ["uid", "owner_kind", "owner_id"])
|
||||
logger.debug("Task updated uid=%s changes=%s", uid, list(changes))
|
||||
|
||||
def delete(self, uid: str) -> bool:
|
||||
deleted = self._table.delete(uid=uid, **self._scope)
|
||||
logger.info("Task deleted uid=%s ok=%s", uid, deleted)
|
||||
return bool(deleted)
|
||||
|
||||
def recover_running(self) -> int:
|
||||
if TABLE not in self._db.tables:
|
||||
return 0
|
||||
stuck = list(self._table.find(status="running", **self._scope))
|
||||
for row in stuck:
|
||||
self._table.update({"uid": row["uid"], "status": "pending", **self._scope},
|
||||
["uid", "owner_kind", "owner_id"])
|
||||
if stuck:
|
||||
logger.info("Recovered %d task(s) stuck in running", len(stuck))
|
||||
return len(stuck)
|
||||
|
||||
def due(self, now_iso: str) -> list[dict[str, Any]]:
|
||||
rows = self._table.find(enabled=True, status="pending", **self._scope)
|
||||
return [row for row in rows if row.get("next_run_at") and row["next_run_at"] <= now_iso]
|
||||
@@ -0,0 +1,88 @@
|
||||
# retoor <retoor@molodetz.nl>
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
|
||||
import httpx
|
||||
|
||||
SCRIPT_STYLE = re.compile(r"<(script|style)[^>]*>.*?</\1>", re.IGNORECASE | re.DOTALL)
|
||||
ANCHOR = re.compile(
|
||||
r"<a\b[^>]*?\bhref\s*=\s*(?:\"([^\"]*)\"|'([^']*)'|([^\s>]+))[^>]*>(.*?)</a>",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
TAG = re.compile(r"<[^>]+>")
|
||||
BLANK_LINES = re.compile(r"\n\s*\n\s*\n+")
|
||||
TRAILING_SPACE = re.compile(r"[ \t]+\n")
|
||||
SKIP_HREF_PREFIXES = ("#", "javascript:")
|
||||
|
||||
HIDDEN = "[hidden]"
|
||||
REDACT_FIELD_KEYS = frozenset({
|
||||
"gateway_upstream_url",
|
||||
"gateway_model",
|
||||
"gateway_vision_url",
|
||||
"gateway_vision_model",
|
||||
})
|
||||
REDACT_STAT_LABELS = frozenset({"Model"})
|
||||
|
||||
|
||||
def redact_backend(obj):
|
||||
if isinstance(obj, dict):
|
||||
if "value" in obj and obj.get("key") in REDACT_FIELD_KEYS:
|
||||
obj["value"] = HIDDEN
|
||||
if "value" in obj and obj.get("label") in REDACT_STAT_LABELS:
|
||||
obj["value"] = HIDDEN
|
||||
for value in obj.values():
|
||||
redact_backend(value)
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
redact_backend(item)
|
||||
return obj
|
||||
|
||||
|
||||
def _render_anchor(match: re.Match[str]) -> str:
|
||||
href = (match.group(1) or match.group(2) or match.group(3) or "").strip()
|
||||
text = html.unescape(TAG.sub(" ", match.group(4))).strip()
|
||||
if not href or href.startswith(SKIP_HREF_PREFIXES):
|
||||
return text
|
||||
if not text:
|
||||
return href
|
||||
return f"{text} ({href})"
|
||||
|
||||
|
||||
def html_to_text(markup: str) -> str:
|
||||
without_blocks = SCRIPT_STYLE.sub(" ", markup)
|
||||
with_links = ANCHOR.sub(_render_anchor, without_blocks)
|
||||
stripped = TAG.sub("\n", with_links)
|
||||
unescaped = html.unescape(stripped)
|
||||
collapsed = TRAILING_SPACE.sub("\n", unescaped)
|
||||
collapsed = BLANK_LINES.sub("\n\n", collapsed)
|
||||
lines = [line.strip() for line in collapsed.splitlines()]
|
||||
return "\n".join(line for line in lines if line).strip()
|
||||
|
||||
|
||||
def truncate(body: str, max_chars: int) -> str:
|
||||
if len(body) <= max_chars:
|
||||
return body
|
||||
return body[:max_chars] + f"\n...[truncated {len(body) - max_chars} chars]"
|
||||
|
||||
|
||||
def format_response(response: httpx.Response) -> str:
|
||||
content_type = response.headers.get("content-type", "")
|
||||
summary = {"status": response.status_code, "url": str(response.url)}
|
||||
|
||||
if "application/json" in content_type:
|
||||
try:
|
||||
summary["body"] = redact_backend(response.json())
|
||||
except ValueError:
|
||||
summary["body"] = response.text
|
||||
return json.dumps(summary, ensure_ascii=False)
|
||||
|
||||
if content_type.startswith("text/") or "html" in content_type or "xml" in content_type:
|
||||
summary["body"] = html_to_text(response.text)
|
||||
return json.dumps(summary, ensure_ascii=False)
|
||||
|
||||
summary["body"] = f"binary response ({len(response.content)} bytes, {content_type})"
|
||||
return json.dumps(summary, ensure_ascii=False)
|
||||
@@ -1,12 +1,21 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from devplacepy.database import get_setting, set_setting
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ServiceManager:
|
||||
def __init__(self):
|
||||
self._services = {}
|
||||
self._owns_lock = False
|
||||
|
||||
def set_lock_owner(self, owns: bool) -> None:
|
||||
self._owns_lock = owns
|
||||
|
||||
def owns_lock(self) -> bool:
|
||||
return self._owns_lock
|
||||
|
||||
def register(self, service) -> None:
|
||||
self._services[service.name] = service
|
||||
@@ -15,30 +24,67 @@ class ServiceManager:
|
||||
def get_service(self, name: str):
|
||||
return self._services.get(name)
|
||||
|
||||
def list_services(self) -> list[dict]:
|
||||
return [
|
||||
{
|
||||
"name": svc.name,
|
||||
"status": svc.status,
|
||||
"interval_seconds": svc.interval_seconds,
|
||||
"last_run": svc.last_run,
|
||||
"next_run": svc.next_run,
|
||||
"uptime": svc.uptime,
|
||||
"log_buffer": list(svc.log_buffer),
|
||||
}
|
||||
for svc in self._services.values()
|
||||
]
|
||||
def describe_all(self) -> list[dict]:
|
||||
return [svc.describe() for svc in self._services.values()]
|
||||
|
||||
async def start_all(self) -> None:
|
||||
for name, svc in self._services.items():
|
||||
await svc.start()
|
||||
logger.info(f"Started service: {name}")
|
||||
def set_enabled(self, name: str, enabled: bool) -> bool:
|
||||
svc = self.get_service(name)
|
||||
if svc is None:
|
||||
return False
|
||||
set_setting(svc.enabled_key, "1" if enabled else "0")
|
||||
logger.info(f"Service {name} enabled={enabled}")
|
||||
return True
|
||||
|
||||
async def stop_all(self) -> None:
|
||||
def send_command(self, name: str, verb: str) -> bool:
|
||||
svc = self.get_service(name)
|
||||
if svc is None:
|
||||
return False
|
||||
raw = get_setting(svc.command_key, "")
|
||||
counter = 0
|
||||
if raw and ":" in raw:
|
||||
try:
|
||||
counter = int(raw.split(":", 1)[1])
|
||||
except ValueError:
|
||||
counter = 0
|
||||
set_setting(svc.command_key, f"{verb}:{counter + 1}")
|
||||
logger.info(f"Service {name} command={verb}")
|
||||
return True
|
||||
|
||||
def save_config(self, name: str, form: dict) -> dict | None:
|
||||
svc = self.get_service(name)
|
||||
if svc is None:
|
||||
return None
|
||||
errors = {}
|
||||
valid = {}
|
||||
for field in svc.all_fields():
|
||||
if field.key not in form:
|
||||
continue
|
||||
raw = form[field.key]
|
||||
if field.secret and raw == "":
|
||||
continue
|
||||
try:
|
||||
valid[field.key] = field.to_storage(field.coerce(raw))
|
||||
except ValueError as e:
|
||||
errors[field.key] = str(e)
|
||||
if errors:
|
||||
return {"ok": False, "errors": errors}
|
||||
for key, value in valid.items():
|
||||
set_setting(key, value)
|
||||
logger.info(f"Service {name} config saved ({len(valid)} fields)")
|
||||
return {"ok": True, "errors": {}}
|
||||
|
||||
def supervise(self) -> None:
|
||||
for svc in self._services.values():
|
||||
svc.start_supervisor()
|
||||
logger.info(f"Supervising service: {svc.name}")
|
||||
|
||||
async def shutdown_all(self) -> None:
|
||||
tasks = []
|
||||
for name, svc in self._services.items():
|
||||
tasks.append(svc.stop())
|
||||
logger.info(f"Stopping service: {name}")
|
||||
for svc in self._services.values():
|
||||
svc.request_shutdown()
|
||||
if svc._task is not None:
|
||||
svc._task.cancel()
|
||||
tasks.append(svc._task)
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
logger.info("All services stopped")
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user