Files
devplacepy/devplacepy/services/devii/agents/evals.py
T
retoor 3bfde87fdb Introduce Devii agents subsystem with supervisor, policy, and evals
- Add agents package: controller, runner, store, tools, actions, evals, policy
- Wire agents into Devii service, session, registry, CLI, and dispatcher
- Extend database schema and config for agent persistence
- Add news and telegram bridge updates plus landing UI and docs
- Cover agents, news, and telegram flows with unit tests
2026-10-26 02:54:44 +01:00

170 lines
6.5 KiB
Python

# retoor <retoor@molodetz.nl>
from __future__ import annotations
from typing import Any
EVALS: tuple[dict[str, Any], ...] = (
{
"uid": "eval-direct-quota",
"channel": "main",
"prompt": "How much of my AI quota have I used today?",
"expect": "direct",
"end_state": "Reply states the quota percentage and turn count, no dollar figure.",
"tools": ("usage_quota",),
},
{
"uid": "eval-direct-time",
"channel": "main",
"prompt": "What time is it in UTC right now, exactly?",
"expect": "direct",
"end_state": "Reply states the current UTC time to the second.",
"tools": ("current_time",),
},
{
"uid": "eval-direct-docs",
"channel": "main",
"prompt": "How do I pair Telegram on this site?",
"expect": "direct",
"end_state": "Reply links the documented pairing page.",
"tools": ("search_docs",),
},
{
"uid": "eval-direct-site-stats",
"channel": "main",
"prompt": "How many members and posts does this site have?",
"expect": "direct",
"end_state": "Reply states member and content totals from one aggregate call.",
"tools": ("site_analytics",),
},
{
"uid": "eval-direct-lesson",
"channel": "main",
"prompt": "Remember that I prefer concise replies with no preamble.",
"expect": "direct",
"end_state": "A behavior rule persists and the next reply is concise.",
"tools": ("update_behavior",),
},
{
"uid": "eval-task-reminder",
"channel": "main",
"prompt": "Remind me in 60 seconds to drink water.",
"expect": "direct",
"end_state": "A notify task exists and fires once with an in-app notification.",
"tools": ("create_task", "list_tasks"),
},
{
"uid": "eval-task-recurring",
"channel": "main",
"prompt": "Every morning at 8 post a good-morning message to the general feed.",
"expect": "direct",
"end_state": "A cron task exists with max_runs and expiry bounds set.",
"tools": ("create_task", "get_task"),
},
{
"uid": "eval-telegram-quick",
"channel": "telegram",
"prompt": "What can you help me with?",
"expect": "direct",
"end_state": "One Telegram message answers; placeholder edits precede it.",
"tools": (),
},
{
"uid": "eval-telegram-quota",
"channel": "telegram",
"prompt": "Have I used much AI quota today?",
"expect": "direct",
"end_state": "One Telegram message states the quota percentage only.",
"tools": ("usage_quota",),
},
{
"uid": "eval-telegram-photo",
"channel": "telegram",
"prompt": "[photo of a cat] What is in this picture?",
"expect": "direct",
"end_state": "One Telegram message describes the image; no base64 persists.",
"tools": (),
},
{
"uid": "eval-research-web",
"channel": "main",
"prompt": "Compare the three most discussed Python web frameworks this week and summarize the tradeoffs.",
"expect": "research",
"end_state": "Reply holds one comparison table with source links; user could chat meanwhile.",
"tools": ("agent_start", "agent_status", "agent_result", "rsearch"),
},
{
"uid": "eval-research-platform",
"channel": "main",
"prompt": "What do people outside this site say about HTMX versus Alpine.js?",
"expect": "research",
"end_state": "Reply summarizes outside discussion with per-platform reach notes.",
"tools": ("agent_start", "agent_result", "social_search"),
},
{
"uid": "eval-files-build",
"channel": "main",
"prompt": "In my project demo-site, add a contact page with a form and wire it into the nav.",
"expect": "files",
"end_state": "Contact page exists in the project, nav links it, no other file touched.",
"tools": ("agent_start", "agent_result", "project_write_file"),
},
{
"uid": "eval-files-edit",
"channel": "main",
"prompt": "In my project demo-site, rename the footer copyright holder across every page.",
"expect": "files",
"end_state": "Every page shows the new holder; nothing else changed.",
"tools": ("agent_start", "agent_result", "project_replace_lines"),
},
{
"uid": "eval-planning-report",
"channel": "main",
"prompt": "Audit who changed admin settings in the last 7 days and report it.",
"expect": "planning-report",
"end_state": "Reply lists the changes with actor and time, grounded in docs and audit.",
"tools": ("agent_start", "agent_result", "audit_log", "search_docs"),
},
{
"uid": "eval-email-triage",
"channel": "main",
"prompt": "Triage my unread inbox mail from this week into urgent and bulk.",
"expect": "email-triage",
"end_state": "A disposition list exists; nothing sent, nothing deleted.",
"tools": ("agent_start", "agent_result", "email_search", "email_mark"),
},
{
"uid": "eval-containers-status",
"channel": "main",
"prompt": "Check every container instance on my projects and report which ones are unhealthy.",
"expect": "containers",
"end_state": "Reply lists each instance with health verdict and log evidence.",
"tools": ("agent_start", "agent_result", "container_list_instances"),
},
{
"uid": "eval-long-mutation",
"channel": "main",
"prompt": "Publish my draft posts about gardening that are older than a week.",
"expect": "long-mutation",
"end_state": "Exactly the matching drafts are published; the rest untouched.",
"tools": ("agent_start", "agent_result"),
},
{
"uid": "eval-needs-user",
"channel": "main",
"prompt": "Research newsletter platforms and recommend one, asking me about budget first.",
"expect": "research",
"end_state": "Run pauses as needs_user, orchestrator asks, answer resumes it to done.",
"tools": ("agent_start", "agent_status", "agent_result", "ui_prompt"),
},
{
"uid": "eval-cancel",
"channel": "main",
"prompt": "Start a deep comparison of static site generators, then cancel it right away.",
"expect": "research",
"end_state": "Run ends cancelled; ledger holds only the spend before cancel.",
"tools": ("agent_start", "agent_cancel", "agent_status"),
},
)