forked from retoor/devplacepy
- Add agents package: controller, runner, store, tools, actions, evals, policy - Wire agents into Devii service, session, registry, CLI, and dispatcher - Extend database schema and config for agent persistence - Add news and telegram bridge updates plus landing UI and docs - Cover agents, news, and telegram flows with unit tests
170 lines
6.5 KiB
Python
170 lines
6.5 KiB
Python
# retoor <retoor@molodetz.nl>
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
|
|
EVALS: tuple[dict[str, Any], ...] = (
|
|
{
|
|
"uid": "eval-direct-quota",
|
|
"channel": "main",
|
|
"prompt": "How much of my AI quota have I used today?",
|
|
"expect": "direct",
|
|
"end_state": "Reply states the quota percentage and turn count, no dollar figure.",
|
|
"tools": ("usage_quota",),
|
|
},
|
|
{
|
|
"uid": "eval-direct-time",
|
|
"channel": "main",
|
|
"prompt": "What time is it in UTC right now, exactly?",
|
|
"expect": "direct",
|
|
"end_state": "Reply states the current UTC time to the second.",
|
|
"tools": ("current_time",),
|
|
},
|
|
{
|
|
"uid": "eval-direct-docs",
|
|
"channel": "main",
|
|
"prompt": "How do I pair Telegram on this site?",
|
|
"expect": "direct",
|
|
"end_state": "Reply links the documented pairing page.",
|
|
"tools": ("search_docs",),
|
|
},
|
|
{
|
|
"uid": "eval-direct-site-stats",
|
|
"channel": "main",
|
|
"prompt": "How many members and posts does this site have?",
|
|
"expect": "direct",
|
|
"end_state": "Reply states member and content totals from one aggregate call.",
|
|
"tools": ("site_analytics",),
|
|
},
|
|
{
|
|
"uid": "eval-direct-lesson",
|
|
"channel": "main",
|
|
"prompt": "Remember that I prefer concise replies with no preamble.",
|
|
"expect": "direct",
|
|
"end_state": "A behavior rule persists and the next reply is concise.",
|
|
"tools": ("update_behavior",),
|
|
},
|
|
{
|
|
"uid": "eval-task-reminder",
|
|
"channel": "main",
|
|
"prompt": "Remind me in 60 seconds to drink water.",
|
|
"expect": "direct",
|
|
"end_state": "A notify task exists and fires once with an in-app notification.",
|
|
"tools": ("create_task", "list_tasks"),
|
|
},
|
|
{
|
|
"uid": "eval-task-recurring",
|
|
"channel": "main",
|
|
"prompt": "Every morning at 8 post a good-morning message to the general feed.",
|
|
"expect": "direct",
|
|
"end_state": "A cron task exists with max_runs and expiry bounds set.",
|
|
"tools": ("create_task", "get_task"),
|
|
},
|
|
{
|
|
"uid": "eval-telegram-quick",
|
|
"channel": "telegram",
|
|
"prompt": "What can you help me with?",
|
|
"expect": "direct",
|
|
"end_state": "One Telegram message answers; placeholder edits precede it.",
|
|
"tools": (),
|
|
},
|
|
{
|
|
"uid": "eval-telegram-quota",
|
|
"channel": "telegram",
|
|
"prompt": "Have I used much AI quota today?",
|
|
"expect": "direct",
|
|
"end_state": "One Telegram message states the quota percentage only.",
|
|
"tools": ("usage_quota",),
|
|
},
|
|
{
|
|
"uid": "eval-telegram-photo",
|
|
"channel": "telegram",
|
|
"prompt": "[photo of a cat] What is in this picture?",
|
|
"expect": "direct",
|
|
"end_state": "One Telegram message describes the image; no base64 persists.",
|
|
"tools": (),
|
|
},
|
|
{
|
|
"uid": "eval-research-web",
|
|
"channel": "main",
|
|
"prompt": "Compare the three most discussed Python web frameworks this week and summarize the tradeoffs.",
|
|
"expect": "research",
|
|
"end_state": "Reply holds one comparison table with source links; user could chat meanwhile.",
|
|
"tools": ("agent_start", "agent_status", "agent_result", "rsearch"),
|
|
},
|
|
{
|
|
"uid": "eval-research-platform",
|
|
"channel": "main",
|
|
"prompt": "What do people outside this site say about HTMX versus Alpine.js?",
|
|
"expect": "research",
|
|
"end_state": "Reply summarizes outside discussion with per-platform reach notes.",
|
|
"tools": ("agent_start", "agent_result", "social_search"),
|
|
},
|
|
{
|
|
"uid": "eval-files-build",
|
|
"channel": "main",
|
|
"prompt": "In my project demo-site, add a contact page with a form and wire it into the nav.",
|
|
"expect": "files",
|
|
"end_state": "Contact page exists in the project, nav links it, no other file touched.",
|
|
"tools": ("agent_start", "agent_result", "project_write_file"),
|
|
},
|
|
{
|
|
"uid": "eval-files-edit",
|
|
"channel": "main",
|
|
"prompt": "In my project demo-site, rename the footer copyright holder across every page.",
|
|
"expect": "files",
|
|
"end_state": "Every page shows the new holder; nothing else changed.",
|
|
"tools": ("agent_start", "agent_result", "project_replace_lines"),
|
|
},
|
|
{
|
|
"uid": "eval-planning-report",
|
|
"channel": "main",
|
|
"prompt": "Audit who changed admin settings in the last 7 days and report it.",
|
|
"expect": "planning-report",
|
|
"end_state": "Reply lists the changes with actor and time, grounded in docs and audit.",
|
|
"tools": ("agent_start", "agent_result", "audit_log", "search_docs"),
|
|
},
|
|
{
|
|
"uid": "eval-email-triage",
|
|
"channel": "main",
|
|
"prompt": "Triage my unread inbox mail from this week into urgent and bulk.",
|
|
"expect": "email-triage",
|
|
"end_state": "A disposition list exists; nothing sent, nothing deleted.",
|
|
"tools": ("agent_start", "agent_result", "email_search", "email_mark"),
|
|
},
|
|
{
|
|
"uid": "eval-containers-status",
|
|
"channel": "main",
|
|
"prompt": "Check every container instance on my projects and report which ones are unhealthy.",
|
|
"expect": "containers",
|
|
"end_state": "Reply lists each instance with health verdict and log evidence.",
|
|
"tools": ("agent_start", "agent_result", "container_list_instances"),
|
|
},
|
|
{
|
|
"uid": "eval-long-mutation",
|
|
"channel": "main",
|
|
"prompt": "Publish my draft posts about gardening that are older than a week.",
|
|
"expect": "long-mutation",
|
|
"end_state": "Exactly the matching drafts are published; the rest untouched.",
|
|
"tools": ("agent_start", "agent_result"),
|
|
},
|
|
{
|
|
"uid": "eval-needs-user",
|
|
"channel": "main",
|
|
"prompt": "Research newsletter platforms and recommend one, asking me about budget first.",
|
|
"expect": "research",
|
|
"end_state": "Run pauses as needs_user, orchestrator asks, answer resumes it to done.",
|
|
"tools": ("agent_start", "agent_status", "agent_result", "ui_prompt"),
|
|
},
|
|
{
|
|
"uid": "eval-cancel",
|
|
"channel": "main",
|
|
"prompt": "Start a deep comparison of static site generators, then cancel it right away.",
|
|
"expect": "research",
|
|
"end_state": "Run ends cancelled; ledger holds only the spend before cancel.",
|
|
"tools": ("agent_start", "agent_cancel", "agent_status"),
|
|
},
|
|
)
|