# retoor from __future__ import annotations from typing import Any EVALS: tuple[dict[str, Any], ...] = ( { "uid": "eval-direct-quota", "channel": "main", "prompt": "How much of my AI quota have I used today?", "expect": "direct", "end_state": "Reply states the quota percentage and turn count, no dollar figure.", "tools": ("usage_quota",), }, { "uid": "eval-direct-time", "channel": "main", "prompt": "What time is it in UTC right now, exactly?", "expect": "direct", "end_state": "Reply states the current UTC time to the second.", "tools": ("current_time",), }, { "uid": "eval-direct-docs", "channel": "main", "prompt": "How do I pair Telegram on this site?", "expect": "direct", "end_state": "Reply links the documented pairing page.", "tools": ("search_docs",), }, { "uid": "eval-direct-site-stats", "channel": "main", "prompt": "How many members and posts does this site have?", "expect": "direct", "end_state": "Reply states member and content totals from one aggregate call.", "tools": ("site_analytics",), }, { "uid": "eval-direct-lesson", "channel": "main", "prompt": "Remember that I prefer concise replies with no preamble.", "expect": "direct", "end_state": "A behavior rule persists and the next reply is concise.", "tools": ("update_behavior",), }, { "uid": "eval-task-reminder", "channel": "main", "prompt": "Remind me in 60 seconds to drink water.", "expect": "direct", "end_state": "A notify task exists and fires once with an in-app notification.", "tools": ("create_task", "list_tasks"), }, { "uid": "eval-task-recurring", "channel": "main", "prompt": "Every morning at 8 post a good-morning message to the general feed.", "expect": "direct", "end_state": "A cron task exists with max_runs and expiry bounds set.", "tools": ("create_task", "get_task"), }, { "uid": "eval-telegram-quick", "channel": "telegram", "prompt": "What can you help me with?", "expect": "direct", "end_state": "One Telegram message answers; placeholder edits precede it.", "tools": (), }, { "uid": "eval-telegram-quota", "channel": "telegram", "prompt": "Have I used much AI quota today?", "expect": "direct", "end_state": "One Telegram message states the quota percentage only.", "tools": ("usage_quota",), }, { "uid": "eval-telegram-photo", "channel": "telegram", "prompt": "[photo of a cat] What is in this picture?", "expect": "direct", "end_state": "One Telegram message describes the image; no base64 persists.", "tools": (), }, { "uid": "eval-research-web", "channel": "main", "prompt": "Compare the three most discussed Python web frameworks this week and summarize the tradeoffs.", "expect": "research", "end_state": "Reply holds one comparison table with source links; user could chat meanwhile.", "tools": ("agent_start", "agent_status", "agent_result", "rsearch"), }, { "uid": "eval-research-platform", "channel": "main", "prompt": "What do people outside this site say about HTMX versus Alpine.js?", "expect": "research", "end_state": "Reply summarizes outside discussion with per-platform reach notes.", "tools": ("agent_start", "agent_result", "social_search"), }, { "uid": "eval-files-build", "channel": "main", "prompt": "In my project demo-site, add a contact page with a form and wire it into the nav.", "expect": "files", "end_state": "Contact page exists in the project, nav links it, no other file touched.", "tools": ("agent_start", "agent_result", "project_write_file"), }, { "uid": "eval-files-edit", "channel": "main", "prompt": "In my project demo-site, rename the footer copyright holder across every page.", "expect": "files", "end_state": "Every page shows the new holder; nothing else changed.", "tools": ("agent_start", "agent_result", "project_replace_lines"), }, { "uid": "eval-planning-report", "channel": "main", "prompt": "Audit who changed admin settings in the last 7 days and report it.", "expect": "planning-report", "end_state": "Reply lists the changes with actor and time, grounded in docs and audit.", "tools": ("agent_start", "agent_result", "audit_log", "search_docs"), }, { "uid": "eval-email-triage", "channel": "main", "prompt": "Triage my unread inbox mail from this week into urgent and bulk.", "expect": "email-triage", "end_state": "A disposition list exists; nothing sent, nothing deleted.", "tools": ("agent_start", "agent_result", "email_search", "email_mark"), }, { "uid": "eval-containers-status", "channel": "main", "prompt": "Check every container instance on my projects and report which ones are unhealthy.", "expect": "containers", "end_state": "Reply lists each instance with health verdict and log evidence.", "tools": ("agent_start", "agent_result", "container_list_instances"), }, { "uid": "eval-long-mutation", "channel": "main", "prompt": "Publish my draft posts about gardening that are older than a week.", "expect": "long-mutation", "end_state": "Exactly the matching drafts are published; the rest untouched.", "tools": ("agent_start", "agent_result"), }, { "uid": "eval-needs-user", "channel": "main", "prompt": "Research newsletter platforms and recommend one, asking me about budget first.", "expect": "research", "end_state": "Run pauses as needs_user, orchestrator asks, answer resumes it to done.", "tools": ("agent_start", "agent_status", "agent_result", "ui_prompt"), }, { "uid": "eval-cancel", "channel": "main", "prompt": "Start a deep comparison of static site generators, then cancel it right away.", "expect": "research", "end_state": "Run ends cancelled; ledger holds only the spend before cancel.", "tools": ("agent_start", "agent_cancel", "agent_status"), }, )