47 lines
5.1 KiB
JSON
47 lines
5.1 KiB
JSON
{"cases": [
|
|
{"id": "boot.vault", "doc": "Fresh home, REPL started then EOF. Compares the seeded settings catalog rows, schema of every table, input_history. audit/records/chunks/tags/edges/shell_sessions are excluded because tai.py snapshots ITSELF into the vault on boot (path of tai.py, 50 KB source as a record); that is a deliberate per-binary difference, to be decided by the foundation agent, not hidden.",
|
|
"symbols": ["function:ensure_schema#schema", "function:boot_settings#seed"],
|
|
"steps": [{"argv": [], "stdin": "", "timeout": 60}],
|
|
"compare": {"stdout": false, "stderr": true, "exit": true, "vault": true,
|
|
"vault_ignore_tables": ["audit", "records", "chunks", "tags", "edges", "shell_sessions", "fts_docs"],
|
|
"vault_ignore_why": "tai.py self-snapshot (audit/records/chunks/tags/edges/fts_docs) and shell session tree (shell_sessions) are volatile per binary and host pid"},
|
|
"normalize": [
|
|
{"on": ["vault"], "pattern": "\"\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d(\\.\\d+)?\\+00:00\"", "replace": "\"<TS>\"", "why": "created/updated/ts columns hold wall-clock ISO timestamps"}
|
|
]},
|
|
{"id": "repl.help", "doc": "/help text in the REPL", "symbols": ["command:/help"],
|
|
"steps": [{"argv": [], "stdin": "/help\n", "timeout": 60}],
|
|
"normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner holds self-backup file name with timestamp, ctx id with pid, and live backend probe results (model counts, latency ms)"}]},
|
|
{"id": "repl.tools", "doc": "/tools lists core and lazy tool names", "symbols": ["command:/tools"],
|
|
"symbols_from_stdout": [{"prefix": "tool:", "pattern": "^ ([a-z_]+)$", "facet": "listed", "step": 0}],
|
|
"steps": [{"argv": [], "stdin": "/tools\n", "timeout": 60}],
|
|
"normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"}]},
|
|
{"id": "repl.tools_named", "doc": "/tools shell shows one tool", "symbols": ["command:/tools"],
|
|
"steps": [{"argv": [], "stdin": "/tools shell\n/tools nosuchtool\n", "timeout": 60}],
|
|
"normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"}]},
|
|
{"id": "repl.unknown_command", "doc": "unknown slash command message", "symbols": [],
|
|
"steps": [{"argv": [], "stdin": "/nonsense\n", "timeout": 60}],
|
|
"normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"}]},
|
|
{"id": "repl.profiles", "doc": "/profiles, /profile, /bots on a fresh home", "symbols": ["command:/profiles", "command:/profile", "command:/bots"],
|
|
"steps": [{"argv": [], "stdin": "/profiles\n/profile\n/bots\n/profile work\n/profiles\n", "timeout": 60}],
|
|
"compare": {"vault": true, "vault_ignore_tables": ["audit", "records", "chunks", "tags", "edges", "shell_sessions", "fts_docs"], "vault_ignore_why": "tai.py self-snapshot and shell session tree, see boot.vault"},
|
|
"normalize": [
|
|
{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"},
|
|
{"on": ["vault"], "pattern": "\"\\d{4}-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d(\\.\\d+)?\\+00:00\"", "replace": "\"<TS>\"", "why": "wall-clock timestamps"}
|
|
]},
|
|
{"id": "repl.skills_secret_schedules", "doc": "listing commands on an empty vault", "symbols": ["command:/skills", "command:/secret", "command:/schedules", "command:/tags", "command:/records"],
|
|
"steps": [{"argv": [], "stdin": "/skills\n/secret list\n/schedules\n/tags\n/records zebra\n", "timeout": 60}],
|
|
"normalize": [{"on": ["stdout"], "pattern": "\\A.*?type /help for commands\\n", "replace": "", "dotall": true, "why": "startup banner is volatile (see repl.help)"}]},
|
|
{"id": "persist.restart_history", "doc": "RESTART PERSISTENCE. Run 1 tells the agent a codeword (real LLM call), run 2 starts a new process with the SAME TAI_PLEXER_PANE and the same TAI_HOME and must resume the history: the events rows of run 1 survive and the model can answer from them. Expected to fail on detz until the foundation agent (store, history resume, plexer pane keyed session) lands. Probabilistic: real free model, 3 attempts, 1 must pass.",
|
|
"llm": true, "attempts": 3, "min_pass": 1,
|
|
"expected_fail": "needs the foundation agent: vault events persistence and history resume across process restarts; detz has no REPL/agent loop yet (2026-10-25)",
|
|
"symbols": ["function:main#restart", "class:Store#events"],
|
|
"steps": [
|
|
{"argv": [], "stdin": "My secret codeword is heliotrope-7314. Just acknowledge it in one short sentence.\n/exit\n", "env": {"TAI_PLEXER_PANE": "7"}, "timeout": 180},
|
|
{"argv": [], "stdin": "What is my secret codeword? Answer with the codeword only.\n/exit\n", "env": {"TAI_PLEXER_PANE": "7"}, "timeout": 180}
|
|
],
|
|
"checks": [
|
|
{"kind": "vault_sql", "sql": "select count(*) from events where text like '%heliotrope-7314%'", "op": ">=", "value": 2},
|
|
{"kind": "stdout_regex", "step": 1, "pattern": "heliotrope-7314"}
|
|
]}
|
|
]}
|