488 lines
28 KiB
Python
488 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""End-to-end differential test: the same scripted conversation runs through
|
|
tai.py (real Store, real Agent, patched urlopen) and through the C++ Agent
|
|
(tests/agent_e2e.cpp: scripted transport, in-memory store). Messages, HTTP
|
|
requests, store events, audit rows and tool results must be identical.
|
|
|
|
python3 tests/agent_e2e.py [--bin build/agent_e2e] [--filter NAME] [-v]
|
|
"""
|
|
import argparse
|
|
import http.client
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import types
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
|
sys.path.insert(0, ROOT)
|
|
sys.dont_write_bytecode = True
|
|
|
|
SAFE_PATH = "/usr/bin:/bin"
|
|
TZ = "Europe/Amsterdam"
|
|
|
|
|
|
def clean_env(home):
|
|
for key in list(os.environ):
|
|
if key.startswith("TAI_") or key in ("MUSE_BIN", "OPENCODE_BIN", "GROK_BIN", "CLAUDE_BIN", "CODEX_BIN", "GEMINI_BIN",
|
|
"OPENROUTER_API_KEY", "DEVPLACE_API_KEY", "TMUX", "TMUX_PANE", "HISTFILE"):
|
|
del os.environ[key]
|
|
os.environ["PATH"] = SAFE_PATH
|
|
os.environ["TZ"] = TZ
|
|
os.environ["TAI_HOME"] = home
|
|
os.environ["TAI_BACKENDS"] = "lab|http://fake.invalid/v1|k|m1"
|
|
os.environ["TAI_OPENROUTER"] = "0"
|
|
os.environ["TAI_POLLINATIONS"] = "0"
|
|
os.environ["TAI_OPENCODE"] = "0"
|
|
os.environ["HOME"] = home
|
|
|
|
|
|
import tai # noqa: E402
|
|
|
|
|
|
# ---------------------------------------------------------------- script builders
|
|
def say(text):
|
|
return {"kind": "stream", "events": [{"choices": [{"delta": {"content": text}}]}]}
|
|
|
|
|
|
def calls(*items, text=None):
|
|
events = []
|
|
if text:
|
|
events.append({"choices": [{"delta": {"content": text}}]})
|
|
for idx, (name, args, cid) in enumerate(items):
|
|
argtext = args if isinstance(args, str) else json.dumps(args)
|
|
half = len(argtext) // 2
|
|
events.append({"choices": [{"delta": {"tool_calls": [{"index": idx, "id": cid, "function": {"name": name, "arguments": argtext[:half]}}]}}]})
|
|
events.append({"choices": [{"delta": {"tool_calls": [{"index": idx, "function": {"arguments": argtext[half:]}}]}}]})
|
|
return {"kind": "stream", "events": events}
|
|
|
|
|
|
def single(text):
|
|
return {"kind": "json", "body": {"choices": [{"message": {"content": text}}]}}
|
|
|
|
|
|
def raw(body, content_type="text/plain"):
|
|
return {"kind": "raw", "body": body, "content_type": content_type}
|
|
|
|
|
|
def status(code, body=""):
|
|
return {"kind": "status", "status": code, "body": body}
|
|
|
|
|
|
def tc(name, args, cid):
|
|
return (name, args, cid)
|
|
|
|
|
|
# ---------------------------------------------------------------- python side
|
|
class ScriptExhausted(BaseException):
|
|
"""Not an Exception: the retry loops of ChatClient must not swallow it."""
|
|
|
|
|
|
class FakeResponse(object):
|
|
def __init__(self, body=b"", lines=None, status=200, content_type=""):
|
|
self._body = body
|
|
self._lines = lines
|
|
self.status = status
|
|
self.headers = {"Content-Type": content_type} if content_type else {}
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
def read(self, n=-1):
|
|
data = self._body if n is None or n < 0 else self._body[:n]
|
|
return data
|
|
|
|
def __iter__(self):
|
|
return iter(self._lines or [])
|
|
|
|
def close(self):
|
|
pass
|
|
|
|
|
|
def python_run(sc, workdir):
|
|
home = os.path.join(workdir, ".home")
|
|
clean_env(home)
|
|
if sc.get("retry_seconds"):
|
|
os.environ["TAI_RETRY_SECONDS"] = str(sc["retry_seconds"])
|
|
os.makedirs(home, exist_ok=True)
|
|
os.chdir(workdir)
|
|
tai.AGENTS.clear()
|
|
tai.AGENTS_NEXT[0] = 1
|
|
requests = []
|
|
queues = {"stream": list(sc.get("script", [])), "single": list(sc.get("single", []))}
|
|
|
|
def fake_urlopen(request, timeout=None):
|
|
url = request.full_url
|
|
if url.endswith("/models"):
|
|
raise urllib.error.URLError("fake")
|
|
body = None
|
|
if request.data is not None:
|
|
try:
|
|
body = json.loads(request.data)
|
|
except ValueError:
|
|
body = request.data.decode("utf-8", "replace")
|
|
requests.append({"method": request.get_method(), "url": url, "body": body})
|
|
streaming = isinstance(body, dict) and bool(body.get("stream"))
|
|
queue = queues["stream" if streaming else "single"]
|
|
if not queue:
|
|
raise ScriptExhausted()
|
|
item = queue.pop(0)
|
|
kind = item["kind"]
|
|
if kind == "transport_error":
|
|
raise urllib.error.URLError("scripted")
|
|
if kind == "status":
|
|
reason = http.client.responses.get(item["status"], "")
|
|
raise urllib.error.HTTPError(url, item["status"], reason, {}, io.BytesIO(item.get("body", "").encode("utf-8")))
|
|
content_type = item.get("content_type", "")
|
|
if kind == "json":
|
|
return FakeResponse(body=json.dumps(item["body"]).encode("utf-8"), content_type=content_type)
|
|
if kind == "raw":
|
|
return FakeResponse(body=item["body"].encode("utf-8"), content_type=content_type)
|
|
lines = []
|
|
for event in item["events"]:
|
|
lines.append(("data: %s\n" % json.dumps(event)).encode("utf-8"))
|
|
lines.append(b"\n")
|
|
lines.append(b"data: [DONE]\n")
|
|
return FakeResponse(lines=lines, content_type=content_type)
|
|
|
|
original = urllib.request.urlopen
|
|
original_sleep = tai.time.sleep
|
|
urllib.request.urlopen = fake_urlopen
|
|
tai.time.sleep = lambda seconds: None
|
|
try:
|
|
args = types.SimpleNamespace(yes=bool(sc.get("auto_approve")), profile=sc.get("profile", "default"), no_opencode=True)
|
|
config = tai.Config(args)
|
|
seal = tai.Seal(config.home, tai.resolve_passphrase()[0])
|
|
store = tai.Store(config, seal)
|
|
presets = sc.get("presets") or {}
|
|
for name, text in (presets.get("systems") or {}).items():
|
|
store.save_system(name, text)
|
|
for name, item in (presets.get("secrets") or {}).items():
|
|
store.save_secret(name, item["value"], {}, [], "default")
|
|
for name, item in (presets.get("bots") or {}).items():
|
|
store.save_bot_system("default", name, item["system"])
|
|
bots = store.load_bots("default")
|
|
bots[name] = {"nicknames": item["nicknames"], "created": "t", "updated": "t"}
|
|
store.save_bots("default", bots)
|
|
events = []
|
|
original_log = store.log_event
|
|
|
|
def wrapped(profile, role, kind, text, tags=()):
|
|
events.append([role, kind, text])
|
|
return original_log(profile, role, kind, text, tags)
|
|
|
|
store.log_event = wrapped
|
|
agent = tai.Agent(config, store, persist=sc.get("persist", True), quiet=True, yolo=bool(sc.get("yolo")))
|
|
if sc.get("deadline_expired"):
|
|
agent.deadline = 1.0
|
|
answers = sc.get("answers")
|
|
if isinstance(answers, list):
|
|
queue = list(answers)
|
|
agent.prompter = lambda kind, text, options: queue.pop(0) if queue else None
|
|
runner_script = sc.get("runner_script")
|
|
if isinstance(runner_script, list):
|
|
rq = list(runner_script)
|
|
agent.runner_override = lambda task, profile, timeout: rq.pop(0) if rq else "runner exhausted"
|
|
out = {}
|
|
try:
|
|
prompts = sc.get("prompts") or [sc.get("prompt")]
|
|
results = []
|
|
for prompt in prompts:
|
|
results.append(agent.run_turn(prompt, capture=True, max_steps=sc.get("max_steps", tai.MAX_STEPS)))
|
|
out["results"] = results
|
|
if sc.get("verify"):
|
|
text, ok = agent.verify_loop(sc["prompt"], results[-1])
|
|
out["verify"] = {"text": text, "ok": ok}
|
|
except ScriptExhausted:
|
|
out["exception"] = "script exhausted"
|
|
except Exception as exc: # noqa: BLE001
|
|
out["exception"] = str(exc)
|
|
out["messages"] = agent.messages
|
|
out["requests"] = requests
|
|
out["events"] = events
|
|
audits = []
|
|
for row in reversed(store.audit_history(None, None, 100)):
|
|
full = store.audit_get(row["id"])
|
|
audits.append({"actor": full["actor"], "action": full["action"], "path": full["path"], "message": full["message"],
|
|
"tags": full["tags"], "old_size": full["old_size"], "new_size": full["new_size"],
|
|
"old": full["old"], "new": full["new"]})
|
|
out["audits"] = audits
|
|
out["tool_calls"] = agent.tools.calls
|
|
out["yolo"] = agent.yolo
|
|
out["timed_out"] = agent.timed_out
|
|
out["system"] = agent.system_message
|
|
store.close()
|
|
return out
|
|
finally:
|
|
urllib.request.urlopen = original
|
|
tai.time.sleep = original_sleep
|
|
|
|
|
|
def cpp_run(sc, workdir, binary):
|
|
path = os.path.join(workdir, "..", "scenario.json")
|
|
with open(path, "w") as handle:
|
|
json.dump(sc, handle)
|
|
env = dict(os.environ)
|
|
env.update({"PATH": SAFE_PATH, "TZ": TZ, "HOME": workdir})
|
|
done = subprocess.run([binary, path, workdir], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, cwd=workdir)
|
|
if done.returncode != 0:
|
|
raise RuntimeError("binary failed: %s %s" % (done.returncode, done.stderr.decode("utf-8", "replace")[-1500:]))
|
|
return json.loads(done.stdout.decode("utf-8"))
|
|
|
|
|
|
def normalize(value):
|
|
text = json.dumps(value, sort_keys=True)
|
|
text = re.sub(r"mem:[0-9a-f]{16}", "mem:ID", text)
|
|
return json.loads(text)
|
|
|
|
|
|
# ---------------------------------------------------------------- scenarios
|
|
def build_scenarios():
|
|
big = "line of text number %d\n" * 1
|
|
scenarios = []
|
|
|
|
def add(name, prompt, script, **kw):
|
|
sc = {"name": name, "prompt": prompt, "script": script, "skill_catalog": tai.skill_catalog({})}
|
|
sc.update(kw)
|
|
scenarios.append(sc)
|
|
|
|
add("plain", "hi there", [say("Hello there")], auto_approve=True)
|
|
add("read_file", "read a.txt", [calls(tc("read_file", {"path": "a.txt"}, "c1")), say("done reading")],
|
|
files={"a.txt": "alpha\nbeta\n"}, auto_approve=True)
|
|
add("read_missing_and_big", "read", [calls(tc("read_file", {"path": "missing.txt"}, "c1"), tc("read_file", {"path": "big.txt"}, "c2")), say("ok")],
|
|
files={"big.txt": "x" * 200001}, auto_approve=True)
|
|
add("write_flow", "write", [
|
|
calls(tc("write_file", {"path": "b.txt", "content": "new"}, "c1")),
|
|
calls(tc("read_file", {"path": "b.txt"}, "c2")),
|
|
calls(tc("write_file", {"path": "b.txt", "content": "second\nversion"}, "c3"),
|
|
tc("write_file", {"path": "sub/dir/c.txt", "content": "nested"}, "c4"),
|
|
tc("write_file", {"path": "", "content": "x"}, "c5")),
|
|
calls(tc("edit_file", {"path": "b.txt", "find": "second", "replace": "third"}, "c6"),
|
|
tc("edit_file", {"path": "b.txt", "find": "zzz", "replace": "q"}, "c7"),
|
|
tc("edit_file", {"path": "b.txt", "find": "", "replace": "q"}, "c8"),
|
|
tc("edit_file", {"path": "nope.txt", "find": "a", "replace": "b"}, "c9")),
|
|
say("finished")], files={"b.txt": "old"}, auto_approve=True)
|
|
add("edit_not_unique", "edit", [
|
|
calls(tc("edit_file", {"path": "c.txt", "find": "x", "replace": "y"}, "c1")),
|
|
calls(tc("read_file", {"path": "c.txt"}, "c2")),
|
|
calls(tc("edit_file", {"path": "c.txt", "find": "x", "replace": "y"}, "c3"),
|
|
tc("edit_file", {"path": "c.txt", "find": "x x x", "replace": "é"}, "c4")),
|
|
say("end")], files={"c.txt": "x x x"}, auto_approve=True)
|
|
add("delete_restore", "delete and restore", [
|
|
calls(tc("delete_file", {"path": "d.txt"}, "c1")),
|
|
calls(tc("read_file", {"path": "d.txt"}, "c2")),
|
|
calls(tc("delete_file", {"path": "d.txt"}, "c3")),
|
|
calls(tc("audit", {}, "c4"), tc("restore", {"id": 1}, "c5"), tc("restore", {"id": 99}, "c6"), tc("restore", {"id": "x"}, "c7")),
|
|
calls(tc("read_file", {"path": "d.txt"}, "c8")),
|
|
say("restored")], files={"d.txt": "keep me"}, auto_approve=True)
|
|
add("shell_basic", "run things", [
|
|
calls(tc("shell", {"command": "echo hello"}, "c1"), tc("shell", {"command": "ls nonexistent_dir_xyz"}, "c2"),
|
|
tc("shell", {"command": "echo curl http://example.com"}, "c3"), tc("shell", {"command": " "}, "c4"),
|
|
tc("shell", {"command": "true"}, "c5"), tc("shell", {"command": "printf 'a\\nb\\n'; exit 3"}, "c6")),
|
|
calls(tc("shell", {"command": "echo x", "secrets": "oops"}, "c7"), tc("shell", {"command": "echo x", "secrets": ["nope"]}, "c8"),
|
|
tc("shell", {"command": "pwd", "workdir": "sub"}, "c9")),
|
|
say("done")], files={"sub/keep.txt": "k"}, auto_approve=True)
|
|
add("shell_long_output", "spill", [
|
|
calls(tc("shell", {"command": "yes line | head -n 3000"}, "c1")), say("done")], auto_approve=True)
|
|
add("unknown_and_bad_args", "odd calls", [
|
|
calls(tc("nonexistent_tool", {}, "c1"), tc("read_file", "this is not json", "c2"), tc("read_file", "[1, 2]", "c3"),
|
|
tc("list_secrets", "[1]", "c4"), tc("bad name!", {}, "c5")),
|
|
say("ok")], auto_approve=True)
|
|
add("loop_stop", "loop", [calls(tc("read_file", {"path": "a.txt"}, "c%d" % i)) for i in range(1, 8)] + [say("never")],
|
|
files={"a.txt": "same"}, auto_approve=True)
|
|
add("secrets", "secrets", [
|
|
calls(tc("store_secret", {"name": "api_key", "value": "supersecret123", "username": "me", "host": "h", "port": 22, "notes": "n", "tags": ["Prod"]}, "c1")),
|
|
calls(tc("list_secrets", {}, "c2"), tc("store_secret", {"name": "bad name", "value": "v"}, "c3"), tc("store_secret", {"name": "x", "value": ""}, "c4"),
|
|
tc("store_secret", {"name": "y", "value": "valid1", "port": "abc"}, "c5"), tc("store_secret", {"name": "y", "value": "valid1", "port": 70000}, "c6"),
|
|
tc("store_secret", {"name": "y", "value": "valid1", "expires": "bogus"}, "c7"), tc("store_secret", {"name": "y", "value": "valid1", "tags": "x"}, "c8")),
|
|
calls(tc("shell", {"command": "echo $TAI_SECRET_API_KEY", "secrets": ["api_key"]}, "c9")),
|
|
calls(tc("delete_secret", {"name": "api_key"}, "c10"), tc("delete_secret", {"name": "missing"}, "c11"), tc("list_secrets", {}, "c12")),
|
|
say("fin")], auto_approve=True)
|
|
add("schedules", "plan", [
|
|
calls(tc("schedule", {"prompt": "do it", "every": 120, "name": "nightly", "tags": ["Ops"]}, "c1"),
|
|
tc("schedule", {"prompt": "x", "at": "2026-12-24T10:00"}, "c2"),
|
|
tc("schedule", {"prompt": "x"}, "c3"), tc("schedule", {"prompt": "x", "every": 5}, "c4"),
|
|
tc("schedule", {"prompt": "x", "at": "garbage"}, "c5"), tc("schedule", {"prompt": "", "every": 100}, "c6"),
|
|
tc("schedule", {"prompt": "x", "every": "abc"}, "c7"), tc("schedule", {"prompt": "x", "every": 100, "profile": "other"}, "c8")),
|
|
calls(tc("schedules", {}, "c9"), tc("unschedule", {"id": 1}, "c10"), tc("unschedule", {"id": 77}, "c11"), tc("schedules", {}, "c12")),
|
|
say("planned")], auto_approve=True)
|
|
add("bots_and_mentions", "make a bot", [
|
|
calls(tc("create_bot", {"name": "helper-bot", "description": "I help.", "rules": "be brief", "nicknames": ["hb"]}, "c1"),
|
|
tc("create_bot", {"name": "main", "description": "x"}, "c2"), tc("create_bot", {"name": "Bad Name", "description": "x"}, "c3"),
|
|
tc("create_bot", {"name": "empty"}, "c4"), tc("create_bot", {"name": "helper-bot", "description": "dup"}, "c5")),
|
|
say("bot created"),
|
|
say("hello from the bot"),
|
|
say("plain third answer")],
|
|
prompts=["make a bot", "@hb say hello", "@nobody hi", "@ "], auto_approve=True)
|
|
add("compaction", "read all", [calls(tc("read_file", {"path": "f%d.txt" % i}, "c%d" % i)) for i in range(1, 14)] + [say("all read")],
|
|
files={"f%d.txt" % i: ("%d " % i) * 95000 for i in range(1, 14)}, single=[single("SUMMARY OF HISTORY")], auto_approve=True)
|
|
add("backend_fail", "will fail", [status(500, "upstream oops")], retry_seconds=1, auto_approve=True)
|
|
add("tools_400_fallback", "fallback", [status(400, "bad tools"), say("worked without tools")], auto_approve=True)
|
|
add("all_permanent_fail", "perm", [status(401, "no key")], auto_approve=True)
|
|
add("ask_user", "ask", [
|
|
calls(tc("ask_user", {"question": "Pick one", "options": ["A", "B", {"label": "C"}, "", 5]}, "c1"),
|
|
tc("ask_user", {"question": "Pick again", "options": ["A", "B"]}, "c2"),
|
|
tc("ask_user", {"question": "Free?"}, "c3"), tc("ask_user", {"question": ""}, "c4"), tc("ask_user", {"question": "q", "options": "no"}, "c5"),
|
|
tc("ask_user", {"question": "closed"}, "c6")),
|
|
say("asked")], answers=["2", "b", " hello world ", None], auto_approve=True)
|
|
add("approval_deny", "dangerous", [
|
|
calls(tc("shell", {"command": "rm -rf somewhere"}, "c1"), tc("shell", {"command": "ls"}, "c2")), say("stopped")],
|
|
answers=["Deny", "Deny"], files={"somewhere/x": "1"})
|
|
add("yolo_always", "yolo", [calls(tc("read_file", {"path": "a.txt"}, "c1")), say("ok")], yolo=True, files={"a.txt": "a"})
|
|
add("verify_ok", "do the thing", [say("I did it")], runner_script=['{"success": true, "failures": [], "commentary": "good"}'], verify=True, auto_approve=True)
|
|
add("verify_retry", "do the thing", [say("first try"), say("second try")],
|
|
runner_script=['{"success": false, "failures": [{"item": "x", "reason": "r", "proposal": "p"}]}', '{"success": true, "failures": []}'],
|
|
verify=True, auto_approve=True)
|
|
add("remember", "remember my name", [
|
|
calls(tc("remember", {"instruction": "The user is called Retoor and likes concise answers."}, "c1"), tc("remember", {"instruction": ""}, "c2")),
|
|
say("remembered")], single=[single("You are tai. The user is called Retoor and likes concise answers. Be professional.")],
|
|
auto_approve=True)
|
|
add("remember_fail", "remember", [calls(tc("remember", {"instruction": "x"}, "c1")), say("sorry")], single=[single("short")], auto_approve=True)
|
|
add("record_flow", "records", [
|
|
calls(tc("record_save", {"content": "alpha beta gamma", "title": "T1", "kind": "note", "tags": ["Zeta"]}, "c1"),
|
|
tc("record_save", {"content": "", "title": "e"}, "c2"), tc("record_save", {"content": "x", "kind": "weird"}, "c3"),
|
|
tc("record_save", {"content": "x", "tags": "no"}, "c4")),
|
|
say("saved")], auto_approve=True)
|
|
add("web_fetch", "fetch", [
|
|
calls(tc("web_fetch", {"url": "ftp://x"}, "c1"), tc("web_fetch", {"url": "http://example.com/a", "method": "TRACE"}, "c2"),
|
|
tc("web_fetch", {"url": "http://example.com/a"}, "c3"), tc("web_fetch", {"url": "http://example.com/b", "raw": True}, "c4"),
|
|
tc("web_fetch", {"url": "http://example.com/c", "headers": {"X-A": "b"}, "method": "post", "body": "{}"}, "c5"),
|
|
tc("web_fetch", {"url": "http://example.com/d", "headers": {"bad name": "b"}}, "c6"),
|
|
tc("web_fetch", {"url": "http://example.com/e", "timeout": "abc"}, "c7"), tc("web_fetch", {"url": "http://example.com/f"}, "c8"),
|
|
tc("web_fetch", {"url": "http://example.com/g"}, "c9")),
|
|
say("fetched")],
|
|
single=[raw("<html><head><style>p{}</style><script>var x=1;</script></head><body><h1>Title & more</h1><p>Para text</p></body></html>", "text/html; charset=utf-8"),
|
|
raw("<b>raw</b> text", "text/plain"), raw("posted", "application/json"),
|
|
status(404, "Not here"), status(500, ""), raw("", "text/html")], auto_approve=True)
|
|
add("web_search", "search", [
|
|
calls(tc("web_search", {"query": "rust news", "images": True}, "c1"), tc("web_search", {"query": ""}, "c2"), tc("web_search", {"query": "x"}, "c3")),
|
|
say("searched")],
|
|
single=[{"kind": "json", "body": {"results": [{"title": "T", "url": "http://u", "description": "d" * 500, "content": "c" * 3000}, {"title": None, "url": "u2"}]}},
|
|
{"kind": "json", "body": {"results": []}}], auto_approve=True)
|
|
add("poll_fork", "fork", [
|
|
calls(tc("fork", {"task": "inspect the repo"}, "c1"), tc("fork", {"task": ""}, "c2"), tc("fork", {"task": "x", "profile": "zzz"}, "c3")),
|
|
calls(tc("poll", {"id": 1, "wait": 5}, "c4"), tc("poll", {"id": 99}, "c5"), tc("poll", {"id": "abc"}, "c6")),
|
|
say("collected")], runner_script=["worker result text"], auto_approve=True)
|
|
add("terminals", "terminals", [
|
|
calls(tc("list_terminals", {}, "c1"), tc("plexer_send", {"text": "x"}, "c2"), tc("plexer_status", {}, "c3"), tc("plexer_read", {}, "c4")),
|
|
say("looked")], auto_approve=True)
|
|
add("persist_off", "worker style", [calls(tc("ask_user", {"question": "q"}, "c1"), tc("read_file", {"path": "a.txt"}, "c2")), say("end")],
|
|
persist=False, files={"a.txt": "z"}, auto_approve=True)
|
|
add("max_steps_stall", "stall", [calls(tc("shell", {"command": "echo %d" % i}, "c%d" % i)) for i in range(1, 5)] + [say("end")],
|
|
max_steps=2, auto_approve=True)
|
|
add("presets_profile", "with presets", [say("ok")], presets={"systems": {"default": "You are a custom bot. " + tai.VAULT_MEMORY_NOTE + " end"}},
|
|
auto_approve=True)
|
|
def ev(delta):
|
|
return {"choices": [{"delta": delta}]}
|
|
|
|
add("stream_quirks", "weird stream", [
|
|
{"kind": "stream", "events": [ev({"reasoning_content": "thinking about it"}), ev({"content": "part one "}), ev({"content": "part two"}),
|
|
ev({"tool_calls": [{"index": 0, "id": "q1", "function": {"name": "read_file", "arguments": "{\"path\": "}}]}),
|
|
ev({"tool_calls": [{"index": 5, "function": {"arguments": "\"a.txt\"}"}}]}),
|
|
{"choices": []}, {"nochoices": True},
|
|
ev({"tool_calls": [{"index": 1, "id": "q2", "function": {"name": "shell<|channel|>commentary", "arguments": "{\"command\": \"echo\"}"}}]}),
|
|
ev({"tool_calls": [{"index": 2, "id": "q3", "function": {"name": "bad name!", "arguments": "{}"}}]}),
|
|
ev({"tool_calls": [{"index": 3, "id": "q4", "function": {"name": "list_secrets", "arguments": "oops not json"}}]})]},
|
|
{"kind": "stream", "events": [ev({"reasoning": "only reasoning, no content"})]}],
|
|
files={"a.txt": "A"}, auto_approve=True)
|
|
add("reasoning_only", "think", [{"kind": "stream", "events": [ev({"reasoning": "deep thought"})]}], auto_approve=True)
|
|
add("empty_reply", "nothing", [{"kind": "stream", "events": []}], auto_approve=True)
|
|
add("store_errors", "probe store tools", [
|
|
calls(tc("record_read", {"id": "mem:0000000000000000"}, "c1"), tc("record_delete", {"id": ""}, "c2"), tc("record_delete", {"id": "mem:none"}, "c3"),
|
|
tc("graph_query", {"node": "bogus"}, "c4"), tc("graph_query", {"node": "mem:0123456789abcdef"}, "c5"),
|
|
tc("graph_link", {"src": "", "dst": "x"}, "c6"), tc("tags", {}, "c7"), tc("tags", {"limit": "x"}, "c8"),
|
|
tc("search", {"query": ""}, "c9"), tc("search", {"query": "x", "kinds": ["bogus"]}, "c10"), tc("search", {"query": "x", "limit": "q"}, "c11"),
|
|
tc("record_search", {"kind": "weird"}, "c12"), tc("audit", {"limit": "z"}, "c13"), tc("release", {"part": "huge"}, "c14"),
|
|
tc("release", {"part": "patch"}, "c15"), tc("install", {"action": "bogus"}, "c16"), tc("sysinfo", {"checks": []}, "c17"),
|
|
tc("mcp_call", {}, "c18"), tc("mcp_enable", {"server": "nope"}, "c19"), tc("mcp_list", {}, "c20"), tc("mcp_tools", {"server": "nope"}, "c21"),
|
|
tc("recall", {"query": ""}, "c22"), tc("recall", {"query": "x", "tags": "no"}, "c23"), tc("load_skill", {"name": "nope"}, "c24")),
|
|
say("probed")], auto_approve=True)
|
|
add("deadline", "late", [say("never reached")], deadline_expired=True, auto_approve=True)
|
|
add("mentions", "pins", [say("one"), say("two"), say("three"), say("four"), say("five")],
|
|
prompts=["@lab hello", "@m1 hello", "@a.txt what is this", "@auto hi", "@lab"], files={"a.txt": "A"}, auto_approve=True)
|
|
add("persist_resume", "session restore", [say("first"), say("second")], prompts=["one", "two"], auto_approve=True)
|
|
add("tool_error_text", "bad things", [
|
|
calls(tc("describe_image", {"source": ""}, "c1"), tc("speak", {"text": ""}, "c2"), tc("create_skill", {"name": "Bad Name", "brief": "x"}, "c3"),
|
|
tc("create_skill", {"name": "ok-name", "brief": ""}, "c4"), tc("install_skill", {"source": ""}, "c5"),
|
|
tc("plexer_type", {"text": ""}, "c6"), tc("plexer_show", {"spec": "no"}, "c7"), tc("tui_screen", {}, "c8"),
|
|
tc("export_terminal_log", {}, "c9"), tc("get_current_terminal_content", {"lines": "x", "target": "nonexistent-pane"}, "c10"),
|
|
tc("tail_terminals", {}, "c11"), tc("web_fetch", {"url": "http://x", "headers": {"A": 1}}, "c12"),
|
|
tc("web_fetch", {"url": "http://x", "body": 5}, "c13"), tc("write_file", {"path": "/proc/forbidden/x", "content": "y"}, "c14")),
|
|
say("done")], auto_approve=True)
|
|
add("legacy_note", "legacy", [say("ok")],
|
|
presets={"systems": {"default": "Old " + tai.LEGACY_PASSWORD_NOTE + " tail"}}, auto_approve=True)
|
|
return scenarios
|
|
|
|
|
|
def run_all(args):
|
|
scenarios = build_scenarios()
|
|
failures = 0
|
|
for sc in scenarios:
|
|
if args.filter and args.filter not in sc["name"]:
|
|
continue
|
|
results = []
|
|
for who in ("python", "cpp"):
|
|
base = tempfile.mkdtemp(prefix="agent-e2e-")
|
|
workdir = os.path.join(base, "work")
|
|
os.makedirs(workdir)
|
|
for rel, content in (sc.get("files") or {}).items():
|
|
path = os.path.join(workdir, rel)
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
with open(path, "w", encoding="utf-8") as handle:
|
|
handle.write(content)
|
|
try:
|
|
if who == "python":
|
|
results.append(normalize(python_run(sc, workdir)))
|
|
else:
|
|
results.append(normalize(cpp_run(sc, workdir, args.bin)))
|
|
except Exception as exc: # noqa: BLE001
|
|
results.append({"harness_error": "%s: %s" % (type(exc).__name__, exc)})
|
|
finally:
|
|
os.chdir(ROOT)
|
|
shutil.rmtree(base, ignore_errors=True)
|
|
py, cpp = results
|
|
# workdir path differs per run: neutralise it
|
|
py_text = re.sub(r"/tmp/agent-e2e-[A-Za-z0-9_]+/work", "<WORK>", json.dumps(py, sort_keys=True))
|
|
cpp_text = re.sub(r"/tmp/agent-e2e-[A-Za-z0-9_]+/work", "<WORK>", json.dumps(cpp, sort_keys=True))
|
|
if py_text == cpp_text:
|
|
sys.stdout.write("ok %s\n" % sc["name"])
|
|
continue
|
|
failures += 1
|
|
sys.stdout.write("FAIL %s\n" % sc["name"])
|
|
pyd, cppd = json.loads(py_text), json.loads(cpp_text)
|
|
for key in sorted(set(pyd) | set(cppd)):
|
|
if pyd.get(key) != cppd.get(key):
|
|
a = json.dumps(pyd.get(key), sort_keys=True)
|
|
b = json.dumps(cppd.get(key), sort_keys=True)
|
|
if args.verbose:
|
|
sys.stdout.write(" [%s]\n python: %s\n cpp: %s\n" % (key, a[:6000], b[:6000]))
|
|
else:
|
|
# first differing position
|
|
pos = next((i for i in range(min(len(a), len(b))) if a[i] != b[i]), min(len(a), len(b)))
|
|
sys.stdout.write(" [%s] differs at char %d\n python: ...%s\n cpp: ...%s\n" % (key, pos, a[max(0, pos - 150):pos + 250], b[max(0, pos - 150):pos + 250]))
|
|
sys.stdout.write("%d failing scenario(s)\n" % failures)
|
|
return 1 if failures else 0
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--bin", default=os.path.join(HERE, "..", "build", "agent_e2e"))
|
|
parser.add_argument("--filter", default="")
|
|
parser.add_argument("-v", "--verbose", action="store_true")
|
|
args = parser.parse_args()
|
|
return run_all(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|