515 lines
29 KiB
Python
515 lines
29 KiB
Python
#!/usr/bin/env python3
|
||
"""Differential test of the C++ agent core against tai.py.
|
||
|
||
Builds cases (curated plus seeded fuzz), computes the expected result with
|
||
tai.py, feeds the same cases to build/agent_diff (tests/agent_diff.cpp) and
|
||
compares the JSON results. Exit 0 only when every case matches.
|
||
|
||
python3 tests/agent_diff.py [--seed N] [--fuzz N] [--bin build/agent_diff] [-v]
|
||
"""
|
||
import argparse
|
||
import hashlib
|
||
import html
|
||
import json
|
||
import os
|
||
import random
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import urllib.parse
|
||
from datetime import datetime, timezone
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
||
# a hermetic environment: no tmux, no plexer, no tui socket, fixed zone
|
||
HOME = tempfile.mkdtemp(prefix="agent-diff-home-")
|
||
os.environ.pop("TMUX", None)
|
||
os.environ.pop("TMUX_PANE", None)
|
||
for key in list(os.environ):
|
||
if key.startswith("TAI_"):
|
||
del os.environ[key]
|
||
os.environ["TAI_HOME"] = HOME
|
||
os.environ["TZ"] = "Europe/Amsterdam"
|
||
os.environ["PATH"] = "/usr/bin:/bin"
|
||
for _name in ("MUSE_BIN", "OPENCODE_BIN", "GROK_BIN", "CLAUDE_BIN", "CODEX_BIN", "GEMINI_BIN"):
|
||
os.environ.pop(_name, None)
|
||
sys.path.insert(0, ROOT)
|
||
sys.dont_write_bytecode = True
|
||
import tai # noqa: E402
|
||
import types # noqa: E402
|
||
import urllib.error # noqa: E402
|
||
import urllib.request # noqa: E402
|
||
|
||
|
||
def _fail_urlopen(*args, **kwargs):
|
||
raise urllib.error.URLError("fake")
|
||
|
||
|
||
def py_chat_candidates(cfg, records, pins):
|
||
original = urllib.request.urlopen
|
||
urllib.request.urlopen = _fail_urlopen
|
||
try:
|
||
home = tempfile.mkdtemp(prefix="agent-diff-chat-")
|
||
cfg = dict(cfg, home=home)
|
||
config = types.SimpleNamespace(**cfg)
|
||
chat = tai.ChatClient(config)
|
||
for provider, model, ok, lat, tps in records:
|
||
chat.health.record(provider, model, ok, lat, tps)
|
||
chat.provider_pin = pins.get("provider")
|
||
chat.model_pin = pins.get("model")
|
||
|
||
def view(c):
|
||
return [c["provider"], c["chat_url"], c["models_url"], c["key"], c["model"], c["profile"], c.get("runner", "")]
|
||
|
||
return {"ordered": [view(c) for c in chat.ordered_candidates()], "all": [view(c) for c in chat.all_candidates()],
|
||
"backends": [list(b) for b in chat.backends()], "pin_note": chat.pin_note()}
|
||
finally:
|
||
urllib.request.urlopen = original
|
||
|
||
|
||
def dumps_sorted(value):
|
||
return json.dumps(value, sort_keys=True, ensure_ascii=True)
|
||
|
||
|
||
def guard(func):
|
||
def run(*args):
|
||
try:
|
||
return func(*args)
|
||
except tai.BackendError as exc:
|
||
return {"backend_error": str(exc), "status": exc.status}
|
||
except Exception as exc: # noqa: BLE001
|
||
return {"exception": str(exc)}
|
||
return run
|
||
|
||
|
||
def verdict_value(text, goal):
|
||
v = tai.GoalVerdict.from_text(text)
|
||
return {"ok": v.ok, "failures": [f.to_dict() for f in v.failures], "commentary": v.commentary,
|
||
"raw": v.raw, "report": v.report(goal), "followup": v.followup()}
|
||
|
||
|
||
def names(picked):
|
||
return [s["function"]["name"] for s in picked]
|
||
|
||
|
||
def int_or_none(value):
|
||
try:
|
||
return int(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def float_or_none(value):
|
||
try:
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def schedule_now(epoch):
|
||
return datetime.fromtimestamp(epoch, timezone.utc)
|
||
|
||
|
||
EXPECT = {
|
||
"chat_candidates": lambda cfg, records, pins: py_chat_candidates(cfg, records, pins),
|
||
"json_roundtrip": guard(lambda text: json.dumps(json.loads(text))),
|
||
"json_compact_sorted": guard(lambda text: json.dumps(json.loads(text), sort_keys=True, separators=(",", ":"))),
|
||
"schemas": lambda: json.dumps(tai.TOOL_SCHEMAS),
|
||
"select_tools": lambda text: names(tai.select_tools(text)),
|
||
"select_tools_extra": lambda text, extra: names(tai.select_tools(text, extra)),
|
||
"tool_catalog": lambda: tai.tool_catalog(),
|
||
"truncate": lambda text, head, tail: tai.truncate(text, head, tail),
|
||
"estimate_tokens": lambda value: tai.estimate_tokens(value),
|
||
"normalize_tag": lambda raw: tai.normalize_tag(raw),
|
||
"normalize_tags": lambda items: tai.normalize_tags(items),
|
||
"singular_noun": lambda word: tai.singular_noun(word),
|
||
"tag_variants": lambda tag: tai.tag_variants(tag),
|
||
"parse_node_id": lambda raw: (lambda r: list(r) if r else None)(tai.parse_node_id(raw)),
|
||
"secret_env_name": lambda name: tai.secret_env_name(name),
|
||
"format_delay": lambda s: tai.format_delay(s),
|
||
"parse_duration": lambda raw: tai.parse_duration(raw),
|
||
"parse_schedule_at": guard(lambda raw: tai.parse_schedule_at(raw)),
|
||
"local_display": lambda raw: tai.local_display(raw),
|
||
"html_unescape": lambda text: html.unescape(text),
|
||
"url_quote_plus": lambda text: urllib.parse.urlencode({"q": text})[2:],
|
||
"chunk_lines": lambda text: tai.chunk_lines(text),
|
||
"collapse_repeated_lines": lambda text: tai.collapse_repeated_lines(text),
|
||
"tool_name_clean": lambda raw: tai.tool_name_clean(raw),
|
||
"tool_args_valid": lambda raw: tai.tool_args_valid(raw),
|
||
"tool_args_text": lambda raw: tai.tool_args_text(raw),
|
||
"action_signature": lambda name, raw: tai.action_signature(name, raw),
|
||
"result_digest": lambda text: tai.result_digest(text),
|
||
"clean_history": lambda messages: tai.clean_history(messages),
|
||
"conversation_tokens": lambda messages, tools=None: tai.conversation_tokens(messages, tools),
|
||
"conversation_text": lambda messages: tai.conversation_text(messages),
|
||
"approval_parse": lambda answer: tai.Approval.parse_allow(answer),
|
||
"approval_affirm": lambda text: tai.Approval.is_affirmative(text),
|
||
"goal_verdict": lambda text, goal: verdict_value(text, goal),
|
||
"extract_json": lambda text: list(tai.GoalVerifier.extract_json(text)),
|
||
"goal_transcript": lambda messages: tai.GoalVerifier.transcript(messages),
|
||
"goal_tried": lambda messages: tai.GoalVerifier.tried_actions(messages),
|
||
"goal_task": lambda goal, final, transcript, tried: tai.GoalVerifier.build_task(goal, final, transcript, tried),
|
||
"muse_prompt": lambda messages, tools: tai.muse_prompt(messages, tools),
|
||
"opencode_prompt": lambda messages, tools: tai.opencode_prompt(messages, tools),
|
||
"opencode_parse": lambda text: tai.opencode_parse(text),
|
||
"opencode_find_answer": lambda text: tai.opencode_find_answer(text),
|
||
"opencode_parse_events": guard(lambda out, code, err: tai.opencode_parse_events(out, code, err)),
|
||
"grok_parse_output": guard(lambda text: tai.grok_parse_output(text)),
|
||
"claude_parse_output": guard(lambda text: tai.claude_parse_output(text)),
|
||
"gemini_parse_output": guard(lambda text: tai.gemini_parse_output(text)),
|
||
"codex_parse_output": guard(lambda out, outfile, used: tai.codex_parse_output(out, outfile, used)),
|
||
"muse_valid_answer": lambda text: tai.muse_valid_answer(text),
|
||
"muse_message_hash": lambda message: tai.muse_message_hash(message),
|
||
"model_weight": lambda entry: tai.model_weight(entry),
|
||
"model_outcome_reward": lambda lat, tps: tai.model_outcome_reward(lat, tps),
|
||
"summarize_errors": lambda errors, shown, width: tai.ChatClient.summarize_errors(errors, shown, width),
|
||
"split_models": lambda spec: tai.split_models(spec),
|
||
"parse_backends": lambda spec: tai.parse_backends(spec),
|
||
"is_free_model": lambda model: tai.is_free_model(model),
|
||
"embedded_failure_message": lambda data: tai.embedded_failure_message(data),
|
||
"stream_status": lambda code, output, elapsed, timed_out: tai.stream_status(code, output, elapsed, timed_out),
|
||
"curl_hint": lambda command: tai.curl_hint(command),
|
||
"format_schedule_lines": lambda items, now: tai.format_schedule_lines(items, schedule_now(now)),
|
||
"py_repr": lambda value: repr(value),
|
||
"py_str": lambda value: str(value),
|
||
"py_strip": lambda text: text.strip(),
|
||
"py_lower": lambda text: text.lower(),
|
||
"py_splitlines": lambda text: text.splitlines(),
|
||
"py_split_ws": lambda text: text.split(),
|
||
"py_int": lambda value: int_or_none(value),
|
||
"py_float": lambda value: float_or_none(value),
|
||
"py_slice": lambda text, start, end: text[start:end],
|
||
"py_len": lambda text: len(text),
|
||
"float_repr": lambda value: repr(float(value)),
|
||
"sha1": lambda text: hashlib.sha1(text.encode("utf-8", "surrogatepass")).hexdigest(),
|
||
"sha256": lambda text: hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest(),
|
||
"utf8_replace": lambda text: text.encode("latin-1").decode("utf-8", "replace"),
|
||
"orientation_block": lambda directory, sandbox: tai.orientation_block(directory or None, sandbox),
|
||
"capability_block": lambda yolo, auto: tai.capability_block(yolo, auto),
|
||
"split_leading_tags": lambda text: list(tai.split_leading_tags(text)),
|
||
"expand_file_mentions": lambda text, directory: tai.expand_file_mentions(text, directory),
|
||
"profile_ok": lambda s: bool(tai.PROFILE_RE.match(s)),
|
||
"bot_ok": lambda s: bool(tai.BOT_RE.match(s)),
|
||
"secret_ok": lambda s: bool(tai.SECRET_RE.match(s)),
|
||
"skill_name_ok": lambda s: bool(tai.SKILL_NAME_RE.match(s)),
|
||
"mcp_server_ok": lambda s: bool(tai.MCP_SERVER_RE.match(s)),
|
||
"create_skill_prompt": lambda name, brief, skill_dir: tai.create_skill_prompt(name, brief, skill_dir),
|
||
"data_texts": lambda: {"default_system": tai.DEFAULT_SYSTEM, "merge_system": tai.MERGE_SYSTEM,
|
||
"compact_system": tai.COMPACT_SYSTEM, "autonomous_note": tai.AUTONOMOUS_NOTE,
|
||
"legacy_password_note": tai.LEGACY_PASSWORD_NOTE, "vault_memory_note": tai.VAULT_MEMORY_NOTE,
|
||
"mcp_resolver_brief": tai.MCP_RESOLVER_BRIEF, "goal_task": tai.GoalVerifier.TASK,
|
||
"curl_hint": tai.CURL_HINT},
|
||
}
|
||
|
||
POOL = ["a", "b", "c", "x", "Y", "Z", "0", "1", "9", " ", " ", "\t", "\n", "\r\n", "-", "_", ".", ",", ";", ":", "!", "?",
|
||
"(", ")", "[", "]", "{", "}", "\"", "'", "\\", "/", "@", "#", "&", "<", ">", "=", "+", "*", "%", "é", "é",
|
||
"ß", "İ", "K", " ", "
", " ", "中", "文", "\U0001f600", "€", "\x7f", "\x1c",
|
||
"TODO", "token", "secret", "readme", "README", "http://x", "curl", "wget", "&", "<", "A", "A", " ",
|
||
"©", "&unknown;", "ies", "ses", "news", "ss", "tags", "record"]
|
||
|
||
|
||
def fuzz_text(rng, max_parts=14):
|
||
return "".join(rng.choice(POOL) for _ in range(rng.randint(0, max_parts)))
|
||
|
||
|
||
def fuzz_json(rng, depth=0):
|
||
kind = rng.randint(0, 9 if depth < 3 else 5)
|
||
if kind == 0:
|
||
return None
|
||
if kind == 1:
|
||
return rng.choice([True, False])
|
||
if kind == 2:
|
||
return rng.randint(-1000, 10 ** 12)
|
||
if kind == 3:
|
||
return rng.choice([0.0, 1.5, -2.25, 1e20, 1e-7, 123456.789, 0.1, 1 / 3])
|
||
if kind in (4, 5):
|
||
return fuzz_text(rng, 6)
|
||
if kind in (6, 7):
|
||
return [fuzz_json(rng, depth + 1) for _ in range(rng.randint(0, 4))]
|
||
return {fuzz_text(rng, 3): fuzz_json(rng, depth + 1) for _ in range(rng.randint(0, 4))}
|
||
|
||
|
||
def fuzz_messages(rng):
|
||
messages = []
|
||
for _ in range(rng.randint(0, 7)):
|
||
role = rng.choice(["system", "user", "assistant", "tool"])
|
||
item = {"role": role, "content": rng.choice([fuzz_text(rng, 5), None, "", fuzz_text(rng, 30)])}
|
||
if role == "assistant" and rng.random() < 0.6:
|
||
calls = []
|
||
for k in range(rng.randint(1, 3)):
|
||
name = rng.choice(["shell", "read_file", "bad name!", "x" * 70, "web_fetch", "", "a.b-c_d"])
|
||
args = rng.choice(['{"a": 1}', "", "not json", "{}", None, {"a": 1}, "[1, 2]"])
|
||
calls.append({"id": "c%d" % rng.randint(0, 3), "type": "function", "function": {"name": name, "arguments": args}})
|
||
item["tool_calls"] = calls
|
||
if role == "tool":
|
||
item["tool_call_id"] = "c%d" % rng.randint(0, 3)
|
||
messages.append(item)
|
||
return messages
|
||
|
||
|
||
PROVIDERS = ["lab", "devplace", "openrouter", "pollinations", "zen", "ollama", "muse", "opencode", "other"]
|
||
MODELS = ["m1", "m2", "molodetz", "free", "muse-spark-x", "a:free", "b-free", "openai-fast", "deepseek/deepseek-v4.1-flash", "z"]
|
||
|
||
|
||
def fuzz_config(rng):
|
||
pick = lambda n: rng.sample(MODELS, rng.randint(0, n)) # noqa: E731
|
||
backends = []
|
||
for i in range(rng.randint(0, 3)):
|
||
backends.append({"label": rng.choice(["lab", "devplace", "Acme", "other"]), "base": "http://h%d/v1" % i, "key": rng.choice(["", "k"]),
|
||
"model": ",".join(pick(2)), "profile": rng.choice(["", "opencode"])})
|
||
return {"model": rng.choice(["muse", "m1"]), "explicit_model": rng.choice(["", "m2", "xx"]), "backends": backends,
|
||
"openrouter_key": rng.choice(["", "key"]), "openrouter_models": pick(3), "retry_seconds": 0, "opencode_enabled": rng.random() < 0.5,
|
||
"opencode_models": pick(2), "devplace_key": rng.choice(["", "dk"]), "devplace_base": "https://devplace.net/openai/v1",
|
||
"devplace_models": pick(2), "openrouter_enabled": rng.random() < 0.7, "pollinations_enabled": rng.random() < 0.7,
|
||
"pollinations_models": pick(2), "zen_enabled": rng.random() < 0.4, "zen_models": pick(3), "ollama_enabled": rng.random() < 0.6,
|
||
"ollama_models": pick(2)}
|
||
|
||
|
||
def fuzz_records(rng):
|
||
out = []
|
||
for _ in range(rng.randint(0, 12)):
|
||
out.append([rng.choice(PROVIDERS), rng.choice(MODELS), rng.random() < 0.6, rng.choice([None, 100.0, 5000.5]), rng.choice([None, 10.0, 80.0])])
|
||
return out
|
||
|
||
|
||
def build_cases(rng, fuzz):
|
||
cases = []
|
||
|
||
def add(fn, *args):
|
||
cases.append({"fn": fn, "args": list(args)})
|
||
|
||
for _ in range(max(40, fuzz // 3)):
|
||
pins = rng.choice([{}, {}, {"provider": rng.choice(PROVIDERS)}, {"model": rng.choice(MODELS)}, {"provider": rng.choice(PROVIDERS), "model": rng.choice(MODELS)}])
|
||
add("chat_candidates", fuzz_config(rng), fuzz_records(rng), pins)
|
||
add("schemas")
|
||
add("tool_catalog")
|
||
add("data_texts")
|
||
prompts = ["", "hello", "please run the tests and fix the failing build", "search the web for rust news", "read the file readme",
|
||
"schedule a reminder every hour", "remember my favorite color", "store a password in the vault", "delete the old file",
|
||
"describe this image screenshot", "fork a subagent to research in parallel", "install the skill from github",
|
||
"mcp server list", "audit changes since yesterday", "release a new version", "create a bot persona", "graph link nodes",
|
||
"tail terminals tmux panes", "what is the system info of this machine?", "translate this poem",
|
||
"SEARCH the WEB", "Kelvin K sign and İstanbul", "tag tags tagging", "ask the user a question and wait"]
|
||
for p in prompts:
|
||
add("select_tools", p)
|
||
for _ in range(fuzz):
|
||
add("select_tools", fuzz_text(rng, 25))
|
||
extra = [{"type": "function", "function": {"name": "mcp_demo_weather", "description": "Fetch the weather forecast for a city", "parameters": {}}},
|
||
{"type": "function", "function": {"name": "mcp_demo_list", "description": "short", "parameters": {}}}]
|
||
for p in ["weather forecast please", "list things", "forecast", ""]:
|
||
add("select_tools_extra", p, extra)
|
||
|
||
for _ in range(fuzz):
|
||
t = fuzz_text(rng, 40)
|
||
add("truncate", t * rng.randint(1, 120), rng.choice([10, 100, 1500]), rng.choice([5, 50, 500]))
|
||
add("estimate_tokens", rng.choice([t, None, "", [1, 2, 3, 4, 5]]))
|
||
add("normalize_tag", t)
|
||
add("normalize_tags", [fuzz_text(rng, 4) for _ in range(rng.randint(0, 25))])
|
||
add("singular_noun", fuzz_text(rng, 3))
|
||
add("tag_variants", fuzz_text(rng, 3))
|
||
add("secret_env_name", t)
|
||
add("html_unescape", t)
|
||
add("url_quote_plus", t)
|
||
add("chunk_lines", "\n".join(fuzz_text(rng, 30) for _ in range(rng.randint(0, 200))))
|
||
add("collapse_repeated_lines", "\n".join(rng.choice(["a", "a", "a", "b", fuzz_text(rng, 2)]) for _ in range(rng.randint(0, 30))))
|
||
add("tool_name_clean", rng.choice([t, None, "shell<|channel|>commentary", " x "]))
|
||
add("tool_args_valid", rng.choice([t, None, "{}", '{"a":1}', {"a": 1}, "", "[1]"]))
|
||
add("tool_args_text", rng.choice([t, None, "", {"a": [1, "é"]}, ["x"], 0, 5]))
|
||
add("action_signature", rng.choice(["shell", "read_file"]), rng.choice(['{"b":1,"a":2}', "nope", None, "", {"z": 1, "a": 2}]))
|
||
add("result_digest", t)
|
||
msgs = fuzz_messages(rng)
|
||
add("clean_history", msgs)
|
||
add("conversation_tokens", msgs)
|
||
add("conversation_text", msgs)
|
||
add("goal_transcript", msgs)
|
||
add("goal_tried", msgs)
|
||
add("approval_parse", t)
|
||
add("approval_affirm", rng.choice([t, "Yes!", " go ahead. ", "LOOKS good", "yes, but no", "ok.?"]))
|
||
add("py_repr", fuzz_json(rng))
|
||
add("py_str", fuzz_json(rng))
|
||
add("py_strip", t)
|
||
add("py_lower", t)
|
||
add("py_splitlines", t)
|
||
add("py_split_ws", t)
|
||
add("py_int", rng.choice([fuzz_json(rng), "12", " 7 ", "1_0", "0x10", "1e3", "-5", "+3", "", "٣"]))
|
||
add("py_float", rng.choice([fuzz_json(rng), "1.5", " 2 ", "1e3", "nan", "inf", "-Infinity", "1_0.5", "0x1p3", "."]))
|
||
add("py_slice", t, rng.choice([None, -3, 0, 2, 50]), rng.choice([None, -1, 0, 4, 99]))
|
||
add("py_len", t)
|
||
add("float_repr", rng.choice([0.1, 1e16, 1e-5, 123456789.123456789, 5e-324, 1.7976931348623157e308, 100.0, 1 / 3, 2.5e22]))
|
||
add("sha1", t)
|
||
add("sha256", t)
|
||
add("split_leading_tags", rng.choice(["@a b", "@a @b rest", "@@x", "plain", "@ x", "@x y", "@a\n@b\tc " + t]))
|
||
add("profile_ok", rng.choice([t, "abc", "abc\n", "a" * 32, "a" * 33, "-x", "a_b-c"]))
|
||
add("bot_ok", rng.choice([t, "abc", "ABC", "a-b_c"]))
|
||
add("secret_ok", rng.choice([t, "a.b-c", "a" * 64, "a" * 65]))
|
||
add("skill_name_ok", rng.choice([t, "ab", "a-b", "a-", "-a", "a" * 64, "a" * 65, "x\n"]))
|
||
add("mcp_server_ok", rng.choice([t, "ok.name", "a" * 64]))
|
||
add("json_roundtrip", json.dumps(fuzz_json(rng), ensure_ascii=rng.random() < 0.5))
|
||
add("json_compact_sorted", json.dumps(fuzz_json(rng)))
|
||
add("model_outcome_reward", rng.choice([None, 0, 120.5, 8000]), rng.choice([None, 0, 12.5, 100]))
|
||
add("model_weight", rng.choice([{}, {"s": 3, "f": 1, "r": 2.5}, {"s": 0, "f": 0}, {"r": 1.0, "s": None, "f": 2}]))
|
||
|
||
for raw in ["", "{", "[1,]", '{"a":1,}', '{"a":1 "b":2}', '{"a" 1}', '"abc', '"a\\q"', '"\\u12"', "tru", "nan", "NaN", "-", "1e", "[1 2]",
|
||
"[1,2] x", '{"a":1}\n', "\t[ ]", '"a\tb"', '{"é": "ü\\ud83d\\ude00"}', '"\\ud800"', "1.5e300", "1" * 30, "-0", "-0.0",
|
||
"[" * 5 + "]" * 5, "{}", '{"a":{"b":[1,2,{"c":null}]}}', '{"a":1,"a":2}', "1.", ".5", "01", "+1"]:
|
||
add("json_roundtrip", raw)
|
||
add("json_compact_sorted", raw)
|
||
|
||
for raw in ["", "2026-10-08T09:00", "2026-10-08 09:00:15.5", "2026-10-08T09:00Z", "2026-10-08T09:00+02:00", "2026-10-08T09:00-0530",
|
||
"2026-10-08", "20261008", "2026-13-01", "2026-02-30", "garbage", "2026-10-08T25:00", "2026-03-29T02:30", "2026-10-25T02:30",
|
||
"2026-10-08T09:00:00.123456789", " 2026-10-08T09:00 ", "2026-10-08t09:00", "2026-10-08T09", "2026-10-08T09:00+02"]:
|
||
add("parse_schedule_at", raw)
|
||
add("local_display", raw)
|
||
for s in [-5, 0, 59, 60, 3599, 3600, 86399, 86400, 10 ** 7]:
|
||
add("format_delay", s)
|
||
for raw in ["", "5", "5s", "10 m", "3h", "2d", "1w", "x", "5x", "1.5h", " 7D ", "99999999999999999999"]:
|
||
add("parse_duration", raw)
|
||
items = [{"id": 1, "status": "pending", "every": 0, "next_run": "2026-10-25T12:00:00+00:00", "tags": ["a", "b"], "name": "x", "prompt": "p" * 100, "last": "done", "result": "r" * 200},
|
||
{"id": 2, "status": "pending", "every": 3600, "next_run": "bad", "tags": [], "name": "", "prompt": "q", "last": "", "result": ""}]
|
||
add("format_schedule_lines", items, 1792924800.0)
|
||
add("format_schedule_lines", [], 1792924800.0)
|
||
|
||
for raw in ["&<>"'", "¬it;", "∉", "�", "€", "€", "�", "�", "�", "&", "&x", "a&b",
|
||
"&;", "&#;", "&#x;", "&#xzz;", "&", "≂̸", "♥&hearts", "€", "A", "A", " x&NBSP;", "&" * 40]:
|
||
add("html_unescape", raw)
|
||
for raw in ["hello world", "a+b=c&d", "é中\U0001f600", "~_.-", "100%"]:
|
||
add("url_quote_plus", raw)
|
||
|
||
for raw in ["", "{}", '```json\n{"success": true, "failures": []}\n```', '{"success": true, "failures": [{"item": "a", "reason": "b", "proposal": "c"}]}',
|
||
'{"success": false, "failures": [{"item": "a"}]}', '{"success": false, "failures": []}', '{"success": "yes", "failures": []}',
|
||
'{"success": true, "failures": [], "commentary": " fine "}', "no json", '{"success": true,', "[1, 2]",
|
||
'text {"success": false, "failures": [{"item": "i", "reason": "r", "proposal": "p"}], "commentary": "c"} tail',
|
||
'{"success": false, "failures": ' + json.dumps([{"item": str(i), "reason": "r", "proposal": "p"} for i in range(12)]) + "}",
|
||
'{"success": true, "failures": "none"}', '{"success": null}', "```\n{}\n```", "```JSON {\"success\": true, \"failures\": []}"]:
|
||
add("goal_verdict", raw, " fix the\tbug ")
|
||
add("extract_json", raw)
|
||
add("goal_task", "do x", "done", "[user] hi", ["a", "b"])
|
||
add("goal_task", "", "", "", [])
|
||
|
||
sample_msgs = [{"role": "system", "content": "s"}, {"role": "user", "content": "do thing\nplease"},
|
||
{"role": "assistant", "content": "ok", "tool_calls": [{"id": "1", "type": "function", "function": {"name": "shell", "arguments": '{"command": "ls"}'}}]},
|
||
{"role": "tool", "tool_call_id": "1", "content": "out"},
|
||
{"role": "user", "content": [{"type": "text", "text": "part one"}, {"type": "text", "text": "part two"}, "x"]}]
|
||
tools = [{"type": "function", "function": {"name": "t", "description": "d", "parameters": {"type": "object", "properties": {}, "required": []}}}]
|
||
add("muse_prompt", sample_msgs, tools)
|
||
add("muse_prompt", sample_msgs, None)
|
||
add("muse_prompt", [], [])
|
||
add("opencode_prompt", sample_msgs, tools)
|
||
for raw in ["", "plain answer", '{"content": "hi", "tool_calls": []}', '```json\n{"content": "x", "tool_calls": [{"name": "a", "arguments": "{}"}]}\n```',
|
||
'noise {"schema": 1} then {"content": "late", "tool_calls": []} tail', '{"tool_calls": []}', '{"nothing": 1}', "{{{", '```\n```',
|
||
"```json\n```", "``````", "```json{\"content\":\"z\"}```", '{"content": "a"} {"content": "b"}']:
|
||
add("opencode_parse", raw)
|
||
add("opencode_find_answer", raw)
|
||
add("grok_parse_output", raw)
|
||
add("claude_parse_output", raw)
|
||
add("gemini_parse_output", raw)
|
||
add("codex_parse_output", raw, "", False)
|
||
add("codex_parse_output", "", raw, True)
|
||
add("codex_parse_output", raw, raw, False)
|
||
add("muse_valid_answer", raw)
|
||
for obj in [{"is_error": True, "result": "bad"}, {"error": "x"}, {"error": {"a": 1}}, {"result": " answer "}, {"text": "t"}, {"structured_output": {"content": "c", "tool_calls": []}},
|
||
{"response": '{"content": "r", "tool_calls": []}'}, {"content": "c", "tool_calls": []}, {"output": "o"}, [1], 5, {"is_error": False, "result": 5}]:
|
||
raw = json.dumps(obj)
|
||
add("grok_parse_output", raw)
|
||
add("claude_parse_output", raw)
|
||
add("gemini_parse_output", raw)
|
||
add("muse_valid_answer", raw)
|
||
add("codex_parse_output", '{"type": "item.completed", "item": {"type": "agent_message", "text": "{\\"content\\": \\"q\\", \\"tool_calls\\": []}"}}\nnoise', "", False)
|
||
add("codex_parse_output", '{"type": "turn.failed", "message": "boom"}', "", False)
|
||
add("codex_parse_output", '{"type": "error", "error": {"message": "deep"}}', "", False)
|
||
add("codex_parse_output", '{"type": "error", "error": {"nomsg": 1}}', "", False)
|
||
add("opencode_parse_events", '{"type": "text", "part": {"text": "hello "}}\n{"type": "text", "part": {"text": "world"}}\nnot json\n', 0, "")
|
||
add("opencode_parse_events", '{"type": "error", "error": {"data": {"message": "m", "statusCode": 429}}}', 1, "")
|
||
add("opencode_parse_events", '{"type": "error", "error": {"name": "X"}}', 1, "")
|
||
add("opencode_parse_events", "", 3, "stderr text " * 40)
|
||
add("opencode_parse_events", "out", 3, "")
|
||
add("opencode_parse_events", '[1]\n{"type": "text"}\n', 0, "e")
|
||
|
||
for ident, errs in enumerate([[], ["a/b: x"], ["a: CLI not found", "b: err1", "b: err1", "c: err2", "d: err3"], ["muse: muse CLI not found"], ["x"] * 3 + ["y" * 500]]):
|
||
add("summarize_errors", errs, 2, 320)
|
||
add("summarize_errors", errs, 6, 900)
|
||
for spec in ["a,b\nc, ,d", "", " x "]:
|
||
add("split_models", spec)
|
||
add("parse_backends", "lab|https://x/v1/|key|m1,m2|opencode;bad|x\nl2|http://h|k2")
|
||
add("parse_backends", "a|b|$NOPE_NOT_SET")
|
||
add("parse_backends", "")
|
||
for model in ["a:free", "x-FREE", "Model-free", "paid", "", "free"]:
|
||
add("is_free_model", model)
|
||
for d in [{"error": {"message": " m "}}, {"error": {"type": "t"}}, {"error": " s "}, {"type": "error", "message": "mm"}, {"type": "error"}, {"ok": 1}, [1], {"error": {}}]:
|
||
add("embedded_failure_message", d)
|
||
for args in [(0, "a\nb\n", 1.234, False), (1, "", 0.05, False), (0, "x" * 2000, 12.55, True), (124, "ü\r\n", 3.25, False)]:
|
||
add("stream_status", *args)
|
||
for command in ["curl http://x", "wget https://y", "curl localhost", "xcurl http://x", "echo curl_http", "WGET HTTP://X", "curl-x http://z"]:
|
||
add("curl_hint", command)
|
||
add("create_skill_prompt", "n", "brief\nmore", "/tmp/skills/n")
|
||
add("capability_block", False, False)
|
||
add("capability_block", True, False)
|
||
add("capability_block", False, True)
|
||
add("orientation_block", "", True)
|
||
add("orientation_block", HOME, False)
|
||
add("orientation_block", "/nonexistent/dir/../x", False)
|
||
workdir = tempfile.mkdtemp(prefix="agent-diff-work-")
|
||
with open(os.path.join(workdir, "a.txt"), "w") as handle:
|
||
handle.write("x")
|
||
for text in ["see @a.txt, and @missing.txt!", "@a.txt)", "mail me@a.txt", "@", "@@a.txt", "x @%s/a.txt y" % workdir, "@~", "no mentions", "@a.txt @a.txt"]:
|
||
add("expand_file_mentions", text, workdir)
|
||
for raw in ["\xc3\xa9", "\xff\xfe", "\xe2\x82", "\xe2\x82\xac", "\xed\xa0\x80", "\xf0\x9f\x98", "\xf0\x9f\x98\x80", "\xc0\x80", "\xf4\x90\x80\x80", "ok", "\xe0\x80\x80\xe0\xa0\x80", "\x80\x80"]:
|
||
add("utf8_replace", raw)
|
||
for value in [None, True, 5, 2.5, "it's", 'say "hi"', "both ' and \"", "tab\there", "\x00\x1f\x7f", "é \U0001f600", [], {}, [1, "a", None, {"k": [True]}], {"a": {"b": 1.0}}, "\ud800"]:
|
||
add("py_repr", value)
|
||
add("py_str", value)
|
||
return cases
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument("--seed", type=int, default=1)
|
||
parser.add_argument("--fuzz", type=int, default=150)
|
||
parser.add_argument("--bin", default=os.path.join(HERE, "..", "build", "agent_diff"))
|
||
parser.add_argument("-v", "--verbose", action="store_true")
|
||
args = parser.parse_args()
|
||
rng = random.Random(args.seed)
|
||
cases = build_cases(rng, args.fuzz)
|
||
payload = "\n".join(json.dumps(case) for case in cases) + "\n"
|
||
env = dict(os.environ)
|
||
done = subprocess.run([args.bin], input=payload.encode("utf-8", "surrogatepass"), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
|
||
if done.returncode != 0:
|
||
sys.stderr.write("binary failed (%s): %s\n" % (done.returncode, done.stderr.decode("utf-8", "replace")[-2000:]))
|
||
return 2
|
||
outputs = done.stdout.decode("utf-8").splitlines()
|
||
if len(outputs) != len(cases):
|
||
sys.stderr.write("expected %d result lines, got %d\n" % (len(cases), len(outputs)))
|
||
return 2
|
||
failures = 0
|
||
per_fn = {}
|
||
for case, line in zip(cases, outputs):
|
||
expected_fn = EXPECT[case["fn"]]
|
||
try:
|
||
expected = expected_fn(*case["args"])
|
||
except Exception as exc: # noqa: BLE001
|
||
expected = {"exception": str(exc)}
|
||
actual = json.loads(line)
|
||
# documented limit: int() results beyond int64 clamp in C++ (Python ints are unbounded)
|
||
if case["fn"] in ("py_int", "parse_duration") and isinstance(expected, int) and not -2 ** 63 <= expected < 2 ** 63:
|
||
expected = 2 ** 63 - 1 if expected > 0 else -2 ** 63
|
||
# normalize through JSON so tuples/ints compare like the C++ side
|
||
expected_norm = json.loads(json.dumps(expected))
|
||
stats = per_fn.setdefault(case["fn"], [0, 0])
|
||
stats[0] += 1
|
||
if dumps_sorted(expected_norm) != dumps_sorted(actual):
|
||
failures += 1
|
||
stats[1] += 1
|
||
if failures <= 40 or args.verbose:
|
||
sys.stdout.write("MISMATCH %s %s\n expected: %s\n actual: %s\n" % (
|
||
case["fn"], json.dumps(case["args"], ensure_ascii=True)[:300], dumps_sorted(expected_norm)[:400], dumps_sorted(actual)[:400]))
|
||
sys.stdout.write("%d cases, %d mismatches\n" % (len(cases), failures))
|
||
for fn in sorted(per_fn):
|
||
total, bad = per_fn[fn]
|
||
if bad:
|
||
sys.stdout.write(" %-26s %4d cases %4d mismatches\n" % (fn, total, bad))
|
||
return 1 if failures else 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|