tai/taicpp/tests/agent_diff.py

#!/usr/bin/env python3
"""Differential test of the C++ agent core against tai.py.

Builds cases (curated plus seeded fuzz), computes the expected result with
tai.py, feeds the same cases to build/agent_diff (tests/agent_diff.cpp) and
compares the JSON results. Exit 0 only when every case matches.

    python3 tests/agent_diff.py [--seed N] [--fuzz N] [--bin build/agent_diff] [-v]
"""
import argparse
import hashlib
import html
import json
import os
import random
import subprocess
import sys
import tempfile
import urllib.parse
from datetime import datetime, timezone

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
# a hermetic environment: no tmux, no plexer, no tui socket, fixed zone
HOME = tempfile.mkdtemp(prefix="agent-diff-home-")
os.environ.pop("TMUX", None)
os.environ.pop("TMUX_PANE", None)
for key in list(os.environ):
    if key.startswith("TAI_"):
        del os.environ[key]
os.environ["TAI_HOME"] = HOME
os.environ["TZ"] = "Europe/Amsterdam"
os.environ["PATH"] = "/usr/bin:/bin"
for _name in ("MUSE_BIN", "OPENCODE_BIN", "GROK_BIN", "CLAUDE_BIN", "CODEX_BIN", "GEMINI_BIN"):
    os.environ.pop(_name, None)
sys.path.insert(0, ROOT)
sys.dont_write_bytecode = True
import tai  # noqa: E402
import types  # noqa: E402
import urllib.error  # noqa: E402
import urllib.request  # noqa: E402


def _fail_urlopen(*args, **kwargs):
    raise urllib.error.URLError("fake")


def py_chat_candidates(cfg, records, pins):
    original = urllib.request.urlopen
    urllib.request.urlopen = _fail_urlopen
    try:
        home = tempfile.mkdtemp(prefix="agent-diff-chat-")
        cfg = dict(cfg, home=home)
        config = types.SimpleNamespace(**cfg)
        chat = tai.ChatClient(config)
        for provider, model, ok, lat, tps in records:
            chat.health.record(provider, model, ok, lat, tps)
        chat.provider_pin = pins.get("provider")
        chat.model_pin = pins.get("model")

        def view(c):
            return [c["provider"], c["chat_url"], c["models_url"], c["key"], c["model"], c["profile"], c.get("runner", "")]

        return {"ordered": [view(c) for c in chat.ordered_candidates()], "all": [view(c) for c in chat.all_candidates()],
                "backends": [list(b) for b in chat.backends()], "pin_note": chat.pin_note()}
    finally:
        urllib.request.urlopen = original


def dumps_sorted(value):
    return json.dumps(value, sort_keys=True, ensure_ascii=True)


def guard(func):
    def run(*args):
        try:
            return func(*args)
        except tai.BackendError as exc:
            return {"backend_error": str(exc), "status": exc.status}
        except Exception as exc:  # noqa: BLE001
            return {"exception": str(exc)}
    return run


def verdict_value(text, goal):
    v = tai.GoalVerdict.from_text(text)
    return {"ok": v.ok, "failures": [f.to_dict() for f in v.failures], "commentary": v.commentary,
            "raw": v.raw, "report": v.report(goal), "followup": v.followup()}


def names(picked):
    return [s["function"]["name"] for s in picked]


def int_or_none(value):
    try:
        return int(value)
    except (TypeError, ValueError):
        return None


def float_or_none(value):
    try:
        return float(value)
    except (TypeError, ValueError):
        return None


def schedule_now(epoch):
    return datetime.fromtimestamp(epoch, timezone.utc)


EXPECT = {
    "chat_candidates": lambda cfg, records, pins: py_chat_candidates(cfg, records, pins),
    "json_roundtrip": guard(lambda text: json.dumps(json.loads(text))),
    "json_compact_sorted": guard(lambda text: json.dumps(json.loads(text), sort_keys=True, separators=(",", ":"))),
    "schemas": lambda: json.dumps(tai.TOOL_SCHEMAS),
    "select_tools": lambda text: names(tai.select_tools(text)),
    "select_tools_extra": lambda text, extra: names(tai.select_tools(text, extra)),
    "tool_catalog": lambda: tai.tool_catalog(),
    "truncate": lambda text, head, tail: tai.truncate(text, head, tail),
    "estimate_tokens": lambda value: tai.estimate_tokens(value),
    "normalize_tag": lambda raw: tai.normalize_tag(raw),
    "normalize_tags": lambda items: tai.normalize_tags(items),
    "singular_noun": lambda word: tai.singular_noun(word),
    "tag_variants": lambda tag: tai.tag_variants(tag),
    "parse_node_id": lambda raw: (lambda r: list(r) if r else None)(tai.parse_node_id(raw)),
    "secret_env_name": lambda name: tai.secret_env_name(name),
    "format_delay": lambda s: tai.format_delay(s),
    "parse_duration": lambda raw: tai.parse_duration(raw),
    "parse_schedule_at": guard(lambda raw: tai.parse_schedule_at(raw)),
    "local_display": lambda raw: tai.local_display(raw),
    "html_unescape": lambda text: html.unescape(text),
    "url_quote_plus": lambda text: urllib.parse.urlencode({"q": text})[2:],
    "chunk_lines": lambda text: tai.chunk_lines(text),
    "collapse_repeated_lines": lambda text: tai.collapse_repeated_lines(text),
    "tool_name_clean": lambda raw: tai.tool_name_clean(raw),
    "tool_args_valid": lambda raw: tai.tool_args_valid(raw),
    "tool_args_text": lambda raw: tai.tool_args_text(raw),
    "action_signature": lambda name, raw: tai.action_signature(name, raw),
    "result_digest": lambda text: tai.result_digest(text),
    "clean_history": lambda messages: tai.clean_history(messages),
    "conversation_tokens": lambda messages, tools=None: tai.conversation_tokens(messages, tools),
    "conversation_text": lambda messages: tai.conversation_text(messages),
    "approval_parse": lambda answer: tai.Approval.parse_allow(answer),
    "approval_affirm": lambda text: tai.Approval.is_affirmative(text),
    "goal_verdict": lambda text, goal: verdict_value(text, goal),
    "extract_json": lambda text: list(tai.GoalVerifier.extract_json(text)),
    "goal_transcript": lambda messages: tai.GoalVerifier.transcript(messages),
    "goal_tried": lambda messages: tai.GoalVerifier.tried_actions(messages),
    "goal_task": lambda goal, final, transcript, tried: tai.GoalVerifier.build_task(goal, final, transcript, tried),
    "muse_prompt": lambda messages, tools: tai.muse_prompt(messages, tools),
    "opencode_prompt": lambda messages, tools: tai.opencode_prompt(messages, tools),
    "opencode_parse": lambda text: tai.opencode_parse(text),
    "opencode_find_answer": lambda text: tai.opencode_find_answer(text),
    "opencode_parse_events": guard(lambda out, code, err: tai.opencode_parse_events(out, code, err)),
    "grok_parse_output": guard(lambda text: tai.grok_parse_output(text)),
    "claude_parse_output": guard(lambda text: tai.claude_parse_output(text)),
    "gemini_parse_output": guard(lambda text: tai.gemini_parse_output(text)),
    "codex_parse_output": guard(lambda out, outfile, used: tai.codex_parse_output(out, outfile, used)),
    "muse_valid_answer": lambda text: tai.muse_valid_answer(text),
    "muse_message_hash": lambda message: tai.muse_message_hash(message),
    "model_weight": lambda entry: tai.model_weight(entry),
    "model_outcome_reward": lambda lat, tps: tai.model_outcome_reward(lat, tps),
    "summarize_errors": lambda errors, shown, width: tai.ChatClient.summarize_errors(errors, shown, width),
    "split_models": lambda spec: tai.split_models(spec),
    "parse_backends": lambda spec: tai.parse_backends(spec),
    "is_free_model": lambda model: tai.is_free_model(model),
    "embedded_failure_message": lambda data: tai.embedded_failure_message(data),
    "stream_status": lambda code, output, elapsed, timed_out: tai.stream_status(code, output, elapsed, timed_out),
    "curl_hint": lambda command: tai.curl_hint(command),
    "format_schedule_lines": lambda items, now: tai.format_schedule_lines(items, schedule_now(now)),
    "py_repr": lambda value: repr(value),
    "py_str": lambda value: str(value),
    "py_strip": lambda text: text.strip(),
    "py_lower": lambda text: text.lower(),
    "py_splitlines": lambda text: text.splitlines(),
    "py_split_ws": lambda text: text.split(),
    "py_int": lambda value: int_or_none(value),
    "py_float": lambda value: float_or_none(value),
    "py_slice": lambda text, start, end: text[start:end],
    "py_len": lambda text: len(text),
    "float_repr": lambda value: repr(float(value)),
    "sha1": lambda text: hashlib.sha1(text.encode("utf-8", "surrogatepass")).hexdigest(),
    "sha256": lambda text: hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest(),
    "utf8_replace": lambda text: text.encode("latin-1").decode("utf-8", "replace"),
    "orientation_block": lambda directory, sandbox: tai.orientation_block(directory or None, sandbox),
    "capability_block": lambda yolo, auto: tai.capability_block(yolo, auto),
    "sessions_block": lambda home, profile: tai.sessions_block(home, profile),
    "split_leading_tags": lambda text: list(tai.split_leading_tags(text)),
    "expand_file_mentions": lambda text, directory: tai.expand_file_mentions(text, directory),
    "profile_ok": lambda s: bool(tai.PROFILE_RE.match(s)),
    "bot_ok": lambda s: bool(tai.BOT_RE.match(s)),
    "secret_ok": lambda s: bool(tai.SECRET_RE.match(s)),
    "skill_name_ok": lambda s: bool(tai.SKILL_NAME_RE.match(s)),
    "mcp_server_ok": lambda s: bool(tai.MCP_SERVER_RE.match(s)),
    "create_skill_prompt": lambda name, brief, skill_dir: tai.create_skill_prompt(name, brief, skill_dir),
    "data_texts": lambda: {"default_system": tai.DEFAULT_SYSTEM, "merge_system": tai.MERGE_SYSTEM,
                           "compact_system": tai.COMPACT_SYSTEM, "autonomous_note": tai.AUTONOMOUS_NOTE,
                           "legacy_password_note": tai.LEGACY_PASSWORD_NOTE, "vault_memory_note": tai.VAULT_MEMORY_NOTE,
                           "mcp_resolver_brief": tai.MCP_RESOLVER_BRIEF, "goal_task": tai.GoalVerifier.TASK,
                           "curl_hint": tai.CURL_HINT},
}

POOL = ["a", "b", "c", "x", "Y", "Z", "0", "1", "9", " ", " ", "\t", "\n", "\r\n", "-", "_", ".", ",", ";", ":", "!", "?",
        "(", ")", "[", "]", "{", "}", "\"", "'", "\\", "/", "@", "#", "&", "<", ">", "=", "+", "*", "%", "é", "é",
        "ß", "İ", "K", " ", "
", "　", "中", "文", "\U0001f600", "€", "\x7f", "\x1c",
        "TODO", "token", "secret", "readme", "README", "http://x", "curl", "wget", "&amp;", "&lt;", "&#65;", "&#x41;", "&nbsp;",
        "&copy", "&unknown;", "ies", "ses", "news", "ss", "tags", "record"]


def fuzz_text(rng, max_parts=14):
    return "".join(rng.choice(POOL) for _ in range(rng.randint(0, max_parts)))


def fuzz_json(rng, depth=0):
    kind = rng.randint(0, 9 if depth < 3 else 5)
    if kind == 0:
        return None
    if kind == 1:
        return rng.choice([True, False])
    if kind == 2:
        return rng.randint(-1000, 10 ** 12)
    if kind == 3:
        return rng.choice([0.0, 1.5, -2.25, 1e20, 1e-7, 123456.789, 0.1, 1 / 3])
    if kind in (4, 5):
        return fuzz_text(rng, 6)
    if kind in (6, 7):
        return [fuzz_json(rng, depth + 1) for _ in range(rng.randint(0, 4))]
    return {fuzz_text(rng, 3): fuzz_json(rng, depth + 1) for _ in range(rng.randint(0, 4))}


def fuzz_messages(rng):
    messages = []
    for _ in range(rng.randint(0, 7)):
        role = rng.choice(["system", "user", "assistant", "tool"])
        item = {"role": role, "content": rng.choice([fuzz_text(rng, 5), None, "", fuzz_text(rng, 30)])}
        if role == "assistant" and rng.random() < 0.6:
            calls = []
            for k in range(rng.randint(1, 3)):
                name = rng.choice(["shell", "read_file", "bad name!", "x" * 70, "web_fetch", "", "a.b-c_d"])
                args = rng.choice(['{"a": 1}', "", "not json", "{}", None, {"a": 1}, "[1, 2]"])
                calls.append({"id": "c%d" % rng.randint(0, 3), "type": "function", "function": {"name": name, "arguments": args}})
            item["tool_calls"] = calls
        if role == "tool":
            item["tool_call_id"] = "c%d" % rng.randint(0, 3)
        messages.append(item)
    return messages


PROVIDERS = ["lab", "devplace", "openrouter", "pollinations", "zen", "ollama", "muse", "opencode", "other"]
MODELS = ["m1", "m2", "molodetz", "free", "muse-spark-x", "a:free", "b-free", "openai-fast", "deepseek/deepseek-v4.1-flash", "z"]


def fuzz_config(rng):
    pick = lambda n: rng.sample(MODELS, rng.randint(0, n))  # noqa: E731
    backends = []
    for i in range(rng.randint(0, 3)):
        backends.append({"label": rng.choice(["lab", "devplace", "Acme", "other"]), "base": "http://h%d/v1" % i, "key": rng.choice(["", "k"]),
                         "model": ",".join(pick(2)), "profile": rng.choice(["", "opencode"])})
    return {"model": rng.choice(["muse", "m1"]), "explicit_model": rng.choice(["", "m2", "xx"]), "backends": backends,
            "openrouter_key": rng.choice(["", "key"]), "openrouter_models": pick(3), "retry_seconds": 0, "opencode_enabled": rng.random() < 0.5,
            "opencode_models": pick(2), "devplace_key": rng.choice(["", "dk"]), "devplace_base": "https://devplace.net/openai/v1",
            "devplace_models": pick(2), "openrouter_enabled": rng.random() < 0.7, "pollinations_enabled": rng.random() < 0.7,
            "pollinations_models": pick(2), "zen_enabled": rng.random() < 0.4, "zen_models": pick(3), "ollama_enabled": rng.random() < 0.6,
            "ollama_models": pick(2)}


def fuzz_records(rng):
    out = []
    for _ in range(rng.randint(0, 12)):
        out.append([rng.choice(PROVIDERS), rng.choice(MODELS), rng.random() < 0.6, rng.choice([None, 100.0, 5000.5]), rng.choice([None, 10.0, 80.0])])
    return out


def build_cases(rng, fuzz):
    cases = []

    def add(fn, *args):
        cases.append({"fn": fn, "args": list(args)})

    for _ in range(max(40, fuzz // 3)):
        pins = rng.choice([{}, {}, {"provider": rng.choice(PROVIDERS)}, {"model": rng.choice(MODELS)}, {"provider": rng.choice(PROVIDERS), "model": rng.choice(MODELS)}])
        add("chat_candidates", fuzz_config(rng), fuzz_records(rng), pins)
    add("schemas")
    add("tool_catalog")
    add("data_texts")
    prompts = ["", "hello", "please run the tests and fix the failing build", "search the web for rust news", "read the file readme",
               "schedule a reminder every hour", "remember my favorite color", "store a password in the vault", "delete the old file",
               "describe this image screenshot", "fork a subagent to research in parallel", "install the skill from github",
               "mcp server list", "audit changes since yesterday", "release a new version", "create a bot persona", "graph link nodes",
               "tail terminals tmux panes", "what is the system info of this machine?", "translate this poem",
               "SEARCH the WEB", "Kelvin K sign and İstanbul", "tag tags tagging", "ask the user a question and wait"]
    for p in prompts:
        add("select_tools", p)
    for _ in range(fuzz):
        add("select_tools", fuzz_text(rng, 25))
    extra = [{"type": "function", "function": {"name": "mcp_demo_weather", "description": "Fetch the weather forecast for a city", "parameters": {}}},
             {"type": "function", "function": {"name": "mcp_demo_list", "description": "short", "parameters": {}}}]
    for p in ["weather forecast please", "list things", "forecast", ""]:
        add("select_tools_extra", p, extra)

    for _ in range(fuzz):
        t = fuzz_text(rng, 40)
        add("truncate", t * rng.randint(1, 120), rng.choice([10, 100, 1500]), rng.choice([5, 50, 500]))
        add("estimate_tokens", rng.choice([t, None, "", [1, 2, 3, 4, 5]]))
        add("normalize_tag", t)
        add("normalize_tags", [fuzz_text(rng, 4) for _ in range(rng.randint(0, 25))])
        add("singular_noun", fuzz_text(rng, 3))
        add("tag_variants", fuzz_text(rng, 3))
        add("secret_env_name", t)
        add("html_unescape", t)
        add("url_quote_plus", t)
        add("chunk_lines", "\n".join(fuzz_text(rng, 30) for _ in range(rng.randint(0, 200))))
        add("collapse_repeated_lines", "\n".join(rng.choice(["a", "a", "a", "b", fuzz_text(rng, 2)]) for _ in range(rng.randint(0, 30))))
        add("tool_name_clean", rng.choice([t, None, "shell<|channel|>commentary", "  x  "]))
        add("tool_args_valid", rng.choice([t, None, "{}", '{"a":1}', {"a": 1}, "", "[1]"]))
        add("tool_args_text", rng.choice([t, None, "", {"a": [1, "é"]}, ["x"], 0, 5]))
        add("action_signature", rng.choice(["shell", "read_file"]), rng.choice(['{"b":1,"a":2}', "nope", None, "", {"z": 1, "a": 2}]))
        add("result_digest", t)
        msgs = fuzz_messages(rng)
        add("clean_history", msgs)
        add("conversation_tokens", msgs)
        add("conversation_text", msgs)
        add("goal_transcript", msgs)
        add("goal_tried", msgs)
        add("approval_parse", t)
        add("approval_affirm", rng.choice([t, "Yes!", "  go   ahead. ", "LOOKS good", "yes, but no", "ok.?"]))
        add("py_repr", fuzz_json(rng))
        add("py_str", fuzz_json(rng))
        add("py_strip", t)
        add("py_lower", t)
        add("py_splitlines", t)
        add("py_split_ws", t)
        add("py_int", rng.choice([fuzz_json(rng), "12", " 7 ", "1_0", "0x10", "1e3", "-5", "+3", "", "٣"]))
        add("py_float", rng.choice([fuzz_json(rng), "1.5", " 2 ", "1e3", "nan", "inf", "-Infinity", "1_0.5", "0x1p3", "."]))
        add("py_slice", t, rng.choice([None, -3, 0, 2, 50]), rng.choice([None, -1, 0, 4, 99]))
        add("py_len", t)
        add("float_repr", rng.choice([0.1, 1e16, 1e-5, 123456789.123456789, 5e-324, 1.7976931348623157e308, 100.0, 1 / 3, 2.5e22]))
        add("sha1", t)
        add("sha256", t)
        add("split_leading_tags", rng.choice(["@a b", "@a @b rest", "@@x", "plain", "@ x", "@x　y", "@a\n@b\tc " + t]))
        add("profile_ok", rng.choice([t, "abc", "abc\n", "a" * 32, "a" * 33, "-x", "a_b-c"]))
        add("bot_ok", rng.choice([t, "abc", "ABC", "a-b_c"]))
        add("secret_ok", rng.choice([t, "a.b-c", "a" * 64, "a" * 65]))
        add("skill_name_ok", rng.choice([t, "ab", "a-b", "a-", "-a", "a" * 64, "a" * 65, "x\n"]))
        add("mcp_server_ok", rng.choice([t, "ok.name", "a" * 64]))
        add("json_roundtrip", json.dumps(fuzz_json(rng), ensure_ascii=rng.random() < 0.5))
        add("json_compact_sorted", json.dumps(fuzz_json(rng)))
        add("model_outcome_reward", rng.choice([None, 0, 120.5, 8000]), rng.choice([None, 0, 12.5, 100]))
        add("model_weight", rng.choice([{}, {"s": 3, "f": 1, "r": 2.5}, {"s": 0, "f": 0}, {"r": 1.0, "s": None, "f": 2}]))

    for raw in ["", "{", "[1,]", '{"a":1,}', '{"a":1 "b":2}', '{"a" 1}', '"abc', '"a\\q"', '"\\u12"', "tru", "nan", "NaN", "-", "1e", "[1 2]",
                "[1,2] x", '{"a":1}\n', "\t[ ]", '"a\tb"', '{"é": "ü\\ud83d\\ude00"}', '"\\ud800"', "1.5e300", "1" * 30, "-0", "-0.0",
                "[" * 5 + "]" * 5, "{}", '{"a":{"b":[1,2,{"c":null}]}}', '{"a":1,"a":2}', "1.", ".5", "01", "+1"]:
        add("json_roundtrip", raw)
        add("json_compact_sorted", raw)

    for raw in ["", "2026-10-08T09:00", "2026-10-08 09:00:15.5", "2026-10-08T09:00Z", "2026-10-08T09:00+02:00", "2026-10-08T09:00-0530",
                "2026-10-08", "20261008", "2026-13-01", "2026-02-30", "garbage", "2026-10-08T25:00", "2026-03-29T02:30", "2026-10-25T02:30",
                "2026-10-08T09:00:00.123456789", " 2026-10-08T09:00 ", "2026-10-08t09:00", "2026-10-08T09", "2026-10-08T09:00+02"]:
        add("parse_schedule_at", raw)
        add("local_display", raw)
    for s in [-5, 0, 59, 60, 3599, 3600, 86399, 86400, 10 ** 7]:
        add("format_delay", s)
    for raw in ["", "5", "5s", "10 m", "3h", "2d", "1w", "x", "5x", "1.5h", " 7D ", "99999999999999999999"]:
        add("parse_duration", raw)
    items = [{"id": 1, "status": "pending", "every": 0, "next_run": "2026-10-25T12:00:00+00:00", "tags": ["a", "b"], "name": "x", "prompt": "p" * 100, "last": "done", "result": "r" * 200},
             {"id": 2, "status": "pending", "every": 3600, "next_run": "bad", "tags": [], "name": "", "prompt": "q", "last": "", "result": ""}]
    add("format_schedule_lines", items, 1792924800.0)
    add("format_schedule_lines", [], 1792924800.0)

    for raw in ["&amp;&lt;&gt;&quot;&#39;", "&notit;", "&notin;", "&#0;", "&#128;", "&#x80;", "&#xD800;", "&#1114112;", "&#99999999999;", "&amp", "&ampx", "a&b",
                "&;", "&#;", "&#x;", "&#xzz;", "&AMP;", "&NotEqualTilde;", "&hearts;&hearts", "&euro;", "&#65", "&#x41", "&nbsp;x&NBSP;", "&" * 40]:
        add("html_unescape", raw)
    for raw in ["hello world", "a+b=c&d", "é中\U0001f600", "~_.-", "100%"]:
        add("url_quote_plus", raw)

    for raw in ["", "{}", '```json\n{"success": true, "failures": []}\n```', '{"success": true, "failures": [{"item": "a", "reason": "b", "proposal": "c"}]}',
                '{"success": false, "failures": [{"item": "a"}]}', '{"success": false, "failures": []}', '{"success": "yes", "failures": []}',
                '{"success": true, "failures": [], "commentary": "  fine  "}', "no json", '{"success": true,', "[1, 2]",
                'text {"success": false, "failures": [{"item": "i", "reason": "r", "proposal": "p"}], "commentary": "c"} tail',
                '{"success": false, "failures": ' + json.dumps([{"item": str(i), "reason": "r", "proposal": "p"} for i in range(12)]) + "}",
                '{"success": true, "failures": "none"}', '{"success": null}', "```\n{}\n```", "```JSON {\"success\": true, \"failures\": []}"]:
        add("goal_verdict", raw, "  fix   the\tbug  ")
        add("extract_json", raw)
    add("goal_task", "do x", "done", "[user] hi", ["a", "b"])
    add("goal_task", "", "", "", [])

    sample_msgs = [{"role": "system", "content": "s"}, {"role": "user", "content": "do thing\nplease"},
                   {"role": "assistant", "content": "ok", "tool_calls": [{"id": "1", "type": "function", "function": {"name": "shell", "arguments": '{"command": "ls"}'}}]},
                   {"role": "tool", "tool_call_id": "1", "content": "out"},
                   {"role": "user", "content": [{"type": "text", "text": "part one"}, {"type": "text", "text": "part two"}, "x"]}]
    tools = [{"type": "function", "function": {"name": "t", "description": "d", "parameters": {"type": "object", "properties": {}, "required": []}}}]
    add("muse_prompt", sample_msgs, tools)
    add("muse_prompt", sample_msgs, None)
    add("muse_prompt", [], [])
    add("opencode_prompt", sample_msgs, tools)
    for raw in ["", "plain answer", '{"content": "hi", "tool_calls": []}', '```json\n{"content": "x", "tool_calls": [{"name": "a", "arguments": "{}"}]}\n```',
                'noise {"schema": 1} then {"content": "late", "tool_calls": []} tail', '{"tool_calls": []}', '{"nothing": 1}', "{{{", '```\n```',
                "```json\n```", "``````", "```json{\"content\":\"z\"}```", '{"content": "a"} {"content": "b"}']:
        add("opencode_parse", raw)
        add("opencode_find_answer", raw)
        add("grok_parse_output", raw)
        add("claude_parse_output", raw)
        add("gemini_parse_output", raw)
        add("codex_parse_output", raw, "", False)
        add("codex_parse_output", "", raw, True)
        add("codex_parse_output", raw, raw, False)
        add("muse_valid_answer", raw)
    for obj in [{"is_error": True, "result": "bad"}, {"error": "x"}, {"error": {"a": 1}}, {"result": " answer "}, {"text": "t"}, {"structured_output": {"content": "c", "tool_calls": []}},
                {"response": '{"content": "r", "tool_calls": []}'}, {"content": "c", "tool_calls": []}, {"output": "o"}, [1], 5, {"is_error": False, "result": 5}]:
        raw = json.dumps(obj)
        add("grok_parse_output", raw)
        add("claude_parse_output", raw)
        add("gemini_parse_output", raw)
        add("muse_valid_answer", raw)
    add("codex_parse_output", '{"type": "item.completed", "item": {"type": "agent_message", "text": "{\\"content\\": \\"q\\", \\"tool_calls\\": []}"}}\nnoise', "", False)
    add("codex_parse_output", '{"type": "turn.failed", "message": "boom"}', "", False)
    add("codex_parse_output", '{"type": "error", "error": {"message": "deep"}}', "", False)
    add("codex_parse_output", '{"type": "error", "error": {"nomsg": 1}}', "", False)
    add("opencode_parse_events", '{"type": "text", "part": {"text": "hello "}}\n{"type": "text", "part": {"text": "world"}}\nnot json\n', 0, "")
    add("opencode_parse_events", '{"type": "error", "error": {"data": {"message": "m", "statusCode": 429}}}', 1, "")
    add("opencode_parse_events", '{"type": "error", "error": {"name": "X"}}', 1, "")
    add("opencode_parse_events", "", 3, "stderr text " * 40)
    add("opencode_parse_events", "out", 3, "")
    add("opencode_parse_events", '[1]\n{"type": "text"}\n', 0, "e")

    for ident, errs in enumerate([[], ["a/b: x"], ["a: CLI not found", "b: err1", "b: err1", "c: err2", "d: err3"], ["muse: muse CLI not found"], ["x"] * 3 + ["y" * 500]]):
        add("summarize_errors", errs, 2, 320)
        add("summarize_errors", errs, 6, 900)
    for spec in ["a,b\nc, ,d", "", " x "]:
        add("split_models", spec)
    add("parse_backends", "lab|https://x/v1/|key|m1,m2|opencode;bad|x\nl2|http://h|k2")
    add("parse_backends", "a|b|$NOPE_NOT_SET")
    add("parse_backends", "")
    for model in ["a:free", "x-FREE", "Model-free", "paid", "", "free"]:
        add("is_free_model", model)
    for d in [{"error": {"message": " m "}}, {"error": {"type": "t"}}, {"error": " s "}, {"type": "error", "message": "mm"}, {"type": "error"}, {"ok": 1}, [1], {"error": {}}]:
        add("embedded_failure_message", d)
    for args in [(0, "a\nb\n", 1.234, False), (1, "", 0.05, False), (0, "x" * 2000, 12.55, True), (124, "ü\r\n", 3.25, False)]:
        add("stream_status", *args)
    for command in ["curl http://x", "wget https://y", "curl localhost", "xcurl http://x", "echo curl_http", "WGET HTTP://X", "curl-x http://z"]:
        add("curl_hint", command)
    add("create_skill_prompt", "n", "brief\nmore", "/tmp/skills/n")
    add("capability_block", False, False)
    add("capability_block", True, False)
    add("capability_block", False, True)
    add("orientation_block", "", True)
    add("orientation_block", HOME, False)
    add("orientation_block", "/nonexistent/dir/../x", False)
    add("sessions_block", HOME, "default")
    add("sessions_block", HOME + "/", "work")
    add("sessions_block", "/nonexistent", "t")
    workdir = tempfile.mkdtemp(prefix="agent-diff-work-")
    with open(os.path.join(workdir, "a.txt"), "w") as handle:
        handle.write("x")
    for text in ["see @a.txt, and @missing.txt!", "@a.txt)", "mail me@a.txt", "@", "@@a.txt", "x @%s/a.txt y" % workdir, "@~", "no mentions", "@a.txt　@a.txt"]:
        add("expand_file_mentions", text, workdir)
    for raw in ["\xc3\xa9", "\xff\xfe", "\xe2\x82", "\xe2\x82\xac", "\xed\xa0\x80", "\xf0\x9f\x98", "\xf0\x9f\x98\x80", "\xc0\x80", "\xf4\x90\x80\x80", "ok", "\xe0\x80\x80\xe0\xa0\x80", "\x80\x80"]:
        add("utf8_replace", raw)
    for value in [None, True, 5, 2.5, "it's", 'say "hi"', "both ' and \"", "tab\there", "\x00\x1f\x7f", "é ​\U0001f600", [], {}, [1, "a", None, {"k": [True]}], {"a": {"b": 1.0}}, "\ud800"]:
        add("py_repr", value)
        add("py_str", value)
    return cases


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--seed", type=int, default=1)
    parser.add_argument("--fuzz", type=int, default=150)
    parser.add_argument("--bin", default=os.path.join(HERE, "..", "build", "agent_diff"))
    parser.add_argument("-v", "--verbose", action="store_true")
    args = parser.parse_args()
    rng = random.Random(args.seed)
    cases = build_cases(rng, args.fuzz)
    payload = "\n".join(json.dumps(case) for case in cases) + "\n"
    env = dict(os.environ)
    done = subprocess.run([args.bin], input=payload.encode("utf-8", "surrogatepass"), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
    if done.returncode != 0:
        sys.stderr.write("binary failed (%s): %s\n" % (done.returncode, done.stderr.decode("utf-8", "replace")[-2000:]))
        return 2
    outputs = done.stdout.decode("utf-8").splitlines()
    if len(outputs) != len(cases):
        sys.stderr.write("expected %d result lines, got %d\n" % (len(cases), len(outputs)))
        return 2
    failures = 0
    per_fn = {}
    for case, line in zip(cases, outputs):
        expected_fn = EXPECT[case["fn"]]
        try:
            expected = expected_fn(*case["args"])
        except Exception as exc:  # noqa: BLE001
            expected = {"exception": str(exc)}
        actual = json.loads(line)
        # documented limit: int() results beyond int64 clamp in C++ (Python ints are unbounded)
        if case["fn"] in ("py_int", "parse_duration") and isinstance(expected, int) and not -2 ** 63 <= expected < 2 ** 63:
            expected = 2 ** 63 - 1 if expected > 0 else -2 ** 63
        # normalize through JSON so tuples/ints compare like the C++ side
        expected_norm = json.loads(json.dumps(expected))
        stats = per_fn.setdefault(case["fn"], [0, 0])
        stats[0] += 1
        if dumps_sorted(expected_norm) != dumps_sorted(actual):
            failures += 1
            stats[1] += 1
            if failures <= 40 or args.verbose:
                sys.stdout.write("MISMATCH %s %s\n  expected: %s\n  actual:   %s\n" % (
                    case["fn"], json.dumps(case["args"], ensure_ascii=True)[:300], dumps_sorted(expected_norm)[:400], dumps_sorted(actual)[:400]))
    sys.stdout.write("%d cases, %d mismatches\n" % (len(cases), failures))
    for fn in sorted(per_fn):
        total, bad = per_fn[fn]
        if bad:
            sys.stdout.write("  %-26s %4d cases %4d mismatches\n" % (fn, total, bad))
    return 1 if failures else 0


if __name__ == "__main__":
    sys.exit(main())