#!/usr/bin/env python3 """Differential test of the C++ agent core against tai.py. Builds cases (curated plus seeded fuzz), computes the expected result with tai.py, feeds the same cases to build/agent_diff (tests/agent_diff.cpp) and compares the JSON results. Exit 0 only when every case matches. python3 tests/agent_diff.py [--seed N] [--fuzz N] [--bin build/agent_diff] [-v] """ import argparse import hashlib import html import json import os import random import subprocess import sys import tempfile import urllib.parse from datetime import datetime, timezone HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.abspath(os.path.join(HERE, "..", "..")) # a hermetic environment: no tmux, no plexer, no tui socket, fixed zone HOME = tempfile.mkdtemp(prefix="agent-diff-home-") os.environ.pop("TMUX", None) os.environ.pop("TMUX_PANE", None) for key in list(os.environ): if key.startswith("TAI_"): del os.environ[key] os.environ["TAI_HOME"] = HOME os.environ["TZ"] = "Europe/Amsterdam" os.environ["PATH"] = "/usr/bin:/bin" for _name in ("MUSE_BIN", "OPENCODE_BIN", "GROK_BIN", "CLAUDE_BIN", "CODEX_BIN", "GEMINI_BIN"): os.environ.pop(_name, None) sys.path.insert(0, ROOT) sys.dont_write_bytecode = True import tai # noqa: E402 import types # noqa: E402 import urllib.error # noqa: E402 import urllib.request # noqa: E402 def _fail_urlopen(*args, **kwargs): raise urllib.error.URLError("fake") def py_chat_candidates(cfg, records, pins): original = urllib.request.urlopen urllib.request.urlopen = _fail_urlopen try: home = tempfile.mkdtemp(prefix="agent-diff-chat-") cfg = dict(cfg, home=home) config = types.SimpleNamespace(**cfg) chat = tai.ChatClient(config) for provider, model, ok, lat, tps in records: chat.health.record(provider, model, ok, lat, tps) chat.provider_pin = pins.get("provider") chat.model_pin = pins.get("model") def view(c): return [c["provider"], c["chat_url"], c["models_url"], c["key"], c["model"], c["profile"], c.get("runner", "")] return {"ordered": [view(c) for c in chat.ordered_candidates()], "all": [view(c) for c in chat.all_candidates()], "backends": [list(b) for b in chat.backends()], "pin_note": chat.pin_note()} finally: urllib.request.urlopen = original def dumps_sorted(value): return json.dumps(value, sort_keys=True, ensure_ascii=True) def guard(func): def run(*args): try: return func(*args) except tai.BackendError as exc: return {"backend_error": str(exc), "status": exc.status} except Exception as exc: # noqa: BLE001 return {"exception": str(exc)} return run def verdict_value(text, goal): v = tai.GoalVerdict.from_text(text) return {"ok": v.ok, "failures": [f.to_dict() for f in v.failures], "commentary": v.commentary, "raw": v.raw, "report": v.report(goal), "followup": v.followup()} def names(picked): return [s["function"]["name"] for s in picked] def int_or_none(value): try: return int(value) except (TypeError, ValueError): return None def float_or_none(value): try: return float(value) except (TypeError, ValueError): return None def schedule_now(epoch): return datetime.fromtimestamp(epoch, timezone.utc) EXPECT = { "chat_candidates": lambda cfg, records, pins: py_chat_candidates(cfg, records, pins), "json_roundtrip": guard(lambda text: json.dumps(json.loads(text))), "json_compact_sorted": guard(lambda text: json.dumps(json.loads(text), sort_keys=True, separators=(",", ":"))), "schemas": lambda: json.dumps(tai.TOOL_SCHEMAS), "select_tools": lambda text: names(tai.select_tools(text)), "select_tools_extra": lambda text, extra: names(tai.select_tools(text, extra)), "tool_catalog": lambda: tai.tool_catalog(), "truncate": lambda text, head, tail: tai.truncate(text, head, tail), "estimate_tokens": lambda value: tai.estimate_tokens(value), "normalize_tag": lambda raw: tai.normalize_tag(raw), "normalize_tags": lambda items: tai.normalize_tags(items), "singular_noun": lambda word: tai.singular_noun(word), "tag_variants": lambda tag: tai.tag_variants(tag), "parse_node_id": lambda raw: (lambda r: list(r) if r else None)(tai.parse_node_id(raw)), "secret_env_name": lambda name: tai.secret_env_name(name), "format_delay": lambda s: tai.format_delay(s), "parse_duration": lambda raw: tai.parse_duration(raw), "parse_schedule_at": guard(lambda raw: tai.parse_schedule_at(raw)), "local_display": lambda raw: tai.local_display(raw), "html_unescape": lambda text: html.unescape(text), "url_quote_plus": lambda text: urllib.parse.urlencode({"q": text})[2:], "chunk_lines": lambda text: tai.chunk_lines(text), "collapse_repeated_lines": lambda text: tai.collapse_repeated_lines(text), "tool_name_clean": lambda raw: tai.tool_name_clean(raw), "tool_args_valid": lambda raw: tai.tool_args_valid(raw), "tool_args_text": lambda raw: tai.tool_args_text(raw), "action_signature": lambda name, raw: tai.action_signature(name, raw), "result_digest": lambda text: tai.result_digest(text), "clean_history": lambda messages: tai.clean_history(messages), "conversation_tokens": lambda messages, tools=None: tai.conversation_tokens(messages, tools), "conversation_text": lambda messages: tai.conversation_text(messages), "approval_parse": lambda answer: tai.Approval.parse_allow(answer), "approval_affirm": lambda text: tai.Approval.is_affirmative(text), "goal_verdict": lambda text, goal: verdict_value(text, goal), "extract_json": lambda text: list(tai.GoalVerifier.extract_json(text)), "goal_transcript": lambda messages: tai.GoalVerifier.transcript(messages), "goal_tried": lambda messages: tai.GoalVerifier.tried_actions(messages), "goal_task": lambda goal, final, transcript, tried: tai.GoalVerifier.build_task(goal, final, transcript, tried), "muse_prompt": lambda messages, tools: tai.muse_prompt(messages, tools), "opencode_prompt": lambda messages, tools: tai.opencode_prompt(messages, tools), "opencode_parse": lambda text: tai.opencode_parse(text), "opencode_find_answer": lambda text: tai.opencode_find_answer(text), "opencode_parse_events": guard(lambda out, code, err: tai.opencode_parse_events(out, code, err)), "grok_parse_output": guard(lambda text: tai.grok_parse_output(text)), "claude_parse_output": guard(lambda text: tai.claude_parse_output(text)), "gemini_parse_output": guard(lambda text: tai.gemini_parse_output(text)), "codex_parse_output": guard(lambda out, outfile, used: tai.codex_parse_output(out, outfile, used)), "muse_valid_answer": lambda text: tai.muse_valid_answer(text), "muse_message_hash": lambda message: tai.muse_message_hash(message), "model_weight": lambda entry: tai.model_weight(entry), "model_outcome_reward": lambda lat, tps: tai.model_outcome_reward(lat, tps), "summarize_errors": lambda errors, shown, width: tai.ChatClient.summarize_errors(errors, shown, width), "split_models": lambda spec: tai.split_models(spec), "parse_backends": lambda spec: tai.parse_backends(spec), "is_free_model": lambda model: tai.is_free_model(model), "embedded_failure_message": lambda data: tai.embedded_failure_message(data), "stream_status": lambda code, output, elapsed, timed_out: tai.stream_status(code, output, elapsed, timed_out), "curl_hint": lambda command: tai.curl_hint(command), "format_schedule_lines": lambda items, now: tai.format_schedule_lines(items, schedule_now(now)), "py_repr": lambda value: repr(value), "py_str": lambda value: str(value), "py_strip": lambda text: text.strip(), "py_lower": lambda text: text.lower(), "py_splitlines": lambda text: text.splitlines(), "py_split_ws": lambda text: text.split(), "py_int": lambda value: int_or_none(value), "py_float": lambda value: float_or_none(value), "py_slice": lambda text, start, end: text[start:end], "py_len": lambda text: len(text), "float_repr": lambda value: repr(float(value)), "sha1": lambda text: hashlib.sha1(text.encode("utf-8", "surrogatepass")).hexdigest(), "sha256": lambda text: hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest(), "utf8_replace": lambda text: text.encode("latin-1").decode("utf-8", "replace"), "orientation_block": lambda directory, sandbox: tai.orientation_block(directory or None, sandbox), "capability_block": lambda yolo, auto: tai.capability_block(yolo, auto), "sessions_block": lambda home, profile: tai.sessions_block(home, profile), "split_leading_tags": lambda text: list(tai.split_leading_tags(text)), "expand_file_mentions": lambda text, directory: tai.expand_file_mentions(text, directory), "profile_ok": lambda s: bool(tai.PROFILE_RE.match(s)), "bot_ok": lambda s: bool(tai.BOT_RE.match(s)), "secret_ok": lambda s: bool(tai.SECRET_RE.match(s)), "skill_name_ok": lambda s: bool(tai.SKILL_NAME_RE.match(s)), "mcp_server_ok": lambda s: bool(tai.MCP_SERVER_RE.match(s)), "create_skill_prompt": lambda name, brief, skill_dir: tai.create_skill_prompt(name, brief, skill_dir), "data_texts": lambda: {"default_system": tai.DEFAULT_SYSTEM, "merge_system": tai.MERGE_SYSTEM, "compact_system": tai.COMPACT_SYSTEM, "autonomous_note": tai.AUTONOMOUS_NOTE, "legacy_password_note": tai.LEGACY_PASSWORD_NOTE, "vault_memory_note": tai.VAULT_MEMORY_NOTE, "mcp_resolver_brief": tai.MCP_RESOLVER_BRIEF, "goal_task": tai.GoalVerifier.TASK, "curl_hint": tai.CURL_HINT}, } POOL = ["a", "b", "c", "x", "Y", "Z", "0", "1", "9", " ", " ", "\t", "\n", "\r\n", "-", "_", ".", ",", ";", ":", "!", "?", "(", ")", "[", "]", "{", "}", "\"", "'", "\\", "/", "@", "#", "&", "<", ">", "=", "+", "*", "%", "é", "é", "ß", "İ", "K", " ", "
", "　", "中", "文", "\U0001f600", "€", "\x7f", "\x1c", "TODO", "token", "secret", "readme", "README", "http://x", "curl", "wget", "&", "<", "A", "A", " ", "©", "&unknown;", "ies", "ses", "news", "ss", "tags", "record"] def fuzz_text(rng, max_parts=14): return "".join(rng.choice(POOL) for _ in range(rng.randint(0, max_parts))) def fuzz_json(rng, depth=0): kind = rng.randint(0, 9 if depth < 3 else 5) if kind == 0: return None if kind == 1: return rng.choice([True, False]) if kind == 2: return rng.randint(-1000, 10 ** 12) if kind == 3: return rng.choice([0.0, 1.5, -2.25, 1e20, 1e-7, 123456.789, 0.1, 1 / 3]) if kind in (4, 5): return fuzz_text(rng, 6) if kind in (6, 7): return [fuzz_json(rng, depth + 1) for _ in range(rng.randint(0, 4))] return {fuzz_text(rng, 3): fuzz_json(rng, depth + 1) for _ in range(rng.randint(0, 4))} def fuzz_messages(rng): messages = [] for _ in range(rng.randint(0, 7)): role = rng.choice(["system", "user", "assistant", "tool"]) item = {"role": role, "content": rng.choice([fuzz_text(rng, 5), None, "", fuzz_text(rng, 30)])} if role == "assistant" and rng.random() < 0.6: calls = [] for k in range(rng.randint(1, 3)): name = rng.choice(["shell", "read_file", "bad name!", "x" * 70, "web_fetch", "", "a.b-c_d"]) args = rng.choice(['{"a": 1}', "", "not json", "{}", None, {"a": 1}, "[1, 2]"]) calls.append({"id": "c%d" % rng.randint(0, 3), "type": "function", "function": {"name": name, "arguments": args}}) item["tool_calls"] = calls if role == "tool": item["tool_call_id"] = "c%d" % rng.randint(0, 3) messages.append(item) return messages PROVIDERS = ["lab", "devplace", "openrouter", "pollinations", "zen", "ollama", "muse", "opencode", "other"] MODELS = ["m1", "m2", "molodetz", "free", "muse-spark-x", "a:free", "b-free", "openai-fast", "deepseek/deepseek-v4.1-flash", "z"] def fuzz_config(rng): pick = lambda n: rng.sample(MODELS, rng.randint(0, n)) # noqa: E731 backends = [] for i in range(rng.randint(0, 3)): backends.append({"label": rng.choice(["lab", "devplace", "Acme", "other"]), "base": "http://h%d/v1" % i, "key": rng.choice(["", "k"]), "model": ",".join(pick(2)), "profile": rng.choice(["", "opencode"])}) return {"model": rng.choice(["muse", "m1"]), "explicit_model": rng.choice(["", "m2", "xx"]), "backends": backends, "openrouter_key": rng.choice(["", "key"]), "openrouter_models": pick(3), "retry_seconds": 0, "opencode_enabled": rng.random() < 0.5, "opencode_models": pick(2), "devplace_key": rng.choice(["", "dk"]), "devplace_base": "https://devplace.net/openai/v1", "devplace_models": pick(2), "openrouter_enabled": rng.random() < 0.7, "pollinations_enabled": rng.random() < 0.7, "pollinations_models": pick(2), "zen_enabled": rng.random() < 0.4, "zen_models": pick(3), "ollama_enabled": rng.random() < 0.6, "ollama_models": pick(2)} def fuzz_records(rng): out = [] for _ in range(rng.randint(0, 12)): out.append([rng.choice(PROVIDERS), rng.choice(MODELS), rng.random() < 0.6, rng.choice([None, 100.0, 5000.5]), rng.choice([None, 10.0, 80.0])]) return out def build_cases(rng, fuzz): cases = [] def add(fn, *args): cases.append({"fn": fn, "args": list(args)}) for _ in range(max(40, fuzz // 3)): pins = rng.choice([{}, {}, {"provider": rng.choice(PROVIDERS)}, {"model": rng.choice(MODELS)}, {"provider": rng.choice(PROVIDERS), "model": rng.choice(MODELS)}]) add("chat_candidates", fuzz_config(rng), fuzz_records(rng), pins) add("schemas") add("tool_catalog") add("data_texts") prompts = ["", "hello", "please run the tests and fix the failing build", "search the web for rust news", "read the file readme", "schedule a reminder every hour", "remember my favorite color", "store a password in the vault", "delete the old file", "describe this image screenshot", "fork a subagent to research in parallel", "install the skill from github", "mcp server list", "audit changes since yesterday", "release a new version", "create a bot persona", "graph link nodes", "tail terminals tmux panes", "what is the system info of this machine?", "translate this poem", "SEARCH the WEB", "Kelvin K sign and İstanbul", "tag tags tagging", "ask the user a question and wait"] for p in prompts: add("select_tools", p) for _ in range(fuzz): add("select_tools", fuzz_text(rng, 25)) extra = [{"type": "function", "function": {"name": "mcp_demo_weather", "description": "Fetch the weather forecast for a city", "parameters": {}}}, {"type": "function", "function": {"name": "mcp_demo_list", "description": "short", "parameters": {}}}] for p in ["weather forecast please", "list things", "forecast", ""]: add("select_tools_extra", p, extra) for _ in range(fuzz): t = fuzz_text(rng, 40) add("truncate", t * rng.randint(1, 120), rng.choice([10, 100, 1500]), rng.choice([5, 50, 500])) add("estimate_tokens", rng.choice([t, None, "", [1, 2, 3, 4, 5]])) add("normalize_tag", t) add("normalize_tags", [fuzz_text(rng, 4) for _ in range(rng.randint(0, 25))]) add("singular_noun", fuzz_text(rng, 3)) add("tag_variants", fuzz_text(rng, 3)) add("secret_env_name", t) add("html_unescape", t) add("url_quote_plus", t) add("chunk_lines", "\n".join(fuzz_text(rng, 30) for _ in range(rng.randint(0, 200)))) add("collapse_repeated_lines", "\n".join(rng.choice(["a", "a", "a", "b", fuzz_text(rng, 2)]) for _ in range(rng.randint(0, 30)))) add("tool_name_clean", rng.choice([t, None, "shell<|channel|>commentary", " x "])) add("tool_args_valid", rng.choice([t, None, "{}", '{"a":1}', {"a": 1}, "", "[1]"])) add("tool_args_text", rng.choice([t, None, "", {"a": [1, "é"]}, ["x"], 0, 5])) add("action_signature", rng.choice(["shell", "read_file"]), rng.choice(['{"b":1,"a":2}', "nope", None, "", {"z": 1, "a": 2}])) add("result_digest", t) msgs = fuzz_messages(rng) add("clean_history", msgs) add("conversation_tokens", msgs) add("conversation_text", msgs) add("goal_transcript", msgs) add("goal_tried", msgs) add("approval_parse", t) add("approval_affirm", rng.choice([t, "Yes!", " go ahead. ", "LOOKS good", "yes, but no", "ok.?"])) add("py_repr", fuzz_json(rng)) add("py_str", fuzz_json(rng)) add("py_strip", t) add("py_lower", t) add("py_splitlines", t) add("py_split_ws", t) add("py_int", rng.choice([fuzz_json(rng), "12", " 7 ", "1_0", "0x10", "1e3", "-5", "+3", "", "٣"])) add("py_float", rng.choice([fuzz_json(rng), "1.5", " 2 ", "1e3", "nan", "inf", "-Infinity", "1_0.5", "0x1p3", "."])) add("py_slice", t, rng.choice([None, -3, 0, 2, 50]), rng.choice([None, -1, 0, 4, 99])) add("py_len", t) add("float_repr", rng.choice([0.1, 1e16, 1e-5, 123456789.123456789, 5e-324, 1.7976931348623157e308, 100.0, 1 / 3, 2.5e22])) add("sha1", t) add("sha256", t) add("split_leading_tags", rng.choice(["@a b", "@a @b rest", "@@x", "plain", "@ x", "@x　y", "@a\n@b\tc " + t])) add("profile_ok", rng.choice([t, "abc", "abc\n", "a" * 32, "a" * 33, "-x", "a_b-c"])) add("bot_ok", rng.choice([t, "abc", "ABC", "a-b_c"])) add("secret_ok", rng.choice([t, "a.b-c", "a" * 64, "a" * 65])) add("skill_name_ok", rng.choice([t, "ab", "a-b", "a-", "-a", "a" * 64, "a" * 65, "x\n"])) add("mcp_server_ok", rng.choice([t, "ok.name", "a" * 64])) add("json_roundtrip", json.dumps(fuzz_json(rng), ensure_ascii=rng.random() < 0.5)) add("json_compact_sorted", json.dumps(fuzz_json(rng))) add("model_outcome_reward", rng.choice([None, 0, 120.5, 8000]), rng.choice([None, 0, 12.5, 100])) add("model_weight", rng.choice([{}, {"s": 3, "f": 1, "r": 2.5}, {"s": 0, "f": 0}, {"r": 1.0, "s": None, "f": 2}])) for raw in ["", "{", "[1,]", '{"a":1,}', '{"a":1 "b":2}', '{"a" 1}', '"abc', '"a\\q"', '"\\u12"', "tru", "nan", "NaN", "-", "1e", "[1 2]", "[1,2] x", '{"a":1}\n', "\t[ ]", '"a\tb"', '{"é": "ü\\ud83d\\ude00"}', '"\\ud800"', "1.5e300", "1" * 30, "-0", "-0.0", "[" * 5 + "]" * 5, "{}", '{"a":{"b":[1,2,{"c":null}]}}', '{"a":1,"a":2}', "1.", ".5", "01", "+1"]: add("json_roundtrip", raw) add("json_compact_sorted", raw) for raw in ["", "2026-10-08T09:00", "2026-10-08 09:00:15.5", "2026-10-08T09:00Z", "2026-10-08T09:00+02:00", "2026-10-08T09:00-0530", "2026-10-08", "20261008", "2026-13-01", "2026-02-30", "garbage", "2026-10-08T25:00", "2026-03-29T02:30", "2026-10-25T02:30", "2026-10-08T09:00:00.123456789", " 2026-10-08T09:00 ", "2026-10-08t09:00", "2026-10-08T09", "2026-10-08T09:00+02"]: add("parse_schedule_at", raw) add("local_display", raw) for s in [-5, 0, 59, 60, 3599, 3600, 86399, 86400, 10 ** 7]: add("format_delay", s) for raw in ["", "5", "5s", "10 m", "3h", "2d", "1w", "x", "5x", "1.5h", " 7D ", "99999999999999999999"]: add("parse_duration", raw) items = [{"id": 1, "status": "pending", "every": 0, "next_run": "2026-10-25T12:00:00+00:00", "tags": ["a", "b"], "name": "x", "prompt": "p" * 100, "last": "done", "result": "r" * 200}, {"id": 2, "status": "pending", "every": 3600, "next_run": "bad", "tags": [], "name": "", "prompt": "q", "last": "", "result": ""}] add("format_schedule_lines", items, 1792924800.0) add("format_schedule_lines", [], 1792924800.0) for raw in ["&<>"'", "¬it;", "∉", "�", "€", "€", "�", "�", "�", "&", "&x", "a&b", "&;", "&#;", "&#x;", "&#xzz;", "&", "≂̸", "♥&hearts", "€", "A", "A", " x&NBSP;", "&" * 40]: add("html_unescape", raw) for raw in ["hello world", "a+b=c&d", "é中\U0001f600", "~_.-", "100%"]: add("url_quote_plus", raw) for raw in ["", "{}", '```json\n{"success": true, "failures": []}\n```', '{"success": true, "failures": [{"item": "a", "reason": "b", "proposal": "c"}]}', '{"success": false, "failures": [{"item": "a"}]}', '{"success": false, "failures": []}', '{"success": "yes", "failures": []}', '{"success": true, "failures": [], "commentary": " fine "}', "no json", '{"success": true,', "[1, 2]", 'text {"success": false, "failures": [{"item": "i", "reason": "r", "proposal": "p"}], "commentary": "c"} tail', '{"success": false, "failures": ' + json.dumps([{"item": str(i), "reason": "r", "proposal": "p"} for i in range(12)]) + "}", '{"success": true, "failures": "none"}', '{"success": null}', "```\n{}\n```", "```JSON {\"success\": true, \"failures\": []}"]: add("goal_verdict", raw, " fix the\tbug ") add("extract_json", raw) add("goal_task", "do x", "done", "[user] hi", ["a", "b"]) add("goal_task", "", "", "", []) sample_msgs = [{"role": "system", "content": "s"}, {"role": "user", "content": "do thing\nplease"}, {"role": "assistant", "content": "ok", "tool_calls": [{"id": "1", "type": "function", "function": {"name": "shell", "arguments": '{"command": "ls"}'}}]}, {"role": "tool", "tool_call_id": "1", "content": "out"}, {"role": "user", "content": [{"type": "text", "text": "part one"}, {"type": "text", "text": "part two"}, "x"]}] tools = [{"type": "function", "function": {"name": "t", "description": "d", "parameters": {"type": "object", "properties": {}, "required": []}}}] add("muse_prompt", sample_msgs, tools) add("muse_prompt", sample_msgs, None) add("muse_prompt", [], []) add("opencode_prompt", sample_msgs, tools) for raw in ["", "plain answer", '{"content": "hi", "tool_calls": []}', '```json\n{"content": "x", "tool_calls": [{"name": "a", "arguments": "{}"}]}\n```', 'noise {"schema": 1} then {"content": "late", "tool_calls": []} tail', '{"tool_calls": []}', '{"nothing": 1}', "{{{", '```\n```', "```json\n```", "``````", "```json{\"content\":\"z\"}```", '{"content": "a"} {"content": "b"}']: add("opencode_parse", raw) add("opencode_find_answer", raw) add("grok_parse_output", raw) add("claude_parse_output", raw) add("gemini_parse_output", raw) add("codex_parse_output", raw, "", False) add("codex_parse_output", "", raw, True) add("codex_parse_output", raw, raw, False) add("muse_valid_answer", raw) for obj in [{"is_error": True, "result": "bad"}, {"error": "x"}, {"error": {"a": 1}}, {"result": " answer "}, {"text": "t"}, {"structured_output": {"content": "c", "tool_calls": []}}, {"response": '{"content": "r", "tool_calls": []}'}, {"content": "c", "tool_calls": []}, {"output": "o"}, [1], 5, {"is_error": False, "result": 5}]: raw = json.dumps(obj) add("grok_parse_output", raw) add("claude_parse_output", raw) add("gemini_parse_output", raw) add("muse_valid_answer", raw) add("codex_parse_output", '{"type": "item.completed", "item": {"type": "agent_message", "text": "{\\"content\\": \\"q\\", \\"tool_calls\\": []}"}}\nnoise', "", False) add("codex_parse_output", '{"type": "turn.failed", "message": "boom"}', "", False) add("codex_parse_output", '{"type": "error", "error": {"message": "deep"}}', "", False) add("codex_parse_output", '{"type": "error", "error": {"nomsg": 1}}', "", False) add("opencode_parse_events", '{"type": "text", "part": {"text": "hello "}}\n{"type": "text", "part": {"text": "world"}}\nnot json\n', 0, "") add("opencode_parse_events", '{"type": "error", "error": {"data": {"message": "m", "statusCode": 429}}}', 1, "") add("opencode_parse_events", '{"type": "error", "error": {"name": "X"}}', 1, "") add("opencode_parse_events", "", 3, "stderr text " * 40) add("opencode_parse_events", "out", 3, "") add("opencode_parse_events", '[1]\n{"type": "text"}\n', 0, "e") for ident, errs in enumerate([[], ["a/b: x"], ["a: CLI not found", "b: err1", "b: err1", "c: err2", "d: err3"], ["muse: muse CLI not found"], ["x"] * 3 + ["y" * 500]]): add("summarize_errors", errs, 2, 320) add("summarize_errors", errs, 6, 900) for spec in ["a,b\nc, ,d", "", " x "]: add("split_models", spec) add("parse_backends", "lab|https://x/v1/|key|m1,m2|opencode;bad|x\nl2|http://h|k2") add("parse_backends", "a|b|$NOPE_NOT_SET") add("parse_backends", "") for model in ["a:free", "x-FREE", "Model-free", "paid", "", "free"]: add("is_free_model", model) for d in [{"error": {"message": " m "}}, {"error": {"type": "t"}}, {"error": " s "}, {"type": "error", "message": "mm"}, {"type": "error"}, {"ok": 1}, [1], {"error": {}}]: add("embedded_failure_message", d) for args in [(0, "a\nb\n", 1.234, False), (1, "", 0.05, False), (0, "x" * 2000, 12.55, True), (124, "ü\r\n", 3.25, False)]: add("stream_status", *args) for command in ["curl http://x", "wget https://y", "curl localhost", "xcurl http://x", "echo curl_http", "WGET HTTP://X", "curl-x http://z"]: add("curl_hint", command) add("create_skill_prompt", "n", "brief\nmore", "/tmp/skills/n") add("capability_block", False, False) add("capability_block", True, False) add("capability_block", False, True) add("orientation_block", "", True) add("orientation_block", HOME, False) add("orientation_block", "/nonexistent/dir/../x", False) add("sessions_block", HOME, "default") add("sessions_block", HOME + "/", "work") add("sessions_block", "/nonexistent", "t") workdir = tempfile.mkdtemp(prefix="agent-diff-work-") with open(os.path.join(workdir, "a.txt"), "w") as handle: handle.write("x") for text in ["see @a.txt, and @missing.txt!", "@a.txt)", "mail me@a.txt", "@", "@@a.txt", "x @%s/a.txt y" % workdir, "@~", "no mentions", "@a.txt　@a.txt"]: add("expand_file_mentions", text, workdir) for raw in ["\xc3\xa9", "\xff\xfe", "\xe2\x82", "\xe2\x82\xac", "\xed\xa0\x80", "\xf0\x9f\x98", "\xf0\x9f\x98\x80", "\xc0\x80", "\xf4\x90\x80\x80", "ok", "\xe0\x80\x80\xe0\xa0\x80", "\x80\x80"]: add("utf8_replace", raw) for value in [None, True, 5, 2.5, "it's", 'say "hi"', "both ' and \"", "tab\there", "\x00\x1f\x7f", "é ​\U0001f600", [], {}, [1, "a", None, {"k": [True]}], {"a": {"b": 1.0}}, "\ud800"]: add("py_repr", value) add("py_str", value) return cases def main(): parser = argparse.ArgumentParser() parser.add_argument("--seed", type=int, default=1) parser.add_argument("--fuzz", type=int, default=150) parser.add_argument("--bin", default=os.path.join(HERE, "..", "build", "agent_diff")) parser.add_argument("-v", "--verbose", action="store_true") args = parser.parse_args() rng = random.Random(args.seed) cases = build_cases(rng, args.fuzz) payload = "\n".join(json.dumps(case) for case in cases) + "\n" env = dict(os.environ) done = subprocess.run([args.bin], input=payload.encode("utf-8", "surrogatepass"), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) if done.returncode != 0: sys.stderr.write("binary failed (%s): %s\n" % (done.returncode, done.stderr.decode("utf-8", "replace")[-2000:])) return 2 outputs = done.stdout.decode("utf-8").splitlines() if len(outputs) != len(cases): sys.stderr.write("expected %d result lines, got %d\n" % (len(cases), len(outputs))) return 2 failures = 0 per_fn = {} for case, line in zip(cases, outputs): expected_fn = EXPECT[case["fn"]] try: expected = expected_fn(*case["args"]) except Exception as exc: # noqa: BLE001 expected = {"exception": str(exc)} actual = json.loads(line) # documented limit: int() results beyond int64 clamp in C++ (Python ints are unbounded) if case["fn"] in ("py_int", "parse_duration") and isinstance(expected, int) and not -2 ** 63 <= expected < 2 ** 63: expected = 2 ** 63 - 1 if expected > 0 else -2 ** 63 # normalize through JSON so tuples/ints compare like the C++ side expected_norm = json.loads(json.dumps(expected)) stats = per_fn.setdefault(case["fn"], [0, 0]) stats[0] += 1 if dumps_sorted(expected_norm) != dumps_sorted(actual): failures += 1 stats[1] += 1 if failures <= 40 or args.verbose: sys.stdout.write("MISMATCH %s %s\n expected: %s\n actual: %s\n" % ( case["fn"], json.dumps(case["args"], ensure_ascii=True)[:300], dumps_sorted(expected_norm)[:400], dumps_sorted(actual)[:400])) sys.stdout.write("%d cases, %d mismatches\n" % (len(cases), failures)) for fn in sorted(per_fn): total, bad = per_fn[fn] if bad: sys.stdout.write(" %-26s %4d cases %4d mismatches\n" % (fn, total, bad)) return 1 if failures else 0 if __name__ == "__main__": sys.exit(main())