tai/taicpp/tests/agent_diff.cpp

// Differential driver for the agent core: reads one JSON case per line from
// stdin ({"fn": name, "args": [...]}) and prints one JSON result per line.
// tests/agent_diff.py computes the same cases with tai.py and compares.
#include "../src/agent/agent.h"
#include "../src/agent/text.h"
#include "../src/agent/data.h"
#include "../src/llm/chat_client.h"
#include "../src/llm/cli_backends.h"
#include "../src/llm/http_transport.h"
#include "../src/llm/helpers.h"
#include "../src/llm/model_health.h"
#include "../src/llm/muse.h"
#include "../src/llm/py.h"
#include <functional>
#include <iostream>
#include <map>
using namespace detz;
using json::Value;

namespace {
Value S(const std::string& s) { return Value(s); }
Value strs(const std::vector<std::string>& v) {
  Value a = Value::array();
  for (const auto& s : v) { a.push(s); }
  return a;
}
Value verdict_value(const agent::GoalVerdict& v, const std::string& goal) {
  Value o = Value::object();
  o.set("ok", v.ok);
  Value fs = Value::array();
  for (const auto& f : v.failures) { fs.push(f.to_dict()); }
  o.set("failures", fs);
  o.set("commentary", v.commentary);
  o.set("raw", v.raw);
  o.set("report", v.report(goal));
  o.set("followup", v.followup());
  return o;
}
Value try_value(const std::function<Value()>& f) {
  try {
    return f();
  } catch (const llm::BackendError& e) {
    Value o = Value::object();
    o.set("backend_error", std::string(e.what()));
    o.set("status", e.status());
    return o;
  } catch (const std::exception& e) {
    Value o = Value::object();
    o.set("exception", std::string(e.what()));
    return o;
  }
}

class FailTransport : public llm::HttpTransport {
 public:
  llm::HttpResult send(const llm::HttpRequest&) override { llm::HttpResult r; r.error = "<urlopen error fake>"; return r; }
  llm::HttpResult stream(const llm::HttpRequest&, const llm::LineSink&) override { llm::HttpResult r; r.error = "<urlopen error fake>"; return r; }
};

std::vector<std::string> str_list(const Value& v) {
  std::vector<std::string> out;
  for (const auto& x : v.items()) { out.push_back(x.str()); }
  return out;
}

Value candidate_view(const llm::Candidate& c) {
  Value a = Value::array();
  a.push(c.provider); a.push(c.chat_url); a.push(c.models_url); a.push(c.key); a.push(c.model); a.push(c.profile);
  a.push(c.runner);
  return a;
}

Value chat_candidates(const Value& a) {
  static FailTransport fail;
  llm::set_transport(&fail);
  const Value& c = a.at(0);
  llm::LlmConfig cfg;
  cfg.home = c.get("home").str();
  cfg.model = c.get("model").str();
  cfg.explicit_model = c.get("explicit_model").str();
  for (const auto& b : c.get("backends").items()) {
    llm::BackendSpec s;
    s.label = b.get("label").str(); s.base = b.get("base").str(); s.key = b.get("key").str();
    s.model = b.get("model").str(); s.profile = b.get("profile").str();
    cfg.backends.push_back(s);
  }
  cfg.openrouter_key = c.get("openrouter_key").str();
  cfg.openrouter_models = str_list(c.get("openrouter_models"));
  cfg.opencode_enabled = c.get("opencode_enabled").truthy();
  cfg.opencode_models = str_list(c.get("opencode_models"));
  cfg.devplace_key = c.get("devplace_key").str();
  cfg.devplace_base = c.get("devplace_base").str();
  cfg.devplace_models = str_list(c.get("devplace_models"));
  cfg.openrouter_enabled = c.get("openrouter_enabled").truthy();
  cfg.pollinations_enabled = c.get("pollinations_enabled").truthy();
  cfg.pollinations_models = str_list(c.get("pollinations_models"));
  cfg.zen_enabled = c.get("zen_enabled").truthy();
  cfg.zen_models = str_list(c.get("zen_models"));
  cfg.ollama_enabled = c.get("ollama_enabled").truthy();
  cfg.ollama_models = str_list(c.get("ollama_models"));
  llm::ChatClient chat(cfg);
  for (const auto& r : a.at(1).items()) {
    std::optional<double> lat, tps;
    if (!r.at(3).is_null()) { lat = r.at(3).as_double(); }
    if (!r.at(4).is_null()) { tps = r.at(4).as_double(); }
    chat.health.record(r.at(0).str(), r.at(1).str(), r.at(2).as_bool(), lat, tps);
  }
  if (a.at(2).get("provider").is_string()) { chat.provider_pin = a.at(2).get("provider").str(); }
  if (a.at(2).get("model").is_string()) { chat.model_pin = a.at(2).get("model").str(); }
  Value out = Value::object();
  Value ordered = Value::array();
  for (const auto& cand : chat.ordered_candidates()) { ordered.push(candidate_view(cand)); }
  out.set("ordered", ordered);
  Value all = Value::array();
  for (const auto& cand : chat.all_candidates()) { all.push(candidate_view(cand)); }
  out.set("all", all);
  Value backs = Value::array();
  for (const auto& b : chat.backends()) {
    Value row = Value::array();
    row.push(b.provider); row.push(b.chat_url); row.push(b.models_url); row.push(b.key); row.push(b.model);
    backs.push(row);
  }
  out.set("backends", backs);
  out.set("pin_note", chat.pin_note());
  return out;
}

using Fn = std::function<Value(const Value&)>;
std::map<std::string, Fn> registry() {
  std::map<std::string, Fn> r;
  r["chat_candidates"] = chat_candidates;
  r["json_roundtrip"] = [](const Value& a) {
    return try_value([&]() { return S(Value::loads(a.at(0).str()).dumps()); });
  };
  r["json_compact_sorted"] = [](const Value& a) {
    return try_value([&]() {
      json::DumpOptions o;
      o.sort_keys = true; o.item_sep = ","; o.key_sep = ":";
      return S(Value::loads(a.at(0).str()).dumps(o));
    });
  };
  r["schemas"] = [](const Value&) {
    Value arr = Value::array();
    for (const auto& s : agent::tool_schemas()) { arr.push(s); }
    return S(arr.dumps());
  };
  r["select_tools"] = [](const Value& a) {
    Value picked = agent::select_tools(a.at(0).str());
    Value names = Value::array();
    for (const auto& s : picked.items()) { names.push(s.get("function").get("name")); }
    return names;
  };
  r["select_tools_extra"] = [](const Value& a) {
    Value picked = agent::select_tools(a.at(0).str(), a.at(1));
    Value names = Value::array();
    for (const auto& s : picked.items()) { names.push(s.get("function").get("name")); }
    return names;
  };
  r["tool_catalog"] = [](const Value&) { return S(agent::tool_catalog()); };
  r["truncate"] = [](const Value& a) {
    return S(llm::truncate(a.at(0).str(), static_cast<size_t>(a.at(1).as_int()), static_cast<size_t>(a.at(2).as_int())));
  };
  r["estimate_tokens"] = [](const Value& a) { return Value(llm::estimate_tokens(a.at(0))); };
  r["normalize_tag"] = [](const Value& a) { return S(agent::normalize_tag(a.at(0).str())); };
  r["normalize_tags"] = [](const Value& a) {
    std::vector<std::string> in;
    for (const auto& x : a.at(0).items()) { in.push_back(x.str()); }
    return strs(agent::normalize_tags(in));
  };
  r["singular_noun"] = [](const Value& a) { return S(agent::singular_noun(a.at(0).str())); };
  r["tag_variants"] = [](const Value& a) { return strs(agent::tag_variants(a.at(0).str())); };
  r["parse_node_id"] = [](const Value& a) {
    auto p = agent::parse_node_id(a.at(0).str());
    if (!p.has_value()) { return Value(); }
    Value o = Value::array();
    o.push(p->first); o.push(p->second);
    return o;
  };
  r["secret_env_name"] = [](const Value& a) { return S(agent::secret_env_name(a.at(0).str())); };
  r["format_delay"] = [](const Value& a) { return S(agent::format_delay(a.at(0).as_int())); };
  r["parse_duration"] = [](const Value& a) {
    auto d = agent::parse_duration(a.at(0).str());
    return d.has_value() ? Value(*d) : Value();
  };
  r["parse_schedule_at"] = [](const Value& a) {
    return try_value([&]() { return S(agent::parse_schedule_at(a.at(0).str())); });
  };
  r["local_display"] = [](const Value& a) { return S(agent::local_display(a.at(0).str())); };
  r["html_unescape"] = [](const Value& a) { return S(agent::html_unescape(a.at(0).str())); };
  r["url_quote_plus"] = [](const Value& a) { return S(agent::url_quote_plus(a.at(0).str())); };
  r["chunk_lines"] = [](const Value& a) { return strs(agent::chunk_lines(a.at(0).str())); };
  r["collapse_repeated_lines"] = [](const Value& a) { return S(agent::collapse_repeated_lines(a.at(0).str())); };
  r["tool_name_clean"] = [](const Value& a) { return S(llm::tool_name_clean(a.at(0))); };
  r["tool_args_valid"] = [](const Value& a) { return S(llm::tool_args_valid(a.at(0))); };
  r["tool_args_text"] = [](const Value& a) { return S(llm::tool_args_text(a.at(0))); };
  r["action_signature"] = [](const Value& a) { return S(llm::action_signature(a.at(0).str(), a.at(1))); };
  r["result_digest"] = [](const Value& a) { return S(llm::result_digest(a.at(0).str())); };
  r["clean_history"] = [](const Value& a) { return llm::clean_history(a.at(0)); };
  r["conversation_tokens"] = [](const Value& a) { return Value(llm::conversation_tokens(a.at(0), a.size() > 1 ? a.at(1) : Value())); };
  r["conversation_text"] = [](const Value& a) { return S(agent::conversation_text(a.at(0))); };
  r["approval_parse"] = [](const Value& a) { return S(agent::Approval::parse_allow(a.at(0).str())); };
  r["approval_affirm"] = [](const Value& a) { return Value(agent::Approval::is_affirmative(a.at(0).str())); };
  r["goal_verdict"] = [](const Value& a) {
    return verdict_value(agent::GoalVerdict::from_text(a.at(0).str()), a.at(1).str());
  };
  r["extract_json"] = [](const Value& a) {
    auto [data, err] = agent::GoalVerifier::extract_json(a.at(0).str());
    Value o = Value::array();
    o.push(data);
    o.push(err.has_value() ? Value(*err) : Value());
    return o;
  };
  r["goal_transcript"] = [](const Value& a) { return S(agent::GoalVerifier::transcript(a.at(0))); };
  r["goal_tried"] = [](const Value& a) { return strs(agent::GoalVerifier::tried_actions(a.at(0))); };
  r["goal_task"] = [](const Value& a) {
    std::vector<std::string> tried;
    for (const auto& x : a.at(3).items()) { tried.push_back(x.str()); }
    return S(agent::GoalVerifier::build_task(a.at(0).str(), a.at(1).str(), a.at(2).str(), tried));
  };
  r["muse_prompt"] = [](const Value& a) { return S(llm::muse_prompt(a.at(0), a.at(1))); };
  r["opencode_prompt"] = [](const Value& a) { return S(llm::opencode_prompt(a.at(0), a.at(1))); };
  r["opencode_parse"] = [](const Value& a) { return llm::opencode_parse(a.at(0).str()); };
  r["opencode_find_answer"] = [](const Value& a) { return llm::opencode_find_answer(a.at(0).str()); };
  r["opencode_parse_events"] = [](const Value& a) {
    return try_value([&]() { return S(llm::opencode_parse_events(a.at(0).str(), static_cast<int>(a.at(1).as_int()), a.at(2).str())); });
  };
  r["grok_parse_output"] = [](const Value& a) { return try_value([&]() { return llm::grok_parse_output(a.at(0).str()); }); };
  r["claude_parse_output"] = [](const Value& a) { return try_value([&]() { return llm::claude_parse_output(a.at(0).str()); }); };
  r["gemini_parse_output"] = [](const Value& a) { return try_value([&]() { return llm::gemini_parse_output(a.at(0).str()); }); };
  r["codex_parse_output"] = [](const Value& a) {
    return try_value([&]() { return llm::codex_parse_output(a.at(0).str(), a.at(1).str(), a.at(2).as_bool()); });
  };
  r["muse_valid_answer"] = [](const Value& a) { return llm::muse_valid_answer(a.at(0).str()); };
  r["muse_message_hash"] = [](const Value& a) { return S(llm::muse_message_hash(a.at(0))); };
  r["model_weight"] = [](const Value& a) { return Value(llm::model_weight(a.at(0))); };
  r["model_outcome_reward"] = [](const Value& a) {
    auto opt = [&](size_t i) -> std::optional<double> { return a.at(i).is_null() ? std::nullopt : std::optional<double>(a.at(i).as_double()); };
    return Value(llm::model_outcome_reward(opt(0), opt(1)));
  };
  r["summarize_errors"] = [](const Value& a) {
    std::vector<std::string> errs;
    for (const auto& x : a.at(0).items()) { errs.push_back(x.str()); }
    return S(llm::ChatClient::summarize_errors(errs, static_cast<size_t>(a.at(1).as_int()), static_cast<size_t>(a.at(2).as_int())));
  };
  r["split_models"] = [](const Value& a) { return strs(llm::split_models(a.at(0).str())); };
  r["parse_backends"] = [](const Value& a) {
    Value arr = Value::array();
    for (const auto& b : llm::parse_backends(a.at(0).str())) {
      Value o = Value::object();
      o.set("label", b.label); o.set("base", b.base); o.set("key", b.key); o.set("model", b.model); o.set("profile", b.profile);
      arr.push(o);
    }
    return arr;
  };
  r["is_free_model"] = [](const Value& a) { return Value(llm::is_free_model(a.at(0).str())); };
  r["embedded_failure_message"] = [](const Value& a) { return S(llm::embedded_failure_message(a.at(0))); };
  r["stream_status"] = [](const Value& a) {
    return S(agent::stream_status(static_cast<int>(a.at(0).as_int()), a.at(1).str(), a.at(2).as_double(), a.at(3).as_bool()));
  };
  r["curl_hint"] = [](const Value& a) { return S(agent::curl_hint(a.at(0).str())); };
  r["format_schedule_lines"] = [](const Value& a) { return S(agent::format_schedule_lines(a.at(0), a.at(1).as_double())); };
  r["py_repr"] = [](const Value& a) { return S(py::repr(a.at(0))); };
  r["py_str"] = [](const Value& a) { return S(py::str(a.at(0))); };
  r["py_strip"] = [](const Value& a) { return S(py::strip(a.at(0).str())); };
  r["py_lower"] = [](const Value& a) { return S(py::lower(a.at(0).str())); };
  r["py_splitlines"] = [](const Value& a) { return strs(py::splitlines(a.at(0).str())); };
  r["py_split_ws"] = [](const Value& a) { return strs(py::split_ws(a.at(0).str())); };
  r["py_int"] = [](const Value& a) {
    auto v = py::to_int(a.at(0));
    return v.has_value() ? Value(*v) : Value();
  };
  r["py_float"] = [](const Value& a) {
    auto v = py::to_float(a.at(0));
    if (!v.has_value()) { return Value(); }
    return Value(*v);
  };
  r["py_slice"] = [](const Value& a) {
    std::optional<long> s, e;
    if (!a.at(1).is_null()) { s = static_cast<long>(a.at(1).as_int()); }
    if (!a.at(2).is_null()) { e = static_cast<long>(a.at(2).as_int()); }
    return S(py::slice(a.at(0).str(), s, e));
  };
  r["py_len"] = [](const Value& a) { return Value(static_cast<int64_t>(py::len(a.at(0).str()))); };
  r["float_repr"] = [](const Value& a) { return S(json::float_repr(a.at(0).as_double())); };
  r["sha1"] = [](const Value& a) { return S(py::sha1_hex(a.at(0).str())); };
  r["sha256"] = [](const Value& a) { return S(py::sha256_hex(a.at(0).str())); };
  r["utf8_replace"] = [](const Value& a) {
    // bytes arrive as a latin-1 style string of code points 0-255
    std::string raw;
    for (uint32_t cp : py::decode(a.at(0).str())) { raw += static_cast<char>(cp); }
    return S(py::utf8_replace(raw));
  };
  r["orientation_block"] = [](const Value& a) { return S(agent::orientation_block(a.at(0).str(), a.at(1).as_bool())); };
  r["capability_block"] = [](const Value& a) { return S(agent::capability_block(a.at(0).as_bool(), a.at(1).as_bool())); };
  r["split_leading_tags"] = [](const Value& a) {
    auto [tags, rest] = agent::split_leading_tags(a.at(0).str());
    Value o = Value::array();
    o.push(strs(tags));
    o.push(rest);
    return o;
  };
  r["expand_file_mentions"] = [](const Value& a) { return S(agent::expand_file_mentions(a.at(0).str(), a.at(1).str())); };
  r["profile_ok"] = [](const Value& a) { return Value(agent::profile_ok(a.at(0).str())); };
  r["bot_ok"] = [](const Value& a) { return Value(agent::bot_ok(a.at(0).str())); };
  r["secret_ok"] = [](const Value& a) { return Value(agent::secret_ok(a.at(0).str())); };
  r["skill_name_ok"] = [](const Value& a) { return Value(agent::skill_name_ok(a.at(0).str())); };
  r["mcp_server_ok"] = [](const Value& a) { return Value(agent::mcp_server_ok(a.at(0).str())); };
  r["create_skill_prompt"] = [](const Value& a) {
    return S(agent::create_skill_prompt(a.at(0).str(), a.at(1).str(), a.at(2).str()));
  };
  r["data_texts"] = [](const Value&) {
    Value o = Value::object();
    o.set("default_system", agent::data::DefaultSystem());
    o.set("merge_system", agent::data::MergeSystem());
    o.set("compact_system", agent::data::CompactSystem());
    o.set("autonomous_note", agent::data::AutonomousNote());
    o.set("legacy_password_note", agent::data::LegacyPasswordNote());
    o.set("vault_memory_note", agent::data::VaultMemoryNote());
    o.set("mcp_resolver_brief", agent::data::McpResolverBrief());
    o.set("goal_task", agent::data::GoalVerifierTask());
    o.set("curl_hint", agent::data::CurlHint());
    return o;
  };
  return r;
}
}  // namespace

int main() {
  auto reg = registry();
  std::string line;
  while (std::getline(std::cin, line)) {
    if (line.empty()) { continue; }
    Value out;
    try {
      Value c = Value::loads(line);
      auto it = reg.find(c.get("fn").str());
      if (it == reg.end()) {
        out = Value::object();
        out.set("missing_fn", c.get("fn"));
      } else {
        out = it->second(c.get("args"));
      }
    } catch (const std::exception& e) {
      out = Value::object();
      out.set("harness_error", std::string(e.what()));
    }
    std::cout << out.dumps() << "\n";
  }
  return 0;
}