tai/taicpp/tests/agent_e2e.cpp

// End-to-end driver: runs one scenario (JSON file) through the C++ Agent with a
// scripted HTTP transport, an in-memory store and a popen shell, then prints
// the observable outcome as JSON. tests/agent_e2e.py runs the same scenario
// through tai.py and compares.
#include "../src/agent/agent.h"
#include "../src/agent/detz_tools.h"
#include "../src/llm/http_transport.h"
#include "../src/llm/py.h"
#include "agent_memstore.h"
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <sstream>
#include <unistd.h>
using namespace detz;
using json::Value;
namespace fs = std::filesystem;

namespace {

struct ScriptExhausted {};   // deliberately not a std::exception: must not be retried

class ScriptedTransport : public llm::HttpTransport {
 public:
  ScriptedTransport(Value script, Value single) : script_(std::move(script)), single_(std::move(single)) {}
  Value requests = Value::array();
  llm::HttpResult send(const llm::HttpRequest& req) override {
    if (req.url.size() >= 7 && req.url.compare(req.url.size() - 7, 7, "/models") == 0) {
      llm::HttpResult r;
      r.error = "<urlopen error fake>";
      return r;
    }
    record(req);
    const Value& item = next(single_, single_pos_);
    return reply(item, nullptr);
  }
  llm::HttpResult stream(const llm::HttpRequest& req, const llm::LineSink& sink) override {
    record(req);
    const Value& item = next(script_, script_pos_);
    return reply(item, &sink);
  }

 private:
  void record(const llm::HttpRequest& req) {
    Value r = Value::object();
    r.set("method", req.method);
    r.set("url", req.url);
    Value body;
    if (req.body.has_value()) {
      try { body = Value::loads(*req.body); } catch (const json::ParseError&) { body = Value(*req.body); }
    }
    r.set("body", body);
    requests.push(r);
  }
  const Value& next(const Value& queue, size_t& pos) {
    if (pos >= queue.size()) { throw ScriptExhausted{}; }
    return queue.at(pos++);
  }
  llm::HttpResult reply(const Value& item, const llm::LineSink* sink) {
    llm::HttpResult r;
    std::string kind = item.get("kind").str();
    if (kind == "transport_error") { r.error = "<urlopen error scripted>"; return r; }
    if (kind == "status") {
      r.status = item.get("status").as_int();
      r.body = item.get("body").str();
      return r;
    }
    r.status = 200;
    if (item.get("content_type").is_string()) { r.content_type = item.get("content_type").str(); }
    if (kind == "json") {
      r.body = item.get("body").dumps();
      return r;
    }
    if (kind == "raw") {
      r.body = item.get("body").str();
      return r;
    }
    // stream: SSE lines
    if (sink == nullptr) { throw std::runtime_error("stream script item consumed by a non-stream call"); }
    for (const auto& ev : item.get("events").items()) {
      if (!(*sink)("data: " + ev.dumps() + "\n")) { return r; }
      if (!(*sink)("\n")) { return r; }
    }
    (*sink)("data: [DONE]\n");
    return r;
  }
  Value script_, single_;
  size_t script_pos_{0}, single_pos_{0};
};

class PopenShell : public agent::ShellPort {
 public:
  agent::RunResult run_live_shell(const std::string& command, const std::string& cwd, const std::map<std::string, std::string>& extra_env,
                                  agent::LiveStreamPort*, const agent::AskFn&) override {
    std::string env;
    for (const auto& kv : extra_env) { env += "export " + kv.first + "='" + kv.second + "'; "; }
    std::string full = "cd '" + cwd + "' && " + env + command + " 2>&1";
    FILE* pipe = popen(("/bin/sh -c '" + escape(full) + "' < /dev/null").c_str(), "r");
    agent::RunResult r;
    if (pipe == nullptr) { r.code = 127; return r; }
    char buf[4096];
    size_t n;
    while ((n = fread(buf, 1, sizeof buf, pipe)) > 0) { r.output.append(buf, n); }
    int status = pclose(pipe);
    r.code = WIFEXITED(status) ? WEXITSTATUS(status) : -1;
    r.output = py::utf8_replace(r.output);
    return r;
  }
  agent::RunResult run_live_argv(const std::vector<std::string>&, agent::LiveStreamPort*, const agent::AskFn&) override {
    throw agent::PortNotWired("shell argv");
  }
  std::unique_ptr<agent::LiveStreamPort> make_live_stream() override { return nullptr; }
  std::vector<std::string> shell_target_paths(const std::string&, const std::string&) override { return {}; }
  std::string ensure_box() override { throw llm::BackendError("no container engine (test)"); }
  llm::ProcResult box_exec(const std::string&, const std::vector<std::string>&, const std::optional<std::string>&, double) override {
    throw agent::PortNotWired("box");
  }
  std::unique_ptr<agent::LaneBoardPort> make_lane_board(const std::string&) override { return nullptr; }

 private:
  static std::string escape(const std::string& s) {
    std::string out;
    for (char c : s) {
      if (c == '\'') { out += "'\\''"; } else { out += c; }
    }
    return out;
  }
};

class FakeSkills : public agent::SkillsPort {
 public:
  explicit FakeSkills(std::string catalog) : catalog_(std::move(catalog)) {}
  Value discover_skills(const std::string&, const std::string&) override { return Value::object(); }
  std::string skill_catalog(const Value&) override { return catalog_; }
  std::string skill_provenance_note(const Value&) override { return ""; }
  std::vector<std::string> skill_extra_files(const std::string&) override { return {}; }
  std::optional<Value> skill_blueprint(const std::string&) override { return std::nullopt; }
  std::string skill_scope_dir(const std::string&, const std::string&, const std::string&, const std::string&) override { return ""; }
  std::string default_skill_scope(const std::string&) override { return "project"; }
  Value parse_skill_file(const std::string&) override { return Value(); }
  bool is_external_skill_root(const std::string&) override { return false; }
  void save_skill_install_meta(const std::string&, const std::string&, const Value&) override {}
 private:
  std::string catalog_;
};

std::string slurp(const std::string& path) {
  std::ifstream in(path, std::ios::binary);
  std::stringstream ss;
  ss << in.rdbuf();
  return ss.str();
}
}  // namespace

int main(int argc, char** argv) {
  agent::DetzTools::set_enabled(false);   // compare the tai.py tool set only
  if (argc < 3) { std::fprintf(stderr, "usage: agent_e2e scenario.json workdir\n"); return 2; }
  Value sc = Value::loads(slurp(argv[1]));
  std::string workdir = argv[2];
  if (chdir(workdir.c_str()) != 0) { std::perror("chdir"); return 2; }
  std::string home = workdir + "/.home";
  fs::create_directories(home);

  agent::AgentConfig cfg;
  cfg.home = home;
  cfg.audio_dir = home + "/audio";
  cfg.profile = sc.get("profile").is_string() ? sc.get("profile").str() : "default";
  cfg.auto_approve = sc.get("auto_approve").truthy();
  cfg.retry_seconds = sc.get("retry_seconds").truthy() ? sc.get("retry_seconds").as_double() : 0.0;
  cfg.opencode_enabled = false;
  cfg.openrouter_enabled = false;
  cfg.pollinations_enabled = false;
  cfg.zen_enabled = false;
  llm::BackendSpec b;
  b.label = "lab"; b.base = "http://fake.invalid/v1"; b.key = "k"; b.model = "m1"; b.profile = "";
  cfg.backends.push_back(b);

  ScriptedTransport transport(sc.get("script"), sc.get("single"));
  llm::set_transport(&transport);
  agent::testing::MemStore store;
  PopenShell shell;
  FakeSkills skills(sc.get("skill_catalog").str_or(""));
  agent::ports().shell = &shell;
  agent::ports().skills = &skills;
  llm::ChatClient::sleeper() = [](double) {};

  if (sc.get("presets").is_object()) {
    // pre-existing vault state: systems and secrets
    for (const auto& kv : sc.get("presets").get("systems").members()) { store.save_system(kv.first, kv.second.str()); }
    for (const auto& kv : sc.get("presets").get("secrets").members()) {
      store.save_secret(kv.first, kv.second.get("value").str(), Value::object(), {}, "default");
    }
    for (const auto& kv : sc.get("presets").get("bots").members()) {
      store.save_bot_system("default", kv.first, kv.second.get("system").str());
      Value bots = store.load_bots("default");
      Value e = Value::object();
      e.set("nicknames", kv.second.get("nicknames"));
      e.set("created", "t");
      e.set("updated", "t");
      bots.set(kv.first, e);
      store.save_bots("default", bots);
    }
  }

  agent::Agent ag(cfg, &store, sc.get("persist").is_bool() ? sc.get("persist").as_bool() : true, true, 0, false, sc.get("yolo").truthy());
  if (sc.get("deadline_expired").truthy()) { ag.deadline = 1.0; }
  Value answers = sc.get("answers");
  size_t answer_pos = 0;
  if (answers.is_array()) {
    ag.prompter = [&](const std::string&, const std::string&, const std::vector<std::string>&) -> std::optional<std::string> {
      if (answer_pos >= answers.size()) { return std::nullopt; }
      const Value& a = answers.at(answer_pos++);
      if (a.is_null()) { return std::nullopt; }
      return a.str();
    };
  }
  Value runner_script = sc.get("runner_script");
  size_t runner_pos = 0;
  if (runner_script.is_array()) {
    ag.runner_override = [&](const std::string&, const std::string&, double) -> std::string {
      if (runner_pos >= runner_script.size()) { return "runner exhausted"; }
      return runner_script.at(runner_pos++).str();
    };
  }
  Value out = Value::object();
  try {
    Value turns = sc.get("prompts").is_array() ? sc.get("prompts") : Value::array();
    if (turns.size() == 0) { turns.push(sc.get("prompt")); }
    Value results = Value::array();
    for (const auto& p : turns.items()) {
      results.push(ag.run_turn(p.str(), true, sc.get("max_steps").truthy() ? static_cast<int>(sc.get("max_steps").as_int()) : agent::MAX_STEPS));
    }
    out.set("results", results);
    if (sc.get("verify").truthy()) {
      auto [text, ok] = ag.verify_loop(sc.get("prompt").str(), results.at(results.size() - 1).str());
      Value v = Value::object();
      v.set("text", text);
      v.set("ok", ok);
      out.set("verify", v);
    }
  } catch (const ScriptExhausted&) {
    out.set("exception", "script exhausted");
  } catch (const std::exception& e) {
    out.set("exception", std::string(e.what()));
  }
  out.set("messages", ag.messages);
  out.set("requests", transport.requests);
  Value events = Value::array();
  for (const auto& e : store.events) {
    Value row = Value::array();
    row.push(e.role); row.push(e.kind); row.push(e.text);
    events.push(row);
  }
  out.set("events", events);
  Value audits = Value::array();
  for (const auto& a : store.audits) {
    Value row = Value::object();
    row.set("actor", a.actor); row.set("action", a.action); row.set("path", a.path); row.set("message", a.message);
    row.set("tags", a.tags);
    row.set("old_size", a.old_size ? Value(*a.old_size) : Value());
    row.set("new_size", a.new_size ? Value(*a.new_size) : Value());
    row.set("old", a.old_text ? Value(*a.old_text) : Value());
    row.set("new", a.new_text ? Value(*a.new_text) : Value());
    audits.push(row);
  }
  out.set("audits", audits);
  out.set("tool_calls", ag.tools.calls);
  out.set("yolo", ag.yolo);
  out.set("timed_out", ag.timed_out);
  out.set("system", ag.system_message);
  std::cout << out.dumps() << "\n";
  return 0;
}