// End-to-end driver: runs one scenario (JSON file) through the C++ Agent with a // scripted HTTP transport, an in-memory store and a popen shell, then prints // the observable outcome as JSON. tests/agent_e2e.py runs the same scenario // through tai.py and compares. #include "../src/agent/agent.h" #include "../src/llm/http_transport.h" #include "../src/llm/py.h" #include "agent_memstore.h" #include #include #include #include #include #include using namespace detz; using json::Value; namespace fs = std::filesystem; namespace { struct ScriptExhausted {}; // deliberately not a std::exception: must not be retried class ScriptedTransport : public llm::HttpTransport { public: ScriptedTransport(Value script, Value single) : script_(std::move(script)), single_(std::move(single)) {} Value requests = Value::array(); llm::HttpResult send(const llm::HttpRequest& req) override { if (req.url.size() >= 7 && req.url.compare(req.url.size() - 7, 7, "/models") == 0) { llm::HttpResult r; r.error = ""; return r; } record(req); const Value& item = next(single_, single_pos_); return reply(item, nullptr); } llm::HttpResult stream(const llm::HttpRequest& req, const llm::LineSink& sink) override { record(req); const Value& item = next(script_, script_pos_); return reply(item, &sink); } private: void record(const llm::HttpRequest& req) { Value r = Value::object(); r.set("method", req.method); r.set("url", req.url); Value body; if (req.body.has_value()) { try { body = Value::loads(*req.body); } catch (const json::ParseError&) { body = Value(*req.body); } } r.set("body", body); requests.push(r); } const Value& next(const Value& queue, size_t& pos) { if (pos >= queue.size()) { throw ScriptExhausted{}; } return queue.at(pos++); } llm::HttpResult reply(const Value& item, const llm::LineSink* sink) { llm::HttpResult r; std::string kind = item.get("kind").str(); if (kind == "transport_error") { r.error = ""; return r; } if (kind == "status") { r.status = item.get("status").as_int(); r.body = item.get("body").str(); return r; } r.status = 200; if (item.get("content_type").is_string()) { r.content_type = item.get("content_type").str(); } if (kind == "json") { r.body = item.get("body").dumps(); return r; } if (kind == "raw") { r.body = item.get("body").str(); return r; } // stream: SSE lines if (sink == nullptr) { throw std::runtime_error("stream script item consumed by a non-stream call"); } for (const auto& ev : item.get("events").items()) { if (!(*sink)("data: " + ev.dumps() + "\n")) { return r; } if (!(*sink)("\n")) { return r; } } (*sink)("data: [DONE]\n"); return r; } Value script_, single_; size_t script_pos_{0}, single_pos_{0}; }; class PopenShell : public agent::ShellPort { public: agent::RunResult run_live_shell(const std::string& command, const std::string& cwd, const std::map& extra_env, agent::LiveStreamPort*, const agent::AskFn&) override { std::string env; for (const auto& kv : extra_env) { env += "export " + kv.first + "='" + kv.second + "'; "; } std::string full = "cd '" + cwd + "' && " + env + command + " 2>&1"; FILE* pipe = popen(("/bin/sh -c '" + escape(full) + "' < /dev/null").c_str(), "r"); agent::RunResult r; if (pipe == nullptr) { r.code = 127; return r; } char buf[4096]; size_t n; while ((n = fread(buf, 1, sizeof buf, pipe)) > 0) { r.output.append(buf, n); } int status = pclose(pipe); r.code = WIFEXITED(status) ? WEXITSTATUS(status) : -1; r.output = py::utf8_replace(r.output); return r; } agent::RunResult run_live_argv(const std::vector&, agent::LiveStreamPort*, const agent::AskFn&) override { throw agent::PortNotWired("shell argv"); } std::unique_ptr make_live_stream() override { return nullptr; } std::vector shell_target_paths(const std::string&, const std::string&) override { return {}; } std::string ensure_box() override { throw llm::BackendError("no container engine (test)"); } llm::ProcResult box_exec(const std::string&, const std::vector&, const std::optional&, double) override { throw agent::PortNotWired("box"); } std::unique_ptr make_lane_board(const std::string&) override { return nullptr; } private: static std::string escape(const std::string& s) { std::string out; for (char c : s) { if (c == '\'') { out += "'\\''"; } else { out += c; } } return out; } }; class FakeSkills : public agent::SkillsPort { public: explicit FakeSkills(std::string catalog) : catalog_(std::move(catalog)) {} Value discover_skills(const std::string&, const std::string&) override { return Value::object(); } std::string skill_catalog(const Value&) override { return catalog_; } std::string skill_provenance_note(const Value&) override { return ""; } std::vector skill_extra_files(const std::string&) override { return {}; } std::optional skill_blueprint(const std::string&) override { return std::nullopt; } std::string skill_scope_dir(const std::string&, const std::string&, const std::string&, const std::string&) override { return ""; } std::string default_skill_scope(const std::string&) override { return "project"; } Value parse_skill_file(const std::string&) override { return Value(); } bool is_external_skill_root(const std::string&) override { return false; } void save_skill_install_meta(const std::string&, const std::string&, const Value&) override {} private: std::string catalog_; }; std::string slurp(const std::string& path) { std::ifstream in(path, std::ios::binary); std::stringstream ss; ss << in.rdbuf(); return ss.str(); } } // namespace int main(int argc, char** argv) { if (argc < 3) { std::fprintf(stderr, "usage: agent_e2e scenario.json workdir\n"); return 2; } Value sc = Value::loads(slurp(argv[1])); std::string workdir = argv[2]; if (chdir(workdir.c_str()) != 0) { std::perror("chdir"); return 2; } std::string home = workdir + "/.home"; fs::create_directories(home); agent::AgentConfig cfg; cfg.home = home; cfg.audio_dir = home + "/audio"; cfg.profile = sc.get("profile").is_string() ? sc.get("profile").str() : "default"; cfg.auto_approve = sc.get("auto_approve").truthy(); cfg.retry_seconds = sc.get("retry_seconds").truthy() ? sc.get("retry_seconds").as_double() : 0.0; cfg.opencode_enabled = false; cfg.openrouter_enabled = false; cfg.pollinations_enabled = false; cfg.zen_enabled = false; llm::BackendSpec b; b.label = "lab"; b.base = "http://fake.invalid/v1"; b.key = "k"; b.model = "m1"; b.profile = ""; cfg.backends.push_back(b); ScriptedTransport transport(sc.get("script"), sc.get("single")); llm::set_transport(&transport); agent::testing::MemStore store; PopenShell shell; FakeSkills skills(sc.get("skill_catalog").str_or("")); agent::ports().shell = &shell; agent::ports().skills = &skills; llm::ChatClient::sleeper() = [](double) {}; if (sc.get("presets").is_object()) { // pre-existing vault state: systems and secrets for (const auto& kv : sc.get("presets").get("systems").members()) { store.save_system(kv.first, kv.second.str()); } for (const auto& kv : sc.get("presets").get("secrets").members()) { store.save_secret(kv.first, kv.second.get("value").str(), Value::object(), {}, "default"); } for (const auto& kv : sc.get("presets").get("bots").members()) { store.save_bot_system("default", kv.first, kv.second.get("system").str()); Value bots = store.load_bots("default"); Value e = Value::object(); e.set("nicknames", kv.second.get("nicknames")); e.set("created", "t"); e.set("updated", "t"); bots.set(kv.first, e); store.save_bots("default", bots); } } agent::Agent ag(cfg, &store, sc.get("persist").is_bool() ? sc.get("persist").as_bool() : true, true, 0, false, sc.get("yolo").truthy()); if (sc.get("deadline_expired").truthy()) { ag.deadline = 1.0; } Value answers = sc.get("answers"); size_t answer_pos = 0; if (answers.is_array()) { ag.prompter = [&](const std::string&, const std::string&, const std::vector&) -> std::optional { if (answer_pos >= answers.size()) { return std::nullopt; } const Value& a = answers.at(answer_pos++); if (a.is_null()) { return std::nullopt; } return a.str(); }; } Value runner_script = sc.get("runner_script"); size_t runner_pos = 0; if (runner_script.is_array()) { ag.runner_override = [&](const std::string&, const std::string&, double) -> std::string { if (runner_pos >= runner_script.size()) { return "runner exhausted"; } return runner_script.at(runner_pos++).str(); }; } Value out = Value::object(); try { Value turns = sc.get("prompts").is_array() ? sc.get("prompts") : Value::array(); if (turns.size() == 0) { turns.push(sc.get("prompt")); } Value results = Value::array(); for (const auto& p : turns.items()) { results.push(ag.run_turn(p.str(), true, sc.get("max_steps").truthy() ? static_cast(sc.get("max_steps").as_int()) : agent::MAX_STEPS)); } out.set("results", results); if (sc.get("verify").truthy()) { auto [text, ok] = ag.verify_loop(sc.get("prompt").str(), results.at(results.size() - 1).str()); Value v = Value::object(); v.set("text", text); v.set("ok", ok); out.set("verify", v); } } catch (const ScriptExhausted&) { out.set("exception", "script exhausted"); } catch (const std::exception& e) { out.set("exception", std::string(e.what())); } out.set("messages", ag.messages); out.set("requests", transport.requests); Value events = Value::array(); for (const auto& e : store.events) { Value row = Value::array(); row.push(e.role); row.push(e.kind); row.push(e.text); events.push(row); } out.set("events", events); Value audits = Value::array(); for (const auto& a : store.audits) { Value row = Value::object(); row.set("actor", a.actor); row.set("action", a.action); row.set("path", a.path); row.set("message", a.message); row.set("tags", a.tags); row.set("old_size", a.old_size ? Value(*a.old_size) : Value()); row.set("new_size", a.new_size ? Value(*a.new_size) : Value()); row.set("old", a.old_text ? Value(*a.old_text) : Value()); row.set("new", a.new_text ? Value(*a.new_text) : Value()); audits.push(row); } out.set("audits", audits); out.set("tool_calls", ag.tools.calls); out.set("yolo", ag.yolo); out.set("timed_out", ag.timed_out); out.set("system", ag.system_message); std::cout << out.dumps() << "\n"; return 0; }