#include "google/optimize/simulator.h" #include "google/errors.h" namespace molo::google::optimize { using clients::body::Json; void SimulationConfig::validate() const { if (max_turns < 1 || max_turns > OptimizeCaps::kMaxSimTurns) throw GoogleError::invalid("max_turns must be 1.." + std::to_string(OptimizeCaps::kMaxSimTurns)); } Simulator::Simulator(std::shared_ptr user_model, std::shared_ptr judge_model) : user_(std::move(user_model)), judge_(std::move(judge_model)) {} SimulationResult Simulator::run(const Scenario& scenario, Agent& agent, VirtualToolbox& tools, const SimulationConfig& config) const { scenario.validate(); config.validate(); tools.reset_counters(); SimulationResult result; result.stop_reason = "max_turns"; std::vector contents; for (int turn = 0; turn < config.max_turns; ++turn) { std::string user_text; if (turn == 0) { user_text = scenario.starting_prompt; } else { auto next = user_.next(scenario, result.conversation.turns); if (!next) { result.stop_reason = "user_done"; break; } user_text = *next; } result.conversation.turns.push_back(Turn{"user", user_text, {}}); contents.push_back(Content::user(user_text)); Turn reply; reply.role = "agent"; bool answered = false; for (int round = 0; round < OptimizeCaps::kMaxToolRoundsPerTurn; ++round) { AgentStep step = agent.step(contents); if (step.calls.empty()) { reply.text = step.text; contents.push_back(Content::model(step.text)); answered = true; break; } Content call_msg; call_msg.role = "model"; Content response_msg; response_msg.role = "user"; bool capped = false; for (const FunctionCall& fc : step.calls) { if (tools.calls() >= OptimizeCaps::kMaxToolCallsPerSession) { capped = true; break; } Json out = tools.call(fc); call_msg.parts.push_back(Part::of_call(fc)); FunctionResponse fr; fr.name = fc.name; fr.id = fc.id; fr.response = out; response_msg.parts.push_back(Part::of_response(fr)); reply.calls.push_back(ToolCallRecord{fc.name, fc.args, out}); } if (capped) { result.stop_reason = "tool_call_cap"; break; } contents.push_back(call_msg); contents.push_back(response_msg); } result.conversation.turns.push_back(reply); if (!answered) { if (result.stop_reason == "max_turns") result.stop_reason = "tool_round_cap"; break; } } result.tool_calls = tools.calls(); result.unknown_tool_calls = tools.unknown_calls(); return result; } SimulationReport Simulator::run_and_score(const Scenario& scenario, Agent& agent, VirtualToolbox& tools, const SimulationConfig& config) const { SimulationReport report; report.scenario = scenario; report.run = run(scenario, agent, tools, config); std::vector success = Autorater::task_success_rubrics(); std::string goal = "Did the agent satisfy the goals of this user plan: " + scenario.plan; if (goal.size() > OptimizeCaps::kMaxRubricChars) goal.resize(OptimizeCaps::kMaxRubricChars); success.push_back({"plan_goals", goal}); report.task_success = judge_.score("task_success", report.run.conversation, success); report.safety = judge_.score("safety", report.run.conversation, Autorater::safety_rubrics()); return report; } Json SimulationReport::to_json() const { Json o = Json::object(); o.set("scenario", scenario.to_json()); o.set("stop_reason", Json(run.stop_reason)); o.set("tool_calls", Json(run.tool_calls)); o.set("unknown_tool_calls", Json(run.unknown_tool_calls)); o.set("turns", Json(static_cast(run.conversation.turns.size()))); o.set("transcript", Json(run.conversation.transcript())); o.set("task_success", task_success.to_json()); o.set("safety", safety.to_json()); o.set("pass", Json(pass())); return o; } } // namespace molo::google::optimize