feat(clients): optimize: evaluation, simulation, observability and optimizer on a discovery-checked call table

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-24 04:40:55 +02:00
co-authored by Claude Sonnet 5.5
parent 6d744a433f
commit 1e2fb400bb
55 changed files with 5099 additions and 3 deletions
+60
View File
@@ -0,0 +1,60 @@
#pragma once
// Simulator: runs a Scenario between a SimulatedUser and an Agent, with
// tools answered by a VirtualToolbox, then scores task success and safety
// over the whole conversation with the Autorater. All loops are capped
// (turns, tool rounds per turn, tool calls per session).
#include <memory>
#include <string>
#include "google/optimize/agent.h"
#include "google/optimize/autorater.h"
#include "google/optimize/caps.h"
#include "google/optimize/scenario.h"
#include "google/optimize/simulated_user.h"
#include "google/optimize/virtual_tools.h"
namespace molo::google::optimize {
struct SimulationConfig {
int max_turns = OptimizeCaps::kDefaultSimTurns; // 1..kMaxSimTurns
void validate() const;
};
struct SimulationResult {
Conversation conversation;
// "user_done" | "max_turns" | "tool_round_cap" | "tool_call_cap"
std::string stop_reason;
int tool_calls = 0;
int unknown_tool_calls = 0;
};
struct SimulationReport {
Scenario scenario;
SimulationResult run;
Score task_success;
Score safety;
bool pass() const { return task_success.pass && safety.pass; }
clients::body::Json to_json() const;
};
class Simulator {
public:
// user_model plays the user; judge_model scores (may be the same object).
Simulator(std::shared_ptr<LanguageModel> user_model,
std::shared_ptr<LanguageModel> judge_model);
SimulationResult run(const Scenario& scenario, Agent& agent, VirtualToolbox& tools,
const SimulationConfig& config = {}) const;
SimulationReport run_and_score(const Scenario& scenario, Agent& agent,
VirtualToolbox& tools,
const SimulationConfig& config = {}) const;
private:
SimulatedUser user_;
Autorater judge_;
};
} // namespace molo::google::optimize