feat(clients): optimize: evaluation, simulation, observability and optimizer on a discovery-checked call table
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
#pragma once
|
||||
|
||||
// Simulator: runs a Scenario between a SimulatedUser and an Agent, with
|
||||
// tools answered by a VirtualToolbox, then scores task success and safety
|
||||
// over the whole conversation with the Autorater. All loops are capped
|
||||
// (turns, tool rounds per turn, tool calls per session).
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "google/optimize/agent.h"
|
||||
#include "google/optimize/autorater.h"
|
||||
#include "google/optimize/caps.h"
|
||||
#include "google/optimize/scenario.h"
|
||||
#include "google/optimize/simulated_user.h"
|
||||
#include "google/optimize/virtual_tools.h"
|
||||
|
||||
namespace molo::google::optimize {
|
||||
|
||||
struct SimulationConfig {
|
||||
int max_turns = OptimizeCaps::kDefaultSimTurns; // 1..kMaxSimTurns
|
||||
void validate() const;
|
||||
};
|
||||
|
||||
struct SimulationResult {
|
||||
Conversation conversation;
|
||||
// "user_done" | "max_turns" | "tool_round_cap" | "tool_call_cap"
|
||||
std::string stop_reason;
|
||||
int tool_calls = 0;
|
||||
int unknown_tool_calls = 0;
|
||||
};
|
||||
|
||||
struct SimulationReport {
|
||||
Scenario scenario;
|
||||
SimulationResult run;
|
||||
Score task_success;
|
||||
Score safety;
|
||||
bool pass() const { return task_success.pass && safety.pass; }
|
||||
clients::body::Json to_json() const;
|
||||
};
|
||||
|
||||
class Simulator {
|
||||
public:
|
||||
// user_model plays the user; judge_model scores (may be the same object).
|
||||
Simulator(std::shared_ptr<LanguageModel> user_model,
|
||||
std::shared_ptr<LanguageModel> judge_model);
|
||||
|
||||
SimulationResult run(const Scenario& scenario, Agent& agent, VirtualToolbox& tools,
|
||||
const SimulationConfig& config = {}) const;
|
||||
|
||||
SimulationReport run_and_score(const Scenario& scenario, Agent& agent,
|
||||
VirtualToolbox& tools,
|
||||
const SimulationConfig& config = {}) const;
|
||||
|
||||
private:
|
||||
SimulatedUser user_;
|
||||
Autorater judge_;
|
||||
};
|
||||
|
||||
} // namespace molo::google::optimize
|
||||
Reference in New Issue
Block a user