61 lines
1.8 KiB
C++
61 lines
1.8 KiB
C++
#pragma once
|
|||
|
|
|
||
|
|
// Simulator: runs a Scenario between a SimulatedUser and an Agent, with
|
||
|
|
// tools answered by a VirtualToolbox, then scores task success and safety
|
||
|
|
// over the whole conversation with the Autorater. All loops are capped
|
||
|
|
// (turns, tool rounds per turn, tool calls per session).
|
||
|
|
|
||
|
|
#include <memory>
|
||
|
|
#include <string>
|
||
|
|
|
||
|
|
#include "google/optimize/agent.h"
|
||
|
|
#include "google/optimize/autorater.h"
|
||
|
|
#include "google/optimize/caps.h"
|
||
|
|
#include "google/optimize/scenario.h"
|
||
|
|
#include "google/optimize/simulated_user.h"
|
||
|
|
#include "google/optimize/virtual_tools.h"
|
||
|
|
|
||
|
|
namespace molo::google::optimize {
|
||
|
|
|
||
|
|
struct SimulationConfig {
|
||
|
|
int max_turns = OptimizeCaps::kDefaultSimTurns; // 1..kMaxSimTurns
|
||
|
|
void validate() const;
|
||
|
|
};
|
||
|
|
|
||
|
|
struct SimulationResult {
|
||
|
|
Conversation conversation;
|
||
|
|
// "user_done" | "max_turns" | "tool_round_cap" | "tool_call_cap"
|
||
|
|
std::string stop_reason;
|
||
|
|
int tool_calls = 0;
|
||
|
|
int unknown_tool_calls = 0;
|
||
|
|
};
|
||
|
|
|
||
|
|
struct SimulationReport {
|
||
|
|
Scenario scenario;
|
||
|
|
SimulationResult run;
|
||
|
|
Score task_success;
|
||
|
|
Score safety;
|
||
|
|
bool pass() const { return task_success.pass && safety.pass; }
|
||
|
|
clients::body::Json to_json() const;
|
||
|
|
};
|
||
|
|
|
||
|
|
class Simulator {
|
||
|
|
public:
|
||
|
|
// user_model plays the user; judge_model scores (may be the same object).
|
||
|
|
Simulator(std::shared_ptr<LanguageModel> user_model,
|
||
|
|
std::shared_ptr<LanguageModel> judge_model);
|
||
|
|
|
||
|
|
SimulationResult run(const Scenario& scenario, Agent& agent, VirtualToolbox& tools,
|
||
|
|
const SimulationConfig& config = {}) const;
|
||
|
|
|
||
|
|
SimulationReport run_and_score(const Scenario& scenario, Agent& agent,
|
||
|
|
VirtualToolbox& tools,
|
||
|
|
const SimulationConfig& config = {}) const;
|
||
|
|
|
||
|
|
private:
|
||
|
|
SimulatedUser user_;
|
||
|
|
Autorater judge_;
|
||
|
|
};
|
||
|
|
|
||
|
|
} // namespace molo::google::optimize
|