#pragma once // Builders for evaluateInstances request bodies and the parser for its // responses. Field names come from the aiplatform v1 Discovery document // (see calls_table.inc and tools/check_discovery.py). Pure functions: no // network, fully testable. #include #include #include #include "clients/body/json.h" namespace molo::google::optimize { struct TrajectoryCall { std::string tool_name; std::string tool_input = "{}"; // Discovery: ToolCall.toolInput is a string }; enum class TrajectoryMetric { kExactMatch, kInOrderMatch, kAnyOrderMatch, kPrecision, kRecall, kSingleToolUse }; struct AutoraterOptions { std::string model; // empty: Google's default autorater int sample_count = 0; // 0: unset; else 1..32 bool flip = false; // pairwise position-bias flip }; struct MetricOutcome { std::string kind; // response union key, e.g. "exactMatchResults" std::vector scores; std::string explanation; std::string choice; // pairwise: BASELINE | CANDIDATE | TIE clients::body::Json raw; }; class EvaluationRequests { public: // LLM-judged, safety. Needs access to the default autorater model. static clients::body::Json safety(const std::string& prediction); // Computation based. static clients::body::Json exact_match( const std::vector>& prediction_reference); static clients::body::Json trajectory(TrajectoryMetric metric, const std::vector& predicted, const std::vector& reference, const std::string& single_tool_name = ""); // Custom LLM-judged metrics. `fields` become the jsonInstance object that // the prompt template's {placeholders} read from. static clients::body::Json pointwise(const std::string& metric_prompt_template, const std::map& fields, const std::string& system_instruction = ""); static clients::body::Json pairwise(const std::string& metric_prompt_template, const std::map& fields, const std::string& baseline_field = "baseline", const std::string& candidate_field = "candidate", const std::string& system_instruction = ""); // Sets autoraterConfig on a request (validates sample_count). static void with_autorater(clients::body::Json& request, const AutoraterOptions& options); // Reads whichever result union member the response carries. static MetricOutcome parse(const clients::body::Json& response); static const char* metric_key(TrajectoryMetric metric); // e.g. "trajectoryExactMatch" }; } // namespace molo::google::optimize