Files
moloapp/src/google/optimize/eval_requests.h
T

70 lines
2.9 KiB
C++
Raw Normal View History

#pragma once
// Builders for evaluateInstances request bodies and the parser for its
// responses. Field names come from the aiplatform v1 Discovery document
// (see calls_table.inc and tools/check_discovery.py). Pure functions: no
// network, fully testable.
#include <map>
#include <string>
#include <vector>
#include "clients/body/json.h"
namespace molo::google::optimize {
struct TrajectoryCall {
std::string tool_name;
std::string tool_input = "{}"; // Discovery: ToolCall.toolInput is a string
};
enum class TrajectoryMetric {
kExactMatch, kInOrderMatch, kAnyOrderMatch, kPrecision, kRecall, kSingleToolUse
};
struct AutoraterOptions {
std::string model; // empty: Google's default autorater
int sample_count = 0; // 0: unset; else 1..32
bool flip = false; // pairwise position-bias flip
};
struct MetricOutcome {
std::string kind; // response union key, e.g. "exactMatchResults"
std::vector<double> scores;
std::string explanation;
std::string choice; // pairwise: BASELINE | CANDIDATE | TIE
clients::body::Json raw;
};
class EvaluationRequests {
public:
// LLM-judged, safety. Needs access to the default autorater model.
static clients::body::Json safety(const std::string& prediction);
// Computation based.
static clients::body::Json exact_match(
const std::vector<std::pair<std::string, std::string>>& prediction_reference);
static clients::body::Json trajectory(TrajectoryMetric metric,
const std::vector<TrajectoryCall>& predicted,
const std::vector<TrajectoryCall>& reference,
const std::string& single_tool_name = "");
// Custom LLM-judged metrics. `fields` become the jsonInstance object that
// the prompt template's {placeholders} read from.
static clients::body::Json pointwise(const std::string& metric_prompt_template,
const std::map<std::string, std::string>& fields,
const std::string& system_instruction = "");
static clients::body::Json pairwise(const std::string& metric_prompt_template,
const std::map<std::string, std::string>& fields,
const std::string& baseline_field = "baseline",
const std::string& candidate_field = "candidate",
const std::string& system_instruction = "");
// Sets autoraterConfig on a request (validates sample_count).
static void with_autorater(clients::body::Json& request, const AutoraterOptions& options);
// Reads whichever result union member the response carries.
static MetricOutcome parse(const clients::body::Json& response);
static const char* metric_key(TrajectoryMetric metric); // e.g. "trajectoryExactMatch"
};
} // namespace molo::google::optimize