feat(clients): optimize: evaluation, simulation, observability and optimizer on a discovery-checked call table
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
#pragma once
|
||||
|
||||
// Builders for evaluateInstances request bodies and the parser for its
|
||||
// responses. Field names come from the aiplatform v1 Discovery document
|
||||
// (see calls_table.inc and tools/check_discovery.py). Pure functions: no
|
||||
// network, fully testable.
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "clients/body/json.h"
|
||||
|
||||
namespace molo::google::optimize {
|
||||
|
||||
struct TrajectoryCall {
|
||||
std::string tool_name;
|
||||
std::string tool_input = "{}"; // Discovery: ToolCall.toolInput is a string
|
||||
};
|
||||
|
||||
enum class TrajectoryMetric {
|
||||
kExactMatch, kInOrderMatch, kAnyOrderMatch, kPrecision, kRecall, kSingleToolUse
|
||||
};
|
||||
|
||||
struct AutoraterOptions {
|
||||
std::string model; // empty: Google's default autorater
|
||||
int sample_count = 0; // 0: unset; else 1..32
|
||||
bool flip = false; // pairwise position-bias flip
|
||||
};
|
||||
|
||||
struct MetricOutcome {
|
||||
std::string kind; // response union key, e.g. "exactMatchResults"
|
||||
std::vector<double> scores;
|
||||
std::string explanation;
|
||||
std::string choice; // pairwise: BASELINE | CANDIDATE | TIE
|
||||
clients::body::Json raw;
|
||||
};
|
||||
|
||||
class EvaluationRequests {
|
||||
public:
|
||||
// LLM-judged, safety. Needs access to the default autorater model.
|
||||
static clients::body::Json safety(const std::string& prediction);
|
||||
// Computation based.
|
||||
static clients::body::Json exact_match(
|
||||
const std::vector<std::pair<std::string, std::string>>& prediction_reference);
|
||||
static clients::body::Json trajectory(TrajectoryMetric metric,
|
||||
const std::vector<TrajectoryCall>& predicted,
|
||||
const std::vector<TrajectoryCall>& reference,
|
||||
const std::string& single_tool_name = "");
|
||||
// Custom LLM-judged metrics. `fields` become the jsonInstance object that
|
||||
// the prompt template's {placeholders} read from.
|
||||
static clients::body::Json pointwise(const std::string& metric_prompt_template,
|
||||
const std::map<std::string, std::string>& fields,
|
||||
const std::string& system_instruction = "");
|
||||
static clients::body::Json pairwise(const std::string& metric_prompt_template,
|
||||
const std::map<std::string, std::string>& fields,
|
||||
const std::string& baseline_field = "baseline",
|
||||
const std::string& candidate_field = "candidate",
|
||||
const std::string& system_instruction = "");
|
||||
// Sets autoraterConfig on a request (validates sample_count).
|
||||
static void with_autorater(clients::body::Json& request, const AutoraterOptions& options);
|
||||
|
||||
// Reads whichever result union member the response carries.
|
||||
static MetricOutcome parse(const clients::body::Json& response);
|
||||
|
||||
static const char* metric_key(TrajectoryMetric metric); // e.g. "trajectoryExactMatch"
|
||||
};
|
||||
|
||||
} // namespace molo::google::optimize
|
||||
Reference in New Issue
Block a user