feat(clients): optimize: evaluation, simulation, observability and optimizer on a discovery-checked call table

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-24 04:40:55 +02:00
co-authored by Claude Sonnet 5.5
parent 6d744a433f
commit 1e2fb400bb
55 changed files with 5099 additions and 3 deletions
+287
View File
@@ -0,0 +1,287 @@
// Call table, request builders and API-class validation. No scripted Google
// replies: everything here is local and deterministic. The table itself is
// verified against Discovery by tools/check_discovery.py.
#include <set>
#include "framework.h"
#include "google/optimize/calls.h"
#include "google/optimize/eval_requests.h"
#include "google/optimize/eval_resources.h"
#include "google/optimize/evaluation_api.h"
#include "google/errors.h"
#include "fakes.h"
using namespace molo::google;
using namespace molo::google::optimize;
using molo::clients::body::Json;
using optimize_test::OfflineClient;
namespace {
CloudContext ctx() {
CloudContext c;
c.project = "test-proj";
c.location = "us-central1";
return c;
}
const char* kSet = "projects/test-proj/locations/us-central1/evaluationSets/s1";
const char* kRun = "projects/test-proj/locations/us-central1/evaluationRuns/r1";
const char* kMetric = "projects/test-proj/locations/us-central1/evaluationMetrics/m1";
bool is_invalid(const std::function<void()>& f) {
try {
f();
} catch (const GoogleError& e) {
return e.http_status() == 400 && e.status() == "INVALID_ARGUMENT";
}
return false;
}
} // namespace
TEST(optimize_calls, table_is_well_formed) {
std::set<std::string> ids;
ASSERT_TRUE(all_calls().size() >= 38);
for (const CallSpec& c : all_calls()) {
ASSERT_TRUE(ids.insert(c.id).second); // unique rpc ids
c.method(); // known verb
ASSERT_TRUE(c.path[0] == '/');
std::string svc = c.service;
ASSERT_TRUE(svc == "aiplatform" || svc == "cloudtrace" || svc == "logging");
if (std::string(c.http) == "GET" || std::string(c.http) == "DELETE")
ASSERT_TRUE(std::string(c.body_fields).empty());
}
ASSERT_THROW(call_spec("nope"), std::logic_error);
ASSERT_TRUE(find_call("nope") == nullptr);
}
TEST(optimize_calls, expand_path_validates) {
const CallSpec& s = call_spec(rpc::kEvaluateInstances);
ASSERT_EQ(expand_path(s, {{"location", "projects/p/locations/l"}}),
"/projects/p/locations/l:evaluateInstances");
ASSERT_TRUE(is_invalid([&] { expand_path(s, {{"location", "../x"}}); }));
ASSERT_TRUE(is_invalid([&] { expand_path(s, {{"location", "locations/l"}}); }));
ASSERT_TRUE(is_invalid([&] { expand_path(s, {{"location", "projects/p/../q"}}); }));
ASSERT_TRUE(is_invalid([&] { expand_path(s, {}); }));
const CallSpec& t = call_spec(rpc::kTraceGet);
ASSERT_EQ(expand_path(t, {{"projectId", "p"}, {"traceId", "abc"}}), "/projects/p/traces/abc");
ASSERT_TRUE(is_invalid([&] { expand_path(t, {{"projectId", "a/b"}, {"traceId", "x"}}); }));
}
TEST(optimize_calls, query_and_body_checked_against_table) {
const CallSpec& list = call_spec(rpc_id("evaluationRuns", "list"));
ASSERT_EQ(with_query(list, "/x", {{"pageSize", "5"}, {"pageToken", "a b"}, {"filter", ""}}),
"/x?pageSize=5&pageToken=a%20b");
ASSERT_TRUE(is_invalid([&] { with_query(list, "/x", {{"evil", "1"}}); }));
const CallSpec& ev = call_spec(rpc::kEvaluateInstances);
Json ok = Json::object();
ok.set("safetyInput", Json::object());
check_body(ev, ok);
Json bad = Json::object();
bad.set("notAField", Json(1));
ASSERT_TRUE(is_invalid([&] { check_body(ev, bad); }));
ASSERT_TRUE(is_invalid([&] { check_body(ev, Json(3)); }));
}
TEST(optimize_requests, every_builder_only_emits_table_fields) {
const CallSpec& ev = call_spec(rpc::kEvaluateInstances);
std::vector<Json> bodies;
bodies.push_back(EvaluationRequests::safety("hello"));
bodies.push_back(EvaluationRequests::exact_match({{"a", "a"}}));
for (auto m : {TrajectoryMetric::kExactMatch, TrajectoryMetric::kInOrderMatch,
TrajectoryMetric::kAnyOrderMatch, TrajectoryMetric::kPrecision,
TrajectoryMetric::kRecall})
bodies.push_back(EvaluationRequests::trajectory(m, {{"f", "{}"}}, {{"f", "{}"}}));
bodies.push_back(EvaluationRequests::trajectory(TrajectoryMetric::kSingleToolUse,
{{"f", "{}"}}, {}, "f"));
bodies.push_back(EvaluationRequests::pointwise("rate {response}", {{"response", "x"}}, "judge"));
bodies.push_back(EvaluationRequests::pairwise("pick {baseline} {candidate}",
{{"baseline", "a"}, {"candidate", "b"}}));
for (Json& b : bodies) {
EvaluationRequests::with_autorater(b, {"gemini-2.5-flash", 3, true});
check_body(ev, b); // top-level fields exist in Discovery's request schema
}
}
TEST(optimize_requests, shapes) {
Json s = EvaluationRequests::safety("hi");
ASSERT_EQ(s.find("safetyInput")->find("instance")->find("prediction")->as_string(), "hi");
Json t = EvaluationRequests::trajectory(TrajectoryMetric::kExactMatch,
{{"search", "{\"q\":1}"}}, {{"search", "{\"q\":1}"}});
const Json& inst = t.find("trajectoryExactMatchInput")->find("instances")->as_array()[0];
ASSERT_EQ(inst.find("predictedTrajectory")->find("toolCalls")->as_array()[0].find("toolName")->as_string(),
"search");
Json p = EvaluationRequests::pointwise("t {a}", {{"a", "1"}});
// jsonInstance is a JSON document inside a string (Discovery: string).
ASSERT_TRUE(p.find("pointwiseMetricInput")->find("instance")->find("jsonInstance")->is_string());
Json c = EvaluationRequests::safety("x");
EvaluationRequests::with_autorater(c, {"m", 2, false});
ASSERT_EQ(c.find("autoraterConfig")->find("samplingCount")->as_int(), 2);
ASSERT_TRUE(c.find("autoraterConfig")->find("flipEnabled") == nullptr);
}
TEST(optimize_requests, validation_and_caps) {
ASSERT_TRUE(is_invalid([] { EvaluationRequests::safety(""); }));
ASSERT_TRUE(is_invalid([] { EvaluationRequests::safety(std::string(20001, 'x')); }));
ASSERT_TRUE(is_invalid([] { EvaluationRequests::exact_match({}); }));
std::vector<TrajectoryCall> many(101, TrajectoryCall{"f", "{}"});
ASSERT_TRUE(is_invalid([&] {
EvaluationRequests::trajectory(TrajectoryMetric::kExactMatch, many, {});
}));
ASSERT_TRUE(is_invalid([] {
EvaluationRequests::trajectory(TrajectoryMetric::kSingleToolUse, {{"f", "{}"}}, {}, "");
}));
ASSERT_TRUE(is_invalid([] { EvaluationRequests::pointwise("", {{"a", "b"}}); }));
ASSERT_TRUE(is_invalid([] { EvaluationRequests::pointwise("t", {}); }));
ASSERT_TRUE(is_invalid([] {
EvaluationRequests::pairwise("t", {{"baseline", "a"}}); // candidate missing
}));
Json r = Json::object();
ASSERT_TRUE(is_invalid([&] { EvaluationRequests::with_autorater(r, {"", 33, false}); }));
}
TEST(optimize_requests, parse_result_unions) {
auto a = EvaluationRequests::parse(Json::parse(
"{\"exactMatchResults\":{\"exactMatchMetricValues\":[{\"score\":1},{\"score\":0}]}}"));
ASSERT_EQ(a.kind, "exactMatchResults");
ASSERT_EQ(a.scores.size(), 2u);
auto b = EvaluationRequests::parse(Json::parse(
"{\"safetyResult\":{\"score\":0.5,\"explanation\":\"meh\",\"confidence\":0.9}}"));
ASSERT_EQ(b.scores[0], 0.5);
ASSERT_EQ(b.explanation, "meh");
auto c = EvaluationRequests::parse(Json::parse(
"{\"pairwiseMetricResult\":{\"pairwiseChoice\":\"TIE\",\"explanation\":\"x\"}}"));
ASSERT_EQ(c.choice, "TIE");
ASSERT_TRUE(EvaluationRequests::parse(Json::object()).kind.empty());
}
TEST(optimize_api, evaluation_api_rejects_before_traffic) {
OfflineClient off;
EvaluationApi api(off.client, ctx());
ASSERT_TRUE(is_invalid([&] { api.evaluate_instances(Json::object()); }));
Json bad = Json::object();
bad.set("nope", Json(1));
ASSERT_TRUE(is_invalid([&] { api.evaluate_instances(bad); }));
ASSERT_TRUE(is_invalid([&] { api.generate_loss_clusters("bad", {{"m", "c"}}); }));
ASSERT_TRUE(is_invalid([&] { api.generate_loss_clusters(kSet, {}); }));
ScenarioRequest r;
r.count = 101;
ASSERT_TRUE(is_invalid([&] { api.generate_user_scenarios(r); }));
r.count = 3;
ASSERT_TRUE(is_invalid([&] { api.generate_user_scenarios(r); })); // no agents
ASSERT_EQ(off.mock->send_count(), 0u);
}
TEST(optimize_api, no_credential_is_loud_503) {
OfflineClient off(false);
EvaluationApi api(off.client, ctx());
try {
api.evaluate(EvaluationRequests::safety("hi"));
FAIL("expected NotConfiguredError");
} catch (const NotConfiguredError& e) {
ASSERT_EQ(e.http_status(), 503);
}
ASSERT_EQ(off.mock->send_count(), 0u);
}
TEST(optimize_api, scenario_body_matches_discovery_shape) {
ScenarioRequest r;
r.agents = Json::parse("{\"root\":{\"agentType\":\"LlmAgent\",\"instruction\":\"help\"}}");
r.root_agent_id = "root";
r.count = 7;
r.instruction = "adversarial";
Json b = EvaluationApi::scenario_body(r);
check_body(call_spec(rpc::kGenerateUserScenarios), b);
const Json* cfg = b.find("userScenarioGenerationConfig");
// Discovery types userScenarioCount as a string (int64).
ASSERT_EQ(cfg->find("userScenarioCount")->as_string(), "7");
r.root_agent_id = "other";
ASSERT_TRUE(is_invalid([&] { EvaluationApi::scenario_body(r); }));
}
TEST(optimize_api, resources_validate_names_verbs_and_caps) {
OfflineClient off;
EvaluationResources res(off.client, ctx());
ASSERT_TRUE(is_invalid([&] { res.get(Collection::kEvaluationRuns, kSet); })); // wrong collection
ASSERT_TRUE(is_invalid([&] { res.get(Collection::kEvaluationRuns, "../../x"); }));
ListOptions bad;
bad.page_size = 0;
ASSERT_TRUE(is_invalid([&] { res.list(Collection::kEvaluationRuns, bad); }));
bad.page_size = 10;
bad.max_pages = 21;
ASSERT_TRUE(is_invalid([&] { res.list(Collection::kEvaluationRuns, bad); }));
bad.max_pages = 1;
bad.filter = std::string(1001, 'x');
ASSERT_TRUE(is_invalid([&] { res.list(Collection::kEvaluationRuns, bad); }));
// evaluationItems/evaluationMetrics/evaluationRuns have no patch in Discovery.
Json body = Json::object();
ASSERT_TRUE(is_invalid([&] { res.patch(Collection::kEvaluationRuns, kRun, body, "displayName"); }));
ASSERT_TRUE(is_invalid([&] { res.action(Collection::kEvaluationRuns, kRun, "activate"); }));
ASSERT_TRUE(is_invalid([&] { res.action(Collection::kOnlineEvaluators, kRun, "suspend"); }));
ASSERT_TRUE(is_invalid([&] { res.create(Collection::kEvaluationRuns, body, "id1"); }));
Json unknown = Json::object();
unknown.set("bogus", Json(1));
ASSERT_TRUE(is_invalid([&] { res.create(Collection::kEvaluationSets, unknown); }));
ASSERT_TRUE(is_invalid([&] { res.delete_resource(Collection::kEvaluationMetrics, kSet); }));
ASSERT_EQ(off.mock->send_count(), 0u);
(void)kMetric;
}
TEST(optimize_api, run_spec_body_matches_discovery) {
EvaluationRunSpec s;
s.display_name = "nightly";
s.evaluation_set = kSet;
s.predefined_metrics = {"final_response_quality_v1"};
s.agent_engine = "projects/test-proj/locations/us-central1/reasoningEngines/e1";
s.simulator_max_turns = 5;
s.simulator_model = "gemini-2.5-flash";
s.autorater_samples = 3;
Json b = s.to_json();
check_body(call_spec(rpc_id("evaluationRuns", "create")), b);
const Json* arc = b.find("inferenceConfigs")->find("candidate")->find("agentRunConfig");
ASSERT_EQ(arc->find("userSimulatorConfig")->find("maxTurn")->as_int(), 5);
ASSERT_EQ(b.find("dataSource")->find("evaluationSet")->as_string(), kSet);
EvaluationRunSpec bad = s;
bad.evaluation_set = "x";
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
bad = s;
bad.predefined_metrics.clear();
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
bad = s;
bad.simulator_max_turns = 13;
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
bad = s;
bad.predefined_metrics.assign(17, "m");
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
}
TEST(optimize_api, online_evaluator_and_item_specs) {
OnlineEvaluatorSpec o;
o.display_name = "live";
o.agent_resource = "projects/test-proj/locations/us-central1/reasoningEngines/e1";
o.metric_resource_names = {kMetric};
o.sampling_percentage = 20;
o.max_samples_per_run = 50;
Json b = o.to_json();
check_body(call_spec(rpc_id("onlineEvaluators", "create")), b);
// int64 is a string in Discovery's JSON mapping.
ASSERT_EQ(b.find("config")->find("maxEvaluatedSamplesPerRun")->as_string(), "50");
OnlineEvaluatorSpec bad = o;
bad.sampling_percentage = 0;
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
bad = o;
bad.max_samples_per_run = 0; // 0 means unbounded at Google: refused
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
bad = o;
bad.metric_resource_names = {"nope"};
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
EvaluationItemSpec item;
item.display_name = "case 1";
item.scenario_start = "hi";
item.scenario_plan = "ask for a refund";
Json ib = item.to_json();
check_body(call_spec(rpc_id("evaluationItems", "create")), ib);
ASSERT_EQ(ib.find("evaluationItemType")->as_string(), "REQUEST");
item.prompt = "also a prompt";
ASSERT_TRUE(is_invalid([&] { item.to_json(); }));
}