// Call table, request builders and API-class validation. No scripted Google // replies: everything here is local and deterministic. The table itself is // verified against Discovery by tools/check_discovery.py. #include #include "framework.h" #include "google/optimize/calls.h" #include "google/optimize/eval_requests.h" #include "google/optimize/eval_resources.h" #include "google/optimize/evaluation_api.h" #include "google/errors.h" #include "fakes.h" using namespace molo::google; using namespace molo::google::optimize; using molo::clients::body::Json; using optimize_test::OfflineClient; namespace { CloudContext ctx() { CloudContext c; c.project = "test-proj"; c.location = "us-central1"; return c; } const char* kSet = "projects/test-proj/locations/us-central1/evaluationSets/s1"; const char* kRun = "projects/test-proj/locations/us-central1/evaluationRuns/r1"; const char* kMetric = "projects/test-proj/locations/us-central1/evaluationMetrics/m1"; bool is_invalid(const std::function& f) { try { f(); } catch (const GoogleError& e) { return e.http_status() == 400 && e.status() == "INVALID_ARGUMENT"; } return false; } } // namespace TEST(optimize_calls, table_is_well_formed) { std::set ids; ASSERT_TRUE(all_calls().size() >= 38); for (const CallSpec& c : all_calls()) { ASSERT_TRUE(ids.insert(c.id).second); // unique rpc ids c.method(); // known verb ASSERT_TRUE(c.path[0] == '/'); std::string svc = c.service; ASSERT_TRUE(svc == "aiplatform" || svc == "cloudtrace" || svc == "logging"); if (std::string(c.http) == "GET" || std::string(c.http) == "DELETE") ASSERT_TRUE(std::string(c.body_fields).empty()); } ASSERT_THROW(call_spec("nope"), std::logic_error); ASSERT_TRUE(find_call("nope") == nullptr); } TEST(optimize_calls, expand_path_validates) { const CallSpec& s = call_spec(rpc::kEvaluateInstances); ASSERT_EQ(expand_path(s, {{"location", "projects/p/locations/l"}}), "/projects/p/locations/l:evaluateInstances"); ASSERT_TRUE(is_invalid([&] { expand_path(s, {{"location", "../x"}}); })); ASSERT_TRUE(is_invalid([&] { expand_path(s, {{"location", "locations/l"}}); })); ASSERT_TRUE(is_invalid([&] { expand_path(s, {{"location", "projects/p/../q"}}); })); ASSERT_TRUE(is_invalid([&] { expand_path(s, {}); })); const CallSpec& t = call_spec(rpc::kTraceGet); ASSERT_EQ(expand_path(t, {{"projectId", "p"}, {"traceId", "abc"}}), "/projects/p/traces/abc"); ASSERT_TRUE(is_invalid([&] { expand_path(t, {{"projectId", "a/b"}, {"traceId", "x"}}); })); } TEST(optimize_calls, query_and_body_checked_against_table) { const CallSpec& list = call_spec(rpc_id("evaluationRuns", "list")); ASSERT_EQ(with_query(list, "/x", {{"pageSize", "5"}, {"pageToken", "a b"}, {"filter", ""}}), "/x?pageSize=5&pageToken=a%20b"); ASSERT_TRUE(is_invalid([&] { with_query(list, "/x", {{"evil", "1"}}); })); const CallSpec& ev = call_spec(rpc::kEvaluateInstances); Json ok = Json::object(); ok.set("safetyInput", Json::object()); check_body(ev, ok); Json bad = Json::object(); bad.set("notAField", Json(1)); ASSERT_TRUE(is_invalid([&] { check_body(ev, bad); })); ASSERT_TRUE(is_invalid([&] { check_body(ev, Json(3)); })); } TEST(optimize_requests, every_builder_only_emits_table_fields) { const CallSpec& ev = call_spec(rpc::kEvaluateInstances); std::vector bodies; bodies.push_back(EvaluationRequests::safety("hello")); bodies.push_back(EvaluationRequests::exact_match({{"a", "a"}})); for (auto m : {TrajectoryMetric::kExactMatch, TrajectoryMetric::kInOrderMatch, TrajectoryMetric::kAnyOrderMatch, TrajectoryMetric::kPrecision, TrajectoryMetric::kRecall}) bodies.push_back(EvaluationRequests::trajectory(m, {{"f", "{}"}}, {{"f", "{}"}})); bodies.push_back(EvaluationRequests::trajectory(TrajectoryMetric::kSingleToolUse, {{"f", "{}"}}, {}, "f")); bodies.push_back(EvaluationRequests::pointwise("rate {response}", {{"response", "x"}}, "judge")); bodies.push_back(EvaluationRequests::pairwise("pick {baseline} {candidate}", {{"baseline", "a"}, {"candidate", "b"}})); for (Json& b : bodies) { EvaluationRequests::with_autorater(b, {"gemini-2.5-flash", 3, true}); check_body(ev, b); // top-level fields exist in Discovery's request schema } } TEST(optimize_requests, shapes) { Json s = EvaluationRequests::safety("hi"); ASSERT_EQ(s.find("safetyInput")->find("instance")->find("prediction")->as_string(), "hi"); Json t = EvaluationRequests::trajectory(TrajectoryMetric::kExactMatch, {{"search", "{\"q\":1}"}}, {{"search", "{\"q\":1}"}}); const Json& inst = t.find("trajectoryExactMatchInput")->find("instances")->as_array()[0]; ASSERT_EQ(inst.find("predictedTrajectory")->find("toolCalls")->as_array()[0].find("toolName")->as_string(), "search"); Json p = EvaluationRequests::pointwise("t {a}", {{"a", "1"}}); // jsonInstance is a JSON document inside a string (Discovery: string). ASSERT_TRUE(p.find("pointwiseMetricInput")->find("instance")->find("jsonInstance")->is_string()); Json c = EvaluationRequests::safety("x"); EvaluationRequests::with_autorater(c, {"m", 2, false}); ASSERT_EQ(c.find("autoraterConfig")->find("samplingCount")->as_int(), 2); ASSERT_TRUE(c.find("autoraterConfig")->find("flipEnabled") == nullptr); } TEST(optimize_requests, validation_and_caps) { ASSERT_TRUE(is_invalid([] { EvaluationRequests::safety(""); })); ASSERT_TRUE(is_invalid([] { EvaluationRequests::safety(std::string(20001, 'x')); })); ASSERT_TRUE(is_invalid([] { EvaluationRequests::exact_match({}); })); std::vector many(101, TrajectoryCall{"f", "{}"}); ASSERT_TRUE(is_invalid([&] { EvaluationRequests::trajectory(TrajectoryMetric::kExactMatch, many, {}); })); ASSERT_TRUE(is_invalid([] { EvaluationRequests::trajectory(TrajectoryMetric::kSingleToolUse, {{"f", "{}"}}, {}, ""); })); ASSERT_TRUE(is_invalid([] { EvaluationRequests::pointwise("", {{"a", "b"}}); })); ASSERT_TRUE(is_invalid([] { EvaluationRequests::pointwise("t", {}); })); ASSERT_TRUE(is_invalid([] { EvaluationRequests::pairwise("t", {{"baseline", "a"}}); // candidate missing })); Json r = Json::object(); ASSERT_TRUE(is_invalid([&] { EvaluationRequests::with_autorater(r, {"", 33, false}); })); } TEST(optimize_requests, parse_result_unions) { auto a = EvaluationRequests::parse(Json::parse( "{\"exactMatchResults\":{\"exactMatchMetricValues\":[{\"score\":1},{\"score\":0}]}}")); ASSERT_EQ(a.kind, "exactMatchResults"); ASSERT_EQ(a.scores.size(), 2u); auto b = EvaluationRequests::parse(Json::parse( "{\"safetyResult\":{\"score\":0.5,\"explanation\":\"meh\",\"confidence\":0.9}}")); ASSERT_EQ(b.scores[0], 0.5); ASSERT_EQ(b.explanation, "meh"); auto c = EvaluationRequests::parse(Json::parse( "{\"pairwiseMetricResult\":{\"pairwiseChoice\":\"TIE\",\"explanation\":\"x\"}}")); ASSERT_EQ(c.choice, "TIE"); ASSERT_TRUE(EvaluationRequests::parse(Json::object()).kind.empty()); } TEST(optimize_api, evaluation_api_rejects_before_traffic) { OfflineClient off; EvaluationApi api(off.client, ctx()); ASSERT_TRUE(is_invalid([&] { api.evaluate_instances(Json::object()); })); Json bad = Json::object(); bad.set("nope", Json(1)); ASSERT_TRUE(is_invalid([&] { api.evaluate_instances(bad); })); ASSERT_TRUE(is_invalid([&] { api.generate_loss_clusters("bad", {{"m", "c"}}); })); ASSERT_TRUE(is_invalid([&] { api.generate_loss_clusters(kSet, {}); })); ScenarioRequest r; r.count = 101; ASSERT_TRUE(is_invalid([&] { api.generate_user_scenarios(r); })); r.count = 3; ASSERT_TRUE(is_invalid([&] { api.generate_user_scenarios(r); })); // no agents ASSERT_EQ(off.mock->send_count(), 0u); } TEST(optimize_api, no_credential_is_loud_503) { OfflineClient off(false); EvaluationApi api(off.client, ctx()); try { api.evaluate(EvaluationRequests::safety("hi")); FAIL("expected NotConfiguredError"); } catch (const NotConfiguredError& e) { ASSERT_EQ(e.http_status(), 503); } ASSERT_EQ(off.mock->send_count(), 0u); } TEST(optimize_api, scenario_body_matches_discovery_shape) { ScenarioRequest r; r.agents = Json::parse("{\"root\":{\"agentType\":\"LlmAgent\",\"instruction\":\"help\"}}"); r.root_agent_id = "root"; r.count = 7; r.instruction = "adversarial"; Json b = EvaluationApi::scenario_body(r); check_body(call_spec(rpc::kGenerateUserScenarios), b); const Json* cfg = b.find("userScenarioGenerationConfig"); // Discovery types userScenarioCount as a string (int64). ASSERT_EQ(cfg->find("userScenarioCount")->as_string(), "7"); r.root_agent_id = "other"; ASSERT_TRUE(is_invalid([&] { EvaluationApi::scenario_body(r); })); } TEST(optimize_api, resources_validate_names_verbs_and_caps) { OfflineClient off; EvaluationResources res(off.client, ctx()); ASSERT_TRUE(is_invalid([&] { res.get(Collection::kEvaluationRuns, kSet); })); // wrong collection ASSERT_TRUE(is_invalid([&] { res.get(Collection::kEvaluationRuns, "../../x"); })); ListOptions bad; bad.page_size = 0; ASSERT_TRUE(is_invalid([&] { res.list(Collection::kEvaluationRuns, bad); })); bad.page_size = 10; bad.max_pages = 21; ASSERT_TRUE(is_invalid([&] { res.list(Collection::kEvaluationRuns, bad); })); bad.max_pages = 1; bad.filter = std::string(1001, 'x'); ASSERT_TRUE(is_invalid([&] { res.list(Collection::kEvaluationRuns, bad); })); // evaluationItems/evaluationMetrics/evaluationRuns have no patch in Discovery. Json body = Json::object(); ASSERT_TRUE(is_invalid([&] { res.patch(Collection::kEvaluationRuns, kRun, body, "displayName"); })); ASSERT_TRUE(is_invalid([&] { res.action(Collection::kEvaluationRuns, kRun, "activate"); })); ASSERT_TRUE(is_invalid([&] { res.action(Collection::kOnlineEvaluators, kRun, "suspend"); })); ASSERT_TRUE(is_invalid([&] { res.create(Collection::kEvaluationRuns, body, "id1"); })); Json unknown = Json::object(); unknown.set("bogus", Json(1)); ASSERT_TRUE(is_invalid([&] { res.create(Collection::kEvaluationSets, unknown); })); ASSERT_TRUE(is_invalid([&] { res.delete_resource(Collection::kEvaluationMetrics, kSet); })); ASSERT_EQ(off.mock->send_count(), 0u); (void)kMetric; } TEST(optimize_api, run_spec_body_matches_discovery) { EvaluationRunSpec s; s.display_name = "nightly"; s.evaluation_set = kSet; s.predefined_metrics = {"final_response_quality_v1"}; s.agent_engine = "projects/test-proj/locations/us-central1/reasoningEngines/e1"; s.simulator_max_turns = 5; s.simulator_model = "gemini-2.5-flash"; s.autorater_samples = 3; Json b = s.to_json(); check_body(call_spec(rpc_id("evaluationRuns", "create")), b); const Json* arc = b.find("inferenceConfigs")->find("candidate")->find("agentRunConfig"); ASSERT_EQ(arc->find("userSimulatorConfig")->find("maxTurn")->as_int(), 5); ASSERT_EQ(b.find("dataSource")->find("evaluationSet")->as_string(), kSet); EvaluationRunSpec bad = s; bad.evaluation_set = "x"; ASSERT_TRUE(is_invalid([&] { bad.to_json(); })); bad = s; bad.predefined_metrics.clear(); ASSERT_TRUE(is_invalid([&] { bad.to_json(); })); bad = s; bad.simulator_max_turns = 13; ASSERT_TRUE(is_invalid([&] { bad.to_json(); })); bad = s; bad.predefined_metrics.assign(17, "m"); ASSERT_TRUE(is_invalid([&] { bad.to_json(); })); } TEST(optimize_api, online_evaluator_and_item_specs) { OnlineEvaluatorSpec o; o.display_name = "live"; o.agent_resource = "projects/test-proj/locations/us-central1/reasoningEngines/e1"; o.metric_resource_names = {kMetric}; o.sampling_percentage = 20; o.max_samples_per_run = 50; Json b = o.to_json(); check_body(call_spec(rpc_id("onlineEvaluators", "create")), b); // int64 is a string in Discovery's JSON mapping. ASSERT_EQ(b.find("config")->find("maxEvaluatedSamplesPerRun")->as_string(), "50"); OnlineEvaluatorSpec bad = o; bad.sampling_percentage = 0; ASSERT_TRUE(is_invalid([&] { bad.to_json(); })); bad = o; bad.max_samples_per_run = 0; // 0 means unbounded at Google: refused ASSERT_TRUE(is_invalid([&] { bad.to_json(); })); bad = o; bad.metric_resource_names = {"nope"}; ASSERT_TRUE(is_invalid([&] { bad.to_json(); })); EvaluationItemSpec item; item.display_name = "case 1"; item.scenario_start = "hi"; item.scenario_plan = "ask for a refund"; Json ib = item.to_json(); check_body(call_spec(rpc_id("evaluationItems", "create")), ib); ASSERT_EQ(ib.find("evaluationItemType")->as_string(), "REQUEST"); item.prompt = "also a prompt"; ASSERT_TRUE(is_invalid([&] { item.to_json(); })); }