288 lines
12 KiB
C++
288 lines
12 KiB
C++
// Call table, request builders and API-class validation. No scripted Google
|
|
// replies: everything here is local and deterministic. The table itself is
|
|
// verified against Discovery by tools/check_discovery.py.
|
|
#include <set>
|
|
|
|
#include "framework.h"
|
|
#include "google/optimize/calls.h"
|
|
#include "google/optimize/eval_requests.h"
|
|
#include "google/optimize/eval_resources.h"
|
|
#include "google/optimize/evaluation_api.h"
|
|
#include "google/errors.h"
|
|
#include "fakes.h"
|
|
|
|
using namespace molo::google;
|
|
using namespace molo::google::optimize;
|
|
using molo::clients::body::Json;
|
|
using optimize_test::OfflineClient;
|
|
|
|
namespace {
|
|
CloudContext ctx() {
|
|
CloudContext c;
|
|
c.project = "test-proj";
|
|
c.location = "us-central1";
|
|
return c;
|
|
}
|
|
const char* kSet = "projects/test-proj/locations/us-central1/evaluationSets/s1";
|
|
const char* kRun = "projects/test-proj/locations/us-central1/evaluationRuns/r1";
|
|
const char* kMetric = "projects/test-proj/locations/us-central1/evaluationMetrics/m1";
|
|
|
|
bool is_invalid(const std::function<void()>& f) {
|
|
try {
|
|
f();
|
|
} catch (const GoogleError& e) {
|
|
return e.http_status() == 400 && e.status() == "INVALID_ARGUMENT";
|
|
}
|
|
return false;
|
|
}
|
|
} // namespace
|
|
|
|
TEST(optimize_calls, table_is_well_formed) {
|
|
std::set<std::string> ids;
|
|
ASSERT_TRUE(all_calls().size() >= 38);
|
|
for (const CallSpec& c : all_calls()) {
|
|
ASSERT_TRUE(ids.insert(c.id).second); // unique rpc ids
|
|
c.method(); // known verb
|
|
ASSERT_TRUE(c.path[0] == '/');
|
|
std::string svc = c.service;
|
|
ASSERT_TRUE(svc == "aiplatform" || svc == "cloudtrace" || svc == "logging");
|
|
if (std::string(c.http) == "GET" || std::string(c.http) == "DELETE")
|
|
ASSERT_TRUE(std::string(c.body_fields).empty());
|
|
}
|
|
ASSERT_THROW(call_spec("nope"), std::logic_error);
|
|
ASSERT_TRUE(find_call("nope") == nullptr);
|
|
}
|
|
|
|
TEST(optimize_calls, expand_path_validates) {
|
|
const CallSpec& s = call_spec(rpc::kEvaluateInstances);
|
|
ASSERT_EQ(expand_path(s, {{"location", "projects/p/locations/l"}}),
|
|
"/projects/p/locations/l:evaluateInstances");
|
|
ASSERT_TRUE(is_invalid([&] { expand_path(s, {{"location", "../x"}}); }));
|
|
ASSERT_TRUE(is_invalid([&] { expand_path(s, {{"location", "locations/l"}}); }));
|
|
ASSERT_TRUE(is_invalid([&] { expand_path(s, {{"location", "projects/p/../q"}}); }));
|
|
ASSERT_TRUE(is_invalid([&] { expand_path(s, {}); }));
|
|
const CallSpec& t = call_spec(rpc::kTraceGet);
|
|
ASSERT_EQ(expand_path(t, {{"projectId", "p"}, {"traceId", "abc"}}), "/projects/p/traces/abc");
|
|
ASSERT_TRUE(is_invalid([&] { expand_path(t, {{"projectId", "a/b"}, {"traceId", "x"}}); }));
|
|
}
|
|
|
|
TEST(optimize_calls, query_and_body_checked_against_table) {
|
|
const CallSpec& list = call_spec(rpc_id("evaluationRuns", "list"));
|
|
ASSERT_EQ(with_query(list, "/x", {{"pageSize", "5"}, {"pageToken", "a b"}, {"filter", ""}}),
|
|
"/x?pageSize=5&pageToken=a%20b");
|
|
ASSERT_TRUE(is_invalid([&] { with_query(list, "/x", {{"evil", "1"}}); }));
|
|
const CallSpec& ev = call_spec(rpc::kEvaluateInstances);
|
|
Json ok = Json::object();
|
|
ok.set("safetyInput", Json::object());
|
|
check_body(ev, ok);
|
|
Json bad = Json::object();
|
|
bad.set("notAField", Json(1));
|
|
ASSERT_TRUE(is_invalid([&] { check_body(ev, bad); }));
|
|
ASSERT_TRUE(is_invalid([&] { check_body(ev, Json(3)); }));
|
|
}
|
|
|
|
TEST(optimize_requests, every_builder_only_emits_table_fields) {
|
|
const CallSpec& ev = call_spec(rpc::kEvaluateInstances);
|
|
std::vector<Json> bodies;
|
|
bodies.push_back(EvaluationRequests::safety("hello"));
|
|
bodies.push_back(EvaluationRequests::exact_match({{"a", "a"}}));
|
|
for (auto m : {TrajectoryMetric::kExactMatch, TrajectoryMetric::kInOrderMatch,
|
|
TrajectoryMetric::kAnyOrderMatch, TrajectoryMetric::kPrecision,
|
|
TrajectoryMetric::kRecall})
|
|
bodies.push_back(EvaluationRequests::trajectory(m, {{"f", "{}"}}, {{"f", "{}"}}));
|
|
bodies.push_back(EvaluationRequests::trajectory(TrajectoryMetric::kSingleToolUse,
|
|
{{"f", "{}"}}, {}, "f"));
|
|
bodies.push_back(EvaluationRequests::pointwise("rate {response}", {{"response", "x"}}, "judge"));
|
|
bodies.push_back(EvaluationRequests::pairwise("pick {baseline} {candidate}",
|
|
{{"baseline", "a"}, {"candidate", "b"}}));
|
|
for (Json& b : bodies) {
|
|
EvaluationRequests::with_autorater(b, {"gemini-2.5-flash", 3, true});
|
|
check_body(ev, b); // top-level fields exist in Discovery's request schema
|
|
}
|
|
}
|
|
|
|
TEST(optimize_requests, shapes) {
|
|
Json s = EvaluationRequests::safety("hi");
|
|
ASSERT_EQ(s.find("safetyInput")->find("instance")->find("prediction")->as_string(), "hi");
|
|
Json t = EvaluationRequests::trajectory(TrajectoryMetric::kExactMatch,
|
|
{{"search", "{\"q\":1}"}}, {{"search", "{\"q\":1}"}});
|
|
const Json& inst = t.find("trajectoryExactMatchInput")->find("instances")->as_array()[0];
|
|
ASSERT_EQ(inst.find("predictedTrajectory")->find("toolCalls")->as_array()[0].find("toolName")->as_string(),
|
|
"search");
|
|
Json p = EvaluationRequests::pointwise("t {a}", {{"a", "1"}});
|
|
// jsonInstance is a JSON document inside a string (Discovery: string).
|
|
ASSERT_TRUE(p.find("pointwiseMetricInput")->find("instance")->find("jsonInstance")->is_string());
|
|
Json c = EvaluationRequests::safety("x");
|
|
EvaluationRequests::with_autorater(c, {"m", 2, false});
|
|
ASSERT_EQ(c.find("autoraterConfig")->find("samplingCount")->as_int(), 2);
|
|
ASSERT_TRUE(c.find("autoraterConfig")->find("flipEnabled") == nullptr);
|
|
}
|
|
|
|
TEST(optimize_requests, validation_and_caps) {
|
|
ASSERT_TRUE(is_invalid([] { EvaluationRequests::safety(""); }));
|
|
ASSERT_TRUE(is_invalid([] { EvaluationRequests::safety(std::string(20001, 'x')); }));
|
|
ASSERT_TRUE(is_invalid([] { EvaluationRequests::exact_match({}); }));
|
|
std::vector<TrajectoryCall> many(101, TrajectoryCall{"f", "{}"});
|
|
ASSERT_TRUE(is_invalid([&] {
|
|
EvaluationRequests::trajectory(TrajectoryMetric::kExactMatch, many, {});
|
|
}));
|
|
ASSERT_TRUE(is_invalid([] {
|
|
EvaluationRequests::trajectory(TrajectoryMetric::kSingleToolUse, {{"f", "{}"}}, {}, "");
|
|
}));
|
|
ASSERT_TRUE(is_invalid([] { EvaluationRequests::pointwise("", {{"a", "b"}}); }));
|
|
ASSERT_TRUE(is_invalid([] { EvaluationRequests::pointwise("t", {}); }));
|
|
ASSERT_TRUE(is_invalid([] {
|
|
EvaluationRequests::pairwise("t", {{"baseline", "a"}}); // candidate missing
|
|
}));
|
|
Json r = Json::object();
|
|
ASSERT_TRUE(is_invalid([&] { EvaluationRequests::with_autorater(r, {"", 33, false}); }));
|
|
}
|
|
|
|
TEST(optimize_requests, parse_result_unions) {
|
|
auto a = EvaluationRequests::parse(Json::parse(
|
|
"{\"exactMatchResults\":{\"exactMatchMetricValues\":[{\"score\":1},{\"score\":0}]}}"));
|
|
ASSERT_EQ(a.kind, "exactMatchResults");
|
|
ASSERT_EQ(a.scores.size(), 2u);
|
|
auto b = EvaluationRequests::parse(Json::parse(
|
|
"{\"safetyResult\":{\"score\":0.5,\"explanation\":\"meh\",\"confidence\":0.9}}"));
|
|
ASSERT_EQ(b.scores[0], 0.5);
|
|
ASSERT_EQ(b.explanation, "meh");
|
|
auto c = EvaluationRequests::parse(Json::parse(
|
|
"{\"pairwiseMetricResult\":{\"pairwiseChoice\":\"TIE\",\"explanation\":\"x\"}}"));
|
|
ASSERT_EQ(c.choice, "TIE");
|
|
ASSERT_TRUE(EvaluationRequests::parse(Json::object()).kind.empty());
|
|
}
|
|
|
|
TEST(optimize_api, evaluation_api_rejects_before_traffic) {
|
|
OfflineClient off;
|
|
EvaluationApi api(off.client, ctx());
|
|
ASSERT_TRUE(is_invalid([&] { api.evaluate_instances(Json::object()); }));
|
|
Json bad = Json::object();
|
|
bad.set("nope", Json(1));
|
|
ASSERT_TRUE(is_invalid([&] { api.evaluate_instances(bad); }));
|
|
ASSERT_TRUE(is_invalid([&] { api.generate_loss_clusters("bad", {{"m", "c"}}); }));
|
|
ASSERT_TRUE(is_invalid([&] { api.generate_loss_clusters(kSet, {}); }));
|
|
ScenarioRequest r;
|
|
r.count = 101;
|
|
ASSERT_TRUE(is_invalid([&] { api.generate_user_scenarios(r); }));
|
|
r.count = 3;
|
|
ASSERT_TRUE(is_invalid([&] { api.generate_user_scenarios(r); })); // no agents
|
|
ASSERT_EQ(off.mock->send_count(), 0u);
|
|
}
|
|
|
|
TEST(optimize_api, no_credential_is_loud_503) {
|
|
OfflineClient off(false);
|
|
EvaluationApi api(off.client, ctx());
|
|
try {
|
|
api.evaluate(EvaluationRequests::safety("hi"));
|
|
FAIL("expected NotConfiguredError");
|
|
} catch (const NotConfiguredError& e) {
|
|
ASSERT_EQ(e.http_status(), 503);
|
|
}
|
|
ASSERT_EQ(off.mock->send_count(), 0u);
|
|
}
|
|
|
|
TEST(optimize_api, scenario_body_matches_discovery_shape) {
|
|
ScenarioRequest r;
|
|
r.agents = Json::parse("{\"root\":{\"agentType\":\"LlmAgent\",\"instruction\":\"help\"}}");
|
|
r.root_agent_id = "root";
|
|
r.count = 7;
|
|
r.instruction = "adversarial";
|
|
Json b = EvaluationApi::scenario_body(r);
|
|
check_body(call_spec(rpc::kGenerateUserScenarios), b);
|
|
const Json* cfg = b.find("userScenarioGenerationConfig");
|
|
// Discovery types userScenarioCount as a string (int64).
|
|
ASSERT_EQ(cfg->find("userScenarioCount")->as_string(), "7");
|
|
r.root_agent_id = "other";
|
|
ASSERT_TRUE(is_invalid([&] { EvaluationApi::scenario_body(r); }));
|
|
}
|
|
|
|
TEST(optimize_api, resources_validate_names_verbs_and_caps) {
|
|
OfflineClient off;
|
|
EvaluationResources res(off.client, ctx());
|
|
ASSERT_TRUE(is_invalid([&] { res.get(Collection::kEvaluationRuns, kSet); })); // wrong collection
|
|
ASSERT_TRUE(is_invalid([&] { res.get(Collection::kEvaluationRuns, "../../x"); }));
|
|
ListOptions bad;
|
|
bad.page_size = 0;
|
|
ASSERT_TRUE(is_invalid([&] { res.list(Collection::kEvaluationRuns, bad); }));
|
|
bad.page_size = 10;
|
|
bad.max_pages = 21;
|
|
ASSERT_TRUE(is_invalid([&] { res.list(Collection::kEvaluationRuns, bad); }));
|
|
bad.max_pages = 1;
|
|
bad.filter = std::string(1001, 'x');
|
|
ASSERT_TRUE(is_invalid([&] { res.list(Collection::kEvaluationRuns, bad); }));
|
|
// evaluationItems/evaluationMetrics/evaluationRuns have no patch in Discovery.
|
|
Json body = Json::object();
|
|
ASSERT_TRUE(is_invalid([&] { res.patch(Collection::kEvaluationRuns, kRun, body, "displayName"); }));
|
|
ASSERT_TRUE(is_invalid([&] { res.action(Collection::kEvaluationRuns, kRun, "activate"); }));
|
|
ASSERT_TRUE(is_invalid([&] { res.action(Collection::kOnlineEvaluators, kRun, "suspend"); }));
|
|
ASSERT_TRUE(is_invalid([&] { res.create(Collection::kEvaluationRuns, body, "id1"); }));
|
|
Json unknown = Json::object();
|
|
unknown.set("bogus", Json(1));
|
|
ASSERT_TRUE(is_invalid([&] { res.create(Collection::kEvaluationSets, unknown); }));
|
|
ASSERT_TRUE(is_invalid([&] { res.delete_resource(Collection::kEvaluationMetrics, kSet); }));
|
|
ASSERT_EQ(off.mock->send_count(), 0u);
|
|
(void)kMetric;
|
|
}
|
|
|
|
TEST(optimize_api, run_spec_body_matches_discovery) {
|
|
EvaluationRunSpec s;
|
|
s.display_name = "nightly";
|
|
s.evaluation_set = kSet;
|
|
s.predefined_metrics = {"final_response_quality_v1"};
|
|
s.agent_engine = "projects/test-proj/locations/us-central1/reasoningEngines/e1";
|
|
s.simulator_max_turns = 5;
|
|
s.simulator_model = "gemini-2.5-flash";
|
|
s.autorater_samples = 3;
|
|
Json b = s.to_json();
|
|
check_body(call_spec(rpc_id("evaluationRuns", "create")), b);
|
|
const Json* arc = b.find("inferenceConfigs")->find("candidate")->find("agentRunConfig");
|
|
ASSERT_EQ(arc->find("userSimulatorConfig")->find("maxTurn")->as_int(), 5);
|
|
ASSERT_EQ(b.find("dataSource")->find("evaluationSet")->as_string(), kSet);
|
|
|
|
EvaluationRunSpec bad = s;
|
|
bad.evaluation_set = "x";
|
|
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
|
|
bad = s;
|
|
bad.predefined_metrics.clear();
|
|
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
|
|
bad = s;
|
|
bad.simulator_max_turns = 13;
|
|
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
|
|
bad = s;
|
|
bad.predefined_metrics.assign(17, "m");
|
|
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
|
|
}
|
|
|
|
TEST(optimize_api, online_evaluator_and_item_specs) {
|
|
OnlineEvaluatorSpec o;
|
|
o.display_name = "live";
|
|
o.agent_resource = "projects/test-proj/locations/us-central1/reasoningEngines/e1";
|
|
o.metric_resource_names = {kMetric};
|
|
o.sampling_percentage = 20;
|
|
o.max_samples_per_run = 50;
|
|
Json b = o.to_json();
|
|
check_body(call_spec(rpc_id("onlineEvaluators", "create")), b);
|
|
// int64 is a string in Discovery's JSON mapping.
|
|
ASSERT_EQ(b.find("config")->find("maxEvaluatedSamplesPerRun")->as_string(), "50");
|
|
OnlineEvaluatorSpec bad = o;
|
|
bad.sampling_percentage = 0;
|
|
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
|
|
bad = o;
|
|
bad.max_samples_per_run = 0; // 0 means unbounded at Google: refused
|
|
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
|
|
bad = o;
|
|
bad.metric_resource_names = {"nope"};
|
|
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
|
|
|
|
EvaluationItemSpec item;
|
|
item.display_name = "case 1";
|
|
item.scenario_start = "hi";
|
|
item.scenario_plan = "ask for a refund";
|
|
Json ib = item.to_json();
|
|
check_body(call_spec(rpc_id("evaluationItems", "create")), ib);
|
|
ASSERT_EQ(ib.find("evaluationItemType")->as_string(), "REQUEST");
|
|
item.prompt = "also a prompt";
|
|
ASSERT_TRUE(is_invalid([&] { item.to_json(); }));
|
|
}
|