#include "google/optimize/eval_resources.h" #include #include "google/errors.h" #include "google/optimize/caps.h" namespace molo::google::optimize { using clients::body::Json; namespace { Json obj() { return Json::object(); } void text_cap(const std::string& s, std::size_t cap, const char* what) { if (s.size() > cap) throw GoogleError::invalid(std::string(what) + " over " + std::to_string(cap) + " characters"); } void need(const std::string& s, const char* what) { if (s.empty()) throw GoogleError::invalid(std::string(what) + " is required"); } } // namespace const char* collection_name(Collection c) { switch (c) { case Collection::kEvaluationSets: return "evaluationSets"; case Collection::kEvaluationItems: return "evaluationItems"; case Collection::kEvaluationMetrics: return "evaluationMetrics"; case Collection::kEvaluationRuns: return "evaluationRuns"; case Collection::kOnlineEvaluators: return "onlineEvaluators"; } return ""; } Json EvaluationRunSpec::to_json() const { need(display_name, "display_name"); text_cap(display_name, 128, "display_name"); if (!is_resource_of(evaluation_set, "evaluationSets")) throw GoogleError::invalid("evaluation_set must be projects/p/locations/l/evaluationSets/id"); if (predefined_metrics.empty() || predefined_metrics.size() > OptimizeCaps::kMaxRunMetrics) throw GoogleError::invalid("1.." + std::to_string(OptimizeCaps::kMaxRunMetrics) + " metrics required"); if (candidate.empty() || candidate.size() > 128) throw GoogleError::invalid("bad candidate name"); if (simulator_max_turns < 0 || simulator_max_turns > OptimizeCaps::kMaxSimTurns) throw GoogleError::invalid("simulator_max_turns must be 0..12"); if (autorater_samples < 0 || autorater_samples > 32) throw GoogleError::invalid("autorater_samples must be 0..32"); text_cap(agent_engine, 300, "agent_engine"); text_cap(model, 300, "model"); Json run = obj(); run.set("displayName", Json(display_name)); Json ds = obj(); ds.set("evaluationSet", Json(evaluation_set)); run.set("dataSource", ds); Json metrics = Json::array(); for (const std::string& m : predefined_metrics) { need(m, "metric name"); text_cap(m, 128, "metric name"); Json p = obj(); p.set("metricSpecName", Json(m)); Json one = obj(); one.set("predefinedMetricSpec", p); metrics.push(one); } Json cfg = obj(); cfg.set("metrics", metrics); if (!autorater_model.empty() || autorater_samples > 0) { Json a = obj(); if (!autorater_model.empty()) a.set("autoraterModel", Json(autorater_model)); if (autorater_samples > 0) a.set("sampleCount", Json(autorater_samples)); cfg.set("autoraterConfig", a); } run.set("evaluationConfig", cfg); Json inf = obj(); if (!model.empty()) inf.set("model", Json(model)); Json arc = obj(); if (!agent_engine.empty()) arc.set("agentEngine", Json(agent_engine)); if (simulator_max_turns > 0 || !simulator_model.empty()) { Json u = obj(); if (simulator_max_turns > 0) u.set("maxTurn", Json(simulator_max_turns)); if (!simulator_model.empty()) u.set("modelName", Json(simulator_model)); arc.set("userSimulatorConfig", u); } if (!arc.as_object().empty()) inf.set("agentRunConfig", arc); if (!inf.as_object().empty()) { Json configs = obj(); configs.set(candidate, inf); run.set("inferenceConfigs", configs); } return run; } Json OnlineEvaluatorSpec::to_json() const { need(display_name, "display_name"); text_cap(display_name, 128, "display_name"); need(agent_resource, "agent_resource"); text_cap(agent_resource, 500, "agent_resource"); if (metric_resource_names.empty() || metric_resource_names.size() > OptimizeCaps::kMaxRunMetrics) throw GoogleError::invalid("1.." + std::to_string(OptimizeCaps::kMaxRunMetrics) + " metrics required"); if (sampling_percentage < 1 || sampling_percentage > OptimizeCaps::kMaxSamplingPercent) throw GoogleError::invalid("sampling_percentage must be 1..100"); if (max_samples_per_run < 1 || max_samples_per_run > 100000) throw GoogleError::invalid("max_samples_per_run must be 1..100000 (0 would be unbounded)"); Json o = obj(); o.set("displayName", Json(display_name)); o.set("agentResource", Json(agent_resource)); Json sources = Json::array(); for (const std::string& m : metric_resource_names) { if (!is_resource_of(m, "evaluationMetrics")) throw GoogleError::invalid("metric must be projects/p/locations/l/evaluationMetrics/id"); Json s = obj(); s.set("metricResourceName", Json(m)); sources.push(s); } o.set("metricSources", sources); Json cfg = obj(); Json rs = obj(); rs.set("percentage", Json(sampling_percentage)); cfg.set("randomSampling", rs); cfg.set("maxEvaluatedSamplesPerRun", Json(std::to_string(max_samples_per_run))); // int64 as string o.set("config", cfg); Json co = obj(); Json ot = obj(); if (!semconv_version.empty()) ot.set("semconvVersion", Json(semconv_version)); co.set("openTelemetry", ot); o.set("cloudObservability", co); return o; } Json EvaluationItemSpec::to_json() const { need(display_name, "display_name"); text_cap(display_name, 128, "display_name"); text_cap(prompt, OptimizeCaps::kMaxTextChars, "prompt"); text_cap(scenario_start, OptimizeCaps::kMaxTextChars, "scenario_start"); text_cap(scenario_plan, OptimizeCaps::kMaxTextChars, "scenario_plan"); bool scenario = !scenario_start.empty() || !scenario_plan.empty(); if (prompt.empty() == !scenario) throw GoogleError::invalid("give either a prompt or a scenario (start and plan), not both"); Json p = obj(); if (scenario) { if (scenario_start.empty() || scenario_plan.empty()) throw GoogleError::invalid("a scenario needs both start and plan"); Json us = obj(); us.set("startingPrompt", Json(scenario_start)); us.set("conversationPlan", Json(scenario_plan)); p.set("userScenario", us); } else { p.set("text", Json(prompt)); } Json req = obj(); req.set("prompt", p); Json item = obj(); item.set("displayName", Json(display_name)); item.set("evaluationItemType", Json("REQUEST")); item.set("evaluationRequest", req); return item; } bool RunStatus::terminal() const { return state == "SUCCEEDED" || state == "FAILED" || state == "CANCELLED"; } EvaluationResources::EvaluationResources(std::shared_ptr client, CloudContext context) : OptimizeService(std::move(client), endpoints::aiplatform(context.location)), context_(std::move(context)) {} const CallSpec& EvaluationResources::spec_for(Collection c, const std::string& method) const { const CallSpec* s = find_call(rpc_id(collection_name(c), method)); if (!s) throw GoogleError::invalid(std::string(collection_name(c)) + " has no " + method + " call"); return *s; } void EvaluationResources::check_name(Collection c, const std::string& name) const { if (!is_resource_of(name, collection_name(c))) throw GoogleError::invalid(std::string("name must be projects/p/locations/l/") + collection_name(c) + "/id"); } Json EvaluationResources::get(Collection c, const std::string& name) const { check_name(c, name); return invoke(spec_for(c, "get"), {{"name", name}}); } Json EvaluationResources::list(Collection c, const ListOptions& o) const { if (o.page_size < 1 || o.page_size > 100) throw GoogleError::invalid("page_size must be 1..100"); if (o.max_pages < 1 || o.max_pages > OptimizeCaps::kMaxPages) throw GoogleError::invalid("max_pages must be 1.." + std::to_string(OptimizeCaps::kMaxPages)); text_cap(o.filter, 1000, "filter"); text_cap(o.order_by, 100, "order_by"); const CallSpec& spec = spec_for(c, "list"); Json all = Json::array(); std::string token; for (std::size_t page = 0; page < o.max_pages; ++page) { Json reply = invoke(spec, {{"parent", context_.parent()}}, {{"pageSize", std::to_string(o.page_size)}, {"filter", o.filter}, {"orderBy", o.order_by}, {"pageToken", token}}); if (const Json* arr = reply.find(collection_name(c)); arr && arr->is_array()) for (const Json& item : arr->as_array()) { if (all.as_array().size() >= OptimizeCaps::kMaxListItems) return all; all.push(item); } const Json* next = reply.find("nextPageToken"); if (!next || !next->is_string() || next->as_string().empty()) break; token = next->as_string(); } return all; } Json EvaluationResources::create(Collection c, const Json& body, const std::string& metric_id) const { std::map q; if (!metric_id.empty()) { if (c != Collection::kEvaluationMetrics || !valid_segment(metric_id)) throw GoogleError::invalid("metric_id only applies to evaluationMetrics and must be a valid id"); q["evaluationMetricId"] = metric_id; } return invoke(spec_for(c, "create"), {{"parent", context_.parent()}}, q, &body); } Json EvaluationResources::patch(Collection c, const std::string& name, const Json& body, const std::string& update_mask) const { check_name(c, name); need(update_mask, "update_mask"); text_cap(update_mask, 500, "update_mask"); return invoke(spec_for(c, "patch"), {{"name", name}}, {{"updateMask", update_mask}}, &body); } Operation EvaluationResources::action(Collection c, const std::string& name, const std::string& verb) const { check_name(c, name); bool ok = (c == Collection::kEvaluationRuns && verb == "cancel") || (c == Collection::kOnlineEvaluators && (verb == "activate" || verb == "suspend")); if (!ok) throw GoogleError::invalid(std::string(collection_name(c)) + " has no action " + verb); Operation op = Operation::from_json(invoke(spec_for(c, verb), {{"name", name}})); if (verb == "cancel") op.done = true; // cancel answers google.protobuf.Empty return op; } Operation EvaluationResources::import_set(const Json& body) const { return Operation::from_json(invoke(spec_for(Collection::kEvaluationSets, "import"), {{"parent", context_.parent()}}, {}, &body)); } Operation EvaluationResources::delete_resource(Collection c, const std::string& name, bool wait, const Sleeper& sleeper) const { check_name(c, name); Operation op = Operation::from_json(invoke(spec_for(c, "delete"), {{"name", name}})); if (wait && !op.done) return wait_operation(*client_, endpoint_, op.name, sleeper); return op; } RunStatus EvaluationResources::run_status(const std::string& run_name) const { RunStatus s; s.run = get(Collection::kEvaluationRuns, run_name); s.name = run_name; s.state = string_at(s.run, "state"); return s; } RunStatus EvaluationResources::wait_run(const std::string& run_name, const Sleeper& sleeper) const { Sleeper sleep = sleeper ? sleeper : [](std::chrono::milliseconds d) { std::this_thread::sleep_for(d); }; for (std::size_t i = 0; i < OptimizeCaps::kMaxRunPolls; ++i) { RunStatus s = run_status(run_name); if (s.terminal()) return s; sleep(OperationCaps::kInterval); } throw GoogleError(504, "DEADLINE_EXCEEDED", "evaluation run still running at the wait cap", endpoint_.name); } } // namespace molo::google::optimize