Files
moloapp/src/google/optimize/eval_resources.cpp
T

273 lines
11 KiB
C++
Raw Normal View History

#include "google/optimize/eval_resources.h"
#include <thread>
#include "google/errors.h"
#include "google/optimize/caps.h"
namespace molo::google::optimize {
using clients::body::Json;
namespace {
Json obj() { return Json::object(); }
void text_cap(const std::string& s, std::size_t cap, const char* what) {
if (s.size() > cap) throw GoogleError::invalid(std::string(what) + " over " + std::to_string(cap) + " characters");
}
void need(const std::string& s, const char* what) {
if (s.empty()) throw GoogleError::invalid(std::string(what) + " is required");
}
} // namespace
const char* collection_name(Collection c) {
switch (c) {
case Collection::kEvaluationSets: return "evaluationSets";
case Collection::kEvaluationItems: return "evaluationItems";
case Collection::kEvaluationMetrics: return "evaluationMetrics";
case Collection::kEvaluationRuns: return "evaluationRuns";
case Collection::kOnlineEvaluators: return "onlineEvaluators";
}
return "";
}
Json EvaluationRunSpec::to_json() const {
need(display_name, "display_name");
text_cap(display_name, 128, "display_name");
if (!is_resource_of(evaluation_set, "evaluationSets"))
throw GoogleError::invalid("evaluation_set must be projects/p/locations/l/evaluationSets/id");
if (predefined_metrics.empty() || predefined_metrics.size() > OptimizeCaps::kMaxRunMetrics)
throw GoogleError::invalid("1.." + std::to_string(OptimizeCaps::kMaxRunMetrics) + " metrics required");
if (candidate.empty() || candidate.size() > 128) throw GoogleError::invalid("bad candidate name");
if (simulator_max_turns < 0 || simulator_max_turns > OptimizeCaps::kMaxSimTurns)
throw GoogleError::invalid("simulator_max_turns must be 0..12");
if (autorater_samples < 0 || autorater_samples > 32)
throw GoogleError::invalid("autorater_samples must be 0..32");
text_cap(agent_engine, 300, "agent_engine");
text_cap(model, 300, "model");
Json run = obj();
run.set("displayName", Json(display_name));
Json ds = obj();
ds.set("evaluationSet", Json(evaluation_set));
run.set("dataSource", ds);
Json metrics = Json::array();
for (const std::string& m : predefined_metrics) {
need(m, "metric name");
text_cap(m, 128, "metric name");
Json p = obj();
p.set("metricSpecName", Json(m));
Json one = obj();
one.set("predefinedMetricSpec", p);
metrics.push(one);
}
Json cfg = obj();
cfg.set("metrics", metrics);
if (!autorater_model.empty() || autorater_samples > 0) {
Json a = obj();
if (!autorater_model.empty()) a.set("autoraterModel", Json(autorater_model));
if (autorater_samples > 0) a.set("sampleCount", Json(autorater_samples));
cfg.set("autoraterConfig", a);
}
run.set("evaluationConfig", cfg);
Json inf = obj();
if (!model.empty()) inf.set("model", Json(model));
Json arc = obj();
if (!agent_engine.empty()) arc.set("agentEngine", Json(agent_engine));
if (simulator_max_turns > 0 || !simulator_model.empty()) {
Json u = obj();
if (simulator_max_turns > 0) u.set("maxTurn", Json(simulator_max_turns));
if (!simulator_model.empty()) u.set("modelName", Json(simulator_model));
arc.set("userSimulatorConfig", u);
}
if (!arc.as_object().empty()) inf.set("agentRunConfig", arc);
if (!inf.as_object().empty()) {
Json configs = obj();
configs.set(candidate, inf);
run.set("inferenceConfigs", configs);
}
return run;
}
Json OnlineEvaluatorSpec::to_json() const {
need(display_name, "display_name");
text_cap(display_name, 128, "display_name");
need(agent_resource, "agent_resource");
text_cap(agent_resource, 500, "agent_resource");
if (metric_resource_names.empty() || metric_resource_names.size() > OptimizeCaps::kMaxRunMetrics)
throw GoogleError::invalid("1.." + std::to_string(OptimizeCaps::kMaxRunMetrics) + " metrics required");
if (sampling_percentage < 1 || sampling_percentage > OptimizeCaps::kMaxSamplingPercent)
throw GoogleError::invalid("sampling_percentage must be 1..100");
if (max_samples_per_run < 1 || max_samples_per_run > 100000)
throw GoogleError::invalid("max_samples_per_run must be 1..100000 (0 would be unbounded)");
Json o = obj();
o.set("displayName", Json(display_name));
o.set("agentResource", Json(agent_resource));
Json sources = Json::array();
for (const std::string& m : metric_resource_names) {
if (!is_resource_of(m, "evaluationMetrics"))
throw GoogleError::invalid("metric must be projects/p/locations/l/evaluationMetrics/id");
Json s = obj();
s.set("metricResourceName", Json(m));
sources.push(s);
}
o.set("metricSources", sources);
Json cfg = obj();
Json rs = obj();
rs.set("percentage", Json(sampling_percentage));
cfg.set("randomSampling", rs);
cfg.set("maxEvaluatedSamplesPerRun", Json(std::to_string(max_samples_per_run))); // int64 as string
o.set("config", cfg);
Json co = obj();
Json ot = obj();
if (!semconv_version.empty()) ot.set("semconvVersion", Json(semconv_version));
co.set("openTelemetry", ot);
o.set("cloudObservability", co);
return o;
}
Json EvaluationItemSpec::to_json() const {
need(display_name, "display_name");
text_cap(display_name, 128, "display_name");
text_cap(prompt, OptimizeCaps::kMaxTextChars, "prompt");
text_cap(scenario_start, OptimizeCaps::kMaxTextChars, "scenario_start");
text_cap(scenario_plan, OptimizeCaps::kMaxTextChars, "scenario_plan");
bool scenario = !scenario_start.empty() || !scenario_plan.empty();
if (prompt.empty() == !scenario)
throw GoogleError::invalid("give either a prompt or a scenario (start and plan), not both");
Json p = obj();
if (scenario) {
if (scenario_start.empty() || scenario_plan.empty())
throw GoogleError::invalid("a scenario needs both start and plan");
Json us = obj();
us.set("startingPrompt", Json(scenario_start));
us.set("conversationPlan", Json(scenario_plan));
p.set("userScenario", us);
} else {
p.set("text", Json(prompt));
}
Json req = obj();
req.set("prompt", p);
Json item = obj();
item.set("displayName", Json(display_name));
item.set("evaluationItemType", Json("REQUEST"));
item.set("evaluationRequest", req);
return item;
}
bool RunStatus::terminal() const {
return state == "SUCCEEDED" || state == "FAILED" || state == "CANCELLED";
}
EvaluationResources::EvaluationResources(std::shared_ptr<GoogleClient> client, CloudContext context)
: OptimizeService(std::move(client), endpoints::aiplatform(context.location)),
context_(std::move(context)) {}
const CallSpec& EvaluationResources::spec_for(Collection c, const std::string& method) const {
const CallSpec* s = find_call(rpc_id(collection_name(c), method));
if (!s) throw GoogleError::invalid(std::string(collection_name(c)) + " has no " + method + " call");
return *s;
}
void EvaluationResources::check_name(Collection c, const std::string& name) const {
if (!is_resource_of(name, collection_name(c)))
throw GoogleError::invalid(std::string("name must be projects/p/locations/l/") +
collection_name(c) + "/id");
}
Json EvaluationResources::get(Collection c, const std::string& name) const {
check_name(c, name);
return invoke(spec_for(c, "get"), {{"name", name}});
}
Json EvaluationResources::list(Collection c, const ListOptions& o) const {
if (o.page_size < 1 || o.page_size > 100) throw GoogleError::invalid("page_size must be 1..100");
if (o.max_pages < 1 || o.max_pages > OptimizeCaps::kMaxPages)
throw GoogleError::invalid("max_pages must be 1.." + std::to_string(OptimizeCaps::kMaxPages));
text_cap(o.filter, 1000, "filter");
text_cap(o.order_by, 100, "order_by");
const CallSpec& spec = spec_for(c, "list");
Json all = Json::array();
std::string token;
for (std::size_t page = 0; page < o.max_pages; ++page) {
Json reply = invoke(spec, {{"parent", context_.parent()}},
{{"pageSize", std::to_string(o.page_size)},
{"filter", o.filter},
{"orderBy", o.order_by},
{"pageToken", token}});
if (const Json* arr = reply.find(collection_name(c)); arr && arr->is_array())
for (const Json& item : arr->as_array()) {
if (all.as_array().size() >= OptimizeCaps::kMaxListItems) return all;
all.push(item);
}
const Json* next = reply.find("nextPageToken");
if (!next || !next->is_string() || next->as_string().empty()) break;
token = next->as_string();
}
return all;
}
Json EvaluationResources::create(Collection c, const Json& body, const std::string& metric_id) const {
std::map<std::string, std::string> q;
if (!metric_id.empty()) {
if (c != Collection::kEvaluationMetrics || !valid_segment(metric_id))
throw GoogleError::invalid("metric_id only applies to evaluationMetrics and must be a valid id");
q["evaluationMetricId"] = metric_id;
}
return invoke(spec_for(c, "create"), {{"parent", context_.parent()}}, q, &body);
}
Json EvaluationResources::patch(Collection c, const std::string& name, const Json& body,
const std::string& update_mask) const {
check_name(c, name);
need(update_mask, "update_mask");
text_cap(update_mask, 500, "update_mask");
return invoke(spec_for(c, "patch"), {{"name", name}}, {{"updateMask", update_mask}}, &body);
}
Operation EvaluationResources::action(Collection c, const std::string& name,
const std::string& verb) const {
check_name(c, name);
bool ok = (c == Collection::kEvaluationRuns && verb == "cancel") ||
(c == Collection::kOnlineEvaluators && (verb == "activate" || verb == "suspend"));
if (!ok) throw GoogleError::invalid(std::string(collection_name(c)) + " has no action " + verb);
Operation op = Operation::from_json(invoke(spec_for(c, verb), {{"name", name}}));
if (verb == "cancel") op.done = true; // cancel answers google.protobuf.Empty
return op;
}
Operation EvaluationResources::import_set(const Json& body) const {
return Operation::from_json(invoke(spec_for(Collection::kEvaluationSets, "import"),
{{"parent", context_.parent()}}, {}, &body));
}
Operation EvaluationResources::delete_resource(Collection c, const std::string& name, bool wait,
const Sleeper& sleeper) const {
check_name(c, name);
Operation op = Operation::from_json(invoke(spec_for(c, "delete"), {{"name", name}}));
if (wait && !op.done) return wait_operation(*client_, endpoint_, op.name, sleeper);
return op;
}
RunStatus EvaluationResources::run_status(const std::string& run_name) const {
RunStatus s;
s.run = get(Collection::kEvaluationRuns, run_name);
s.name = run_name;
s.state = string_at(s.run, "state");
return s;
}
RunStatus EvaluationResources::wait_run(const std::string& run_name, const Sleeper& sleeper) const {
Sleeper sleep = sleeper ? sleeper : [](std::chrono::milliseconds d) { std::this_thread::sleep_for(d); };
for (std::size_t i = 0; i < OptimizeCaps::kMaxRunPolls; ++i) {
RunStatus s = run_status(run_name);
if (s.terminal()) return s;
sleep(OperationCaps::kInterval);
}
throw GoogleError(504, "DEADLINE_EXCEEDED", "evaluation run still running at the wait cap",
endpoint_.name);
}
} // namespace molo::google::optimize