273 lines
11 KiB
C++
273 lines
11 KiB
C++
#include "google/optimize/eval_resources.h"
|
|||
|
|
|
||
|
|
#include <thread>
|
||
|
|
|
||
|
|
#include "google/errors.h"
|
||
|
|
#include "google/optimize/caps.h"
|
||
|
|
|
||
|
|
namespace molo::google::optimize {
|
||
|
|
|
||
|
|
using clients::body::Json;
|
||
|
|
|
||
|
|
namespace {
|
||
|
|
Json obj() { return Json::object(); }
|
||
|
|
void text_cap(const std::string& s, std::size_t cap, const char* what) {
|
||
|
|
if (s.size() > cap) throw GoogleError::invalid(std::string(what) + " over " + std::to_string(cap) + " characters");
|
||
|
|
}
|
||
|
|
void need(const std::string& s, const char* what) {
|
||
|
|
if (s.empty()) throw GoogleError::invalid(std::string(what) + " is required");
|
||
|
|
}
|
||
|
|
} // namespace
|
||
|
|
|
||
|
|
const char* collection_name(Collection c) {
|
||
|
|
switch (c) {
|
||
|
|
case Collection::kEvaluationSets: return "evaluationSets";
|
||
|
|
case Collection::kEvaluationItems: return "evaluationItems";
|
||
|
|
case Collection::kEvaluationMetrics: return "evaluationMetrics";
|
||
|
|
case Collection::kEvaluationRuns: return "evaluationRuns";
|
||
|
|
case Collection::kOnlineEvaluators: return "onlineEvaluators";
|
||
|
|
}
|
||
|
|
return "";
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EvaluationRunSpec::to_json() const {
|
||
|
|
need(display_name, "display_name");
|
||
|
|
text_cap(display_name, 128, "display_name");
|
||
|
|
if (!is_resource_of(evaluation_set, "evaluationSets"))
|
||
|
|
throw GoogleError::invalid("evaluation_set must be projects/p/locations/l/evaluationSets/id");
|
||
|
|
if (predefined_metrics.empty() || predefined_metrics.size() > OptimizeCaps::kMaxRunMetrics)
|
||
|
|
throw GoogleError::invalid("1.." + std::to_string(OptimizeCaps::kMaxRunMetrics) + " metrics required");
|
||
|
|
if (candidate.empty() || candidate.size() > 128) throw GoogleError::invalid("bad candidate name");
|
||
|
|
if (simulator_max_turns < 0 || simulator_max_turns > OptimizeCaps::kMaxSimTurns)
|
||
|
|
throw GoogleError::invalid("simulator_max_turns must be 0..12");
|
||
|
|
if (autorater_samples < 0 || autorater_samples > 32)
|
||
|
|
throw GoogleError::invalid("autorater_samples must be 0..32");
|
||
|
|
text_cap(agent_engine, 300, "agent_engine");
|
||
|
|
text_cap(model, 300, "model");
|
||
|
|
|
||
|
|
Json run = obj();
|
||
|
|
run.set("displayName", Json(display_name));
|
||
|
|
Json ds = obj();
|
||
|
|
ds.set("evaluationSet", Json(evaluation_set));
|
||
|
|
run.set("dataSource", ds);
|
||
|
|
|
||
|
|
Json metrics = Json::array();
|
||
|
|
for (const std::string& m : predefined_metrics) {
|
||
|
|
need(m, "metric name");
|
||
|
|
text_cap(m, 128, "metric name");
|
||
|
|
Json p = obj();
|
||
|
|
p.set("metricSpecName", Json(m));
|
||
|
|
Json one = obj();
|
||
|
|
one.set("predefinedMetricSpec", p);
|
||
|
|
metrics.push(one);
|
||
|
|
}
|
||
|
|
Json cfg = obj();
|
||
|
|
cfg.set("metrics", metrics);
|
||
|
|
if (!autorater_model.empty() || autorater_samples > 0) {
|
||
|
|
Json a = obj();
|
||
|
|
if (!autorater_model.empty()) a.set("autoraterModel", Json(autorater_model));
|
||
|
|
if (autorater_samples > 0) a.set("sampleCount", Json(autorater_samples));
|
||
|
|
cfg.set("autoraterConfig", a);
|
||
|
|
}
|
||
|
|
run.set("evaluationConfig", cfg);
|
||
|
|
|
||
|
|
Json inf = obj();
|
||
|
|
if (!model.empty()) inf.set("model", Json(model));
|
||
|
|
Json arc = obj();
|
||
|
|
if (!agent_engine.empty()) arc.set("agentEngine", Json(agent_engine));
|
||
|
|
if (simulator_max_turns > 0 || !simulator_model.empty()) {
|
||
|
|
Json u = obj();
|
||
|
|
if (simulator_max_turns > 0) u.set("maxTurn", Json(simulator_max_turns));
|
||
|
|
if (!simulator_model.empty()) u.set("modelName", Json(simulator_model));
|
||
|
|
arc.set("userSimulatorConfig", u);
|
||
|
|
}
|
||
|
|
if (!arc.as_object().empty()) inf.set("agentRunConfig", arc);
|
||
|
|
if (!inf.as_object().empty()) {
|
||
|
|
Json configs = obj();
|
||
|
|
configs.set(candidate, inf);
|
||
|
|
run.set("inferenceConfigs", configs);
|
||
|
|
}
|
||
|
|
return run;
|
||
|
|
}
|
||
|
|
|
||
|
|
Json OnlineEvaluatorSpec::to_json() const {
|
||
|
|
need(display_name, "display_name");
|
||
|
|
text_cap(display_name, 128, "display_name");
|
||
|
|
need(agent_resource, "agent_resource");
|
||
|
|
text_cap(agent_resource, 500, "agent_resource");
|
||
|
|
if (metric_resource_names.empty() || metric_resource_names.size() > OptimizeCaps::kMaxRunMetrics)
|
||
|
|
throw GoogleError::invalid("1.." + std::to_string(OptimizeCaps::kMaxRunMetrics) + " metrics required");
|
||
|
|
if (sampling_percentage < 1 || sampling_percentage > OptimizeCaps::kMaxSamplingPercent)
|
||
|
|
throw GoogleError::invalid("sampling_percentage must be 1..100");
|
||
|
|
if (max_samples_per_run < 1 || max_samples_per_run > 100000)
|
||
|
|
throw GoogleError::invalid("max_samples_per_run must be 1..100000 (0 would be unbounded)");
|
||
|
|
Json o = obj();
|
||
|
|
o.set("displayName", Json(display_name));
|
||
|
|
o.set("agentResource", Json(agent_resource));
|
||
|
|
Json sources = Json::array();
|
||
|
|
for (const std::string& m : metric_resource_names) {
|
||
|
|
if (!is_resource_of(m, "evaluationMetrics"))
|
||
|
|
throw GoogleError::invalid("metric must be projects/p/locations/l/evaluationMetrics/id");
|
||
|
|
Json s = obj();
|
||
|
|
s.set("metricResourceName", Json(m));
|
||
|
|
sources.push(s);
|
||
|
|
}
|
||
|
|
o.set("metricSources", sources);
|
||
|
|
Json cfg = obj();
|
||
|
|
Json rs = obj();
|
||
|
|
rs.set("percentage", Json(sampling_percentage));
|
||
|
|
cfg.set("randomSampling", rs);
|
||
|
|
cfg.set("maxEvaluatedSamplesPerRun", Json(std::to_string(max_samples_per_run))); // int64 as string
|
||
|
|
o.set("config", cfg);
|
||
|
|
Json co = obj();
|
||
|
|
Json ot = obj();
|
||
|
|
if (!semconv_version.empty()) ot.set("semconvVersion", Json(semconv_version));
|
||
|
|
co.set("openTelemetry", ot);
|
||
|
|
o.set("cloudObservability", co);
|
||
|
|
return o;
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EvaluationItemSpec::to_json() const {
|
||
|
|
need(display_name, "display_name");
|
||
|
|
text_cap(display_name, 128, "display_name");
|
||
|
|
text_cap(prompt, OptimizeCaps::kMaxTextChars, "prompt");
|
||
|
|
text_cap(scenario_start, OptimizeCaps::kMaxTextChars, "scenario_start");
|
||
|
|
text_cap(scenario_plan, OptimizeCaps::kMaxTextChars, "scenario_plan");
|
||
|
|
bool scenario = !scenario_start.empty() || !scenario_plan.empty();
|
||
|
|
if (prompt.empty() == !scenario)
|
||
|
|
throw GoogleError::invalid("give either a prompt or a scenario (start and plan), not both");
|
||
|
|
Json p = obj();
|
||
|
|
if (scenario) {
|
||
|
|
if (scenario_start.empty() || scenario_plan.empty())
|
||
|
|
throw GoogleError::invalid("a scenario needs both start and plan");
|
||
|
|
Json us = obj();
|
||
|
|
us.set("startingPrompt", Json(scenario_start));
|
||
|
|
us.set("conversationPlan", Json(scenario_plan));
|
||
|
|
p.set("userScenario", us);
|
||
|
|
} else {
|
||
|
|
p.set("text", Json(prompt));
|
||
|
|
}
|
||
|
|
Json req = obj();
|
||
|
|
req.set("prompt", p);
|
||
|
|
Json item = obj();
|
||
|
|
item.set("displayName", Json(display_name));
|
||
|
|
item.set("evaluationItemType", Json("REQUEST"));
|
||
|
|
item.set("evaluationRequest", req);
|
||
|
|
return item;
|
||
|
|
}
|
||
|
|
|
||
|
|
bool RunStatus::terminal() const {
|
||
|
|
return state == "SUCCEEDED" || state == "FAILED" || state == "CANCELLED";
|
||
|
|
}
|
||
|
|
|
||
|
|
EvaluationResources::EvaluationResources(std::shared_ptr<GoogleClient> client, CloudContext context)
|
||
|
|
: OptimizeService(std::move(client), endpoints::aiplatform(context.location)),
|
||
|
|
context_(std::move(context)) {}
|
||
|
|
|
||
|
|
const CallSpec& EvaluationResources::spec_for(Collection c, const std::string& method) const {
|
||
|
|
const CallSpec* s = find_call(rpc_id(collection_name(c), method));
|
||
|
|
if (!s) throw GoogleError::invalid(std::string(collection_name(c)) + " has no " + method + " call");
|
||
|
|
return *s;
|
||
|
|
}
|
||
|
|
|
||
|
|
void EvaluationResources::check_name(Collection c, const std::string& name) const {
|
||
|
|
if (!is_resource_of(name, collection_name(c)))
|
||
|
|
throw GoogleError::invalid(std::string("name must be projects/p/locations/l/") +
|
||
|
|
collection_name(c) + "/id");
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EvaluationResources::get(Collection c, const std::string& name) const {
|
||
|
|
check_name(c, name);
|
||
|
|
return invoke(spec_for(c, "get"), {{"name", name}});
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EvaluationResources::list(Collection c, const ListOptions& o) const {
|
||
|
|
if (o.page_size < 1 || o.page_size > 100) throw GoogleError::invalid("page_size must be 1..100");
|
||
|
|
if (o.max_pages < 1 || o.max_pages > OptimizeCaps::kMaxPages)
|
||
|
|
throw GoogleError::invalid("max_pages must be 1.." + std::to_string(OptimizeCaps::kMaxPages));
|
||
|
|
text_cap(o.filter, 1000, "filter");
|
||
|
|
text_cap(o.order_by, 100, "order_by");
|
||
|
|
const CallSpec& spec = spec_for(c, "list");
|
||
|
|
Json all = Json::array();
|
||
|
|
std::string token;
|
||
|
|
for (std::size_t page = 0; page < o.max_pages; ++page) {
|
||
|
|
Json reply = invoke(spec, {{"parent", context_.parent()}},
|
||
|
|
{{"pageSize", std::to_string(o.page_size)},
|
||
|
|
{"filter", o.filter},
|
||
|
|
{"orderBy", o.order_by},
|
||
|
|
{"pageToken", token}});
|
||
|
|
if (const Json* arr = reply.find(collection_name(c)); arr && arr->is_array())
|
||
|
|
for (const Json& item : arr->as_array()) {
|
||
|
|
if (all.as_array().size() >= OptimizeCaps::kMaxListItems) return all;
|
||
|
|
all.push(item);
|
||
|
|
}
|
||
|
|
const Json* next = reply.find("nextPageToken");
|
||
|
|
if (!next || !next->is_string() || next->as_string().empty()) break;
|
||
|
|
token = next->as_string();
|
||
|
|
}
|
||
|
|
return all;
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EvaluationResources::create(Collection c, const Json& body, const std::string& metric_id) const {
|
||
|
|
std::map<std::string, std::string> q;
|
||
|
|
if (!metric_id.empty()) {
|
||
|
|
if (c != Collection::kEvaluationMetrics || !valid_segment(metric_id))
|
||
|
|
throw GoogleError::invalid("metric_id only applies to evaluationMetrics and must be a valid id");
|
||
|
|
q["evaluationMetricId"] = metric_id;
|
||
|
|
}
|
||
|
|
return invoke(spec_for(c, "create"), {{"parent", context_.parent()}}, q, &body);
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EvaluationResources::patch(Collection c, const std::string& name, const Json& body,
|
||
|
|
const std::string& update_mask) const {
|
||
|
|
check_name(c, name);
|
||
|
|
need(update_mask, "update_mask");
|
||
|
|
text_cap(update_mask, 500, "update_mask");
|
||
|
|
return invoke(spec_for(c, "patch"), {{"name", name}}, {{"updateMask", update_mask}}, &body);
|
||
|
|
}
|
||
|
|
|
||
|
|
Operation EvaluationResources::action(Collection c, const std::string& name,
|
||
|
|
const std::string& verb) const {
|
||
|
|
check_name(c, name);
|
||
|
|
bool ok = (c == Collection::kEvaluationRuns && verb == "cancel") ||
|
||
|
|
(c == Collection::kOnlineEvaluators && (verb == "activate" || verb == "suspend"));
|
||
|
|
if (!ok) throw GoogleError::invalid(std::string(collection_name(c)) + " has no action " + verb);
|
||
|
|
Operation op = Operation::from_json(invoke(spec_for(c, verb), {{"name", name}}));
|
||
|
|
if (verb == "cancel") op.done = true; // cancel answers google.protobuf.Empty
|
||
|
|
return op;
|
||
|
|
}
|
||
|
|
|
||
|
|
Operation EvaluationResources::import_set(const Json& body) const {
|
||
|
|
return Operation::from_json(invoke(spec_for(Collection::kEvaluationSets, "import"),
|
||
|
|
{{"parent", context_.parent()}}, {}, &body));
|
||
|
|
}
|
||
|
|
|
||
|
|
Operation EvaluationResources::delete_resource(Collection c, const std::string& name, bool wait,
|
||
|
|
const Sleeper& sleeper) const {
|
||
|
|
check_name(c, name);
|
||
|
|
Operation op = Operation::from_json(invoke(spec_for(c, "delete"), {{"name", name}}));
|
||
|
|
if (wait && !op.done) return wait_operation(*client_, endpoint_, op.name, sleeper);
|
||
|
|
return op;
|
||
|
|
}
|
||
|
|
|
||
|
|
RunStatus EvaluationResources::run_status(const std::string& run_name) const {
|
||
|
|
RunStatus s;
|
||
|
|
s.run = get(Collection::kEvaluationRuns, run_name);
|
||
|
|
s.name = run_name;
|
||
|
|
s.state = string_at(s.run, "state");
|
||
|
|
return s;
|
||
|
|
}
|
||
|
|
|
||
|
|
RunStatus EvaluationResources::wait_run(const std::string& run_name, const Sleeper& sleeper) const {
|
||
|
|
Sleeper sleep = sleeper ? sleeper : [](std::chrono::milliseconds d) { std::this_thread::sleep_for(d); };
|
||
|
|
for (std::size_t i = 0; i < OptimizeCaps::kMaxRunPolls; ++i) {
|
||
|
|
RunStatus s = run_status(run_name);
|
||
|
|
if (s.terminal()) return s;
|
||
|
|
sleep(OperationCaps::kInterval);
|
||
|
|
}
|
||
|
|
throw GoogleError(504, "DEADLINE_EXCEEDED", "evaluation run still running at the wait cap",
|
||
|
|
endpoint_.name);
|
||
|
|
}
|
||
|
|
|
||
|
|
} // namespace molo::google::optimize
|