332 lines
15 KiB
C++
332 lines
15 KiB
C++
#include "google/models/endpoint_admin.h"
|
|||
|
|
|
||
|
|
#include <cstring>
|
||
|
|
|
||
|
|
#include "clients/url.h"
|
||
|
|
#include "google/models/admin_checks.h"
|
||
|
|
#include "google/models/open_models.h"
|
||
|
|
|
||
|
|
namespace molo::google::models {
|
||
|
|
|
||
|
|
using clients::body::Json;
|
||
|
|
|
||
|
|
namespace {
|
||
|
|
|
||
|
|
bool base64_chars(const std::string& s) {
|
||
|
|
for (char c : s)
|
||
|
|
if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') ||
|
||
|
|
c == '+' || c == '/' || c == '=' || c == '-' || c == '_'))
|
||
|
|
return false;
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
Json with(const char* key, const Json& value, Json body = Json::object()) {
|
||
|
|
body.set(key, value);
|
||
|
|
return body;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string q_int(const char* key, int value) {
|
||
|
|
return std::string(key) + "=" + std::to_string(value);
|
||
|
|
}
|
||
|
|
|
||
|
|
} // namespace
|
||
|
|
|
||
|
|
EndpointAdmin::EndpointAdmin(std::shared_ptr<GoogleClient> client, VertexTarget target,
|
||
|
|
bool beta)
|
||
|
|
: VertexService(std::move(client), std::move(target), "aiplatform"), beta_(beta) {}
|
||
|
|
|
||
|
|
PlannedCall EndpointAdmin::plan_call(const std::string& call_id,
|
||
|
|
const std::string& endpoint_id, const Json* body,
|
||
|
|
const std::string& query,
|
||
|
|
const CallPlanner::Args& extra) const {
|
||
|
|
CallPlanner::Args args = extra;
|
||
|
|
if (!endpoint_id.empty()) args["endpoint"] = endpoint_id;
|
||
|
|
const bool twin = beta_ && std::strcmp(CallPlanner::spec(call_id).version, "v1") == 0;
|
||
|
|
return planner_.plan(call_id, args, body, query, twin);
|
||
|
|
}
|
||
|
|
|
||
|
|
Operation EndpointAdmin::run_operation(const PlannedCall& call) const {
|
||
|
|
return Operation::from_json(run(call));
|
||
|
|
}
|
||
|
|
|
||
|
|
// --- lifecycle ---------------------------------------------------------------
|
||
|
|
|
||
|
|
Operation EndpointAdmin::create(const Json& endpoint, const std::string& endpoint_id) const {
|
||
|
|
AdminChecks::object(endpoint, "endpoint");
|
||
|
|
std::string q;
|
||
|
|
if (!endpoint_id.empty()) {
|
||
|
|
VertexTarget::require_segment(endpoint_id, "endpointId");
|
||
|
|
q = "endpointId=" + clients::Url::encode_component(endpoint_id);
|
||
|
|
}
|
||
|
|
return run_operation(plan_call("endpoint.create", "", &endpoint, q));
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EndpointAdmin::patch(const std::string& endpoint_id, const Json& endpoint,
|
||
|
|
const std::string& update_mask) const {
|
||
|
|
AdminChecks::object(endpoint, "endpoint");
|
||
|
|
AdminChecks::mask(update_mask);
|
||
|
|
return run(plan_call("endpoint.patch", endpoint_id, &endpoint,
|
||
|
|
"updateMask=" + clients::Url::encode_component(update_mask)));
|
||
|
|
}
|
||
|
|
|
||
|
|
Operation EndpointAdmin::update(const std::string& endpoint_id, const Json& endpoint) const {
|
||
|
|
AdminChecks::object(endpoint, "endpoint");
|
||
|
|
Json body = with("endpoint", endpoint);
|
||
|
|
return run_operation(plan_call("endpoint.update", endpoint_id, &body));
|
||
|
|
}
|
||
|
|
|
||
|
|
Operation EndpointAdmin::destroy(const std::string& endpoint_id, bool confirm) const {
|
||
|
|
AdminChecks::confirm(confirm, "deleting an endpoint");
|
||
|
|
return run_operation(plan_call("endpoint.delete", endpoint_id));
|
||
|
|
}
|
||
|
|
|
||
|
|
Operation EndpointAdmin::deploy_model(const std::string& endpoint_id, const Json& deployed_model,
|
||
|
|
const Json& traffic_split, bool confirm) const {
|
||
|
|
AdminChecks::confirm(confirm, "deploying a model (billable)");
|
||
|
|
AdminChecks::object(deployed_model, "deployedModel");
|
||
|
|
Json body = with("deployedModel", deployed_model);
|
||
|
|
if (traffic_split.is_object()) body.set("trafficSplit", AdminChecks::object(traffic_split, "trafficSplit"));
|
||
|
|
return run_operation(plan_call("endpoint.deploy_model", endpoint_id, &body));
|
||
|
|
}
|
||
|
|
|
||
|
|
Operation EndpointAdmin::undeploy_model(const std::string& endpoint_id,
|
||
|
|
const std::string& deployed_model_id,
|
||
|
|
const Json& traffic_split, bool confirm) const {
|
||
|
|
AdminChecks::confirm(confirm, "undeploying a model");
|
||
|
|
VertexTarget::require_segment(deployed_model_id, "deployedModelId");
|
||
|
|
Json body = with("deployedModelId", Json(deployed_model_id));
|
||
|
|
if (traffic_split.is_object()) body.set("trafficSplit", AdminChecks::object(traffic_split, "trafficSplit"));
|
||
|
|
return run_operation(plan_call("endpoint.undeploy_model", endpoint_id, &body));
|
||
|
|
}
|
||
|
|
|
||
|
|
Operation EndpointAdmin::mutate_deployed_model(const std::string& endpoint_id,
|
||
|
|
const Json& deployed_model,
|
||
|
|
const std::string& update_mask,
|
||
|
|
bool confirm) const {
|
||
|
|
AdminChecks::confirm(confirm, "mutating a deployed model");
|
||
|
|
AdminChecks::object(deployed_model, "deployedModel");
|
||
|
|
AdminChecks::mask(update_mask);
|
||
|
|
Json body = with("deployedModel", deployed_model);
|
||
|
|
body.set("updateMask", Json(update_mask));
|
||
|
|
return run_operation(plan_call("endpoint.mutate_deployed_model", endpoint_id, &body));
|
||
|
|
}
|
||
|
|
|
||
|
|
// --- inference ---------------------------------------------------------------
|
||
|
|
|
||
|
|
Json EndpointAdmin::compute_tokens(const std::string& endpoint_id, const Json& contents,
|
||
|
|
const std::string& model) const {
|
||
|
|
if (!contents.is_array() || contents.as_array().empty() ||
|
||
|
|
contents.as_array().size() > ModelsCaps::kMaxContents)
|
||
|
|
throw GoogleError::invalid("contents must be an array of 1.." +
|
||
|
|
std::to_string(ModelsCaps::kMaxContents) + " items");
|
||
|
|
AdminChecks::size(contents, "contents");
|
||
|
|
Json body = with("contents", contents);
|
||
|
|
if (!model.empty()) {
|
||
|
|
VertexTarget::require_segment(model, "model");
|
||
|
|
body.set("model", Json(model));
|
||
|
|
}
|
||
|
|
return run(plan_call("endpoint.compute_tokens", endpoint_id, &body));
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EndpointAdmin::direct_predict(const std::string& endpoint_id, const Json& inputs,
|
||
|
|
const Json& parameters) const {
|
||
|
|
if (!inputs.is_array() || inputs.as_array().empty() ||
|
||
|
|
inputs.as_array().size() > ModelsCaps::kMaxInstances)
|
||
|
|
throw GoogleError::invalid("inputs must be an array of 1.." +
|
||
|
|
std::to_string(ModelsCaps::kMaxInstances) + " items");
|
||
|
|
AdminChecks::size(inputs, "inputs");
|
||
|
|
Json body = with("inputs", inputs);
|
||
|
|
if (parameters.is_object()) body.set("parameters", parameters);
|
||
|
|
return run(plan_call("endpoint.direct_predict", endpoint_id, &body));
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EndpointAdmin::direct_raw_predict(const std::string& endpoint_id,
|
||
|
|
const std::string& method_name,
|
||
|
|
const std::string& input_base64) const {
|
||
|
|
if (method_name.empty() || method_name.size() > 256)
|
||
|
|
throw GoogleError::invalid("methodName must be 1..256 characters");
|
||
|
|
for (char c : method_name)
|
||
|
|
if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') ||
|
||
|
|
c == '/' || c == '.' || c == '_' || c == '-'))
|
||
|
|
throw GoogleError::invalid("methodName has invalid characters");
|
||
|
|
if (input_base64.size() > ModelsCaps::kMaxAdminBodyChars || !base64_chars(input_base64))
|
||
|
|
throw GoogleError::invalid("input must be base64 within " +
|
||
|
|
std::to_string(ModelsCaps::kMaxAdminBodyChars) + " characters");
|
||
|
|
Json body = with("methodName", Json(method_name));
|
||
|
|
body.set("input", Json(input_base64));
|
||
|
|
return run(plan_call("endpoint.direct_raw_predict", endpoint_id, &body));
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EndpointAdmin::explain(const std::string& endpoint_id, const Json& instances,
|
||
|
|
const Json& parameters, const std::string& deployed_model_id) const {
|
||
|
|
if (!instances.is_array() || instances.as_array().empty() ||
|
||
|
|
instances.as_array().size() > ModelsCaps::kMaxInstances)
|
||
|
|
throw GoogleError::invalid("instances must be an array of 1.." +
|
||
|
|
std::to_string(ModelsCaps::kMaxInstances) + " items");
|
||
|
|
AdminChecks::size(instances, "instances");
|
||
|
|
Json body = with("instances", instances);
|
||
|
|
if (parameters.is_object()) body.set("parameters", parameters);
|
||
|
|
if (!deployed_model_id.empty()) {
|
||
|
|
VertexTarget::require_segment(deployed_model_id, "deployedModelId");
|
||
|
|
body.set("deployedModelId", Json(deployed_model_id));
|
||
|
|
}
|
||
|
|
return run(plan_call("endpoint.explain", endpoint_id, &body));
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EndpointAdmin::invoke(const std::string& endpoint_id, const std::string& invoke_id,
|
||
|
|
const Json& body) const {
|
||
|
|
AdminChecks::object(body, "body");
|
||
|
|
return run(plan_call("endpoint.invoke", endpoint_id, &body, "", {{"invoke", invoke_id}}));
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EndpointAdmin::invoke_deployed_model(const std::string& endpoint_id,
|
||
|
|
const std::string& deployed_model_id,
|
||
|
|
const std::string& invoke_id, const Json& body) const {
|
||
|
|
AdminChecks::object(body, "body");
|
||
|
|
return run(plan_call("endpoint.deployed_model_invoke", endpoint_id, &body, "",
|
||
|
|
{{"deployed", deployed_model_id}, {"invoke", invoke_id}}));
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EndpointAdmin::chat_completions(const std::string& endpoint_id, const Json& body) const {
|
||
|
|
AdminChecks::object(body, "body");
|
||
|
|
return run(plan_call("endpoint.chat_completions", endpoint_id, &body));
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EndpointAdmin::openapi(const std::string& kind, const Json& body) const {
|
||
|
|
AdminChecks::object(body, "body");
|
||
|
|
if (kind != "completions" && kind != "embeddings" && kind != "responses")
|
||
|
|
throw GoogleError::invalid("kind must be completions, embeddings or responses");
|
||
|
|
return run(plan_call("endpoint.openapi_" + kind, "", &body));
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EndpointAdmin::science_inference(const Json& body) const {
|
||
|
|
AdminChecks::object(body, "body");
|
||
|
|
return run(plan_call("endpoint.science_inference", "", &body));
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EndpointAdmin::get_response(const std::string& endpoint_id,
|
||
|
|
const std::string& response_id) const {
|
||
|
|
return run(plan_call("endpoint.response_get", endpoint_id, nullptr, "",
|
||
|
|
{{"response", response_id}}));
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EndpointAdmin::delete_response(const std::string& endpoint_id,
|
||
|
|
const std::string& response_id, bool confirm) const {
|
||
|
|
AdminChecks::confirm(confirm, "deleting a stored response");
|
||
|
|
return run(plan_call("endpoint.response_delete", endpoint_id, nullptr, "",
|
||
|
|
{{"response", response_id}}));
|
||
|
|
}
|
||
|
|
|
||
|
|
// --- operations --------------------------------------------------------------
|
||
|
|
|
||
|
|
Operation EndpointAdmin::operation(const std::string& endpoint_id,
|
||
|
|
const std::string& operation_id) const {
|
||
|
|
return run_operation(plan_call("endpoint.operation_get", endpoint_id, nullptr, "",
|
||
|
|
{{"operation", operation_id}}));
|
||
|
|
}
|
||
|
|
|
||
|
|
std::vector<Operation> EndpointAdmin::list_operations(const std::string& endpoint_id,
|
||
|
|
std::size_t max_pages,
|
||
|
|
const std::string& filter) const {
|
||
|
|
if (max_pages < 1 || max_pages > ModelsCaps::kMaxOperationPages)
|
||
|
|
throw GoogleError::invalid("max_pages outside 1.." +
|
||
|
|
std::to_string(ModelsCaps::kMaxOperationPages));
|
||
|
|
if (filter.size() > 256) throw GoogleError::invalid("filter over 256 characters");
|
||
|
|
std::vector<Operation> out;
|
||
|
|
std::string token;
|
||
|
|
for (std::size_t page = 0; page < max_pages; ++page) {
|
||
|
|
std::string q = "pageSize=" + std::to_string(ModelsCaps::kMaxPageSize);
|
||
|
|
if (!filter.empty()) q += "&filter=" + clients::Url::encode_component(filter);
|
||
|
|
if (!token.empty()) q += "&pageToken=" + clients::Url::encode_component(token);
|
||
|
|
Json reply = run(plan_call("endpoint.operation_list", endpoint_id, nullptr, q));
|
||
|
|
if (const Json* items = reply.find("operations"); items && items->is_array())
|
||
|
|
for (const Json& item : items->as_array()) {
|
||
|
|
if (out.size() >= ModelsCaps::kMaxModelsReturned) return out;
|
||
|
|
out.push_back(Operation::from_json(item));
|
||
|
|
}
|
||
|
|
token = json_str(reply, "nextPageToken");
|
||
|
|
if (token.empty()) break;
|
||
|
|
}
|
||
|
|
return out;
|
||
|
|
}
|
||
|
|
|
||
|
|
void EndpointAdmin::cancel_operation(const std::string& endpoint_id,
|
||
|
|
const std::string& operation_id) const {
|
||
|
|
run(plan_call("endpoint.operation_cancel", endpoint_id, nullptr, "",
|
||
|
|
{{"operation", operation_id}}));
|
||
|
|
}
|
||
|
|
|
||
|
|
Operation EndpointAdmin::wait_operation(const std::string& endpoint_id,
|
||
|
|
const std::string& operation_id,
|
||
|
|
int timeout_seconds) const {
|
||
|
|
if (timeout_seconds < 1 || timeout_seconds > ModelsCaps::kMaxOperationTimeoutSeconds)
|
||
|
|
throw GoogleError::invalid("timeout outside 1.." +
|
||
|
|
std::to_string(ModelsCaps::kMaxOperationTimeoutSeconds) +
|
||
|
|
" seconds");
|
||
|
|
return run_operation(plan_call("endpoint.operation_wait", endpoint_id, nullptr,
|
||
|
|
"timeout=" + std::to_string(timeout_seconds) + "s",
|
||
|
|
{{"operation", operation_id}}));
|
||
|
|
}
|
||
|
|
|
||
|
|
void EndpointAdmin::delete_operation(const std::string& endpoint_id,
|
||
|
|
const std::string& operation_id, bool confirm) const {
|
||
|
|
AdminChecks::confirm(confirm, "deleting an operation record");
|
||
|
|
run(plan_call("endpoint.operation_delete", endpoint_id, nullptr, "",
|
||
|
|
{{"operation", operation_id}}));
|
||
|
|
}
|
||
|
|
|
||
|
|
// --- v1beta1 only ------------------------------------------------------------
|
||
|
|
|
||
|
|
Json EndpointAdmin::list_beta(const std::string& read_mask, const std::string& gdc_zone) const {
|
||
|
|
std::string q = "pageSize=" + std::to_string(ModelsCaps::kMaxPageSize);
|
||
|
|
if (!read_mask.empty()) {
|
||
|
|
AdminChecks::mask(read_mask);
|
||
|
|
q += "&readMask=" + clients::Url::encode_component(read_mask);
|
||
|
|
}
|
||
|
|
if (!gdc_zone.empty()) {
|
||
|
|
VertexTarget::require_segment(gdc_zone, "gdcZone");
|
||
|
|
q += "&gdcZone=" + clients::Url::encode_component(gdc_zone);
|
||
|
|
}
|
||
|
|
return run(plan_call("endpoint.beta_list", "", nullptr, q));
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EndpointAdmin::get_iam_policy(const std::string& endpoint_id, int policy_version) const {
|
||
|
|
if (policy_version < 0 || policy_version > 3)
|
||
|
|
throw GoogleError::invalid("policy version outside 0..3");
|
||
|
|
return run(plan_call("endpoint.beta_get_iam_policy", endpoint_id, nullptr,
|
||
|
|
policy_version ? q_int("options.requestedPolicyVersion", policy_version)
|
||
|
|
: ""));
|
||
|
|
}
|
||
|
|
|
||
|
|
Json EndpointAdmin::set_iam_policy(const std::string& endpoint_id, const Json& policy,
|
||
|
|
bool confirm) const {
|
||
|
|
AdminChecks::confirm(confirm, "replacing an IAM policy");
|
||
|
|
AdminChecks::object(policy, "policy");
|
||
|
|
Json body = with("policy", policy);
|
||
|
|
return run(plan_call("endpoint.beta_set_iam_policy", endpoint_id, &body));
|
||
|
|
}
|
||
|
|
|
||
|
|
std::vector<std::string> EndpointAdmin::test_iam_permissions(
|
||
|
|
const std::string& endpoint_id, const std::vector<std::string>& permissions) const {
|
||
|
|
if (permissions.empty() || permissions.size() > ModelsCaps::kMaxPermissions)
|
||
|
|
throw GoogleError::invalid("1.." + std::to_string(ModelsCaps::kMaxPermissions) +
|
||
|
|
" permissions");
|
||
|
|
std::string q;
|
||
|
|
for (const std::string& p : permissions) {
|
||
|
|
if (p.empty() || p.size() > 128) throw GoogleError::invalid("bad permission");
|
||
|
|
for (char c : p)
|
||
|
|
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ||
|
||
|
|
c == '.' || c == '_'))
|
||
|
|
throw GoogleError::invalid("bad permission");
|
||
|
|
q += (q.empty() ? "permissions=" : "&permissions=") + clients::Url::encode_component(p);
|
||
|
|
}
|
||
|
|
Json reply = run(plan_call("endpoint.beta_test_iam_permissions", endpoint_id, nullptr, q));
|
||
|
|
std::vector<std::string> out;
|
||
|
|
if (const Json* a = reply.find("permissions"); a && a->is_array())
|
||
|
|
for (const Json& p : a->as_array())
|
||
|
|
if (p.is_string()) out.push_back(p.as_string());
|
||
|
|
return out;
|
||
|
|
}
|
||
|
|
|
||
|
|
} // namespace molo::google::models
|