#pragma once // EvaluationApi: the location-level evaluation RPCs of aiplatform v1: // evaluateInstances, evaluateDataset, generateInstanceRubrics, // generateUserScenarios, generateLossClusters. Resource CRUD (sets, items, // metrics, runs, online evaluators) is EvaluationResources. #include #include #include "google/operation.h" #include "google/optimize/eval_requests.h" #include "google/optimize/optimize_service.h" #include "google/optimize/scenario.h" #include "google/resource.h" namespace molo::google::optimize { struct ScenarioRequest { // Discovery: agents is a map agentId -> AgentConfig. Pass the agent // definitions as Json ({agentId: {agentType, instruction, description, // tools, subAgents, agentId}}) and the root agent id. clients::body::Json agents = clients::body::Json::object(); std::string root_agent_id; int count = 5; // 1..kMaxScenarios (Google: max 100) std::string instruction; // simulationInstruction std::string environment_data; // environmentData std::string model_name; // empty: Google's default }; class EvaluationApi : public OptimizeService { public: EvaluationApi(std::shared_ptr client, CloudContext context); const CloudContext& context() const { return context_; } // Raw request (use EvaluationRequests builders). Unknown top-level fields // are rejected locally. `location` is not sent in the body. clients::body::Json evaluate_instances(const clients::body::Json& request) const; MetricOutcome evaluate(const clients::body::Json& request) const; // Long-running (returns the operation). wait=true polls to completion. Operation evaluate_dataset(const clients::body::Json& request, bool wait = false, const Sleeper& sleeper = nullptr) const; Operation generate_loss_clusters(const std::string& evaluation_set, const std::vector>& metric_candidate, bool wait = false, const Sleeper& sleeper = nullptr) const; clients::body::Json generate_instance_rubrics(const clients::body::Json& request) const; std::vector generate_user_scenarios(const ScenarioRequest& request) const; // Body of generateUserScenarios, exposed so it can be checked offline. static clients::body::Json scenario_body(const ScenarioRequest& request); Operation wait(const Operation& op, const Sleeper& sleeper = nullptr) const; private: CloudContext context_; }; } // namespace molo::google::optimize