feat(clients): optimize: evaluation, simulation, observability and optimizer on a discovery-checked call table

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-24 04:40:55 +02:00
co-authored by Claude Sonnet 5.5
parent 6d744a433f
commit 1e2fb400bb
55 changed files with 5099 additions and 3 deletions
+62
View File
@@ -0,0 +1,62 @@
#pragma once
// EvaluationApi: the location-level evaluation RPCs of aiplatform v1:
// evaluateInstances, evaluateDataset, generateInstanceRubrics,
// generateUserScenarios, generateLossClusters. Resource CRUD (sets, items,
// metrics, runs, online evaluators) is EvaluationResources.
#include <string>
#include <vector>
#include "google/operation.h"
#include "google/optimize/eval_requests.h"
#include "google/optimize/optimize_service.h"
#include "google/optimize/scenario.h"
#include "google/resource.h"
namespace molo::google::optimize {
struct ScenarioRequest {
// Discovery: agents is a map agentId -> AgentConfig. Pass the agent
// definitions as Json ({agentId: {agentType, instruction, description,
// tools, subAgents, agentId}}) and the root agent id.
clients::body::Json agents = clients::body::Json::object();
std::string root_agent_id;
int count = 5; // 1..kMaxScenarios (Google: max 100)
std::string instruction; // simulationInstruction
std::string environment_data; // environmentData
std::string model_name; // empty: Google's default
};
class EvaluationApi : public OptimizeService {
public:
EvaluationApi(std::shared_ptr<GoogleClient> client, CloudContext context);
const CloudContext& context() const { return context_; }
// Raw request (use EvaluationRequests builders). Unknown top-level fields
// are rejected locally. `location` is not sent in the body.
clients::body::Json evaluate_instances(const clients::body::Json& request) const;
MetricOutcome evaluate(const clients::body::Json& request) const;
// Long-running (returns the operation). wait=true polls to completion.
Operation evaluate_dataset(const clients::body::Json& request, bool wait = false,
const Sleeper& sleeper = nullptr) const;
Operation generate_loss_clusters(const std::string& evaluation_set,
const std::vector<std::pair<std::string, std::string>>&
metric_candidate,
bool wait = false, const Sleeper& sleeper = nullptr) const;
clients::body::Json generate_instance_rubrics(const clients::body::Json& request) const;
std::vector<Scenario> generate_user_scenarios(const ScenarioRequest& request) const;
// Body of generateUserScenarios, exposed so it can be checked offline.
static clients::body::Json scenario_body(const ScenarioRequest& request);
Operation wait(const Operation& op, const Sleeper& sleeper = nullptr) const;
private:
CloudContext context_;
};
} // namespace molo::google::optimize