feat(clients): optimize: evaluation, simulation, observability and optimizer on a discovery-checked call table
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
#pragma once
|
||||
|
||||
// EvaluationApi: the location-level evaluation RPCs of aiplatform v1:
|
||||
// evaluateInstances, evaluateDataset, generateInstanceRubrics,
|
||||
// generateUserScenarios, generateLossClusters. Resource CRUD (sets, items,
|
||||
// metrics, runs, online evaluators) is EvaluationResources.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "google/operation.h"
|
||||
#include "google/optimize/eval_requests.h"
|
||||
#include "google/optimize/optimize_service.h"
|
||||
#include "google/optimize/scenario.h"
|
||||
#include "google/resource.h"
|
||||
|
||||
namespace molo::google::optimize {
|
||||
|
||||
struct ScenarioRequest {
|
||||
// Discovery: agents is a map agentId -> AgentConfig. Pass the agent
|
||||
// definitions as Json ({agentId: {agentType, instruction, description,
|
||||
// tools, subAgents, agentId}}) and the root agent id.
|
||||
clients::body::Json agents = clients::body::Json::object();
|
||||
std::string root_agent_id;
|
||||
int count = 5; // 1..kMaxScenarios (Google: max 100)
|
||||
std::string instruction; // simulationInstruction
|
||||
std::string environment_data; // environmentData
|
||||
std::string model_name; // empty: Google's default
|
||||
};
|
||||
|
||||
class EvaluationApi : public OptimizeService {
|
||||
public:
|
||||
EvaluationApi(std::shared_ptr<GoogleClient> client, CloudContext context);
|
||||
|
||||
const CloudContext& context() const { return context_; }
|
||||
|
||||
// Raw request (use EvaluationRequests builders). Unknown top-level fields
|
||||
// are rejected locally. `location` is not sent in the body.
|
||||
clients::body::Json evaluate_instances(const clients::body::Json& request) const;
|
||||
MetricOutcome evaluate(const clients::body::Json& request) const;
|
||||
|
||||
// Long-running (returns the operation). wait=true polls to completion.
|
||||
Operation evaluate_dataset(const clients::body::Json& request, bool wait = false,
|
||||
const Sleeper& sleeper = nullptr) const;
|
||||
Operation generate_loss_clusters(const std::string& evaluation_set,
|
||||
const std::vector<std::pair<std::string, std::string>>&
|
||||
metric_candidate,
|
||||
bool wait = false, const Sleeper& sleeper = nullptr) const;
|
||||
|
||||
clients::body::Json generate_instance_rubrics(const clients::body::Json& request) const;
|
||||
std::vector<Scenario> generate_user_scenarios(const ScenarioRequest& request) const;
|
||||
|
||||
// Body of generateUserScenarios, exposed so it can be checked offline.
|
||||
static clients::body::Json scenario_body(const ScenarioRequest& request);
|
||||
|
||||
Operation wait(const Operation& op, const Sleeper& sleeper = nullptr) const;
|
||||
|
||||
private:
|
||||
CloudContext context_;
|
||||
};
|
||||
|
||||
} // namespace molo::google::optimize
|
||||
Reference in New Issue
Block a user