63 lines
2.6 KiB
C++
63 lines
2.6 KiB
C++
#pragma once
|
|||
|
|
|
||
|
|
// EvaluationApi: the location-level evaluation RPCs of aiplatform v1:
|
||
|
|
// evaluateInstances, evaluateDataset, generateInstanceRubrics,
|
||
|
|
// generateUserScenarios, generateLossClusters. Resource CRUD (sets, items,
|
||
|
|
// metrics, runs, online evaluators) is EvaluationResources.
|
||
|
|
|
||
|
|
#include <string>
|
||
|
|
#include <vector>
|
||
|
|
|
||
|
|
#include "google/operation.h"
|
||
|
|
#include "google/optimize/eval_requests.h"
|
||
|
|
#include "google/optimize/optimize_service.h"
|
||
|
|
#include "google/optimize/scenario.h"
|
||
|
|
#include "google/resource.h"
|
||
|
|
|
||
|
|
namespace molo::google::optimize {
|
||
|
|
|
||
|
|
struct ScenarioRequest {
|
||
|
|
// Discovery: agents is a map agentId -> AgentConfig. Pass the agent
|
||
|
|
// definitions as Json ({agentId: {agentType, instruction, description,
|
||
|
|
// tools, subAgents, agentId}}) and the root agent id.
|
||
|
|
clients::body::Json agents = clients::body::Json::object();
|
||
|
|
std::string root_agent_id;
|
||
|
|
int count = 5; // 1..kMaxScenarios (Google: max 100)
|
||
|
|
std::string instruction; // simulationInstruction
|
||
|
|
std::string environment_data; // environmentData
|
||
|
|
std::string model_name; // empty: Google's default
|
||
|
|
};
|
||
|
|
|
||
|
|
class EvaluationApi : public OptimizeService {
|
||
|
|
public:
|
||
|
|
EvaluationApi(std::shared_ptr<GoogleClient> client, CloudContext context);
|
||
|
|
|
||
|
|
const CloudContext& context() const { return context_; }
|
||
|
|
|
||
|
|
// Raw request (use EvaluationRequests builders). Unknown top-level fields
|
||
|
|
// are rejected locally. `location` is not sent in the body.
|
||
|
|
clients::body::Json evaluate_instances(const clients::body::Json& request) const;
|
||
|
|
MetricOutcome evaluate(const clients::body::Json& request) const;
|
||
|
|
|
||
|
|
// Long-running (returns the operation). wait=true polls to completion.
|
||
|
|
Operation evaluate_dataset(const clients::body::Json& request, bool wait = false,
|
||
|
|
const Sleeper& sleeper = nullptr) const;
|
||
|
|
Operation generate_loss_clusters(const std::string& evaluation_set,
|
||
|
|
const std::vector<std::pair<std::string, std::string>>&
|
||
|
|
metric_candidate,
|
||
|
|
bool wait = false, const Sleeper& sleeper = nullptr) const;
|
||
|
|
|
||
|
|
clients::body::Json generate_instance_rubrics(const clients::body::Json& request) const;
|
||
|
|
std::vector<Scenario> generate_user_scenarios(const ScenarioRequest& request) const;
|
||
|
|
|
||
|
|
// Body of generateUserScenarios, exposed so it can be checked offline.
|
||
|
|
static clients::body::Json scenario_body(const ScenarioRequest& request);
|
||
|
|
|
||
|
|
Operation wait(const Operation& op, const Sleeper& sleeper = nullptr) const;
|
||
|
|
|
||
|
|
private:
|
||
|
|
CloudContext context_;
|
||
|
|
};
|
||
|
|
|
||
|
|
} // namespace molo::google::optimize
|