feat(clients): optimize: evaluation, simulation, observability and optimizer on a discovery-checked call table
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,13 +1,369 @@
|
||||
#include "../AgentDemos.hpp"
|
||||
|
||||
#include <functional>
|
||||
|
||||
#include "../../fw/Router.hpp"
|
||||
#include "../../fw/Templates.hpp"
|
||||
#include "google/credentials.h"
|
||||
#include "google/optimize/autorater.h"
|
||||
#include "google/optimize/calls.h"
|
||||
#include "google/optimize/clusterer.h"
|
||||
#include "google/optimize/direct_model.h"
|
||||
#include "google/optimize/eval_requests.h"
|
||||
#include "google/optimize/eval_resources.h"
|
||||
#include "google/optimize/evaluation_api.h"
|
||||
#include "google/optimize/observability.h"
|
||||
#include "google/optimize/optimizer.h"
|
||||
#include "google/optimize/simulator.h"
|
||||
|
||||
namespace molo::example::app {
|
||||
|
||||
// Agent Platform: optimize. Stub: the module agent fills this in.
|
||||
// Agent Platform: optimize (evaluation, simulation, observability, optimizer)
|
||||
// over src/google/optimize. Routes under /agent/optimize. GET answers without
|
||||
// a credential; everything that calls Google is a POST (503
|
||||
// google_not_configured without credentials, 422 on bad input).
|
||||
// Gap: live OAuth-only reads (runs, traces, logs) are verified by discovery
|
||||
// only, see src/google/optimize/SPEC.md.
|
||||
|
||||
namespace {
|
||||
|
||||
namespace opt = google::optimize;
|
||||
using Json = clients::body::Json;
|
||||
|
||||
[[noreturn]] void rethrow(const google::GoogleError& e) {
|
||||
if (e.http_status() == 503 && e.status() == "NOT_CONFIGURED")
|
||||
throw fw::HttpError(503, "google_not_configured", e.message());
|
||||
if (e.service().empty() && e.status() == "INVALID_ARGUMENT")
|
||||
throw fw::HttpError(422, "validation_error", e.message());
|
||||
throw fw::HttpError(502, "google_error", e.status() + ": " + e.message());
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
auto guarded(F&& f) -> decltype(f()) {
|
||||
try {
|
||||
return f();
|
||||
} catch (const google::GoogleError& e) {
|
||||
rethrow(e);
|
||||
}
|
||||
}
|
||||
|
||||
fw::Response json_out(const Json& doc) {
|
||||
fw::Response res = fw::JsonResponse(doc.dump(), 200);
|
||||
res.headers.set("Cache-Control", "no-store");
|
||||
return res;
|
||||
}
|
||||
|
||||
Json parse_body(const fw::Request& req) {
|
||||
try {
|
||||
Json doc = Json::parse(req.body);
|
||||
if (doc.is_object()) return doc;
|
||||
} catch (const clients::body::JsonError&) {
|
||||
}
|
||||
throw fw::HttpError(422, "validation_error", "body must be a JSON object");
|
||||
}
|
||||
|
||||
std::string text_field(const Json& body, const char* name, bool required,
|
||||
std::size_t max = opt::OptimizeCaps::kMaxTextChars) {
|
||||
const Json* v = body.find(name);
|
||||
std::string out = v && v->is_string() ? v->as_string() : "";
|
||||
if (required && out.empty())
|
||||
throw fw::HttpError(422, "validation_error", std::string(name) + " is required");
|
||||
if (out.size() > max)
|
||||
throw fw::HttpError(422, "validation_error",
|
||||
std::string(name) + " over " + std::to_string(max) + " characters");
|
||||
return out;
|
||||
}
|
||||
|
||||
int int_field(const Json& body, const char* name, int fallback, int lo, int hi) {
|
||||
const Json* v = body.find(name);
|
||||
if (!v) return fallback;
|
||||
int n = 0;
|
||||
if (v->is_number()) n = static_cast<int>(v->as_int());
|
||||
else if (v->is_string()) {
|
||||
try { n = std::stoi(v->as_string()); } catch (...) { n = lo - 1; }
|
||||
} else n = lo - 1;
|
||||
if (n < lo || n > hi)
|
||||
throw fw::HttpError(422, "validation_error", std::string(name) + " must be " +
|
||||
std::to_string(lo) + ".." + std::to_string(hi));
|
||||
return n;
|
||||
}
|
||||
|
||||
std::vector<opt::TrajectoryCall> tool_list(const std::string& csv) {
|
||||
std::vector<opt::TrajectoryCall> out;
|
||||
for (const std::string& name : opt::split_list(csv)) out.push_back({name, "{}"});
|
||||
return out;
|
||||
}
|
||||
|
||||
google::CloudContext require_context() {
|
||||
google::CloudContext ctx = google::CloudContext::from_env();
|
||||
if (!ctx.valid())
|
||||
throw fw::HttpError(503, "google_not_configured",
|
||||
"GOOGLE_CLOUD_PROJECT is not set or malformed");
|
||||
return ctx;
|
||||
}
|
||||
|
||||
const char kPage[] = R"OPT({% extends "base.html" %}
|
||||
{% block content %}
|
||||
<h1>Agent optimize</h1>
|
||||
<p class="hint">Evaluate, simulate, observe and optimize agents. Pick an
|
||||
action, fill the fields, send. Calls that reach Google need credentials.</p>
|
||||
<section class="opt">
|
||||
<label class="inline">Action
|
||||
<select id="o-action"></select></label>
|
||||
<p id="o-help" class="hint"></p>
|
||||
<div id="o-fields"></div>
|
||||
<button id="o-send" type="button" class="primary">Send</button>
|
||||
<p id="o-error" class="error" role="alert"></p>
|
||||
<pre id="o-out" class="opt-out"></pre>
|
||||
</section>
|
||||
<script src="/embedded/agent_optimize.js"></script>
|
||||
{% endblock %}
|
||||
)OPT";
|
||||
|
||||
} // namespace
|
||||
|
||||
void register_agent_optimize_demo(fw::Router& router, Services& services) {
|
||||
(void)router;
|
||||
(void)services;
|
||||
if (!services.google) services.google = google::GoogleClient::from_env();
|
||||
auto gclient = services.google;
|
||||
fw::Environment* env = services.templates.get();
|
||||
env->add_template("agent_optimize.html", kPage);
|
||||
|
||||
services.add_feature("agent optimize", "/agent/optimize");
|
||||
|
||||
auto model_for = [gclient]() {
|
||||
return google::optimize::DirectModel::from_env(gclient);
|
||||
};
|
||||
|
||||
router.get("/agent/optimize", "agent-optimize-page", [env](fw::Request req) {
|
||||
(void)req;
|
||||
fw::TemplateCtx ctx;
|
||||
ctx["title"] = std::string("Agent optimize");
|
||||
return fw::Response(200, env->render("agent_optimize.html", ctx), "text/html; charset=utf-8");
|
||||
});
|
||||
|
||||
router.get("/agent/optimize/status", "agent-optimize-status", [gclient](fw::Request req) {
|
||||
(void)req;
|
||||
google::CloudContext ctx = google::CloudContext::from_env();
|
||||
Json calls = Json::array();
|
||||
for (const auto& c : opt::all_calls()) {
|
||||
Json o = Json::object();
|
||||
o.set("id", Json(c.id));
|
||||
o.set("http", Json(c.http));
|
||||
o.set("path", Json(c.path));
|
||||
o.set("service", Json(std::string(c.service) + " " + c.version));
|
||||
calls.push(o);
|
||||
}
|
||||
Json caps = Json::object();
|
||||
caps.set("max_sim_turns", Json(opt::OptimizeCaps::kMaxSimTurns));
|
||||
caps.set("max_scenarios", Json(static_cast<int>(opt::OptimizeCaps::kMaxScenarios)));
|
||||
caps.set("max_failures", Json(static_cast<int>(opt::OptimizeCaps::kMaxFailures)));
|
||||
caps.set("max_clusters", Json(static_cast<int>(opt::OptimizeCaps::kMaxClusters)));
|
||||
caps.set("max_optimize_rounds", Json(opt::OptimizeCaps::kMaxOptimizeRounds));
|
||||
caps.set("max_log_entries", Json(static_cast<int>(opt::OptimizeCaps::kMaxLogEntries)));
|
||||
Json out = Json::object();
|
||||
out.set("configured", Json(gclient->configured()));
|
||||
out.set("project_set", Json(ctx.valid()));
|
||||
out.set("location", Json(ctx.location));
|
||||
out.set("calls", calls);
|
||||
out.set("caps", caps);
|
||||
return json_out(out);
|
||||
});
|
||||
|
||||
// Managed evaluateInstances: safety | exact_match | trajectory.
|
||||
router.post("/agent/optimize/evaluate", "agent-optimize-evaluate", [gclient](fw::Request req) {
|
||||
Json body = parse_body(req);
|
||||
std::string kind = text_field(body, "kind", true, 32);
|
||||
Json request;
|
||||
try {
|
||||
if (kind == "safety") {
|
||||
request = opt::EvaluationRequests::safety(text_field(body, "prediction", true));
|
||||
} else if (kind == "exact_match") {
|
||||
request = opt::EvaluationRequests::exact_match(
|
||||
{{text_field(body, "prediction", true), text_field(body, "reference", true)}});
|
||||
} else if (kind == "trajectory") {
|
||||
request = opt::EvaluationRequests::trajectory(
|
||||
opt::TrajectoryMetric::kExactMatch, tool_list(text_field(body, "predicted", true, 2000)),
|
||||
tool_list(text_field(body, "reference", true, 2000)));
|
||||
} else {
|
||||
throw fw::HttpError(422, "validation_error", "kind must be safety, exact_match or trajectory");
|
||||
}
|
||||
} catch (const google::GoogleError& e) {
|
||||
rethrow(e);
|
||||
}
|
||||
google::CloudContext ctx = require_context();
|
||||
return guarded([&] {
|
||||
opt::EvaluationApi api(gclient, ctx);
|
||||
opt::MetricOutcome out = api.evaluate(request);
|
||||
Json res = Json::object();
|
||||
res.set("kind", Json(out.kind));
|
||||
Json scores = Json::array();
|
||||
for (double s : out.scores) scores.push(Json(s));
|
||||
res.set("scores", scores);
|
||||
res.set("explanation", Json(out.explanation));
|
||||
return json_out(res);
|
||||
});
|
||||
});
|
||||
|
||||
router.post("/agent/optimize/scenarios", "agent-optimize-scenarios", [gclient](fw::Request req) {
|
||||
Json body = parse_body(req);
|
||||
opt::ScenarioRequest sr;
|
||||
std::string instruction = text_field(body, "agent_instruction", true, opt::OptimizeCaps::kMaxInstructionChars);
|
||||
Json agent = Json::object();
|
||||
agent.set("agentId", Json("root"));
|
||||
agent.set("agentType", Json("LlmAgent"));
|
||||
agent.set("instruction", Json(instruction));
|
||||
sr.agents.set("root", agent);
|
||||
sr.root_agent_id = "root";
|
||||
sr.count = int_field(body, "count", 5, 1, static_cast<int>(opt::OptimizeCaps::kMaxScenarios));
|
||||
sr.instruction = text_field(body, "instruction", false);
|
||||
google::CloudContext ctx = require_context();
|
||||
return guarded([&] {
|
||||
opt::EvaluationApi api(gclient, ctx);
|
||||
Json arr = Json::array();
|
||||
for (const opt::Scenario& s : api.generate_user_scenarios(sr)) arr.push(s.to_json());
|
||||
Json out = Json::object();
|
||||
out.set("scenarios", arr);
|
||||
return json_out(out);
|
||||
});
|
||||
});
|
||||
|
||||
router.post("/agent/optimize/runs", "agent-optimize-runs", [gclient](fw::Request req) {
|
||||
Json body = parse_body(req);
|
||||
opt::ListOptions lo;
|
||||
lo.page_size = int_field(body, "limit", 10, 1, 100);
|
||||
lo.max_pages = 1;
|
||||
google::CloudContext ctx = require_context();
|
||||
return guarded([&] {
|
||||
opt::EvaluationResources res(gclient, ctx);
|
||||
Json out = Json::object();
|
||||
out.set("evaluationRuns", res.list(opt::Collection::kEvaluationRuns, lo));
|
||||
return json_out(out);
|
||||
});
|
||||
});
|
||||
|
||||
router.post("/agent/optimize/traces", "agent-optimize-traces", [gclient](fw::Request req) {
|
||||
Json body = parse_body(req);
|
||||
opt::TraceQuery q;
|
||||
q.page_size = int_field(body, "limit", 20, 1, 100);
|
||||
q.max_pages = 1;
|
||||
q.max_traces = static_cast<std::size_t>(q.page_size);
|
||||
q.filter = text_field(body, "filter", false, 1000);
|
||||
google::CloudContext ctx = require_context();
|
||||
return guarded([&] {
|
||||
opt::TraceReader reader(gclient);
|
||||
std::vector<opt::Trace> traces = reader.list(ctx.project, q);
|
||||
Json out = opt::summarize(traces, {}).to_json();
|
||||
Json ids = Json::array();
|
||||
for (const opt::Trace& t : traces) ids.push(Json(t.trace_id));
|
||||
out.set("trace_ids", ids);
|
||||
return json_out(out);
|
||||
});
|
||||
});
|
||||
|
||||
router.post("/agent/optimize/logs", "agent-optimize-logs", [gclient](fw::Request req) {
|
||||
Json body = parse_body(req);
|
||||
opt::LogQuery q;
|
||||
q.page_size = int_field(body, "limit", 20, 1, 100);
|
||||
q.max_entries = static_cast<std::size_t>(q.page_size);
|
||||
q.filter = text_field(body, "filter", false, 2000);
|
||||
std::string trace_id = text_field(body, "trace_id", false, 64);
|
||||
google::CloudContext ctx = require_context();
|
||||
q.resource_names = {"projects/" + ctx.project};
|
||||
return guarded([&] {
|
||||
if (!trace_id.empty()) q.filter = opt::LogQuery::trace_filter(ctx.project, trace_id);
|
||||
opt::LogReader reader(gclient);
|
||||
std::vector<opt::LogEntry> entries = reader.entries(q);
|
||||
Json out = opt::summarize({}, entries).to_json();
|
||||
Json arr = Json::array();
|
||||
for (const opt::LogEntry& e : entries) {
|
||||
Json o = Json::object();
|
||||
o.set("timestamp", Json(e.timestamp));
|
||||
o.set("severity", Json(e.severity));
|
||||
o.set("text", Json(e.text));
|
||||
arr.push(o);
|
||||
}
|
||||
out.set("entries", arr);
|
||||
return json_out(out);
|
||||
});
|
||||
});
|
||||
|
||||
// Local autorater over a pasted transcript ("user: ..." / "agent: ...").
|
||||
router.post("/agent/optimize/score", "agent-optimize-score", [model_for](fw::Request req) {
|
||||
Json body = parse_body(req);
|
||||
std::string metric = text_field(body, "metric", false, 32);
|
||||
if (metric.empty()) metric = "task_success";
|
||||
if (metric != "task_success" && metric != "safety")
|
||||
throw fw::HttpError(422, "validation_error", "metric must be task_success or safety");
|
||||
return guarded([&] {
|
||||
opt::Conversation c = opt::Conversation::parse_transcript(text_field(body, "transcript", true));
|
||||
opt::Autorater rater(model_for());
|
||||
opt::Score s = rater.score(metric, c, metric == "safety" ? opt::Autorater::safety_rubrics()
|
||||
: opt::Autorater::task_success_rubrics());
|
||||
return json_out(s.to_json());
|
||||
});
|
||||
});
|
||||
|
||||
// One failure per line.
|
||||
router.post("/agent/optimize/cluster", "agent-optimize-cluster", [model_for](fw::Request req) {
|
||||
Json body = parse_body(req);
|
||||
int max_clusters = int_field(body, "max_clusters", 5, 1, static_cast<int>(opt::OptimizeCaps::kMaxClusters));
|
||||
std::string text = text_field(body, "failures", true, opt::OptimizeCaps::kMaxTranscriptChars);
|
||||
std::vector<opt::Failure> failures;
|
||||
std::size_t pos = 0;
|
||||
while (pos < text.size()) {
|
||||
std::size_t end = text.find('\n', pos);
|
||||
if (end == std::string::npos) end = text.size();
|
||||
std::string line = text.substr(pos, end - pos);
|
||||
pos = end + 1;
|
||||
if (line.empty()) continue;
|
||||
failures.push_back({"f" + std::to_string(failures.size() + 1), line, ""});
|
||||
}
|
||||
return guarded([&] {
|
||||
opt::FailureClusterer fc(model_for());
|
||||
return json_out(fc.cluster(failures, static_cast<std::size_t>(max_clusters)).to_json());
|
||||
});
|
||||
});
|
||||
|
||||
router.post("/agent/optimize/suggest", "agent-optimize-suggest", [model_for](fw::Request req) {
|
||||
Json body = parse_body(req);
|
||||
std::string instruction = text_field(body, "instruction", true, opt::OptimizeCaps::kMaxInstructionChars);
|
||||
std::string text = text_field(body, "failures", true, opt::OptimizeCaps::kMaxTranscriptChars);
|
||||
std::vector<opt::Failure> failures;
|
||||
std::size_t pos = 0;
|
||||
while (pos < text.size()) {
|
||||
std::size_t end = text.find('\n', pos);
|
||||
if (end == std::string::npos) end = text.size();
|
||||
std::string line = text.substr(pos, end - pos);
|
||||
pos = end + 1;
|
||||
if (!line.empty()) failures.push_back({"f" + std::to_string(failures.size() + 1), line, ""});
|
||||
}
|
||||
return guarded([&] {
|
||||
auto model = model_for();
|
||||
opt::ClusterReport clusters = opt::FailureClusterer(model).cluster(failures, 5);
|
||||
Json out = opt::InstructionOptimizer(model).suggest(instruction, clusters, failures).to_json();
|
||||
out.set("clusters", clusters.to_json());
|
||||
return json_out(out);
|
||||
});
|
||||
});
|
||||
|
||||
// Simulated user against a ModelAgent built from the instruction.
|
||||
router.post("/agent/optimize/simulate", "agent-optimize-simulate", [model_for](fw::Request req) {
|
||||
Json body = parse_body(req);
|
||||
std::string instruction = text_field(body, "instruction", true, opt::OptimizeCaps::kMaxInstructionChars);
|
||||
opt::Scenario sc;
|
||||
sc.title = text_field(body, "title", false, 200);
|
||||
sc.starting_prompt = text_field(body, "starting_prompt", true);
|
||||
sc.plan = text_field(body, "plan", true);
|
||||
opt::SimulationConfig cfg;
|
||||
cfg.max_turns = int_field(body, "max_turns", opt::OptimizeCaps::kDefaultSimTurns, 1, opt::OptimizeCaps::kMaxSimTurns);
|
||||
return guarded([&] {
|
||||
auto model = model_for();
|
||||
opt::ModelAgent agent(model, instruction);
|
||||
opt::VirtualToolbox tools;
|
||||
opt::Simulator sim(model, model);
|
||||
return json_out(sim.run_and_score(sc, agent, tools, cfg).to_json());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace molo::example::app
|
||||
|
||||
Reference in New Issue
Block a user