Files
moloapp/example/app/agent/OptimizeDemo.cpp
T

370 lines
15 KiB
C++

#include "../AgentDemos.hpp"
#include <functional>
#include "../../fw/Router.hpp"
#include "../../fw/Templates.hpp"
#include "google/credentials.h"
#include "google/optimize/autorater.h"
#include "google/optimize/calls.h"
#include "google/optimize/clusterer.h"
#include "google/optimize/direct_model.h"
#include "google/optimize/eval_requests.h"
#include "google/optimize/eval_resources.h"
#include "google/optimize/evaluation_api.h"
#include "google/optimize/observability.h"
#include "google/optimize/optimizer.h"
#include "google/optimize/simulator.h"
namespace molo::example::app {
// Agent Platform: optimize (evaluation, simulation, observability, optimizer)
// over src/google/optimize. Routes under /agent/optimize. GET answers without
// a credential; everything that calls Google is a POST (503
// google_not_configured without credentials, 422 on bad input).
// Gap: live OAuth-only reads (runs, traces, logs) are verified by discovery
// only, see src/google/optimize/SPEC.md.
namespace {
namespace opt = google::optimize;
using Json = clients::body::Json;
[[noreturn]] void rethrow(const google::GoogleError& e) {
if (e.http_status() == 503 && e.status() == "NOT_CONFIGURED")
throw fw::HttpError(503, "google_not_configured", e.message());
if (e.service().empty() && e.status() == "INVALID_ARGUMENT")
throw fw::HttpError(422, "validation_error", e.message());
throw fw::HttpError(502, "google_error", e.status() + ": " + e.message());
}
template <typename F>
auto guarded(F&& f) -> decltype(f()) {
try {
return f();
} catch (const google::GoogleError& e) {
rethrow(e);
}
}
fw::Response json_out(const Json& doc) {
fw::Response res = fw::JsonResponse(doc.dump(), 200);
res.headers.set("Cache-Control", "no-store");
return res;
}
Json parse_body(const fw::Request& req) {
try {
Json doc = Json::parse(req.body);
if (doc.is_object()) return doc;
} catch (const clients::body::JsonError&) {
}
throw fw::HttpError(422, "validation_error", "body must be a JSON object");
}
std::string text_field(const Json& body, const char* name, bool required,
std::size_t max = opt::OptimizeCaps::kMaxTextChars) {
const Json* v = body.find(name);
std::string out = v && v->is_string() ? v->as_string() : "";
if (required && out.empty())
throw fw::HttpError(422, "validation_error", std::string(name) + " is required");
if (out.size() > max)
throw fw::HttpError(422, "validation_error",
std::string(name) + " over " + std::to_string(max) + " characters");
return out;
}
int int_field(const Json& body, const char* name, int fallback, int lo, int hi) {
const Json* v = body.find(name);
if (!v) return fallback;
int n = 0;
if (v->is_number()) n = static_cast<int>(v->as_int());
else if (v->is_string()) {
try { n = std::stoi(v->as_string()); } catch (...) { n = lo - 1; }
} else n = lo - 1;
if (n < lo || n > hi)
throw fw::HttpError(422, "validation_error", std::string(name) + " must be " +
std::to_string(lo) + ".." + std::to_string(hi));
return n;
}
std::vector<opt::TrajectoryCall> tool_list(const std::string& csv) {
std::vector<opt::TrajectoryCall> out;
for (const std::string& name : opt::split_list(csv)) out.push_back({name, "{}"});
return out;
}
google::CloudContext require_context() {
google::CloudContext ctx = google::CloudContext::from_env();
if (!ctx.valid())
throw fw::HttpError(503, "google_not_configured",
"GOOGLE_CLOUD_PROJECT is not set or malformed");
return ctx;
}
const char kPage[] = R"OPT({% extends "base.html" %}
{% block content %}
<h1>Agent optimize</h1>
<p class="hint">Evaluate, simulate, observe and optimize agents. Pick an
action, fill the fields, send. Calls that reach Google need credentials.</p>
<section class="opt">
<label class="inline">Action
<select id="o-action"></select></label>
<p id="o-help" class="hint"></p>
<div id="o-fields"></div>
<button id="o-send" type="button" class="primary">Send</button>
<p id="o-error" class="error" role="alert"></p>
<pre id="o-out" class="opt-out"></pre>
</section>
<script src="/embedded/agent_optimize.js"></script>
{% endblock %}
)OPT";
} // namespace
void register_agent_optimize_demo(fw::Router& router, Services& services) {
if (!services.google) services.google = google::GoogleClient::from_env();
auto gclient = services.google;
fw::Environment* env = services.templates.get();
env->add_template("agent_optimize.html", kPage);
services.add_feature("agent optimize", "/agent/optimize");
auto model_for = [gclient]() {
return google::optimize::DirectModel::from_env(gclient);
};
router.get("/agent/optimize", "agent-optimize-page", [env](fw::Request req) {
(void)req;
fw::TemplateCtx ctx;
ctx["title"] = std::string("Agent optimize");
return fw::Response(200, env->render("agent_optimize.html", ctx), "text/html; charset=utf-8");
});
router.get("/agent/optimize/status", "agent-optimize-status", [gclient](fw::Request req) {
(void)req;
google::CloudContext ctx = google::CloudContext::from_env();
Json calls = Json::array();
for (const auto& c : opt::all_calls()) {
Json o = Json::object();
o.set("id", Json(c.id));
o.set("http", Json(c.http));
o.set("path", Json(c.path));
o.set("service", Json(std::string(c.service) + " " + c.version));
calls.push(o);
}
Json caps = Json::object();
caps.set("max_sim_turns", Json(opt::OptimizeCaps::kMaxSimTurns));
caps.set("max_scenarios", Json(static_cast<int>(opt::OptimizeCaps::kMaxScenarios)));
caps.set("max_failures", Json(static_cast<int>(opt::OptimizeCaps::kMaxFailures)));
caps.set("max_clusters", Json(static_cast<int>(opt::OptimizeCaps::kMaxClusters)));
caps.set("max_optimize_rounds", Json(opt::OptimizeCaps::kMaxOptimizeRounds));
caps.set("max_log_entries", Json(static_cast<int>(opt::OptimizeCaps::kMaxLogEntries)));
Json out = Json::object();
out.set("configured", Json(gclient->configured()));
out.set("project_set", Json(ctx.valid()));
out.set("location", Json(ctx.location));
out.set("calls", calls);
out.set("caps", caps);
return json_out(out);
});
// Managed evaluateInstances: safety | exact_match | trajectory.
router.post("/agent/optimize/evaluate", "agent-optimize-evaluate", [gclient](fw::Request req) {
Json body = parse_body(req);
std::string kind = text_field(body, "kind", true, 32);
Json request;
try {
if (kind == "safety") {
request = opt::EvaluationRequests::safety(text_field(body, "prediction", true));
} else if (kind == "exact_match") {
request = opt::EvaluationRequests::exact_match(
{{text_field(body, "prediction", true), text_field(body, "reference", true)}});
} else if (kind == "trajectory") {
request = opt::EvaluationRequests::trajectory(
opt::TrajectoryMetric::kExactMatch, tool_list(text_field(body, "predicted", true, 2000)),
tool_list(text_field(body, "reference", true, 2000)));
} else {
throw fw::HttpError(422, "validation_error", "kind must be safety, exact_match or trajectory");
}
} catch (const google::GoogleError& e) {
rethrow(e);
}
google::CloudContext ctx = require_context();
return guarded([&] {
opt::EvaluationApi api(gclient, ctx);
opt::MetricOutcome out = api.evaluate(request);
Json res = Json::object();
res.set("kind", Json(out.kind));
Json scores = Json::array();
for (double s : out.scores) scores.push(Json(s));
res.set("scores", scores);
res.set("explanation", Json(out.explanation));
return json_out(res);
});
});
router.post("/agent/optimize/scenarios", "agent-optimize-scenarios", [gclient](fw::Request req) {
Json body = parse_body(req);
opt::ScenarioRequest sr;
std::string instruction = text_field(body, "agent_instruction", true, opt::OptimizeCaps::kMaxInstructionChars);
Json agent = Json::object();
agent.set("agentId", Json("root"));
agent.set("agentType", Json("LlmAgent"));
agent.set("instruction", Json(instruction));
sr.agents.set("root", agent);
sr.root_agent_id = "root";
sr.count = int_field(body, "count", 5, 1, static_cast<int>(opt::OptimizeCaps::kMaxScenarios));
sr.instruction = text_field(body, "instruction", false);
google::CloudContext ctx = require_context();
return guarded([&] {
opt::EvaluationApi api(gclient, ctx);
Json arr = Json::array();
for (const opt::Scenario& s : api.generate_user_scenarios(sr)) arr.push(s.to_json());
Json out = Json::object();
out.set("scenarios", arr);
return json_out(out);
});
});
router.post("/agent/optimize/runs", "agent-optimize-runs", [gclient](fw::Request req) {
Json body = parse_body(req);
opt::ListOptions lo;
lo.page_size = int_field(body, "limit", 10, 1, 100);
lo.max_pages = 1;
google::CloudContext ctx = require_context();
return guarded([&] {
opt::EvaluationResources res(gclient, ctx);
Json out = Json::object();
out.set("evaluationRuns", res.list(opt::Collection::kEvaluationRuns, lo));
return json_out(out);
});
});
router.post("/agent/optimize/traces", "agent-optimize-traces", [gclient](fw::Request req) {
Json body = parse_body(req);
opt::TraceQuery q;
q.page_size = int_field(body, "limit", 20, 1, 100);
q.max_pages = 1;
q.max_traces = static_cast<std::size_t>(q.page_size);
q.filter = text_field(body, "filter", false, 1000);
google::CloudContext ctx = require_context();
return guarded([&] {
opt::TraceReader reader(gclient);
std::vector<opt::Trace> traces = reader.list(ctx.project, q);
Json out = opt::summarize(traces, {}).to_json();
Json ids = Json::array();
for (const opt::Trace& t : traces) ids.push(Json(t.trace_id));
out.set("trace_ids", ids);
return json_out(out);
});
});
router.post("/agent/optimize/logs", "agent-optimize-logs", [gclient](fw::Request req) {
Json body = parse_body(req);
opt::LogQuery q;
q.page_size = int_field(body, "limit", 20, 1, 100);
q.max_entries = static_cast<std::size_t>(q.page_size);
q.filter = text_field(body, "filter", false, 2000);
std::string trace_id = text_field(body, "trace_id", false, 64);
google::CloudContext ctx = require_context();
q.resource_names = {"projects/" + ctx.project};
return guarded([&] {
if (!trace_id.empty()) q.filter = opt::LogQuery::trace_filter(ctx.project, trace_id);
opt::LogReader reader(gclient);
std::vector<opt::LogEntry> entries = reader.entries(q);
Json out = opt::summarize({}, entries).to_json();
Json arr = Json::array();
for (const opt::LogEntry& e : entries) {
Json o = Json::object();
o.set("timestamp", Json(e.timestamp));
o.set("severity", Json(e.severity));
o.set("text", Json(e.text));
arr.push(o);
}
out.set("entries", arr);
return json_out(out);
});
});
// Local autorater over a pasted transcript ("user: ..." / "agent: ...").
router.post("/agent/optimize/score", "agent-optimize-score", [model_for](fw::Request req) {
Json body = parse_body(req);
std::string metric = text_field(body, "metric", false, 32);
if (metric.empty()) metric = "task_success";
if (metric != "task_success" && metric != "safety")
throw fw::HttpError(422, "validation_error", "metric must be task_success or safety");
return guarded([&] {
opt::Conversation c = opt::Conversation::parse_transcript(text_field(body, "transcript", true));
opt::Autorater rater(model_for());
opt::Score s = rater.score(metric, c, metric == "safety" ? opt::Autorater::safety_rubrics()
: opt::Autorater::task_success_rubrics());
return json_out(s.to_json());
});
});
// One failure per line.
router.post("/agent/optimize/cluster", "agent-optimize-cluster", [model_for](fw::Request req) {
Json body = parse_body(req);
int max_clusters = int_field(body, "max_clusters", 5, 1, static_cast<int>(opt::OptimizeCaps::kMaxClusters));
std::string text = text_field(body, "failures", true, opt::OptimizeCaps::kMaxTranscriptChars);
std::vector<opt::Failure> failures;
std::size_t pos = 0;
while (pos < text.size()) {
std::size_t end = text.find('\n', pos);
if (end == std::string::npos) end = text.size();
std::string line = text.substr(pos, end - pos);
pos = end + 1;
if (line.empty()) continue;
failures.push_back({"f" + std::to_string(failures.size() + 1), line, ""});
}
return guarded([&] {
opt::FailureClusterer fc(model_for());
return json_out(fc.cluster(failures, static_cast<std::size_t>(max_clusters)).to_json());
});
});
router.post("/agent/optimize/suggest", "agent-optimize-suggest", [model_for](fw::Request req) {
Json body = parse_body(req);
std::string instruction = text_field(body, "instruction", true, opt::OptimizeCaps::kMaxInstructionChars);
std::string text = text_field(body, "failures", true, opt::OptimizeCaps::kMaxTranscriptChars);
std::vector<opt::Failure> failures;
std::size_t pos = 0;
while (pos < text.size()) {
std::size_t end = text.find('\n', pos);
if (end == std::string::npos) end = text.size();
std::string line = text.substr(pos, end - pos);
pos = end + 1;
if (!line.empty()) failures.push_back({"f" + std::to_string(failures.size() + 1), line, ""});
}
return guarded([&] {
auto model = model_for();
opt::ClusterReport clusters = opt::FailureClusterer(model).cluster(failures, 5);
Json out = opt::InstructionOptimizer(model).suggest(instruction, clusters, failures).to_json();
out.set("clusters", clusters.to_json());
return json_out(out);
});
});
// Simulated user against a ModelAgent built from the instruction.
router.post("/agent/optimize/simulate", "agent-optimize-simulate", [model_for](fw::Request req) {
Json body = parse_body(req);
std::string instruction = text_field(body, "instruction", true, opt::OptimizeCaps::kMaxInstructionChars);
opt::Scenario sc;
sc.title = text_field(body, "title", false, 200);
sc.starting_prompt = text_field(body, "starting_prompt", true);
sc.plan = text_field(body, "plan", true);
opt::SimulationConfig cfg;
cfg.max_turns = int_field(body, "max_turns", opt::OptimizeCaps::kDefaultSimTurns, 1, opt::OptimizeCaps::kMaxSimTurns);
return guarded([&] {
auto model = model_for();
opt::ModelAgent agent(model, instruction);
opt::VirtualToolbox tools;
opt::Simulator sim(model, model);
return json_out(sim.run_and_score(sc, agent, tools, cfg).to_json());
});
});
}
} // namespace molo::example::app