#include "../AgentDemos.hpp" #include #include "../../fw/Router.hpp" #include "../../fw/Templates.hpp" #include "google/credentials.h" #include "google/optimize/autorater.h" #include "google/optimize/calls.h" #include "google/optimize/clusterer.h" #include "google/optimize/model_factory.h" #include "google/optimize/eval_requests.h" #include "google/optimize/eval_resources.h" #include "google/optimize/evaluation_api.h" #include "google/optimize/observability.h" #include "google/optimize/operations_api.h" #include "google/optimize/telemetry_writer.h" #include "google/optimize/optimizer.h" #include "google/optimize/simulator.h" namespace molo::example::app { // Agent Platform: optimize (evaluation, simulation, observability, optimizer) // over src/google/optimize. Routes under /agent/optimize. GET answers without // a credential; everything that calls Google is a POST (503 // google_not_configured without credentials, 422 on bad input). // Gap: live OAuth-only reads (runs, traces, logs) are verified by discovery // only, see src/google/optimize/SPEC.md. namespace { namespace opt = google::optimize; using Json = clients::body::Json; [[noreturn]] void rethrow(const google::GoogleError& e) { if (e.http_status() == 503 && e.status() == "NOT_CONFIGURED") throw fw::HttpError(503, "google_not_configured", e.message()); if (e.service().empty() && e.status() == "INVALID_ARGUMENT") throw fw::HttpError(422, "validation_error", e.message()); throw fw::HttpError(502, "google_error", e.status() + ": " + e.message()); } template auto guarded(F&& f) -> decltype(f()) { try { return f(); } catch (const google::GoogleError& e) { rethrow(e); } } fw::Response json_out(const Json& doc) { fw::Response res = fw::JsonResponse(doc.dump(), 200); res.headers.set("Cache-Control", "no-store"); return res; } Json parse_body(const fw::Request& req) { try { Json doc = Json::parse(req.body); if (doc.is_object()) return doc; } catch (const clients::body::JsonError&) { } throw fw::HttpError(422, "validation_error", "body must be a JSON object"); } std::string text_field(const Json& body, const char* name, bool required, std::size_t max = opt::OptimizeCaps::kMaxTextChars) { const Json* v = body.find(name); std::string out = v && v->is_string() ? v->as_string() : ""; if (required && out.empty()) throw fw::HttpError(422, "validation_error", std::string(name) + " is required"); if (out.size() > max) throw fw::HttpError(422, "validation_error", std::string(name) + " over " + std::to_string(max) + " characters"); return out; } int int_field(const Json& body, const char* name, int fallback, int lo, int hi) { const Json* v = body.find(name); if (!v) return fallback; int n = 0; if (v->is_number()) n = static_cast(v->as_int()); else if (v->is_string()) { try { n = std::stoi(v->as_string()); } catch (...) { n = lo - 1; } } else n = lo - 1; if (n < lo || n > hi) throw fw::HttpError(422, "validation_error", std::string(name) + " must be " + std::to_string(lo) + ".." + std::to_string(hi)); return n; } std::vector tool_list(const std::string& csv) { std::vector out; for (const std::string& name : opt::split_list(csv)) out.push_back({name, "{}"}); return out; } google::CloudContext require_context() { google::CloudContext ctx = google::CloudContext::from_env(); if (!ctx.valid()) throw fw::HttpError(503, "google_not_configured", "GOOGLE_CLOUD_PROJECT is not set or malformed"); return ctx; } const char kPage[] = R"OPT({% extends "base.html" %} {% block content %}

Agent optimize

Evaluate, simulate, observe and optimize agents. Pick an action, fill the fields, send. Calls that reach Google need credentials.


{% endblock %} )OPT"; } // namespace void register_agent_optimize_demo(fw::Router& router, Services& services) { if (!services.google) services.google = google::GoogleClient::from_env(); auto gclient = services.google; fw::Environment* env = services.templates.get(); env->add_template("agent_optimize.html", kPage); services.add_feature("agent optimize", "/agent/optimize"); auto model_for = [gclient]() { return google::optimize::make_gemini_model(gclient); }; router.get("/agent/optimize", "agent-optimize-page", [env](fw::Request req) { (void)req; fw::TemplateCtx ctx; ctx["title"] = std::string("Agent optimize"); return fw::Response(200, env->render("agent_optimize.html", ctx), "text/html; charset=utf-8"); }); router.get("/agent/optimize/status", "agent-optimize-status", [gclient](fw::Request req) { (void)req; google::CloudContext ctx = google::CloudContext::from_env(); Json calls = Json::array(); for (const auto& c : opt::all_calls()) { Json o = Json::object(); o.set("id", Json(c.id)); o.set("http", Json(c.http)); o.set("path", Json(c.path)); o.set("service", Json(std::string(c.service) + " " + c.version)); calls.push(o); } Json caps = Json::object(); caps.set("max_sim_turns", Json(opt::OptimizeCaps::kMaxSimTurns)); caps.set("max_scenarios", Json(static_cast(opt::OptimizeCaps::kMaxScenarios))); caps.set("max_failures", Json(static_cast(opt::OptimizeCaps::kMaxFailures))); caps.set("max_clusters", Json(static_cast(opt::OptimizeCaps::kMaxClusters))); caps.set("max_optimize_rounds", Json(opt::OptimizeCaps::kMaxOptimizeRounds)); caps.set("max_log_entries", Json(static_cast(opt::OptimizeCaps::kMaxLogEntries))); Json out = Json::object(); out.set("configured", Json(gclient->configured())); out.set("project_set", Json(ctx.valid())); out.set("location", Json(ctx.location)); out.set("calls", calls); out.set("caps", caps); return json_out(out); }); // Managed evaluateInstances: safety | exact_match | trajectory. router.post("/agent/optimize/evaluate", "agent-optimize-evaluate", [gclient](fw::Request req) { Json body = parse_body(req); std::string kind = text_field(body, "kind", true, 32); Json request; try { if (kind == "safety") { request = opt::EvaluationRequests::safety(text_field(body, "prediction", true)); } else if (kind == "exact_match") { request = opt::EvaluationRequests::exact_match( {{text_field(body, "prediction", true), text_field(body, "reference", true)}}); } else if (kind == "trajectory") { request = opt::EvaluationRequests::trajectory( opt::TrajectoryMetric::kExactMatch, tool_list(text_field(body, "predicted", true, 2000)), tool_list(text_field(body, "reference", true, 2000))); } else { throw fw::HttpError(422, "validation_error", "kind must be safety, exact_match or trajectory"); } } catch (const google::GoogleError& e) { rethrow(e); } google::CloudContext ctx = require_context(); return guarded([&] { opt::EvaluationApi api(gclient, ctx); opt::MetricOutcome out = api.evaluate(request); Json res = Json::object(); res.set("kind", Json(out.kind)); Json scores = Json::array(); for (double s : out.scores) scores.push(Json(s)); res.set("scores", scores); res.set("explanation", Json(out.explanation)); return json_out(res); }); }); router.post("/agent/optimize/scenarios", "agent-optimize-scenarios", [gclient](fw::Request req) { Json body = parse_body(req); opt::ScenarioRequest sr; std::string instruction = text_field(body, "agent_instruction", true, opt::OptimizeCaps::kMaxInstructionChars); Json agent = Json::object(); agent.set("agentId", Json("root")); agent.set("agentType", Json("LlmAgent")); agent.set("instruction", Json(instruction)); sr.agents.set("root", agent); sr.root_agent_id = "root"; sr.count = int_field(body, "count", 5, 1, static_cast(opt::OptimizeCaps::kMaxScenarios)); sr.instruction = text_field(body, "instruction", false); google::CloudContext ctx = require_context(); return guarded([&] { opt::EvaluationApi api(gclient, ctx); Json arr = Json::array(); for (const opt::Scenario& s : api.generate_user_scenarios(sr)) arr.push(s.to_json()); Json out = Json::object(); out.set("scenarios", arr); return json_out(out); }); }); router.post("/agent/optimize/runs", "agent-optimize-runs", [gclient](fw::Request req) { Json body = parse_body(req); opt::ListOptions lo; lo.page_size = int_field(body, "limit", 10, 1, 100); lo.max_pages = 1; google::CloudContext ctx = require_context(); return guarded([&] { opt::EvaluationResources res(gclient, ctx); Json out = Json::object(); out.set("evaluationRuns", res.list(opt::Collection::kEvaluationRuns, lo)); return json_out(out); }); }); router.post("/agent/optimize/traces", "agent-optimize-traces", [gclient](fw::Request req) { Json body = parse_body(req); opt::TraceQuery q; q.page_size = int_field(body, "limit", 20, 1, 100); q.max_pages = 1; q.max_traces = static_cast(q.page_size); q.filter = text_field(body, "filter", false, 1000); google::CloudContext ctx = require_context(); return guarded([&] { opt::TraceReader reader(gclient); std::vector traces = reader.list(ctx.project, q); Json out = opt::summarize(traces, {}).to_json(); Json ids = Json::array(); for (const opt::Trace& t : traces) ids.push(Json(t.trace_id)); out.set("trace_ids", ids); return json_out(out); }); }); router.post("/agent/optimize/logs", "agent-optimize-logs", [gclient](fw::Request req) { Json body = parse_body(req); opt::LogQuery q; q.page_size = int_field(body, "limit", 20, 1, 100); q.max_entries = static_cast(q.page_size); q.filter = text_field(body, "filter", false, 2000); std::string trace_id = text_field(body, "trace_id", false, 64); google::CloudContext ctx = require_context(); q.resource_names = {"projects/" + ctx.project}; return guarded([&] { if (!trace_id.empty()) q.filter = opt::LogQuery::trace_filter(ctx.project, trace_id); opt::LogReader reader(gclient); std::vector entries = reader.entries(q); Json out = opt::summarize({}, entries).to_json(); Json arr = Json::array(); for (const opt::LogEntry& e : entries) { Json o = Json::object(); o.set("timestamp", Json(e.timestamp)); o.set("severity", Json(e.severity)); o.set("text", Json(e.text)); arr.push(o); } out.set("entries", arr); return json_out(out); }); }); // generateSyntheticData: `request` is the Discovery request object (or its // JSON text); top-level fields are checked against the call table. router.post("/agent/optimize/synthetic", "agent-optimize-synthetic", [gclient](fw::Request req) { Json body = parse_body(req); const Json* r = body.find("request"); Json request; if (r && r->is_object()) request = *r; else if (r && r->is_string()) { try { request = Json::parse(r->as_string()); } catch (const clients::body::JsonError&) {} } if (!request.is_object() || request.as_object().empty()) throw fw::HttpError(422, "validation_error", "request must be a non-empty JSON object"); google::CloudContext ctx = require_context(); return guarded([&] { return json_out(opt::EvaluationApi(gclient, ctx).generate_synthetic_data(request)); }); }); // Operations of online evaluators (get, list, server-side wait, cancel; // delete is a DELETE route and removes the operation record only). auto op_json = [](const google::Operation& op) { Json o = Json::object(); o.set("name", Json(op.name)); o.set("done", Json(op.done)); if (op.failed()) o.set("error_message", Json(op.error_message)); return o; }; router.post("/agent/optimize/operations/list", "agent-optimize-ops-list", [gclient, op_json](fw::Request req) { Json body = parse_body(req); std::string evaluator = text_field(body, "evaluator", true, 512); int limit = int_field(body, "limit", 20, 1, 100); google::CloudContext ctx = require_context(); return guarded([&] { opt::OperationPage page = opt::OnlineEvaluatorOperations(gclient, ctx).list(evaluator, "", limit); Json arr = Json::array(); for (const google::Operation& op : page.operations) arr.push(op_json(op)); Json out = Json::object(); out.set("operations", arr); out.set("count", Json(static_cast(page.operations.size()))); out.set("truncated", Json(page.truncated)); out.set("next_page_token", Json(page.next_page_token)); return json_out(out); }); }); router.post("/agent/optimize/operations/get", "agent-optimize-ops-get", [gclient](fw::Request req) { Json body = parse_body(req); std::string name = text_field(body, "name", true, 512); google::CloudContext ctx = require_context(); return guarded([&] { return json_out(opt::OnlineEvaluatorOperations(gclient, ctx).get(name)); }); }); router.post("/agent/optimize/operations/wait", "agent-optimize-ops-wait", [gclient, op_json](fw::Request req) { Json body = parse_body(req); std::string name = text_field(body, "name", true, 512); int seconds = int_field(body, "timeout_seconds", 30, 1, opt::OptimizeCaps::kMaxServerWaitSeconds); google::CloudContext ctx = require_context(); return guarded([&] { return json_out(op_json(opt::OnlineEvaluatorOperations(gclient, ctx).wait_server(name, seconds))); }); }); router.post("/agent/optimize/operations/cancel", "agent-optimize-ops-cancel", [gclient](fw::Request req) { Json body = parse_body(req); std::string name = text_field(body, "name", true, 512); google::CloudContext ctx = require_context(); return guarded([&] { opt::OnlineEvaluatorOperations(gclient, ctx).cancel(name); Json out = Json::object(); out.set("cancel_requested", Json(true)); return json_out(out); }); }); router.del("/agent/optimize/operations", "agent-optimize-ops-delete", [gclient](fw::Request req) { std::string name = req.query_or("name", ""); if (name.empty() || !opt::OnlineEvaluatorOperations::is_operation_name(name)) throw fw::HttpError(422, "validation_error", "name must be an online evaluator operation name"); google::CloudContext ctx = require_context(); return guarded([&] { opt::OnlineEvaluatorOperations(gclient, ctx).remove(name); Json out = Json::object(); out.set("deleted", Json(true)); return json_out(out); }); }); // entries:write. A dry run (nothing stored) unless confirm is "write". router.post("/agent/optimize/logs/write", "agent-optimize-logs-write", [gclient](fw::Request req) { Json body = parse_body(req); std::string log_id = text_field(body, "log_id", true, 100); std::string text = text_field(body, "text", true, 4000); std::string confirm = text_field(body, "confirm", false, 16); if (!google::valid_segment(log_id)) throw fw::HttpError(422, "validation_error", "log_id has invalid characters"); google::CloudContext ctx = require_context(); bool real = confirm == "write"; Json entry = Json::object(); entry.set("textPayload", Json(text)); Json entries = Json::array(); entries.push(entry); Json resource = Json::object(); resource.set("type", Json("global")); Json request = Json::object(); request.set("logName", Json("projects/" + ctx.project + "/logs/" + log_id)); request.set("resource", resource); request.set("entries", entries); request.set("dryRun", Json(!real)); return guarded([&] { opt::LogWriter(gclient).write(request); Json out = Json::object(); out.set("dry_run", Json(!real)); out.set("written", Json(real)); return json_out(out); }); }); // Local autorater over a pasted transcript ("user: ..." / "agent: ..."). router.post("/agent/optimize/score", "agent-optimize-score", [model_for](fw::Request req) { Json body = parse_body(req); std::string metric = text_field(body, "metric", false, 32); if (metric.empty()) metric = "task_success"; if (metric != "task_success" && metric != "safety") throw fw::HttpError(422, "validation_error", "metric must be task_success or safety"); return guarded([&] { opt::Conversation c = opt::Conversation::parse_transcript(text_field(body, "transcript", true)); opt::Autorater rater(model_for()); opt::Score s = rater.score(metric, c, metric == "safety" ? opt::Autorater::safety_rubrics() : opt::Autorater::task_success_rubrics()); return json_out(s.to_json()); }); }); // One failure per line. router.post("/agent/optimize/cluster", "agent-optimize-cluster", [model_for](fw::Request req) { Json body = parse_body(req); int max_clusters = int_field(body, "max_clusters", 5, 1, static_cast(opt::OptimizeCaps::kMaxClusters)); std::string text = text_field(body, "failures", true, opt::OptimizeCaps::kMaxTranscriptChars); std::vector failures; std::size_t pos = 0; while (pos < text.size()) { std::size_t end = text.find('\n', pos); if (end == std::string::npos) end = text.size(); std::string line = text.substr(pos, end - pos); pos = end + 1; if (line.empty()) continue; failures.push_back({"f" + std::to_string(failures.size() + 1), line, ""}); } return guarded([&] { opt::FailureClusterer fc(model_for()); return json_out(fc.cluster(failures, static_cast(max_clusters)).to_json()); }); }); router.post("/agent/optimize/suggest", "agent-optimize-suggest", [model_for](fw::Request req) { Json body = parse_body(req); std::string instruction = text_field(body, "instruction", true, opt::OptimizeCaps::kMaxInstructionChars); std::string text = text_field(body, "failures", true, opt::OptimizeCaps::kMaxTranscriptChars); std::vector failures; std::size_t pos = 0; while (pos < text.size()) { std::size_t end = text.find('\n', pos); if (end == std::string::npos) end = text.size(); std::string line = text.substr(pos, end - pos); pos = end + 1; if (!line.empty()) failures.push_back({"f" + std::to_string(failures.size() + 1), line, ""}); } return guarded([&] { auto model = model_for(); opt::ClusterReport clusters = opt::FailureClusterer(model).cluster(failures, 5); Json out = opt::InstructionOptimizer(model).suggest(instruction, clusters, failures).to_json(); out.set("clusters", clusters.to_json()); return json_out(out); }); }); // Simulated user against a ModelAgent built from the instruction. router.post("/agent/optimize/simulate", "agent-optimize-simulate", [model_for](fw::Request req) { Json body = parse_body(req); std::string instruction = text_field(body, "instruction", true, opt::OptimizeCaps::kMaxInstructionChars); opt::Scenario sc; sc.title = text_field(body, "title", false, 200); sc.starting_prompt = text_field(body, "starting_prompt", true); sc.plan = text_field(body, "plan", true); opt::SimulationConfig cfg; cfg.max_turns = int_field(body, "max_turns", opt::OptimizeCaps::kDefaultSimTurns, 1, opt::OptimizeCaps::kMaxSimTurns); return guarded([&] { auto model = model_for(); opt::ModelAgent agent(model, instruction); opt::VirtualToolbox tools; opt::Simulator sim(model, model); return json_out(sim.run_and_score(sc, agent, tools, cfg).to_json()); }); }); } } // namespace molo::example::app