Merge optimize module (evaluation, simulation, observability, optimizer)

This commit is contained in:
2026-10-24 04:41:09 +02:00
55 changed files with 5099 additions and 3 deletions
@@ -0,0 +1,30 @@
// Route: POST /agent/optimize/cluster (local failure clustering)
#include "support/GoogleRig.hpp"
using namespace molo::example::test;
int main() {
const char* json = "application/json";
GoogleRig keyed;
Fixture f(keyed.client);
check(f.client->post("/agent/optimize/cluster", "{}", json).status == 422,
"missing failures is 422");
check(f.client->post("/agent/optimize/cluster",
"{\"failures\":\"a\\nb\",\"max_clusters\":11}", json).status == 422,
"max_clusters over 10 is 422");
check(f.client->post("/agent/optimize/cluster",
"{\"failures\":\"a\",\"max_clusters\":0}", json).status == 422,
"max_clusters 0 is 422");
check(f.client->post("/agent/optimize/cluster", "{\"failures\":\"\\n\\n\"}", json).status == 422,
"only blank lines is 422");
check(keyed.mock->send_count() == 0, "rejected input never reached Google");
check(f.client->get("/agent/optimize/cluster").status == 405, "get is 405");
GoogleRig none(false);
Fixture g(none.client);
auto res = g.client->post("/agent/optimize/cluster", "{\"failures\":\"timeout\\nwrong tool\"}", json);
check(res.status == 503 && contains(res.body, "google_not_configured"),
"no credential is a loud 503");
check(none.mock->send_count() == 0, "no network without a credential");
return finish("agent/optimize/cluster");
}
@@ -0,0 +1,47 @@
// Route: POST /agent/optimize/evaluate (managed evaluateInstances)
#include <cstdlib>
#include "support/GoogleRig.hpp"
using namespace molo::example::test;
int main() {
setenv("GOOGLE_CLOUD_PROJECT", "test-proj", 1);
const char* json = "application/json";
GoogleRig keyed;
Fixture f(keyed.client);
check(f.client->post("/agent/optimize/evaluate", "{\"kind\":\"voodoo\"}", json).status == 422,
"unknown kind is 422");
check(f.client->post("/agent/optimize/evaluate", "{\"kind\":\"safety\"}", json).status == 422,
"missing prediction is 422");
check(f.client->post("/agent/optimize/evaluate", "not json", json).status == 422,
"non-JSON body is 422");
check(f.client->post("/agent/optimize/evaluate",
"{\"kind\":\"safety\",\"prediction\":\"" + std::string(20001, 'x') + "\"}", json)
.status == 422,
"prediction over the 20000 character cap is 422");
check(f.client->post("/agent/optimize/evaluate", "{\"kind\":\"trajectory\",\"predicted\":\"a\"}", json)
.status == 422,
"trajectory without reference is 422");
check(keyed.mock->send_count() == 0, "rejected input never reached Google");
check(f.client->get("/agent/optimize/evaluate").status == 405, "get is 405");
// Credential present but the project id is malformed: loud 503, no traffic.
setenv("GOOGLE_CLOUD_PROJECT", "Not A Project!", 1);
auto bad = f.client->post("/agent/optimize/evaluate",
"{\"kind\":\"safety\",\"prediction\":\"hi\"}", json);
check(bad.status == 503 && contains(bad.body, "google_not_configured"),
"malformed project is a 503");
check(keyed.mock->send_count() == 0, "side effect: no traffic without a valid project");
setenv("GOOGLE_CLOUD_PROJECT", "test-proj", 1);
GoogleRig none(false);
Fixture g(none.client);
auto res = g.client->post("/agent/optimize/evaluate",
"{\"kind\":\"safety\",\"prediction\":\"hi\"}", json);
check(res.status == 503 && contains(res.body, "google_not_configured"),
"no credential is a loud 503");
check(none.mock->send_count() == 0, "no network without a credential");
return finish("agent/optimize/evaluate");
}
@@ -0,0 +1,30 @@
// Route: POST /agent/optimize/logs (Cloud Logging entries:list)
#include <cstdlib>
#include "support/GoogleRig.hpp"
using namespace molo::example::test;
int main() {
setenv("GOOGLE_CLOUD_PROJECT", "test-proj", 1);
const char* json = "application/json";
GoogleRig keyed;
Fixture f(keyed.client);
check(f.client->post("/agent/optimize/logs", "{\"limit\":101}", json).status == 422,
"limit over 100 is 422");
check(f.client->post("/agent/optimize/logs",
"{\"filter\":\"" + std::string(2001, 'x') + "\"}", json).status == 422,
"filter over 2000 characters is 422");
check(f.client->post("/agent/optimize/logs", "{\"trace_id\":\"not-hex\"}", json).status == 422,
"malformed trace id is 422");
check(keyed.mock->send_count() == 0, "rejected input never reached Google");
check(f.client->get("/agent/optimize/logs").status == 405, "get is 405");
GoogleRig none(false);
Fixture g(none.client);
auto res = g.client->post("/agent/optimize/logs", "{\"limit\":5}", json);
check(res.status == 503 && contains(res.body, "google_not_configured"),
"no credential is a loud 503");
check(none.mock->send_count() == 0, "no network without a credential");
return finish("agent/optimize/logs");
}
@@ -0,0 +1,37 @@
// Route: GET /agent/optimize and /agent/optimize/status (page, script, call table)
#include <cstdlib>
#include "support/GoogleRig.hpp"
using namespace molo::example::test;
int main() {
setenv("GOOGLE_CLOUD_PROJECT", "test-proj", 1);
GoogleRig rig(false);
Fixture f(rig.client);
auto page = f.client->get("/agent/optimize");
check(page.status == 200, "page status 200 without a credential");
check(contains(page.headers.get("Content-Type"), "text/html"), "html");
check(contains(page.body, "width=device-width"), "responsive viewport");
check(contains(page.body, "id=\"o-action\""), "action dropdown (no wide button group)");
check(contains(page.body, "/embedded/agent_optimize.js"), "script linked");
auto js = f.client->get("/embedded/agent_optimize.js");
check(js.status == 200 && contains(js.body, "/agent/optimize/"), "script served from the bundle");
auto st = f.client->get("/agent/optimize/status");
check(st.status == 200, "status 200 without a credential");
check(contains(st.headers.get("Content-Type"), "json"), "status is json");
check(contains(st.body, "\"configured\":false"), "reports no credential");
check(contains(st.body, "\"project_set\":true"), "reports the project is set, never its value");
check(!contains(st.body, "test-proj"), "project id never echoed");
check(contains(st.body, "aiplatform.projects.locations.evaluateInstances"),
"call table listed");
check(contains(st.body, "cloudtrace.projects.traces.list"), "trace read listed");
check(contains(st.body, "\"max_sim_turns\":12"), "caps listed");
check(f.client->post("/agent/optimize").status == 405, "post to the page is 405");
check(f.client->post("/agent/optimize/status").status == 405, "post to status is 405");
check(rig.mock->send_count() == 0, "side effect: GET routes make no Google call");
return finish("agent/optimize/page");
}
@@ -0,0 +1,28 @@
// Route: POST /agent/optimize/runs (read-only list of evaluation runs)
#include <cstdlib>
#include "support/GoogleRig.hpp"
using namespace molo::example::test;
int main() {
setenv("GOOGLE_CLOUD_PROJECT", "test-proj", 1);
const char* json = "application/json";
GoogleRig keyed;
Fixture f(keyed.client);
check(f.client->post("/agent/optimize/runs", "{\"limit\":0}", json).status == 422,
"limit 0 is 422");
check(f.client->post("/agent/optimize/runs", "{\"limit\":101}", json).status == 422,
"limit over the page cap of 100 is 422");
check(f.client->post("/agent/optimize/runs", "[]", json).status == 422, "array body is 422");
check(keyed.mock->send_count() == 0, "rejected input never reached Google");
check(f.client->get("/agent/optimize/runs").status == 405, "get is 405");
GoogleRig none(false);
Fixture g(none.client);
auto res = g.client->post("/agent/optimize/runs", "{\"limit\":5}", json);
check(res.status == 503 && contains(res.body, "google_not_configured"),
"no credential is a loud 503");
check(none.mock->send_count() == 0, "no network without a credential");
return finish("agent/optimize/runs");
}
@@ -0,0 +1,35 @@
// Route: POST /agent/optimize/scenarios (generateUserScenarios)
#include <cstdlib>
#include "support/GoogleRig.hpp"
using namespace molo::example::test;
int main() {
setenv("GOOGLE_CLOUD_PROJECT", "test-proj", 1);
const char* json = "application/json";
GoogleRig keyed;
Fixture f(keyed.client);
check(f.client->post("/agent/optimize/scenarios", "{}", json).status == 422,
"missing agent_instruction is 422");
check(f.client->post("/agent/optimize/scenarios",
"{\"agent_instruction\":\"help\",\"count\":101}", json).status == 422,
"count over the Google ceiling of 100 is 422");
check(f.client->post("/agent/optimize/scenarios",
"{\"agent_instruction\":\"help\",\"count\":0}", json).status == 422,
"count 0 is 422");
check(f.client->post("/agent/optimize/scenarios",
"{\"agent_instruction\":\"help\",\"count\":\"many\"}", json).status == 422,
"non-numeric count is 422");
check(keyed.mock->send_count() == 0, "rejected input never reached Google");
check(f.client->get("/agent/optimize/scenarios").status == 405, "get is 405");
GoogleRig none(false);
Fixture g(none.client);
auto res = g.client->post("/agent/optimize/scenarios",
"{\"agent_instruction\":\"help\",\"count\":3}", json);
check(res.status == 503 && contains(res.body, "google_not_configured"),
"no credential is a loud 503");
check(none.mock->send_count() == 0, "no network without a credential");
return finish("agent/optimize/scenarios");
}
@@ -0,0 +1,34 @@
// Route: POST /agent/optimize/score (local multi-turn autorater)
#include "support/GoogleRig.hpp"
using namespace molo::example::test;
int main() {
const char* json = "application/json";
GoogleRig keyed;
Fixture f(keyed.client);
check(f.client->post("/agent/optimize/score", "{}", json).status == 422,
"missing transcript is 422");
check(f.client->post("/agent/optimize/score",
"{\"transcript\":\"no role prefix\"}", json).status == 422,
"transcript without roles is 422");
check(f.client->post("/agent/optimize/score",
"{\"transcript\":\"user: hi\\nagent: hello\",\"metric\":\"vibes\"}", json)
.status == 422,
"unknown metric is 422");
std::string many;
for (int i = 0; i < 51; ++i) many += "user: x\\n";
check(f.client->post("/agent/optimize/score", "{\"transcript\":\"" + many + "\"}", json).status == 422,
"more than 50 turns is 422");
check(keyed.mock->send_count() == 0, "rejected input never reached Google");
check(f.client->get("/agent/optimize/score").status == 405, "get is 405");
GoogleRig none(false);
Fixture g(none.client);
auto res = g.client->post("/agent/optimize/score",
"{\"transcript\":\"user: hi\\nagent: hello\"}", json);
check(res.status == 503 && contains(res.body, "google_not_configured"),
"no credential is a loud 503");
check(none.mock->send_count() == 0, "no network without a credential");
return finish("agent/optimize/score");
}
@@ -0,0 +1,32 @@
// Route: POST /agent/optimize/simulate (simulated user vs a model agent, scored)
#include "support/GoogleRig.hpp"
using namespace molo::example::test;
int main() {
const char* json = "application/json";
GoogleRig keyed;
Fixture f(keyed.client);
const std::string ok =
"\"instruction\":\"help\",\"starting_prompt\":\"hi\",\"plan\":\"ask for a refund\"";
check(f.client->post("/agent/optimize/simulate", "{" + ok + ",\"max_turns\":13}", json).status == 422,
"max_turns over 12 is 422");
check(f.client->post("/agent/optimize/simulate", "{" + ok + ",\"max_turns\":0}", json).status == 422,
"max_turns 0 is 422");
check(f.client->post("/agent/optimize/simulate",
"{\"instruction\":\"help\",\"starting_prompt\":\"hi\"}", json).status == 422,
"missing plan is 422");
check(f.client->post("/agent/optimize/simulate", "{\"plan\":\"p\",\"starting_prompt\":\"hi\"}", json)
.status == 422,
"missing instruction is 422");
check(keyed.mock->send_count() == 0, "rejected input never reached Google");
check(f.client->get("/agent/optimize/simulate").status == 405, "get is 405");
GoogleRig none(false);
Fixture g(none.client);
auto res = g.client->post("/agent/optimize/simulate", "{" + ok + "}", json);
check(res.status == 503 && contains(res.body, "google_not_configured"),
"no credential is a loud 503");
check(none.mock->send_count() == 0, "no network without a credential");
return finish("agent/optimize/simulate");
}
@@ -0,0 +1,29 @@
// Route: POST /agent/optimize/suggest (cluster failures, suggest an instruction)
#include "support/GoogleRig.hpp"
using namespace molo::example::test;
int main() {
const char* json = "application/json";
GoogleRig keyed;
Fixture f(keyed.client);
check(f.client->post("/agent/optimize/suggest", "{\"failures\":\"x\"}", json).status == 422,
"missing instruction is 422");
check(f.client->post("/agent/optimize/suggest", "{\"instruction\":\"be nice\"}", json).status == 422,
"missing failures is 422");
check(f.client->post("/agent/optimize/suggest",
"{\"instruction\":\"" + std::string(8001, 'x') + "\",\"failures\":\"x\"}", json)
.status == 422,
"instruction over the 8000 character cap is 422");
check(keyed.mock->send_count() == 0, "rejected input never reached Google");
check(f.client->get("/agent/optimize/suggest").status == 405, "get is 405");
GoogleRig none(false);
Fixture g(none.client);
auto res = g.client->post("/agent/optimize/suggest",
"{\"instruction\":\"be nice\",\"failures\":\"rude reply\"}", json);
check(res.status == 503 && contains(res.body, "google_not_configured"),
"no credential is a loud 503");
check(none.mock->send_count() == 0, "no network without a credential");
return finish("agent/optimize/suggest");
}
@@ -0,0 +1,30 @@
// Route: POST /agent/optimize/traces (Cloud Trace v1 list, dashboard numbers)
#include <cstdlib>
#include "support/GoogleRig.hpp"
using namespace molo::example::test;
int main() {
setenv("GOOGLE_CLOUD_PROJECT", "test-proj", 1);
const char* json = "application/json";
GoogleRig keyed;
Fixture f(keyed.client);
check(f.client->post("/agent/optimize/traces", "{\"limit\":0}", json).status == 422,
"limit 0 is 422");
check(f.client->post("/agent/optimize/traces", "{\"limit\":101}", json).status == 422,
"limit over 100 is 422");
check(f.client->post("/agent/optimize/traces",
"{\"filter\":\"" + std::string(1001, 'x') + "\"}", json).status == 422,
"filter over 1000 characters is 422");
check(keyed.mock->send_count() == 0, "rejected input never reached Google");
check(f.client->get("/agent/optimize/traces").status == 405, "get is 405");
GoogleRig none(false);
Fixture g(none.client);
auto res = g.client->post("/agent/optimize/traces", "{\"limit\":5}", json);
check(res.status == 503 && contains(res.body, "google_not_configured"),
"no credential is a loud 503");
check(none.mock->send_count() == 0, "no network without a credential");
return finish("agent/optimize/traces");
}