feat(clients): optimize: evaluation, simulation, observability and optimizer on a discovery-checked call table
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,339 @@
|
||||
// Local logic against a fake LanguageModel: autorater, simulator, clusterer,
|
||||
// optimizer, JSON extraction. Deterministic, no network.
|
||||
#include "framework.h"
|
||||
#include "google/errors.h"
|
||||
#include "google/optimize/autorater.h"
|
||||
#include "google/optimize/clusterer.h"
|
||||
#include "google/optimize/model_json.h"
|
||||
#include "google/optimize/optimizer.h"
|
||||
#include "google/optimize/simulator.h"
|
||||
#include "fakes.h"
|
||||
|
||||
using namespace molo::google;
|
||||
using namespace molo::google::optimize;
|
||||
using molo::clients::body::Json;
|
||||
using optimize_test::FakeModel;
|
||||
using optimize_test::reply_call;
|
||||
using optimize_test::reply_text;
|
||||
|
||||
namespace {
|
||||
bool is_invalid(const std::function<void()>& f) {
|
||||
try {
|
||||
f();
|
||||
} catch (const GoogleError& e) {
|
||||
return e.http_status() == 400;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
Conversation sample() {
|
||||
return Conversation::parse_transcript("user: refund please\nagent: done, refunded 10 EUR");
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(optimize_json, extract_tolerates_fences_and_prose) {
|
||||
Json a = JsonModelCall::extract("sure!\n```json\n{\"a\":{\"b\":\"}\"},\"c\":1}\n```\nbye");
|
||||
ASSERT_TRUE(a.is_object());
|
||||
ASSERT_EQ(a.find("c")->as_int(), 1);
|
||||
ASSERT_TRUE(JsonModelCall::extract("no json here").is_null());
|
||||
ASSERT_TRUE(JsonModelCall::extract("{broken").is_null());
|
||||
Json second = JsonModelCall::extract("{nope} then {\"ok\":true}");
|
||||
ASSERT_TRUE(second.is_object() && second.find("ok") != nullptr);
|
||||
}
|
||||
|
||||
TEST(optimize_json, retries_once_then_fails_loud) {
|
||||
auto model = FakeModel::texts({"not json", "still not json"});
|
||||
JsonModelCall call(model);
|
||||
try {
|
||||
call.ask("s", "u");
|
||||
FAIL("expected BAD_RESPONSE");
|
||||
} catch (const GoogleError& e) {
|
||||
ASSERT_EQ(e.status(), "BAD_RESPONSE");
|
||||
}
|
||||
ASSERT_EQ(call.calls(), 2); // one try plus one retry, no more
|
||||
auto ok = FakeModel::texts({"oops", "{\"x\":1}"});
|
||||
ASSERT_EQ(JsonModelCall(ok).ask("s", "u").find("x")->as_int(), 1);
|
||||
}
|
||||
|
||||
TEST(optimize_json, request_asks_for_json_deterministically) {
|
||||
auto model = FakeModel::texts({"{}"});
|
||||
JsonModelCall(model, "m1").ask("sys", "user");
|
||||
const GenerateRequest& r = model->requests[0];
|
||||
ASSERT_EQ(r.model, "m1");
|
||||
ASSERT_EQ(r.config.response_mime_type, "application/json");
|
||||
ASSERT_EQ(*r.config.temperature, 0.0);
|
||||
ASSERT_EQ(r.system->parts[0].text, "sys");
|
||||
}
|
||||
|
||||
TEST(optimize_conversation, parse_and_limits) {
|
||||
Conversation c = Conversation::parse_transcript("user: hi\nthere\nagent: hello");
|
||||
ASSERT_EQ(c.turns.size(), 2u);
|
||||
ASSERT_EQ(c.turns[0].text, "hi\nthere");
|
||||
ASSERT_TRUE(is_invalid([] { Conversation::parse_transcript(""); }));
|
||||
ASSERT_TRUE(is_invalid([] { Conversation::parse_transcript("hello without role"); }));
|
||||
std::string many;
|
||||
for (int i = 0; i < 51; ++i) many += "user: x\n";
|
||||
ASSERT_TRUE(is_invalid([&] { Conversation::parse_transcript(many); }));
|
||||
Conversation big;
|
||||
big.turns.push_back({"user", std::string(20000, 'a'), {}});
|
||||
big.turns.push_back({"agent", std::string(20000, 'b'), {}});
|
||||
big.turns.push_back({"user", std::string(20000, 'c'), {}});
|
||||
big.turns.push_back({"agent", std::string(20000, 'd'), {}});
|
||||
ASSERT_TRUE(big.transcript().find("[transcript truncated]") != std::string::npos);
|
||||
}
|
||||
|
||||
TEST(optimize_autorater, scores_whole_conversation_by_rubric) {
|
||||
auto model = FakeModel::texts({
|
||||
"{\"verdicts\":[{\"id\":\"goal_reached\",\"pass\":true,\"explanation\":\"ok\"},"
|
||||
"{\"id\":\"no_contradiction\",\"pass\":true,\"explanation\":\"ok\"},"
|
||||
"{\"id\":\"no_unneeded_steps\",\"pass\":false,\"explanation\":\"asked twice\"}]}"});
|
||||
Autorater rater(model);
|
||||
Score s = rater.score("task_success", sample(), Autorater::task_success_rubrics());
|
||||
ASSERT_FALSE(s.pass);
|
||||
ASSERT_TRUE(s.value > 0.66 && s.value < 0.67);
|
||||
ASSERT_TRUE(s.explanation.find("asked twice") != std::string::npos);
|
||||
// One call covers the whole conversation, and the prompt carries every turn.
|
||||
ASSERT_EQ(model->requests.size(), 1u);
|
||||
const std::string& prompt = model->requests[0].contents[0].text();
|
||||
ASSERT_TRUE(prompt.find("user: refund please") != std::string::npos);
|
||||
ASSERT_TRUE(prompt.find("agent: done, refunded") != std::string::npos);
|
||||
}
|
||||
|
||||
TEST(optimize_autorater, missing_verdict_fails_loudly) {
|
||||
auto model = FakeModel::texts({"{\"verdicts\":[{\"id\":\"goal_reached\",\"pass\":true}]}"});
|
||||
Score s = Autorater(model).score("t", sample(), Autorater::task_success_rubrics());
|
||||
ASSERT_FALSE(s.pass);
|
||||
ASSERT_EQ(s.verdicts.size(), 3u);
|
||||
ASSERT_EQ(s.verdicts[1].explanation, "no verdict returned");
|
||||
}
|
||||
|
||||
TEST(optimize_autorater, validation) {
|
||||
auto model = FakeModel::texts({"{}"});
|
||||
Autorater rater(model);
|
||||
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {}); }));
|
||||
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {{"bad id", "q"}}); }));
|
||||
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {{"a", "q"}, {"a", "q2"}}); }));
|
||||
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {{"a", ""}}); }));
|
||||
ASSERT_TRUE(is_invalid([&] { rater.score("t", Conversation{}, {{"a", "q"}}); }));
|
||||
std::vector<Rubric> many;
|
||||
for (int i = 0; i < 17; ++i) many.push_back({"r" + std::to_string(i), "q"});
|
||||
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), many); }));
|
||||
ASSERT_EQ(model->requests.size(), 0u); // nothing reached the model
|
||||
}
|
||||
|
||||
TEST(optimize_tools, fixed_unknown_and_model_played) {
|
||||
auto model = FakeModel::texts({"{\"balance\":42}"});
|
||||
VirtualToolbox box(model);
|
||||
box.add({"lookup", "finds things", Json(), Json::parse("{\"found\":true}")});
|
||||
box.add({"balance", "account balance", Json(), Json()});
|
||||
FunctionCall a;
|
||||
a.name = "lookup";
|
||||
ASSERT_EQ(box.call(a).find("found")->as_bool(), true);
|
||||
FunctionCall b;
|
||||
b.name = "balance";
|
||||
b.args.set("account", Json("x"));
|
||||
ASSERT_EQ(box.call(b).find("balance")->as_int(), 42);
|
||||
FunctionCall c;
|
||||
c.name = "ghost";
|
||||
ASSERT_EQ(box.call(c).find("error")->as_string(), "unknown_tool");
|
||||
ASSERT_EQ(box.calls(), 3);
|
||||
ASSERT_EQ(box.unknown_calls(), 1);
|
||||
ASSERT_EQ(box.declarations()[0].functions.size(), 2u);
|
||||
ASSERT_TRUE(is_invalid([&] { box.add({"lookup", "", Json(), Json()}); }));
|
||||
ASSERT_TRUE(is_invalid([&] { box.add({"a b", "", Json(), Json()}); }));
|
||||
ASSERT_TRUE(is_invalid([&] { box.add({"t", "", Json(), Json(3)}); }));
|
||||
}
|
||||
|
||||
namespace {
|
||||
// Scripted user: first message comes from the scenario, then "more", then done.
|
||||
std::shared_ptr<FakeModel> user_model(int messages) {
|
||||
auto n = std::make_shared<int>(0);
|
||||
return std::make_shared<FakeModel>([n, messages](const GenerateRequest&) {
|
||||
if (++*n > messages) return reply_text("{\"done\":true,\"message\":\"\"}");
|
||||
return reply_text("{\"done\":false,\"message\":\"follow up " + std::to_string(*n) + "\"}");
|
||||
});
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(optimize_simulator, runs_tools_and_stops_when_user_is_done) {
|
||||
auto agent_model = std::make_shared<FakeModel>([](const GenerateRequest& req) {
|
||||
// First request of a turn ends in the user's text, the follow-up request
|
||||
// after a tool call ends in a functionResponse.
|
||||
const Content& last = req.contents.back();
|
||||
if (!last.parts.empty() && last.parts[0].function_response)
|
||||
return reply_text("Your balance is 42.");
|
||||
if (last.text().find("balance") != std::string::npos)
|
||||
return reply_call("balance", Json::object());
|
||||
return reply_text("How can I help?");
|
||||
});
|
||||
ModelAgent agent(agent_model, "You are a bank bot.", {});
|
||||
VirtualToolbox box;
|
||||
box.add({"balance", "returns balance", Json(), Json::parse("{\"balance\":42}")});
|
||||
Scenario sc{"balance check", "what is my balance", "check the balance, then thank the agent"};
|
||||
Simulator sim(user_model(1), FakeModel::texts({"{}"}));
|
||||
SimulationResult r = sim.run(sc, agent, box, SimulationConfig{5});
|
||||
ASSERT_EQ(r.stop_reason, "user_done");
|
||||
ASSERT_EQ(r.tool_calls, 1);
|
||||
ASSERT_EQ(r.conversation.turns.size(), 4u); // user, agent, user, agent
|
||||
ASSERT_EQ(r.conversation.turns[1].text, "Your balance is 42.");
|
||||
ASSERT_EQ(r.conversation.turns[1].calls[0].name, "balance");
|
||||
ASSERT_EQ(r.conversation.turns[1].calls[0].result.find("balance")->as_int(), 42);
|
||||
}
|
||||
|
||||
TEST(optimize_simulator, caps_turns_tool_rounds_and_tool_calls) {
|
||||
auto chatty = std::make_shared<FakeModel>(
|
||||
[](const GenerateRequest&) { return reply_text("ok"); });
|
||||
ModelAgent agent(chatty, "x");
|
||||
VirtualToolbox box;
|
||||
Scenario sc{"t", "hi", "keep talking"};
|
||||
Simulator endless(user_model(1000), FakeModel::texts({"{}"}));
|
||||
SimulationResult r = endless.run(sc, agent, box, SimulationConfig{3});
|
||||
ASSERT_EQ(r.stop_reason, "max_turns");
|
||||
ASSERT_EQ(r.conversation.turns.size(), 6u);
|
||||
ASSERT_TRUE(is_invalid([&] { endless.run(sc, agent, box, SimulationConfig{0}); }));
|
||||
ASSERT_TRUE(is_invalid([&] { endless.run(sc, agent, box, SimulationConfig{13}); }));
|
||||
|
||||
// An agent that calls tools forever hits the per-turn round cap.
|
||||
auto looper = std::make_shared<FakeModel>(
|
||||
[](const GenerateRequest&) { return reply_call("ping", Json::object()); });
|
||||
ModelAgent loop_agent(looper, "x");
|
||||
VirtualToolbox pingbox;
|
||||
pingbox.add({"ping", "p", Json(), Json::parse("{}")});
|
||||
SimulationResult lr = endless.run(sc, loop_agent, pingbox, SimulationConfig{5});
|
||||
ASSERT_EQ(lr.stop_reason, "tool_round_cap");
|
||||
ASSERT_EQ(lr.tool_calls, 5);
|
||||
|
||||
Scenario bad{"t", "", "plan"};
|
||||
ASSERT_TRUE(is_invalid([&] { endless.run(bad, agent, box); }));
|
||||
}
|
||||
|
||||
TEST(optimize_simulator, scores_task_success_and_safety) {
|
||||
auto agent_model = std::make_shared<FakeModel>(
|
||||
[](const GenerateRequest&) { return reply_text("Here you go."); });
|
||||
ModelAgent agent(agent_model, "x");
|
||||
VirtualToolbox box;
|
||||
auto judge = std::make_shared<FakeModel>([](const GenerateRequest& req) {
|
||||
std::string prompt = req.contents[0].text();
|
||||
bool safety = prompt.find("harmful") != std::string::npos;
|
||||
std::string v = "{\"verdicts\":[";
|
||||
// Every rubric id appears in the prompt as "- id: question".
|
||||
std::size_t pos = 0;
|
||||
bool first = true;
|
||||
while ((pos = prompt.find("\n- ", pos)) != std::string::npos) {
|
||||
std::size_t colon = prompt.find(':', pos);
|
||||
std::string id = prompt.substr(pos + 3, colon - pos - 3);
|
||||
v += std::string(first ? "" : ",") + "{\"id\":\"" + id + "\",\"pass\":" +
|
||||
(safety || id != "plan_goals" ? "true" : "false") + ",\"explanation\":\"e\"}";
|
||||
first = false;
|
||||
pos = colon;
|
||||
}
|
||||
return reply_text(v + "]}");
|
||||
});
|
||||
Simulator sim(user_model(0), judge);
|
||||
SimulationReport rep = sim.run_and_score({"t", "hi", "get a thing"}, agent, box);
|
||||
ASSERT_TRUE(rep.safety.pass);
|
||||
ASSERT_FALSE(rep.task_success.pass); // plan_goals failed
|
||||
ASSERT_FALSE(rep.pass());
|
||||
ASSERT_EQ(rep.to_json().find("pass")->as_bool(), false);
|
||||
ASSERT_EQ(rep.task_success.verdicts.size(), 4u); // 3 built-in plus plan_goals
|
||||
}
|
||||
|
||||
TEST(optimize_clusterer, verifies_model_output) {
|
||||
auto model = FakeModel::texts({
|
||||
"{\"clusters\":["
|
||||
"{\"label\":\"wrong tool\",\"description\":\"d\",\"failure_ids\":[\"f1\",\"f2\",\"ghost\"]},"
|
||||
"{\"label\":\"rude\",\"description\":\"d\",\"failure_ids\":[\"f2\",\"f3\"]},"
|
||||
"{\"label\":\"empty\",\"description\":\"d\",\"failure_ids\":[\"ghost2\"]}]}"});
|
||||
FailureClusterer fc(model);
|
||||
ClusterReport r = fc.cluster({{"f1", "called wrong tool", ""}, {"f2", "wrong tool again", ""},
|
||||
{"f3", "rude tone", ""}, {"f4", "timeout", ""}});
|
||||
ASSERT_EQ(r.clusters.size(), 2u);
|
||||
ASSERT_EQ(r.clusters[0].label, "wrong tool");
|
||||
ASSERT_EQ(r.clusters[0].failure_ids.size(), 2u); // ghost dropped
|
||||
ASSERT_EQ(r.clusters[1].failure_ids.size(), 1u); // f2 already used
|
||||
ASSERT_EQ(r.unassigned.size(), 1u);
|
||||
ASSERT_EQ(r.unassigned[0], "f4"); // loud, not silently placed
|
||||
ASSERT_EQ(r.to_json().find("unassigned")->as_array().size(), 1u);
|
||||
}
|
||||
|
||||
TEST(optimize_clusterer, caps_and_validation) {
|
||||
auto model = FakeModel::texts({"{\"clusters\":[]}"});
|
||||
FailureClusterer fc(model);
|
||||
ASSERT_TRUE(is_invalid([&] { fc.cluster({}); }));
|
||||
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "x", ""}, {"a", "y", ""}}); }));
|
||||
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"bad id", "x", ""}}); }));
|
||||
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "", ""}}); }));
|
||||
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "x", ""}}, 0); }));
|
||||
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "x", ""}}, 11); }));
|
||||
std::vector<Failure> many;
|
||||
for (int i = 0; i < 205; ++i) many.push_back({"f" + std::to_string(i), "boom", ""});
|
||||
ClusterReport r = fc.cluster(many);
|
||||
ASSERT_EQ(r.dropped, 5u);
|
||||
ASSERT_EQ(r.unassigned.size(), 200u);
|
||||
// More clusters than the cap are cut and their members become unassigned.
|
||||
auto wide = FakeModel::texts({
|
||||
"{\"clusters\":[{\"label\":\"a\",\"failure_ids\":[\"x\"]},"
|
||||
"{\"label\":\"b\",\"failure_ids\":[\"y\"]}]}"});
|
||||
ClusterReport w = FailureClusterer(wide).cluster({{"x", "s", ""}, {"y", "s", ""}}, 1);
|
||||
ASSERT_EQ(w.clusters.size(), 1u);
|
||||
ASSERT_EQ(w.unassigned.size(), 1u);
|
||||
}
|
||||
|
||||
namespace {
|
||||
ClusterReport one_cluster() {
|
||||
ClusterReport r;
|
||||
r.clusters.push_back({"wrong tool", "calls search instead of lookup", {"f1"}});
|
||||
return r;
|
||||
}
|
||||
std::vector<Failure> failures() { return {{"f1", "used search for an order lookup", ""}}; }
|
||||
} // namespace
|
||||
|
||||
TEST(optimize_optimizer, suggest_returns_refined_instruction) {
|
||||
auto model = FakeModel::texts({"{\"instruction\":\"Use lookup for orders.\",\"rationale\":\"r\"}"});
|
||||
Suggestion s = InstructionOptimizer(model).suggest("Be helpful.", one_cluster(), failures());
|
||||
ASSERT_EQ(s.instruction, "Use lookup for orders.");
|
||||
ASSERT_TRUE(s.changed);
|
||||
ASSERT_FALSE(s.improved); // nothing was measured
|
||||
const std::string& prompt = model->requests[0].contents[0].text();
|
||||
ASSERT_TRUE(prompt.find("Be helpful.") != std::string::npos);
|
||||
ASSERT_TRUE(prompt.find("wrong tool") != std::string::npos);
|
||||
}
|
||||
|
||||
TEST(optimize_optimizer, optimize_never_returns_a_worse_instruction) {
|
||||
auto model = FakeModel::texts({"{\"instruction\":\"worse\",\"rationale\":\"r\"}"});
|
||||
Suggestion s = InstructionOptimizer(model).optimize(
|
||||
"orig", one_cluster(), failures(), [](const std::string& i) { return i == "orig" ? 0.6 : 0.3; });
|
||||
ASSERT_EQ(s.instruction, "orig");
|
||||
ASSERT_FALSE(s.improved);
|
||||
ASSERT_FALSE(s.changed);
|
||||
|
||||
auto better = FakeModel::texts({"{\"instruction\":\"v2\",\"rationale\":\"r2\"}",
|
||||
"{\"instruction\":\"v3\",\"rationale\":\"r3\"}"});
|
||||
Suggestion b = InstructionOptimizer(better).optimize(
|
||||
"orig", one_cluster(), failures(),
|
||||
[](const std::string& i) { return i == "orig" ? 0.2 : i == "v2" ? 0.5 : 0.7; }, 3);
|
||||
ASSERT_EQ(b.instruction, "v3");
|
||||
ASSERT_TRUE(b.improved);
|
||||
ASSERT_EQ(b.rounds, 3); // round 3: the model repeated v3, unchanged, search ends
|
||||
ASSERT_EQ(b.best_score, 0.7);
|
||||
ASSERT_EQ(b.baseline_score, 0.2);
|
||||
}
|
||||
|
||||
TEST(optimize_optimizer, validation_and_caps) {
|
||||
auto model = FakeModel::texts({"{\"instruction\":\"\"}"});
|
||||
InstructionOptimizer opt(model);
|
||||
ASSERT_TRUE(is_invalid([&] { opt.suggest("", one_cluster(), failures()); }));
|
||||
ASSERT_TRUE(is_invalid([&] { opt.suggest(std::string(8001, 'x'), one_cluster(), failures()); }));
|
||||
ASSERT_TRUE(is_invalid([&] { opt.suggest("x", ClusterReport{}, failures()); }));
|
||||
auto scorer = [](const std::string&) { return 0.0; };
|
||||
ASSERT_TRUE(is_invalid([&] { opt.optimize("x", one_cluster(), failures(), scorer, 0); }));
|
||||
ASSERT_TRUE(is_invalid([&] { opt.optimize("x", one_cluster(), failures(), scorer, 4); }));
|
||||
ASSERT_TRUE(is_invalid([&] { opt.optimize("x", one_cluster(), failures(), nullptr); }));
|
||||
try {
|
||||
opt.suggest("x", one_cluster(), failures()); // model returned an empty instruction
|
||||
FAIL("expected BAD_RESPONSE");
|
||||
} catch (const GoogleError& e) {
|
||||
ASSERT_EQ(e.status(), "BAD_RESPONSE");
|
||||
}
|
||||
auto huge = FakeModel::texts({"{\"instruction\":\"" + std::string(8001, 'y') + "\"}"});
|
||||
ASSERT_THROW(InstructionOptimizer(huge).suggest("x", one_cluster(), failures()), GoogleError);
|
||||
}
|
||||
Reference in New Issue
Block a user