Files
moloapp/tests/src/google/optimize/test_local.cpp
T

340 lines
15 KiB
C++
Raw Normal View History

// Local logic against a fake LanguageModel: autorater, simulator, clusterer,
// optimizer, JSON extraction. Deterministic, no network.
#include "framework.h"
#include "google/errors.h"
#include "google/optimize/autorater.h"
#include "google/optimize/clusterer.h"
#include "google/optimize/model_json.h"
#include "google/optimize/optimizer.h"
#include "google/optimize/simulator.h"
#include "fakes.h"
using namespace molo::google;
using namespace molo::google::optimize;
using molo::clients::body::Json;
using optimize_test::FakeModel;
using optimize_test::reply_call;
using optimize_test::reply_text;
namespace {
bool is_invalid(const std::function<void()>& f) {
try {
f();
} catch (const GoogleError& e) {
return e.http_status() == 400;
}
return false;
}
Conversation sample() {
return Conversation::parse_transcript("user: refund please\nagent: done, refunded 10 EUR");
}
} // namespace
TEST(optimize_json, extract_tolerates_fences_and_prose) {
Json a = JsonModelCall::extract("sure!\n```json\n{\"a\":{\"b\":\"}\"},\"c\":1}\n```\nbye");
ASSERT_TRUE(a.is_object());
ASSERT_EQ(a.find("c")->as_int(), 1);
ASSERT_TRUE(JsonModelCall::extract("no json here").is_null());
ASSERT_TRUE(JsonModelCall::extract("{broken").is_null());
Json second = JsonModelCall::extract("{nope} then {\"ok\":true}");
ASSERT_TRUE(second.is_object() && second.find("ok") != nullptr);
}
TEST(optimize_json, retries_once_then_fails_loud) {
auto model = FakeModel::texts({"not json", "still not json"});
JsonModelCall call(model);
try {
call.ask("s", "u");
FAIL("expected BAD_RESPONSE");
} catch (const GoogleError& e) {
ASSERT_EQ(e.status(), "BAD_RESPONSE");
}
ASSERT_EQ(call.calls(), 2); // one try plus one retry, no more
auto ok = FakeModel::texts({"oops", "{\"x\":1}"});
ASSERT_EQ(JsonModelCall(ok).ask("s", "u").find("x")->as_int(), 1);
}
TEST(optimize_json, request_asks_for_json_deterministically) {
auto model = FakeModel::texts({"{}"});
JsonModelCall(model, "m1").ask("sys", "user");
const GenerateRequest& r = model->requests[0];
ASSERT_EQ(r.model, "m1");
ASSERT_EQ(r.config.response_mime_type, "application/json");
ASSERT_EQ(*r.config.temperature, 0.0);
ASSERT_EQ(r.system->parts[0].text, "sys");
}
TEST(optimize_conversation, parse_and_limits) {
Conversation c = Conversation::parse_transcript("user: hi\nthere\nagent: hello");
ASSERT_EQ(c.turns.size(), 2u);
ASSERT_EQ(c.turns[0].text, "hi\nthere");
ASSERT_TRUE(is_invalid([] { Conversation::parse_transcript(""); }));
ASSERT_TRUE(is_invalid([] { Conversation::parse_transcript("hello without role"); }));
std::string many;
for (int i = 0; i < 51; ++i) many += "user: x\n";
ASSERT_TRUE(is_invalid([&] { Conversation::parse_transcript(many); }));
Conversation big;
big.turns.push_back({"user", std::string(20000, 'a'), {}});
big.turns.push_back({"agent", std::string(20000, 'b'), {}});
big.turns.push_back({"user", std::string(20000, 'c'), {}});
big.turns.push_back({"agent", std::string(20000, 'd'), {}});
ASSERT_TRUE(big.transcript().find("[transcript truncated]") != std::string::npos);
}
TEST(optimize_autorater, scores_whole_conversation_by_rubric) {
auto model = FakeModel::texts({
"{\"verdicts\":[{\"id\":\"goal_reached\",\"pass\":true,\"explanation\":\"ok\"},"
"{\"id\":\"no_contradiction\",\"pass\":true,\"explanation\":\"ok\"},"
"{\"id\":\"no_unneeded_steps\",\"pass\":false,\"explanation\":\"asked twice\"}]}"});
Autorater rater(model);
Score s = rater.score("task_success", sample(), Autorater::task_success_rubrics());
ASSERT_FALSE(s.pass);
ASSERT_TRUE(s.value > 0.66 && s.value < 0.67);
ASSERT_TRUE(s.explanation.find("asked twice") != std::string::npos);
// One call covers the whole conversation, and the prompt carries every turn.
ASSERT_EQ(model->requests.size(), 1u);
const std::string& prompt = model->requests[0].contents[0].text();
ASSERT_TRUE(prompt.find("user: refund please") != std::string::npos);
ASSERT_TRUE(prompt.find("agent: done, refunded") != std::string::npos);
}
TEST(optimize_autorater, missing_verdict_fails_loudly) {
auto model = FakeModel::texts({"{\"verdicts\":[{\"id\":\"goal_reached\",\"pass\":true}]}"});
Score s = Autorater(model).score("t", sample(), Autorater::task_success_rubrics());
ASSERT_FALSE(s.pass);
ASSERT_EQ(s.verdicts.size(), 3u);
ASSERT_EQ(s.verdicts[1].explanation, "no verdict returned");
}
TEST(optimize_autorater, validation) {
auto model = FakeModel::texts({"{}"});
Autorater rater(model);
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {}); }));
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {{"bad id", "q"}}); }));
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {{"a", "q"}, {"a", "q2"}}); }));
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {{"a", ""}}); }));
ASSERT_TRUE(is_invalid([&] { rater.score("t", Conversation{}, {{"a", "q"}}); }));
std::vector<Rubric> many;
for (int i = 0; i < 17; ++i) many.push_back({"r" + std::to_string(i), "q"});
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), many); }));
ASSERT_EQ(model->requests.size(), 0u); // nothing reached the model
}
TEST(optimize_tools, fixed_unknown_and_model_played) {
auto model = FakeModel::texts({"{\"balance\":42}"});
VirtualToolbox box(model);
box.add({"lookup", "finds things", Json(), Json::parse("{\"found\":true}")});
box.add({"balance", "account balance", Json(), Json()});
FunctionCall a;
a.name = "lookup";
ASSERT_EQ(box.call(a).find("found")->as_bool(), true);
FunctionCall b;
b.name = "balance";
b.args.set("account", Json("x"));
ASSERT_EQ(box.call(b).find("balance")->as_int(), 42);
FunctionCall c;
c.name = "ghost";
ASSERT_EQ(box.call(c).find("error")->as_string(), "unknown_tool");
ASSERT_EQ(box.calls(), 3);
ASSERT_EQ(box.unknown_calls(), 1);
ASSERT_EQ(box.declarations()[0].functions.size(), 2u);
ASSERT_TRUE(is_invalid([&] { box.add({"lookup", "", Json(), Json()}); }));
ASSERT_TRUE(is_invalid([&] { box.add({"a b", "", Json(), Json()}); }));
ASSERT_TRUE(is_invalid([&] { box.add({"t", "", Json(), Json(3)}); }));
}
namespace {
// Scripted user: first message comes from the scenario, then "more", then done.
std::shared_ptr<FakeModel> user_model(int messages) {
auto n = std::make_shared<int>(0);
return std::make_shared<FakeModel>([n, messages](const GenerateRequest&) {
if (++*n > messages) return reply_text("{\"done\":true,\"message\":\"\"}");
return reply_text("{\"done\":false,\"message\":\"follow up " + std::to_string(*n) + "\"}");
});
}
} // namespace
TEST(optimize_simulator, runs_tools_and_stops_when_user_is_done) {
auto agent_model = std::make_shared<FakeModel>([](const GenerateRequest& req) {
// First request of a turn ends in the user's text, the follow-up request
// after a tool call ends in a functionResponse.
const Content& last = req.contents.back();
if (!last.parts.empty() && last.parts[0].function_response)
return reply_text("Your balance is 42.");
if (last.text().find("balance") != std::string::npos)
return reply_call("balance", Json::object());
return reply_text("How can I help?");
});
ModelAgent agent(agent_model, "You are a bank bot.", {});
VirtualToolbox box;
box.add({"balance", "returns balance", Json(), Json::parse("{\"balance\":42}")});
Scenario sc{"balance check", "what is my balance", "check the balance, then thank the agent"};
Simulator sim(user_model(1), FakeModel::texts({"{}"}));
SimulationResult r = sim.run(sc, agent, box, SimulationConfig{5});
ASSERT_EQ(r.stop_reason, "user_done");
ASSERT_EQ(r.tool_calls, 1);
ASSERT_EQ(r.conversation.turns.size(), 4u); // user, agent, user, agent
ASSERT_EQ(r.conversation.turns[1].text, "Your balance is 42.");
ASSERT_EQ(r.conversation.turns[1].calls[0].name, "balance");
ASSERT_EQ(r.conversation.turns[1].calls[0].result.find("balance")->as_int(), 42);
}
TEST(optimize_simulator, caps_turns_tool_rounds_and_tool_calls) {
auto chatty = std::make_shared<FakeModel>(
[](const GenerateRequest&) { return reply_text("ok"); });
ModelAgent agent(chatty, "x");
VirtualToolbox box;
Scenario sc{"t", "hi", "keep talking"};
Simulator endless(user_model(1000), FakeModel::texts({"{}"}));
SimulationResult r = endless.run(sc, agent, box, SimulationConfig{3});
ASSERT_EQ(r.stop_reason, "max_turns");
ASSERT_EQ(r.conversation.turns.size(), 6u);
ASSERT_TRUE(is_invalid([&] { endless.run(sc, agent, box, SimulationConfig{0}); }));
ASSERT_TRUE(is_invalid([&] { endless.run(sc, agent, box, SimulationConfig{13}); }));
// An agent that calls tools forever hits the per-turn round cap.
auto looper = std::make_shared<FakeModel>(
[](const GenerateRequest&) { return reply_call("ping", Json::object()); });
ModelAgent loop_agent(looper, "x");
VirtualToolbox pingbox;
pingbox.add({"ping", "p", Json(), Json::parse("{}")});
SimulationResult lr = endless.run(sc, loop_agent, pingbox, SimulationConfig{5});
ASSERT_EQ(lr.stop_reason, "tool_round_cap");
ASSERT_EQ(lr.tool_calls, 5);
Scenario bad{"t", "", "plan"};
ASSERT_TRUE(is_invalid([&] { endless.run(bad, agent, box); }));
}
TEST(optimize_simulator, scores_task_success_and_safety) {
auto agent_model = std::make_shared<FakeModel>(
[](const GenerateRequest&) { return reply_text("Here you go."); });
ModelAgent agent(agent_model, "x");
VirtualToolbox box;
auto judge = std::make_shared<FakeModel>([](const GenerateRequest& req) {
std::string prompt = req.contents[0].text();
bool safety = prompt.find("harmful") != std::string::npos;
std::string v = "{\"verdicts\":[";
// Every rubric id appears in the prompt as "- id: question".
std::size_t pos = 0;
bool first = true;
while ((pos = prompt.find("\n- ", pos)) != std::string::npos) {
std::size_t colon = prompt.find(':', pos);
std::string id = prompt.substr(pos + 3, colon - pos - 3);
v += std::string(first ? "" : ",") + "{\"id\":\"" + id + "\",\"pass\":" +
(safety || id != "plan_goals" ? "true" : "false") + ",\"explanation\":\"e\"}";
first = false;
pos = colon;
}
return reply_text(v + "]}");
});
Simulator sim(user_model(0), judge);
SimulationReport rep = sim.run_and_score({"t", "hi", "get a thing"}, agent, box);
ASSERT_TRUE(rep.safety.pass);
ASSERT_FALSE(rep.task_success.pass); // plan_goals failed
ASSERT_FALSE(rep.pass());
ASSERT_EQ(rep.to_json().find("pass")->as_bool(), false);
ASSERT_EQ(rep.task_success.verdicts.size(), 4u); // 3 built-in plus plan_goals
}
TEST(optimize_clusterer, verifies_model_output) {
auto model = FakeModel::texts({
"{\"clusters\":["
"{\"label\":\"wrong tool\",\"description\":\"d\",\"failure_ids\":[\"f1\",\"f2\",\"ghost\"]},"
"{\"label\":\"rude\",\"description\":\"d\",\"failure_ids\":[\"f2\",\"f3\"]},"
"{\"label\":\"empty\",\"description\":\"d\",\"failure_ids\":[\"ghost2\"]}]}"});
FailureClusterer fc(model);
ClusterReport r = fc.cluster({{"f1", "called wrong tool", ""}, {"f2", "wrong tool again", ""},
{"f3", "rude tone", ""}, {"f4", "timeout", ""}});
ASSERT_EQ(r.clusters.size(), 2u);
ASSERT_EQ(r.clusters[0].label, "wrong tool");
ASSERT_EQ(r.clusters[0].failure_ids.size(), 2u); // ghost dropped
ASSERT_EQ(r.clusters[1].failure_ids.size(), 1u); // f2 already used
ASSERT_EQ(r.unassigned.size(), 1u);
ASSERT_EQ(r.unassigned[0], "f4"); // loud, not silently placed
ASSERT_EQ(r.to_json().find("unassigned")->as_array().size(), 1u);
}
TEST(optimize_clusterer, caps_and_validation) {
auto model = FakeModel::texts({"{\"clusters\":[]}"});
FailureClusterer fc(model);
ASSERT_TRUE(is_invalid([&] { fc.cluster({}); }));
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "x", ""}, {"a", "y", ""}}); }));
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"bad id", "x", ""}}); }));
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "", ""}}); }));
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "x", ""}}, 0); }));
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "x", ""}}, 11); }));
std::vector<Failure> many;
for (int i = 0; i < 205; ++i) many.push_back({"f" + std::to_string(i), "boom", ""});
ClusterReport r = fc.cluster(many);
ASSERT_EQ(r.dropped, 5u);
ASSERT_EQ(r.unassigned.size(), 200u);
// More clusters than the cap are cut and their members become unassigned.
auto wide = FakeModel::texts({
"{\"clusters\":[{\"label\":\"a\",\"failure_ids\":[\"x\"]},"
"{\"label\":\"b\",\"failure_ids\":[\"y\"]}]}"});
ClusterReport w = FailureClusterer(wide).cluster({{"x", "s", ""}, {"y", "s", ""}}, 1);
ASSERT_EQ(w.clusters.size(), 1u);
ASSERT_EQ(w.unassigned.size(), 1u);
}
namespace {
ClusterReport one_cluster() {
ClusterReport r;
r.clusters.push_back({"wrong tool", "calls search instead of lookup", {"f1"}});
return r;
}
std::vector<Failure> failures() { return {{"f1", "used search for an order lookup", ""}}; }
} // namespace
TEST(optimize_optimizer, suggest_returns_refined_instruction) {
auto model = FakeModel::texts({"{\"instruction\":\"Use lookup for orders.\",\"rationale\":\"r\"}"});
Suggestion s = InstructionOptimizer(model).suggest("Be helpful.", one_cluster(), failures());
ASSERT_EQ(s.instruction, "Use lookup for orders.");
ASSERT_TRUE(s.changed);
ASSERT_FALSE(s.improved); // nothing was measured
const std::string& prompt = model->requests[0].contents[0].text();
ASSERT_TRUE(prompt.find("Be helpful.") != std::string::npos);
ASSERT_TRUE(prompt.find("wrong tool") != std::string::npos);
}
TEST(optimize_optimizer, optimize_never_returns_a_worse_instruction) {
auto model = FakeModel::texts({"{\"instruction\":\"worse\",\"rationale\":\"r\"}"});
Suggestion s = InstructionOptimizer(model).optimize(
"orig", one_cluster(), failures(), [](const std::string& i) { return i == "orig" ? 0.6 : 0.3; });
ASSERT_EQ(s.instruction, "orig");
ASSERT_FALSE(s.improved);
ASSERT_FALSE(s.changed);
auto better = FakeModel::texts({"{\"instruction\":\"v2\",\"rationale\":\"r2\"}",
"{\"instruction\":\"v3\",\"rationale\":\"r3\"}"});
Suggestion b = InstructionOptimizer(better).optimize(
"orig", one_cluster(), failures(),
[](const std::string& i) { return i == "orig" ? 0.2 : i == "v2" ? 0.5 : 0.7; }, 3);
ASSERT_EQ(b.instruction, "v3");
ASSERT_TRUE(b.improved);
ASSERT_EQ(b.rounds, 3); // round 3: the model repeated v3, unchanged, search ends
ASSERT_EQ(b.best_score, 0.7);
ASSERT_EQ(b.baseline_score, 0.2);
}
TEST(optimize_optimizer, validation_and_caps) {
auto model = FakeModel::texts({"{\"instruction\":\"\"}"});
InstructionOptimizer opt(model);
ASSERT_TRUE(is_invalid([&] { opt.suggest("", one_cluster(), failures()); }));
ASSERT_TRUE(is_invalid([&] { opt.suggest(std::string(8001, 'x'), one_cluster(), failures()); }));
ASSERT_TRUE(is_invalid([&] { opt.suggest("x", ClusterReport{}, failures()); }));
auto scorer = [](const std::string&) { return 0.0; };
ASSERT_TRUE(is_invalid([&] { opt.optimize("x", one_cluster(), failures(), scorer, 0); }));
ASSERT_TRUE(is_invalid([&] { opt.optimize("x", one_cluster(), failures(), scorer, 4); }));
ASSERT_TRUE(is_invalid([&] { opt.optimize("x", one_cluster(), failures(), nullptr); }));
try {
opt.suggest("x", one_cluster(), failures()); // model returned an empty instruction
FAIL("expected BAD_RESPONSE");
} catch (const GoogleError& e) {
ASSERT_EQ(e.status(), "BAD_RESPONSE");
}
auto huge = FakeModel::texts({"{\"instruction\":\"" + std::string(8001, 'y') + "\"}"});
ASSERT_THROW(InstructionOptimizer(huge).suggest("x", one_cluster(), failures()), GoogleError);
}