// Local logic against a fake LanguageModel: autorater, simulator, clusterer, // optimizer, JSON extraction. Deterministic, no network. #include "framework.h" #include "google/errors.h" #include "google/optimize/autorater.h" #include "google/optimize/clusterer.h" #include "google/optimize/model_json.h" #include "google/optimize/optimizer.h" #include "google/optimize/simulator.h" #include "fakes.h" using namespace molo::google; using namespace molo::google::optimize; using molo::clients::body::Json; using optimize_test::FakeModel; using optimize_test::reply_call; using optimize_test::reply_text; namespace { bool is_invalid(const std::function& f) { try { f(); } catch (const GoogleError& e) { return e.http_status() == 400; } return false; } Conversation sample() { return Conversation::parse_transcript("user: refund please\nagent: done, refunded 10 EUR"); } } // namespace TEST(optimize_json, extract_tolerates_fences_and_prose) { Json a = JsonModelCall::extract("sure!\n```json\n{\"a\":{\"b\":\"}\"},\"c\":1}\n```\nbye"); ASSERT_TRUE(a.is_object()); ASSERT_EQ(a.find("c")->as_int(), 1); ASSERT_TRUE(JsonModelCall::extract("no json here").is_null()); ASSERT_TRUE(JsonModelCall::extract("{broken").is_null()); Json second = JsonModelCall::extract("{nope} then {\"ok\":true}"); ASSERT_TRUE(second.is_object() && second.find("ok") != nullptr); } TEST(optimize_json, retries_once_then_fails_loud) { auto model = FakeModel::texts({"not json", "still not json"}); JsonModelCall call(model); try { call.ask("s", "u"); FAIL("expected BAD_RESPONSE"); } catch (const GoogleError& e) { ASSERT_EQ(e.status(), "BAD_RESPONSE"); } ASSERT_EQ(call.calls(), 2); // one try plus one retry, no more auto ok = FakeModel::texts({"oops", "{\"x\":1}"}); ASSERT_EQ(JsonModelCall(ok).ask("s", "u").find("x")->as_int(), 1); } TEST(optimize_json, request_asks_for_json_deterministically) { auto model = FakeModel::texts({"{}"}); JsonModelCall(model, "m1").ask("sys", "user"); const GenerateRequest& r = model->requests[0]; ASSERT_EQ(r.model, "m1"); ASSERT_EQ(r.config.response_mime_type, "application/json"); ASSERT_EQ(*r.config.temperature, 0.0); ASSERT_EQ(r.system->parts[0].text, "sys"); } TEST(optimize_conversation, parse_and_limits) { Conversation c = Conversation::parse_transcript("user: hi\nthere\nagent: hello"); ASSERT_EQ(c.turns.size(), 2u); ASSERT_EQ(c.turns[0].text, "hi\nthere"); ASSERT_TRUE(is_invalid([] { Conversation::parse_transcript(""); })); ASSERT_TRUE(is_invalid([] { Conversation::parse_transcript("hello without role"); })); std::string many; for (int i = 0; i < 51; ++i) many += "user: x\n"; ASSERT_TRUE(is_invalid([&] { Conversation::parse_transcript(many); })); Conversation big; big.turns.push_back({"user", std::string(20000, 'a'), {}}); big.turns.push_back({"agent", std::string(20000, 'b'), {}}); big.turns.push_back({"user", std::string(20000, 'c'), {}}); big.turns.push_back({"agent", std::string(20000, 'd'), {}}); ASSERT_TRUE(big.transcript().find("[transcript truncated]") != std::string::npos); } TEST(optimize_autorater, scores_whole_conversation_by_rubric) { auto model = FakeModel::texts({ "{\"verdicts\":[{\"id\":\"goal_reached\",\"pass\":true,\"explanation\":\"ok\"}," "{\"id\":\"no_contradiction\",\"pass\":true,\"explanation\":\"ok\"}," "{\"id\":\"no_unneeded_steps\",\"pass\":false,\"explanation\":\"asked twice\"}]}"}); Autorater rater(model); Score s = rater.score("task_success", sample(), Autorater::task_success_rubrics()); ASSERT_FALSE(s.pass); ASSERT_TRUE(s.value > 0.66 && s.value < 0.67); ASSERT_TRUE(s.explanation.find("asked twice") != std::string::npos); // One call covers the whole conversation, and the prompt carries every turn. ASSERT_EQ(model->requests.size(), 1u); const std::string& prompt = model->requests[0].contents[0].text(); ASSERT_TRUE(prompt.find("user: refund please") != std::string::npos); ASSERT_TRUE(prompt.find("agent: done, refunded") != std::string::npos); } TEST(optimize_autorater, missing_verdict_fails_loudly) { auto model = FakeModel::texts({"{\"verdicts\":[{\"id\":\"goal_reached\",\"pass\":true}]}"}); Score s = Autorater(model).score("t", sample(), Autorater::task_success_rubrics()); ASSERT_FALSE(s.pass); ASSERT_EQ(s.verdicts.size(), 3u); ASSERT_EQ(s.verdicts[1].explanation, "no verdict returned"); } TEST(optimize_autorater, validation) { auto model = FakeModel::texts({"{}"}); Autorater rater(model); ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {}); })); ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {{"bad id", "q"}}); })); ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {{"a", "q"}, {"a", "q2"}}); })); ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {{"a", ""}}); })); ASSERT_TRUE(is_invalid([&] { rater.score("t", Conversation{}, {{"a", "q"}}); })); std::vector many; for (int i = 0; i < 17; ++i) many.push_back({"r" + std::to_string(i), "q"}); ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), many); })); ASSERT_EQ(model->requests.size(), 0u); // nothing reached the model } TEST(optimize_tools, fixed_unknown_and_model_played) { auto model = FakeModel::texts({"{\"balance\":42}"}); VirtualToolbox box(model); box.add({"lookup", "finds things", Json(), Json::parse("{\"found\":true}")}); box.add({"balance", "account balance", Json(), Json()}); FunctionCall a; a.name = "lookup"; ASSERT_EQ(box.call(a).find("found")->as_bool(), true); FunctionCall b; b.name = "balance"; b.args.set("account", Json("x")); ASSERT_EQ(box.call(b).find("balance")->as_int(), 42); FunctionCall c; c.name = "ghost"; ASSERT_EQ(box.call(c).find("error")->as_string(), "unknown_tool"); ASSERT_EQ(box.calls(), 3); ASSERT_EQ(box.unknown_calls(), 1); ASSERT_EQ(box.declarations()[0].functions.size(), 2u); ASSERT_TRUE(is_invalid([&] { box.add({"lookup", "", Json(), Json()}); })); ASSERT_TRUE(is_invalid([&] { box.add({"a b", "", Json(), Json()}); })); ASSERT_TRUE(is_invalid([&] { box.add({"t", "", Json(), Json(3)}); })); } namespace { // Scripted user: first message comes from the scenario, then "more", then done. std::shared_ptr user_model(int messages) { auto n = std::make_shared(0); return std::make_shared([n, messages](const GenerateRequest&) { if (++*n > messages) return reply_text("{\"done\":true,\"message\":\"\"}"); return reply_text("{\"done\":false,\"message\":\"follow up " + std::to_string(*n) + "\"}"); }); } } // namespace TEST(optimize_simulator, runs_tools_and_stops_when_user_is_done) { auto agent_model = std::make_shared([](const GenerateRequest& req) { // First request of a turn ends in the user's text, the follow-up request // after a tool call ends in a functionResponse. const Content& last = req.contents.back(); if (!last.parts.empty() && last.parts[0].function_response) return reply_text("Your balance is 42."); if (last.text().find("balance") != std::string::npos) return reply_call("balance", Json::object()); return reply_text("How can I help?"); }); ModelAgent agent(agent_model, "You are a bank bot.", {}); VirtualToolbox box; box.add({"balance", "returns balance", Json(), Json::parse("{\"balance\":42}")}); Scenario sc{"balance check", "what is my balance", "check the balance, then thank the agent"}; Simulator sim(user_model(1), FakeModel::texts({"{}"})); SimulationResult r = sim.run(sc, agent, box, SimulationConfig{5}); ASSERT_EQ(r.stop_reason, "user_done"); ASSERT_EQ(r.tool_calls, 1); ASSERT_EQ(r.conversation.turns.size(), 4u); // user, agent, user, agent ASSERT_EQ(r.conversation.turns[1].text, "Your balance is 42."); ASSERT_EQ(r.conversation.turns[1].calls[0].name, "balance"); ASSERT_EQ(r.conversation.turns[1].calls[0].result.find("balance")->as_int(), 42); } TEST(optimize_simulator, caps_turns_tool_rounds_and_tool_calls) { auto chatty = std::make_shared( [](const GenerateRequest&) { return reply_text("ok"); }); ModelAgent agent(chatty, "x"); VirtualToolbox box; Scenario sc{"t", "hi", "keep talking"}; Simulator endless(user_model(1000), FakeModel::texts({"{}"})); SimulationResult r = endless.run(sc, agent, box, SimulationConfig{3}); ASSERT_EQ(r.stop_reason, "max_turns"); ASSERT_EQ(r.conversation.turns.size(), 6u); ASSERT_TRUE(is_invalid([&] { endless.run(sc, agent, box, SimulationConfig{0}); })); ASSERT_TRUE(is_invalid([&] { endless.run(sc, agent, box, SimulationConfig{13}); })); // An agent that calls tools forever hits the per-turn round cap. auto looper = std::make_shared( [](const GenerateRequest&) { return reply_call("ping", Json::object()); }); ModelAgent loop_agent(looper, "x"); VirtualToolbox pingbox; pingbox.add({"ping", "p", Json(), Json::parse("{}")}); SimulationResult lr = endless.run(sc, loop_agent, pingbox, SimulationConfig{5}); ASSERT_EQ(lr.stop_reason, "tool_round_cap"); ASSERT_EQ(lr.tool_calls, 5); Scenario bad{"t", "", "plan"}; ASSERT_TRUE(is_invalid([&] { endless.run(bad, agent, box); })); } TEST(optimize_simulator, scores_task_success_and_safety) { auto agent_model = std::make_shared( [](const GenerateRequest&) { return reply_text("Here you go."); }); ModelAgent agent(agent_model, "x"); VirtualToolbox box; auto judge = std::make_shared([](const GenerateRequest& req) { std::string prompt = req.contents[0].text(); bool safety = prompt.find("harmful") != std::string::npos; std::string v = "{\"verdicts\":["; // Every rubric id appears in the prompt as "- id: question". std::size_t pos = 0; bool first = true; while ((pos = prompt.find("\n- ", pos)) != std::string::npos) { std::size_t colon = prompt.find(':', pos); std::string id = prompt.substr(pos + 3, colon - pos - 3); v += std::string(first ? "" : ",") + "{\"id\":\"" + id + "\",\"pass\":" + (safety || id != "plan_goals" ? "true" : "false") + ",\"explanation\":\"e\"}"; first = false; pos = colon; } return reply_text(v + "]}"); }); Simulator sim(user_model(0), judge); SimulationReport rep = sim.run_and_score({"t", "hi", "get a thing"}, agent, box); ASSERT_TRUE(rep.safety.pass); ASSERT_FALSE(rep.task_success.pass); // plan_goals failed ASSERT_FALSE(rep.pass()); ASSERT_EQ(rep.to_json().find("pass")->as_bool(), false); ASSERT_EQ(rep.task_success.verdicts.size(), 4u); // 3 built-in plus plan_goals } TEST(optimize_clusterer, verifies_model_output) { auto model = FakeModel::texts({ "{\"clusters\":[" "{\"label\":\"wrong tool\",\"description\":\"d\",\"failure_ids\":[\"f1\",\"f2\",\"ghost\"]}," "{\"label\":\"rude\",\"description\":\"d\",\"failure_ids\":[\"f2\",\"f3\"]}," "{\"label\":\"empty\",\"description\":\"d\",\"failure_ids\":[\"ghost2\"]}]}"}); FailureClusterer fc(model); ClusterReport r = fc.cluster({{"f1", "called wrong tool", ""}, {"f2", "wrong tool again", ""}, {"f3", "rude tone", ""}, {"f4", "timeout", ""}}); ASSERT_EQ(r.clusters.size(), 2u); ASSERT_EQ(r.clusters[0].label, "wrong tool"); ASSERT_EQ(r.clusters[0].failure_ids.size(), 2u); // ghost dropped ASSERT_EQ(r.clusters[1].failure_ids.size(), 1u); // f2 already used ASSERT_EQ(r.unassigned.size(), 1u); ASSERT_EQ(r.unassigned[0], "f4"); // loud, not silently placed ASSERT_EQ(r.to_json().find("unassigned")->as_array().size(), 1u); } TEST(optimize_clusterer, caps_and_validation) { auto model = FakeModel::texts({"{\"clusters\":[]}"}); FailureClusterer fc(model); ASSERT_TRUE(is_invalid([&] { fc.cluster({}); })); ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "x", ""}, {"a", "y", ""}}); })); ASSERT_TRUE(is_invalid([&] { fc.cluster({{"bad id", "x", ""}}); })); ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "", ""}}); })); ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "x", ""}}, 0); })); ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "x", ""}}, 11); })); std::vector many; for (int i = 0; i < 205; ++i) many.push_back({"f" + std::to_string(i), "boom", ""}); ClusterReport r = fc.cluster(many); ASSERT_EQ(r.dropped, 5u); ASSERT_EQ(r.unassigned.size(), 200u); // More clusters than the cap are cut and their members become unassigned. auto wide = FakeModel::texts({ "{\"clusters\":[{\"label\":\"a\",\"failure_ids\":[\"x\"]}," "{\"label\":\"b\",\"failure_ids\":[\"y\"]}]}"}); ClusterReport w = FailureClusterer(wide).cluster({{"x", "s", ""}, {"y", "s", ""}}, 1); ASSERT_EQ(w.clusters.size(), 1u); ASSERT_EQ(w.unassigned.size(), 1u); } namespace { ClusterReport one_cluster() { ClusterReport r; r.clusters.push_back({"wrong tool", "calls search instead of lookup", {"f1"}}); return r; } std::vector failures() { return {{"f1", "used search for an order lookup", ""}}; } } // namespace TEST(optimize_optimizer, suggest_returns_refined_instruction) { auto model = FakeModel::texts({"{\"instruction\":\"Use lookup for orders.\",\"rationale\":\"r\"}"}); Suggestion s = InstructionOptimizer(model).suggest("Be helpful.", one_cluster(), failures()); ASSERT_EQ(s.instruction, "Use lookup for orders."); ASSERT_TRUE(s.changed); ASSERT_FALSE(s.improved); // nothing was measured const std::string& prompt = model->requests[0].contents[0].text(); ASSERT_TRUE(prompt.find("Be helpful.") != std::string::npos); ASSERT_TRUE(prompt.find("wrong tool") != std::string::npos); } TEST(optimize_optimizer, optimize_never_returns_a_worse_instruction) { auto model = FakeModel::texts({"{\"instruction\":\"worse\",\"rationale\":\"r\"}"}); Suggestion s = InstructionOptimizer(model).optimize( "orig", one_cluster(), failures(), [](const std::string& i) { return i == "orig" ? 0.6 : 0.3; }); ASSERT_EQ(s.instruction, "orig"); ASSERT_FALSE(s.improved); ASSERT_FALSE(s.changed); auto better = FakeModel::texts({"{\"instruction\":\"v2\",\"rationale\":\"r2\"}", "{\"instruction\":\"v3\",\"rationale\":\"r3\"}"}); Suggestion b = InstructionOptimizer(better).optimize( "orig", one_cluster(), failures(), [](const std::string& i) { return i == "orig" ? 0.2 : i == "v2" ? 0.5 : 0.7; }, 3); ASSERT_EQ(b.instruction, "v3"); ASSERT_TRUE(b.improved); ASSERT_EQ(b.rounds, 3); // round 3: the model repeated v3, unchanged, search ends ASSERT_EQ(b.best_score, 0.7); ASSERT_EQ(b.baseline_score, 0.2); } TEST(optimize_optimizer, validation_and_caps) { auto model = FakeModel::texts({"{\"instruction\":\"\"}"}); InstructionOptimizer opt(model); ASSERT_TRUE(is_invalid([&] { opt.suggest("", one_cluster(), failures()); })); ASSERT_TRUE(is_invalid([&] { opt.suggest(std::string(8001, 'x'), one_cluster(), failures()); })); ASSERT_TRUE(is_invalid([&] { opt.suggest("x", ClusterReport{}, failures()); })); auto scorer = [](const std::string&) { return 0.0; }; ASSERT_TRUE(is_invalid([&] { opt.optimize("x", one_cluster(), failures(), scorer, 0); })); ASSERT_TRUE(is_invalid([&] { opt.optimize("x", one_cluster(), failures(), scorer, 4); })); ASSERT_TRUE(is_invalid([&] { opt.optimize("x", one_cluster(), failures(), nullptr); })); try { opt.suggest("x", one_cluster(), failures()); // model returned an empty instruction FAIL("expected BAD_RESPONSE"); } catch (const GoogleError& e) { ASSERT_EQ(e.status(), "BAD_RESPONSE"); } auto huge = FakeModel::texts({"{\"instruction\":\"" + std::string(8001, 'y') + "\"}"}); ASSERT_THROW(InstructionOptimizer(huge).suggest("x", one_cluster(), failures()), GoogleError); }