feat(clients): optimize: evaluation, simulation, observability and optimizer on a discovery-checked call table

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-24 04:40:55 +02:00
co-authored by Claude Sonnet 5.5
parent 6d744a433f
commit 1e2fb400bb
55 changed files with 5099 additions and 3 deletions
+81
View File
@@ -0,0 +1,81 @@
#pragma once
// Test doubles for the optimize module: a fake LanguageModel and clients
// that can never reach the network.
#include <functional>
#include <memory>
#include <string>
#include <vector>
#include "clients/transports/mock_transport.h"
#include "google/client.h"
#include "google/credentials.h"
#include "google/gemini_types.h"
namespace optimize_test {
using namespace molo::google;
inline GenerateResponse reply_text(const std::string& text) {
GenerateResponse r;
Candidate c;
c.content = Content::model(text);
r.candidates.push_back(c);
return r;
}
inline GenerateResponse reply_call(const std::string& name, molo::clients::body::Json args) {
GenerateResponse r;
Candidate c;
c.content.role = "model";
FunctionCall fc;
fc.name = name;
fc.args = std::move(args);
c.content.parts.push_back(Part::of_call(fc));
r.candidates.push_back(c);
return r;
}
// Answers from a function of the request; records every request.
class FakeModel : public LanguageModel {
public:
using Fn = std::function<GenerateResponse(const GenerateRequest&)>;
explicit FakeModel(Fn fn) : fn_(std::move(fn)) {}
static std::shared_ptr<FakeModel> texts(std::vector<std::string> replies) {
auto idx = std::make_shared<std::size_t>(0);
return std::make_shared<FakeModel>([replies, idx](const GenerateRequest&) {
std::string t = *idx < replies.size() ? replies[*idx] : replies.back();
++*idx;
return reply_text(t);
});
}
GenerateResponse generate(const GenerateRequest& request) override {
requests.push_back(request);
return fn_(request);
}
void stream(const GenerateRequest& request,
const std::function<bool(const GenerateResponse&)>& on_chunk) override {
on_chunk(generate(request));
}
std::vector<GenerateRequest> requests;
private:
Fn fn_;
};
// A GoogleClient over an empty MockTransport. With a credential it would
// still fail any send (no expectations), so tests only use it for calls that
// must be rejected before traffic.
struct OfflineClient {
molo::clients::transports::MockTransport* mock = nullptr;
std::shared_ptr<GoogleClient> client;
explicit OfflineClient(bool with_credential = true) {
auto t = std::make_unique<molo::clients::transports::MockTransport>();
mock = t.get();
client = std::make_shared<GoogleClient>(
std::move(t), with_credential ? Credentials::bearer("offline-token") : nullptr);
}
};
} // namespace optimize_test
+287
View File
@@ -0,0 +1,287 @@
// Call table, request builders and API-class validation. No scripted Google
// replies: everything here is local and deterministic. The table itself is
// verified against Discovery by tools/check_discovery.py.
#include <set>
#include "framework.h"
#include "google/optimize/calls.h"
#include "google/optimize/eval_requests.h"
#include "google/optimize/eval_resources.h"
#include "google/optimize/evaluation_api.h"
#include "google/errors.h"
#include "fakes.h"
using namespace molo::google;
using namespace molo::google::optimize;
using molo::clients::body::Json;
using optimize_test::OfflineClient;
namespace {
CloudContext ctx() {
CloudContext c;
c.project = "test-proj";
c.location = "us-central1";
return c;
}
const char* kSet = "projects/test-proj/locations/us-central1/evaluationSets/s1";
const char* kRun = "projects/test-proj/locations/us-central1/evaluationRuns/r1";
const char* kMetric = "projects/test-proj/locations/us-central1/evaluationMetrics/m1";
bool is_invalid(const std::function<void()>& f) {
try {
f();
} catch (const GoogleError& e) {
return e.http_status() == 400 && e.status() == "INVALID_ARGUMENT";
}
return false;
}
} // namespace
TEST(optimize_calls, table_is_well_formed) {
std::set<std::string> ids;
ASSERT_TRUE(all_calls().size() >= 38);
for (const CallSpec& c : all_calls()) {
ASSERT_TRUE(ids.insert(c.id).second); // unique rpc ids
c.method(); // known verb
ASSERT_TRUE(c.path[0] == '/');
std::string svc = c.service;
ASSERT_TRUE(svc == "aiplatform" || svc == "cloudtrace" || svc == "logging");
if (std::string(c.http) == "GET" || std::string(c.http) == "DELETE")
ASSERT_TRUE(std::string(c.body_fields).empty());
}
ASSERT_THROW(call_spec("nope"), std::logic_error);
ASSERT_TRUE(find_call("nope") == nullptr);
}
TEST(optimize_calls, expand_path_validates) {
const CallSpec& s = call_spec(rpc::kEvaluateInstances);
ASSERT_EQ(expand_path(s, {{"location", "projects/p/locations/l"}}),
"/projects/p/locations/l:evaluateInstances");
ASSERT_TRUE(is_invalid([&] { expand_path(s, {{"location", "../x"}}); }));
ASSERT_TRUE(is_invalid([&] { expand_path(s, {{"location", "locations/l"}}); }));
ASSERT_TRUE(is_invalid([&] { expand_path(s, {{"location", "projects/p/../q"}}); }));
ASSERT_TRUE(is_invalid([&] { expand_path(s, {}); }));
const CallSpec& t = call_spec(rpc::kTraceGet);
ASSERT_EQ(expand_path(t, {{"projectId", "p"}, {"traceId", "abc"}}), "/projects/p/traces/abc");
ASSERT_TRUE(is_invalid([&] { expand_path(t, {{"projectId", "a/b"}, {"traceId", "x"}}); }));
}
TEST(optimize_calls, query_and_body_checked_against_table) {
const CallSpec& list = call_spec(rpc_id("evaluationRuns", "list"));
ASSERT_EQ(with_query(list, "/x", {{"pageSize", "5"}, {"pageToken", "a b"}, {"filter", ""}}),
"/x?pageSize=5&pageToken=a%20b");
ASSERT_TRUE(is_invalid([&] { with_query(list, "/x", {{"evil", "1"}}); }));
const CallSpec& ev = call_spec(rpc::kEvaluateInstances);
Json ok = Json::object();
ok.set("safetyInput", Json::object());
check_body(ev, ok);
Json bad = Json::object();
bad.set("notAField", Json(1));
ASSERT_TRUE(is_invalid([&] { check_body(ev, bad); }));
ASSERT_TRUE(is_invalid([&] { check_body(ev, Json(3)); }));
}
TEST(optimize_requests, every_builder_only_emits_table_fields) {
const CallSpec& ev = call_spec(rpc::kEvaluateInstances);
std::vector<Json> bodies;
bodies.push_back(EvaluationRequests::safety("hello"));
bodies.push_back(EvaluationRequests::exact_match({{"a", "a"}}));
for (auto m : {TrajectoryMetric::kExactMatch, TrajectoryMetric::kInOrderMatch,
TrajectoryMetric::kAnyOrderMatch, TrajectoryMetric::kPrecision,
TrajectoryMetric::kRecall})
bodies.push_back(EvaluationRequests::trajectory(m, {{"f", "{}"}}, {{"f", "{}"}}));
bodies.push_back(EvaluationRequests::trajectory(TrajectoryMetric::kSingleToolUse,
{{"f", "{}"}}, {}, "f"));
bodies.push_back(EvaluationRequests::pointwise("rate {response}", {{"response", "x"}}, "judge"));
bodies.push_back(EvaluationRequests::pairwise("pick {baseline} {candidate}",
{{"baseline", "a"}, {"candidate", "b"}}));
for (Json& b : bodies) {
EvaluationRequests::with_autorater(b, {"gemini-2.5-flash", 3, true});
check_body(ev, b); // top-level fields exist in Discovery's request schema
}
}
TEST(optimize_requests, shapes) {
Json s = EvaluationRequests::safety("hi");
ASSERT_EQ(s.find("safetyInput")->find("instance")->find("prediction")->as_string(), "hi");
Json t = EvaluationRequests::trajectory(TrajectoryMetric::kExactMatch,
{{"search", "{\"q\":1}"}}, {{"search", "{\"q\":1}"}});
const Json& inst = t.find("trajectoryExactMatchInput")->find("instances")->as_array()[0];
ASSERT_EQ(inst.find("predictedTrajectory")->find("toolCalls")->as_array()[0].find("toolName")->as_string(),
"search");
Json p = EvaluationRequests::pointwise("t {a}", {{"a", "1"}});
// jsonInstance is a JSON document inside a string (Discovery: string).
ASSERT_TRUE(p.find("pointwiseMetricInput")->find("instance")->find("jsonInstance")->is_string());
Json c = EvaluationRequests::safety("x");
EvaluationRequests::with_autorater(c, {"m", 2, false});
ASSERT_EQ(c.find("autoraterConfig")->find("samplingCount")->as_int(), 2);
ASSERT_TRUE(c.find("autoraterConfig")->find("flipEnabled") == nullptr);
}
TEST(optimize_requests, validation_and_caps) {
ASSERT_TRUE(is_invalid([] { EvaluationRequests::safety(""); }));
ASSERT_TRUE(is_invalid([] { EvaluationRequests::safety(std::string(20001, 'x')); }));
ASSERT_TRUE(is_invalid([] { EvaluationRequests::exact_match({}); }));
std::vector<TrajectoryCall> many(101, TrajectoryCall{"f", "{}"});
ASSERT_TRUE(is_invalid([&] {
EvaluationRequests::trajectory(TrajectoryMetric::kExactMatch, many, {});
}));
ASSERT_TRUE(is_invalid([] {
EvaluationRequests::trajectory(TrajectoryMetric::kSingleToolUse, {{"f", "{}"}}, {}, "");
}));
ASSERT_TRUE(is_invalid([] { EvaluationRequests::pointwise("", {{"a", "b"}}); }));
ASSERT_TRUE(is_invalid([] { EvaluationRequests::pointwise("t", {}); }));
ASSERT_TRUE(is_invalid([] {
EvaluationRequests::pairwise("t", {{"baseline", "a"}}); // candidate missing
}));
Json r = Json::object();
ASSERT_TRUE(is_invalid([&] { EvaluationRequests::with_autorater(r, {"", 33, false}); }));
}
TEST(optimize_requests, parse_result_unions) {
auto a = EvaluationRequests::parse(Json::parse(
"{\"exactMatchResults\":{\"exactMatchMetricValues\":[{\"score\":1},{\"score\":0}]}}"));
ASSERT_EQ(a.kind, "exactMatchResults");
ASSERT_EQ(a.scores.size(), 2u);
auto b = EvaluationRequests::parse(Json::parse(
"{\"safetyResult\":{\"score\":0.5,\"explanation\":\"meh\",\"confidence\":0.9}}"));
ASSERT_EQ(b.scores[0], 0.5);
ASSERT_EQ(b.explanation, "meh");
auto c = EvaluationRequests::parse(Json::parse(
"{\"pairwiseMetricResult\":{\"pairwiseChoice\":\"TIE\",\"explanation\":\"x\"}}"));
ASSERT_EQ(c.choice, "TIE");
ASSERT_TRUE(EvaluationRequests::parse(Json::object()).kind.empty());
}
TEST(optimize_api, evaluation_api_rejects_before_traffic) {
OfflineClient off;
EvaluationApi api(off.client, ctx());
ASSERT_TRUE(is_invalid([&] { api.evaluate_instances(Json::object()); }));
Json bad = Json::object();
bad.set("nope", Json(1));
ASSERT_TRUE(is_invalid([&] { api.evaluate_instances(bad); }));
ASSERT_TRUE(is_invalid([&] { api.generate_loss_clusters("bad", {{"m", "c"}}); }));
ASSERT_TRUE(is_invalid([&] { api.generate_loss_clusters(kSet, {}); }));
ScenarioRequest r;
r.count = 101;
ASSERT_TRUE(is_invalid([&] { api.generate_user_scenarios(r); }));
r.count = 3;
ASSERT_TRUE(is_invalid([&] { api.generate_user_scenarios(r); })); // no agents
ASSERT_EQ(off.mock->send_count(), 0u);
}
TEST(optimize_api, no_credential_is_loud_503) {
OfflineClient off(false);
EvaluationApi api(off.client, ctx());
try {
api.evaluate(EvaluationRequests::safety("hi"));
FAIL("expected NotConfiguredError");
} catch (const NotConfiguredError& e) {
ASSERT_EQ(e.http_status(), 503);
}
ASSERT_EQ(off.mock->send_count(), 0u);
}
TEST(optimize_api, scenario_body_matches_discovery_shape) {
ScenarioRequest r;
r.agents = Json::parse("{\"root\":{\"agentType\":\"LlmAgent\",\"instruction\":\"help\"}}");
r.root_agent_id = "root";
r.count = 7;
r.instruction = "adversarial";
Json b = EvaluationApi::scenario_body(r);
check_body(call_spec(rpc::kGenerateUserScenarios), b);
const Json* cfg = b.find("userScenarioGenerationConfig");
// Discovery types userScenarioCount as a string (int64).
ASSERT_EQ(cfg->find("userScenarioCount")->as_string(), "7");
r.root_agent_id = "other";
ASSERT_TRUE(is_invalid([&] { EvaluationApi::scenario_body(r); }));
}
TEST(optimize_api, resources_validate_names_verbs_and_caps) {
OfflineClient off;
EvaluationResources res(off.client, ctx());
ASSERT_TRUE(is_invalid([&] { res.get(Collection::kEvaluationRuns, kSet); })); // wrong collection
ASSERT_TRUE(is_invalid([&] { res.get(Collection::kEvaluationRuns, "../../x"); }));
ListOptions bad;
bad.page_size = 0;
ASSERT_TRUE(is_invalid([&] { res.list(Collection::kEvaluationRuns, bad); }));
bad.page_size = 10;
bad.max_pages = 21;
ASSERT_TRUE(is_invalid([&] { res.list(Collection::kEvaluationRuns, bad); }));
bad.max_pages = 1;
bad.filter = std::string(1001, 'x');
ASSERT_TRUE(is_invalid([&] { res.list(Collection::kEvaluationRuns, bad); }));
// evaluationItems/evaluationMetrics/evaluationRuns have no patch in Discovery.
Json body = Json::object();
ASSERT_TRUE(is_invalid([&] { res.patch(Collection::kEvaluationRuns, kRun, body, "displayName"); }));
ASSERT_TRUE(is_invalid([&] { res.action(Collection::kEvaluationRuns, kRun, "activate"); }));
ASSERT_TRUE(is_invalid([&] { res.action(Collection::kOnlineEvaluators, kRun, "suspend"); }));
ASSERT_TRUE(is_invalid([&] { res.create(Collection::kEvaluationRuns, body, "id1"); }));
Json unknown = Json::object();
unknown.set("bogus", Json(1));
ASSERT_TRUE(is_invalid([&] { res.create(Collection::kEvaluationSets, unknown); }));
ASSERT_TRUE(is_invalid([&] { res.delete_resource(Collection::kEvaluationMetrics, kSet); }));
ASSERT_EQ(off.mock->send_count(), 0u);
(void)kMetric;
}
TEST(optimize_api, run_spec_body_matches_discovery) {
EvaluationRunSpec s;
s.display_name = "nightly";
s.evaluation_set = kSet;
s.predefined_metrics = {"final_response_quality_v1"};
s.agent_engine = "projects/test-proj/locations/us-central1/reasoningEngines/e1";
s.simulator_max_turns = 5;
s.simulator_model = "gemini-2.5-flash";
s.autorater_samples = 3;
Json b = s.to_json();
check_body(call_spec(rpc_id("evaluationRuns", "create")), b);
const Json* arc = b.find("inferenceConfigs")->find("candidate")->find("agentRunConfig");
ASSERT_EQ(arc->find("userSimulatorConfig")->find("maxTurn")->as_int(), 5);
ASSERT_EQ(b.find("dataSource")->find("evaluationSet")->as_string(), kSet);
EvaluationRunSpec bad = s;
bad.evaluation_set = "x";
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
bad = s;
bad.predefined_metrics.clear();
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
bad = s;
bad.simulator_max_turns = 13;
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
bad = s;
bad.predefined_metrics.assign(17, "m");
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
}
TEST(optimize_api, online_evaluator_and_item_specs) {
OnlineEvaluatorSpec o;
o.display_name = "live";
o.agent_resource = "projects/test-proj/locations/us-central1/reasoningEngines/e1";
o.metric_resource_names = {kMetric};
o.sampling_percentage = 20;
o.max_samples_per_run = 50;
Json b = o.to_json();
check_body(call_spec(rpc_id("onlineEvaluators", "create")), b);
// int64 is a string in Discovery's JSON mapping.
ASSERT_EQ(b.find("config")->find("maxEvaluatedSamplesPerRun")->as_string(), "50");
OnlineEvaluatorSpec bad = o;
bad.sampling_percentage = 0;
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
bad = o;
bad.max_samples_per_run = 0; // 0 means unbounded at Google: refused
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
bad = o;
bad.metric_resource_names = {"nope"};
ASSERT_TRUE(is_invalid([&] { bad.to_json(); }));
EvaluationItemSpec item;
item.display_name = "case 1";
item.scenario_start = "hi";
item.scenario_plan = "ask for a refund";
Json ib = item.to_json();
check_body(call_spec(rpc_id("evaluationItems", "create")), ib);
ASSERT_EQ(ib.find("evaluationItemType")->as_string(), "REQUEST");
item.prompt = "also a prompt";
ASSERT_TRUE(is_invalid([&] { item.to_json(); }));
}
+117
View File
@@ -0,0 +1,117 @@
// Opt-in live checks. Run with MOLO_LIVE_NET=1 plus credentials; otherwise
// each test prints a loud SKIP. Never prints a credential or the project id.
// Metered but tiny: one computation-metric call, one autorater call.
#include <cstdlib>
#include <iostream>
#include "framework.h"
#include "google/credentials.h"
#include "google/optimize/autorater.h"
#include "google/optimize/direct_model.h"
#include "google/optimize/eval_resources.h"
#include "google/optimize/evaluation_api.h"
#include "google/optimize/observability.h"
using namespace molo::google;
using namespace molo::google::optimize;
namespace {
bool live(const char* name, bool need_project) {
if (!std::getenv("MOLO_LIVE_NET") || Credentials::env().empty() ||
(need_project && Credentials::env("GOOGLE_CLOUD_PROJECT").empty())) {
std::cout << "SKIP " << name
<< " (set MOLO_LIVE_NET=1, a Google credential"
<< (need_project ? " and GOOGLE_CLOUD_PROJECT" : "") << ")\n";
return false;
}
return true;
}
} // namespace
TEST(optimize_live, evaluate_instances_computation_metrics) {
if (!live("optimize_live.evaluate_instances_computation_metrics", true)) return;
// Verified 2026-10-24: an express key is accepted for computation metrics.
EvaluationApi api(GoogleClient::from_env(), CloudContext::from_env());
MetricOutcome match = api.evaluate(EvaluationRequests::exact_match({{"a", "a"}, {"a", "b"}}));
ASSERT_EQ(match.kind, "exactMatchResults");
ASSERT_EQ(match.scores.size(), 2u);
ASSERT_EQ(match.scores[0], 1.0);
ASSERT_EQ(match.scores[1], 0.0);
MetricOutcome traj = api.evaluate(EvaluationRequests::trajectory(
TrajectoryMetric::kExactMatch, {{"f", "{}"}}, {{"f", "{}"}}));
ASSERT_EQ(traj.kind, "trajectoryExactMatchResults");
ASSERT_EQ(traj.scores.at(0), 1.0);
}
TEST(optimize_live, resource_lists_need_oauth) {
if (!live("optimize_live.resource_lists_need_oauth", true)) return;
EvaluationResources res(GoogleClient::from_env(), CloudContext::from_env());
if (Credentials::env("GOOGLE_ACCESS_TOKEN").empty()) {
// Verified 2026-10-24: API keys are refused with 401 UNAUTHENTICATED.
try {
res.list(Collection::kEvaluationRuns, ListOptions{1, "", "", 1});
FAIL("an API key should be refused");
} catch (const GoogleError& e) {
ASSERT_EQ(e.http_status(), 401);
}
return;
}
// Read-only lists of every evaluation collection, one item per page.
for (Collection c : {Collection::kEvaluationSets, Collection::kEvaluationItems,
Collection::kEvaluationMetrics, Collection::kEvaluationRuns,
Collection::kOnlineEvaluators}) {
molo::clients::body::Json items = res.list(c, ListOptions{1, "", "", 1});
ASSERT_TRUE(items.is_array());
std::cout << " " << collection_name(c) << ": " << items.as_array().size() << " item(s)\n";
}
}
TEST(optimize_live, observability_reads_with_oauth) {
if (!live("optimize_live.observability_reads_with_oauth", true)) return;
if (Credentials::env("GOOGLE_ACCESS_TOKEN").empty()) {
std::cout << "SKIP optimize_live.observability_reads_with_oauth (needs GOOGLE_ACCESS_TOKEN)\n";
return;
}
auto client = GoogleClient::from_env();
std::string project = CloudContext::from_env().project;
LogQuery lq;
lq.resource_names = {"projects/" + project};
lq.page_size = 1;
lq.max_entries = 1;
std::vector<LogEntry> logs = LogReader(client).entries(lq);
ASSERT_TRUE(logs.size() <= 1);
TraceQuery tq;
tq.page_size = 1;
tq.max_pages = 1;
tq.max_traces = 1;
std::vector<Trace> traces = TraceReader(client).list(project, tq);
ASSERT_TRUE(traces.size() <= 1);
std::cout << " logs " << logs.size() << ", traces " << traces.size() << "\n";
}
TEST(optimize_live, llm_judged_evaluate_instances) {
if (!live("optimize_live.llm_judged_evaluate_instances", true)) return;
EvaluationApi api(GoogleClient::from_env(), CloudContext::from_env());
// One tiny LLM-judged instance. Outcome is recorded, not asserted: the
// default autorater model may be unavailable to a new project (404).
try {
MetricOutcome out = api.evaluate(EvaluationRequests::safety("Hello, how can I help you today?"));
std::cout << " safety result: " << out.kind << " scores " << out.scores.size() << "\n";
ASSERT_EQ(out.kind, "safetyResult");
} catch (const GoogleError& e) {
std::cout << " safety evaluateInstances refused: " << e.http_status() << " " << e.status() << "\n";
}
}
TEST(optimize_live, autorater_parses_a_real_model_reply) {
if (!live("optimize_live.autorater_parses_a_real_model_reply", false)) return;
auto model = DirectModel::from_env(GoogleClient::from_env());
Autorater rater(model);
Conversation c = Conversation::parse_transcript(
"user: What is 2+2?\nagent: 4.", "live");
Score s = rater.score("task_success", c, Autorater::task_success_rubrics());
ASSERT_EQ(s.verdicts.size(), 3u);
for (const RubricVerdict& v : s.verdicts)
ASSERT_TRUE(v.explanation != "no verdict returned"); // the reply parsed
std::cout << " autorater score " << s.value << "\n";
}
+339
View File
@@ -0,0 +1,339 @@
// Local logic against a fake LanguageModel: autorater, simulator, clusterer,
// optimizer, JSON extraction. Deterministic, no network.
#include "framework.h"
#include "google/errors.h"
#include "google/optimize/autorater.h"
#include "google/optimize/clusterer.h"
#include "google/optimize/model_json.h"
#include "google/optimize/optimizer.h"
#include "google/optimize/simulator.h"
#include "fakes.h"
using namespace molo::google;
using namespace molo::google::optimize;
using molo::clients::body::Json;
using optimize_test::FakeModel;
using optimize_test::reply_call;
using optimize_test::reply_text;
namespace {
bool is_invalid(const std::function<void()>& f) {
try {
f();
} catch (const GoogleError& e) {
return e.http_status() == 400;
}
return false;
}
Conversation sample() {
return Conversation::parse_transcript("user: refund please\nagent: done, refunded 10 EUR");
}
} // namespace
TEST(optimize_json, extract_tolerates_fences_and_prose) {
Json a = JsonModelCall::extract("sure!\n```json\n{\"a\":{\"b\":\"}\"},\"c\":1}\n```\nbye");
ASSERT_TRUE(a.is_object());
ASSERT_EQ(a.find("c")->as_int(), 1);
ASSERT_TRUE(JsonModelCall::extract("no json here").is_null());
ASSERT_TRUE(JsonModelCall::extract("{broken").is_null());
Json second = JsonModelCall::extract("{nope} then {\"ok\":true}");
ASSERT_TRUE(second.is_object() && second.find("ok") != nullptr);
}
TEST(optimize_json, retries_once_then_fails_loud) {
auto model = FakeModel::texts({"not json", "still not json"});
JsonModelCall call(model);
try {
call.ask("s", "u");
FAIL("expected BAD_RESPONSE");
} catch (const GoogleError& e) {
ASSERT_EQ(e.status(), "BAD_RESPONSE");
}
ASSERT_EQ(call.calls(), 2); // one try plus one retry, no more
auto ok = FakeModel::texts({"oops", "{\"x\":1}"});
ASSERT_EQ(JsonModelCall(ok).ask("s", "u").find("x")->as_int(), 1);
}
TEST(optimize_json, request_asks_for_json_deterministically) {
auto model = FakeModel::texts({"{}"});
JsonModelCall(model, "m1").ask("sys", "user");
const GenerateRequest& r = model->requests[0];
ASSERT_EQ(r.model, "m1");
ASSERT_EQ(r.config.response_mime_type, "application/json");
ASSERT_EQ(*r.config.temperature, 0.0);
ASSERT_EQ(r.system->parts[0].text, "sys");
}
TEST(optimize_conversation, parse_and_limits) {
Conversation c = Conversation::parse_transcript("user: hi\nthere\nagent: hello");
ASSERT_EQ(c.turns.size(), 2u);
ASSERT_EQ(c.turns[0].text, "hi\nthere");
ASSERT_TRUE(is_invalid([] { Conversation::parse_transcript(""); }));
ASSERT_TRUE(is_invalid([] { Conversation::parse_transcript("hello without role"); }));
std::string many;
for (int i = 0; i < 51; ++i) many += "user: x\n";
ASSERT_TRUE(is_invalid([&] { Conversation::parse_transcript(many); }));
Conversation big;
big.turns.push_back({"user", std::string(20000, 'a'), {}});
big.turns.push_back({"agent", std::string(20000, 'b'), {}});
big.turns.push_back({"user", std::string(20000, 'c'), {}});
big.turns.push_back({"agent", std::string(20000, 'd'), {}});
ASSERT_TRUE(big.transcript().find("[transcript truncated]") != std::string::npos);
}
TEST(optimize_autorater, scores_whole_conversation_by_rubric) {
auto model = FakeModel::texts({
"{\"verdicts\":[{\"id\":\"goal_reached\",\"pass\":true,\"explanation\":\"ok\"},"
"{\"id\":\"no_contradiction\",\"pass\":true,\"explanation\":\"ok\"},"
"{\"id\":\"no_unneeded_steps\",\"pass\":false,\"explanation\":\"asked twice\"}]}"});
Autorater rater(model);
Score s = rater.score("task_success", sample(), Autorater::task_success_rubrics());
ASSERT_FALSE(s.pass);
ASSERT_TRUE(s.value > 0.66 && s.value < 0.67);
ASSERT_TRUE(s.explanation.find("asked twice") != std::string::npos);
// One call covers the whole conversation, and the prompt carries every turn.
ASSERT_EQ(model->requests.size(), 1u);
const std::string& prompt = model->requests[0].contents[0].text();
ASSERT_TRUE(prompt.find("user: refund please") != std::string::npos);
ASSERT_TRUE(prompt.find("agent: done, refunded") != std::string::npos);
}
TEST(optimize_autorater, missing_verdict_fails_loudly) {
auto model = FakeModel::texts({"{\"verdicts\":[{\"id\":\"goal_reached\",\"pass\":true}]}"});
Score s = Autorater(model).score("t", sample(), Autorater::task_success_rubrics());
ASSERT_FALSE(s.pass);
ASSERT_EQ(s.verdicts.size(), 3u);
ASSERT_EQ(s.verdicts[1].explanation, "no verdict returned");
}
TEST(optimize_autorater, validation) {
auto model = FakeModel::texts({"{}"});
Autorater rater(model);
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {}); }));
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {{"bad id", "q"}}); }));
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {{"a", "q"}, {"a", "q2"}}); }));
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), {{"a", ""}}); }));
ASSERT_TRUE(is_invalid([&] { rater.score("t", Conversation{}, {{"a", "q"}}); }));
std::vector<Rubric> many;
for (int i = 0; i < 17; ++i) many.push_back({"r" + std::to_string(i), "q"});
ASSERT_TRUE(is_invalid([&] { rater.score("t", sample(), many); }));
ASSERT_EQ(model->requests.size(), 0u); // nothing reached the model
}
TEST(optimize_tools, fixed_unknown_and_model_played) {
auto model = FakeModel::texts({"{\"balance\":42}"});
VirtualToolbox box(model);
box.add({"lookup", "finds things", Json(), Json::parse("{\"found\":true}")});
box.add({"balance", "account balance", Json(), Json()});
FunctionCall a;
a.name = "lookup";
ASSERT_EQ(box.call(a).find("found")->as_bool(), true);
FunctionCall b;
b.name = "balance";
b.args.set("account", Json("x"));
ASSERT_EQ(box.call(b).find("balance")->as_int(), 42);
FunctionCall c;
c.name = "ghost";
ASSERT_EQ(box.call(c).find("error")->as_string(), "unknown_tool");
ASSERT_EQ(box.calls(), 3);
ASSERT_EQ(box.unknown_calls(), 1);
ASSERT_EQ(box.declarations()[0].functions.size(), 2u);
ASSERT_TRUE(is_invalid([&] { box.add({"lookup", "", Json(), Json()}); }));
ASSERT_TRUE(is_invalid([&] { box.add({"a b", "", Json(), Json()}); }));
ASSERT_TRUE(is_invalid([&] { box.add({"t", "", Json(), Json(3)}); }));
}
namespace {
// Scripted user: first message comes from the scenario, then "more", then done.
std::shared_ptr<FakeModel> user_model(int messages) {
auto n = std::make_shared<int>(0);
return std::make_shared<FakeModel>([n, messages](const GenerateRequest&) {
if (++*n > messages) return reply_text("{\"done\":true,\"message\":\"\"}");
return reply_text("{\"done\":false,\"message\":\"follow up " + std::to_string(*n) + "\"}");
});
}
} // namespace
TEST(optimize_simulator, runs_tools_and_stops_when_user_is_done) {
auto agent_model = std::make_shared<FakeModel>([](const GenerateRequest& req) {
// First request of a turn ends in the user's text, the follow-up request
// after a tool call ends in a functionResponse.
const Content& last = req.contents.back();
if (!last.parts.empty() && last.parts[0].function_response)
return reply_text("Your balance is 42.");
if (last.text().find("balance") != std::string::npos)
return reply_call("balance", Json::object());
return reply_text("How can I help?");
});
ModelAgent agent(agent_model, "You are a bank bot.", {});
VirtualToolbox box;
box.add({"balance", "returns balance", Json(), Json::parse("{\"balance\":42}")});
Scenario sc{"balance check", "what is my balance", "check the balance, then thank the agent"};
Simulator sim(user_model(1), FakeModel::texts({"{}"}));
SimulationResult r = sim.run(sc, agent, box, SimulationConfig{5});
ASSERT_EQ(r.stop_reason, "user_done");
ASSERT_EQ(r.tool_calls, 1);
ASSERT_EQ(r.conversation.turns.size(), 4u); // user, agent, user, agent
ASSERT_EQ(r.conversation.turns[1].text, "Your balance is 42.");
ASSERT_EQ(r.conversation.turns[1].calls[0].name, "balance");
ASSERT_EQ(r.conversation.turns[1].calls[0].result.find("balance")->as_int(), 42);
}
TEST(optimize_simulator, caps_turns_tool_rounds_and_tool_calls) {
auto chatty = std::make_shared<FakeModel>(
[](const GenerateRequest&) { return reply_text("ok"); });
ModelAgent agent(chatty, "x");
VirtualToolbox box;
Scenario sc{"t", "hi", "keep talking"};
Simulator endless(user_model(1000), FakeModel::texts({"{}"}));
SimulationResult r = endless.run(sc, agent, box, SimulationConfig{3});
ASSERT_EQ(r.stop_reason, "max_turns");
ASSERT_EQ(r.conversation.turns.size(), 6u);
ASSERT_TRUE(is_invalid([&] { endless.run(sc, agent, box, SimulationConfig{0}); }));
ASSERT_TRUE(is_invalid([&] { endless.run(sc, agent, box, SimulationConfig{13}); }));
// An agent that calls tools forever hits the per-turn round cap.
auto looper = std::make_shared<FakeModel>(
[](const GenerateRequest&) { return reply_call("ping", Json::object()); });
ModelAgent loop_agent(looper, "x");
VirtualToolbox pingbox;
pingbox.add({"ping", "p", Json(), Json::parse("{}")});
SimulationResult lr = endless.run(sc, loop_agent, pingbox, SimulationConfig{5});
ASSERT_EQ(lr.stop_reason, "tool_round_cap");
ASSERT_EQ(lr.tool_calls, 5);
Scenario bad{"t", "", "plan"};
ASSERT_TRUE(is_invalid([&] { endless.run(bad, agent, box); }));
}
TEST(optimize_simulator, scores_task_success_and_safety) {
auto agent_model = std::make_shared<FakeModel>(
[](const GenerateRequest&) { return reply_text("Here you go."); });
ModelAgent agent(agent_model, "x");
VirtualToolbox box;
auto judge = std::make_shared<FakeModel>([](const GenerateRequest& req) {
std::string prompt = req.contents[0].text();
bool safety = prompt.find("harmful") != std::string::npos;
std::string v = "{\"verdicts\":[";
// Every rubric id appears in the prompt as "- id: question".
std::size_t pos = 0;
bool first = true;
while ((pos = prompt.find("\n- ", pos)) != std::string::npos) {
std::size_t colon = prompt.find(':', pos);
std::string id = prompt.substr(pos + 3, colon - pos - 3);
v += std::string(first ? "" : ",") + "{\"id\":\"" + id + "\",\"pass\":" +
(safety || id != "plan_goals" ? "true" : "false") + ",\"explanation\":\"e\"}";
first = false;
pos = colon;
}
return reply_text(v + "]}");
});
Simulator sim(user_model(0), judge);
SimulationReport rep = sim.run_and_score({"t", "hi", "get a thing"}, agent, box);
ASSERT_TRUE(rep.safety.pass);
ASSERT_FALSE(rep.task_success.pass); // plan_goals failed
ASSERT_FALSE(rep.pass());
ASSERT_EQ(rep.to_json().find("pass")->as_bool(), false);
ASSERT_EQ(rep.task_success.verdicts.size(), 4u); // 3 built-in plus plan_goals
}
TEST(optimize_clusterer, verifies_model_output) {
auto model = FakeModel::texts({
"{\"clusters\":["
"{\"label\":\"wrong tool\",\"description\":\"d\",\"failure_ids\":[\"f1\",\"f2\",\"ghost\"]},"
"{\"label\":\"rude\",\"description\":\"d\",\"failure_ids\":[\"f2\",\"f3\"]},"
"{\"label\":\"empty\",\"description\":\"d\",\"failure_ids\":[\"ghost2\"]}]}"});
FailureClusterer fc(model);
ClusterReport r = fc.cluster({{"f1", "called wrong tool", ""}, {"f2", "wrong tool again", ""},
{"f3", "rude tone", ""}, {"f4", "timeout", ""}});
ASSERT_EQ(r.clusters.size(), 2u);
ASSERT_EQ(r.clusters[0].label, "wrong tool");
ASSERT_EQ(r.clusters[0].failure_ids.size(), 2u); // ghost dropped
ASSERT_EQ(r.clusters[1].failure_ids.size(), 1u); // f2 already used
ASSERT_EQ(r.unassigned.size(), 1u);
ASSERT_EQ(r.unassigned[0], "f4"); // loud, not silently placed
ASSERT_EQ(r.to_json().find("unassigned")->as_array().size(), 1u);
}
TEST(optimize_clusterer, caps_and_validation) {
auto model = FakeModel::texts({"{\"clusters\":[]}"});
FailureClusterer fc(model);
ASSERT_TRUE(is_invalid([&] { fc.cluster({}); }));
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "x", ""}, {"a", "y", ""}}); }));
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"bad id", "x", ""}}); }));
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "", ""}}); }));
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "x", ""}}, 0); }));
ASSERT_TRUE(is_invalid([&] { fc.cluster({{"a", "x", ""}}, 11); }));
std::vector<Failure> many;
for (int i = 0; i < 205; ++i) many.push_back({"f" + std::to_string(i), "boom", ""});
ClusterReport r = fc.cluster(many);
ASSERT_EQ(r.dropped, 5u);
ASSERT_EQ(r.unassigned.size(), 200u);
// More clusters than the cap are cut and their members become unassigned.
auto wide = FakeModel::texts({
"{\"clusters\":[{\"label\":\"a\",\"failure_ids\":[\"x\"]},"
"{\"label\":\"b\",\"failure_ids\":[\"y\"]}]}"});
ClusterReport w = FailureClusterer(wide).cluster({{"x", "s", ""}, {"y", "s", ""}}, 1);
ASSERT_EQ(w.clusters.size(), 1u);
ASSERT_EQ(w.unassigned.size(), 1u);
}
namespace {
ClusterReport one_cluster() {
ClusterReport r;
r.clusters.push_back({"wrong tool", "calls search instead of lookup", {"f1"}});
return r;
}
std::vector<Failure> failures() { return {{"f1", "used search for an order lookup", ""}}; }
} // namespace
TEST(optimize_optimizer, suggest_returns_refined_instruction) {
auto model = FakeModel::texts({"{\"instruction\":\"Use lookup for orders.\",\"rationale\":\"r\"}"});
Suggestion s = InstructionOptimizer(model).suggest("Be helpful.", one_cluster(), failures());
ASSERT_EQ(s.instruction, "Use lookup for orders.");
ASSERT_TRUE(s.changed);
ASSERT_FALSE(s.improved); // nothing was measured
const std::string& prompt = model->requests[0].contents[0].text();
ASSERT_TRUE(prompt.find("Be helpful.") != std::string::npos);
ASSERT_TRUE(prompt.find("wrong tool") != std::string::npos);
}
TEST(optimize_optimizer, optimize_never_returns_a_worse_instruction) {
auto model = FakeModel::texts({"{\"instruction\":\"worse\",\"rationale\":\"r\"}"});
Suggestion s = InstructionOptimizer(model).optimize(
"orig", one_cluster(), failures(), [](const std::string& i) { return i == "orig" ? 0.6 : 0.3; });
ASSERT_EQ(s.instruction, "orig");
ASSERT_FALSE(s.improved);
ASSERT_FALSE(s.changed);
auto better = FakeModel::texts({"{\"instruction\":\"v2\",\"rationale\":\"r2\"}",
"{\"instruction\":\"v3\",\"rationale\":\"r3\"}"});
Suggestion b = InstructionOptimizer(better).optimize(
"orig", one_cluster(), failures(),
[](const std::string& i) { return i == "orig" ? 0.2 : i == "v2" ? 0.5 : 0.7; }, 3);
ASSERT_EQ(b.instruction, "v3");
ASSERT_TRUE(b.improved);
ASSERT_EQ(b.rounds, 3); // round 3: the model repeated v3, unchanged, search ends
ASSERT_EQ(b.best_score, 0.7);
ASSERT_EQ(b.baseline_score, 0.2);
}
TEST(optimize_optimizer, validation_and_caps) {
auto model = FakeModel::texts({"{\"instruction\":\"\"}"});
InstructionOptimizer opt(model);
ASSERT_TRUE(is_invalid([&] { opt.suggest("", one_cluster(), failures()); }));
ASSERT_TRUE(is_invalid([&] { opt.suggest(std::string(8001, 'x'), one_cluster(), failures()); }));
ASSERT_TRUE(is_invalid([&] { opt.suggest("x", ClusterReport{}, failures()); }));
auto scorer = [](const std::string&) { return 0.0; };
ASSERT_TRUE(is_invalid([&] { opt.optimize("x", one_cluster(), failures(), scorer, 0); }));
ASSERT_TRUE(is_invalid([&] { opt.optimize("x", one_cluster(), failures(), scorer, 4); }));
ASSERT_TRUE(is_invalid([&] { opt.optimize("x", one_cluster(), failures(), nullptr); }));
try {
opt.suggest("x", one_cluster(), failures()); // model returned an empty instruction
FAIL("expected BAD_RESPONSE");
} catch (const GoogleError& e) {
ASSERT_EQ(e.status(), "BAD_RESPONSE");
}
auto huge = FakeModel::texts({"{\"instruction\":\"" + std::string(8001, 'y') + "\"}"});
ASSERT_THROW(InstructionOptimizer(huge).suggest("x", one_cluster(), failures()), GoogleError);
}
@@ -0,0 +1,163 @@
// Observability: validation, request bodies, local parsing and dashboard
// math. No Google replies are scripted; parsing tests feed documents in the
// shape of the Discovery schemas straight to the parsers.
#include "framework.h"
#include "google/errors.h"
#include "google/optimize/observability.h"
#include "fakes.h"
using namespace molo::google;
using namespace molo::google::optimize;
using molo::clients::body::Json;
using optimize_test::OfflineClient;
namespace {
bool is_invalid(const std::function<void()>& f) {
try {
f();
} catch (const GoogleError& e) {
return e.http_status() == 400;
}
return false;
}
const char* kTrace = "0123456789abcdef0123456789abcdef";
} // namespace
TEST(optimize_observability, trace_reader_validates_before_traffic) {
OfflineClient off;
TraceReader r(off.client);
ASSERT_TRUE(is_invalid([&] { r.get("p", "short"); }));
ASSERT_TRUE(is_invalid([&] { r.get("UPPER", kTrace); }));
ASSERT_TRUE(is_invalid([&] { r.get("a/b", kTrace); }));
TraceQuery q;
q.view = "FULL";
ASSERT_TRUE(is_invalid([&] { r.list("p", q); }));
q = TraceQuery{};
q.page_size = 101;
ASSERT_TRUE(is_invalid([&] { r.list("p", q); }));
q = TraceQuery{};
q.max_traces = 501;
ASSERT_TRUE(is_invalid([&] { r.list("p", q); }));
q = TraceQuery{};
q.start_time = "yesterday";
ASSERT_TRUE(is_invalid([&] { r.list("p", q); }));
q = TraceQuery{};
q.filter = std::string(1001, 'x');
ASSERT_TRUE(is_invalid([&] { r.list("p", q); }));
ASSERT_EQ(off.mock->send_count(), 0u);
ASSERT_TRUE(TraceReader::valid_trace_id(kTrace));
ASSERT_FALSE(TraceReader::valid_trace_id("zzzz456789abcdef0123456789abcdef"));
}
TEST(optimize_observability, no_credential_is_loud) {
OfflineClient off(false);
TraceReader r(off.client);
ASSERT_THROW(r.get("proj", kTrace), NotConfiguredError);
LogReader l(off.client);
LogQuery q;
q.resource_names = {"projects/proj"};
ASSERT_THROW(l.entries(q), NotConfiguredError);
ASSERT_EQ(off.mock->send_count(), 0u);
}
TEST(optimize_observability, log_request_body_matches_discovery) {
LogQuery q;
q.resource_names = {"projects/proj"};
q.filter = LogQuery::trace_filter("proj", kTrace);
q.page_size = 20;
Json b = LogReader::request_body(q, "tok");
check_body(call_spec(rpc::kLogEntriesList), b);
ASSERT_EQ(b.find("orderBy")->as_string(), "timestamp desc");
ASSERT_EQ(b.find("pageToken")->as_string(), "tok");
ASSERT_EQ(b.find("filter")->as_string(),
std::string("trace=\"projects/proj/traces/") + kTrace + "\"");
LogQuery bad = q;
bad.resource_names.clear();
ASSERT_TRUE(is_invalid([&] { LogReader::request_body(bad, ""); }));
bad = q;
bad.resource_names.assign(101, "projects/proj");
ASSERT_TRUE(is_invalid([&] { LogReader::request_body(bad, ""); }));
bad = q;
bad.resource_names = {"folders/1"};
ASSERT_TRUE(is_invalid([&] { LogReader::request_body(bad, ""); }));
bad = q;
bad.filter = std::string(20001, 'x');
ASSERT_TRUE(is_invalid([&] { LogReader::request_body(bad, ""); }));
bad = q;
bad.page_size = 101;
ASSERT_TRUE(is_invalid([&] { LogReader::request_body(bad, ""); }));
bad = q;
bad.max_entries = 1001;
ASSERT_TRUE(is_invalid([&] { LogReader::request_body(bad, ""); }));
ASSERT_TRUE(is_invalid([] { LogQuery::trace_filter("proj\"", kTrace); }));
}
TEST(optimize_observability, trace_parse_and_root) {
Trace t = Trace::from_json(Json::parse(
"{\"projectId\":\"p\",\"traceId\":\"" + std::string(kTrace) + "\",\"spans\":["
"{\"spanId\":\"1\",\"name\":\"invoke_agent\",\"startTime\":\"2026-10-24T10:00:00.000Z\","
"\"endTime\":\"2026-10-24T10:00:02.500Z\",\"kind\":\"RPC_SERVER\"},"
"{\"spanId\":\"2\",\"parentSpanId\":\"1\",\"name\":\"call_llm\","
"\"startTime\":\"2026-10-24T10:00:00.100Z\",\"endTime\":\"2026-10-24T10:00:01.100Z\","
"\"labels\":{\"gen_ai.request.model\":\"gemini\"}}]}"));
ASSERT_EQ(t.spans.size(), 2u);
ASSERT_EQ(t.root()->name, "invoke_agent");
ASSERT_EQ(t.root()->duration_ms(), 2500);
ASSERT_EQ(t.spans[1].duration_ms(), 1000);
ASSERT_EQ(t.spans[1].labels.at("gen_ai.request.model"), "gemini");
Span bad;
bad.start_time = "garbage";
ASSERT_EQ(bad.duration_ms(), 0);
}
TEST(optimize_observability, span_cap) {
Json spans = Json::array();
for (int i = 0; i < 2100; ++i) {
Json s = Json::object();
s.set("spanId", Json(std::to_string(i)));
spans.push(s);
}
Json doc = Json::object();
doc.set("spans", spans);
ASSERT_EQ(Trace::from_json(doc).spans.size(), 2000u);
}
TEST(optimize_observability, conversation_from_logs_and_dashboard) {
std::vector<LogEntry> logs;
logs.push_back(LogEntry::from_json(Json::parse(
"{\"severity\":\"INFO\",\"jsonPayload\":{\"role\":\"user\",\"content\":\"hi\"}}")));
logs.push_back(LogEntry::from_json(Json::parse(
"{\"severity\":\"INFO\",\"jsonPayload\":{\"role\":\"model\",\"text\":\"hello\"}}")));
logs.push_back(LogEntry::from_json(Json::parse(
"{\"severity\":\"ERROR\",\"textPayload\":\"boom\"}")));
logs.push_back(LogEntry::from_json(Json::parse(
"{\"jsonPayload\":{\"role\":\"system\",\"content\":\"ignored\"}}")));
Conversation c = ConversationBuilder::from_logs(logs, "c1");
ASSERT_EQ(c.turns.size(), 2u);
ASSERT_EQ(c.turns[1].role, "agent");
std::vector<Trace> traces;
for (int ms : {100, 200, 300, 400, 1000}) {
Trace t;
Span s;
s.span_id = "1";
s.name = "invoke_agent";
s.start_time = "2026-10-24T10:00:00.000Z";
char end[40];
snprintf(end, sizeof end, "2026-10-24T10:00:%02d.%03dZ", ms / 1000, ms % 1000);
s.end_time = end;
t.spans.push_back(s);
traces.push_back(t);
}
DashboardData d = summarize(traces, logs);
ASSERT_EQ(d.traces, 5u);
ASSERT_EQ(d.p50_ms, 300);
ASSERT_EQ(d.p95_ms, 1000);
ASSERT_EQ(d.top_spans[0].first, "invoke_agent");
ASSERT_EQ(d.log_severity.at("INFO"), 2u);
ASSERT_EQ(d.log_severity.at("DEFAULT"), 1u);
ASSERT_EQ(d.to_json().find("traces")->as_int(), 5);
DashboardData empty = summarize({}, {});
ASSERT_EQ(empty.p95_ms, 0);
}