feat(clients): optimize: evaluation, simulation, observability and optimizer on a discovery-checked call table
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,163 @@
|
||||
// Observability: validation, request bodies, local parsing and dashboard
|
||||
// math. No Google replies are scripted; parsing tests feed documents in the
|
||||
// shape of the Discovery schemas straight to the parsers.
|
||||
#include "framework.h"
|
||||
#include "google/errors.h"
|
||||
#include "google/optimize/observability.h"
|
||||
#include "fakes.h"
|
||||
|
||||
using namespace molo::google;
|
||||
using namespace molo::google::optimize;
|
||||
using molo::clients::body::Json;
|
||||
using optimize_test::OfflineClient;
|
||||
|
||||
namespace {
|
||||
bool is_invalid(const std::function<void()>& f) {
|
||||
try {
|
||||
f();
|
||||
} catch (const GoogleError& e) {
|
||||
return e.http_status() == 400;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
const char* kTrace = "0123456789abcdef0123456789abcdef";
|
||||
} // namespace
|
||||
|
||||
TEST(optimize_observability, trace_reader_validates_before_traffic) {
|
||||
OfflineClient off;
|
||||
TraceReader r(off.client);
|
||||
ASSERT_TRUE(is_invalid([&] { r.get("p", "short"); }));
|
||||
ASSERT_TRUE(is_invalid([&] { r.get("UPPER", kTrace); }));
|
||||
ASSERT_TRUE(is_invalid([&] { r.get("a/b", kTrace); }));
|
||||
TraceQuery q;
|
||||
q.view = "FULL";
|
||||
ASSERT_TRUE(is_invalid([&] { r.list("p", q); }));
|
||||
q = TraceQuery{};
|
||||
q.page_size = 101;
|
||||
ASSERT_TRUE(is_invalid([&] { r.list("p", q); }));
|
||||
q = TraceQuery{};
|
||||
q.max_traces = 501;
|
||||
ASSERT_TRUE(is_invalid([&] { r.list("p", q); }));
|
||||
q = TraceQuery{};
|
||||
q.start_time = "yesterday";
|
||||
ASSERT_TRUE(is_invalid([&] { r.list("p", q); }));
|
||||
q = TraceQuery{};
|
||||
q.filter = std::string(1001, 'x');
|
||||
ASSERT_TRUE(is_invalid([&] { r.list("p", q); }));
|
||||
ASSERT_EQ(off.mock->send_count(), 0u);
|
||||
ASSERT_TRUE(TraceReader::valid_trace_id(kTrace));
|
||||
ASSERT_FALSE(TraceReader::valid_trace_id("zzzz456789abcdef0123456789abcdef"));
|
||||
}
|
||||
|
||||
TEST(optimize_observability, no_credential_is_loud) {
|
||||
OfflineClient off(false);
|
||||
TraceReader r(off.client);
|
||||
ASSERT_THROW(r.get("proj", kTrace), NotConfiguredError);
|
||||
LogReader l(off.client);
|
||||
LogQuery q;
|
||||
q.resource_names = {"projects/proj"};
|
||||
ASSERT_THROW(l.entries(q), NotConfiguredError);
|
||||
ASSERT_EQ(off.mock->send_count(), 0u);
|
||||
}
|
||||
|
||||
TEST(optimize_observability, log_request_body_matches_discovery) {
|
||||
LogQuery q;
|
||||
q.resource_names = {"projects/proj"};
|
||||
q.filter = LogQuery::trace_filter("proj", kTrace);
|
||||
q.page_size = 20;
|
||||
Json b = LogReader::request_body(q, "tok");
|
||||
check_body(call_spec(rpc::kLogEntriesList), b);
|
||||
ASSERT_EQ(b.find("orderBy")->as_string(), "timestamp desc");
|
||||
ASSERT_EQ(b.find("pageToken")->as_string(), "tok");
|
||||
ASSERT_EQ(b.find("filter")->as_string(),
|
||||
std::string("trace=\"projects/proj/traces/") + kTrace + "\"");
|
||||
|
||||
LogQuery bad = q;
|
||||
bad.resource_names.clear();
|
||||
ASSERT_TRUE(is_invalid([&] { LogReader::request_body(bad, ""); }));
|
||||
bad = q;
|
||||
bad.resource_names.assign(101, "projects/proj");
|
||||
ASSERT_TRUE(is_invalid([&] { LogReader::request_body(bad, ""); }));
|
||||
bad = q;
|
||||
bad.resource_names = {"folders/1"};
|
||||
ASSERT_TRUE(is_invalid([&] { LogReader::request_body(bad, ""); }));
|
||||
bad = q;
|
||||
bad.filter = std::string(20001, 'x');
|
||||
ASSERT_TRUE(is_invalid([&] { LogReader::request_body(bad, ""); }));
|
||||
bad = q;
|
||||
bad.page_size = 101;
|
||||
ASSERT_TRUE(is_invalid([&] { LogReader::request_body(bad, ""); }));
|
||||
bad = q;
|
||||
bad.max_entries = 1001;
|
||||
ASSERT_TRUE(is_invalid([&] { LogReader::request_body(bad, ""); }));
|
||||
ASSERT_TRUE(is_invalid([] { LogQuery::trace_filter("proj\"", kTrace); }));
|
||||
}
|
||||
|
||||
TEST(optimize_observability, trace_parse_and_root) {
|
||||
Trace t = Trace::from_json(Json::parse(
|
||||
"{\"projectId\":\"p\",\"traceId\":\"" + std::string(kTrace) + "\",\"spans\":["
|
||||
"{\"spanId\":\"1\",\"name\":\"invoke_agent\",\"startTime\":\"2026-10-24T10:00:00.000Z\","
|
||||
"\"endTime\":\"2026-10-24T10:00:02.500Z\",\"kind\":\"RPC_SERVER\"},"
|
||||
"{\"spanId\":\"2\",\"parentSpanId\":\"1\",\"name\":\"call_llm\","
|
||||
"\"startTime\":\"2026-10-24T10:00:00.100Z\",\"endTime\":\"2026-10-24T10:00:01.100Z\","
|
||||
"\"labels\":{\"gen_ai.request.model\":\"gemini\"}}]}"));
|
||||
ASSERT_EQ(t.spans.size(), 2u);
|
||||
ASSERT_EQ(t.root()->name, "invoke_agent");
|
||||
ASSERT_EQ(t.root()->duration_ms(), 2500);
|
||||
ASSERT_EQ(t.spans[1].duration_ms(), 1000);
|
||||
ASSERT_EQ(t.spans[1].labels.at("gen_ai.request.model"), "gemini");
|
||||
Span bad;
|
||||
bad.start_time = "garbage";
|
||||
ASSERT_EQ(bad.duration_ms(), 0);
|
||||
}
|
||||
|
||||
TEST(optimize_observability, span_cap) {
|
||||
Json spans = Json::array();
|
||||
for (int i = 0; i < 2100; ++i) {
|
||||
Json s = Json::object();
|
||||
s.set("spanId", Json(std::to_string(i)));
|
||||
spans.push(s);
|
||||
}
|
||||
Json doc = Json::object();
|
||||
doc.set("spans", spans);
|
||||
ASSERT_EQ(Trace::from_json(doc).spans.size(), 2000u);
|
||||
}
|
||||
|
||||
TEST(optimize_observability, conversation_from_logs_and_dashboard) {
|
||||
std::vector<LogEntry> logs;
|
||||
logs.push_back(LogEntry::from_json(Json::parse(
|
||||
"{\"severity\":\"INFO\",\"jsonPayload\":{\"role\":\"user\",\"content\":\"hi\"}}")));
|
||||
logs.push_back(LogEntry::from_json(Json::parse(
|
||||
"{\"severity\":\"INFO\",\"jsonPayload\":{\"role\":\"model\",\"text\":\"hello\"}}")));
|
||||
logs.push_back(LogEntry::from_json(Json::parse(
|
||||
"{\"severity\":\"ERROR\",\"textPayload\":\"boom\"}")));
|
||||
logs.push_back(LogEntry::from_json(Json::parse(
|
||||
"{\"jsonPayload\":{\"role\":\"system\",\"content\":\"ignored\"}}")));
|
||||
Conversation c = ConversationBuilder::from_logs(logs, "c1");
|
||||
ASSERT_EQ(c.turns.size(), 2u);
|
||||
ASSERT_EQ(c.turns[1].role, "agent");
|
||||
|
||||
std::vector<Trace> traces;
|
||||
for (int ms : {100, 200, 300, 400, 1000}) {
|
||||
Trace t;
|
||||
Span s;
|
||||
s.span_id = "1";
|
||||
s.name = "invoke_agent";
|
||||
s.start_time = "2026-10-24T10:00:00.000Z";
|
||||
char end[40];
|
||||
snprintf(end, sizeof end, "2026-10-24T10:00:%02d.%03dZ", ms / 1000, ms % 1000);
|
||||
s.end_time = end;
|
||||
t.spans.push_back(s);
|
||||
traces.push_back(t);
|
||||
}
|
||||
DashboardData d = summarize(traces, logs);
|
||||
ASSERT_EQ(d.traces, 5u);
|
||||
ASSERT_EQ(d.p50_ms, 300);
|
||||
ASSERT_EQ(d.p95_ms, 1000);
|
||||
ASSERT_EQ(d.top_spans[0].first, "invoke_agent");
|
||||
ASSERT_EQ(d.log_severity.at("INFO"), 2u);
|
||||
ASSERT_EQ(d.log_severity.at("DEFAULT"), 1u);
|
||||
ASSERT_EQ(d.to_json().find("traces")->as_int(), 5);
|
||||
DashboardData empty = summarize({}, {});
|
||||
ASSERT_EQ(empty.p95_ms, 0);
|
||||
}
|
||||
Reference in New Issue
Block a user