// Opt-in live checks. Run with MOLO_LIVE_NET=1 plus credentials; otherwise // each test prints a loud SKIP. Never prints a credential or the project id. // Metered but tiny: one computation-metric call, one autorater call. #include #include #include "framework.h" #include "google/credentials.h" #include "google/optimize/autorater.h" #include "google/optimize/model_factory.h" #include "google/optimize/eval_resources.h" #include "google/optimize/evaluation_api.h" #include "google/optimize/observability.h" using namespace molo::google; using namespace molo::google::optimize; namespace { bool live(const char* name, bool need_project) { if (!std::getenv("MOLO_LIVE_NET") || Credentials::env().empty() || (need_project && Credentials::env("GOOGLE_CLOUD_PROJECT").empty())) { std::cout << "SKIP " << name << " (set MOLO_LIVE_NET=1, a Google credential" << (need_project ? " and GOOGLE_CLOUD_PROJECT" : "") << ")\n"; return false; } return true; } } // namespace TEST(optimize_live, evaluate_instances_computation_metrics) { if (!live("optimize_live.evaluate_instances_computation_metrics", true)) return; // Verified 2026-10-24: an express key is accepted for computation metrics. EvaluationApi api(GoogleClient::from_env(), CloudContext::from_env()); MetricOutcome match = api.evaluate(EvaluationRequests::exact_match({{"a", "a"}, {"a", "b"}})); ASSERT_EQ(match.kind, "exactMatchResults"); ASSERT_EQ(match.scores.size(), 2u); ASSERT_EQ(match.scores[0], 1.0); ASSERT_EQ(match.scores[1], 0.0); MetricOutcome traj = api.evaluate(EvaluationRequests::trajectory( TrajectoryMetric::kExactMatch, {{"f", "{}"}}, {{"f", "{}"}})); ASSERT_EQ(traj.kind, "trajectoryExactMatchResults"); ASSERT_EQ(traj.scores.at(0), 1.0); } TEST(optimize_live, resource_lists_need_oauth) { if (!live("optimize_live.resource_lists_need_oauth", true)) return; EvaluationResources res(GoogleClient::from_env(), CloudContext::from_env()); if (Credentials::env("GOOGLE_ACCESS_TOKEN").empty()) { // Verified 2026-10-24: API keys are refused with 401 UNAUTHENTICATED. try { res.list(Collection::kEvaluationRuns, ListOptions{1, "", "", 1}); FAIL("an API key should be refused"); } catch (const GoogleError& e) { ASSERT_EQ(e.http_status(), 401); } return; } // Read-only lists of every evaluation collection, one item per page. for (Collection c : {Collection::kEvaluationSets, Collection::kEvaluationItems, Collection::kEvaluationMetrics, Collection::kEvaluationRuns, Collection::kOnlineEvaluators}) { molo::clients::body::Json items = res.list(c, ListOptions{1, "", "", 1}); ASSERT_TRUE(items.is_array()); std::cout << " " << collection_name(c) << ": " << items.as_array().size() << " item(s)\n"; } } TEST(optimize_live, observability_reads_with_oauth) { if (!live("optimize_live.observability_reads_with_oauth", true)) return; if (Credentials::env("GOOGLE_ACCESS_TOKEN").empty()) { std::cout << "SKIP optimize_live.observability_reads_with_oauth (needs GOOGLE_ACCESS_TOKEN)\n"; return; } auto client = GoogleClient::from_env(); std::string project = CloudContext::from_env().project; LogQuery lq; lq.resource_names = {"projects/" + project}; lq.page_size = 1; lq.max_entries = 1; std::vector logs = LogReader(client).entries(lq); ASSERT_TRUE(logs.size() <= 1); TraceQuery tq; tq.page_size = 1; tq.max_pages = 1; tq.max_traces = 1; std::vector traces = TraceReader(client).list(project, tq); ASSERT_TRUE(traces.size() <= 1); std::cout << " logs " << logs.size() << ", traces " << traces.size() << "\n"; } TEST(optimize_live, llm_judged_evaluate_instances) { if (!live("optimize_live.llm_judged_evaluate_instances", true)) return; EvaluationApi api(GoogleClient::from_env(), CloudContext::from_env()); // One tiny LLM-judged instance. Outcome is recorded, not asserted: the // default autorater model may be unavailable to a new project (404). try { MetricOutcome out = api.evaluate(EvaluationRequests::safety("Hello, how can I help you today?")); std::cout << " safety result: " << out.kind << " scores " << out.scores.size() << "\n"; ASSERT_EQ(out.kind, "safetyResult"); } catch (const GoogleError& e) { std::cout << " safety evaluateInstances refused: " << e.http_status() << " " << e.status() << "\n"; } } TEST(optimize_live, autorater_parses_a_real_model_reply) { if (!live("optimize_live.autorater_parses_a_real_model_reply", false)) return; auto model = make_gemini_model(GoogleClient::from_env()); Autorater rater(model); Conversation c = Conversation::parse_transcript( "user: What is 2+2?\nagent: 4.", "live"); Score s = rater.score("task_success", c, Autorater::task_success_rubrics()); ASSERT_EQ(s.verdicts.size(), 3u); for (const RubricVerdict& v : s.verdicts) ASSERT_TRUE(v.explanation != "no verdict returned"); // the reply parsed std::cout << " autorater score " << s.value << "\n"; }