2026-10-24 04:40:55 +02:00
|
|
|
// Opt-in live checks. Run with MOLO_LIVE_NET=1 plus credentials; otherwise
|
|
|
|
|
// each test prints a loud SKIP. Never prints a credential or the project id.
|
|
|
|
|
// Metered but tiny: one computation-metric call, one autorater call.
|
|
|
|
|
#include <cstdlib>
|
|
|
|
|
#include <iostream>
|
|
|
|
|
|
|
|
|
|
#include "framework.h"
|
|
|
|
|
#include "google/credentials.h"
|
|
|
|
|
#include "google/optimize/autorater.h"
|
2026-10-24 05:43:13 +02:00
|
|
|
#include "google/optimize/model_factory.h"
|
2026-10-24 04:40:55 +02:00
|
|
|
#include "google/optimize/eval_resources.h"
|
|
|
|
|
#include "google/optimize/evaluation_api.h"
|
|
|
|
|
#include "google/optimize/observability.h"
|
|
|
|
|
|
|
|
|
|
using namespace molo::google;
|
|
|
|
|
using namespace molo::google::optimize;
|
|
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
bool live(const char* name, bool need_project) {
|
|
|
|
|
if (!std::getenv("MOLO_LIVE_NET") || Credentials::env().empty() ||
|
|
|
|
|
(need_project && Credentials::env("GOOGLE_CLOUD_PROJECT").empty())) {
|
|
|
|
|
std::cout << "SKIP " << name
|
|
|
|
|
<< " (set MOLO_LIVE_NET=1, a Google credential"
|
|
|
|
|
<< (need_project ? " and GOOGLE_CLOUD_PROJECT" : "") << ")\n";
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
} // namespace
|
|
|
|
|
|
|
|
|
|
TEST(optimize_live, evaluate_instances_computation_metrics) {
|
|
|
|
|
if (!live("optimize_live.evaluate_instances_computation_metrics", true)) return;
|
|
|
|
|
// Verified 2026-10-24: an express key is accepted for computation metrics.
|
|
|
|
|
EvaluationApi api(GoogleClient::from_env(), CloudContext::from_env());
|
|
|
|
|
MetricOutcome match = api.evaluate(EvaluationRequests::exact_match({{"a", "a"}, {"a", "b"}}));
|
|
|
|
|
ASSERT_EQ(match.kind, "exactMatchResults");
|
|
|
|
|
ASSERT_EQ(match.scores.size(), 2u);
|
|
|
|
|
ASSERT_EQ(match.scores[0], 1.0);
|
|
|
|
|
ASSERT_EQ(match.scores[1], 0.0);
|
|
|
|
|
MetricOutcome traj = api.evaluate(EvaluationRequests::trajectory(
|
|
|
|
|
TrajectoryMetric::kExactMatch, {{"f", "{}"}}, {{"f", "{}"}}));
|
|
|
|
|
ASSERT_EQ(traj.kind, "trajectoryExactMatchResults");
|
|
|
|
|
ASSERT_EQ(traj.scores.at(0), 1.0);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
TEST(optimize_live, resource_lists_need_oauth) {
|
|
|
|
|
if (!live("optimize_live.resource_lists_need_oauth", true)) return;
|
|
|
|
|
EvaluationResources res(GoogleClient::from_env(), CloudContext::from_env());
|
|
|
|
|
if (Credentials::env("GOOGLE_ACCESS_TOKEN").empty()) {
|
|
|
|
|
// Verified 2026-10-24: API keys are refused with 401 UNAUTHENTICATED.
|
|
|
|
|
try {
|
|
|
|
|
res.list(Collection::kEvaluationRuns, ListOptions{1, "", "", 1});
|
|
|
|
|
FAIL("an API key should be refused");
|
|
|
|
|
} catch (const GoogleError& e) {
|
|
|
|
|
ASSERT_EQ(e.http_status(), 401);
|
|
|
|
|
}
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
// Read-only lists of every evaluation collection, one item per page.
|
|
|
|
|
for (Collection c : {Collection::kEvaluationSets, Collection::kEvaluationItems,
|
|
|
|
|
Collection::kEvaluationMetrics, Collection::kEvaluationRuns,
|
|
|
|
|
Collection::kOnlineEvaluators}) {
|
|
|
|
|
molo::clients::body::Json items = res.list(c, ListOptions{1, "", "", 1});
|
|
|
|
|
ASSERT_TRUE(items.is_array());
|
|
|
|
|
std::cout << " " << collection_name(c) << ": " << items.as_array().size() << " item(s)\n";
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
TEST(optimize_live, observability_reads_with_oauth) {
|
|
|
|
|
if (!live("optimize_live.observability_reads_with_oauth", true)) return;
|
|
|
|
|
if (Credentials::env("GOOGLE_ACCESS_TOKEN").empty()) {
|
|
|
|
|
std::cout << "SKIP optimize_live.observability_reads_with_oauth (needs GOOGLE_ACCESS_TOKEN)\n";
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
auto client = GoogleClient::from_env();
|
|
|
|
|
std::string project = CloudContext::from_env().project;
|
|
|
|
|
LogQuery lq;
|
|
|
|
|
lq.resource_names = {"projects/" + project};
|
|
|
|
|
lq.page_size = 1;
|
|
|
|
|
lq.max_entries = 1;
|
|
|
|
|
std::vector<LogEntry> logs = LogReader(client).entries(lq);
|
|
|
|
|
ASSERT_TRUE(logs.size() <= 1);
|
|
|
|
|
TraceQuery tq;
|
|
|
|
|
tq.page_size = 1;
|
|
|
|
|
tq.max_pages = 1;
|
|
|
|
|
tq.max_traces = 1;
|
|
|
|
|
std::vector<Trace> traces = TraceReader(client).list(project, tq);
|
|
|
|
|
ASSERT_TRUE(traces.size() <= 1);
|
|
|
|
|
std::cout << " logs " << logs.size() << ", traces " << traces.size() << "\n";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
TEST(optimize_live, llm_judged_evaluate_instances) {
|
|
|
|
|
if (!live("optimize_live.llm_judged_evaluate_instances", true)) return;
|
|
|
|
|
EvaluationApi api(GoogleClient::from_env(), CloudContext::from_env());
|
|
|
|
|
// One tiny LLM-judged instance. Outcome is recorded, not asserted: the
|
|
|
|
|
// default autorater model may be unavailable to a new project (404).
|
|
|
|
|
try {
|
|
|
|
|
MetricOutcome out = api.evaluate(EvaluationRequests::safety("Hello, how can I help you today?"));
|
|
|
|
|
std::cout << " safety result: " << out.kind << " scores " << out.scores.size() << "\n";
|
|
|
|
|
ASSERT_EQ(out.kind, "safetyResult");
|
|
|
|
|
} catch (const GoogleError& e) {
|
|
|
|
|
std::cout << " safety evaluateInstances refused: " << e.http_status() << " " << e.status() << "\n";
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
TEST(optimize_live, autorater_parses_a_real_model_reply) {
|
|
|
|
|
if (!live("optimize_live.autorater_parses_a_real_model_reply", false)) return;
|
2026-10-24 05:43:13 +02:00
|
|
|
auto model = make_gemini_model(GoogleClient::from_env());
|
2026-10-24 04:40:55 +02:00
|
|
|
Autorater rater(model);
|
|
|
|
|
Conversation c = Conversation::parse_transcript(
|
|
|
|
|
"user: What is 2+2?\nagent: 4.", "live");
|
|
|
|
|
Score s = rater.score("task_success", c, Autorater::task_success_rubrics());
|
|
|
|
|
ASSERT_EQ(s.verdicts.size(), 3u);
|
|
|
|
|
for (const RubricVerdict& v : s.verdicts)
|
|
|
|
|
ASSERT_TRUE(v.explanation != "no verdict returned"); // the reply parsed
|
|
|
|
|
std::cout << " autorater score " << s.value << "\n";
|
|
|
|
|
}
|