feat(clients): optimize: evaluation, simulation, observability and optimizer on a discovery-checked call table
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,117 @@
|
||||
// Opt-in live checks. Run with MOLO_LIVE_NET=1 plus credentials; otherwise
|
||||
// each test prints a loud SKIP. Never prints a credential or the project id.
|
||||
// Metered but tiny: one computation-metric call, one autorater call.
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
|
||||
#include "framework.h"
|
||||
#include "google/credentials.h"
|
||||
#include "google/optimize/autorater.h"
|
||||
#include "google/optimize/direct_model.h"
|
||||
#include "google/optimize/eval_resources.h"
|
||||
#include "google/optimize/evaluation_api.h"
|
||||
#include "google/optimize/observability.h"
|
||||
|
||||
using namespace molo::google;
|
||||
using namespace molo::google::optimize;
|
||||
|
||||
namespace {
|
||||
bool live(const char* name, bool need_project) {
|
||||
if (!std::getenv("MOLO_LIVE_NET") || Credentials::env().empty() ||
|
||||
(need_project && Credentials::env("GOOGLE_CLOUD_PROJECT").empty())) {
|
||||
std::cout << "SKIP " << name
|
||||
<< " (set MOLO_LIVE_NET=1, a Google credential"
|
||||
<< (need_project ? " and GOOGLE_CLOUD_PROJECT" : "") << ")\n";
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(optimize_live, evaluate_instances_computation_metrics) {
|
||||
if (!live("optimize_live.evaluate_instances_computation_metrics", true)) return;
|
||||
// Verified 2026-10-24: an express key is accepted for computation metrics.
|
||||
EvaluationApi api(GoogleClient::from_env(), CloudContext::from_env());
|
||||
MetricOutcome match = api.evaluate(EvaluationRequests::exact_match({{"a", "a"}, {"a", "b"}}));
|
||||
ASSERT_EQ(match.kind, "exactMatchResults");
|
||||
ASSERT_EQ(match.scores.size(), 2u);
|
||||
ASSERT_EQ(match.scores[0], 1.0);
|
||||
ASSERT_EQ(match.scores[1], 0.0);
|
||||
MetricOutcome traj = api.evaluate(EvaluationRequests::trajectory(
|
||||
TrajectoryMetric::kExactMatch, {{"f", "{}"}}, {{"f", "{}"}}));
|
||||
ASSERT_EQ(traj.kind, "trajectoryExactMatchResults");
|
||||
ASSERT_EQ(traj.scores.at(0), 1.0);
|
||||
}
|
||||
|
||||
TEST(optimize_live, resource_lists_need_oauth) {
|
||||
if (!live("optimize_live.resource_lists_need_oauth", true)) return;
|
||||
EvaluationResources res(GoogleClient::from_env(), CloudContext::from_env());
|
||||
if (Credentials::env("GOOGLE_ACCESS_TOKEN").empty()) {
|
||||
// Verified 2026-10-24: API keys are refused with 401 UNAUTHENTICATED.
|
||||
try {
|
||||
res.list(Collection::kEvaluationRuns, ListOptions{1, "", "", 1});
|
||||
FAIL("an API key should be refused");
|
||||
} catch (const GoogleError& e) {
|
||||
ASSERT_EQ(e.http_status(), 401);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Read-only lists of every evaluation collection, one item per page.
|
||||
for (Collection c : {Collection::kEvaluationSets, Collection::kEvaluationItems,
|
||||
Collection::kEvaluationMetrics, Collection::kEvaluationRuns,
|
||||
Collection::kOnlineEvaluators}) {
|
||||
molo::clients::body::Json items = res.list(c, ListOptions{1, "", "", 1});
|
||||
ASSERT_TRUE(items.is_array());
|
||||
std::cout << " " << collection_name(c) << ": " << items.as_array().size() << " item(s)\n";
|
||||
}
|
||||
}
|
||||
|
||||
TEST(optimize_live, observability_reads_with_oauth) {
|
||||
if (!live("optimize_live.observability_reads_with_oauth", true)) return;
|
||||
if (Credentials::env("GOOGLE_ACCESS_TOKEN").empty()) {
|
||||
std::cout << "SKIP optimize_live.observability_reads_with_oauth (needs GOOGLE_ACCESS_TOKEN)\n";
|
||||
return;
|
||||
}
|
||||
auto client = GoogleClient::from_env();
|
||||
std::string project = CloudContext::from_env().project;
|
||||
LogQuery lq;
|
||||
lq.resource_names = {"projects/" + project};
|
||||
lq.page_size = 1;
|
||||
lq.max_entries = 1;
|
||||
std::vector<LogEntry> logs = LogReader(client).entries(lq);
|
||||
ASSERT_TRUE(logs.size() <= 1);
|
||||
TraceQuery tq;
|
||||
tq.page_size = 1;
|
||||
tq.max_pages = 1;
|
||||
tq.max_traces = 1;
|
||||
std::vector<Trace> traces = TraceReader(client).list(project, tq);
|
||||
ASSERT_TRUE(traces.size() <= 1);
|
||||
std::cout << " logs " << logs.size() << ", traces " << traces.size() << "\n";
|
||||
}
|
||||
|
||||
TEST(optimize_live, llm_judged_evaluate_instances) {
|
||||
if (!live("optimize_live.llm_judged_evaluate_instances", true)) return;
|
||||
EvaluationApi api(GoogleClient::from_env(), CloudContext::from_env());
|
||||
// One tiny LLM-judged instance. Outcome is recorded, not asserted: the
|
||||
// default autorater model may be unavailable to a new project (404).
|
||||
try {
|
||||
MetricOutcome out = api.evaluate(EvaluationRequests::safety("Hello, how can I help you today?"));
|
||||
std::cout << " safety result: " << out.kind << " scores " << out.scores.size() << "\n";
|
||||
ASSERT_EQ(out.kind, "safetyResult");
|
||||
} catch (const GoogleError& e) {
|
||||
std::cout << " safety evaluateInstances refused: " << e.http_status() << " " << e.status() << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
TEST(optimize_live, autorater_parses_a_real_model_reply) {
|
||||
if (!live("optimize_live.autorater_parses_a_real_model_reply", false)) return;
|
||||
auto model = DirectModel::from_env(GoogleClient::from_env());
|
||||
Autorater rater(model);
|
||||
Conversation c = Conversation::parse_transcript(
|
||||
"user: What is 2+2?\nagent: 4.", "live");
|
||||
Score s = rater.score("task_success", c, Autorater::task_success_rubrics());
|
||||
ASSERT_EQ(s.verdicts.size(), 3u);
|
||||
for (const RubricVerdict& v : s.verdicts)
|
||||
ASSERT_TRUE(v.explanation != "no verdict returned"); // the reply parsed
|
||||
std::cout << " autorater score " << s.value << "\n";
|
||||
}
|
||||
Reference in New Issue
Block a user