feat(clients): sessions+registry+optimize+armor: close gaps, loud list truncation, shared Gemini adapter
sessions: operations get/list/cancel/delete/wait for engine, session, memory and serving profile; servingProfiles CRUD. registry: ListResult with truncated flag, v1alpha twins, gateway/policy operations, authz IAM, deny policy update, boundary patch/bindings. optimize: make_gemini_model over models::VertexModel (DirectModel removed), generateSyntheticData, online evaluator operations, trace/log writers. armor: SecurityAdmin for all SCC v2 methods (attack exposure excluded), documented agent threat categories, floor settings fixed to location global after live run. check_discovery.py of all four enforce coverage. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,284 @@
|
||||
// Optimize gap closing: the shared Gemini adapter (models::VertexModel) behind
|
||||
// make_gemini_model, online evaluator operations, generateSyntheticData and
|
||||
// the telemetry writers. Local only: requests are read back from the mock,
|
||||
// nothing answers (no scripted Google replies).
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "framework.h"
|
||||
#include "clients/transports/mock_transport.h"
|
||||
#include "google/credentials.h"
|
||||
#include "google/errors.h"
|
||||
#include "google/models/vertex_model.h"
|
||||
#include "google/optimize/calls.h"
|
||||
#include "google/optimize/evaluation_api.h"
|
||||
#include "google/optimize/model_factory.h"
|
||||
#include "google/optimize/operations_api.h"
|
||||
#include "google/optimize/telemetry_writer.h"
|
||||
|
||||
using namespace molo::google;
|
||||
using namespace molo::google::optimize;
|
||||
using molo::clients::body::Json;
|
||||
using molo::clients::transports::MockTransport;
|
||||
|
||||
namespace {
|
||||
|
||||
struct Rig {
|
||||
MockTransport* mock;
|
||||
std::shared_ptr<GoogleClient> client;
|
||||
explicit Rig(bool with_credential = true) {
|
||||
auto t = std::make_unique<MockTransport>();
|
||||
mock = t.get();
|
||||
client = std::make_shared<GoogleClient>(
|
||||
std::move(t), with_credential ? Credentials::bearer("tok") : nullptr);
|
||||
}
|
||||
};
|
||||
|
||||
CloudContext ctx() {
|
||||
CloudContext c;
|
||||
c.project = "p";
|
||||
c.location = "us-central1";
|
||||
return c;
|
||||
}
|
||||
|
||||
bool has(const std::string& hay, const std::string& needle) {
|
||||
return hay.find(needle) != std::string::npos;
|
||||
}
|
||||
|
||||
MockTransport::SeenRequest first_sent(Rig& rig, const std::function<void()>& fn) {
|
||||
std::size_t before = rig.mock->seen().size();
|
||||
try {
|
||||
fn();
|
||||
} catch (...) {
|
||||
}
|
||||
auto all = rig.mock->seen();
|
||||
if (all.size() <= before) FAIL("no request reached the transport");
|
||||
return all[before];
|
||||
}
|
||||
|
||||
bool invalid_without_traffic(Rig& rig, const std::function<void()>& fn) {
|
||||
std::size_t before = rig.mock->send_count();
|
||||
try {
|
||||
fn();
|
||||
} catch (const GoogleError& e) {
|
||||
return e.status() == "INVALID_ARGUMENT" && rig.mock->send_count() == before;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Sets or clears environment variables for one test and restores them, so
|
||||
// later (live) tests still see the caller's credentials.
|
||||
class EnvGuard {
|
||||
public:
|
||||
EnvGuard() {
|
||||
for (const char* k : {"GOOGLE_ACCESS_TOKEN", "GOOGLE_CLOUD_PROJECT"}) {
|
||||
const char* v = std::getenv(k);
|
||||
saved_.push_back({k, v != nullptr, v ? v : ""});
|
||||
}
|
||||
}
|
||||
~EnvGuard() {
|
||||
for (const auto& s : saved_) {
|
||||
if (s.had) setenv(s.key.c_str(), s.value.c_str(), 1);
|
||||
else unsetenv(s.key.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
struct Saved {
|
||||
std::string key;
|
||||
bool had;
|
||||
std::string value;
|
||||
};
|
||||
std::vector<Saved> saved_;
|
||||
};
|
||||
|
||||
const char* kEval = "projects/p/locations/us-central1/onlineEvaluators/oe1";
|
||||
const std::string kOp = std::string(kEval) + "/operations/op1";
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST(optimize_model, factory_returns_the_shared_vertex_adapter) {
|
||||
Rig rig;
|
||||
auto model = make_gemini_model(rig.client, "gemini-2.5-flash");
|
||||
auto* vertex = dynamic_cast<models::VertexModel*>(model.get());
|
||||
ASSERT_TRUE(vertex != nullptr);
|
||||
ASSERT_EQ(vertex->default_model(), std::string("gemini-2.5-flash"));
|
||||
ASSERT_TRUE(dynamic_cast<models::VertexModel*>(make_gemini_model(rig.client).get()) != nullptr);
|
||||
}
|
||||
|
||||
TEST(optimize_model, oauth_with_a_project_uses_the_global_project_path) {
|
||||
EnvGuard guard;
|
||||
setenv("GOOGLE_ACCESS_TOKEN", "tok", 1);
|
||||
setenv("GOOGLE_CLOUD_PROJECT", "p1", 1);
|
||||
Rig rig;
|
||||
auto model = make_gemini_model(rig.client, "m1");
|
||||
GenerateRequest req;
|
||||
req.contents.push_back(Content::user("hi"));
|
||||
auto r = first_sent(rig, [&] { model->generate(req); });
|
||||
ASSERT_EQ(r.method, std::string("POST"));
|
||||
ASSERT_TRUE(has(r.target, "/projects/p1/locations/global/publishers/google/models/m1:generateContent"));
|
||||
}
|
||||
|
||||
TEST(optimize_model, without_a_token_it_is_vertex_express) {
|
||||
EnvGuard guard;
|
||||
unsetenv("GOOGLE_ACCESS_TOKEN");
|
||||
setenv("GOOGLE_CLOUD_PROJECT", "p1", 1);
|
||||
Rig rig;
|
||||
auto model = make_gemini_model(rig.client, "m1");
|
||||
GenerateRequest req;
|
||||
req.contents.push_back(Content::user("hi"));
|
||||
auto r = first_sent(rig, [&] { model->generate(req); });
|
||||
ASSERT_TRUE(has(r.target, "/publishers/google/models/m1:generateContent"));
|
||||
ASSERT_FALSE(has(r.target, "/projects/"));
|
||||
// A bad model name is refused before any traffic.
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { make_gemini_model(rig.client, "a/b"); }));
|
||||
}
|
||||
|
||||
TEST(optimize_operations, names_and_bounds) {
|
||||
ASSERT_TRUE(OnlineEvaluatorOperations::is_operation_name(kOp));
|
||||
for (const char* bad : {"", "operations/1", "projects/p/locations/l/onlineEvaluators/e",
|
||||
"projects/p/locations/l/evaluationRuns/e/operations/o",
|
||||
"projects/p/locations/l/onlineEvaluators/e/operations/",
|
||||
"projects/p/locations/l/onlineEvaluators/e/operations/..",
|
||||
"projects/p/locations/l/onlineEvaluators/e/operations/o/x"})
|
||||
ASSERT_FALSE(OnlineEvaluatorOperations::is_operation_name(bad));
|
||||
Rig rig;
|
||||
OnlineEvaluatorOperations ops(rig.client, ctx());
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.get("nope"); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.cancel("nope"); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.remove("nope"); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.wait_server(kOp, 0); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.wait_server(kOp, 61); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.list("projects/p/locations/l/evaluationRuns/r"); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.list(kEval, "", 101); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.list(kEval, std::string(1001, 'x')); }));
|
||||
Rig off(false);
|
||||
OnlineEvaluatorOperations n(off.client, ctx());
|
||||
ASSERT_THROW(n.get(kOp), NotConfiguredError);
|
||||
ASSERT_EQ(off.mock->send_count(), std::size_t(0));
|
||||
}
|
||||
|
||||
TEST(optimize_operations, requests_use_the_declared_rows) {
|
||||
Rig rig;
|
||||
OnlineEvaluatorOperations ops(rig.client, ctx());
|
||||
auto r = first_sent(rig, [&] { ops.get(kOp); });
|
||||
ASSERT_EQ(r.method, std::string("GET"));
|
||||
ASSERT_TRUE(has(r.target, "/v1/" + kOp));
|
||||
r = first_sent(rig, [&] { ops.cancel(kOp); });
|
||||
ASSERT_EQ(r.method, std::string("POST"));
|
||||
ASSERT_TRUE(has(r.target, kOp + ":cancel"));
|
||||
r = first_sent(rig, [&] { ops.remove(kOp); });
|
||||
ASSERT_EQ(r.method, std::string("DELETE"));
|
||||
r = first_sent(rig, [&] { ops.wait_server(kOp, 30); });
|
||||
ASSERT_EQ(r.method, std::string("POST"));
|
||||
ASSERT_TRUE(has(r.target, kOp + ":wait?timeout=30s"));
|
||||
r = first_sent(rig, [&] { ops.list(kEval, "done=true", 20); });
|
||||
ASSERT_TRUE(has(r.target, std::string(kEval) + "/operations?"));
|
||||
ASSERT_TRUE(has(r.target, "pageSize=20"));
|
||||
}
|
||||
|
||||
TEST(optimize_synthetic_data, request_is_checked_against_the_table) {
|
||||
Rig rig;
|
||||
EvaluationApi api(rig.client, ctx());
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { api.generate_synthetic_data(Json::object()); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] {
|
||||
api.generate_synthetic_data(Json::parse("{\"notAField\":1}"));
|
||||
}));
|
||||
const CallSpec& spec = call_spec(rpc::kGenerateSyntheticData);
|
||||
std::string field = split_list(spec.body_fields).front();
|
||||
Json body = Json::object();
|
||||
body.set(field, Json::object());
|
||||
auto r = first_sent(rig, [&] { api.generate_synthetic_data(body); });
|
||||
ASSERT_EQ(r.method, std::string("POST"));
|
||||
ASSERT_TRUE(has(r.target, "/v1/projects/p/locations/us-central1:generateSyntheticData"));
|
||||
}
|
||||
|
||||
TEST(optimize_writers, trace_writer_validates_then_uses_v1_and_v2) {
|
||||
Rig rig;
|
||||
TraceWriter w(rig.client);
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.patch_traces("BAD ID", Json::object()); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.patch_traces("p", Json::parse("{\"traces\":{}}")); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.batch_write("p", Json::parse("{\"spans\":[]}")); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.batch_write("p", Json::parse("{\"other\":[1]}")); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.create_span("x", Json::object()); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] {
|
||||
w.create_span("projects/p/traces/abc/spans/s1", Json::parse("{\"name\":\"other\"}"));
|
||||
}));
|
||||
auto r = first_sent(rig, [&] {
|
||||
w.patch_traces("p", Json::parse("{\"traces\":{\"traces\":[{\"traceId\":\"a\"}]}}"));
|
||||
});
|
||||
ASSERT_EQ(r.method, std::string("PATCH"));
|
||||
ASSERT_TRUE(has(r.target, "/v1/projects/p/traces"));
|
||||
r = first_sent(rig, [&] { w.batch_write("p", Json::parse("{\"spans\":[{\"name\":\"n\"}]}")); });
|
||||
ASSERT_EQ(r.method, std::string("POST"));
|
||||
ASSERT_TRUE(has(r.target, "/v2/projects/p/traces:batchWrite"));
|
||||
r = first_sent(rig, [&] {
|
||||
w.create_span("projects/p/traces/abc/spans/s1", Json::parse("{\"name\":\"projects/p/traces/abc/spans/s1\"}"));
|
||||
});
|
||||
ASSERT_TRUE(has(r.target, "/v2/projects/p/traces/abc/spans/s1"));
|
||||
}
|
||||
|
||||
TEST(optimize_writers, log_writer_validates_then_uses_the_table) {
|
||||
Rig rig;
|
||||
LogWriter w(rig.client);
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.write(Json::object()); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.write(Json::parse("{\"entries\":[]}")); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.write(Json::parse("{\"bogus\":[1]}")); }));
|
||||
Json many = Json::array();
|
||||
for (int i = 0; i < 1001; ++i) many.push(Json::object());
|
||||
Json big = Json::object();
|
||||
big.set("entries", many);
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.write(big); }));
|
||||
ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.copy(Json::parse("{\"name\":\"x\"}")); }));
|
||||
auto r = first_sent(rig, [&] {
|
||||
w.write(Json::parse("{\"dryRun\":true,\"logName\":\"projects/p/logs/l\",\"entries\":[{\"textPayload\":\"x\"}]}"));
|
||||
});
|
||||
ASSERT_EQ(r.method, std::string("POST"));
|
||||
ASSERT_TRUE(has(r.target, "/v2/entries:write"));
|
||||
ASSERT_TRUE(has(r.body, "\"dryRun\":true"));
|
||||
r = first_sent(rig, [&] { w.copy(Json::parse("{\"name\":\"projects/p/locations/global/buckets/b\",\"destination\":\"storage.googleapis.com/b2\"}")); });
|
||||
ASSERT_TRUE(has(r.target, "/v2/entries:copy"));
|
||||
}
|
||||
|
||||
// Read-only live probe (MOLO_LIVE_NET=1, GOOGLE_ACCESS_TOKEN, GOOGLE_CLOUD_PROJECT;
|
||||
// loud SKIP otherwise). Observed 2026-10-24: an operations list under a
|
||||
// numeric, absent online evaluator id answers 200 with no operations; a
|
||||
// non numeric evaluator id is 400 INVALID_ARGUMENT from Google's LRO layer.
|
||||
TEST(optimize_live_gaps, online_evaluator_operations_list_and_shared_model) {
|
||||
std::string project = Credentials::env("GOOGLE_CLOUD_PROJECT");
|
||||
if (!std::getenv("MOLO_LIVE_NET") || Credentials::env("GOOGLE_ACCESS_TOKEN").empty() ||
|
||||
project.empty()) {
|
||||
std::cout << "SKIP optimize_live_gaps (set MOLO_LIVE_NET=1, GOOGLE_ACCESS_TOKEN and "
|
||||
"GOOGLE_CLOUD_PROJECT)\n";
|
||||
return;
|
||||
}
|
||||
auto client = GoogleClient::from_env();
|
||||
CloudContext c;
|
||||
c.project = project;
|
||||
c.location = "us-central1";
|
||||
OnlineEvaluatorOperations ops(client, c);
|
||||
OperationPage page = ops.list("projects/" + project + "/locations/us-central1/onlineEvaluators/1");
|
||||
ASSERT_EQ(page.operations.size(), std::size_t(0));
|
||||
ASSERT_FALSE(page.truncated);
|
||||
// The shared adapter answers a real generateContent (a few tokens).
|
||||
auto model = make_gemini_model(client);
|
||||
GenerateRequest req;
|
||||
req.contents.push_back(Content::user("Reply with the single word: ok"));
|
||||
GenerateResponse res = model->generate(req);
|
||||
ASSERT_FALSE(res.text().empty());
|
||||
std::cout << " shared VertexModel reply chars: " << res.text().size() << "\n";
|
||||
}
|
||||
|
||||
TEST(optimize_table, new_calls_are_rows) {
|
||||
for (const char* id : {rpc::kGenerateSyntheticData, rpc::kOnlineEvaluatorOpGet,
|
||||
rpc::kOnlineEvaluatorOpList, rpc::kOnlineEvaluatorOpCancel,
|
||||
rpc::kOnlineEvaluatorOpDelete, rpc::kOnlineEvaluatorOpWait,
|
||||
rpc::kTracePatch, rpc::kTraceBatchWrite, rpc::kTraceCreateSpan,
|
||||
rpc::kLogEntriesWrite, rpc::kLogEntriesCopy})
|
||||
ASSERT_TRUE(find_call(id) != nullptr);
|
||||
ASSERT_TRUE(all_calls().size() >= 46);
|
||||
}
|
||||
@@ -7,7 +7,7 @@
|
||||
#include "framework.h"
|
||||
#include "google/credentials.h"
|
||||
#include "google/optimize/autorater.h"
|
||||
#include "google/optimize/direct_model.h"
|
||||
#include "google/optimize/model_factory.h"
|
||||
#include "google/optimize/eval_resources.h"
|
||||
#include "google/optimize/evaluation_api.h"
|
||||
#include "google/optimize/observability.h"
|
||||
@@ -105,7 +105,7 @@ TEST(optimize_live, llm_judged_evaluate_instances) {
|
||||
|
||||
TEST(optimize_live, autorater_parses_a_real_model_reply) {
|
||||
if (!live("optimize_live.autorater_parses_a_real_model_reply", false)) return;
|
||||
auto model = DirectModel::from_env(GoogleClient::from_env());
|
||||
auto model = make_gemini_model(GoogleClient::from_env());
|
||||
Autorater rater(model);
|
||||
Conversation c = Conversation::parse_transcript(
|
||||
"user: What is 2+2?\nagent: 4.", "live");
|
||||
|
||||
Reference in New Issue
Block a user