Files
moloapp/tests/src/google/models/test_models.cpp
T

452 lines
19 KiB
C++
Raw Normal View History

// Deterministic local tests for the models module: request construction
// checked against the call table (which tools/check_discovery.py checks
// against Google's Discovery documents), id validation, caps, confirm gates,
// not-configured behavior. No Google replies are scripted anywhere.
#include <memory>
#include <set>
#include <sstream>
#include "clients/transports/mock_transport.h"
#include "framework.h"
#include "google/credentials.h"
#include "google/errors.h"
#include "google/models/calls.h"
#include "google/models/caps.h"
#include "google/models/claude_models.h"
#include "google/models/gemini.h"
#include "google/models/media.h"
#include "google/models/model_garden.h"
#include "google/models/open_models.h"
#include "google/models/vertex_model.h"
using namespace molo::google;
using namespace molo::google::models;
using molo::clients::body::Json;
using molo::clients::transports::MockTransport;
namespace {
std::shared_ptr<GoogleClient> client(bool configured = true,
MockTransport** raw = nullptr) {
auto t = std::make_unique<MockTransport>();
if (raw) *raw = t.get();
return std::make_shared<GoogleClient>(
std::move(t), configured ? Credentials::bearer("tok") : nullptr);
}
VertexTarget project() {
CloudContext c;
c.project = "demo-proj";
c.location = "us-central1";
return VertexTarget::project(c);
}
std::set<std::string> table_fields(const std::string& id) {
std::set<std::string> out;
std::istringstream in(CallPlanner::spec(id).fields);
for (std::string f; in >> f;) out.insert(f);
return out;
}
// Every key of `body` (and of generationConfig) must be a field the call
// table declares for that call.
void expect_keys_declared(const PlannedCall& call, const std::string& id) {
ASSERT_TRUE(call.has_body);
auto fields = table_fields(id);
for (const auto& [k, v] : call.body.as_object()) {
if (!fields.count(k)) FAIL("body key not in call table: " + k);
bool nested = false;
for (const auto& f : fields)
if (f.rfind(k + ".", 0) == 0) nested = true;
if (nested && v.is_object())
for (const auto& [k2, v2] : v.as_object()) {
(void)v2;
if (!fields.count(k + "." + k2)) FAIL("nested key not in call table: " + k + "." + k2);
}
}
}
GenerateRequest rich() {
GenerateRequest r;
r.system = Content::user("be brief");
r.contents.push_back(Content::user("hello"));
Tool t;
t.google_search = true;
r.tools.push_back(t);
r.tool_mode = "AUTO";
r.safety.push_back({"HARM_CATEGORY_HARASSMENT", "BLOCK_NONE"});
r.config.temperature = 0.2;
r.config.top_p = 0.9;
r.config.top_k = 20;
r.config.max_output_tokens = 64;
r.config.candidate_count = 1;
r.config.thinking_budget = 0;
r.config.stop_sequences = {"END"};
r.config.response_mime_type = "application/json";
return r;
}
} // namespace
TEST(models_table, rows_are_well_formed_and_unique) {
std::set<std::string> ids;
for (const CallSpec& s : kCalls) {
ASSERT_TRUE(ids.insert(s.id).second);
std::string http = s.http;
ASSERT_TRUE(http == "GET" || http == "POST");
std::string v = s.version;
ASSERT_TRUE(v == "v1" || v == "v1beta1");
std::string p = s.prefix;
ASSERT_TRUE(p == "" || p == "L" || p == "R");
ASSERT_TRUE(std::string(s.rpc).rfind("aiplatform.", 0) == 0);
ASSERT_TRUE(std::string(s.suffix)[0] == '/' || std::string(s.suffix)[0] == ':');
}
ASSERT_EQ(ids.size(), kCallCount);
}
TEST(models_paths, express_and_project_forms) {
Gemini ex(client(), VertexTarget::express());
PlannedCall a = ex.plan_generate("gemini-3.5-flash", rich());
ASSERT_EQ(a.url(),
"https://aiplatform.googleapis.com/v1/publishers/google/models/gemini-3.5-flash:generateContent");
ASSERT_TRUE(a.method == molo::clients::Method::kPost);
Gemini pr(client(), project());
PlannedCall b = pr.plan_generate("gemini-3.5-flash", rich());
ASSERT_EQ(b.url(),
"https://us-central1-aiplatform.googleapis.com/v1/projects/demo-proj/locations/us-central1/"
"publishers/google/models/gemini-3.5-flash:generateContent");
CloudContext g;
g.project = "demo-proj";
g.location = "global";
Gemini gl(client(), VertexTarget::project(g));
ASSERT_EQ(gl.plan_generate("m", rich()).url(),
"https://aiplatform.googleapis.com/v1/projects/demo-proj/locations/global/"
"publishers/google/models/m:generateContent");
}
TEST(models_paths, stream_has_alt_sse) {
Gemini ex(client(), VertexTarget::express());
PlannedCall c = ex.plan_stream("m", rich());
ASSERT_EQ(c.path, "/publishers/google/models/m:streamGenerateContent?alt=sse");
}
TEST(models_paths, bodies_use_only_declared_fields) {
Gemini g(client(), project());
expect_keys_declared(g.plan_generate("m", rich()), "gemini.generate");
expect_keys_declared(g.plan_stream("m", rich()), "gemini.stream");
expect_keys_declared(g.plan_count_tokens("m", rich()), "gemini.count_tokens");
expect_keys_declared(g.plan_compute_tokens("m", {Content::user("x")}), "gemini.compute_tokens");
EmbedOptions o;
o.task_type = "RETRIEVAL_QUERY";
o.title = "t";
o.output_dimensionality = 8;
o.auto_truncate = true;
expect_keys_declared(g.plan_embed_content("e", "text", o), "gemini.embed_content");
expect_keys_declared(g.plan_embed("e", {"a", "b"}, o), "gemini.embed_predict");
// countTokens must not carry safety or tool config.
PlannedCall ct = g.plan_count_tokens("m", rich());
ASSERT_FALSE(ct.body.has("safetySettings"));
ASSERT_FALSE(ct.body.has("toolConfig"));
// embedContent nested config names
Json cfg = *g.plan_embed_content("e", "text", o).body.find("embedContentConfig");
for (const auto& [k, v] : cfg.as_object()) {
(void)v;
ASSERT_TRUE(table_fields("gemini.embed_content").count("embedContentConfig." + k) == 1);
}
OpenModels om(client(), project());
Json inst = Json::array();
inst.push(OpenModels::gemma_instance("hi"));
expect_keys_declared(om.plan_predict("77", inst, Json::object()), "endpoint.predict");
expect_keys_declared(om.plan_server_streaming("77", inst, Json::object()),
"endpoint.server_streaming_predict");
expect_keys_declared(om.plan_generate("endpoint.generate_content", "77", rich()),
"endpoint.generate_content");
expect_keys_declared(om.plan_long_running("77", inst, Json::object()),
"endpoint.predict_long_running");
expect_keys_declared(om.plan_fetch_operation("77", "projects/p/locations/l/operations/1"),
"endpoint.fetch_predict_operation");
MediaGenerator mg(client(), project());
ImageRequest ir;
ir.prompt = "cat";
ir.negative_prompt = "dog";
expect_keys_declared(mg.plan_image("imagen", ir), "media.predict");
MusicRequest mr;
mr.prompt = "calm";
expect_keys_declared(mg.plan_music("lyria-002", mr), "media.predict");
VideoRequest vr;
vr.prompt = "waves";
vr.duration_seconds = 8;
vr.storage_uri = "gs://bucket/out";
expect_keys_declared(mg.plan_start_video("veo", vr), "media.predict_long_running");
expect_keys_declared(mg.plan_video_status("veo", "projects/p/locations/l/operations/1"),
"media.fetch_predict_operation");
ModelGarden mgd(client(), project());
DeployOptions d;
d.publisher_model = "publishers/google/models/gemma3@gemma-3-1b-it";
d.model_display_name = "m";
d.endpoint_display_name = "e";
d.accept_eula = true;
d.confirm = true;
expect_keys_declared(mgd.plan_deploy(d), "garden.deploy");
expect_keys_declared(mgd.plan_eula("garden.eula_check", d.publisher_model), "garden.eula_check");
}
TEST(models_paths, model_garden_and_endpoints) {
ModelGarden mg(client(), VertexTarget::express());
ListModelsOptions o;
PlannedCall l = mg.plan_list_page("google", o, "tok/en");
ASSERT_EQ(l.endpoint.base_url, "https://aiplatform.googleapis.com/v1beta1");
ASSERT_TRUE(l.path.rfind("/publishers/google/models?pageSize=50", 0) == 0);
ASSERT_TRUE(l.path.find("pageToken=tok%2Fen") != std::string::npos);
PlannedCall g = mg.plan_get("anthropic", "claude-x", true);
ASSERT_EQ(g.url(),
"https://aiplatform.googleapis.com/v1/publishers/anthropic/models/claude-x"
"?view=PUBLISHER_MODEL_VIEW_FULL");
ModelGarden pg(client(), project());
ASSERT_EQ(pg.plan_fetch_config("anthropic", "claude-x").url(),
"https://us-central1-aiplatform.googleapis.com/v1beta1/projects/demo-proj/locations/"
"us-central1/publishers/anthropic/models/claude-x:fetchPublisherModelConfig");
ASSERT_EQ(pg.plan_eula("garden.eula_check", "publishers/google/models/gemma3").url(),
"https://aiplatform.googleapis.com/v1beta1/projects/demo-proj/modelGardenEula:check");
OpenModels om(client(), project());
ASSERT_EQ(om.plan_get("77").path,
"/projects/demo-proj/locations/us-central1/endpoints/77");
ASSERT_EQ(om.plan_raw("endpoint.raw_predict", "77", Json::object()).path,
"/projects/demo-proj/locations/us-central1/endpoints/77:rawPredict");
}
TEST(models_paths, claude_is_project_only) {
ClaudeRequest req;
req.model = "claude-sonnet-4-6";
req.messages.push_back({"user", "hi"});
ClaudeModels c(client(), project());
PlannedCall p = c.plan_generate(req);
ASSERT_EQ(p.url(),
"https://us-central1-aiplatform.googleapis.com/v1/projects/demo-proj/locations/us-central1/"
"publishers/anthropic/models/claude-sonnet-4-6:rawPredict");
ASSERT_EQ(c.plan_stream(req).path.substr(c.plan_stream(req).path.rfind(':')), ":streamRawPredict");
ASSERT_EQ(p.body.find("anthropic_version")->as_string(), "vertex-2023-10-16");
ASSERT_FALSE(p.body.has("stream"));
ASSERT_TRUE(c.plan_stream(req).body.find("stream")->as_bool());
ClaudeModels ex(client(), VertexTarget::express());
ASSERT_THROW(ex.plan_generate(req), GoogleError);
}
TEST(models_paths, project_only_calls_refused_in_express) {
Gemini ex(client(), VertexTarget::express());
ASSERT_THROW(ex.plan_embed_content("e", "x", {}), GoogleError); // no express embedContent
OpenModels om(client(), VertexTarget::express());
ASSERT_THROW(om.plan_get("77"), GoogleError);
ASSERT_THROW(om.plan_list_page(1, "", ""), GoogleError);
ModelGarden mg(client(), VertexTarget::express());
ASSERT_THROW(mg.plan_fetch_config("google", "m"), GoogleError);
ASSERT_THROW(mg.plan_eula("garden.eula_check", "publishers/google/models/m"), GoogleError);
// express supports these (verified by Discovery and live):
ex.plan_count_tokens("m", rich());
ex.plan_compute_tokens("m", {Content::user("x")});
ex.plan_embed("e", {"x"}, {});
MediaGenerator media(client(), VertexTarget::express());
VideoRequest v;
v.prompt = "x";
media.plan_start_video("veo", v);
}
TEST(models_validation, ids_never_reach_a_path) {
Gemini g(client(), VertexTarget::express());
for (const char* bad : {"", "..", "a/b", "a b", "a?x=1", "a:generateContent", "m%2f"})
ASSERT_THROW(g.plan_generate(bad, rich()), GoogleError);
OpenModels om(client(), project());
for (const char* bad : {"", "..", "7/8", "7:predict"})
ASSERT_THROW(om.plan_get(bad), GoogleError);
ModelGarden mg(client(), VertexTarget::express());
ASSERT_THROW(mg.plan_get("a/b", "m", false), GoogleError);
ASSERT_THROW(mg.plan_eula("garden.eula_check", "models/x"), GoogleError);
}
TEST(models_validation, gemini_caps) {
Gemini g(client(), VertexTarget::express());
GenerateRequest empty;
ASSERT_THROW(g.plan_generate("m", empty), GoogleError);
GenerateRequest big = rich();
big.config.max_output_tokens = ModelsCaps::kMaxOutputTokens + 1;
ASSERT_THROW(g.plan_generate("m", big), GoogleError);
big.config.max_output_tokens = 0;
ASSERT_THROW(g.plan_generate("m", big), GoogleError);
GenerateRequest many;
for (std::size_t i = 0; i <= ModelsCaps::kMaxContents; ++i)
many.contents.push_back(Content::user("x"));
ASSERT_THROW(g.plan_generate("m", many), GoogleError);
GenerateRequest huge;
huge.contents.push_back(Content::user(std::string(ModelsCaps::kMaxTextChars + 1, 'x')));
ASSERT_THROW(g.plan_count_tokens("m", huge), GoogleError);
GenerateRequest cand = rich();
cand.config.candidate_count = ModelsCaps::kMaxCandidates + 1;
ASSERT_THROW(g.plan_generate("m", cand), GoogleError);
ASSERT_THROW(g.plan_compute_tokens("m", {}), GoogleError);
}
TEST(models_validation, embeddings) {
Gemini g(client(), project());
ASSERT_THROW(g.plan_embed("e", {}, {}), GoogleError);
ASSERT_THROW(g.plan_embed("e", std::vector<std::string>(ModelsCaps::kMaxEmbedBatch + 1, "x"), {}),
GoogleError);
ASSERT_THROW(g.plan_embed("e", {""}, {}), GoogleError);
ASSERT_THROW(g.plan_embed("e", {std::string(ModelsCaps::kMaxEmbedChars + 1, 'x')}, {}),
GoogleError);
EmbedOptions bad;
bad.task_type = "drop table";
ASSERT_THROW(g.plan_embed_content("e", "x", bad), GoogleError);
bad = {};
bad.output_dimensionality = ModelsCaps::kMaxEmbedDimensions + 1;
ASSERT_THROW(g.plan_embed_content("e", "x", bad), GoogleError);
ASSERT_THROW(g.plan_embed_content("e", "", {}), GoogleError);
}
TEST(models_validation, claude) {
ClaudeModels c(client(), project());
ClaudeRequest req;
req.model = "claude-sonnet-4-6";
ASSERT_THROW(c.plan_generate(req), GoogleError); // no messages
req.messages.push_back({"assistant", "first"});
ASSERT_THROW(c.plan_generate(req), GoogleError); // must start with user
req.messages[0] = {"user", "x"};
req.max_tokens = ModelsCaps::kMaxClaudeTokens + 1;
ASSERT_THROW(c.plan_generate(req), GoogleError);
req.max_tokens = 2000;
req.thinking_budget = 500;
ASSERT_THROW(c.plan_generate(req), GoogleError);
req.thinking_budget = 1024;
c.plan_generate(req);
req.thinking_budget = 0;
req.temperature = 1.5;
ASSERT_THROW(c.plan_generate(req), GoogleError);
req.temperature.reset();
req.messages[0].role = "system";
ASSERT_THROW(c.plan_generate(req), GoogleError);
req.messages[0].role = "user";
req.model = "a/b";
ASSERT_THROW(c.plan_generate(req), GoogleError);
ASSERT_EQ(ClaudeModels::text_delta(Json::parse(
"{\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"a\"}}")),
"a");
ASSERT_EQ(ClaudeModels::text_delta(Json::parse("{\"type\":\"message_stop\"}")), "");
}
TEST(models_validation, open_models) {
OpenModels om(client(), project());
Json inst = Json::array();
ASSERT_THROW(om.plan_predict("77", inst, Json()), GoogleError);
ASSERT_THROW(om.plan_predict("77", Json::object(), Json()), GoogleError);
Json many = Json::array();
for (std::size_t i = 0; i <= ModelsCaps::kMaxInstances; ++i)
many.push(OpenModels::gemma_instance("x"));
ASSERT_THROW(om.plan_predict("77", many, Json()), GoogleError);
ASSERT_THROW(OpenModels::gemma_instance(""), GoogleError);
ASSERT_THROW(OpenModels::gemma_instance("x", ModelsCaps::kMaxGemmaTokens + 1), GoogleError);
ASSERT_THROW(OpenModels::gemma_instance("x", 8, 3.0), GoogleError);
ASSERT_THROW(om.plan_list_page(0, "", ""), GoogleError);
ASSERT_THROW(om.plan_list_page(ModelsCaps::kMaxListPages + 1, "", ""), GoogleError);
ASSERT_THROW(om.plan_raw("endpoint.raw_predict", "77", Json::array()), GoogleError);
ASSERT_THROW(om.plan_fetch_operation("77", "../x"), GoogleError);
ASSERT_THROW(om.plan_fetch_operation("77", ""), GoogleError);
ASSERT_THROW(om.plan_server_streaming("77", Json::array(), Json()), GoogleError);
ASSERT_THROW(om.invoke_publisher("google", "m", "a/b", Json::object()), GoogleError);
}
TEST(models_validation, media) {
MediaGenerator g(client(), VertexTarget::express());
ImageRequest i;
ASSERT_THROW(g.plan_image("m", i), GoogleError); // empty prompt
i.prompt = std::string(ModelsCaps::kMaxPromptChars + 1, 'x');
ASSERT_THROW(g.plan_image("m", i), GoogleError);
i.prompt = "x";
i.count = ModelsCaps::kMaxImages + 1;
ASSERT_THROW(g.plan_image("m", i), GoogleError);
i.count = 1;
i.aspect_ratio = "1:1;drop";
ASSERT_THROW(g.plan_image("m", i), GoogleError);
MusicRequest m;
m.prompt = "x";
m.count = 2;
m.seed = 4;
ASSERT_THROW(g.plan_music("m", m), GoogleError);
VideoRequest v;
v.prompt = "x";
v.duration_seconds = ModelsCaps::kMaxVideoSeconds + 1;
ASSERT_THROW(g.plan_start_video("m", v), GoogleError);
v.duration_seconds = 4;
v.storage_uri = "http://evil";
ASSERT_THROW(g.plan_start_video("m", v), GoogleError);
ASSERT_THROW(g.plan_video_status("m", "../etc"), GoogleError);
ASSERT_THROW(g.plan_image("../m", ImageRequest{"x", 1, "", ""}), GoogleError);
}
TEST(models_validation, destructive_calls_need_confirm) {
ModelGarden mg(client(), project());
DeployOptions d;
d.publisher_model = "publishers/google/models/gemma3";
ASSERT_THROW(mg.plan_deploy(d), GoogleError); // confirm is false by default
ASSERT_THROW(mg.accept_eula("publishers/google/models/gemma3", false), GoogleError);
d.confirm = true;
mg.plan_deploy(d);
d.publisher_model = "x; rm -rf";
ASSERT_THROW(mg.plan_deploy(d), GoogleError);
ListModelsOptions o;
o.page_size = ModelsCaps::kMaxPageSize + 1;
ASSERT_THROW(mg.plan_list_page("google", o), GoogleError);
o = {};
o.max_pages = ModelsCaps::kMaxListPages + 1;
ASSERT_THROW(mg.plan_list_page("google", o), GoogleError);
o = {};
o.order_by = "name; drop";
ASSERT_THROW(mg.plan_list_page("google", o), GoogleError);
}
TEST(models_validation, target_and_vertex_model) {
CloudContext bad;
bad.project = "Bad Project!";
ASSERT_THROW(VertexTarget::project(bad), GoogleError);
ASSERT_THROW(VertexTarget::express().parent_path(), GoogleError);
auto gem = std::make_shared<Gemini>(client(), VertexTarget::express());
ASSERT_THROW(VertexModel(gem, "a/b"), GoogleError);
VertexModel ok(gem, "gemini-3.5-flash");
LanguageModel& lm = ok; // the seam agents code against
(void)lm;
ASSERT_EQ(ok.default_model(), "gemini-3.5-flash");
}
TEST(models_notconfigured, every_service_refuses_without_a_credential_and_never_sends) {
MockTransport* mock = nullptr;
auto c = client(false, &mock);
GenerateRequest r = rich();
ASSERT_THROW(Gemini(c, VertexTarget::express()).generate("m", r), NotConfiguredError);
ASSERT_THROW(Gemini(c, VertexTarget::express()).embed("e", {"x"}), NotConfiguredError);
ASSERT_THROW(ModelGarden(c, VertexTarget::express()).list("google"), NotConfiguredError);
ASSERT_THROW(OpenModels(c, project()).list_endpoints(), NotConfiguredError);
ClaudeRequest cr;
cr.model = "claude-sonnet-4-6";
cr.messages.push_back({"user", "hi"});
ASSERT_THROW(ClaudeModels(c, project()).generate(cr), NotConfiguredError);
ImageRequest ir;
ir.prompt = "x";
ASSERT_THROW(MediaGenerator(c, VertexTarget::express()).generate_image("m", ir),
NotConfiguredError);
ASSERT_EQ(mock->send_count(), 0u);
}
TEST(models_notconfigured, invalid_input_wins_before_any_traffic) {
MockTransport* mock = nullptr;
auto c = client(true, &mock);
GenerateRequest empty;
ASSERT_THROW(Gemini(c, VertexTarget::express()).generate("m", empty), GoogleError);
ASSERT_THROW(Gemini(c, VertexTarget::express()).stream("../x", rich(), nullptr), GoogleError);
ASSERT_EQ(mock->send_count(), 0u);
}