Merge models module (Model Garden, Gemini on Vertex, partner and open models, media)
This commit is contained in:
@@ -0,0 +1,134 @@
|
||||
// Opt-in live checks against Google (MOLO_LIVE_NET=1 plus a credential).
|
||||
// Tiny and read-only: a 20 token generation, countTokens, computeTokens, one
|
||||
// embedding. Nothing is created, changed or deleted. Express keys (AQ.) only
|
||||
// work for the generateContent family on aiplatform.googleapis.com; the
|
||||
// OAuth-only reads additionally need GOOGLE_ACCESS_TOKEN and
|
||||
// GOOGLE_CLOUD_PROJECT. Without them the test prints a loud SKIP.
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
|
||||
#include "framework.h"
|
||||
#include "google/credentials.h"
|
||||
#include "google/errors.h"
|
||||
#include "google/models/claude_models.h"
|
||||
#include "google/models/gemini.h"
|
||||
#include "google/models/model_garden.h"
|
||||
#include "google/models/open_models.h"
|
||||
|
||||
using namespace molo::google;
|
||||
using namespace molo::google::models;
|
||||
|
||||
namespace {
|
||||
bool live() { return std::getenv("MOLO_LIVE_NET") != nullptr; }
|
||||
std::string model() {
|
||||
std::string m = Credentials::env("GOOGLE_GENAI_MODEL");
|
||||
return m.empty() ? "gemini-3.5-flash" : m;
|
||||
}
|
||||
GenerateRequest tiny() {
|
||||
GenerateRequest r;
|
||||
r.contents.push_back(Content::user("Say hello in one short sentence."));
|
||||
r.config.max_output_tokens = 20;
|
||||
r.config.thinking_budget = 0;
|
||||
return r;
|
||||
}
|
||||
// Shared project quota can answer 429 for generation; that still proves the
|
||||
// route and credential, so it is noted, not failed. Anything else fails.
|
||||
template <typename F>
|
||||
void tolerate_quota(const char* what, F&& f) {
|
||||
try {
|
||||
f();
|
||||
} catch (const GoogleError& e) {
|
||||
if (e.http_status() != 429) throw;
|
||||
std::cout << " note: " << what << " reached Google, quota exhausted (429)\n";
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(models_live, express_gemini_family) {
|
||||
if (!live() || Credentials::env().empty()) {
|
||||
std::cout << "SKIP models_live.express_gemini_family (set MOLO_LIVE_NET=1 and GOOGLE_API_KEY)\n";
|
||||
return;
|
||||
}
|
||||
if (!Credentials::env("GOOGLE_ACCESS_TOKEN").empty()) {
|
||||
// A bearer token beats the key and Google rejects it on project-less paths.
|
||||
std::cout << "SKIP models_live.express_gemini_family (GOOGLE_ACCESS_TOKEN is set; unset it to test express keys)\n";
|
||||
return;
|
||||
}
|
||||
auto gemini = std::make_shared<Gemini>(GoogleClient::from_env(), VertexTarget::express());
|
||||
tolerate_quota("express generate", [&] {
|
||||
ASSERT_TRUE(!gemini->generate(model(), tiny()).text().empty());
|
||||
});
|
||||
tolerate_quota("express stream", [&] {
|
||||
std::string streamed;
|
||||
gemini->stream(model(), tiny(), [&](const GenerateResponse& c) {
|
||||
streamed += c.text();
|
||||
return true;
|
||||
});
|
||||
ASSERT_TRUE(!streamed.empty());
|
||||
});
|
||||
CountTokensResult n = gemini->count_tokens(model(), tiny());
|
||||
ASSERT_TRUE(n.total_tokens > 0);
|
||||
auto toks = gemini->compute_tokens(model(), tiny().contents);
|
||||
ASSERT_TRUE(!toks.empty() && !toks[0].tokens.empty());
|
||||
auto emb = gemini->embed("text-embedding-005", {"hello"});
|
||||
ASSERT_TRUE(emb.size() == 1 && emb[0].values.size() > 10);
|
||||
}
|
||||
|
||||
TEST(models_live, project_scoped_count_tokens) {
|
||||
std::string project = Credentials::env("GOOGLE_CLOUD_PROJECT");
|
||||
if (!live() || Credentials::env().empty() || project.empty()) {
|
||||
std::cout << "SKIP models_live.project_scoped_count_tokens (set MOLO_LIVE_NET=1, GOOGLE_API_KEY, GOOGLE_CLOUD_PROJECT)\n";
|
||||
return;
|
||||
}
|
||||
CloudContext c;
|
||||
c.project = project;
|
||||
c.location = "global";
|
||||
Gemini gemini(GoogleClient::from_env(), VertexTarget::project(c));
|
||||
ASSERT_TRUE(gemini.count_tokens(model(), tiny()).total_tokens > 0);
|
||||
}
|
||||
|
||||
TEST(models_live, oauth_reads) {
|
||||
std::string project = Credentials::env("GOOGLE_CLOUD_PROJECT");
|
||||
if (!live() || Credentials::env("GOOGLE_ACCESS_TOKEN").empty() || project.empty()) {
|
||||
std::cout << "SKIP models_live.oauth_reads (needs MOLO_LIVE_NET=1, GOOGLE_ACCESS_TOKEN, GOOGLE_CLOUD_PROJECT: "
|
||||
"Model Garden and endpoints refuse API keys)\n";
|
||||
return;
|
||||
}
|
||||
CloudContext c;
|
||||
c.project = project;
|
||||
auto client = GoogleClient::from_env();
|
||||
ModelGarden garden(client, VertexTarget::project(c));
|
||||
ListModelsOptions o;
|
||||
o.page_size = 5;
|
||||
o.max_pages = 1;
|
||||
ASSERT_TRUE(!garden.list("google", o).empty());
|
||||
PublisherModel m = garden.get("google", model());
|
||||
ASSERT_TRUE(m.name.find(model()) != std::string::npos);
|
||||
OpenModels endpoints(client, VertexTarget::project(c));
|
||||
endpoints.list_endpoints(1); // may be empty; must not throw
|
||||
// embedContent exists only project scoped; this project has it in us-central1.
|
||||
Gemini gemini(client, VertexTarget::project(c));
|
||||
ASSERT_TRUE(!gemini.embed_content("gemini-embedding-001", "hello").values.empty());
|
||||
// Project scoped generation and tokens at the global location.
|
||||
CloudContext g;
|
||||
g.project = project;
|
||||
g.location = "global";
|
||||
Gemini global(client, VertexTarget::project(g));
|
||||
tolerate_quota("project generate", [&] {
|
||||
ASSERT_TRUE(!global.generate(model(), tiny()).text().empty());
|
||||
});
|
||||
ASSERT_TRUE(!global.compute_tokens(model(), tiny().contents).empty());
|
||||
// Claude rawPredict, 16 tokens. 429 (project quota) still proves the route,
|
||||
// auth and body were accepted; any other failure is a real failure.
|
||||
ClaudeModels claude(client, VertexTarget::project(g));
|
||||
ClaudeRequest cr;
|
||||
cr.model = "claude-sonnet-4-6";
|
||||
cr.messages.push_back({"user", "Say hi"});
|
||||
cr.max_tokens = 16;
|
||||
try {
|
||||
ASSERT_TRUE(!claude.generate(cr).text.empty());
|
||||
} catch (const GoogleError& e) {
|
||||
if (e.http_status() != 429) throw;
|
||||
std::cout << " note: claude rawPredict reached Google, project quota exhausted (429)\n";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
// Deterministic local tests for the models module: request construction
|
||||
// checked against the call table (which tools/check_discovery.py checks
|
||||
// against Google's Discovery documents), id validation, caps, confirm gates,
|
||||
// not-configured behavior. No Google replies are scripted anywhere.
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
|
||||
#include "clients/transports/mock_transport.h"
|
||||
#include "framework.h"
|
||||
#include "google/credentials.h"
|
||||
#include "google/errors.h"
|
||||
#include "google/models/calls.h"
|
||||
#include "google/models/caps.h"
|
||||
#include "google/models/claude_models.h"
|
||||
#include "google/models/gemini.h"
|
||||
#include "google/models/media.h"
|
||||
#include "google/models/model_garden.h"
|
||||
#include "google/models/open_models.h"
|
||||
#include "google/models/vertex_model.h"
|
||||
|
||||
using namespace molo::google;
|
||||
using namespace molo::google::models;
|
||||
using molo::clients::body::Json;
|
||||
using molo::clients::transports::MockTransport;
|
||||
|
||||
namespace {
|
||||
|
||||
std::shared_ptr<GoogleClient> client(bool configured = true,
|
||||
MockTransport** raw = nullptr) {
|
||||
auto t = std::make_unique<MockTransport>();
|
||||
if (raw) *raw = t.get();
|
||||
return std::make_shared<GoogleClient>(
|
||||
std::move(t), configured ? Credentials::bearer("tok") : nullptr);
|
||||
}
|
||||
|
||||
VertexTarget project() {
|
||||
CloudContext c;
|
||||
c.project = "demo-proj";
|
||||
c.location = "us-central1";
|
||||
return VertexTarget::project(c);
|
||||
}
|
||||
|
||||
std::set<std::string> table_fields(const std::string& id) {
|
||||
std::set<std::string> out;
|
||||
std::istringstream in(CallPlanner::spec(id).fields);
|
||||
for (std::string f; in >> f;) out.insert(f);
|
||||
return out;
|
||||
}
|
||||
|
||||
// Every key of `body` (and of generationConfig) must be a field the call
|
||||
// table declares for that call.
|
||||
void expect_keys_declared(const PlannedCall& call, const std::string& id) {
|
||||
ASSERT_TRUE(call.has_body);
|
||||
auto fields = table_fields(id);
|
||||
for (const auto& [k, v] : call.body.as_object()) {
|
||||
if (!fields.count(k)) FAIL("body key not in call table: " + k);
|
||||
bool nested = false;
|
||||
for (const auto& f : fields)
|
||||
if (f.rfind(k + ".", 0) == 0) nested = true;
|
||||
if (nested && v.is_object())
|
||||
for (const auto& [k2, v2] : v.as_object()) {
|
||||
(void)v2;
|
||||
if (!fields.count(k + "." + k2)) FAIL("nested key not in call table: " + k + "." + k2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
GenerateRequest rich() {
|
||||
GenerateRequest r;
|
||||
r.system = Content::user("be brief");
|
||||
r.contents.push_back(Content::user("hello"));
|
||||
Tool t;
|
||||
t.google_search = true;
|
||||
r.tools.push_back(t);
|
||||
r.tool_mode = "AUTO";
|
||||
r.safety.push_back({"HARM_CATEGORY_HARASSMENT", "BLOCK_NONE"});
|
||||
r.config.temperature = 0.2;
|
||||
r.config.top_p = 0.9;
|
||||
r.config.top_k = 20;
|
||||
r.config.max_output_tokens = 64;
|
||||
r.config.candidate_count = 1;
|
||||
r.config.thinking_budget = 0;
|
||||
r.config.stop_sequences = {"END"};
|
||||
r.config.response_mime_type = "application/json";
|
||||
return r;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST(models_table, rows_are_well_formed_and_unique) {
|
||||
std::set<std::string> ids;
|
||||
for (const CallSpec& s : kCalls) {
|
||||
ASSERT_TRUE(ids.insert(s.id).second);
|
||||
std::string http = s.http;
|
||||
ASSERT_TRUE(http == "GET" || http == "POST");
|
||||
std::string v = s.version;
|
||||
ASSERT_TRUE(v == "v1" || v == "v1beta1");
|
||||
std::string p = s.prefix;
|
||||
ASSERT_TRUE(p == "" || p == "L" || p == "R");
|
||||
ASSERT_TRUE(std::string(s.rpc).rfind("aiplatform.", 0) == 0);
|
||||
ASSERT_TRUE(std::string(s.suffix)[0] == '/' || std::string(s.suffix)[0] == ':');
|
||||
}
|
||||
ASSERT_EQ(ids.size(), kCallCount);
|
||||
}
|
||||
|
||||
TEST(models_paths, express_and_project_forms) {
|
||||
Gemini ex(client(), VertexTarget::express());
|
||||
PlannedCall a = ex.plan_generate("gemini-3.5-flash", rich());
|
||||
ASSERT_EQ(a.url(),
|
||||
"https://aiplatform.googleapis.com/v1/publishers/google/models/gemini-3.5-flash:generateContent");
|
||||
ASSERT_TRUE(a.method == molo::clients::Method::kPost);
|
||||
|
||||
Gemini pr(client(), project());
|
||||
PlannedCall b = pr.plan_generate("gemini-3.5-flash", rich());
|
||||
ASSERT_EQ(b.url(),
|
||||
"https://us-central1-aiplatform.googleapis.com/v1/projects/demo-proj/locations/us-central1/"
|
||||
"publishers/google/models/gemini-3.5-flash:generateContent");
|
||||
|
||||
CloudContext g;
|
||||
g.project = "demo-proj";
|
||||
g.location = "global";
|
||||
Gemini gl(client(), VertexTarget::project(g));
|
||||
ASSERT_EQ(gl.plan_generate("m", rich()).url(),
|
||||
"https://aiplatform.googleapis.com/v1/projects/demo-proj/locations/global/"
|
||||
"publishers/google/models/m:generateContent");
|
||||
}
|
||||
|
||||
TEST(models_paths, stream_has_alt_sse) {
|
||||
Gemini ex(client(), VertexTarget::express());
|
||||
PlannedCall c = ex.plan_stream("m", rich());
|
||||
ASSERT_EQ(c.path, "/publishers/google/models/m:streamGenerateContent?alt=sse");
|
||||
}
|
||||
|
||||
TEST(models_paths, bodies_use_only_declared_fields) {
|
||||
Gemini g(client(), project());
|
||||
expect_keys_declared(g.plan_generate("m", rich()), "gemini.generate");
|
||||
expect_keys_declared(g.plan_stream("m", rich()), "gemini.stream");
|
||||
expect_keys_declared(g.plan_count_tokens("m", rich()), "gemini.count_tokens");
|
||||
expect_keys_declared(g.plan_compute_tokens("m", {Content::user("x")}), "gemini.compute_tokens");
|
||||
EmbedOptions o;
|
||||
o.task_type = "RETRIEVAL_QUERY";
|
||||
o.title = "t";
|
||||
o.output_dimensionality = 8;
|
||||
o.auto_truncate = true;
|
||||
expect_keys_declared(g.plan_embed_content("e", "text", o), "gemini.embed_content");
|
||||
expect_keys_declared(g.plan_embed("e", {"a", "b"}, o), "gemini.embed_predict");
|
||||
// countTokens must not carry safety or tool config.
|
||||
PlannedCall ct = g.plan_count_tokens("m", rich());
|
||||
ASSERT_FALSE(ct.body.has("safetySettings"));
|
||||
ASSERT_FALSE(ct.body.has("toolConfig"));
|
||||
// embedContent nested config names
|
||||
Json cfg = *g.plan_embed_content("e", "text", o).body.find("embedContentConfig");
|
||||
for (const auto& [k, v] : cfg.as_object()) {
|
||||
(void)v;
|
||||
ASSERT_TRUE(table_fields("gemini.embed_content").count("embedContentConfig." + k) == 1);
|
||||
}
|
||||
|
||||
OpenModels om(client(), project());
|
||||
Json inst = Json::array();
|
||||
inst.push(OpenModels::gemma_instance("hi"));
|
||||
expect_keys_declared(om.plan_predict("77", inst, Json::object()), "endpoint.predict");
|
||||
expect_keys_declared(om.plan_server_streaming("77", inst, Json::object()),
|
||||
"endpoint.server_streaming_predict");
|
||||
expect_keys_declared(om.plan_generate("endpoint.generate_content", "77", rich()),
|
||||
"endpoint.generate_content");
|
||||
expect_keys_declared(om.plan_long_running("77", inst, Json::object()),
|
||||
"endpoint.predict_long_running");
|
||||
expect_keys_declared(om.plan_fetch_operation("77", "projects/p/locations/l/operations/1"),
|
||||
"endpoint.fetch_predict_operation");
|
||||
|
||||
MediaGenerator mg(client(), project());
|
||||
ImageRequest ir;
|
||||
ir.prompt = "cat";
|
||||
ir.negative_prompt = "dog";
|
||||
expect_keys_declared(mg.plan_image("imagen", ir), "media.predict");
|
||||
MusicRequest mr;
|
||||
mr.prompt = "calm";
|
||||
expect_keys_declared(mg.plan_music("lyria-002", mr), "media.predict");
|
||||
VideoRequest vr;
|
||||
vr.prompt = "waves";
|
||||
vr.duration_seconds = 8;
|
||||
vr.storage_uri = "gs://bucket/out";
|
||||
expect_keys_declared(mg.plan_start_video("veo", vr), "media.predict_long_running");
|
||||
expect_keys_declared(mg.plan_video_status("veo", "projects/p/locations/l/operations/1"),
|
||||
"media.fetch_predict_operation");
|
||||
|
||||
ModelGarden mgd(client(), project());
|
||||
DeployOptions d;
|
||||
d.publisher_model = "publishers/google/models/gemma3@gemma-3-1b-it";
|
||||
d.model_display_name = "m";
|
||||
d.endpoint_display_name = "e";
|
||||
d.accept_eula = true;
|
||||
d.confirm = true;
|
||||
expect_keys_declared(mgd.plan_deploy(d), "garden.deploy");
|
||||
expect_keys_declared(mgd.plan_eula("garden.eula_check", d.publisher_model), "garden.eula_check");
|
||||
}
|
||||
|
||||
TEST(models_paths, model_garden_and_endpoints) {
|
||||
ModelGarden mg(client(), VertexTarget::express());
|
||||
ListModelsOptions o;
|
||||
PlannedCall l = mg.plan_list_page("google", o, "tok/en");
|
||||
ASSERT_EQ(l.endpoint.base_url, "https://aiplatform.googleapis.com/v1beta1");
|
||||
ASSERT_TRUE(l.path.rfind("/publishers/google/models?pageSize=50", 0) == 0);
|
||||
ASSERT_TRUE(l.path.find("pageToken=tok%2Fen") != std::string::npos);
|
||||
PlannedCall g = mg.plan_get("anthropic", "claude-x", true);
|
||||
ASSERT_EQ(g.url(),
|
||||
"https://aiplatform.googleapis.com/v1/publishers/anthropic/models/claude-x"
|
||||
"?view=PUBLISHER_MODEL_VIEW_FULL");
|
||||
|
||||
ModelGarden pg(client(), project());
|
||||
ASSERT_EQ(pg.plan_fetch_config("anthropic", "claude-x").url(),
|
||||
"https://us-central1-aiplatform.googleapis.com/v1beta1/projects/demo-proj/locations/"
|
||||
"us-central1/publishers/anthropic/models/claude-x:fetchPublisherModelConfig");
|
||||
ASSERT_EQ(pg.plan_eula("garden.eula_check", "publishers/google/models/gemma3").url(),
|
||||
"https://aiplatform.googleapis.com/v1beta1/projects/demo-proj/modelGardenEula:check");
|
||||
|
||||
OpenModels om(client(), project());
|
||||
ASSERT_EQ(om.plan_get("77").path,
|
||||
"/projects/demo-proj/locations/us-central1/endpoints/77");
|
||||
ASSERT_EQ(om.plan_raw("endpoint.raw_predict", "77", Json::object()).path,
|
||||
"/projects/demo-proj/locations/us-central1/endpoints/77:rawPredict");
|
||||
}
|
||||
|
||||
TEST(models_paths, claude_is_project_only) {
|
||||
ClaudeRequest req;
|
||||
req.model = "claude-sonnet-4-6";
|
||||
req.messages.push_back({"user", "hi"});
|
||||
ClaudeModels c(client(), project());
|
||||
PlannedCall p = c.plan_generate(req);
|
||||
ASSERT_EQ(p.url(),
|
||||
"https://us-central1-aiplatform.googleapis.com/v1/projects/demo-proj/locations/us-central1/"
|
||||
"publishers/anthropic/models/claude-sonnet-4-6:rawPredict");
|
||||
ASSERT_EQ(c.plan_stream(req).path.substr(c.plan_stream(req).path.rfind(':')), ":streamRawPredict");
|
||||
ASSERT_EQ(p.body.find("anthropic_version")->as_string(), "vertex-2023-10-16");
|
||||
ASSERT_FALSE(p.body.has("stream"));
|
||||
ASSERT_TRUE(c.plan_stream(req).body.find("stream")->as_bool());
|
||||
ClaudeModels ex(client(), VertexTarget::express());
|
||||
ASSERT_THROW(ex.plan_generate(req), GoogleError);
|
||||
}
|
||||
|
||||
TEST(models_paths, project_only_calls_refused_in_express) {
|
||||
Gemini ex(client(), VertexTarget::express());
|
||||
ASSERT_THROW(ex.plan_embed_content("e", "x", {}), GoogleError); // no express embedContent
|
||||
OpenModels om(client(), VertexTarget::express());
|
||||
ASSERT_THROW(om.plan_get("77"), GoogleError);
|
||||
ASSERT_THROW(om.plan_list_page(1, "", ""), GoogleError);
|
||||
ModelGarden mg(client(), VertexTarget::express());
|
||||
ASSERT_THROW(mg.plan_fetch_config("google", "m"), GoogleError);
|
||||
ASSERT_THROW(mg.plan_eula("garden.eula_check", "publishers/google/models/m"), GoogleError);
|
||||
// express supports these (verified by Discovery and live):
|
||||
ex.plan_count_tokens("m", rich());
|
||||
ex.plan_compute_tokens("m", {Content::user("x")});
|
||||
ex.plan_embed("e", {"x"}, {});
|
||||
MediaGenerator media(client(), VertexTarget::express());
|
||||
VideoRequest v;
|
||||
v.prompt = "x";
|
||||
media.plan_start_video("veo", v);
|
||||
}
|
||||
|
||||
TEST(models_validation, ids_never_reach_a_path) {
|
||||
Gemini g(client(), VertexTarget::express());
|
||||
for (const char* bad : {"", "..", "a/b", "a b", "a?x=1", "a:generateContent", "m%2f"})
|
||||
ASSERT_THROW(g.plan_generate(bad, rich()), GoogleError);
|
||||
OpenModels om(client(), project());
|
||||
for (const char* bad : {"", "..", "7/8", "7:predict"})
|
||||
ASSERT_THROW(om.plan_get(bad), GoogleError);
|
||||
ModelGarden mg(client(), VertexTarget::express());
|
||||
ASSERT_THROW(mg.plan_get("a/b", "m", false), GoogleError);
|
||||
ASSERT_THROW(mg.plan_eula("garden.eula_check", "models/x"), GoogleError);
|
||||
}
|
||||
|
||||
TEST(models_validation, gemini_caps) {
|
||||
Gemini g(client(), VertexTarget::express());
|
||||
GenerateRequest empty;
|
||||
ASSERT_THROW(g.plan_generate("m", empty), GoogleError);
|
||||
GenerateRequest big = rich();
|
||||
big.config.max_output_tokens = ModelsCaps::kMaxOutputTokens + 1;
|
||||
ASSERT_THROW(g.plan_generate("m", big), GoogleError);
|
||||
big.config.max_output_tokens = 0;
|
||||
ASSERT_THROW(g.plan_generate("m", big), GoogleError);
|
||||
GenerateRequest many;
|
||||
for (std::size_t i = 0; i <= ModelsCaps::kMaxContents; ++i)
|
||||
many.contents.push_back(Content::user("x"));
|
||||
ASSERT_THROW(g.plan_generate("m", many), GoogleError);
|
||||
GenerateRequest huge;
|
||||
huge.contents.push_back(Content::user(std::string(ModelsCaps::kMaxTextChars + 1, 'x')));
|
||||
ASSERT_THROW(g.plan_count_tokens("m", huge), GoogleError);
|
||||
GenerateRequest cand = rich();
|
||||
cand.config.candidate_count = ModelsCaps::kMaxCandidates + 1;
|
||||
ASSERT_THROW(g.plan_generate("m", cand), GoogleError);
|
||||
ASSERT_THROW(g.plan_compute_tokens("m", {}), GoogleError);
|
||||
}
|
||||
|
||||
TEST(models_validation, embeddings) {
|
||||
Gemini g(client(), project());
|
||||
ASSERT_THROW(g.plan_embed("e", {}, {}), GoogleError);
|
||||
ASSERT_THROW(g.plan_embed("e", std::vector<std::string>(ModelsCaps::kMaxEmbedBatch + 1, "x"), {}),
|
||||
GoogleError);
|
||||
ASSERT_THROW(g.plan_embed("e", {""}, {}), GoogleError);
|
||||
ASSERT_THROW(g.plan_embed("e", {std::string(ModelsCaps::kMaxEmbedChars + 1, 'x')}, {}),
|
||||
GoogleError);
|
||||
EmbedOptions bad;
|
||||
bad.task_type = "drop table";
|
||||
ASSERT_THROW(g.plan_embed_content("e", "x", bad), GoogleError);
|
||||
bad = {};
|
||||
bad.output_dimensionality = ModelsCaps::kMaxEmbedDimensions + 1;
|
||||
ASSERT_THROW(g.plan_embed_content("e", "x", bad), GoogleError);
|
||||
ASSERT_THROW(g.plan_embed_content("e", "", {}), GoogleError);
|
||||
}
|
||||
|
||||
TEST(models_validation, claude) {
|
||||
ClaudeModels c(client(), project());
|
||||
ClaudeRequest req;
|
||||
req.model = "claude-sonnet-4-6";
|
||||
ASSERT_THROW(c.plan_generate(req), GoogleError); // no messages
|
||||
req.messages.push_back({"assistant", "first"});
|
||||
ASSERT_THROW(c.plan_generate(req), GoogleError); // must start with user
|
||||
req.messages[0] = {"user", "x"};
|
||||
req.max_tokens = ModelsCaps::kMaxClaudeTokens + 1;
|
||||
ASSERT_THROW(c.plan_generate(req), GoogleError);
|
||||
req.max_tokens = 2000;
|
||||
req.thinking_budget = 500;
|
||||
ASSERT_THROW(c.plan_generate(req), GoogleError);
|
||||
req.thinking_budget = 1024;
|
||||
c.plan_generate(req);
|
||||
req.thinking_budget = 0;
|
||||
req.temperature = 1.5;
|
||||
ASSERT_THROW(c.plan_generate(req), GoogleError);
|
||||
req.temperature.reset();
|
||||
req.messages[0].role = "system";
|
||||
ASSERT_THROW(c.plan_generate(req), GoogleError);
|
||||
req.messages[0].role = "user";
|
||||
req.model = "a/b";
|
||||
ASSERT_THROW(c.plan_generate(req), GoogleError);
|
||||
ASSERT_EQ(ClaudeModels::text_delta(Json::parse(
|
||||
"{\"type\":\"content_block_delta\",\"delta\":{\"type\":\"text_delta\",\"text\":\"a\"}}")),
|
||||
"a");
|
||||
ASSERT_EQ(ClaudeModels::text_delta(Json::parse("{\"type\":\"message_stop\"}")), "");
|
||||
}
|
||||
|
||||
TEST(models_validation, open_models) {
|
||||
OpenModels om(client(), project());
|
||||
Json inst = Json::array();
|
||||
ASSERT_THROW(om.plan_predict("77", inst, Json()), GoogleError);
|
||||
ASSERT_THROW(om.plan_predict("77", Json::object(), Json()), GoogleError);
|
||||
Json many = Json::array();
|
||||
for (std::size_t i = 0; i <= ModelsCaps::kMaxInstances; ++i)
|
||||
many.push(OpenModels::gemma_instance("x"));
|
||||
ASSERT_THROW(om.plan_predict("77", many, Json()), GoogleError);
|
||||
ASSERT_THROW(OpenModels::gemma_instance(""), GoogleError);
|
||||
ASSERT_THROW(OpenModels::gemma_instance("x", ModelsCaps::kMaxGemmaTokens + 1), GoogleError);
|
||||
ASSERT_THROW(OpenModels::gemma_instance("x", 8, 3.0), GoogleError);
|
||||
ASSERT_THROW(om.plan_list_page(0, "", ""), GoogleError);
|
||||
ASSERT_THROW(om.plan_list_page(ModelsCaps::kMaxListPages + 1, "", ""), GoogleError);
|
||||
ASSERT_THROW(om.plan_raw("endpoint.raw_predict", "77", Json::array()), GoogleError);
|
||||
ASSERT_THROW(om.plan_fetch_operation("77", "../x"), GoogleError);
|
||||
ASSERT_THROW(om.plan_fetch_operation("77", ""), GoogleError);
|
||||
ASSERT_THROW(om.plan_server_streaming("77", Json::array(), Json()), GoogleError);
|
||||
ASSERT_THROW(om.invoke_publisher("google", "m", "a/b", Json::object()), GoogleError);
|
||||
}
|
||||
|
||||
TEST(models_validation, media) {
|
||||
MediaGenerator g(client(), VertexTarget::express());
|
||||
ImageRequest i;
|
||||
ASSERT_THROW(g.plan_image("m", i), GoogleError); // empty prompt
|
||||
i.prompt = std::string(ModelsCaps::kMaxPromptChars + 1, 'x');
|
||||
ASSERT_THROW(g.plan_image("m", i), GoogleError);
|
||||
i.prompt = "x";
|
||||
i.count = ModelsCaps::kMaxImages + 1;
|
||||
ASSERT_THROW(g.plan_image("m", i), GoogleError);
|
||||
i.count = 1;
|
||||
i.aspect_ratio = "1:1;drop";
|
||||
ASSERT_THROW(g.plan_image("m", i), GoogleError);
|
||||
MusicRequest m;
|
||||
m.prompt = "x";
|
||||
m.count = 2;
|
||||
m.seed = 4;
|
||||
ASSERT_THROW(g.plan_music("m", m), GoogleError);
|
||||
VideoRequest v;
|
||||
v.prompt = "x";
|
||||
v.duration_seconds = ModelsCaps::kMaxVideoSeconds + 1;
|
||||
ASSERT_THROW(g.plan_start_video("m", v), GoogleError);
|
||||
v.duration_seconds = 4;
|
||||
v.storage_uri = "http://evil";
|
||||
ASSERT_THROW(g.plan_start_video("m", v), GoogleError);
|
||||
ASSERT_THROW(g.plan_video_status("m", "../etc"), GoogleError);
|
||||
ASSERT_THROW(g.plan_image("../m", ImageRequest{"x", 1, "", ""}), GoogleError);
|
||||
}
|
||||
|
||||
TEST(models_validation, destructive_calls_need_confirm) {
|
||||
ModelGarden mg(client(), project());
|
||||
DeployOptions d;
|
||||
d.publisher_model = "publishers/google/models/gemma3";
|
||||
ASSERT_THROW(mg.plan_deploy(d), GoogleError); // confirm is false by default
|
||||
ASSERT_THROW(mg.accept_eula("publishers/google/models/gemma3", false), GoogleError);
|
||||
d.confirm = true;
|
||||
mg.plan_deploy(d);
|
||||
d.publisher_model = "x; rm -rf";
|
||||
ASSERT_THROW(mg.plan_deploy(d), GoogleError);
|
||||
ListModelsOptions o;
|
||||
o.page_size = ModelsCaps::kMaxPageSize + 1;
|
||||
ASSERT_THROW(mg.plan_list_page("google", o), GoogleError);
|
||||
o = {};
|
||||
o.max_pages = ModelsCaps::kMaxListPages + 1;
|
||||
ASSERT_THROW(mg.plan_list_page("google", o), GoogleError);
|
||||
o = {};
|
||||
o.order_by = "name; drop";
|
||||
ASSERT_THROW(mg.plan_list_page("google", o), GoogleError);
|
||||
}
|
||||
|
||||
TEST(models_validation, target_and_vertex_model) {
|
||||
CloudContext bad;
|
||||
bad.project = "Bad Project!";
|
||||
ASSERT_THROW(VertexTarget::project(bad), GoogleError);
|
||||
ASSERT_THROW(VertexTarget::express().parent_path(), GoogleError);
|
||||
auto gem = std::make_shared<Gemini>(client(), VertexTarget::express());
|
||||
ASSERT_THROW(VertexModel(gem, "a/b"), GoogleError);
|
||||
VertexModel ok(gem, "gemini-3.5-flash");
|
||||
LanguageModel& lm = ok; // the seam agents code against
|
||||
(void)lm;
|
||||
ASSERT_EQ(ok.default_model(), "gemini-3.5-flash");
|
||||
}
|
||||
|
||||
TEST(models_notconfigured, every_service_refuses_without_a_credential_and_never_sends) {
|
||||
MockTransport* mock = nullptr;
|
||||
auto c = client(false, &mock);
|
||||
GenerateRequest r = rich();
|
||||
ASSERT_THROW(Gemini(c, VertexTarget::express()).generate("m", r), NotConfiguredError);
|
||||
ASSERT_THROW(Gemini(c, VertexTarget::express()).embed("e", {"x"}), NotConfiguredError);
|
||||
ASSERT_THROW(ModelGarden(c, VertexTarget::express()).list("google"), NotConfiguredError);
|
||||
ASSERT_THROW(OpenModels(c, project()).list_endpoints(), NotConfiguredError);
|
||||
ClaudeRequest cr;
|
||||
cr.model = "claude-sonnet-4-6";
|
||||
cr.messages.push_back({"user", "hi"});
|
||||
ASSERT_THROW(ClaudeModels(c, project()).generate(cr), NotConfiguredError);
|
||||
ImageRequest ir;
|
||||
ir.prompt = "x";
|
||||
ASSERT_THROW(MediaGenerator(c, VertexTarget::express()).generate_image("m", ir),
|
||||
NotConfiguredError);
|
||||
ASSERT_EQ(mock->send_count(), 0u);
|
||||
}
|
||||
|
||||
TEST(models_notconfigured, invalid_input_wins_before_any_traffic) {
|
||||
MockTransport* mock = nullptr;
|
||||
auto c = client(true, &mock);
|
||||
GenerateRequest empty;
|
||||
ASSERT_THROW(Gemini(c, VertexTarget::express()).generate("m", empty), GoogleError);
|
||||
ASSERT_THROW(Gemini(c, VertexTarget::express()).stream("../x", rich(), nullptr), GoogleError);
|
||||
ASSERT_EQ(mock->send_count(), 0u);
|
||||
}
|
||||
Reference in New Issue
Block a user