feat(clients): models+runtime: close gaps (all Discovery methods wrapped, live re-verification)
models: EndpointAdmin and PublisherAdmin wrap every remaining method of the owned resources (84 call table rows, 132 Discovery methods covered, v1beta1 twins, IAM, operations, responses, export, config, enableModel); planner rejects undeclared body keys; VertexModel::TurnBuilder replays model turns with every thoughtSignature; Lyria WAV mime from bytes; veo-3.1 default. runtime: v1beta1 feedbackEntries, A2A gateway (40 rpcs), a2aTasks, task store records, scoped operations, beta twin flag (103 rows, 214 methods covered); checker verifies twins, a2aEndpoint patterns and coverage. example: admin routes for both parts. SPECs: live results moved from Unverified to Verified live (Veo, Lyria, thought signatures, v1beta1 routes). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,318 @@
|
||||
// Deterministic local tests for the admin surface of the models module
|
||||
// (EndpointAdmin, PublisherAdmin) and for the thought signature replay of
|
||||
// VertexModel. Requests are compared to the call table, which
|
||||
// tools/check_discovery.py checks against Google's Discovery documents. No
|
||||
// Google replies are scripted anywhere.
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
|
||||
#include "clients/transports/mock_transport.h"
|
||||
#include "framework.h"
|
||||
#include "google/credentials.h"
|
||||
#include "google/errors.h"
|
||||
#include "google/models/calls.h"
|
||||
#include "google/models/caps.h"
|
||||
#include "google/models/endpoint_admin.h"
|
||||
#include "google/models/gemini.h"
|
||||
#include "google/models/media.h"
|
||||
#include "google/models/publisher_admin.h"
|
||||
#include "google/models/vertex_model.h"
|
||||
|
||||
using namespace molo::google;
|
||||
using namespace molo::google::models;
|
||||
using molo::clients::body::Json;
|
||||
using molo::clients::transports::MockTransport;
|
||||
|
||||
namespace {
|
||||
|
||||
std::shared_ptr<GoogleClient> client(bool configured, MockTransport** raw = nullptr) {
|
||||
auto t = std::make_unique<MockTransport>();
|
||||
if (raw) *raw = t.get();
|
||||
return std::make_shared<GoogleClient>(std::move(t),
|
||||
configured ? Credentials::bearer("tok") : nullptr);
|
||||
}
|
||||
|
||||
VertexTarget project() {
|
||||
CloudContext c;
|
||||
c.project = "demo-proj";
|
||||
c.location = "us-central1";
|
||||
return VertexTarget::project(c);
|
||||
}
|
||||
|
||||
const std::string kEp =
|
||||
"https://us-central1-aiplatform.googleapis.com/v1/projects/demo-proj/locations/us-central1";
|
||||
const std::string kEpBeta =
|
||||
"https://us-central1-aiplatform.googleapis.com/v1beta1/projects/demo-proj/locations/us-central1";
|
||||
|
||||
// True when the call is rejected locally (invalid) and nothing was sent.
|
||||
template <typename F>
|
||||
bool rejected(MockTransport* mock, F&& f) {
|
||||
std::size_t before = mock->send_count();
|
||||
try {
|
||||
f();
|
||||
} catch (const GoogleError& e) {
|
||||
return e.http_status() == 400 && e.status() == "INVALID_ARGUMENT" &&
|
||||
mock->send_count() == before;
|
||||
} catch (...) {
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
Json obj(const char* k, Json v) {
|
||||
Json o = Json::object();
|
||||
o.set(k, std::move(v));
|
||||
return o;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST(models_admin, every_table_row_plans_to_a_concrete_url) {
|
||||
CallPlanner planner(project());
|
||||
CallPlanner express(VertexTarget::express());
|
||||
CallPlanner::Args args = {{"publisher", "google"}, {"model", "m1"}, {"endpoint", "11"},
|
||||
{"operation", "22"}, {"response", "33"}, {"deployed", "44"},
|
||||
{"invoke", "55"}};
|
||||
std::size_t rows = 0;
|
||||
for (const CallSpec& s : kCalls) {
|
||||
Json body = Json::object();
|
||||
std::istringstream in(s.fields);
|
||||
for (std::string f; in >> f;)
|
||||
if (f != "-" && f[0] != '?' && f.find('.') == std::string::npos) body.set(f, Json(1));
|
||||
PlannedCall p = planner.plan(s.id, args, body.as_object().empty() ? nullptr : &body);
|
||||
ASSERT_TRUE(p.url().find('{') == std::string::npos);
|
||||
ASSERT_TRUE(p.url().find("demo-proj") != std::string::npos || std::string(s.prefix) == "");
|
||||
if (s.express) {
|
||||
PlannedCall e = express.plan(s.id, args, body.as_object().empty() ? nullptr : &body);
|
||||
ASSERT_TRUE(e.url().find("projects") == std::string::npos);
|
||||
}
|
||||
++rows;
|
||||
}
|
||||
ASSERT_EQ(rows, kCallCount);
|
||||
ASSERT_TRUE(kCallCount >= 84);
|
||||
}
|
||||
|
||||
TEST(models_admin, endpoint_urls_and_beta_twin) {
|
||||
EndpointAdmin v1(client(false), project());
|
||||
EndpointAdmin b(client(false), project(), true);
|
||||
Json body = obj("displayName", Json("e"));
|
||||
ASSERT_EQ(v1.plan_call("endpoint.create", "", &body, "endpointId=e1").url(),
|
||||
kEp + "/endpoints?endpointId=e1");
|
||||
ASSERT_EQ(b.plan_call("endpoint.create", "", &body, "endpointId=e1").url(),
|
||||
kEpBeta + "/endpoints?endpointId=e1");
|
||||
ASSERT_EQ(v1.plan_call("endpoint.deploy_model", "11", nullptr).url(),
|
||||
kEp + "/endpoints/11:deployModel");
|
||||
ASSERT_EQ(v1.plan_call("endpoint.operation_wait", "11", nullptr, "timeout=5s", {{"operation", "9"}})
|
||||
.url(),
|
||||
kEp + "/endpoints/11/operations/9:wait?timeout=5s");
|
||||
ASSERT_EQ(v1.plan_call("endpoint.deployed_model_invoke", "11", nullptr, "",
|
||||
{{"deployed", "4"}, {"invoke", "predict"}})
|
||||
.url(),
|
||||
kEp + "/endpoints/11/deployedModels/4/invoke/predict");
|
||||
ASSERT_EQ(v1.plan_call("endpoint.openapi_embeddings", "").url(),
|
||||
kEp + "/endpoints/openapi/embeddings");
|
||||
// beta only rows are beta in either mode
|
||||
ASSERT_EQ(v1.plan_call("endpoint.beta_get_iam_policy", "11").url(),
|
||||
kEpBeta + "/endpoints/11:getIamPolicy");
|
||||
ASSERT_TRUE(v1.plan_call("endpoint.delete", "11").method == molo::clients::Method::kDelete);
|
||||
// the planner refuses a beta twin for a non endpoint row
|
||||
CallPlanner p(project());
|
||||
ASSERT_THROW(p.plan("gemini.generate", {{"publisher", "google"}, {"model", "m"}}, nullptr, "", true),
|
||||
GoogleError);
|
||||
}
|
||||
|
||||
TEST(models_admin, stray_body_keys_are_rejected_by_the_planner) {
|
||||
EndpointAdmin a(client(false), project());
|
||||
Json bad = obj("notAField", Json(1));
|
||||
ASSERT_THROW(a.plan_call("endpoint.update", "11", &bad), GoogleError);
|
||||
Json ok = obj("endpoint", Json::object());
|
||||
a.plan_call("endpoint.update", "11", &ok);
|
||||
// raw rows ("-") take any body
|
||||
a.plan_call("endpoint.chat_completions", "11", &bad);
|
||||
}
|
||||
|
||||
TEST(models_admin, endpoint_validation_and_confirm_gates) {
|
||||
MockTransport* mock = nullptr;
|
||||
EndpointAdmin a(client(true, &mock), project());
|
||||
Json o = Json::object();
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.destroy("11", false); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.destroy("a/b", true); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.deploy_model("11", obj("model", Json("m")), o, false); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.deploy_model("11", Json("not an object"), o, true); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.undeploy_model("11", "d1", o, false); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.undeploy_model("11", "d/1", o, true); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.mutate_deployed_model("11", o, "bad mask!", true); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.mutate_deployed_model("11", o, "minReplicaCount", false); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.patch("11", o, ""); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.patch("11", Json::array(), "displayName"); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.create(o, "bad id"); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.delete_operation("11", "5", false); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.delete_response("11", "5", false); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.set_iam_policy("11", o, false); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.wait_operation("11", "5", 0); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] {
|
||||
a.wait_operation("11", "5", ModelsCaps::kMaxOperationTimeoutSeconds + 1);
|
||||
}));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.list_operations("11", ModelsCaps::kMaxOperationPages + 1); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.get_iam_policy("11", 4); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.test_iam_permissions("11", {}); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.test_iam_permissions("11", {"a b"}); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.openapi("images", o); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.direct_predict("11", Json::array()); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.direct_raw_predict("11", "bad method", "AAAA"); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.direct_raw_predict("11", "pkg.Svc/M", "not base64!"); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.explain("11", Json::array()); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.compute_tokens("11", Json::array()); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.invoke("11", "../x", o); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.get_response("11", "a/b"); }));
|
||||
Json big = obj("endpoint", Json(std::string(ModelsCaps::kMaxAdminBodyChars + 1, 'x')));
|
||||
ASSERT_TRUE(rejected(mock, [&] { a.update("11", big); }));
|
||||
ASSERT_EQ(mock->send_count(), 0u);
|
||||
}
|
||||
|
||||
TEST(models_admin, publisher_urls_validation_and_gates) {
|
||||
MockTransport* mock = nullptr;
|
||||
PublisherAdmin p(client(true, &mock), project());
|
||||
ASSERT_EQ(p.plan_call("beta.export", {{"publisher", "google"}, {"model", "gemma3"}}).url(),
|
||||
kEpBeta + "/publishers/google/models/gemma3:export");
|
||||
ASSERT_EQ(p.plan_call("beta.enable_model", {{"publisher", "google"}, {"model", "gemma3"}}).url(),
|
||||
"https://aiplatform.googleapis.com/v1beta1/projects/demo-proj/publishers/google/models/gemma3:enableModel");
|
||||
ASSERT_EQ(p.plan_call("publisher.responses_compact", {{"publisher", "google"}}).url(),
|
||||
kEp + "/publishers/google/v1/responses/compact");
|
||||
ASSERT_EQ(p.plan_call("beta.responses_get", {{"publisher", "google"}, {"response", "r1"}}).url(),
|
||||
kEpBeta + "/publishers/google/v1/responses/r1");
|
||||
ASSERT_EQ(p.plan_call("publisher.server_streaming_predict",
|
||||
{{"publisher", "google"}, {"model", "m"}})
|
||||
.url(),
|
||||
kEp + "/publishers/google/models/m:serverStreamingPredict");
|
||||
Json o = Json::object();
|
||||
ASSERT_TRUE(rejected(mock, [&] { p.export_model("google", "m", "gs://b/x", false); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { p.export_model("google", "m", "http://b/x", true); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { p.set_config("google", "m", o, "x", false); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { p.set_config("google", "m", o, "", true); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { p.enable_model("google", "m", "svc", false); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { p.enable_model("google", "m", "bad service!", true); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { p.responses_delete("google", "r1", false); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { p.responses_get("a/b", "r1"); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { p.get_iam_policy("google", "m", 9); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { p.beta_post("beta.generate", "google", "m", o); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] { p.beta_compute_tokens("google", "m", Json::array()); }));
|
||||
ASSERT_TRUE(rejected(mock, [&] {
|
||||
p.server_streaming_predict("google", "m", Json::array(), o, nullptr);
|
||||
}));
|
||||
ASSERT_EQ(mock->send_count(), 0u);
|
||||
}
|
||||
|
||||
TEST(models_admin, unconfigured_admin_calls_refuse_and_never_send) {
|
||||
MockTransport* mock = nullptr;
|
||||
auto c = client(false, &mock);
|
||||
EndpointAdmin a(c, project());
|
||||
PublisherAdmin p(c, project());
|
||||
Json o = Json::object();
|
||||
ASSERT_THROW(a.destroy("11", true), NotConfiguredError);
|
||||
ASSERT_THROW(a.list_operations("11"), NotConfiguredError);
|
||||
ASSERT_THROW(a.operation("11", "5"), NotConfiguredError);
|
||||
ASSERT_THROW(a.openapi("completions", o), NotConfiguredError);
|
||||
ASSERT_THROW(a.get_iam_policy("11"), NotConfiguredError);
|
||||
ASSERT_THROW(p.get_iam_policy("google", "m"), NotConfiguredError);
|
||||
ASSERT_THROW(p.responses_get("google", "r1"), NotConfiguredError);
|
||||
ASSERT_THROW(p.beta_get("google", "m"), NotConfiguredError);
|
||||
ASSERT_EQ(mock->send_count(), 0u);
|
||||
}
|
||||
|
||||
// --- thought signatures ------------------------------------------------------
|
||||
|
||||
namespace {
|
||||
GenerateResponse chunk(std::vector<Part> parts) {
|
||||
GenerateResponse r;
|
||||
Candidate c;
|
||||
c.content.role = "model";
|
||||
c.content.parts = std::move(parts);
|
||||
r.candidates.push_back(std::move(c));
|
||||
return r;
|
||||
}
|
||||
Part text(const std::string& t, const std::string& sig = "", bool thought = false) {
|
||||
Part p = Part::of_text(t);
|
||||
p.thought_signature = sig;
|
||||
p.thought = thought;
|
||||
return p;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(models_signature, turn_builder_keeps_every_signature) {
|
||||
VertexModel::TurnBuilder b;
|
||||
b.add(chunk({text("Hel")}));
|
||||
b.add(chunk({text("lo ")}));
|
||||
b.add(chunk({text("world", "SIG1")})); // carries a signature
|
||||
ASSERT_EQ(b.content().parts.size(), 1u);
|
||||
ASSERT_EQ(b.content().parts[0].text, "Hello world");
|
||||
ASSERT_EQ(b.signatures(), 1u);
|
||||
// a part that has a signature is never merged into
|
||||
b.add(chunk({text(" more")}));
|
||||
ASSERT_EQ(b.content().parts.size(), 2u);
|
||||
ASSERT_EQ(b.content().parts[0].thought_signature, "SIG1");
|
||||
// a function call stays a separate part with its signature
|
||||
Part call = Part::of_call(FunctionCall{"get_weather", obj("city", Json("Paris")), ""});
|
||||
call.thought_signature = "SIG3";
|
||||
b.add(chunk({call}));
|
||||
ASSERT_EQ(b.content().parts.size(), 3u);
|
||||
ASSERT_EQ(b.content().parts[2].thought_signature, "SIG3");
|
||||
ASSERT_EQ(b.signatures(), 2u);
|
||||
// a signature that arrives alone attaches to the text before it
|
||||
VertexModel::TurnBuilder late;
|
||||
late.add(chunk({text("answer")}));
|
||||
late.add(chunk({text("", "LATE")}));
|
||||
ASSERT_EQ(late.content().parts.size(), 1u);
|
||||
ASSERT_EQ(late.content().parts[0].thought_signature, "LATE");
|
||||
// the stream ends with an empty text chunk: it must not become an empty part
|
||||
VertexModel::TurnBuilder tail;
|
||||
tail.add(chunk({call}));
|
||||
tail.add(chunk({text("")}));
|
||||
ASSERT_EQ(tail.content().parts.size(), 1u);
|
||||
// thought text and answer text are not merged
|
||||
VertexModel::TurnBuilder t;
|
||||
t.add(chunk({text("thinking", "", true)}));
|
||||
t.add(chunk({text("answer")}));
|
||||
ASSERT_EQ(t.content().parts.size(), 2u);
|
||||
}
|
||||
|
||||
TEST(models_signature, replayed_turn_sends_the_signature_back) {
|
||||
Part call = Part::of_call(FunctionCall{"get_weather", obj("city", Json("Paris")), "c1"});
|
||||
call.thought_signature = "c2ln-base64";
|
||||
GenerateResponse reply = chunk({call});
|
||||
Content turn = VertexModel::model_turn(reply);
|
||||
GenerateRequest r;
|
||||
r.contents.push_back(Content::user("weather in Paris?"));
|
||||
Part fr = Part::of_response(FunctionResponse{"get_weather", obj("temp", Json(20)), "c1"});
|
||||
Content user;
|
||||
user.role = "user";
|
||||
user.parts.push_back(fr);
|
||||
VertexModel::replay(r, turn, user);
|
||||
ASSERT_EQ(r.contents.size(), 3u);
|
||||
Gemini g(client(false), VertexTarget::express());
|
||||
PlannedCall p = g.plan_generate("gemini-3.5-flash", r);
|
||||
const Json& parts = *p.body.find("contents")->as_array()[1].find("parts");
|
||||
ASSERT_EQ(parts.as_array()[0].find("thoughtSignature")->as_string(), "c2ln-base64");
|
||||
// a signature-only part is a valid part (validate does not call it empty)
|
||||
GenerateRequest only;
|
||||
Content m;
|
||||
m.role = "model";
|
||||
Part sig;
|
||||
sig.thought_signature = "abc";
|
||||
m.parts.push_back(sig);
|
||||
only.contents.push_back(Content::user("x"));
|
||||
only.contents.push_back(m);
|
||||
g.plan_generate("gemini-3.5-flash", only);
|
||||
ASSERT_THROW(VertexModel::model_turn(GenerateResponse{}), GoogleError);
|
||||
}
|
||||
|
||||
TEST(models_media, lyria_wav_bytes_name_their_mime) {
|
||||
// "RIFF....WAVE" is what lyria-002 returned live (bytesBase64Encoded only).
|
||||
Json item = Json::object();
|
||||
item.set("bytesBase64Encoded", Json("UklGRnwAwABXQVZFZm10IBAAAAABAAIAgLsAAADuAgAEABAA"));
|
||||
Json list = Json::array();
|
||||
list.push(item);
|
||||
auto items = MediaGenerator::items_from(list);
|
||||
ASSERT_EQ(items.size(), 1u);
|
||||
ASSERT_EQ(items[0].mime, "audio/wav");
|
||||
}
|
||||
Reference in New Issue
Block a user