From a3396bfc689e7af2605690c92e95dfb704978fa7 Mon Sep 17 00:00:00 2001 From: retoor Date: Sat, 24 Oct 2026 05:43:13 +0200 Subject: [PATCH] feat(clients): sessions+registry+optimize+armor: close gaps, loud list truncation, shared Gemini adapter sessions: operations get/list/cancel/delete/wait for engine, session, memory and serving profile; servingProfiles CRUD. registry: ListResult with truncated flag, v1alpha twins, gateway/policy operations, authz IAM, deny policy update, boundary patch/bindings. optimize: make_gemini_model over models::VertexModel (DirectModel removed), generateSyntheticData, online evaluator operations, trace/log writers. armor: SecurityAdmin for all SCC v2 methods (attack exposure excluded), documented agent threat categories, floor settings fixed to location global after live run. check_discovery.py of all four enforce coverage. Co-Authored-By: Claude Sonnet 5.5 --- example/app/agent/ArmorDemo.cpp | 138 ++++++ example/app/agent/OptimizeDemo.cpp | 117 ++++- example/app/agent/RegistryDemo.cpp | 133 +++++- example/app/agent/SessionsDemo.cpp | 142 ++++++ example/assets/agent_armor.js | 10 +- example/assets/agent_optimize.js | 9 + example/assets/agent_registry.js | 18 +- src/google/armor/SPEC.md | 103 +++- src/google/armor/agent_categories.cpp | 105 +++++ src/google/armor/agent_categories.h | 39 ++ src/google/armor/calls.h | 186 ++++++-- src/google/armor/caps.h | 2 +- src/google/armor/findings.cpp | 13 +- src/google/armor/model_armor.cpp | 12 +- src/google/armor/scc_name.cpp | 57 +++ src/google/armor/scc_name.h | 47 ++ src/google/armor/security_admin.cpp | 439 ++++++++++++++++++ src/google/armor/security_admin.h | 131 ++++++ src/google/armor/tools/check_discovery.py | 30 +- src/google/optimize/SPEC.md | 66 ++- src/google/optimize/calls.h | 11 + src/google/optimize/calls_table.inc | 24 + src/google/optimize/caps.h | 5 + src/google/optimize/direct_model.cpp | 47 -- src/google/optimize/direct_model.h | 38 -- src/google/optimize/evaluation_api.cpp | 9 + src/google/optimize/evaluation_api.h | 3 + src/google/optimize/model_factory.cpp | 25 + src/google/optimize/model_factory.h | 26 ++ src/google/optimize/operations_api.cpp | 95 ++++ src/google/optimize/operations_api.h | 42 ++ src/google/optimize/optimize_service.cpp | 12 +- src/google/optimize/optimize_service.h | 9 +- src/google/optimize/telemetry_writer.cpp | 100 ++++ src/google/optimize/telemetry_writer.h | 53 +++ src/google/optimize/tools/check_discovery.py | 21 + src/google/registry/SPEC.md | 49 +- src/google/registry/agent_gateway.cpp | 20 +- src/google/registry/agent_gateway.h | 9 +- src/google/registry/agent_identity.cpp | 56 ++- src/google/registry/agent_identity.h | 22 +- src/google/registry/agent_registry.cpp | 83 ++-- src/google/registry/agent_registry.h | 37 +- src/google/registry/call_runner.cpp | 64 ++- src/google/registry/call_runner.h | 45 +- src/google/registry/calls.cpp | 42 ++ src/google/registry/gateway_policy.cpp | 44 +- src/google/registry/gateway_policy.h | 14 +- src/google/registry/tools/check_discovery.py | 32 +- src/google/sessions/SPEC.md | 77 ++- src/google/sessions/calls.h | 37 +- src/google/sessions/caps.h | 3 + src/google/sessions/engine_service.cpp | 66 +-- src/google/sessions/engine_service.h | 32 +- src/google/sessions/memory_bank.cpp | 5 + src/google/sessions/memory_bank.h | 4 + src/google/sessions/names.cpp | 37 ++ src/google/sessions/names.h | 19 + src/google/sessions/operation_names.cpp | 92 ++++ src/google/sessions/operation_names.h | 44 ++ src/google/sessions/operation_service.cpp | 123 +++++ src/google/sessions/operation_service.h | 72 +++ src/google/sessions/serving_profile.cpp | 165 +++++++ src/google/sessions/serving_profile.h | 82 ++++ src/google/sessions/sessions.cpp | 5 + src/google/sessions/sessions.h | 4 + src/google/sessions/tools/check_discovery.py | 48 +- tests/example/agent/armor/scc_test.cpp | 96 ++++ tests/example/agent/optimize/gaps_test.cpp | 71 +++ tests/example/agent/registry/iam_test.cpp | 27 ++ tests/example/agent/registry/list_test.cpp | 12 + .../agent/registry/operations_test.cpp | 39 ++ tests/example/agent/registry/page_test.cpp | 3 + tests/example/agent/registry/status_test.cpp | 1 + tests/example/agent/sessions/calls_test.cpp | 44 ++ tests/example/agent/sessions/page_test.cpp | 3 + tests/src/google/armor/test_armor.cpp | 49 +- tests/src/google/armor/test_armor_gaps.cpp | 330 +++++++++++++ tests/src/google/optimize/test_gaps.cpp | 284 +++++++++++ tests/src/google/optimize/test_live.cpp | 4 +- .../google/registry/test_registry_gaps.cpp | 299 ++++++++++++ .../google/registry/test_registry_live.cpp | 34 +- .../google/sessions/test_sessions_gaps.cpp | 353 ++++++++++++++ 83 files changed, 5020 insertions(+), 378 deletions(-) create mode 100644 src/google/armor/agent_categories.cpp create mode 100644 src/google/armor/agent_categories.h create mode 100644 src/google/armor/scc_name.cpp create mode 100644 src/google/armor/scc_name.h create mode 100644 src/google/armor/security_admin.cpp create mode 100644 src/google/armor/security_admin.h delete mode 100644 src/google/optimize/direct_model.cpp delete mode 100644 src/google/optimize/direct_model.h create mode 100644 src/google/optimize/model_factory.cpp create mode 100644 src/google/optimize/model_factory.h create mode 100644 src/google/optimize/operations_api.cpp create mode 100644 src/google/optimize/operations_api.h create mode 100644 src/google/optimize/telemetry_writer.cpp create mode 100644 src/google/optimize/telemetry_writer.h create mode 100644 src/google/sessions/operation_names.cpp create mode 100644 src/google/sessions/operation_names.h create mode 100644 src/google/sessions/operation_service.cpp create mode 100644 src/google/sessions/operation_service.h create mode 100644 src/google/sessions/serving_profile.cpp create mode 100644 src/google/sessions/serving_profile.h create mode 100644 tests/example/agent/armor/scc_test.cpp create mode 100644 tests/example/agent/optimize/gaps_test.cpp create mode 100644 tests/example/agent/registry/iam_test.cpp create mode 100644 tests/example/agent/registry/operations_test.cpp create mode 100644 tests/src/google/armor/test_armor_gaps.cpp create mode 100644 tests/src/google/optimize/test_gaps.cpp create mode 100644 tests/src/google/registry/test_registry_gaps.cpp create mode 100644 tests/src/google/sessions/test_sessions_gaps.cpp diff --git a/example/app/agent/ArmorDemo.cpp b/example/app/agent/ArmorDemo.cpp index 8069447..da39820 100644 --- a/example/app/agent/ArmorDemo.cpp +++ b/example/app/agent/ArmorDemo.cpp @@ -9,12 +9,14 @@ #include "../../fw/Templates.hpp" #include "../../fw/Validate.hpp" #include "clients/body/json.h" +#include "google/armor/agent_categories.h" #include "google/armor/armor_util.h" #include "google/armor/caps.h" #include "google/armor/dashboard.h" #include "google/armor/findings.h" #include "google/armor/guarded_model.h" #include "google/armor/model_armor.h" +#include "google/armor/security_admin.h" #include "google/credentials.h" #include "google/endpoints.h" @@ -157,6 +159,8 @@ OAuth access token, not an API key.

+ + @@ -164,6 +168,12 @@ OAuth access token, not an API key.

+ +

@@ -354,6 +364,12 @@ void register_agent_armor_demo(fw::Router& router, Services& services) {
     q.severity = field(in, "severity");
     q.state = field(in, "state");
     if (!field(in, "category").empty()) q.categories = {field(in, "category")};
+    // family selects the documented Agent Platform Threat Detection categories.
+    std::string family = field(in, "family");
+    if (family == "runtime") q.categories = armor::AgentCategories::strings(armor::AgentFamily::kRuntime);
+    else if (family == "control_plane") q.categories = armor::AgentCategories::strings(armor::AgentFamily::kControlPlane);
+    else if (family == "agent") q.categories = armor::AgentCategories::strings();
+    else if (!family.empty()) bad("family must be runtime, control_plane or agent");
     q.order_by = field(in, "order_by");
     q.page_size = int_field(in, "page_size", 0, armor::ArmorCaps::kMaxFindingsPageSize);
     q.max_pages = static_cast(int_field(in, "max_pages", 1, armor::ArmorCaps::kMaxFindingPages));
@@ -378,6 +394,8 @@ void register_agent_armor_demo(fw::Router& router, Services& services) {
       Json o = Json::object();
       o.set("findings", items);
       o.set("truncated", page.truncated);
+      o.set("count", static_cast(page.findings.size()));
+      o.set("cap_findings", static_cast(armor::ArmorCaps::kMaxFindings));
       o.set("total_size", static_cast(page.total_size));
       return json_out(o);
     });
@@ -393,6 +411,126 @@ void register_agent_armor_demo(fw::Router& router, Services& services) {
       return json_out(dash.summarize(q).to_json());
     });
   });
+
+  // ---- Documented Agent Threat Detection categories (no credential needed).
+  router.get("/agent/armor/categories", "agent-armor-categories", [](fw::Request req) {
+    (void)req;
+    Json arr = Json::array();
+    for (const armor::AgentCategory& c : armor::AgentCategories::all()) {
+      Json o = Json::object();
+      o.set("category", c.category);
+      o.set("alt", c.alt);
+      o.set("module", c.module);
+      o.set("family", c.family == armor::AgentFamily::kRuntime ? "runtime" : "control_plane");
+      arr.push(o);
+    }
+    Json o = Json::object();
+    o.set("count", static_cast(arr.as_array().size()));
+    o.set("anomaly_detection", "no documented category string; recognised by the agentAnomaly member");
+    o.set("categories", arr);
+    return json_out(o);
+  });
+
+  // ---- Security Command Center administration (SecurityAdmin).
+  auto kind_of = [](const Form& in) {
+    std::string k = field(in, "kind");
+    if (k == "muteConfigs") return armor::SccCollection::kMuteConfigs;
+    if (k == "notificationConfigs") return armor::SccCollection::kNotificationConfigs;
+    if (k == "bigQueryExports") return armor::SccCollection::kBigQueryExports;
+    bad("kind must be muteConfigs, notificationConfigs or bigQueryExports");
+  };
+
+  router.post("/agent/armor/scc/availability", "agent-armor-scc-availability", [client](fw::Request req) {
+    Form in = fw::parse_json_object_flat(req.body);
+    std::string scope = field(in, "scope");
+    if (scope.empty()) {
+      google::CloudContext ctx = google::CloudContext::from_env();
+      if (!ctx.project.empty()) scope = "projects/" + ctx.project;
+    }
+    return guarded([&] {
+      Json o = Json::object();
+      o.set("scope", scope);
+      o.set("state", armor::availability_name(armor::SecurityAdmin(client).availability(scope)));
+      return json_out(o);
+    });
+  });
+
+  router.post("/agent/armor/scc/configs/list", "agent-armor-scc-configs-list", [client, kind_of](fw::Request req) {
+    Form in = fw::parse_json_object_flat(req.body);
+    armor::SccCollection kind = kind_of(in);
+    std::string parent = field(in, "parent");
+    return guarded([&] { return json_out(armor::SecurityAdmin(client).list_configs(kind, parent).to_json()); });
+  });
+
+  router.post("/agent/armor/scc/configs/get", "agent-armor-scc-configs-get", [client, kind_of](fw::Request req) {
+    Form in = fw::parse_json_object_flat(req.body);
+    armor::SccCollection kind = kind_of(in);
+    std::string name = field(in, "name");
+    return guarded([&] { return json_out(armor::SecurityAdmin(client).get_config(kind, name)); });
+  });
+
+  // Destructive and explicit: one config, DELETE only.
+  router.del("/agent/armor/scc/configs", "agent-armor-scc-configs-delete", [client](fw::Request req) {
+    Form in = {{"kind", req.query_or("kind", "")}};
+    armor::SccCollection kind;
+    if (in["kind"] == "muteConfigs") kind = armor::SccCollection::kMuteConfigs;
+    else if (in["kind"] == "notificationConfigs") kind = armor::SccCollection::kNotificationConfigs;
+    else if (in["kind"] == "bigQueryExports") kind = armor::SccCollection::kBigQueryExports;
+    else bad("kind must be muteConfigs, notificationConfigs or bigQueryExports");
+    std::string name = req.query_or("name", "");
+    return guarded([&] {
+      armor::SecurityAdmin(client).delete_config(kind, name);
+      Json o = Json::object();
+      o.set("deleted", true);
+      return json_out(o);
+    });
+  });
+
+  router.post("/agent/armor/scc/finding/state", "agent-armor-scc-finding-state", [client](fw::Request req) {
+    Form in = fw::parse_json_object_flat(req.body);
+    std::string state = field(in, "state");
+    if (state != "ACTIVE" && state != "INACTIVE") bad("state must be ACTIVE or INACTIVE");
+    std::string finding = field(in, "finding");
+    return guarded([&] {
+      return json_out(armor::SecurityAdmin(client).set_finding_state(
+          finding, state == "ACTIVE" ? armor::FindingState::kActive : armor::FindingState::kInactive));
+    });
+  });
+
+  router.post("/agent/armor/scc/finding/mute", "agent-armor-scc-finding-mute", [client](fw::Request req) {
+    Form in = fw::parse_json_object_flat(req.body);
+    std::string mute = field(in, "mute");
+    if (mute != "MUTED" && mute != "UNMUTED" && mute != "UNDEFINED") bad("mute must be MUTED, UNMUTED or UNDEFINED");
+    std::string finding = field(in, "finding");
+    return guarded([&] {
+      return json_out(armor::SecurityAdmin(client).set_finding_mute(
+          finding, mute == "MUTED" ? armor::MuteValue::kMuted
+                   : mute == "UNMUTED" ? armor::MuteValue::kUnmuted : armor::MuteValue::kUndefined));
+    });
+  });
+
+  // Mass change: needs a filter and confirm=bulk-mute.
+  router.post("/agent/armor/scc/bulk-mute", "agent-armor-scc-bulk-mute", [client](fw::Request req) {
+    Form in = fw::parse_json_object_flat(req.body);
+    if (field(in, "confirm") != "bulk-mute") bad("confirm must be bulk-mute");
+    std::string value = field(in, "value");
+    if (value != "MUTED" && value != "UNDEFINED") bad("value must be MUTED or UNDEFINED");
+    std::string parent = field(in, "parent"), filter = field(in, "filter");
+    return guarded([&] {
+      google::Operation op = armor::SecurityAdmin(client).bulk_mute(
+          parent, filter, value == "MUTED" ? armor::BulkMuteValue::kMuted : armor::BulkMuteValue::kUndefined);
+      Json o = Json::object();
+      o.set("operation", op.name);
+      o.set("done", op.done);
+      return json_out(o);
+    });
+  });
+
+  router.post("/agent/armor/scc/operations/list", "agent-armor-scc-operations", [client](fw::Request req) {
+    Form in = fw::parse_json_object_flat(req.body);
+    std::string org = field(in, "organization");
+    return guarded([&] { return json_out(armor::SecurityAdmin(client).list_operations(org).to_json()); });
+  });
 }
 
 }  // namespace molo::example::app
diff --git a/example/app/agent/OptimizeDemo.cpp b/example/app/agent/OptimizeDemo.cpp
index 8d56412..5e9ffec 100644
--- a/example/app/agent/OptimizeDemo.cpp
+++ b/example/app/agent/OptimizeDemo.cpp
@@ -8,11 +8,13 @@
 #include "google/optimize/autorater.h"
 #include "google/optimize/calls.h"
 #include "google/optimize/clusterer.h"
-#include "google/optimize/direct_model.h"
+#include "google/optimize/model_factory.h"
 #include "google/optimize/eval_requests.h"
 #include "google/optimize/eval_resources.h"
 #include "google/optimize/evaluation_api.h"
 #include "google/optimize/observability.h"
+#include "google/optimize/operations_api.h"
+#include "google/optimize/telemetry_writer.h"
 #include "google/optimize/optimizer.h"
 #include "google/optimize/simulator.h"
 
@@ -131,7 +133,7 @@ void register_agent_optimize_demo(fw::Router& router, Services& services) {
   services.add_feature("agent optimize", "/agent/optimize");
 
   auto model_for = [gclient]() {
-    return google::optimize::DirectModel::from_env(gclient);
+    return google::optimize::make_gemini_model(gclient);
   };
 
   router.get("/agent/optimize", "agent-optimize-page", [env](fw::Request req) {
@@ -287,6 +289,117 @@ void register_agent_optimize_demo(fw::Router& router, Services& services) {
     });
   });
 
+  // generateSyntheticData: `request` is the Discovery request object (or its
+  // JSON text); top-level fields are checked against the call table.
+  router.post("/agent/optimize/synthetic", "agent-optimize-synthetic", [gclient](fw::Request req) {
+    Json body = parse_body(req);
+    const Json* r = body.find("request");
+    Json request;
+    if (r && r->is_object()) request = *r;
+    else if (r && r->is_string()) {
+      try { request = Json::parse(r->as_string()); } catch (const clients::body::JsonError&) {}
+    }
+    if (!request.is_object() || request.as_object().empty())
+      throw fw::HttpError(422, "validation_error", "request must be a non-empty JSON object");
+    google::CloudContext ctx = require_context();
+    return guarded([&] { return json_out(opt::EvaluationApi(gclient, ctx).generate_synthetic_data(request)); });
+  });
+
+  // Operations of online evaluators (get, list, server-side wait, cancel;
+  // delete is a DELETE route and removes the operation record only).
+  auto op_json = [](const google::Operation& op) {
+    Json o = Json::object();
+    o.set("name", Json(op.name));
+    o.set("done", Json(op.done));
+    if (op.failed()) o.set("error_message", Json(op.error_message));
+    return o;
+  };
+  router.post("/agent/optimize/operations/list", "agent-optimize-ops-list", [gclient, op_json](fw::Request req) {
+    Json body = parse_body(req);
+    std::string evaluator = text_field(body, "evaluator", true, 512);
+    int limit = int_field(body, "limit", 20, 1, 100);
+    google::CloudContext ctx = require_context();
+    return guarded([&] {
+      opt::OperationPage page = opt::OnlineEvaluatorOperations(gclient, ctx).list(evaluator, "", limit);
+      Json arr = Json::array();
+      for (const google::Operation& op : page.operations) arr.push(op_json(op));
+      Json out = Json::object();
+      out.set("operations", arr);
+      out.set("count", Json(static_cast(page.operations.size())));
+      out.set("truncated", Json(page.truncated));
+      out.set("next_page_token", Json(page.next_page_token));
+      return json_out(out);
+    });
+  });
+  router.post("/agent/optimize/operations/get", "agent-optimize-ops-get", [gclient](fw::Request req) {
+    Json body = parse_body(req);
+    std::string name = text_field(body, "name", true, 512);
+    google::CloudContext ctx = require_context();
+    return guarded([&] { return json_out(opt::OnlineEvaluatorOperations(gclient, ctx).get(name)); });
+  });
+  router.post("/agent/optimize/operations/wait", "agent-optimize-ops-wait", [gclient, op_json](fw::Request req) {
+    Json body = parse_body(req);
+    std::string name = text_field(body, "name", true, 512);
+    int seconds = int_field(body, "timeout_seconds", 30, 1, opt::OptimizeCaps::kMaxServerWaitSeconds);
+    google::CloudContext ctx = require_context();
+    return guarded([&] {
+      return json_out(op_json(opt::OnlineEvaluatorOperations(gclient, ctx).wait_server(name, seconds)));
+    });
+  });
+  router.post("/agent/optimize/operations/cancel", "agent-optimize-ops-cancel", [gclient](fw::Request req) {
+    Json body = parse_body(req);
+    std::string name = text_field(body, "name", true, 512);
+    google::CloudContext ctx = require_context();
+    return guarded([&] {
+      opt::OnlineEvaluatorOperations(gclient, ctx).cancel(name);
+      Json out = Json::object();
+      out.set("cancel_requested", Json(true));
+      return json_out(out);
+    });
+  });
+  router.del("/agent/optimize/operations", "agent-optimize-ops-delete", [gclient](fw::Request req) {
+    std::string name = req.query_or("name", "");
+    if (name.empty() || !opt::OnlineEvaluatorOperations::is_operation_name(name))
+      throw fw::HttpError(422, "validation_error", "name must be an online evaluator operation name");
+    google::CloudContext ctx = require_context();
+    return guarded([&] {
+      opt::OnlineEvaluatorOperations(gclient, ctx).remove(name);
+      Json out = Json::object();
+      out.set("deleted", Json(true));
+      return json_out(out);
+    });
+  });
+
+  // entries:write. A dry run (nothing stored) unless confirm is "write".
+  router.post("/agent/optimize/logs/write", "agent-optimize-logs-write", [gclient](fw::Request req) {
+    Json body = parse_body(req);
+    std::string log_id = text_field(body, "log_id", true, 100);
+    std::string text = text_field(body, "text", true, 4000);
+    std::string confirm = text_field(body, "confirm", false, 16);
+    if (!google::valid_segment(log_id))
+      throw fw::HttpError(422, "validation_error", "log_id has invalid characters");
+    google::CloudContext ctx = require_context();
+    bool real = confirm == "write";
+    Json entry = Json::object();
+    entry.set("textPayload", Json(text));
+    Json entries = Json::array();
+    entries.push(entry);
+    Json resource = Json::object();
+    resource.set("type", Json("global"));
+    Json request = Json::object();
+    request.set("logName", Json("projects/" + ctx.project + "/logs/" + log_id));
+    request.set("resource", resource);
+    request.set("entries", entries);
+    request.set("dryRun", Json(!real));
+    return guarded([&] {
+      opt::LogWriter(gclient).write(request);
+      Json out = Json::object();
+      out.set("dry_run", Json(!real));
+      out.set("written", Json(real));
+      return json_out(out);
+    });
+  });
+
   // Local autorater over a pasted transcript ("user: ..." / "agent: ...").
   router.post("/agent/optimize/score", "agent-optimize-score", [model_for](fw::Request req) {
     Json body = parse_body(req);
diff --git a/example/app/agent/RegistryDemo.cpp b/example/app/agent/RegistryDemo.cpp
index 0d35d25..669a120 100644
--- a/example/app/agent/RegistryDemo.cpp
+++ b/example/app/agent/RegistryDemo.cpp
@@ -53,6 +53,19 @@ fw::Response json_reply(const Json& j, int status = 200) {
   return res;
 }
 
+// A list answer. Truncation is part of the answer, never hidden: `truncated`
+// is true when more existed past the stated cap, `cap_items` names the cap.
+Json list_answer(const reg::ListResult& list, const char* items_key) {
+  Json j = Json::object();
+  j.set("count", Json(static_cast(list.size())));
+  j.set("pages", Json(static_cast(list.pages)));
+  j.set("truncated", Json(list.truncated));
+  j.set("next_page_token", Json(list.next_page_token));
+  j.set("cap_items", Json(static_cast(reg::RegistryCaps::kMaxItems)));
+  j.set(items_key, list.items);
+  return j;
+}
+
 std::string field(std::map& in, const char* key) {
   auto it = in.find(key);
   return it == in.end() ? "" : it->second;
@@ -87,7 +100,9 @@ class RegistryDemo {
   void require_credential() const {
     if (!configured()) throw google::NotConfiguredError("registry");
   }
-  reg::AgentRegistry registry() const { return reg::AgentRegistry(client_, ready()); }
+  reg::AgentRegistry registry(reg::RegistryVersion v = reg::RegistryVersion::kV1) const {
+    return reg::AgentRegistry(client_, ready(), v);
+  }
   reg::AgentGateway gateway() const { return reg::AgentGateway(client_, ready()); }
   reg::GatewayPolicies policies() const { return reg::GatewayPolicies(client_, ready()); }
   reg::AgentIdentity identity() const { return reg::AgentIdentity(client_, ready()); }
@@ -122,6 +137,11 @@ here is a local list: the registry API has no approve method.

+
@@ -133,10 +153,14 @@ here is a local list: the registry API has no approve method.

+ + + +

 
@@ -169,6 +193,7 @@ void register_agent_registry_demo(fw::Router& router, Services& services) {
     Json caps = Json::object();
     caps.set("page_size", Json(reg::RegistryCaps::kPageSize));
     caps.set("max_pages", Json(static_cast(reg::RegistryCaps::kMaxPages)));
+    caps.set("max_items", Json(static_cast(reg::RegistryCaps::kMaxItems)));
     caps.set("max_ledger_entries", Json(static_cast(reg::RegistryCaps::kMaxLedgerEntries)));
     caps.set("max_audit_entries", Json(static_cast(reg::RegistryCaps::kMaxAuditEntries)));
     j.set("caps", caps);
@@ -259,15 +284,17 @@ void register_agent_registry_demo(fw::Router& router, Services& services) {
                                    reg::AssetKind::kSkills, reg::AssetKind::kBindings,
                                    reg::AssetKind::kPublishers});
     bool approved_only = field(in, "approved_only") == "true";
+    std::string version = field(in, "version");
+    if (!version.empty() && version != "v1" && version != "v1alpha") bad("version must be v1 or v1alpha");
     return guarded([&] {
-      reg::AgentRegistry registry = demo->registry();
-      Json items = approved_only ? registry.list_approved(kind, demo->ledger())
-                                 : registry.list(kind);
-      Json j = Json::object();
+      reg::AgentRegistry registry = demo->registry(version == "v1alpha" ? reg::RegistryVersion::kV1Alpha
+                                                                        : reg::RegistryVersion::kV1);
+      reg::ListResult list = approved_only ? registry.list_approved(kind, demo->ledger())
+                                           : registry.list(kind);
+      Json j = list_answer(list, "items");
       j.set("kind", Json(reg::asset_collection(kind)));
+      j.set("version", Json(registry.version()));
       j.set("approved_only", Json(approved_only));
-      j.set("count", Json(static_cast(items.as_array().size())));
-      j.set("items", items);
       return json_reply(j);
     });
   });
@@ -377,11 +404,7 @@ void register_agent_registry_demo(fw::Router& router, Services& services) {
   // ---- Gateways and policies (reads, plus an explicit delete).
   router.post("/agent/registry/gateways/list", "agent-registry-gateways", [demo](fw::Request) {
     return guarded([&] {
-      Json items = demo->gateway().list();
-      Json j = Json::object();
-      j.set("count", Json(static_cast(items.as_array().size())));
-      j.set("gateways", items);
-      return json_reply(j);
+      return json_reply(list_answer(demo->gateway().list(), "gateways"));
     });
   });
 
@@ -400,14 +423,92 @@ void register_agent_registry_demo(fw::Router& router, Services& services) {
 
   router.post("/agent/registry/policies/list", "agent-registry-policies", [demo](fw::Request) {
     return guarded([&] {
-      Json items = demo->policies().list();
-      Json j = Json::object();
-      j.set("count", Json(static_cast(items.as_array().size())));
-      j.set("policies", items);
+      return json_reply(list_answer(demo->policies().list(), "policies"));
+    });
+  });
+
+  // ---- Operations of the three registry-side hosts. Reads are POST (they
+  // call out); delete is DELETE and explicit (the operation record only).
+  auto scope_of = [](std::map& in) {
+    std::string s = field(in, "scope");
+    if (s != "registry" && s != "gateways" && s != "policies")
+      bad("scope must be registry, gateways or policies");
+    return s;
+  };
+  router.post("/agent/registry/operations/list", "agent-registry-operations", [demo, scope_of](fw::Request req) {
+    auto in = fw::parse_json_object_flat(req.body);
+    std::string scope = scope_of(in);
+    return guarded([&] {
+      reg::ListResult ops = scope == "registry" ? demo->registry().operations()
+                            : scope == "gateways" ? demo->gateway().operations()
+                                                  : demo->policies().operations();
+      Json j = list_answer(ops, "operations");
+      j.set("scope", Json(scope));
       return json_reply(j);
     });
   });
 
+  router.post("/agent/registry/operations/cancel", "agent-registry-operation-cancel",
+              [demo, scope_of](fw::Request req) {
+    auto in = fw::parse_json_object_flat(req.body);
+    std::string scope = scope_of(in);
+    std::string id = field(in, "id");
+    if (!google::valid_segment(id)) bad("id is empty or has invalid characters");
+    return guarded([&] {
+      Json r = scope == "registry" ? demo->registry().cancel_operation(id)
+               : scope == "gateways" ? demo->gateway().cancel_operation(id)
+                                     : demo->policies().cancel_operation(id);
+      Json j = Json::object();
+      j.set("scope", Json(scope));
+      j.set("cancel_requested", Json(true));
+      j.set("reply", r);
+      return json_reply(j);
+    });
+  });
+
+  router.del("/agent/registry/operations", "agent-registry-operation-delete", [demo](fw::Request req) {
+    std::string scope = req.query_or("scope", "");
+    std::string id = req.query_or("id", "");
+    if (scope != "registry" && scope != "gateways" && scope != "policies")
+      bad("scope must be registry, gateways or policies");
+    if (!google::valid_segment(id)) bad("id is empty or has invalid characters");
+    return guarded([&] {
+      Json r = scope == "registry" ? demo->registry().delete_operation(id)
+               : scope == "gateways" ? demo->gateway().delete_operation(id)
+                                     : demo->policies().delete_operation(id);
+      Json j = Json::object();
+      j.set("deleted", Json(true));
+      j.set("reply", r);
+      return json_reply(j);
+    });
+  });
+
+  // ---- IAM deny policies (iam v2) and boundary bindings (iam v3).
+  router.post("/agent/registry/deny-policies/list", "agent-registry-deny-policies", [demo](fw::Request req) {
+    auto in = fw::parse_json_object_flat(req.body);
+    std::string number = field(in, "project_number");
+    if (number.empty() || number.size() > 20 || number.find_first_not_of("0123456789") != std::string::npos)
+      bad("project_number must be numeric");
+    return guarded([&] {
+      return json_reply(list_answer(
+          demo->identity().deny_policies(reg::AgentIdentity::attachment_for_project(number)),
+          "policies"));
+    });
+  });
+
+  router.post("/agent/registry/boundaries/bindings", "agent-registry-boundary-bindings",
+              [demo](fw::Request req) {
+    auto in = fw::parse_json_object_flat(req.body);
+    std::string org = field(in, "org_id");
+    std::string id = field(in, "id");
+    if (org.empty() || org.size() > 20 || org.find_first_not_of("0123456789") != std::string::npos)
+      bad("org_id must be numeric");
+    if (!google::valid_segment(id)) bad("id is empty or has invalid characters");
+    return guarded([&] {
+      return json_reply(list_answer(demo->identity().boundary_bindings(org, id), "bindings"));
+    });
+  });
+
   // ---- Identity of one deployed agent (engine id).
   router.post("/agent/registry/identity", "agent-registry-identity", [demo](fw::Request req) {
     auto in = fw::parse_json_object_flat(req.body);
diff --git a/example/app/agent/SessionsDemo.cpp b/example/app/agent/SessionsDemo.cpp
index bf81883..68773aa 100644
--- a/example/app/agent/SessionsDemo.cpp
+++ b/example/app/agent/SessionsDemo.cpp
@@ -9,6 +9,7 @@
 #include "google/sessions/caps.h"
 #include "google/sessions/calls.h"
 #include "google/sessions/memory_bank.h"
+#include "google/sessions/serving_profile.h"
 #include "google/sessions/sessions.h"
 
 namespace molo::example::app {
@@ -52,6 +53,15 @@ Every call below is a JSON POST (or DELETE) and answers 503
 DELETE /agent/sessions/memories/memoryquery: engine, id
 POST /agent/sessions/memories/profilesengine, scope_key, scope_value
 POST /agent/sessions/memories/purgeengine, filter, force?, confirm? (dry run by default)
+POST /agent/sessions/operations/listengine, parent? (engine, session or memory name), filter?
+POST /agent/sessions/operations/getengine, name
+POST /agent/sessions/operations/waitengine, name, timeout_seconds (server-side wait, 1..60)
+POST /agent/sessions/operations/cancelengine, name
+DELETE /agent/sessions/operations/operationquery: engine, name
+POST /agent/sessions/serving-profiles/listlocation? (id or projects/P/locations/L)
+POST /agent/sessions/serving-profiles/getlocation?, profile_id
+POST /agent/sessions/serving-profiles/createlocation?, profile_id, display_name, scope, description?
+DELETE /agent/sessions/serving-profiles/profilequery: location?, id
 
 
 
@@ -106,6 +116,29 @@ class SessionsRoutes { return gs::MemoryBank(client_, engine_of(engine)); } + // A full "projects/P/locations/L" name, or a location id (project from + // GOOGLE_CLOUD_PROJECT, empty: GOOGLE_CLOUD_LOCATION or us-central1). + gs::LocationName location_of(const std::string& value) const { + if (value.rfind("projects/", 0) == 0) return gs::LocationName(value); + google::CloudContext ctx = google::CloudContext::from_env(); + if (!value.empty()) ctx.location = value; + return gs::LocationName(ctx.parent()); + } + gs::ServingProfiles profiles(const std::string& location) const { + return gs::ServingProfiles(client_, location_of(location)); + } + + static GJson operation_json(const google::Operation& op) { + GJson o = GJson::object(); + o.set("name", GJson(op.name)); + o.set("done", GJson(op.done)); + if (op.failed()) { + o.set("error_code", GJson(op.error_code)); + o.set("error_message", GJson(op.error_message)); + } + return o; + } + static gs::StringMap scope_of(const Flat& in) { gs::StringMap scope; if (!field(in, "scope_key").empty() || !field(in, "scope_value").empty()) @@ -147,6 +180,8 @@ void register_agent_sessions_demo(fw::Router& router, Services& services) { caps.set("max_session_id_chars", GJson(static_cast(gs::SessionsCaps::kMaxIdChars))); caps.set("max_list_items", GJson(static_cast(gs::SessionsCaps::kMaxListItems))); caps.set("max_top_k", GJson(gs::MemoryCaps::kMaxTopK)); + caps.set("max_server_wait_seconds", + GJson(static_cast(gs::SessionsCaps::kMaxServerWaitSeconds))); caps.set("max_fact_chars", GJson(static_cast(gs::MemoryCaps::kMaxFactChars))); out.set("caps", caps); return answer(out); @@ -355,6 +390,113 @@ void register_agent_sessions_demo(fw::Router& router, Services& services) { return answer(out); }); }); + + // Operations (engine, session, memory) ----------------------------------- + + router.post("/agent/sessions/operations/list", "agent-sessions-operations-list", + [routes](fw::Request req) { + Flat in = fw::parse_json_object_flat(req.body); + return guarded([&] { + gs::OperationListOptions options; + options.filter = field(in, "filter"); + GJson arr = GJson::array(); + for (const google::Operation& op : + routes->sessions(field(in, "engine")).list_operations(field(in, "parent"), options)) + arr.push(SessionsRoutes::operation_json(op)); + GJson out = GJson::object(); + out.set("operations", arr); + return answer(out); + }); + }); + + router.post("/agent/sessions/operations/get", "agent-sessions-operations-get", + [routes](fw::Request req) { + Flat in = fw::parse_json_object_flat(req.body); + return guarded([&] { + return answer(routes->sessions(field(in, "engine")).operation(field(in, "name"))); + }); + }); + + // One server-side wait (POST :wait); done=false when the timeout passed first. + router.post("/agent/sessions/operations/wait", "agent-sessions-operations-wait", + [routes](fw::Request req) { + Flat in = fw::parse_json_object_flat(req.body); + return guarded([&] { + int seconds = field(in, "timeout_seconds").empty() + ? 30 + : std::atoi(field(in, "timeout_seconds").c_str()); + return answer(SessionsRoutes::operation_json( + routes->sessions(field(in, "engine")).wait_server(field(in, "name"), seconds))); + }); + }); + + router.post("/agent/sessions/operations/cancel", "agent-sessions-operations-cancel", + [routes](fw::Request req) { + Flat in = fw::parse_json_object_flat(req.body); + return guarded([&] { + routes->sessions(field(in, "engine")).cancel_operation(field(in, "name")); + GJson out = GJson::object(); + out.set("cancel_requested", GJson(true)); + return answer(out); + }); + }); + + router.del("/agent/sessions/operations/operation", "agent-sessions-operations-delete", + [routes](fw::Request req) { + return guarded([&] { + routes->sessions(req.query_or("engine", "")).delete_operation(req.query_or("name", "")); + GJson out = GJson::object(); + out.set("deleted", GJson(true)); + return answer(out); + }); + }); + + // Serving profiles (location level) --------------------------------------- + + router.post("/agent/sessions/serving-profiles/list", "agent-serving-profiles-list", + [routes](fw::Request req) { + Flat in = fw::parse_json_object_flat(req.body); + return guarded([&] { + GJson arr = GJson::array(); + for (const gs::ServingProfile& p : routes->profiles(field(in, "location")).list()) + arr.push(p.to_json()); + GJson out = GJson::object(); + out.set("serving_profiles", arr); + return answer(out); + }); + }); + + router.post("/agent/sessions/serving-profiles/get", "agent-serving-profiles-get", + [routes](fw::Request req) { + Flat in = fw::parse_json_object_flat(req.body); + return guarded([&] { + return answer( + routes->profiles(field(in, "location")).get(field(in, "profile_id")).to_json()); + }); + }); + + router.post("/agent/sessions/serving-profiles/create", "agent-serving-profiles-create", + [routes](fw::Request req) { + Flat in = fw::parse_json_object_flat(req.body); + return guarded([&] { + gs::ServingProfileSpec spec; + spec.profile_id = field(in, "profile_id"); + spec.display_name = field(in, "display_name"); + spec.description = field(in, "description"); + spec.scope = gs::serving_scope_from(field(in, "scope")); + return answer(routes->profiles(field(in, "location")).create(spec).to_json(), 201); + }); + }); + + router.del("/agent/sessions/serving-profiles/profile", "agent-serving-profiles-delete", + [routes](fw::Request req) { + return guarded([&] { + routes->profiles(req.query_or("location", "")).remove(req.query_or("id", "")); + GJson out = GJson::object(); + out.set("deleted", GJson(true)); + return answer(out); + }); + }); } } // namespace molo::example::app diff --git a/example/assets/agent_armor.js b/example/assets/agent_armor.js index ba9b2ad..4ebc088 100644 --- a/example/assets/agent_armor.js +++ b/example/assets/agent_armor.js @@ -16,13 +16,15 @@ "templates/create": ["a-template"], "templates/delete": ["a-template"], "floor/get": ["a-scope"], - "findings": ["a-scope", "a-severity"], - "dashboard": ["a-scope", "a-severity"] + "findings": ["a-scope", "a-severity", "a-family"], + "dashboard": ["a-scope", "a-severity", "a-family"], + "scc/availability": ["a-scope"], + "scc/configs/list": ["a-scope", "a-kind"] }; function sync() { var shown = uses[action.value] || []; - ["a-template", "a-text", "a-scope", "a-severity"].forEach(function (id) { + ["a-template", "a-text", "a-scope", "a-severity", "a-family", "a-kind"].forEach(function (id) { $(id).parentNode.style.display = shown.indexOf(id) >= 0 ? "" : "none"; }); } @@ -34,6 +36,8 @@ if (text.value) b.text = text.value; if (scope.value) b.scope = scope.value; if (severity.value) b.severity = severity.value; + if ($("a-family").value) b.family = $("a-family").value; + if (a === "scc/configs/list") { b.kind = $("a-kind").value; b.parent = scope.value; delete b.scope; } if (a === "templates/create") { b.pi = "MEDIUM_AND_ABOVE"; b.malicious_uri = "true"; diff --git a/example/assets/agent_optimize.js b/example/assets/agent_optimize.js index 51b8865..de61b5f 100644 --- a/example/assets/agent_optimize.js +++ b/example/assets/agent_optimize.js @@ -17,6 +17,15 @@ fields: [["agent_instruction", "area"], ["instruction", "text"], ["count", "text:5"]] }, runs: { help: "List evaluation runs (needs OAuth).", fields: [["limit", "text:10"]] }, traces: { help: "Cloud Trace summary (needs OAuth).", fields: [["limit", "text:20"], ["filter", "text"]] }, + synthetic: { help: "generateSyntheticData: paste the request JSON (needs OAuth).", + fields: [["request", "area:{ ... }"]] }, + "operations/list": { help: "Operations of an online evaluator (needs OAuth).", + fields: [["evaluator", "text:projects/P/locations/L/onlineEvaluators/ID"], ["limit", "text:20"]] }, + "operations/get": { help: "One online evaluator operation.", fields: [["name", "text"]] }, + "operations/wait": { help: "Server-side wait, 1..60 seconds.", + fields: [["name", "text"], ["timeout_seconds", "text:30"]] }, + "logs/write": { help: "Write one log entry. Dry run unless confirm is write.", + fields: [["log_id", "text:agent-check"], ["text", "text"], ["confirm", "text"]] }, logs: { help: "Cloud Logging entries (needs OAuth).", fields: [["limit", "text:20"], ["trace_id", "text"], ["filter", "text"]] } }; var $ = function (id) { return document.getElementById(id); }; diff --git a/example/assets/agent_registry.js b/example/assets/agent_registry.js index eaa0464..d3fa237 100644 --- a/example/assets/agent_registry.js +++ b/example/assets/agent_registry.js @@ -28,13 +28,25 @@ function get(path) { return fetch(path).then(function (res) { return res.json(); }); } + // A list that hit its cap says so loudly; it is never presented as complete. + function banner(j) { + var el = $("r-truncated"); + if (j && j.truncated === true) { + el.textContent = "TRUNCATED: showing " + j.count + " of more than " + j.cap_items + + " items (" + j.pages + " pages). Narrow the query or continue from the page token."; + el.hidden = false; + } else { + el.hidden = true; + el.textContent = ""; + } + } function run(promise) { - promise.then(function (j) { show(JSON.stringify(j, null, 2), false); }) + promise.then(function (j) { banner(j); show(JSON.stringify(j, null, 2), false); }) .catch(function (e) { show(e.message, true); }); } $("r-list").addEventListener("click", function () { - run(post("/agent/registry/list", { kind: $("r-kind").value })); + run(post("/agent/registry/list", { kind: $("r-kind").value, version: $("r-version").value })); }); $("r-search").addEventListener("click", function () { run(post("/agent/registry/search", { kind: $("r-kind").value, q: $("r-q").value })); @@ -47,6 +59,8 @@ if (which === "gateways") run(post("/agent/registry/gateways/list")); else if (which === "policies") run(post("/agent/registry/policies/list")); else if (which === "identity") run(post("/agent/registry/identity", { engine_id: $("r-q").value })); + else if (which.indexOf("ops-") === 0) + run(post("/agent/registry/operations/list", { scope: which.substring(4) })); else if (which === "approvals") run(get("/agent/registry/approvals")); else run(get("/agent/registry/audit")); }); diff --git a/src/google/armor/SPEC.md b/src/google/armor/SPEC.md index 2c2958c..de216ca 100644 --- a/src/google/armor/SPEC.md +++ b/src/google/armor/SPEC.md @@ -19,6 +19,19 @@ module. Finding resource carries typed `agent`, `agentAnomaly`, `agentSessions`, `agentDataAccessEvents`, `aiModel` and `vulnerability` members, which are read into `Finding`. +- Security Command Center administration, `SecurityAdmin` (every v2 method + except the attack exposure engine): mute configs, notification configs and + BigQuery exports (list, get, create, patch, delete), finding state, mute, + patch, security marks, create, external systems, bulk mute, findings export, + sources create/patch and source IAM, asset security marks, organization + operations (get, list, cancel, delete). Attack paths, valued resources, + simulations and resource value configs are excluded on purpose: they are + cloud posture risk scoring, not agent security (`EXCLUDED_RE` in + `tools/check_discovery.py`). +- Agent Threat Detection categories: `AgentCategories` holds the documented + category strings (runtime detectors and control plane detectors) with module + names. Agent Anomaly Detection has no documented category string: its + findings are the ones with the typed `agentAnomaly` member. - Agent Security dashboard: a console product with no dashboard API. `AgentSecurityDashboard` is a LOCAL abstraction that aggregates listed findings (counts by severity, category, class, state, per-asset worst @@ -40,7 +53,10 @@ env: `GOOGLE_CLOUD_PROJECT`, optional `GOOGLE_CLOUD_LOCATION`. | File | Concern | |---|---| | `caps.h` | Static ceilings (`ArmorCaps`) | -| `calls.h/.cpp` | The ONE table of declared calls, schemas, enums; path matcher | +| `calls.h/.cpp` | The ONE table of declared calls (144 rows: rpc, verb, path, query, request schema, allowed body fields), schemas, enums; path matcher | +| `scc_name.h/.cpp` | `SccName`: a validated SCC resource name; derives the table row (scope, `locations`, `sources`) from the name | +| `security_admin.h/.cpp` | `SecurityAdmin : Service`: every SCC v2 administration method, `SccList`, `SccAvailability` probe | +| `agent_categories.h/.cpp` | `AgentCategories`: documented Agent Platform Threat Detection categories and module names | | `tools/check_discovery.py` | Fails when `calls.h` and Google's Discovery documents disagree | | `armor_types.h/.cpp` | `SanitizationResult`, `FilterOutcome`, enums | | `template.h/.cpp` | `Template`, `FilterConfig`, `TemplateMetadata`, `FloorSetting`, rule sets | @@ -62,8 +78,8 @@ env: `GOOGLE_CLOUD_PROJECT`, optional `GOOGLE_CLOUD_LOCATION`. | Labels per template | 64 (63 chars each) | | Update mask paths | 16 | | Detail strings per filter result | 50 | -| Findings page size (Google max) / pages / results | 1000 / 10 / 5000 | -| Categories in a filter | 20 | +| Findings page size (Google max) / pages / results | 1000 / 10 / 5000; config and operation lists use 100 / 10 / 5000 and report `truncated` | +| Categories in a filter | 100 (all documented agent categories fit) | | Dashboard distinct assets | 500 | ## 4. Contracts @@ -77,8 +93,19 @@ env: `GOOGLE_CLOUD_PROJECT`, optional `GOOGLE_CLOUD_LOCATION`. `global` (`https://modelarmor.googleapis.com/v1`); templates and sanitize reject `global`. Security Command Center: `https://securitycenter.googleapis.com/v2`. - Destructive: `delete_template` is an explicit method; the demo route also - requires `confirm=delete`. Nothing here creates, updates or deletes - findings, sources or any Security Command Center resource. + requires `confirm=delete`. `SecurityAdmin` mutations are explicit methods + (`delete_config`, `bulk_mute`, `set_finding_state`, ...), each checked + against its table row (query parameters, top-level body fields, update mask + roots) before traffic. `bulk_mute` refuses an empty filter and only mutes or + un-mutes (`MUTED`, `UNDEFINED`); its demo route needs `confirm=bulk-mute`. + `cancel_operation` / `delete_operation` read the operation first. Floor + settings are read and written at location `global` on the global host only + (found live, see 6). +- Lists never truncate silently: findings (`FindingPage::truncated`), config + and operation lists (`SccList::truncated` with `next_page_token`). +- Not activated is a state, not a crash: `SecurityAdmin::availability` maps + 403 API disabled, 404 on `sources` and 400 `FAILED_PRECONDITION` (SCC enabled + but not activated for the project) to `SccAvailability`. - `retry ids`: `requestId` is accepted on create, patch and delete as a non-zero UUID. - Sanitization results are typed; an unknown or missing `invocationResult` @@ -97,13 +124,13 @@ env: `GOOGLE_CLOUD_PROJECT`, optional `GOOGLE_CLOUD_LOCATION`. ## 5. Verified against discovery -`python3 -I src/google/armor/tools/check_discovery.py` fetches +`python3 -I src/google/armor/tools/check_discovery.py` (coverage: every method of both Discovery documents is a table row except the excluded attack exposure engine) fetches `https://modelarmor.googleapis.com/$discovery/rest?version=v1` and `https://securitycenter.googleapis.com/$discovery/rest?version=v2` and checks -every row of `calls.h`: 31 calls (rpc id, HTTP method, path, query -parameters), 66 schemas (every field this module reads or writes) and 13 enum -value sets. Result on 2026-10-24: `OK: 31 calls, 66 schemas, 13 enums -checked, 0 problems`. Unit tests additionally prove that every request the +every row of `calls.h`: 144 calls (rpc id, HTTP method, path, query +parameters, request schema, request fields), 71 schemas (every field this +module reads or writes) and 17 enum value sets. Result on 2026-10-24: `OK: +144 calls, 71 schemas, 17 enums checked, 0 problems`. Unit tests additionally prove that every request the client puts on the wire matches a declared call and only uses declared query parameters, and that every key of every request body built here is a field of its Discovery schema (nested schemas included). @@ -122,6 +149,34 @@ and messages, `filterVersionSelector`, `modalities`, `dataResidencyCompliant`; Date 2026-10-24, project from `GOOGLE_CLOUD_PROJECT`, no resource created, updated or deleted. +**Re-run after the owner enabled Model Armor and Security Command Center +(service-account token, read-only):** + +| Call | Result | +|---|---| +| Model Armor `GET https://modelarmor.us-central1.rep.googleapis.com/v1/projects/P/locations/us-central1/templates` | 200 `{}` (no template exists) | +| `GET https://modelarmor.googleapis.com/v1/projects/P/locations/global/floorSetting` | 200 `{name, createTime, updateTime}` (no filterConfig: enforcement off) | +| `GET .../locations/us-central1/floorSetting` on the regional host | 400 INVALID_ARGUMENT "Malformed name" (floor settings exist only at `global`) | +| `GET https://modelarmor.googleapis.com/v1/projects/P/locations?pageSize=5` | 200, a list of locations (`name`, `locationId`) | +| `GET https://modelarmor.us-central1.rep.googleapis.com/v1/projects/P/locations` | 403 "Read access to project denied" (list is global-host only) | +| `GET .../locations/us-central1` on the regional host / `.../locations/global` on the global host | 200 `{name, locationId}` | +| `GET .../locations/us-central1` on the global host | 403 | +| `sanitizeUserPrompt` against an absent template id | 400 INVALID_ARGUMENT "The template named '...' was not found" (request shape accepted; absent template is 400, not 404) | +| SCC v2 `GET projects/P/sources`, `sources/-/findings`, `sources/-/locations/global/findings` (also v1 sources) | 404 NOT_FOUND "Requested entity was not found." | +| SCC v2 `GET projects/P/muteConfigs`, `.../locations/global/muteConfigs`, `.../locations/global/notificationConfigs`, `.../locations/global/bigQueryExports` | 400 FAILED_PRECONDITION "Precondition check failed." | +| Cloud Resource Manager `GET v3/projects/P` | 403 API not enabled (so the project's organization could not be read) | + +Reading: the API is enabled but Security Command Center is not activated for +this project (no organization or SCC tier reachable): sources do not resolve +and every config call fails its precondition. Findings, groups, mute configs, +notification configs, exports and every `SecurityAdmin` happy path are +therefore NOT seen live; they stay Discovery-verified. `SecurityAdmin::availability` +reports `not_activated` for exactly this state, and the opt-in test +`google_live_armor.oauth_token_reads_templates_and_findings` prints it. +Model Armor code fixed from these runs: floor settings now use location +`global` on the global host, `list_locations` uses the global host, +`get_location` the location's own host. + | Call | Credential | Result | |---|---|---| | Model Armor `templates.list` | express key (`AQ.`) | 401 UNAUTHENTICATED (API keys are not supported); `google_live_armor.express_key_is_refused` | @@ -155,8 +210,25 @@ disabled API loudly. Run it once the APIs are enabled. Nothing neither Discovery nor docs state: -- Finding `category` strings for Agent Threat Detection and Agent Anomaly - Detection: not read; callers pass categories they see in the console. +- Agent Anomaly Detection category strings: neither the anomaly overview page + nor the threat pages name one. Settled by documentation reading + (2026-10-24): Agent Platform Threat Detection (runtime, Preview) and the AI + agent control plane detectors publish the category strings in + `agent_categories.cpp` (module names `AGENT_ENGINE_*`); anomaly findings are + recognised by the typed `agentAnomaly` member, not by category. The pages + disagree on a few spellings (CloudSQL / Cloud SQL, "Evidence of Port + Scanning from AI Agent" / "AI Agent Evidence of Port Scanning", "Remove Bulk + Data From Disk" / "from Disk", "cryptocurrency" / "crypto", "Abuse of Sudo" / + "Attempt to Abuse Sudo"): both are listed and filtered. No finding was ever + seen live, so which spelling Google emits is unverified. +- `SecurityAdmin` response bodies (configs, operations, marks) are returned as + raw Json and were never seen; request fields are checked against Discovery + only. Notification configs and BigQuery exports exist only under + `locations` in v2 (Discovery), mute configs with and without. +- Whether `findings:bulkMute` accepts `UNMUTED` (Discovery's enum for + `BulkMuteFindingsRequest.muteState` is `MUTED`, `UNDEFINED`): only those are sent. +- Sanitize against a floor setting without a template: not possible (a + template name is required); no default template exists. - That agent assets always contain `reasoningEngines` in `resourceName` (dashboard default filter; override with `resource_contains`). - Server-side behavior of the `resourceName:"x"` substring operator (AIP-160 @@ -178,6 +250,9 @@ Nothing neither Discovery nor docs state: - Streaming sanitize (`streamingMode`, gRPC `StreamSanitize*`) is not implemented; `GuardedModel` buffers instead. - No asset inventory (Cloud Asset Inventory is a different API). -- Findings mutation (mute, state), notification configs, BigQuery exports, - mute configs and attack paths are out of scope (read-only module). +- Attack exposure engine (attack paths, valued resources, simulations, + resource value configs) is excluded on purpose (posture risk scoring). +- Security Command Center is not activated for the test project, so no + findings, group, config or admin response was ever seen. Owner: whoever + activates SCC at the organization; tests `google_live_armor.*`. - Token refresh is the caller's job (foundation gap). diff --git a/src/google/armor/agent_categories.cpp b/src/google/armor/agent_categories.cpp new file mode 100644 index 0000000..a6a5c15 --- /dev/null +++ b/src/google/armor/agent_categories.cpp @@ -0,0 +1,105 @@ +#include "google/armor/agent_categories.h" + +namespace molo::google::armor { + +namespace { +constexpr AgentFamily R = AgentFamily::kRuntime; +constexpr AgentFamily C = AgentFamily::kControlPlane; + +// clang-format off +const std::vector kTable = { + // Control plane (Event Threat Detection), enabled by default. + {"Discovery: Evidence of Port Scanning from AI Agent", "Discovery: AI Agent Evidence of Port Scanning", "AGENT_ENGINE_PORT_SCANNING_EVIDENCE", C}, + {"Discovery: AI Agent Unauthorized Service Account API Call", "", "AGENT_ENGINE_UNAUTHORIZED_SERVICE_ACCOUNT_API_CALL", C}, + {"Discovery: AI Agent Unauthorized Service Account API Call Affecting Sensitive Data", "", "", C}, + {"Discovery: AI Agent Service Account Self-Investigation", "", "AGENT_ENGINE_IAM_ANOMALOUS_BEHAVIOR_SERVICE_ACCOUNT_GETS_OWN_IAM_POLICY", C}, + {"Credential Access: AI Agent Anomalous Access to Metadata Service", "", "AGENT_ENGINE_ANOMALOUS_ACCESS_TO_METADATA_SERVICE", C}, + {"Exfiltration: AI Agent Initiated BigQuery VPC Perimeter Violation", "", "AGENT_ENGINE_BIG_QUERY_EXFIL_VPC_PERIMETER_VIOLATION", C}, + {"Exfiltration: AI Agent Initiated BigQuery Data Exfiltration to External Table", "", "AGENT_ENGINE_BIG_QUERY_EXFIL_TO_EXTERNAL_TABLE", C}, + {"Exfiltration: AI Agent Initiated Sensitive BigQuery Data Exfiltration to External Table", "", "", C}, + {"Exfiltration: AI Agent Initiated CloudSQL Exfiltration to Public Bucket", "Exfiltration: AI Agent Initiated Cloud SQL Exfiltration to Public Bucket", "AGENT_ENGINE_CLOUDSQL_EXFIL_EXPORT_TO_PUBLIC_GCS", C}, + {"Exfiltration: AI Agent Initiated CloudSQL Exfiltration to External Bucket", "Exfiltration: AI Agent Initiated Cloud SQL Exfiltration to External Bucket", "AGENT_ENGINE_CLOUDSQL_EXFIL_EXPORT_TO_EXTERNAL_GCS", C}, + {"Exfiltration: AI Agent Initiated BigQuery Data Extraction", "", "AGENT_ENGINE_BIG_QUERY_EXFIL_TO_CLOUD_STORAGE", C}, + {"Exfiltration: AI Agent Initiated Sensitive BigQuery Data Extraction", "", "", C}, + {"Initial Access: AI Agent Identity Excessive Permission Denied Actions", "", "AGENT_ENGINE_EXCESSIVE_FAILED_ATTEMPT", C}, + {"Privilege Escalation: AI Agent Token Generation Using signJwt", "", "AGENT_ENGINE_SUSPICIOUS_TOKEN_GENERATION_SIGN_JWT", C}, + {"Privilege Escalation: AI Agent Token Generation Using Implicit Delegation", "", "AGENT_ENGINE_SUSPICIOUS_TOKEN_GENERATION_IMPLICIT_DELEGATION", C}, + {"Privilege Escalation: AI Agent Cross-Project OpenID Token Generation", "", "AGENT_ENGINE_SUSPICIOUS_TOKEN_GENERATION_CROSS_PROJECT_OPENID", C}, + {"Privilege Escalation: AI Agent Cross-Project Access Token Generation", "", "AGENT_ENGINE_SUSPICIOUS_TOKEN_GENERATION_CROSS_PROJECT_ACCESS_TOKEN", C}, + // Runtime (Agent Platform Threat Detection, Preview). + {"Command and Control: Steganography Tool Detected", "", "AGENT_ENGINE_STEGANOGRAPHY_TOOL_DETECTED", R}, + {"Credential Access: Find Google Cloud Credentials", "", "AGENT_ENGINE_FIND_GCP_CREDENTIALS", R}, + {"Credential Access: GPG Key Reconnaissance", "", "AGENT_ENGINE_GPG_KEY_RECONNAISSANCE", R}, + {"Credential Access: Search Private Keys or Passwords", "", "AGENT_ENGINE_SEARCH_PRIVATE_KEYS_OR_PASSWORDS", R}, + {"Defense Evasion: Base64 ELF File Command Line", "", "AGENT_ENGINE_BASE64_ELF_FILE_CMDLINE", R}, + {"Defense Evasion: Base64 Encoded Python Script Executed", "", "AGENT_ENGINE_BASE64_ENCODED_PYTHON_SCRIPT_EXECUTED", R}, + {"Defense Evasion: Base64 Encoded Shell Script Executed", "", "AGENT_ENGINE_BASE64_ENCODED_SHELL_SCRIPT_EXECUTED", R}, + {"Defense Evasion: Launch Code Compiler Tool In Container", "", "AGENT_ENGINE_LAUNCH_CODE_COMPILER_TOOL_IN_CONTAINER", R}, + {"Execution: Added Malicious Binary Executed", "", "AGENT_ENGINE_ADDED_MALICIOUS_BINARY_EXECUTED", R}, + {"Execution: Added Malicious Library Loaded", "", "AGENT_ENGINE_ADDED_MALICIOUS_LIBRARY_LOADED", R}, + {"Execution: Added Malicious Skill Executed", "", "AGENT_ENGINE_ADDED_MALICIOUS_SKILL_EXECUTED", R}, + {"Execution: Added Malicious Skill Loaded", "", "AGENT_ENGINE_ADDED_MALICIOUS_SKILL_LOADED", R}, + {"Execution: Built in Malicious Binary Executed", "", "AGENT_ENGINE_BUILT_IN_MALICIOUS_BINARY_EXECUTED", R}, + {"Execution: Container Escape", "", "AGENT_ENGINE_CONTAINER_ESCAPE", R}, + {"Execution: Fileless Execution in /memfd:", "", "AGENT_ENGINE_FILELESS_EXECUTION_DETECTION_MEMFD", R}, + {"Execution: Kubernetes Attack Tool Execution", "", "AGENT_ENGINE_KUBERNETES_ATTACK_TOOL_EXECUTION", R}, + {"Execution: Local Reconnaissance Tool Execution", "", "AGENT_ENGINE_LOCAL_RECONNAISSANCE_TOOL_EXECUTION", R}, + {"Execution: Malicious Python Executed", "", "AGENT_ENGINE_MALICIOUS_PYTHON_EXECUTED", R}, + {"Execution: Modified Malicious Binary Executed", "", "AGENT_ENGINE_MODIFIED_MALICIOUS_BINARY_EXECUTED", R}, + {"Execution: Modified Malicious Library Loaded", "", "AGENT_ENGINE_MODIFIED_MALICIOUS_LIBRARY_LOADED", R}, + {"Execution: Modified Malicious Skill Executed", "", "AGENT_ENGINE_MODIFIED_MALICIOUS_SKILL_EXECUTED", R}, + {"Execution: Modified Malicious Skill Loaded", "", "AGENT_ENGINE_MODIFIED_MALICIOUS_SKILL_LOADED", R}, + {"Malicious Script Executed", "", "AGENT_ENGINE_MALICIOUS_SCRIPT_EXECUTED", R}, + {"Malicious URL Observed", "", "AGENT_ENGINE_MALICIOUS_URL_OBSERVED", R}, + {"Reverse Shell", "", "AGENT_ENGINE_REVERSE_SHELL", R}, + {"Unexpected Child Shell", "", "AGENT_ENGINE_UNEXPECTED_CHILD_SHELL", R}, + {"Execution: Netcat Remote Code Execution in Container", "", "AGENT_ENGINE_NETCAT_REMOTE_CODE_EXECUTION_IN_CONTAINER", R}, + {"Execution: Possible Arbitrary Command Execution through CUPS (CVE-2024-47177)", "", "AGENT_ENGINE_POSSIBLE_ARBITRARY_COMMAND_EXECUTION_THROUGH_CUPS", R}, + {"Execution: Possible Remote Command Execution Detected", "", "AGENT_ENGINE_POSSIBLE_REMOTE_COMMAND_EXECUTION_DETECTED", R}, + {"Execution: Program Run with Disallowed HTTP Proxy Env", "", "AGENT_ENGINE_PROGRAM_RUN_WITH_DISALLOWED_HTTP_PROXY_ENV", R}, + {"Execution: Socat Reverse Shell Detected", "", "AGENT_ENGINE_SOCAT_REVERSE_SHELL_DETECTED", R}, + {"Execution: Suspicious OpenSSL Shared Object Loaded", "", "AGENT_ENGINE_SUSPICIOUS_OPENSSL_SHARED_OBJECT_LOADED", R}, + {"Exfiltration: Launch Remote File Copy Tools in Container", "", "AGENT_ENGINE_LAUNCH_REMOTE_FILE_COPY_TOOLS_IN_CONTAINER", R}, + {"Impact: Detect Malicious Cmdlines", "", "AGENT_ENGINE_DETECT_MALICIOUS_CMDLINES", R}, + {"Impact: Remove Bulk Data From Disk", "Impact: Remove Bulk Data from Disk", "AGENT_ENGINE_REMOVE_BULK_DATA_FROM_DISK", R}, + {"Impact: Suspicious cryptocurrency mining activity using the Stratum Protocol", "Impact: Suspicious crypto mining activity using the Stratum Protocol", "AGENT_ENGINE_DETECT_CRYPTO_MINERS_USING_STRATUM_PROTOCOL", R}, + {"Privilege Escalation: Abuse of Sudo For Privilege Escalation (CVE-2019-14287)", "Privilege Escalation: Attempt to Abuse Sudo For Privilege Escalation (CVE-2019-14287)", "AGENT_ENGINE_ABUSE_SUDO_FOR_PRIVILEGE_ESCALATION", R}, + {"Privilege Escalation: Polkit Local Privilege Escalation Vulnerability (CVE-2021-4034)", "", "AGENT_ENGINE_POLKIT_LOCAL_PRIVILEGE_ESCALATION_VULNERABILITY", R}, + {"Privilege Escalation: Sudo Potential Privilege Escalation (CVE-2021-3156)", "", "AGENT_ENGINE_SUDO_POTENTIAL_PRIVILEGE_ESCALATION", R}, +}; +// clang-format on +} // namespace + +const std::vector& AgentCategories::all() { return kTable; } + +std::vector AgentCategories::strings(AgentFamily family) { + std::vector out; + for (const AgentCategory& c : kTable) { + if (c.family != family) continue; + out.push_back(c.category); + if (c.alt[0]) out.push_back(c.alt); + } + return out; +} + +std::vector AgentCategories::strings() { + std::vector out = strings(AgentFamily::kControlPlane); + std::vector rt = strings(AgentFamily::kRuntime); + out.insert(out.end(), rt.begin(), rt.end()); + return out; +} + +const AgentCategory* AgentCategories::find(const std::string& category) { + for (const AgentCategory& c : kTable) + if (category == c.category || (c.alt[0] && category == c.alt)) return &c; + return nullptr; +} + +const AgentCategory* AgentCategories::find_module(const std::string& module) { + if (module.empty()) return nullptr; + for (const AgentCategory& c : kTable) + if (module == c.module) return &c; + return nullptr; +} + +} // namespace molo::google::armor diff --git a/src/google/armor/agent_categories.h b/src/google/armor/agent_categories.h new file mode 100644 index 0000000..a02b414 --- /dev/null +++ b/src/google/armor/agent_categories.h @@ -0,0 +1,39 @@ +#pragma once +// Security Command Center finding categories of Agent Platform Threat +// Detection (runtime detectors, module names AGENT_ENGINE_*) and of the Event +// Threat Detection control plane detectors for AI agents, read from the +// official pages (see SPEC section Sources): +// https://docs.cloud.google.com/security-command-center/docs/agent-engine-threat-detection-overview +// https://docs.cloud.google.com/security-command-center/docs/ai-threats +// The two pages spell a few categories differently (for example "CloudSQL" +// and "Cloud SQL"); both spellings are listed and both are filtered on. +// Agent Anomaly Detection has no documented category string: its findings are +// recognised by the typed `agentAnomaly` member of the Finding. +#include +#include + +namespace molo::google::armor { + +enum class AgentFamily { kRuntime, kControlPlane }; + +struct AgentCategory { + const char* category; // As on the agent-engine-threat-detection-overview page + const char* alt; // Spelling on the ai-threats page when it differs, else "" + const char* module; // Event Threat Detection / runtime module name + AgentFamily family; +}; + +class AgentCategories { + public: + static const std::vector& all(); + // Every category string of a family, both spellings, for FindingQuery.categories. + static std::vector strings(AgentFamily family); + static std::vector strings(); // both families + // The entry whose category or alt equals `category`, or nullptr. + static const AgentCategory* find(const std::string& category); + static bool is_agent_threat(const std::string& category) { return find(category) != nullptr; } + // Entry by module name (the Finding.moduleName value), or nullptr. + static const AgentCategory* find_module(const std::string& module); +}; + +} // namespace molo::google::armor diff --git a/src/google/armor/calls.h b/src/google/armor/calls.h index 21f318a..a1413d6 100644 --- a/src/google/armor/calls.h +++ b/src/google/armor/calls.h @@ -18,6 +18,8 @@ struct CallSpec { const char* http; // GET | POST | PATCH | DELETE const char* path; // Discovery `path` (relative to the service root) const char* query; // Comma separated query parameters this module sends + const char* request; // Discovery request schema name, "" when no body + const char* fields; // Comma separated top-level request fields allowed }; struct SchemaSpec { @@ -37,37 +39,150 @@ struct EnumSpec { // clang-format off inline constexpr CallSpec kCalls[] = { - {"modelarmor", "v1", "modelarmor.projects.locations.templates.create", "POST", "v1/{+parent}/templates", "templateId,requestId"}, - {"modelarmor", "v1", "modelarmor.projects.locations.templates.get", "GET", "v1/{+name}", ""}, - {"modelarmor", "v1", "modelarmor.projects.locations.templates.list", "GET", "v1/{+parent}/templates", "pageSize,pageToken,filter,orderBy"}, - {"modelarmor", "v1", "modelarmor.projects.locations.templates.patch", "PATCH", "v1/{+name}", "updateMask,requestId"}, - {"modelarmor", "v1", "modelarmor.projects.locations.templates.delete", "DELETE", "v1/{+name}", "requestId"}, - {"modelarmor", "v1", "modelarmor.projects.locations.templates.sanitizeUserPrompt", "POST", "v1/{+name}:sanitizeUserPrompt", ""}, - {"modelarmor", "v1", "modelarmor.projects.locations.templates.sanitizeModelResponse", "POST", "v1/{+name}:sanitizeModelResponse", ""}, - {"modelarmor", "v1", "modelarmor.projects.locations.getFloorSetting", "GET", "v1/{+name}", ""}, - {"modelarmor", "v1", "modelarmor.projects.locations.updateFloorSetting", "PATCH", "v1/{+name}", "updateMask"}, - {"modelarmor", "v1", "modelarmor.folders.locations.getFloorSetting", "GET", "v1/{+name}", ""}, - {"modelarmor", "v1", "modelarmor.folders.locations.updateFloorSetting", "PATCH", "v1/{+name}", "updateMask"}, - {"modelarmor", "v1", "modelarmor.organizations.locations.getFloorSetting", "GET", "v1/{+name}", ""}, - {"modelarmor", "v1", "modelarmor.organizations.locations.updateFloorSetting", "PATCH", "v1/{+name}", "updateMask"}, - {"modelarmor", "v1", "modelarmor.projects.locations.list", "GET", "v1/{+name}/locations", "pageToken"}, - {"modelarmor", "v1", "modelarmor.projects.locations.get", "GET", "v1/{+name}", ""}, - {"securitycenter", "v2", "securitycenter.organizations.sources.findings.list", "GET", "v2/{+parent}/findings", "pageSize,pageToken,filter,orderBy,fieldMask"}, - {"securitycenter", "v2", "securitycenter.folders.sources.findings.list", "GET", "v2/{+parent}/findings", "pageSize,pageToken,filter,orderBy,fieldMask"}, - {"securitycenter", "v2", "securitycenter.projects.sources.findings.list", "GET", "v2/{+parent}/findings", "pageSize,pageToken,filter,orderBy,fieldMask"}, - {"securitycenter", "v2", "securitycenter.organizations.sources.locations.findings.list", "GET", "v2/{+parent}/findings", "pageSize,pageToken,filter,orderBy,fieldMask"}, - {"securitycenter", "v2", "securitycenter.folders.sources.locations.findings.list", "GET", "v2/{+parent}/findings", "pageSize,pageToken,filter,orderBy,fieldMask"}, - {"securitycenter", "v2", "securitycenter.projects.sources.locations.findings.list", "GET", "v2/{+parent}/findings", "pageSize,pageToken,filter,orderBy,fieldMask"}, - {"securitycenter", "v2", "securitycenter.organizations.sources.findings.group", "POST", "v2/{+parent}/findings:group", ""}, - {"securitycenter", "v2", "securitycenter.folders.sources.findings.group", "POST", "v2/{+parent}/findings:group", ""}, - {"securitycenter", "v2", "securitycenter.projects.sources.findings.group", "POST", "v2/{+parent}/findings:group", ""}, - {"securitycenter", "v2", "securitycenter.organizations.sources.locations.findings.group", "POST", "v2/{+parent}/findings:group", ""}, - {"securitycenter", "v2", "securitycenter.folders.sources.locations.findings.group", "POST", "v2/{+parent}/findings:group", ""}, - {"securitycenter", "v2", "securitycenter.projects.sources.locations.findings.group", "POST", "v2/{+parent}/findings:group", ""}, - {"securitycenter", "v2", "securitycenter.organizations.sources.list", "GET", "v2/{+parent}/sources", "pageSize,pageToken"}, - {"securitycenter", "v2", "securitycenter.folders.sources.list", "GET", "v2/{+parent}/sources", "pageSize,pageToken"}, - {"securitycenter", "v2", "securitycenter.projects.sources.list", "GET", "v2/{+parent}/sources", "pageSize,pageToken"}, - {"securitycenter", "v2", "securitycenter.organizations.sources.get", "GET", "v2/{+name}", ""}, + {"modelarmor", "v1", "modelarmor.projects.locations.templates.create", "POST", "v1/{+parent}/templates", "templateId,requestId", "Template", "templateMetadata,name,labels,filterConfig"}, + {"modelarmor", "v1", "modelarmor.projects.locations.templates.get", "GET", "v1/{+name}", "", "", ""}, + {"modelarmor", "v1", "modelarmor.projects.locations.templates.list", "GET", "v1/{+parent}/templates", "pageSize,pageToken,filter,orderBy", "", ""}, + {"modelarmor", "v1", "modelarmor.projects.locations.templates.patch", "PATCH", "v1/{+name}", "updateMask,requestId", "Template", "templateMetadata,name,labels,filterConfig"}, + {"modelarmor", "v1", "modelarmor.projects.locations.templates.delete", "DELETE", "v1/{+name}", "requestId", "", ""}, + {"modelarmor", "v1", "modelarmor.projects.locations.templates.sanitizeUserPrompt", "POST", "v1/{+name}:sanitizeUserPrompt", "", "SanitizeUserPromptRequest", "userPromptData,streamingMode,multiLanguageDetectionMetadata"}, + {"modelarmor", "v1", "modelarmor.projects.locations.templates.sanitizeModelResponse", "POST", "v1/{+name}:sanitizeModelResponse", "", "SanitizeModelResponseRequest", "streamingMode,multiLanguageDetectionMetadata,modelResponseData,userPrompt"}, + {"modelarmor", "v1", "modelarmor.projects.locations.getFloorSetting", "GET", "v1/{+name}", "", "", ""}, + {"modelarmor", "v1", "modelarmor.projects.locations.updateFloorSetting", "PATCH", "v1/{+name}", "updateMask", "FloorSetting", "aiPlatformFloorSetting,name,integratedServices,floorSettingMetadata,enableFloorSettingEnforcement,googleMcpServerFloorSetting,filterConfig"}, + {"modelarmor", "v1", "modelarmor.folders.locations.getFloorSetting", "GET", "v1/{+name}", "", "", ""}, + {"modelarmor", "v1", "modelarmor.folders.locations.updateFloorSetting", "PATCH", "v1/{+name}", "updateMask", "FloorSetting", "aiPlatformFloorSetting,name,integratedServices,floorSettingMetadata,enableFloorSettingEnforcement,googleMcpServerFloorSetting,filterConfig"}, + {"modelarmor", "v1", "modelarmor.organizations.locations.getFloorSetting", "GET", "v1/{+name}", "", "", ""}, + {"modelarmor", "v1", "modelarmor.organizations.locations.updateFloorSetting", "PATCH", "v1/{+name}", "updateMask", "FloorSetting", "aiPlatformFloorSetting,name,integratedServices,floorSettingMetadata,enableFloorSettingEnforcement,googleMcpServerFloorSetting,filterConfig"}, + {"modelarmor", "v1", "modelarmor.projects.locations.list", "GET", "v1/{+name}/locations", "pageToken", "", ""}, + {"modelarmor", "v1", "modelarmor.projects.locations.get", "GET", "v1/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.sources.findings.list", "GET", "v2/{+parent}/findings", "pageSize,pageToken,filter,orderBy,fieldMask", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.sources.findings.list", "GET", "v2/{+parent}/findings", "pageSize,pageToken,filter,orderBy,fieldMask", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.sources.findings.list", "GET", "v2/{+parent}/findings", "pageSize,pageToken,filter,orderBy,fieldMask", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.sources.locations.findings.list", "GET", "v2/{+parent}/findings", "pageSize,pageToken,filter,orderBy,fieldMask", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.sources.locations.findings.list", "GET", "v2/{+parent}/findings", "pageSize,pageToken,filter,orderBy,fieldMask", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.sources.locations.findings.list", "GET", "v2/{+parent}/findings", "pageSize,pageToken,filter,orderBy,fieldMask", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.sources.findings.group", "POST", "v2/{+parent}/findings:group", "", "GroupFindingsRequest", "pageToken,groupBy,pageSize,filter"}, + {"securitycenter", "v2", "securitycenter.folders.sources.findings.group", "POST", "v2/{+parent}/findings:group", "", "GroupFindingsRequest", "pageToken,groupBy,pageSize,filter"}, + {"securitycenter", "v2", "securitycenter.projects.sources.findings.group", "POST", "v2/{+parent}/findings:group", "", "GroupFindingsRequest", "pageToken,groupBy,pageSize,filter"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.locations.findings.group", "POST", "v2/{+parent}/findings:group", "", "GroupFindingsRequest", "pageToken,groupBy,pageSize,filter"}, + {"securitycenter", "v2", "securitycenter.folders.sources.locations.findings.group", "POST", "v2/{+parent}/findings:group", "", "GroupFindingsRequest", "pageToken,groupBy,pageSize,filter"}, + {"securitycenter", "v2", "securitycenter.projects.sources.locations.findings.group", "POST", "v2/{+parent}/findings:group", "", "GroupFindingsRequest", "pageToken,groupBy,pageSize,filter"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.list", "GET", "v2/{+parent}/sources", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.sources.list", "GET", "v2/{+parent}/sources", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.sources.list", "GET", "v2/{+parent}/sources", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.sources.get", "GET", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.assets.updateSecurityMarks", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2SecurityMarks", "name,marks,canonicalName"}, + {"securitycenter", "v2", "securitycenter.folders.findings.bulkMute", "POST", "v2/{+parent}/findings:bulkMute", "", "BulkMuteFindingsRequest", "filter,muteState"}, + {"securitycenter", "v2", "securitycenter.folders.locations.bigQueryExports.create", "POST", "v2/{+parent}/bigQueryExports", "bigQueryExportId", "GoogleCloudSecuritycenterV2BigQueryExport", "name,filter,dataset,deletionNotificationsEnabled,description"}, + {"securitycenter", "v2", "securitycenter.folders.locations.bigQueryExports.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.locations.bigQueryExports.get", "GET", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.locations.bigQueryExports.list", "GET", "v2/{+parent}/bigQueryExports", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.locations.bigQueryExports.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2BigQueryExport", "name,filter,dataset,deletionNotificationsEnabled,description"}, + {"securitycenter", "v2", "securitycenter.folders.locations.findings.bulkMute", "POST", "v2/{+parent}/findings:bulkMute", "", "BulkMuteFindingsRequest", "filter,muteState"}, + {"securitycenter", "v2", "securitycenter.folders.locations.muteConfigs.create", "POST", "v2/{+parent}/muteConfigs", "muteConfigId", "GoogleCloudSecuritycenterV2MuteConfig", "description,name,type,filter,expiryTime"}, + {"securitycenter", "v2", "securitycenter.folders.locations.muteConfigs.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.locations.muteConfigs.get", "GET", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.locations.muteConfigs.list", "GET", "v2/{+parent}/muteConfigs", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.locations.muteConfigs.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2MuteConfig", "description,name,type,filter,expiryTime"}, + {"securitycenter", "v2", "securitycenter.folders.locations.notificationConfigs.create", "POST", "v2/{+parent}/notificationConfigs", "configId", "NotificationConfig", "streamingConfig,pubsubTopic,description,name,deletionNotificationsEnabled"}, + {"securitycenter", "v2", "securitycenter.folders.locations.notificationConfigs.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.locations.notificationConfigs.get", "GET", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.locations.notificationConfigs.list", "GET", "v2/{+parent}/notificationConfigs", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.locations.notificationConfigs.patch", "PATCH", "v2/{+name}", "updateMask", "NotificationConfig", "streamingConfig,pubsubTopic,description,name,deletionNotificationsEnabled"}, + {"securitycenter", "v2", "securitycenter.folders.muteConfigs.create", "POST", "v2/{+parent}/muteConfigs", "muteConfigId", "GoogleCloudSecuritycenterV2MuteConfig", "description,name,type,filter,expiryTime"}, + {"securitycenter", "v2", "securitycenter.folders.muteConfigs.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.muteConfigs.get", "GET", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.muteConfigs.list", "GET", "v2/{+parent}/muteConfigs", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.folders.muteConfigs.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2MuteConfig", "description,name,type,filter,expiryTime"}, + {"securitycenter", "v2", "securitycenter.folders.sources.findings.externalSystems.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2ExternalSystem", "caseUri,name,assignees,externalUid,casePriority,status,caseSla,caseCloseTime,caseCreateTime,externalSystemUpdateTime,ticketInfo"}, + {"securitycenter", "v2", "securitycenter.folders.sources.findings.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2Finding", "iamBindings,eventTime,loadBalancers,findingClass,secret,database,moduleName,securityPosture,dataRetentionDeletionEvents,indicator,vertexAi,kubernetes,chokepoint,cloudArmor,exfiltration,files,notebook,toxicCombination,iamDetails,agentAnomaly,name,artifactGuardPolicies,compliances,processes,backupDisasterRecovery,parent,severity,externalExposure,connections,groupMemberships,attackExposure,disk,discoveredWorkload,agentDataAccessEvents,policyViolationSummary,logEntries,muteInitiator,vulnerability,externalUri,category,agent,mitreAttack,resourceName,orgPolicies,networks,kernelRootkit,dataAccessEvents,agentSessions,application,ipRules,job,cloudDlpInspection,aiModel,nextSteps,access,mute,cloudDlpDataProfile,affectedResources,complianceDetails,dataFlowEvents,containers,sourceProperties,description"}, + {"securitycenter", "v2", "securitycenter.folders.sources.findings.setMute", "POST", "v2/{+name}:setMute", "", "SetMuteRequest", "mute"}, + {"securitycenter", "v2", "securitycenter.folders.sources.findings.setState", "POST", "v2/{+name}:setState", "", "SetFindingStateRequest", "state"}, + {"securitycenter", "v2", "securitycenter.folders.sources.findings.updateSecurityMarks", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2SecurityMarks", "name,marks,canonicalName"}, + {"securitycenter", "v2", "securitycenter.folders.sources.locations.findings.export", "POST", "v2/{+parent}/findings:export", "", "ExportFindingsRequest", "bigQueryDestination"}, + {"securitycenter", "v2", "securitycenter.folders.sources.locations.findings.externalSystems.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2ExternalSystem", "caseUri,name,assignees,externalUid,casePriority,status,caseSla,caseCloseTime,caseCreateTime,externalSystemUpdateTime,ticketInfo"}, + {"securitycenter", "v2", "securitycenter.folders.sources.locations.findings.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2Finding", "iamBindings,eventTime,loadBalancers,findingClass,secret,database,moduleName,securityPosture,dataRetentionDeletionEvents,indicator,vertexAi,kubernetes,chokepoint,cloudArmor,exfiltration,files,notebook,toxicCombination,iamDetails,agentAnomaly,name,artifactGuardPolicies,compliances,processes,backupDisasterRecovery,parent,severity,externalExposure,connections,groupMemberships,attackExposure,disk,discoveredWorkload,agentDataAccessEvents,policyViolationSummary,logEntries,muteInitiator,vulnerability,externalUri,category,agent,mitreAttack,resourceName,orgPolicies,networks,kernelRootkit,dataAccessEvents,agentSessions,application,ipRules,job,cloudDlpInspection,aiModel,nextSteps,access,mute,cloudDlpDataProfile,affectedResources,complianceDetails,dataFlowEvents,containers,sourceProperties,description"}, + {"securitycenter", "v2", "securitycenter.folders.sources.locations.findings.setMute", "POST", "v2/{+name}:setMute", "", "SetMuteRequest", "mute"}, + {"securitycenter", "v2", "securitycenter.folders.sources.locations.findings.setState", "POST", "v2/{+name}:setState", "", "SetFindingStateRequest", "state"}, + {"securitycenter", "v2", "securitycenter.folders.sources.locations.findings.updateSecurityMarks", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2SecurityMarks", "name,marks,canonicalName"}, + {"securitycenter", "v2", "securitycenter.organizations.assets.updateSecurityMarks", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2SecurityMarks", "name,marks,canonicalName"}, + {"securitycenter", "v2", "securitycenter.organizations.findings.bulkMute", "POST", "v2/{+parent}/findings:bulkMute", "", "BulkMuteFindingsRequest", "filter,muteState"}, + {"securitycenter", "v2", "securitycenter.organizations.locations.bigQueryExports.create", "POST", "v2/{+parent}/bigQueryExports", "bigQueryExportId", "GoogleCloudSecuritycenterV2BigQueryExport", "name,filter,dataset,deletionNotificationsEnabled,description"}, + {"securitycenter", "v2", "securitycenter.organizations.locations.bigQueryExports.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.locations.bigQueryExports.get", "GET", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.locations.bigQueryExports.list", "GET", "v2/{+parent}/bigQueryExports", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.locations.bigQueryExports.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2BigQueryExport", "name,filter,dataset,deletionNotificationsEnabled,description"}, + {"securitycenter", "v2", "securitycenter.organizations.locations.findings.bulkMute", "POST", "v2/{+parent}/findings:bulkMute", "", "BulkMuteFindingsRequest", "filter,muteState"}, + {"securitycenter", "v2", "securitycenter.organizations.locations.muteConfigs.create", "POST", "v2/{+parent}/muteConfigs", "muteConfigId", "GoogleCloudSecuritycenterV2MuteConfig", "description,name,type,filter,expiryTime"}, + {"securitycenter", "v2", "securitycenter.organizations.locations.muteConfigs.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.locations.muteConfigs.get", "GET", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.locations.muteConfigs.list", "GET", "v2/{+parent}/muteConfigs", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.locations.muteConfigs.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2MuteConfig", "description,name,type,filter,expiryTime"}, + {"securitycenter", "v2", "securitycenter.organizations.locations.notificationConfigs.create", "POST", "v2/{+parent}/notificationConfigs", "configId", "NotificationConfig", "streamingConfig,pubsubTopic,description,name,deletionNotificationsEnabled"}, + {"securitycenter", "v2", "securitycenter.organizations.locations.notificationConfigs.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.locations.notificationConfigs.get", "GET", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.locations.notificationConfigs.list", "GET", "v2/{+parent}/notificationConfigs", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.locations.notificationConfigs.patch", "PATCH", "v2/{+name}", "updateMask", "NotificationConfig", "streamingConfig,pubsubTopic,description,name,deletionNotificationsEnabled"}, + {"securitycenter", "v2", "securitycenter.organizations.muteConfigs.create", "POST", "v2/{+parent}/muteConfigs", "muteConfigId", "GoogleCloudSecuritycenterV2MuteConfig", "description,name,type,filter,expiryTime"}, + {"securitycenter", "v2", "securitycenter.organizations.muteConfigs.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.muteConfigs.get", "GET", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.muteConfigs.list", "GET", "v2/{+parent}/muteConfigs", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.muteConfigs.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2MuteConfig", "description,name,type,filter,expiryTime"}, + {"securitycenter", "v2", "securitycenter.organizations.operations.cancel", "POST", "v2/{+name}:cancel", "", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.operations.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.operations.get", "GET", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.operations.list", "GET", "v2/{+name}", "filter,pageSize,pageToken,returnPartialSuccess", "", ""}, + {"securitycenter", "v2", "securitycenter.organizations.sources.create", "POST", "v2/{+parent}/sources", "", "Source", "canonicalName,name,displayName,description"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.findings.create", "POST", "v2/{+parent}/findings", "findingId", "GoogleCloudSecuritycenterV2Finding", "iamBindings,eventTime,loadBalancers,findingClass,secret,database,moduleName,securityPosture,dataRetentionDeletionEvents,indicator,vertexAi,kubernetes,chokepoint,cloudArmor,exfiltration,files,notebook,toxicCombination,iamDetails,agentAnomaly,name,artifactGuardPolicies,compliances,processes,backupDisasterRecovery,parent,severity,externalExposure,connections,groupMemberships,attackExposure,disk,discoveredWorkload,agentDataAccessEvents,policyViolationSummary,logEntries,muteInitiator,vulnerability,externalUri,category,agent,mitreAttack,resourceName,orgPolicies,networks,kernelRootkit,dataAccessEvents,agentSessions,application,ipRules,job,cloudDlpInspection,aiModel,nextSteps,access,mute,cloudDlpDataProfile,affectedResources,complianceDetails,dataFlowEvents,containers,sourceProperties,description"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.findings.externalSystems.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2ExternalSystem", "caseUri,name,assignees,externalUid,casePriority,status,caseSla,caseCloseTime,caseCreateTime,externalSystemUpdateTime,ticketInfo"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.findings.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2Finding", "iamBindings,eventTime,loadBalancers,findingClass,secret,database,moduleName,securityPosture,dataRetentionDeletionEvents,indicator,vertexAi,kubernetes,chokepoint,cloudArmor,exfiltration,files,notebook,toxicCombination,iamDetails,agentAnomaly,name,artifactGuardPolicies,compliances,processes,backupDisasterRecovery,parent,severity,externalExposure,connections,groupMemberships,attackExposure,disk,discoveredWorkload,agentDataAccessEvents,policyViolationSummary,logEntries,muteInitiator,vulnerability,externalUri,category,agent,mitreAttack,resourceName,orgPolicies,networks,kernelRootkit,dataAccessEvents,agentSessions,application,ipRules,job,cloudDlpInspection,aiModel,nextSteps,access,mute,cloudDlpDataProfile,affectedResources,complianceDetails,dataFlowEvents,containers,sourceProperties,description"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.findings.setMute", "POST", "v2/{+name}:setMute", "", "SetMuteRequest", "mute"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.findings.setState", "POST", "v2/{+name}:setState", "", "SetFindingStateRequest", "state"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.findings.updateSecurityMarks", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2SecurityMarks", "name,marks,canonicalName"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.getIamPolicy", "POST", "v2/{+resource}:getIamPolicy", "", "GetIamPolicyRequest", "options"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.locations.findings.create", "POST", "v2/{+parent}/findings", "findingId", "GoogleCloudSecuritycenterV2Finding", "iamBindings,eventTime,loadBalancers,findingClass,secret,database,moduleName,securityPosture,dataRetentionDeletionEvents,indicator,vertexAi,kubernetes,chokepoint,cloudArmor,exfiltration,files,notebook,toxicCombination,iamDetails,agentAnomaly,name,artifactGuardPolicies,compliances,processes,backupDisasterRecovery,parent,severity,externalExposure,connections,groupMemberships,attackExposure,disk,discoveredWorkload,agentDataAccessEvents,policyViolationSummary,logEntries,muteInitiator,vulnerability,externalUri,category,agent,mitreAttack,resourceName,orgPolicies,networks,kernelRootkit,dataAccessEvents,agentSessions,application,ipRules,job,cloudDlpInspection,aiModel,nextSteps,access,mute,cloudDlpDataProfile,affectedResources,complianceDetails,dataFlowEvents,containers,sourceProperties,description"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.locations.findings.export", "POST", "v2/{+parent}/findings:export", "", "ExportFindingsRequest", "bigQueryDestination"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.locations.findings.externalSystems.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2ExternalSystem", "caseUri,name,assignees,externalUid,casePriority,status,caseSla,caseCloseTime,caseCreateTime,externalSystemUpdateTime,ticketInfo"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.locations.findings.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2Finding", "iamBindings,eventTime,loadBalancers,findingClass,secret,database,moduleName,securityPosture,dataRetentionDeletionEvents,indicator,vertexAi,kubernetes,chokepoint,cloudArmor,exfiltration,files,notebook,toxicCombination,iamDetails,agentAnomaly,name,artifactGuardPolicies,compliances,processes,backupDisasterRecovery,parent,severity,externalExposure,connections,groupMemberships,attackExposure,disk,discoveredWorkload,agentDataAccessEvents,policyViolationSummary,logEntries,muteInitiator,vulnerability,externalUri,category,agent,mitreAttack,resourceName,orgPolicies,networks,kernelRootkit,dataAccessEvents,agentSessions,application,ipRules,job,cloudDlpInspection,aiModel,nextSteps,access,mute,cloudDlpDataProfile,affectedResources,complianceDetails,dataFlowEvents,containers,sourceProperties,description"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.locations.findings.setMute", "POST", "v2/{+name}:setMute", "", "SetMuteRequest", "mute"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.locations.findings.setState", "POST", "v2/{+name}:setState", "", "SetFindingStateRequest", "state"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.locations.findings.updateSecurityMarks", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2SecurityMarks", "name,marks,canonicalName"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.patch", "PATCH", "v2/{+name}", "updateMask", "Source", "canonicalName,name,displayName,description"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.setIamPolicy", "POST", "v2/{+resource}:setIamPolicy", "", "SetIamPolicyRequest", "updateMask,policy"}, + {"securitycenter", "v2", "securitycenter.organizations.sources.testIamPermissions", "POST", "v2/{+resource}:testIamPermissions", "", "TestIamPermissionsRequest", "permissions"}, + {"securitycenter", "v2", "securitycenter.projects.assets.updateSecurityMarks", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2SecurityMarks", "name,marks,canonicalName"}, + {"securitycenter", "v2", "securitycenter.projects.findings.bulkMute", "POST", "v2/{+parent}/findings:bulkMute", "", "BulkMuteFindingsRequest", "filter,muteState"}, + {"securitycenter", "v2", "securitycenter.projects.locations.bigQueryExports.create", "POST", "v2/{+parent}/bigQueryExports", "bigQueryExportId", "GoogleCloudSecuritycenterV2BigQueryExport", "name,filter,dataset,deletionNotificationsEnabled,description"}, + {"securitycenter", "v2", "securitycenter.projects.locations.bigQueryExports.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.locations.bigQueryExports.get", "GET", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.locations.bigQueryExports.list", "GET", "v2/{+parent}/bigQueryExports", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.locations.bigQueryExports.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2BigQueryExport", "name,filter,dataset,deletionNotificationsEnabled,description"}, + {"securitycenter", "v2", "securitycenter.projects.locations.findings.bulkMute", "POST", "v2/{+parent}/findings:bulkMute", "", "BulkMuteFindingsRequest", "filter,muteState"}, + {"securitycenter", "v2", "securitycenter.projects.locations.muteConfigs.create", "POST", "v2/{+parent}/muteConfigs", "muteConfigId", "GoogleCloudSecuritycenterV2MuteConfig", "description,name,type,filter,expiryTime"}, + {"securitycenter", "v2", "securitycenter.projects.locations.muteConfigs.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.locations.muteConfigs.get", "GET", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.locations.muteConfigs.list", "GET", "v2/{+parent}/muteConfigs", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.locations.muteConfigs.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2MuteConfig", "description,name,type,filter,expiryTime"}, + {"securitycenter", "v2", "securitycenter.projects.locations.notificationConfigs.create", "POST", "v2/{+parent}/notificationConfigs", "configId", "NotificationConfig", "streamingConfig,pubsubTopic,description,name,deletionNotificationsEnabled"}, + {"securitycenter", "v2", "securitycenter.projects.locations.notificationConfigs.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.locations.notificationConfigs.get", "GET", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.locations.notificationConfigs.list", "GET", "v2/{+parent}/notificationConfigs", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.locations.notificationConfigs.patch", "PATCH", "v2/{+name}", "updateMask", "NotificationConfig", "streamingConfig,pubsubTopic,description,name,deletionNotificationsEnabled"}, + {"securitycenter", "v2", "securitycenter.projects.muteConfigs.create", "POST", "v2/{+parent}/muteConfigs", "muteConfigId", "GoogleCloudSecuritycenterV2MuteConfig", "description,name,type,filter,expiryTime"}, + {"securitycenter", "v2", "securitycenter.projects.muteConfigs.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.muteConfigs.get", "GET", "v2/{+name}", "", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.muteConfigs.list", "GET", "v2/{+parent}/muteConfigs", "pageSize,pageToken", "", ""}, + {"securitycenter", "v2", "securitycenter.projects.muteConfigs.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2MuteConfig", "description,name,type,filter,expiryTime"}, + {"securitycenter", "v2", "securitycenter.projects.sources.findings.externalSystems.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2ExternalSystem", "caseUri,name,assignees,externalUid,casePriority,status,caseSla,caseCloseTime,caseCreateTime,externalSystemUpdateTime,ticketInfo"}, + {"securitycenter", "v2", "securitycenter.projects.sources.findings.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2Finding", "iamBindings,eventTime,loadBalancers,findingClass,secret,database,moduleName,securityPosture,dataRetentionDeletionEvents,indicator,vertexAi,kubernetes,chokepoint,cloudArmor,exfiltration,files,notebook,toxicCombination,iamDetails,agentAnomaly,name,artifactGuardPolicies,compliances,processes,backupDisasterRecovery,parent,severity,externalExposure,connections,groupMemberships,attackExposure,disk,discoveredWorkload,agentDataAccessEvents,policyViolationSummary,logEntries,muteInitiator,vulnerability,externalUri,category,agent,mitreAttack,resourceName,orgPolicies,networks,kernelRootkit,dataAccessEvents,agentSessions,application,ipRules,job,cloudDlpInspection,aiModel,nextSteps,access,mute,cloudDlpDataProfile,affectedResources,complianceDetails,dataFlowEvents,containers,sourceProperties,description"}, + {"securitycenter", "v2", "securitycenter.projects.sources.findings.setMute", "POST", "v2/{+name}:setMute", "", "SetMuteRequest", "mute"}, + {"securitycenter", "v2", "securitycenter.projects.sources.findings.setState", "POST", "v2/{+name}:setState", "", "SetFindingStateRequest", "state"}, + {"securitycenter", "v2", "securitycenter.projects.sources.findings.updateSecurityMarks", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2SecurityMarks", "name,marks,canonicalName"}, + {"securitycenter", "v2", "securitycenter.projects.sources.locations.findings.export", "POST", "v2/{+parent}/findings:export", "", "ExportFindingsRequest", "bigQueryDestination"}, + {"securitycenter", "v2", "securitycenter.projects.sources.locations.findings.externalSystems.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2ExternalSystem", "caseUri,name,assignees,externalUid,casePriority,status,caseSla,caseCloseTime,caseCreateTime,externalSystemUpdateTime,ticketInfo"}, + {"securitycenter", "v2", "securitycenter.projects.sources.locations.findings.patch", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2Finding", "iamBindings,eventTime,loadBalancers,findingClass,secret,database,moduleName,securityPosture,dataRetentionDeletionEvents,indicator,vertexAi,kubernetes,chokepoint,cloudArmor,exfiltration,files,notebook,toxicCombination,iamDetails,agentAnomaly,name,artifactGuardPolicies,compliances,processes,backupDisasterRecovery,parent,severity,externalExposure,connections,groupMemberships,attackExposure,disk,discoveredWorkload,agentDataAccessEvents,policyViolationSummary,logEntries,muteInitiator,vulnerability,externalUri,category,agent,mitreAttack,resourceName,orgPolicies,networks,kernelRootkit,dataAccessEvents,agentSessions,application,ipRules,job,cloudDlpInspection,aiModel,nextSteps,access,mute,cloudDlpDataProfile,affectedResources,complianceDetails,dataFlowEvents,containers,sourceProperties,description"}, + {"securitycenter", "v2", "securitycenter.projects.sources.locations.findings.setMute", "POST", "v2/{+name}:setMute", "", "SetMuteRequest", "mute"}, + {"securitycenter", "v2", "securitycenter.projects.sources.locations.findings.setState", "POST", "v2/{+name}:setState", "", "SetFindingStateRequest", "state"}, + {"securitycenter", "v2", "securitycenter.projects.sources.locations.findings.updateSecurityMarks", "PATCH", "v2/{+name}", "updateMask", "GoogleCloudSecuritycenterV2SecurityMarks", "name,marks,canonicalName"}, }; // Request bodies (what to_json emits) and response documents (what from_json reads). @@ -138,6 +253,11 @@ inline constexpr SchemaSpec kSchemas[] = { {"securitycenter", "v2", "GoogleCloudSecuritycenterV2Cve", "id"}, {"securitycenter", "v2", "ListSourcesResponse", "sources,nextPageToken"}, {"securitycenter", "v2", "Source", "name,displayName,description,canonicalName"}, + {"securitycenter", "v2", "ListMuteConfigsResponse", "muteConfigs,nextPageToken"}, + {"securitycenter", "v2", "ListNotificationConfigsResponse", "notificationConfigs,nextPageToken"}, + {"securitycenter", "v2", "ListBigQueryExportsResponse", "bigQueryExports,nextPageToken"}, + {"securitycenter", "v2", "ListOperationsResponse", "operations,nextPageToken"}, + {"securitycenter", "v2", "Operation", "name,done,error,response,metadata"}, }; inline constexpr EnumSpec kEnums[] = { @@ -153,6 +273,10 @@ inline constexpr EnumSpec kEnums[] = { {"modelarmor", "v1", "ByteDataItem", "byteDataType", "PLAINTEXT_UTF8,PDF,WORD_DOCUMENT,EXCEL_DOCUMENT,POWERPOINT_DOCUMENT,TXT,CSV,IMAGE"}, {"securitycenter", "v2", "GoogleCloudSecuritycenterV2Finding", "state", "ACTIVE,INACTIVE"}, {"securitycenter", "v2", "GoogleCloudSecuritycenterV2Finding", "severity", "CRITICAL,HIGH,MEDIUM,LOW"}, + {"securitycenter", "v2", "SetMuteRequest", "mute", "MUTED,UNMUTED,UNDEFINED"}, + {"securitycenter", "v2", "SetFindingStateRequest", "state", "ACTIVE,INACTIVE"}, + {"securitycenter", "v2", "BulkMuteFindingsRequest", "muteState", "MUTED,UNDEFINED"}, + {"securitycenter", "v2", "GoogleCloudSecuritycenterV2MuteConfig", "type", "STATIC,DYNAMIC"}, {"securitycenter", "v2", "GoogleCloudSecuritycenterV2Finding", "findingClass", "THREAT,VULNERABILITY,MISCONFIGURATION,OBSERVATION"}, }; // clang-format on diff --git a/src/google/armor/caps.h b/src/google/armor/caps.h index 972bf82..4e80112 100644 --- a/src/google/armor/caps.h +++ b/src/google/armor/caps.h @@ -34,7 +34,7 @@ struct ArmorCaps { static constexpr std::size_t kMaxFindingPages = 10; static constexpr std::size_t kMaxFindings = 5000; static constexpr std::size_t kMaxFilterValueChars = 256; - static constexpr std::size_t kMaxFilterCategories = 20; + static constexpr std::size_t kMaxFilterCategories = 100; static constexpr std::size_t kMaxSources = 100; // Distinct assets reported by the local dashboard. static constexpr std::size_t kMaxDashboardAssets = 500; diff --git a/src/google/armor/findings.cpp b/src/google/armor/findings.cpp index 768f6f6..74ca79e 100644 --- a/src/google/armor/findings.cpp +++ b/src/google/armor/findings.cpp @@ -33,6 +33,17 @@ bool safe_value(const std::string& v) { return true; } +// Finding categories are display names such as "Execution: Container Escape" or +// "Execution: Possible Remote Command Execution Detected": printable ASCII with +// spaces, colons and parentheses, never a quote or backslash (the value is +// placed inside a quoted filter string). +bool safe_category(const std::string& v) { + if (v.empty() || v.size() > ArmorCaps::kMaxFilterValueChars) return false; + for (unsigned char c : v) + if (c < 0x20 || c > 0x7e || c == '"' || c == '\\') return false; + return true; +} + // Comma separated field paths: letters, digits, '_' and '.'. bool safe_path_list(const std::string& v) { if (v.empty() || v.size() > ArmorCaps::kMaxFilterValueChars) return false; @@ -166,7 +177,7 @@ void FindingQuery::validate() const { if (categories.size() > ArmorCaps::kMaxFilterCategories) throw GoogleError::invalid("too many categories"); for (const auto& c : categories) - if (!safe_value(c)) throw GoogleError::invalid("bad category"); + if (!safe_category(c)) throw GoogleError::invalid("bad category"); if (!state.empty() && !one_of(state, {"ACTIVE", "INACTIVE"})) throw GoogleError::invalid("state must be ACTIVE or INACTIVE"); if (!severity.empty() && !one_of(severity, {"CRITICAL", "HIGH", "MEDIUM", "LOW"})) diff --git a/src/google/armor/model_armor.cpp b/src/google/armor/model_armor.cpp index c479dfc..00d6972 100644 --- a/src/google/armor/model_armor.cpp +++ b/src/google/armor/model_armor.cpp @@ -104,16 +104,17 @@ std::string ModelArmor::template_path(const std::string& id) const { return "/" + context_.child("templates", id); } -ServiceEndpoint ModelArmor::floor_endpoint() const { - return endpoint_for(context_.location); -} +// Observed live 2026-10-24: floor settings exist only at location "global" +// on the global host (a regional name answers 400 "Malformed name"), and +// projects.locations.list answers 403 on a regional host. +ServiceEndpoint ModelArmor::floor_endpoint() const { return endpoint_for("global"); } std::string ModelArmor::floor_path(const std::string& scope) const { std::string s = scope; context_.parent(); // Validates project and location. if (s.empty()) s = "projects/" + context_.project; if (!valid_scope(s)) throw GoogleError::invalid("bad floor setting scope"); - return "/" + s + "/locations/" + context_.location + "/floorSetting"; + return "/" + s + "/locations/global/floorSetting"; } std::string ModelArmor::request_id_query(const std::string& request_id, char sep) { @@ -216,7 +217,8 @@ std::vector ModelArmor::list_locations() { } ArmorLocation ModelArmor::get_location() { - Json l = client_->get_json(floor_endpoint(), "/" + context_.parent()); + // A location is served by its own host (regional or global). + Json l = client_->get_json(endpoint_for(context_.location), "/" + context_.parent()); return {string_at(l, "name"), string_at(l, "locationId"), string_at(l, "displayName")}; } diff --git a/src/google/armor/scc_name.cpp b/src/google/armor/scc_name.cpp new file mode 100644 index 0000000..b560fcb --- /dev/null +++ b/src/google/armor/scc_name.cpp @@ -0,0 +1,57 @@ +#include "google/armor/scc_name.h" + +#include "google/errors.h" +#include "google/resource.h" + +namespace molo::google::armor { + +SccName SccName::parse(const std::string& name) { + if (name.empty() || name.size() > 512 || name.front() == '/' || + name.find("..") != std::string::npos || name.find("//") != std::string::npos) + throw GoogleError::invalid("bad Security Command Center resource name"); + SccName out; + out.full_ = name; + std::string cur; + for (char c : name) { + if (c == '/') { + out.segments_.push_back(cur); + cur.clear(); + } else { + cur.push_back(c); + } + } + out.segments_.push_back(cur); + const auto& s = out.segments_; + if (s.size() < 2 || s.size() > 12) + throw GoogleError::invalid("resource name needs 2..12 segments"); + for (const std::string& seg : s) + if (!valid_segment(seg)) throw GoogleError::invalid("resource name has an invalid segment"); + if (s[0] != "organizations" && s[0] != "folders" && s[0] != "projects") + throw GoogleError::invalid("resource name must start with organizations/, folders/ or projects/"); + out.kind_ = s[0]; + if (s.size() > 2 && s[2] == "locations") out.has_location_ = true; + if (s.size() > 4 && s[2] == "sources" && s[4] == "locations") out.has_location_ = true; + return out; +} + +bool SccName::names_collection(const std::string& collection) const { + for (std::size_t i = 2; i < segments_.size(); ++i) + if (segments_[i] == collection) return true; + return false; +} + +bool SccName::ends_with_id_of(const std::string& collection) const { + return segments_.size() >= 4 && segments_[segments_.size() - 2] == collection; +} + +std::string SccName::rpc_for(const std::string& tail) const { + std::string base = "securitycenter." + kind_ + "."; + const std::string sources = "sources."; + if (tail.compare(0, sources.size(), sources) == 0) + return base + sources + (has_location_ ? "locations." : "") + tail.substr(sources.size()); + return base + (has_location_ ? "locations." : "") + tail; +} + +const CallSpec* SccName::row(const std::string& tail) const { return find_rpc(rpc_for(tail)); } + +} // namespace molo::google::armor diff --git a/src/google/armor/scc_name.h b/src/google/armor/scc_name.h new file mode 100644 index 0000000..1e48929 --- /dev/null +++ b/src/google/armor/scc_name.h @@ -0,0 +1,47 @@ +#pragma once +// A validated Security Command Center v2 resource name or parent and the +// table row that serves a method for it. Discovery lists each method per +// scope (organizations, folders, projects) and, for most, again under +// `locations`; the row is derived from the name itself so a caller can never +// reach a different scope than the one it named: +// projects/p/locations/global/muteConfigs/x -> securitycenter.projects.locations.muteConfigs.* +// projects/p/muteConfigs/x -> securitycenter.projects.muteConfigs.* +// organizations/1/sources/2/findings/f -> securitycenter.organizations.sources.findings.* +// organizations/1/sources/2/locations/l/findings/f -> ...sources.locations.findings.* +#include +#include + +#include "google/armor/calls.h" + +namespace molo::google::armor { + +class SccName { + public: + // Throws GoogleError::invalid when the name is not a well formed resource + // name (scope kind, valid id, valid segments, at most 12, no ".."). + static SccName parse(const std::string& name); + + const std::string& str() const { return full_; } + const std::string& kind() const { return kind_; } // organizations|folders|projects + const std::vector& segments() const { return segments_; } + // True when the name goes through a `locations/{l}` segment (either directly + // under the scope, or under a source). + bool has_location() const { return has_location_; } + // True when the last segment is `collection` (a parent names the scope, a + // resource name ends with collection/id). + bool ends_with_id_of(const std::string& collection) const; + bool names_collection(const std::string& collection) const; + + // The declared row for a method tail such as "muteConfigs.get" or + // "sources.findings.setState"; nullptr when this scope/location has none. + const CallSpec* row(const std::string& tail) const; + // The rpc id this name would use for the tail (not checked against the table). + std::string rpc_for(const std::string& tail) const; + + private: + std::string full_, kind_; + std::vector segments_; + bool has_location_ = false; +}; + +} // namespace molo::google::armor diff --git a/src/google/armor/security_admin.cpp b/src/google/armor/security_admin.cpp new file mode 100644 index 0000000..c0e1893 --- /dev/null +++ b/src/google/armor/security_admin.cpp @@ -0,0 +1,439 @@ +#include "google/armor/security_admin.h" + +#include +#include + +#include "clients/url.h" +#include "google/armor/caps.h" +#include "google/armor/findings.h" +#include "google/errors.h" +#include "google/resource.h" + +namespace molo::google::armor { + +using clients::body::Json; + +namespace { + +[[noreturn]] void bad(const std::string& m) { throw GoogleError::invalid(m); } + +std::set csv(const char* list, char sep = ',') { + std::set out; + std::string cur; + for (const char* p = list; *p; ++p) { + if (*p == sep) { + if (!cur.empty()) out.insert(cur); + cur.clear(); + } else { + cur.push_back(*p); + } + } + if (!cur.empty()) out.insert(cur); + return out; +} + +void check_mask(const CallSpec& row, const std::string& mask) { + if (mask.empty() || mask.size() > 256) bad("update_mask is required (max 256 characters)"); + std::set roots = csv(row.fields); + std::size_t pos = 0; + while (pos <= mask.size()) { + std::size_t end = mask.find(',', pos); + if (end == std::string::npos) end = mask.size(); + std::string path = mask.substr(pos, end - pos); + if (path.empty()) bad("update_mask has an empty path"); + for (char c : path) + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '_' || c == '.')) + bad("update_mask allows letters, digits, '_' and '.'"); + // Security marks masks name map keys ("marks.k"); other masks name fields. + std::string root = path.substr(0, path.find('.')); + if (!roots.count(root)) bad("update_mask field not allowed for this call: " + path); + pos = end + 1; + } +} + +void check_id(const std::string& id, const char* what) { + if (!valid_segment(id)) bad(std::string(what) + " must be 1-128 of A-Z a-z 0-9 . _ -"); +} + +const char* id_param(SccCollection c) { + switch (c) { + case SccCollection::kMuteConfigs: return "muteConfigId"; + case SccCollection::kNotificationConfigs: return "configId"; + case SccCollection::kBigQueryExports: return "bigQueryExportId"; + } + return "muteConfigId"; +} + +} // namespace + +const char* scc_collection_name(SccCollection c) { + switch (c) { + case SccCollection::kMuteConfigs: return "muteConfigs"; + case SccCollection::kNotificationConfigs: return "notificationConfigs"; + case SccCollection::kBigQueryExports: return "bigQueryExports"; + } + return "muteConfigs"; +} + +const char* finding_state_name(FindingState s) { + return s == FindingState::kActive ? "ACTIVE" : "INACTIVE"; +} + +const char* mute_value_name(MuteValue v) { + switch (v) { + case MuteValue::kMuted: return "MUTED"; + case MuteValue::kUnmuted: return "UNMUTED"; + case MuteValue::kUndefined: return "UNDEFINED"; + } + return "UNDEFINED"; +} + +const char* availability_name(SccAvailability a) { + switch (a) { + case SccAvailability::kAvailable: return "available"; + case SccAvailability::kNotActivated: return "not_activated"; + case SccAvailability::kApiDisabled: return "api_disabled"; + case SccAvailability::kDenied: return "denied"; + case SccAvailability::kUnknown: return "unknown"; + } + return "unknown"; +} + +Json SccList::to_json() const { + Json o = Json::object(); + o.set("count", static_cast(items.as_array().size())); + o.set("pages", static_cast(pages)); + o.set("truncated", truncated); + o.set("next_page_token", next_page_token); + o.set("items", items); + return o; +} + +SecurityAdmin::SecurityAdmin(std::shared_ptr client) + : Service(std::move(client), SecurityFindings::endpoint()) {} + +namespace { + +struct Planned { + const CallSpec* row; + std::string path; // leading slash, no version, with query + clients::Method method; +}; + +Planned plan(const std::string& tail, const std::string& resource, const Json* body, + const std::map& query) { + SccName n = SccName::parse(resource); + const CallSpec* row = n.row(tail); + if (!row) + bad(tail + " is not available for " + n.kind() + (n.has_location() ? " with a location" : " without a location")); + std::set allowed = csv(row->query); + std::string q; + for (const auto& [k, v] : query) { + if (!allowed.count(k)) bad("query parameter " + k + " is not in " + row->rpc); + if (v.empty()) continue; + if (v.size() > 2048) bad("query value too long"); + for (char c : v) + if (static_cast(c) < 0x20) bad("query value has control characters"); + q += (q.empty() ? "?" : "&") + k + "=" + clients::Url::encode_component(v); + } + std::string request = row->request; + if (!request.empty()) { + if (!body || !body->is_object()) bad(std::string(row->rpc) + " needs a JSON object body"); + std::set fields = csv(row->fields); + for (const auto& [k, v] : body->as_object()) { + (void)v; + if (!fields.count(k)) bad("field " + k + " is not part of " + request); + } + if (body->dump().size() > 256 * 1024) bad("request body over 256 KiB"); + } else if (body) { + bad(std::string(row->rpc) + " takes no body"); + } + // Discovery path "v2/{+parent}/muteConfigs": drop the version, fill the one variable. + std::string tmpl = row->path; + tmpl.erase(0, tmpl.find('/')); + std::size_t open = tmpl.find('{'), close = tmpl.find('}'); + if (open == std::string::npos || close == std::string::npos) bad("bad row path"); + std::string path = tmpl.substr(0, open) + n.str() + tmpl.substr(close + 1) + q; + std::string verb = row->http; + clients::Method m = verb == "POST" ? clients::Method::kPost + : verb == "PATCH" ? clients::Method::kPatch + : verb == "DELETE" ? clients::Method::kDelete + : clients::Method::kGet; + return {row, path, m}; +} + +} // namespace + +Json SecurityAdmin::run(const std::string& tail, const std::string& resource, const Json* body, + const std::map& query) const { + Planned p = plan(tail, resource, body, query); + Json empty = Json::object(); + const Json* send = body; + if (!send && p.method == clients::Method::kPost) send = ∅ // cancel and friends + return client_->request(p.method, endpoint_, p.path, send); +} + +std::string SecurityAdmin::describe(const std::string& tail, const std::string& resource, + const std::map& query) const { + Json empty = Json::object(); + // Describing a call with a body still validates the body shape: use {} and + // let callers that need fields pass them through run(). + Planned p = [&] { + try { + return plan(tail, resource, nullptr, query); + } catch (const GoogleError&) { + return plan(tail, resource, &empty, query); + } + }(); + return std::string(p.row->http) + " " + p.path; +} + +void SecurityAdmin::expect_name(const SccName& n, const std::string& collection, bool resource) { + if (resource) { + if (!n.ends_with_id_of(collection)) bad("name must end with /" + collection + "/{id}"); + } else if (n.names_collection(collection)) { + bad("parent must not include /" + collection); + } +} + +SccList SecurityAdmin::list_all_pages(const std::string& tail, const std::string& parent, + const std::string& items_key) const { + SccList out; + std::string token; + while (true) { + std::map q = {{"pageSize", std::to_string(ArmorCaps::kMaxTemplatePageSize)}}; + if (!token.empty()) q["pageToken"] = token; + Json reply = run(tail, parent, nullptr, q); + ++out.pages; + if (const Json* arr = reply.is_object() ? reply.find(items_key) : nullptr; arr && arr->is_array()) + for (const Json& item : arr->as_array()) { + if (out.items.as_array().size() >= ArmorCaps::kMaxFindings) { + out.truncated = true; + return out; + } + out.items.push(item); + } + const Json* next = reply.is_object() ? reply.find("nextPageToken") : nullptr; + token = next && next->is_string() ? next->as_string() : ""; + if (token.empty()) return out; + if (out.pages >= ArmorCaps::kMaxFindingPages) { + out.truncated = true; + out.next_page_token = token; + return out; + } + } +} + +// ---- Configs + +SccList SecurityAdmin::list_configs(SccCollection c, const std::string& parent) const { + expect_name(SccName::parse(parent), scc_collection_name(c), false); + return list_all_pages(std::string(scc_collection_name(c)) + ".list", parent, + scc_collection_name(c)); +} + +Json SecurityAdmin::get_config(SccCollection c, const std::string& name) const { + expect_name(SccName::parse(name), scc_collection_name(c), true); + return run(std::string(scc_collection_name(c)) + ".get", name); +} + +Json SecurityAdmin::create_config(SccCollection c, const std::string& parent, + const std::string& id, const Json& body) const { + check_id(id, "config id"); + expect_name(SccName::parse(parent), scc_collection_name(c), false); + return run(std::string(scc_collection_name(c)) + ".create", parent, &body, + {{id_param(c), id}}); +} + +Json SecurityAdmin::update_config(SccCollection c, const std::string& name, const Json& body, + const std::string& update_mask) const { + SccName n = SccName::parse(name); + expect_name(n, scc_collection_name(c), true); + std::string tail = std::string(scc_collection_name(c)) + ".patch"; + if (const CallSpec* row = n.row(tail)) check_mask(*row, update_mask); + return run(tail, name, &body, {{"updateMask", update_mask}}); +} + +void SecurityAdmin::delete_config(SccCollection c, const std::string& name) const { + expect_name(SccName::parse(name), scc_collection_name(c), true); + run(std::string(scc_collection_name(c)) + ".delete", name); +} + +// ---- Findings + +Json SecurityAdmin::set_finding_state(const std::string& finding, FindingState state) const { + expect_name(SccName::parse(finding), "findings", true); + Json body = Json::object(); + body.set("state", finding_state_name(state)); + return run("sources.findings.setState", finding, &body); +} + +Json SecurityAdmin::set_finding_mute(const std::string& finding, MuteValue mute) const { + expect_name(SccName::parse(finding), "findings", true); + Json body = Json::object(); + body.set("mute", mute_value_name(mute)); + return run("sources.findings.setMute", finding, &body); +} + +Json SecurityAdmin::patch_finding(const std::string& finding, const Json& body, + const std::string& update_mask) const { + SccName n = SccName::parse(finding); + expect_name(n, "findings", true); + if (const CallSpec* row = n.row("sources.findings.patch")) check_mask(*row, update_mask); + return run("sources.findings.patch", finding, &body, {{"updateMask", update_mask}}); +} + +Json SecurityAdmin::update_finding_marks(const std::string& finding, const Json& marks, + const std::string& update_mask) const { + SccName n = SccName::parse(finding); + expect_name(n, "findings", true); + // The resource is ".../findings/{f}" and the call patches its securityMarks. + if (const CallSpec* row = n.row("sources.findings.updateSecurityMarks")) check_mask(*row, update_mask); + return run("sources.findings.updateSecurityMarks", finding + "/securityMarks", &marks, + {{"updateMask", update_mask}}); +} + +Json SecurityAdmin::create_finding(const std::string& source, const std::string& finding_id, + const Json& body) const { + check_id(finding_id, "finding id"); + expect_name(SccName::parse(source), "findings", false); + return run("sources.findings.create", source, &body, {{"findingId", finding_id}}); +} + +Json SecurityAdmin::patch_external_system(const std::string& external_system, const Json& body, + const std::string& update_mask) const { + SccName n = SccName::parse(external_system); + expect_name(n, "externalSystems", true); + if (const CallSpec* row = n.row("sources.findings.externalSystems.patch")) check_mask(*row, update_mask); + return run("sources.findings.externalSystems.patch", external_system, &body, + {{"updateMask", update_mask}}); +} + +Operation SecurityAdmin::bulk_mute(const std::string& parent, const std::string& filter, + BulkMuteValue value) const { + if (filter.empty() || filter.size() > ArmorCaps::kMaxFilterValueChars * 4) + bad("bulk mute needs a filter (1.." + std::to_string(ArmorCaps::kMaxFilterValueChars * 4) + + " characters); it never mutes everything"); + expect_name(SccName::parse(parent), "findings", false); + Json body = Json::object(); + body.set("filter", filter); + body.set("muteState", value == BulkMuteValue::kMuted ? "MUTED" : "UNDEFINED"); + return Operation::from_json(run("findings.bulkMute", parent, &body)); +} + +Operation SecurityAdmin::export_findings(const std::string& source, const Json& body) const { + SccName n = SccName::parse(source); + if (!n.has_location()) bad("findings:export needs a source under a location"); + return Operation::from_json(run("sources.findings.export", source, &body)); +} + +// ---- Sources + +Json SecurityAdmin::create_source(const std::string& organization, const Json& body) const { + SccName n = SccName::parse(organization); + if (n.kind() != "organizations" || n.segments().size() != 2) + bad("sources are created under organizations/{id}"); + return run("sources.create", organization, &body); +} + +Json SecurityAdmin::update_source(const std::string& source, const Json& body, + const std::string& update_mask) const { + SccName n = SccName::parse(source); + expect_name(n, "sources", true); + if (const CallSpec* row = n.row("sources.patch")) check_mask(*row, update_mask); + return run("sources.patch", source, &body, {{"updateMask", update_mask}}); +} + +Json SecurityAdmin::source_iam_policy(const std::string& source) const { + expect_name(SccName::parse(source), "sources", true); + Json body = Json::object(); + return run("sources.getIamPolicy", source, &body); +} + +Json SecurityAdmin::set_source_iam_policy(const std::string& source, const Json& policy) const { + expect_name(SccName::parse(source), "sources", true); + if (!policy.is_object()) bad("policy must be a JSON object"); + source_iam_policy(source); // Read first. + Json body = Json::object(); + body.set("policy", policy); + return run("sources.setIamPolicy", source, &body); +} + +Json SecurityAdmin::test_source_iam(const std::string& source, + const std::vector& permissions) const { + expect_name(SccName::parse(source), "sources", true); + if (permissions.empty() || permissions.size() > 64) bad("1..64 permissions"); + Json arr = Json::array(); + for (const std::string& p : permissions) { + if (p.empty() || p.size() > 256) bad("permission empty or over 256 characters"); + arr.push(p); + } + Json body = Json::object(); + body.set("permissions", arr); + return run("sources.testIamPermissions", source, &body); +} + +Json SecurityAdmin::update_asset_marks(const std::string& marks_name, const Json& marks, + const std::string& update_mask) const { + SccName n = SccName::parse(marks_name); + if (!n.names_collection("assets") || marks_name.size() < 14 || + marks_name.compare(marks_name.size() - 14, 14, "/securityMarks") != 0) + bad("name must be {scope}/assets/{id}/securityMarks"); + if (const CallSpec* row = n.row("assets.updateSecurityMarks")) check_mask(*row, update_mask); + return run("assets.updateSecurityMarks", marks_name, &marks, {{"updateMask", update_mask}}); +} + +// ---- Operations + +Json SecurityAdmin::get_operation(const std::string& name) const { + if (!SccName::parse(name).ends_with_id_of("operations")) bad("name must be organizations/{id}/operations/{op}"); + return run("operations.get", name); +} + +SccList SecurityAdmin::list_operations(const std::string& organization) const { + // Discovery: operations.list takes the collection name itself as {+name}. + SccName n = SccName::parse(organization); + if (n.kind() != "organizations" || n.segments().size() != 2) + bad("operations are listed under organizations/{id}"); + return list_all_pages("operations.list", organization + "/operations", "operations"); +} + +void SecurityAdmin::cancel_operation(const std::string& name) const { + get_operation(name); // Read first: a missing operation is a clear 404. + run("operations.cancel", name); +} + +void SecurityAdmin::delete_operation(const std::string& name) const { + get_operation(name); + run("operations.delete", name); +} + +Operation SecurityAdmin::wait_operation_done(const std::string& name) const { + if (!SccName::parse(name).ends_with_id_of("operations")) bad("name must be organizations/{id}/operations/{op}"); + return google::wait_operation(*client_, endpoint_, name); +} + +// ---- Probe + +SccAvailability SecurityAdmin::availability(const std::string& scope) const { + SccName n = SccName::parse(scope); + if (n.segments().size() != 2) bad("scope must be organizations/{id}, folders/{id} or projects/{id}"); + try { + run("sources.list", scope, nullptr, {{"pageSize", "1"}}); + return SccAvailability::kAvailable; + } catch (const NotConfiguredError&) { + throw; + } catch (const GoogleError& e) { + if (e.service().empty() && e.status() == "INVALID_ARGUMENT") throw; + const std::string& m = e.message(); + if (e.http_status() == 403 && + (m.find("has not been used") != std::string::npos || m.find("disabled") != std::string::npos)) + return SccAvailability::kApiDisabled; + if (e.http_status() == 403) return SccAvailability::kDenied; + if (e.http_status() == 404 || e.status() == "FAILED_PRECONDITION") return SccAvailability::kNotActivated; + return SccAvailability::kUnknown; + } +} + +} // namespace molo::google::armor diff --git a/src/google/armor/security_admin.h b/src/google/armor/security_admin.h new file mode 100644 index 0000000..106d31d --- /dev/null +++ b/src/google/armor/security_admin.h @@ -0,0 +1,131 @@ +#pragma once +// Security Command Center v2 beyond reading findings: mute configs, +// notification configs, BigQuery exports, finding state/mute/marks, bulk +// mute, findings export, sources, asset marks, source IAM and organization +// operations. Every method is a row of the call table in calls.h, picked by +// SccName from the resource it is given; query parameters and top-level body +// fields are checked against the row before any traffic. Every mutation is an +// explicit method. Lists report truncation in their result type. +// Out of scope on purpose (calls.h / tools/check_discovery.py): the attack +// exposure engine (attack paths, valued resources, simulations, resource value +// configs), which is cloud posture risk scoring and not agent security. +#include +#include +#include +#include + +#include "clients/body/json.h" +#include "google/armor/scc_name.h" +#include "google/operation.h" +#include "google/service.h" + +namespace molo::google::armor { + +enum class SccCollection { kMuteConfigs, kNotificationConfigs, kBigQueryExports }; +enum class FindingState { kActive, kInactive }; +enum class MuteValue { kMuted, kUnmuted, kUndefined }; // setMute +enum class BulkMuteValue { kMuted, kUndefined }; // bulkMute + +const char* scc_collection_name(SccCollection c); // "muteConfigs", ... +const char* finding_state_name(FindingState s); +const char* mute_value_name(MuteValue v); + +// A paged list. Never silently cut: past the caps `truncated` is true and +// `next_page_token` continues. +struct SccList { + clients::body::Json items = clients::body::Json::array(); + bool truncated = false; + std::string next_page_token; + std::size_t pages = 0; + clients::body::Json to_json() const; +}; + +// Why Security Command Center did not answer, from a real probe (see SPEC): +// an enabled API without SCC activation answers 404 (sources) or 400 +// FAILED_PRECONDITION (configs); a disabled API 403. +enum class SccAvailability { kAvailable, kNotActivated, kApiDisabled, kDenied, kUnknown }; +const char* availability_name(SccAvailability a); + +class SecurityAdmin : public Service { + public: + explicit SecurityAdmin(std::shared_ptr client); + + // ---- Generic table-driven call (tail like "muteConfigs.get"). The public + // typed methods below are thin wrappers; this is for tools and tests. + clients::body::Json run(const std::string& tail, const std::string& resource, + const clients::body::Json* body = nullptr, + const std::map& query = {}) const; + // The request the typed call would send: "METHOD path", no network. + std::string describe(const std::string& tail, const std::string& resource, + const std::map& query = {}) const; + + // ---- Configs: parent is "projects/p[/locations/l]" (also folders, + // organizations); notification configs and BigQuery exports exist only + // with a location. A name is parent + "/" + collection + "/" + id. + SccList list_configs(SccCollection c, const std::string& parent) const; + clients::body::Json get_config(SccCollection c, const std::string& name) const; + clients::body::Json create_config(SccCollection c, const std::string& parent, + const std::string& id, + const clients::body::Json& body) const; + clients::body::Json update_config(SccCollection c, const std::string& name, + const clients::body::Json& body, + const std::string& update_mask) const; + // Destructive: explicit, one config. + void delete_config(SccCollection c, const std::string& name) const; + + // ---- Findings. A finding name is ".../sources/{s}[/locations/{l}]/findings/{f}". + clients::body::Json set_finding_state(const std::string& finding, FindingState state) const; + clients::body::Json set_finding_mute(const std::string& finding, MuteValue mute) const; + clients::body::Json patch_finding(const std::string& finding, + const clients::body::Json& body, + const std::string& update_mask) const; + clients::body::Json update_finding_marks(const std::string& finding, + const clients::body::Json& marks, + const std::string& update_mask) const; + clients::body::Json create_finding(const std::string& source, const std::string& finding_id, + const clients::body::Json& body) const; + clients::body::Json patch_external_system(const std::string& external_system, + const clients::body::Json& body, + const std::string& update_mask) const; + // Mutes or unmutes every finding matching `filter` (required, never empty). + // Long-running; returns the operation (not waited). + Operation bulk_mute(const std::string& parent, const std::string& filter, + BulkMuteValue value) const; + // findings:export (locations only), long-running; body {"bigQueryDestination":{...}}. + Operation export_findings(const std::string& source, const clients::body::Json& body) const; + + // ---- Sources (organization scope) and IAM on them. + clients::body::Json create_source(const std::string& organization, + const clients::body::Json& body) const; + clients::body::Json update_source(const std::string& source, const clients::body::Json& body, + const std::string& update_mask) const; + clients::body::Json source_iam_policy(const std::string& source) const; + clients::body::Json set_source_iam_policy(const std::string& source, + const clients::body::Json& policy) const; + clients::body::Json test_source_iam(const std::string& source, + const std::vector& permissions) const; + + // ---- Asset security marks (assets.updateSecurityMarks). + clients::body::Json update_asset_marks(const std::string& marks_name, + const clients::body::Json& marks, + const std::string& update_mask) const; + + // ---- Organization operations (get, list, cancel, delete, wait by polling). + clients::body::Json get_operation(const std::string& name) const; + SccList list_operations(const std::string& organization) const; + void cancel_operation(const std::string& name) const; + void delete_operation(const std::string& name) const; // record only + Operation wait_operation_done(const std::string& name) const; + + // ---- Probe: what does Security Command Center say for this scope? + // Read-only (sources.list, one page of one). Never throws for a Google + // refusal; validation errors still throw. + SccAvailability availability(const std::string& scope) const; + + private: + SccList list_all_pages(const std::string& tail, const std::string& parent, + const std::string& items_key) const; + static void expect_name(const SccName& n, const std::string& collection, bool resource); +}; + +} // namespace molo::google::armor diff --git a/src/google/armor/tools/check_discovery.py b/src/google/armor/tools/check_discovery.py index 1587b85..1f64a03 100644 --- a/src/google/armor/tools/check_discovery.py +++ b/src/google/armor/tools/check_discovery.py @@ -19,6 +19,13 @@ ROW = re.compile(r'^\s*\{((?:"[^"]*"(?:,\s*)?)+)\},\s*$') STR = re.compile(r'"([^"]*)"') +# Excluded on purpose (scope): the attack exposure engine of Security Command +# Center (attack paths, valued resources, simulations, resource value configs) +# is cloud posture risk scoring, not agent security. Every other Security +# Command Center v2 method and every Model Armor method is a table row. +EXCLUDED_RE = re.compile(r"attackPaths|valuedResources|simulations|resourceValueConfigs") + + def rows(): calls, schemas, enums = [], [], [] for line in open(HEADER, encoding="utf-8"): @@ -26,7 +33,7 @@ def rows(): if not m: continue cells = STR.findall(m.group(1)) - {6: calls, 4: schemas, 5: enums}.get(len(cells), []).append(cells) + {8: calls, 4: schemas, 5: enums}.get(len(cells), []).append(cells) return calls, schemas, enums @@ -70,7 +77,7 @@ def main(): return docs[key] index = {} - for service, version, rpc, http, path, query in calls: + for service, version, rpc, http, path, query, request, fields in calls: d = doc(service, version) idx = index.setdefault((service, version), methods(d)) m = idx.get(rpc) @@ -84,6 +91,25 @@ def main(): for q in filter(None, query.split(",")): if q not in m.get("parameters", {}): errors.append("%s: query parameter %s missing" % (rpc, q)) + ref = (m.get("request") or {}).get("$ref", "") + if ref != request: + errors.append("%s: request schema %r != declared %r" % (rpc, ref, request)) + if ref: + props = doc(service, version)["schemas"].get(ref, {}).get("properties", {}) + for f in filter(None, fields.split(",")): + if f not in props: + errors.append("%s: request field %s not in %s" % (rpc, f, ref)) + elif fields: + errors.append("%s: fields declared for a call without a request body" % rpc) + # Coverage: every discovery method must be a table row or be excluded. + declared = {(c[0], c[1], c[2]) for c in calls} + for (service, version), idx in index.items(): + for rpc in sorted(idx): + if (service, version, rpc) in declared: + continue + if EXCLUDED_RE.search(rpc): + continue + errors.append("discovery method not in the table: " + rpc) for service, version, schema, fields in schemas: s = doc(service, version)["schemas"].get(schema) if not s: diff --git a/src/google/optimize/SPEC.md b/src/google/optimize/SPEC.md index 9a9d0f5..7693f8e 100644 --- a/src/google/optimize/SPEC.md +++ b/src/google/optimize/SPEC.md @@ -34,7 +34,9 @@ optimizer; those are local abstractions and are named local in code and here. | `observability.h/.cpp` | `TraceReader`, `LogReader`, `Trace`, `Span`, `LogEntry`, `ConversationBuilder`, `summarize` | | `conversation.h/.cpp` | `Conversation`, `Turn`, transcript parse and render | | `model_json.h/.cpp` | `JsonModelCall`: one JSON object from a `LanguageModel`, one retry | -| `direct_model.h/.cpp` | `DirectModel : LanguageModel` over `GoogleClient` (private to this module) | +| `model_factory.h/.cpp` | `make_gemini_model`: the one Gemini adapter, `models::VertexModel` from `src/google/models`, target picked from the credentials (the former private `DirectModel` is gone) | +| `operations_api.h/.cpp` | `OnlineEvaluatorOperations`: get, list (loud truncation flag), cancel, delete, server-side wait for online evaluator operations | +| `telemetry_writer.h/.cpp` | `TraceWriter` (patchTraces, batchWrite, createSpan) and `LogWriter` (entries:write with dryRun, entries:copy) | | `autorater.h/.cpp` | `Autorater`, `Rubric`, `Score` | | `scenario.h/.cpp` | `Scenario` (mirrors Google's `UserScenario`) | | `agent.h/.cpp` | `Agent` seam, `ModelAgent` | @@ -97,23 +99,37 @@ optimizer; those are local abstractions and are named local in code and here. - Simulator loops are bounded by turns, tool rounds and tool calls; the stop reason is reported (`user_done`, `max_turns`, `tool_round_cap`, `tool_call_cap`). -- Credentials: this module never reads or prints them. `DirectModel::from_env` - picks the endpoint: OAuth token plus project gives the project-scoped - Vertex path (global location), an express key the Vertex publisher path, - otherwise Generative Language. +- Credentials: this module never reads or prints them. `make_gemini_model` + builds a `models::VertexModel` (so there is one Gemini adapter in the + platform): an OAuth token (`GOOGLE_ACCESS_TOKEN`) with a valid + `GOOGLE_CLOUD_PROJECT` gives the project scoped Vertex path in location + `global`, anything else Vertex express mode (publisher path). Behavior + change from `DirectModel`: the Generative Language endpoint (non Vertex API + keys) is no longer used; such a key now fails at Google on the Vertex path. + Streaming is real (`?alt=sse`) instead of one chunk. +- Write calls are explicit classes (`TraceWriter`, `LogWriter`), bodies are raw + Json checked against the table and capped (1 MiB, 1000 log entries, 2000 + spans). `LogWriter::write` supports `dryRun`; the demo route is a dry run + unless `confirm` is `write`. `entries.tail` is excluded: an endless server + stream has no cap. `OnlineEvaluatorOperations::list` stops at 20 pages / + 1000 items and says `truncated` with the next page token. ## 5. Example routes (`/agent/optimize`) GET `/agent/optimize` (page, matrix row), GET `/agent/optimize/status`. POST (503 `google_not_configured` without credentials or a valid `GOOGLE_CLOUD_PROJECT`, 422 on bad input, 502 `google_error` upstream): -`evaluate`, `scenarios`, `runs`, `traces`, `logs` (Google APIs) and `score`, +`evaluate`, `scenarios`, `runs`, `traces`, `logs`, `synthetic`, `operations/list`, `operations/get`, `operations/wait`, `operations/cancel`, `logs/write` (Google APIs; DELETE `operations?name=` removes an operation record) and `score`, `cluster`, `suggest`, `simulate` (local logic on a Gemini model). ## 6. Verified against discovery -`python3 -I src/google/optimize/tools/check_discovery.py`, run 2026-10-24: -`OK: 38 calls and 64 schemas match discovery`. It fetches +`python3 -I src/google/optimize/tools/check_discovery.py`, run 2026-10-24 +(gap-closing round): `OK: 46 calls and 64 schemas match discovery`. It also +enforces coverage: every aiplatform v1 `evaluat*`, `generate*` and +`onlineEvaluators*` method, every Cloud Trace v1 and v2 method and every +`logging.entries*` method must be a table row or be listed in `EXCLUDED` +(only `logging.entries.tail`, endless stream). It fetches `https://{aiplatform,cloudtrace,logging}.googleapis.com/$discovery/rest` (aiplatform v1, cloudtrace v1 and v2, logging v2) and fails on any declared call absent or different in verb, path, request fields or query parameters, @@ -141,7 +157,21 @@ One run of the OAuth live suite showed 1 failed test of 5 (name not captured); the immediate rerun and a later run passed 5 of 5. Treat as an unexplained transient until reproduced. -Not run live (would create or change resources): create/patch/delete/import +Gap-closing round, 2026-10-24, service-account token and express key, +read-only (`optimize_live_gaps.*`, `optimize_live.*`): + +| Call | Result | +|---|---| +| `GET onlineEvaluators/1/operations?pageSize=1` (absent evaluator) | 200, no operations | +| `GET onlineEvaluators/probe/operations` | 400 INVALID_ARGUMENT: the LRO layer wants a numeric evaluator id ("must be a Long") | +| `make_gemini_model` (`models::VertexModel`) `generate`, OAuth token, project path, location global | 200, a reply | +| `Autorater` through `make_gemini_model` with the express key (no token) | verdicts parsed, score 1 | +| `Autorater` through `make_gemini_model` with the token | verdicts parsed, score 1 | +| `evaluate_instances` exactMatch / trajectory (key), five resource lists (token) | unchanged from above: 200 | + +Not run live (would create or change resources): `generateSyntheticData`, the +online evaluator operation cancel, delete and wait, `patchTraces`, `batchWrite`, +`createSpan`, `entries:write`, `entries:copy`, create/patch/delete/import of sets, items, metrics, runs, online evaluators, `evaluateDataset`, `generateInstanceRubrics`, `generateUserScenarios`, `generateLossClusters`. These are verified by discovery only. @@ -164,12 +194,24 @@ These are verified by discovery only. - The default autorater model for LLM-judged metrics is unavailable to the test project (see 7); `AutoraterOptions::model` is the workaround, untested. - v1beta1 differences are not checked. +- `generateSyntheticData`: request and reply shapes are raw Json; the reply type + is not typed here and was never seen. +- Online evaluator operation names: the id of an evaluator in an operation + name must be numeric at Google's LRO layer (400 seen); evaluator ids Google + assigns were never observed because none exists. +- `entries:copy` destination string format and `entries:write` resource labels + are passed through, only top-level fields are checked. +- The shared adapter is Vertex only: a non Vertex API key (Generative Language + `AIza` key) is not routed to the Generative Language endpoint any more. ## 9. Gaps -- Cloud Trace v2 `batchWrite` and `createSpan` and Cloud Logging - `entries.write/tail` are not wrapped: agents write through OTel exporters. - The v2 rows are not in the table because nothing calls them. +- `logging.entries.tail` (endless server stream) is not wrapped; the write + calls (`TraceWriter`, `LogWriter`) exist for tools, agents normally write + through OTel exporters. +- Trace and log lists (`TraceReader::list`, `LogReader::entries`) still stop at + their caps without a `truncated` flag; only the new operations list reports + truncation. Owner: `observability.cpp`. - Dashboards: only numbers (`DashboardData`); no Cloud Monitoring reads. - `evaluateDataset`, `generateInstanceRubrics`, `evaluationMetrics` and item bodies are passed as raw `Json` (checked at the top level against diff --git a/src/google/optimize/calls.h b/src/google/optimize/calls.h index 96504ed..42bd565 100644 --- a/src/google/optimize/calls.h +++ b/src/google/optimize/calls.h @@ -38,6 +38,17 @@ inline constexpr const char* kEvaluateDataset = "aiplatform.projects.locations.e inline constexpr const char* kGenerateInstanceRubrics = "aiplatform.projects.locations.generateInstanceRubrics"; inline constexpr const char* kGenerateUserScenarios = "aiplatform.projects.locations.generateUserScenarios"; inline constexpr const char* kGenerateLossClusters = "aiplatform.projects.locations.generateLossClusters"; +inline constexpr const char* kGenerateSyntheticData = "aiplatform.projects.locations.generateSyntheticData"; +inline constexpr const char* kOnlineEvaluatorOpGet = "aiplatform.projects.locations.onlineEvaluators.operations.get"; +inline constexpr const char* kOnlineEvaluatorOpList = "aiplatform.projects.locations.onlineEvaluators.operations.list"; +inline constexpr const char* kOnlineEvaluatorOpCancel = "aiplatform.projects.locations.onlineEvaluators.operations.cancel"; +inline constexpr const char* kOnlineEvaluatorOpDelete = "aiplatform.projects.locations.onlineEvaluators.operations.delete"; +inline constexpr const char* kOnlineEvaluatorOpWait = "aiplatform.projects.locations.onlineEvaluators.operations.wait"; +inline constexpr const char* kTracePatch = "cloudtrace.projects.patchTraces"; +inline constexpr const char* kTraceBatchWrite = "cloudtrace.projects.traces.batchWrite"; +inline constexpr const char* kTraceCreateSpan = "cloudtrace.projects.traces.spans.createSpan"; +inline constexpr const char* kLogEntriesWrite = "logging.entries.write"; +inline constexpr const char* kLogEntriesCopy = "logging.entries.copy"; inline constexpr const char* kOperationGet = "aiplatform.projects.locations.operations.get"; inline constexpr const char* kTraceGet = "cloudtrace.projects.traces.get"; inline constexpr const char* kTraceList = "cloudtrace.projects.traces.list"; diff --git a/src/google/optimize/calls_table.inc b/src/google/optimize/calls_table.inc index d25d9dc..7f79f3d 100644 --- a/src/google/optimize/calls_table.inc +++ b/src/google/optimize/calls_table.inc @@ -115,3 +115,27 @@ {"logging.entries.list", "POST", "/entries:list", "logging", "v2", "filter,orderBy,pageSize,pageToken,projectIds,resourceNames", ""}, +{"aiplatform.projects.locations.generateSyntheticData", "POST", "/{+location}:generateSyntheticData", "aiplatform", "v1", + "count,examples,outputFieldSpecs,taskDescription", + ""}, +{"aiplatform.projects.locations.onlineEvaluators.operations.cancel", "POST", "/{+name}:cancel", "aiplatform", "v1", + "", + ""}, +{"aiplatform.projects.locations.onlineEvaluators.operations.delete", "DELETE", "/{+name}", "aiplatform", "v1", + "", + ""}, +{"aiplatform.projects.locations.onlineEvaluators.operations.get", "GET", "/{+name}", "aiplatform", "v1", + "", + ""}, +{"aiplatform.projects.locations.onlineEvaluators.operations.list", "GET", "/{+name}/operations", "aiplatform", "v1", + "", + "filter,pageSize,pageToken,returnPartialSuccess"}, +{"aiplatform.projects.locations.onlineEvaluators.operations.wait", "POST", "/{+name}:wait", "aiplatform", "v1", + "", + "timeout"}, +{"logging.entries.write", "POST", "/entries:write", "logging", "v2", + "dryRun,entries,labels,logName,partialSuccess,resource", + ""}, +{"logging.entries.copy", "POST", "/entries:copy", "logging", "v2", + "destination,filter,name", + ""}, diff --git a/src/google/optimize/caps.h b/src/google/optimize/caps.h index 183293a..f21b525 100644 --- a/src/google/optimize/caps.h +++ b/src/google/optimize/caps.h @@ -36,6 +36,11 @@ struct OptimizeCaps { static constexpr std::size_t kMaxLogPageSize = 100; static constexpr std::size_t kMaxResourceNames = 100; // Cloud Logging limit static constexpr std::size_t kMaxFilterChars = 20000; // Cloud Logging limit + // Write side (telemetry, operations). + static constexpr std::size_t kMaxWriteEntries = 1000; // log entries per write + static constexpr std::size_t kMaxWriteBytes = 1024 * 1024; // serialized write body + static constexpr std::size_t kMaxSpansPerWrite = 2000; + static constexpr int kMaxServerWaitSeconds = 60; // one POST :wait // Clustering and optimization. static constexpr std::size_t kMaxFailures = 200; static constexpr std::size_t kMaxClusters = 10; diff --git a/src/google/optimize/direct_model.cpp b/src/google/optimize/direct_model.cpp deleted file mode 100644 index 4beff0b..0000000 --- a/src/google/optimize/direct_model.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include "google/optimize/direct_model.h" - -#include "google/credentials.h" -#include "google/endpoints.h" -#include "google/errors.h" -#include "google/generative.h" -#include "google/resource.h" -#include "google/voice_backend.h" - -namespace molo::google::optimize { - -DirectModel::DirectModel(std::shared_ptr client, std::string model, - ServiceEndpoint endpoint) - : Service(std::move(client), std::move(endpoint)), - model_(std::move(model)) {} - -std::shared_ptr DirectModel::from_env(std::shared_ptr client, - std::string model) { - // OAuth tokens need the project-scoped Vertex path (express paths refuse them). - CloudContext ctx = CloudContext::from_env(); - if (!Credentials::env("GOOGLE_ACCESS_TOKEN").empty() && ctx.valid()) { - auto m = std::make_shared(std::move(client), std::move(model), - endpoints::aiplatform("global")); - m->path_prefix_ = "/projects/" + ctx.project + "/locations/global/publishers/google/models/"; - return m; - } - ServiceEndpoint ep = voice_backend_from_env() == VoiceBackend::kVertex - ? endpoints::vertex_generative() - : endpoints::generative(); - return std::make_shared(std::move(client), std::move(model), std::move(ep)); -} - -GenerateResponse DirectModel::generate(const GenerateRequest& request) { - std::string model = !request.model.empty() ? request.model - : !model_.empty() ? model_ - : Generative::default_model(); - if (!valid_segment(model)) throw GoogleError::invalid("bad model name"); - return GenerateResponse::from_json( - call(path_prefix_ + model + ":generateContent", request.to_json())); -} - -void DirectModel::stream(const GenerateRequest& request, - const std::function& on_chunk) { - on_chunk(generate(request)); -} - -} // namespace molo::google::optimize diff --git a/src/google/optimize/direct_model.h b/src/google/optimize/direct_model.h deleted file mode 100644 index a8e4f7e..0000000 --- a/src/google/optimize/direct_model.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -// DirectModel: a LanguageModel over GoogleClient (Gemini generateContent on -// the Generative Language endpoint). Private to this module so optimize -// needs no other agent-platform module; any LanguageModel can replace it. -// Streaming is one chunk (the full reply). - -#include -#include - -#include "google/gemini_types.h" -#include "google/service.h" - -namespace molo::google::optimize { - -class DirectModel : public Service, public LanguageModel { - public: - // Empty model: GOOGLE_GENAI_MODEL or the built-in default. - explicit DirectModel(std::shared_ptr client, std::string model = "", - ServiceEndpoint endpoint = endpoints::generative()); - - // Endpoint by credential: an OAuth token (GOOGLE_ACCESS_TOKEN) with a valid - // GOOGLE_CLOUD_PROJECT uses the project-scoped Vertex path; Vertex express - // keys ("AQ.", or GOOGLE_API_BACKEND=vertex) the Vertex publisher endpoint; - // everything else the Generative Language endpoint. - static std::shared_ptr from_env(std::shared_ptr client, - std::string model = ""); - - GenerateResponse generate(const GenerateRequest& request) override; - void stream(const GenerateRequest& request, - const std::function& on_chunk) override; - - private: - std::string model_; - std::string path_prefix_ = "/models/"; -}; - -} // namespace molo::google::optimize diff --git a/src/google/optimize/evaluation_api.cpp b/src/google/optimize/evaluation_api.cpp index 217f3e9..deacb01 100644 --- a/src/google/optimize/evaluation_api.cpp +++ b/src/google/optimize/evaluation_api.cpp @@ -65,6 +65,15 @@ Json EvaluationApi::generate_instance_rubrics(const Json& request) const { &request); } +Json EvaluationApi::generate_synthetic_data(const Json& request) const { + if (!request.is_object() || request.as_object().empty()) + throw GoogleError::invalid("synthetic data request is empty"); + if (request.dump().size() > OptimizeCaps::kMaxWriteBytes) + throw GoogleError::invalid("synthetic data request is over the size cap"); + return invoke(call_spec(rpc::kGenerateSyntheticData), {{"location", context_.parent()}}, {}, + &request); +} + Json EvaluationApi::scenario_body(const ScenarioRequest& r) { if (r.count < 1 || r.count > static_cast(OptimizeCaps::kMaxScenarios)) throw GoogleError::invalid("count must be 1.." + std::to_string(OptimizeCaps::kMaxScenarios)); diff --git a/src/google/optimize/evaluation_api.h b/src/google/optimize/evaluation_api.h index 31a7a67..fbba087 100644 --- a/src/google/optimize/evaluation_api.h +++ b/src/google/optimize/evaluation_api.h @@ -48,6 +48,9 @@ class EvaluationApi : public OptimizeService { bool wait = false, const Sleeper& sleeper = nullptr) const; clients::body::Json generate_instance_rubrics(const clients::body::Json& request) const; + // generateSyntheticData: raw request (top-level fields checked against the + // table); the reply is returned as Json, its shape is not typed (Unverified). + clients::body::Json generate_synthetic_data(const clients::body::Json& request) const; std::vector generate_user_scenarios(const ScenarioRequest& request) const; // Body of generateUserScenarios, exposed so it can be checked offline. diff --git a/src/google/optimize/model_factory.cpp b/src/google/optimize/model_factory.cpp new file mode 100644 index 0000000..645aa65 --- /dev/null +++ b/src/google/optimize/model_factory.cpp @@ -0,0 +1,25 @@ +#include "google/optimize/model_factory.h" + +#include "google/credentials.h" +#include "google/generative.h" +#include "google/models/gemini.h" +#include "google/models/target.h" +#include "google/models/vertex_model.h" +#include "google/resource.h" + +namespace molo::google::optimize { + +std::shared_ptr make_gemini_model(std::shared_ptr client, + std::string model) { + if (model.empty()) model = Generative::default_model(); + CloudContext ctx = CloudContext::from_env(); + models::VertexTarget target = models::VertexTarget::express(); + if (!Credentials::env("GOOGLE_ACCESS_TOKEN").empty() && ctx.valid()) { + ctx.location = "global"; + target = models::VertexTarget::project(std::move(ctx)); + } + auto gemini = std::make_shared(std::move(client), std::move(target)); + return std::make_shared(std::move(gemini), std::move(model)); +} + +} // namespace molo::google::optimize diff --git a/src/google/optimize/model_factory.h b/src/google/optimize/model_factory.h new file mode 100644 index 0000000..7301f49 --- /dev/null +++ b/src/google/optimize/model_factory.h @@ -0,0 +1,26 @@ +#pragma once + +// The one way this module gets a Gemini LanguageModel: models::VertexModel +// from src/google/models (the single Gemini adapter of the platform). The +// scoring and simulation classes still code against LanguageModel only; this +// factory just picks the Vertex target from the credentials. + +#include +#include + +#include "google/client.h" +#include "google/gemini_types.h" + +namespace molo::google::optimize { + +// Target by credential: +// - OAuth token (GOOGLE_ACCESS_TOKEN) with a valid GOOGLE_CLOUD_PROJECT: the +// project scoped Vertex path in location "global" (what the previous +// private adapter used). +// - otherwise Vertex express mode (API key, publisher path). +// Empty `model`: Generative::default_model() (GOOGLE_GENAI_MODEL or built in). +// Nothing is sent here; credential problems surface on the first generate(). +std::shared_ptr make_gemini_model(std::shared_ptr client, + std::string model = ""); + +} // namespace molo::google::optimize diff --git a/src/google/optimize/operations_api.cpp b/src/google/optimize/operations_api.cpp new file mode 100644 index 0000000..ea323d2 --- /dev/null +++ b/src/google/optimize/operations_api.cpp @@ -0,0 +1,95 @@ +#include "google/optimize/operations_api.h" + +#include "clients/url.h" +#include "google/errors.h" +#include "google/optimize/caps.h" + +namespace molo::google::optimize { + +using clients::body::Json; + +OnlineEvaluatorOperations::OnlineEvaluatorOperations(std::shared_ptr client, + CloudContext context) + : OptimizeService(std::move(client), endpoints::aiplatform(context.location)) { + (void)context; +} + +bool OnlineEvaluatorOperations::is_operation_name(const std::string& name) { + // projects/p/locations/l/onlineEvaluators/id/operations/op + if (!is_resource_of(name.substr(0, name.rfind("/operations/")), "onlineEvaluators")) return false; + std::size_t at = name.rfind("/operations/"); + return at != std::string::npos && valid_segment(name.substr(at + 12)); +} + +namespace { +void require_operation(const std::string& name) { + if (!OnlineEvaluatorOperations::is_operation_name(name)) + throw GoogleError::invalid( + "operation must be projects/p/locations/l/onlineEvaluators/id/operations/op"); +} +} // namespace + +Json OnlineEvaluatorOperations::get(const std::string& name) const { + require_operation(name); + return invoke(call_spec(rpc::kOnlineEvaluatorOpGet), {{"name", name}}); +} + +OperationPage OnlineEvaluatorOperations::list(const std::string& evaluator_name, + const std::string& filter, int page_size) const { + if (!is_resource_of(evaluator_name, "onlineEvaluators")) + throw GoogleError::invalid("evaluator must be projects/p/locations/l/onlineEvaluators/id"); + if (filter.size() > 1000) throw GoogleError::invalid("filter over 1000 characters"); + if (page_size < 0 || page_size > 100) throw GoogleError::invalid("page_size must be 0..100"); + const CallSpec& spec = call_spec(rpc::kOnlineEvaluatorOpList); + OperationPage out; + std::string token; + for (std::size_t page = 0;; ++page) { + Json reply = invoke(spec, {{"name", evaluator_name}}, + {{"filter", filter}, + {"pageSize", page_size > 0 ? std::to_string(page_size) : ""}, + {"pageToken", token}}); + if (const Json* arr = reply.is_object() ? reply.find("operations") : nullptr; + arr && arr->is_array()) + for (const Json& o : arr->as_array()) { + if (out.operations.size() >= OptimizeCaps::kMaxListItems) { + out.truncated = true; + return out; + } + out.operations.push_back(Operation::from_json(o)); + } + const Json* next = reply.is_object() ? reply.find("nextPageToken") : nullptr; + token = next && next->is_string() ? next->as_string() : ""; + if (token.empty()) return out; + if (page + 1 >= OptimizeCaps::kMaxPages) { + out.truncated = true; + out.next_page_token = token; + return out; + } + } +} + +void OnlineEvaluatorOperations::cancel(const std::string& name) const { + require_operation(name); + invoke(call_spec(rpc::kOnlineEvaluatorOpCancel), {{"name", name}}); +} + +void OnlineEvaluatorOperations::remove(const std::string& name) const { + require_operation(name); + invoke(call_spec(rpc::kOnlineEvaluatorOpDelete), {{"name", name}}); +} + +Operation OnlineEvaluatorOperations::wait_server(const std::string& name, + int timeout_seconds) const { + require_operation(name); + if (timeout_seconds < 1 || timeout_seconds > OptimizeCaps::kMaxServerWaitSeconds) + throw GoogleError::invalid("timeout_seconds must be 1.." + + std::to_string(OptimizeCaps::kMaxServerWaitSeconds)); + Operation op = Operation::from_json(invoke(call_spec(rpc::kOnlineEvaluatorOpWait), + {{"name", name}}, + {{"timeout", std::to_string(timeout_seconds) + "s"}})); + if (op.done && op.failed()) + throw GoogleError(502, "OPERATION_FAILED", op.error_message, endpoint_.name); + return op; +} + +} // namespace molo::google::optimize diff --git a/src/google/optimize/operations_api.h b/src/google/optimize/operations_api.h new file mode 100644 index 0000000..6df5244 --- /dev/null +++ b/src/google/optimize/operations_api.h @@ -0,0 +1,42 @@ +#pragma once + +// Operations of online evaluators (aiplatform v1 +// projects.locations.onlineEvaluators.operations: get, list, cancel, delete, +// wait). An online evaluator's activate, suspend and create calls return +// long-running operations that live under the evaluator. + +#include +#include + +#include "google/operation.h" +#include "google/optimize/optimize_service.h" +#include "google/resource.h" + +namespace molo::google::optimize { + +struct OperationPage { + std::vector operations; + bool truncated = false; // More existed past the page/item caps. + std::string next_page_token; // Continues a truncated list. +}; + +class OnlineEvaluatorOperations : public OptimizeService { + public: + OnlineEvaluatorOperations(std::shared_ptr client, CloudContext context); + + // Names: projects/{p}/locations/{l}/onlineEvaluators/{id}/operations/{op}. + static bool is_operation_name(const std::string& name); + + clients::body::Json get(const std::string& operation_name) const; + // Operations under projects/{p}/locations/{l}/onlineEvaluators/{id}. + OperationPage list(const std::string& evaluator_name, const std::string& filter = "", + int page_size = 0) const; + void cancel(const std::string& operation_name) const; + // Destructive: removes the operation record only. + void remove(const std::string& operation_name) const; + // One server-side wait, 1..OptimizeCaps::kMaxServerWaitSeconds; the reply + // may be not done. A failed operation throws. + Operation wait_server(const std::string& operation_name, int timeout_seconds) const; +}; + +} // namespace molo::google::optimize diff --git a/src/google/optimize/optimize_service.cpp b/src/google/optimize/optimize_service.cpp index 98a3356..7851bb9 100644 --- a/src/google/optimize/optimize_service.cpp +++ b/src/google/optimize/optimize_service.cpp @@ -6,18 +6,18 @@ namespace molo::google::optimize { using clients::body::Json; -Json OptimizeService::invoke(const CallSpec& spec, - const std::map& path_params, - const std::map& query, - const Json* body) const { +Json OptimizeService::invoke_on(const ServiceEndpoint& endpoint, const CallSpec& spec, + const std::map& path_params, + const std::map& query, + const Json* body) const { std::string path = with_query(spec, expand_path(spec, path_params), query); if (body) check_body(spec, *body); else if (std::string(spec.http) == "POST") { // A POST without fields (cancel, activate, suspend) sends {}. Json empty = Json::object(); - return client_->request(spec.method(), endpoint_, path, &empty); + return client_->request(spec.method(), endpoint, path, &empty); } - return client_->request(spec.method(), endpoint_, path, body); + return client_->request(spec.method(), endpoint, path, body); } } // namespace molo::google::optimize diff --git a/src/google/optimize/optimize_service.h b/src/google/optimize/optimize_service.h index 7b3f339..a8d2c57 100644 --- a/src/google/optimize/optimize_service.h +++ b/src/google/optimize/optimize_service.h @@ -22,7 +22,14 @@ class OptimizeService : public Service { clients::body::Json invoke(const CallSpec& spec, const std::map& path_params, const std::map& query = {}, - const clients::body::Json* body = nullptr) const; + const clients::body::Json* body = nullptr) const { + return invoke_on(endpoint_, spec, path_params, query, body); + } + // Same, on another endpoint (a service with calls in two versions). + clients::body::Json invoke_on(const ServiceEndpoint& endpoint, const CallSpec& spec, + const std::map& path_params, + const std::map& query = {}, + const clients::body::Json* body = nullptr) const; }; } // namespace molo::google::optimize diff --git a/src/google/optimize/telemetry_writer.cpp b/src/google/optimize/telemetry_writer.cpp new file mode 100644 index 0000000..6f93cef --- /dev/null +++ b/src/google/optimize/telemetry_writer.cpp @@ -0,0 +1,100 @@ +#include "google/optimize/telemetry_writer.h" + +#include "google/errors.h" +#include "google/optimize/caps.h" +#include "google/resource.h" + +namespace molo::google::optimize { + +using clients::body::Json; + +namespace { + +void require_project(const std::string& project) { + CloudContext c; + c.project = project; + if (!c.valid()) throw GoogleError::invalid("bad project id"); +} + +void require_object(const Json& body, const char* what) { + if (!body.is_object() || body.as_object().empty()) + throw GoogleError::invalid(std::string(what) + " must be a non-empty JSON object"); + if (body.dump().size() > OptimizeCaps::kMaxWriteBytes) + throw GoogleError::invalid(std::string(what) + " is over " + + std::to_string(OptimizeCaps::kMaxWriteBytes) + " bytes"); +} + +std::size_t array_size(const Json& body, const char* key) { + const Json* arr = body.is_object() ? body.find(key) : nullptr; + return arr && arr->is_array() ? arr->as_array().size() : 0; +} + +} // namespace + +TraceWriter::TraceWriter(std::shared_ptr client) + : OptimizeService(std::move(client), + endpoints::custom("cloudtrace", "https://cloudtrace.googleapis.com/v1")), + v2_(endpoints::custom("cloudtrace-v2", "https://cloudtrace.googleapis.com/v2")) {} + +void TraceWriter::patch_traces(const std::string& project, const Json& traces) const { + require_project(project); + require_object(traces, "traces request"); + const Json* inner = traces.find("traces"); + std::size_t n = inner && inner->is_object() ? array_size(*inner, "traces") : 0; + if (n < 1 || n > OptimizeCaps::kMaxTraces) + throw GoogleError::invalid("traces.traces needs 1.." + + std::to_string(OptimizeCaps::kMaxTraces) + " traces"); + invoke(call_spec(rpc::kTracePatch), {{"projectId", project}}, {}, &traces); +} + +void TraceWriter::batch_write(const std::string& project, const Json& spans) const { + require_project(project); + require_object(spans, "spans request"); + std::size_t n = array_size(spans, "spans"); + if (n < 1 || n > OptimizeCaps::kMaxSpansPerWrite) + throw GoogleError::invalid("spans needs 1.." + std::to_string(OptimizeCaps::kMaxSpansPerWrite) + + " entries"); + invoke_on(v2_, call_spec(rpc::kTraceBatchWrite), {{"name", "projects/" + project}}, {}, &spans); +} + +Json TraceWriter::create_span(const std::string& name, const Json& span) const { + // projects/{p}/traces/{trace}/spans/{span} + std::size_t t = name.find("/traces/"), s = name.find("/spans/"); + if (name.rfind("projects/", 0) != 0 || t == std::string::npos || s == std::string::npos || s < t) + throw GoogleError::invalid("span name must be projects/{p}/traces/{trace}/spans/{span}"); + require_project(name.substr(9, t - 9)); + if (!valid_segment(name.substr(t + 8, s - t - 8)) || !valid_segment(name.substr(s + 7))) + throw GoogleError::invalid("span name has an invalid trace or span id"); + require_object(span, "span"); + const Json* own = span.find("name"); + if (!own || !own->is_string() || own->as_string() != name) + throw GoogleError::invalid("span.name must equal the span name in the path"); + return invoke_on(v2_, call_spec(rpc::kTraceCreateSpan), {{"name", name}}, {}, &span); +} + +LogWriter::LogWriter(std::shared_ptr client) + : OptimizeService(std::move(client), + endpoints::custom("logging", "https://logging.googleapis.com/v2")) {} + +Json LogWriter::write(const Json& request) const { + require_object(request, "log write request"); + std::size_t n = array_size(request, "entries"); + if (n < 1 || n > OptimizeCaps::kMaxWriteEntries) + throw GoogleError::invalid("entries needs 1.." + std::to_string(OptimizeCaps::kMaxWriteEntries) + + " items"); + return invoke(call_spec(rpc::kLogEntriesWrite), {}, {}, &request); +} + +Operation LogWriter::copy(const Json& request) const { + require_object(request, "log copy request"); + for (const char* k : {"name", "destination"}) { + const Json* v = request.find(k); + if (!v || !v->is_string() || v->as_string().empty() || v->as_string().size() > 1024) + throw GoogleError::invalid(std::string("copy needs a string ") + k + " (max 1024)"); + } + if (const Json* f = request.find("filter"); f && f->is_string() && f->as_string().size() > OptimizeCaps::kMaxFilterChars) + throw GoogleError::invalid("filter over the character cap"); + return Operation::from_json(invoke(call_spec(rpc::kLogEntriesCopy), {}, {}, &request)); +} + +} // namespace molo::google::optimize diff --git a/src/google/optimize/telemetry_writer.h b/src/google/optimize/telemetry_writer.h new file mode 100644 index 0000000..fea120b --- /dev/null +++ b/src/google/optimize/telemetry_writer.h @@ -0,0 +1,53 @@ +#pragma once + +// Write side of Agent Observability, for tests and tools that must push a +// span or a log entry themselves (agents normally write through their OTel +// exporter). Every method mutates Google state and is explicit; bodies are +// raw Json whose top-level fields are checked against the call table and +// whose size is capped. +// +// - TraceWriter: Cloud Trace v1 patchTraces; v2 batchWrite and createSpan. +// - LogWriter: Cloud Logging v2 entries.write (dryRun available) and +// entries.copy (long-running). +// entries.tail is not wrapped: it is an endless server stream (see SPEC). + +#include + +#include "google/operation.h" +#include "google/optimize/optimize_service.h" + +namespace molo::google::optimize { + +class TraceWriter : public OptimizeService { + public: + explicit TraceWriter(std::shared_ptr client); + + // PATCH v1 projects/{project}/traces. `traces` is {"traces":[Trace,...]} + // (PatchTracesRequest.traces), at most OptimizeCaps::kMaxTraces traces. + void patch_traces(const std::string& project, const clients::body::Json& traces) const; + // POST v2 projects/{project}/traces:batchWrite with {"spans":[Span,...]}, + // 1..OptimizeCaps::kMaxSpansPerWrite spans. + void batch_write(const std::string& project, const clients::body::Json& spans) const; + // POST v2 {name}: name is projects/{p}/traces/{traceId}/spans/{spanId}; + // `span` is the Span body (its `name` must equal `name`). + clients::body::Json create_span(const std::string& name, const clients::body::Json& span) const; + + private: + ServiceEndpoint v2_; +}; + +class LogWriter : public OptimizeService { + public: + explicit LogWriter(std::shared_ptr client); + + // entries:write. `request` carries logName, resource, labels, entries, + // partialSuccess, dryRun. 1..kMaxWriteEntries entries. Prefer dryRun=true + // first: nothing is stored. + clients::body::Json write(const clients::body::Json& request) const; + // entries:copy: copies matching entries to a Cloud Storage destination. + // `request` carries name (source bucket view), filter, destination. + // Returns the long-running operation (not waited). + Operation copy(const clients::body::Json& request) const; +}; + +} // namespace molo::google::optimize diff --git a/src/google/optimize/tools/check_discovery.py b/src/google/optimize/tools/check_discovery.py index 6a3a2ce..4946849 100644 --- a/src/google/optimize/tools/check_discovery.py +++ b/src/google/optimize/tools/check_discovery.py @@ -111,6 +111,21 @@ ENUM_VALUES = [ ] +# Coverage: every Discovery method with one of these id prefixes must be a +# table row or be listed in EXCLUDED with its reason. +COVERED = [ + ('aiplatform', 'v1', ['aiplatform.projects.locations.evaluat', + 'aiplatform.projects.locations.generate', + 'aiplatform.projects.locations.onlineEvaluators']), + ('cloudtrace', 'v1', ['']), + ('cloudtrace', 'v2', ['']), + ('logging', 'v2', ['logging.entries']), +] +EXCLUDED = { + 'logging.entries.tail': 'server-streaming RPC with no end; this module has no unbounded stream (hard caps rule)', +} + + def fetch(url): with urllib.request.urlopen(url, timeout=60) as r: return json.load(r) @@ -156,6 +171,12 @@ def main(): want_query = sorted(k for k, v in m.get('parameters', {}).items() if v['location'] == 'query') if sorted(x for x in query_csv.split(',') if x) != want_query: errors.append('%s: query params differ from discovery' % rpc_id) + declared = {(r[3], r[4], r[0]) for r in rows} + for svc, ver, prefixes in COVERED: + for m in methods(docs[(svc, ver)]): + if any(m['id'].startswith(p) for p in prefixes) and (svc, ver, m['id']) not in declared \ + and m['id'] not in EXCLUDED: + errors.append('discovery method %s is not in the table' % m['id']) for svc, ver, schema, fields in SCHEMA_FIELDS: s = docs[(svc, ver)]['schemas'].get(schema) if s is None: diff --git a/src/google/registry/SPEC.md b/src/google/registry/SPEC.md index 267e95b..00f2869 100644 --- a/src/google/registry/SPEC.md +++ b/src/google/registry/SPEC.md @@ -14,8 +14,8 @@ Foundation used: `GoogleClient`, `Service`, `GoogleError`, `CloudContext`, |---|---| | `caps.h` | Static ceilings | | `ids.h/.cpp` | `Ids`: validation of ids, asset names, template names, request ids, free text | -| `calls.h/.cpp` | `Calls` and THE call table (86 rows): method, path, discovery service/version, rpc id, request schema, query params, request fields | -| `call_runner.h/.cpp` | `CallRunner`: renders and validates a table call, paging, mutation + operation wait, `update_mask` | +| `calls.h/.cpp` | `Calls` and THE call table (128 rows): method, path, discovery service/version, rpc id, request schema, query params, request fields | +| `call_runner.h/.cpp` | `CallRunner`: renders and validates a table call, paging (`ListPager`, `ListResult`), mutation + operation wait, `update_mask` | | `operation_runner.h/.cpp` | `OperationRunner`: finish a long-running operation, 404 existence probe | | `registry_service.h/.cpp` | `RegistryService : Service`: base of the four services | | `service_record.h/.cpp` | `ServiceRecord`, `Interface`: body of `services.create/patch` | @@ -34,7 +34,7 @@ Foundation used: `GoogleClient`, `Service`, `GoogleError`, `CloudContext`, | Knob | Value | |---|---| -| List page size / pages / items | 100 / 10 / 1000 per list call | +| List page size / pages / items | 100 / 10 / 1000 per list call; past a cap the result says `truncated` and carries the next page token (see 3) | | displayName / skill displayName / description | 63 / 128 / 2048 (Google's limits) | | Spec content | 10 KiB (Google's limit) | | Service id | 4..63 of `[a-z0-9-]`, starts with a letter | @@ -57,6 +57,17 @@ Foundation used: `GoogleClient`, `Service`, `GoogleError`, `CloudContext`, call, `pageSize` outside 1..100, a zero or malformed `requestId`, a body on a bodiless call, a missing body, or a top-level body field not declared for the call. +- Lists never truncate silently. Every list method returns `ListResult` + (`items`, `truncated`, `next_page_token`, `pages`, `to_json()`); `CallRunner` + pages with `ListPager` up to 10 pages / 1000 items and sets `truncated` (and + the continuation token) when the server still had more. A caller-supplied + `pageToken` is refused: the runner owns paging. `list_approved` keeps the + flag. The demo routes put `count`, `pages`, `truncated`, `next_page_token` + and `cap_items` in every list answer and the page shows a TRUNCATED banner. +- Two API versions. Discovery lists every registry resource in v1 and in + v1alpha (skills, revisions and publishers in v1alpha only). Both are in the + call table (`agentregistry.v1.*` and `agentregistry.v1alpha.*`); + `RegistryVersion` picks the rows for `AgentRegistry`, v1 by default. - Read before mutate: `register_service`, `create_skill`, `create_binding`, `create` (gateway), `create_policy`, `create_deny_policy`, `create_boundary` GET first and answer 409 `ALREADY_EXISTS` for an existing id. Update and @@ -97,9 +108,11 @@ Foundation used: `GoogleClient`, `Service`, `GoogleError`, `CloudContext`, GET (no credential needed): `/agent/registry` page, `/status`, `/calls`, `/approvals`, `/audit`. POST: `/list`, `/get`, `/search`, `/approve`, `/deprecate`, `/reject`, `/skills/state`, `/gateways/list`, `/gateways/get`, -`/policies/list`, `/identity`, `/identity/audit` (503 `google_not_configured` -without a credential, 422 on bad input) and `/identity/principal` (local, no -credential). DELETE: `/services/{id}`, `/gateways/{id}`. `/approve` and +`/policies/list`, `/identity`, `/identity/audit`, `/operations/list`, `/operations/cancel`, `/deny-policies/list`, +`/boundaries/bindings` (503 `google_not_configured` without a credential, 422 +on bad input) and `/identity/principal` (local, no credential). `/list` takes +`version` (`v1` or `v1alpha`). DELETE: `/services/{id}`, `/gateways/{id}`, +`/operations?scope=registry|gateways|policies&id=` (the operation record only). `/approve` and `/deprecate` read the asset from Google first. ## Sources @@ -119,8 +132,15 @@ credential). DELETE: `/services/{id}`, `/gateways/{id}`. `/approve` and ## Verified against discovery -`python3 -I src/google/registry/tools/check_discovery.py`, run 2026-10-24: -`checked 86 calls across 8 discovery documents` / `OK`. For every row it +`python3 -I src/google/registry/tools/check_discovery.py`, run 2026-10-24 +(gap-closing round): `checked 128 calls across 8 discovery documents, 0 +discovery methods unwrapped` / `OK`. Coverage is enforced: every discovery +method of agentregistry v1 and v1alpha, networkservices +`agentGateways` and `operations`, networksecurity `authzPolicies` and +`operations`, iam v2 (deny policies) and iam v3 `principalAccessBoundaryPolicies` +must be a table row or be listed in `EXCLUDED` (empty). aiplatform and +logging are deliberately narrow subsets (reasoningEngines get/list/IAM, `entries:list`) +and are not covered. For every row it confirms: rpc method id exists, HTTP method, path (`version + path`), the exact set of query parameters, the request schema name, and that every declared request field is a property of that schema. Discovery findings that @@ -147,6 +167,15 @@ resource created, changed or deleted. Rerun: `MOLO_LIVE_NET=1` plus | networkservices v1 `agentGateways` list | 401 UNAUTHENTICATED | 403 SERVICE_DISABLED: Network Services API not enabled | | networksecurity v1 `authzPolicies` list | 401 UNAUTHENTICATED | 403 SERVICE_DISABLED: Network Security API not enabled | +Re-probed 2026-10-24 (gap-closing round, service-account token, read-only): +agentregistry v1 `agents` and v1alpha `operations`, networkservices +`agentGateways` and `operations`, networksecurity `authzPolicies`, and iam v2 +`denypolicies` (project number 474586702855) all still answer 403 +PERMISSION_DENIED "... has not been used in project ... or it is disabled". +The new v1alpha, operations and deny policy reads in +`registry_live.v1alpha_operations_and_deny_policy_reads` are therefore +Discovery-verified only, with a loud opt-in test ready. + What this proves: hosts, versions and collection paths exist (a wrong path would be 404, not an auth or enablement error) and registry/gateway/policy APIs need OAuth. What is NOT live-verified: any registry, gateway or policy @@ -177,7 +206,9 @@ Neither discovery nor the docs state these; each is a choice made here: beyond the shared schema shape. - The deny-policy permission and principal string checks are stricter than Google's (shape checks only). -- 1000-item list ceiling: a larger registry is silently truncated at the cap. +- v1alpha rows were generated from the v1alpha discovery document, keeping the v1 field order where a v1 twin exists. Their record builders (`ServiceRecord`, `BindingRecord`) emit v1 shapes; a field that v1alpha lacks fails loudly (`field ... is not declared`), a v1alpha-only field cannot be sent. No v1alpha body was ever sent. +- `iam.v2.policies.update` (PUT) needs the policy `etag`: `update_deny_policy` reads it first. Whether IAM accepts an empty etag is unread. +- `searchPolicyBindings` (iam v3) is listed under key `policyBindings` from the response schema; no live reply was seen (organization scope, no org access). ## Gaps diff --git a/src/google/registry/agent_gateway.cpp b/src/google/registry/agent_gateway.cpp index 050f2bf..50387b8 100644 --- a/src/google/registry/agent_gateway.cpp +++ b/src/google/registry/agent_gateway.cpp @@ -101,7 +101,7 @@ ServiceEndpoint AgentGateway::default_endpoint() { AgentGateway::AgentGateway(std::shared_ptr client, CloudContext context) : RegistryService(std::move(client), default_endpoint(), std::move(context)) {} -Json AgentGateway::list() const { +ListResult AgentGateway::list() const { return runner_.list("networkservices.v1.agentGateways.list", "agentGateways", with("parent", context_.parent())); } @@ -148,4 +148,22 @@ Json AgentGateway::operation(const std::string& operation_id) const { with("name", context_.parent() + "/operations/" + operation_id)); } +ListResult AgentGateway::operations() const { + return runner_.list("networkservices.v1.operations.list", "operations", + with("name", context_.parent())); +} + +Json AgentGateway::cancel_operation(const std::string& operation_id) const { + operation(operation_id); // Read first: a missing operation is a clear 404. + CallArgs a = with("name", context_.parent() + "/operations/" + operation_id); + a.body = Json::object(); + return runner_.invoke("networkservices.v1.operations.cancel", a); +} + +Json AgentGateway::delete_operation(const std::string& operation_id) const { + operation(operation_id); + return runner_.invoke("networkservices.v1.operations.delete", + with("name", context_.parent() + "/operations/" + operation_id)); +} + } // namespace molo::google::registry diff --git a/src/google/registry/agent_gateway.h b/src/google/registry/agent_gateway.h index 0aedcd2..dd1b3ad 100644 --- a/src/google/registry/agent_gateway.h +++ b/src/google/registry/agent_gateway.h @@ -41,7 +41,8 @@ class AgentGateway : public RegistryService { static ServiceEndpoint default_endpoint(); // networkservices .../v1 // Reads first. - clients::body::Json list() const; + // Paged up to the registry caps; ListResult::truncated is loud. + ListResult list() const; clients::body::Json get(const std::string& id) const; // Mutations are explicit and read first: create refuses an existing id, @@ -56,7 +57,13 @@ class AgentGateway : public RegistryService { // `etag` guards against a concurrent change; empty sends none. clients::body::Json remove_gateway(const std::string& id, const std::string& etag = "") const; + // ---- Operations of the networkservices location (get, list, cancel, + // delete). cancel and delete read the operation first. clients::body::Json operation(const std::string& operation_id) const; + ListResult operations() const; + clients::body::Json cancel_operation(const std::string& operation_id) const; + // Destructive: removes the operation record, never the gateway. + clients::body::Json delete_operation(const std::string& operation_id) const; }; } // namespace molo::google::registry diff --git a/src/google/registry/agent_identity.cpp b/src/google/registry/agent_identity.cpp index 438df85..fa940e9 100644 --- a/src/google/registry/agent_identity.cpp +++ b/src/google/registry/agent_identity.cpp @@ -74,11 +74,13 @@ Json DenyPolicySpec::to_json() const { Ids::require_text(display_name, RegistryCaps::kMaxDisplayName, "displayName"); if (rules.empty() || rules.size() > RegistryCaps::kMaxPolicyRules) throw GoogleError::invalid("a deny policy needs 1..5 rules"); + Ids::require_text(etag, 256, "etag"); Json j = Json::object(); if (!display_name.empty()) j.set("displayName", Json(display_name)); Json r = Json::array(); for (const auto& rule : rules) r.push(rule.to_json()); j.set("rules", r); + if (!etag.empty()) j.set("etag", Json(etag)); return j; } @@ -113,7 +115,7 @@ ServiceEndpoint AgentIdentity::default_endpoint(const std::string& location) { AgentIdentity::AgentIdentity(std::shared_ptr client, CloudContext context) : RegistryService(std::move(client), default_endpoint(context.location), context) {} -Json AgentIdentity::engines() const { +ListResult AgentIdentity::engines() const { return runner_.list("aiplatform.v1.reasoningEngines.list", "reasoningEngines", with("parent", context_.parent())); } @@ -185,7 +187,7 @@ std::string AgentIdentity::boundary_parent(const std::string& org_id) { return "organizations/" + org_id + "/locations/global"; } -Json AgentIdentity::deny_policies(const std::string& attachment) const { +ListResult AgentIdentity::deny_policies(const std::string& attachment) const { return runner_.list("iam.v2.policies.listPolicies", "policies", with("parent", policy_parent(attachment))); } @@ -214,7 +216,7 @@ Json AgentIdentity::delete_deny_policy(const std::string& attachment, const std: return runner_.mutate("iam.v2.policies.delete", a); } -Json AgentIdentity::boundaries(const std::string& org_id) const { +ListResult AgentIdentity::boundaries(const std::string& org_id) const { return runner_.list("iam.v3.principalAccessBoundaryPolicies.list", "principalAccessBoundaryPolicies", with("parent", boundary_parent(org_id))); } @@ -243,4 +245,52 @@ Json AgentIdentity::delete_boundary(const std::string& org_id, const std::string with("name", boundary_parent(org_id) + "/principalAccessBoundaryPolicies/" + id)); } +Json AgentIdentity::update_deny_policy(const std::string& attachment, const std::string& id, + const DenyPolicySpec& spec) const { + Ids::require_segment(id, "policy id"); + Json current = deny_policy(attachment, id); // Must exist; supplies the etag. + DenyPolicySpec body = spec; + if (body.etag.empty()) body.etag = string_at(current, "etag"); + CallArgs a = with("name", policy_name(attachment, id)); + a.body = body.to_json(); + runner_.mutate("iam.v2.policies.update", a); + return deny_policy(attachment, id); +} + +Json AgentIdentity::deny_policy_operation(const std::string& operation_name) const { + // The name comes from a mutation reply: policies/.../operations/{id}. + if (operation_name.rfind("policies/", 0) != 0 || + operation_name.find("/operations/") == std::string::npos) + throw GoogleError::invalid("operation name must look like policies/.../operations/{id}"); + return runner_.invoke("iam.v2.policies.operations.get", with("name", operation_name)); +} + +Json AgentIdentity::engine_operation(const std::string& operation_id) const { + Ids::require_segment(operation_id, "operation id"); + return runner_.invoke("aiplatform.v1.operations.get", + with("name", context_.parent() + "/operations/" + operation_id)); +} + +Json AgentIdentity::update_boundary(const std::string& org_id, const std::string& id, + const AccessBoundarySpec& spec, + const std::vector& mask) const { + Ids::require_segment(id, "policy id"); + CallArgs a = with("name", boundary_parent(org_id) + "/principalAccessBoundaryPolicies/" + id); + a.body = spec.to_json(); + a.query["updateMask"] = + CallRunner::update_mask("iam.v3.principalAccessBoundaryPolicies.patch", mask); + boundary(org_id, id); // Must exist (checked after the input, before the write). + runner_.mutate("iam.v3.principalAccessBoundaryPolicies.patch", a); + return boundary(org_id, id); +} + +ListResult AgentIdentity::boundary_bindings(const std::string& org_id, + const std::string& id) const { + Ids::require_segment(id, "policy id"); + // searchPolicyBindings: the policy bindings that attach this boundary. + return runner_.list("iam.v3.principalAccessBoundaryPolicies.searchPolicyBindings", + "policyBindings", + with("name", boundary_parent(org_id) + "/principalAccessBoundaryPolicies/" + id)); +} + } // namespace molo::google::registry diff --git a/src/google/registry/agent_identity.h b/src/google/registry/agent_identity.h index 67ed73a..70a578d 100644 --- a/src/google/registry/agent_identity.h +++ b/src/google/registry/agent_identity.h @@ -43,6 +43,7 @@ struct DenyRuleSpec { struct DenyPolicySpec { std::string display_name; + std::string etag; // update only; empty: the current etag is read and used std::vector rules; // At least one. clients::body::Json to_json() const; @@ -63,7 +64,7 @@ class AgentIdentity : public RegistryService { static ServiceEndpoint default_endpoint(const std::string& location); // ---- Reading identities (aiplatform v1; an API key reaches these). - clients::body::Json engines() const; + ListResult engines() const; IdentityInfo identity(const std::string& engine_id) const; clients::body::Json engine_iam_policy(const std::string& engine_id) const; @@ -74,19 +75,34 @@ class AgentIdentity : public RegistryService { // point, e.g. attachment_for_project("123"). static std::string attachment_for_project(const std::string& project_number); static std::string attachment_for_organization(const std::string& org_id); - clients::body::Json deny_policies(const std::string& attachment) const; + ListResult deny_policies(const std::string& attachment) const; clients::body::Json deny_policy(const std::string& attachment, const std::string& id) const; clients::body::Json create_deny_policy(const std::string& attachment, const std::string& id, const DenyPolicySpec& spec) const; + // PUT: replaces displayName and rules. Reads the policy first (it must + // exist) and sends its etag unless spec.etag is set. + clients::body::Json update_deny_policy(const std::string& attachment, const std::string& id, + const DenyPolicySpec& spec) const; clients::body::Json delete_deny_policy(const std::string& attachment, const std::string& id, const std::string& etag = "") const; + // Long-running operation of the IAM v2 policies API. + // `operation_name` is policies/.../operations/{id}, as a mutation returns it. + clients::body::Json deny_policy_operation(const std::string& operation_name) const; + // Long-running operation of the aiplatform location (reasoning engines). + clients::body::Json engine_operation(const std::string& operation_id) const; // ---- Principal access boundary policies (iam v3, organization scope). - clients::body::Json boundaries(const std::string& org_id) const; + ListResult boundaries(const std::string& org_id) const; clients::body::Json boundary(const std::string& org_id, const std::string& id) const; clients::body::Json create_boundary(const std::string& org_id, const std::string& id, const AccessBoundarySpec& spec) const; + // PATCH: mask names AccessBoundary fields (displayName, details, ...). + clients::body::Json update_boundary(const std::string& org_id, const std::string& id, + const AccessBoundarySpec& spec, + const std::vector& mask) const; clients::body::Json delete_boundary(const std::string& org_id, const std::string& id) const; + // searchPolicyBindings: bindings that attach this boundary. + ListResult boundary_bindings(const std::string& org_id, const std::string& id) const; private: static std::string policy_parent(const std::string& attachment); diff --git a/src/google/registry/agent_registry.cpp b/src/google/registry/agent_registry.cpp index 2aece64..edbef03 100644 --- a/src/google/registry/agent_registry.cpp +++ b/src/google/registry/agent_registry.cpp @@ -33,18 +33,27 @@ bool asset_kind_from(const std::string& c, AssetKind& out) { return false; } -ServiceEndpoint AgentRegistry::default_endpoint() { - return endpoints::custom("agentregistry", "https://agentregistry.googleapis.com/v1"); +ServiceEndpoint AgentRegistry::default_endpoint(RegistryVersion version) { + return version == RegistryVersion::kV1Alpha + ? endpoints::custom("agentregistry-v1alpha", + "https://agentregistry.googleapis.com/v1alpha") + : endpoints::custom("agentregistry", "https://agentregistry.googleapis.com/v1"); } -AgentRegistry::AgentRegistry(std::shared_ptr client, CloudContext context) - : RegistryService(std::move(client), default_endpoint(), std::move(context)) {} +AgentRegistry::AgentRegistry(std::shared_ptr client, CloudContext context, + RegistryVersion version) + : RegistryService(std::move(client), default_endpoint(version), std::move(context)), + version_(version == RegistryVersion::kV1Alpha ? "v1alpha" : "v1") {} + +std::string AgentRegistry::rid(const std::string& tail) const { + return "agentregistry." + version_ + "." + tail; +} std::string AgentRegistry::call_id(AssetKind kind, const char* verb) const { // Skills (CRUD, search, revisions) and publishers exist in v1alpha only. bool alpha = kind == AssetKind::kSkills || kind == AssetKind::kPublishers; - return std::string("agentregistry.") + (alpha ? "v1alpha." : "v1.") + asset_collection(kind) + - "." + verb; + std::string base = alpha ? "agentregistry.v1alpha." : rid(""); + return base + asset_collection(kind) + "." + verb; } std::string AgentRegistry::name_of(AssetKind kind, const std::string& id) const { @@ -55,18 +64,18 @@ CallArgs AgentRegistry::parent_args() const { return with("parent", context_.par // ---- Reads -Json AgentRegistry::locations() const { - return runner_.list("agentregistry.v1.locations.list", "locations", +ListResult AgentRegistry::locations() const { + return runner_.list(rid("locations.list"), "locations", with("name", "projects/" + context_.project)); } Json AgentRegistry::location(const std::string& id) const { Ids::require_segment(id, "location id"); - return runner_.invoke("agentregistry.v1.locations.get", + return runner_.invoke(rid("locations.get"), with("name", "projects/" + context_.project + "/locations/" + id)); } -Json AgentRegistry::list(AssetKind kind, const std::map& query) const { +ListResult AgentRegistry::list(AssetKind kind, const std::map& query) const { CallArgs a = parent_args(); for (const auto& kv : query) { if (kv.first == "filter") Ids::require_text(kv.second, RegistryCaps::kMaxFilterChars, "filter"); @@ -124,8 +133,10 @@ Json AgentRegistry::tools(const std::string& mcp_server_id) const { return t && t->is_array() ? *t : Json::array(); } -Json AgentRegistry::list_approved(AssetKind kind, const ApprovalLedger& ledger) const { - return ledger.restrict_to_approved(list(kind)); +ListResult AgentRegistry::list_approved(AssetKind kind, const ApprovalLedger& ledger) const { + ListResult all = list(kind); + all.items = ledger.restrict_to_approved(all.items); // truncated stays as listed + return all; } // ---- Services @@ -140,7 +151,7 @@ Json AgentRegistry::register_service(const std::string& id, const ServiceRecord& put_query(a, "requestId", request_id); if (exists([&] { return get(AssetKind::kServices, id); })) throw GoogleError(409, "ALREADY_EXISTS", "service " + id + " is already registered"); - runner_.mutate("agentregistry.v1.services.create", a); + runner_.mutate(rid("services.create"), a); return get(AssetKind::kServices, id); } @@ -150,17 +161,17 @@ Json AgentRegistry::update_service(const std::string& id, const ServiceRecord& r Ids::require_service_id(id); CallArgs a = with("name", name_of(AssetKind::kServices, id)); a.body = record.to_json(); - a.query["updateMask"] = CallRunner::update_mask("agentregistry.v1.services.patch", mask); + a.query["updateMask"] = CallRunner::update_mask(rid("services.patch"), mask); put_query(a, "requestId", request_id); get(AssetKind::kServices, id); // Must exist. - runner_.mutate("agentregistry.v1.services.patch", a); + runner_.mutate(rid("services.patch"), a); return get(AssetKind::kServices, id); } Json AgentRegistry::remove_service(const std::string& id) const { Ids::require_service_id(id); get(AssetKind::kServices, id); - return runner_.mutate("agentregistry.v1.services.delete", + return runner_.mutate(rid("services.delete"), with("name", name_of(AssetKind::kServices, id))); } @@ -197,7 +208,7 @@ Json AgentRegistry::delete_skill(const std::string& id, bool force) const { return runner_.mutate("agentregistry.v1alpha.skills.delete", a); } -Json AgentRegistry::list_revisions(const std::string& skill_id) const { +ListResult AgentRegistry::list_revisions(const std::string& skill_id) const { Ids::require_segment(skill_id, "skill id"); return runner_.list("agentregistry.v1alpha.skills.revisions.list", "skillRevisions", with("parent", name_of(AssetKind::kSkills, skill_id))); @@ -237,7 +248,7 @@ Json AgentRegistry::create_binding(const std::string& id, const BindingRecord& b a.query["bindingId"] = id; if (exists([&] { return get(AssetKind::kBindings, id); })) throw GoogleError(409, "ALREADY_EXISTS", "binding " + id + " already exists"); - runner_.mutate("agentregistry.v1.bindings.create", a); + runner_.mutate(rid("bindings.create"), a); return get(AssetKind::kBindings, id); } @@ -246,20 +257,20 @@ Json AgentRegistry::update_binding(const std::string& id, const BindingRecord& b Ids::require_segment(id, "binding id"); CallArgs a = with("name", name_of(AssetKind::kBindings, id)); a.body = binding.to_json(); - a.query["updateMask"] = CallRunner::update_mask("agentregistry.v1.bindings.patch", mask); + a.query["updateMask"] = CallRunner::update_mask(rid("bindings.patch"), mask); get(AssetKind::kBindings, id); - runner_.mutate("agentregistry.v1.bindings.patch", a); + runner_.mutate(rid("bindings.patch"), a); return get(AssetKind::kBindings, id); } Json AgentRegistry::delete_binding(const std::string& id) const { Ids::require_segment(id, "binding id"); get(AssetKind::kBindings, id); - return runner_.mutate("agentregistry.v1.bindings.delete", + return runner_.mutate(rid("bindings.delete"), with("name", name_of(AssetKind::kBindings, id))); } -Json AgentRegistry::fetch_available_bindings(const std::string& source_id, +ListResult AgentRegistry::fetch_available_bindings(const std::string& source_id, const std::string& target_id) const { if (!source_id.empty() && !BindingRecord::valid_identifier(source_id, true)) throw GoogleError::invalid("source id must be urn:agent:{publisher}:{namespace}:{name}"); @@ -268,15 +279,15 @@ Json AgentRegistry::fetch_available_bindings(const std::string& source_id, CallArgs a = parent_args(); put_query(a, "sourceIdentifier", source_id); put_query(a, "targetIdentifier", target_id); - return runner_.list("agentregistry.v1.bindings.fetchAvailable", "bindings", a); + return runner_.list(rid("bindings.fetchAvailable"), "bindings", a); } // ---- AI applications -Json AgentRegistry::application_assets(const std::string& application_id, AssetKind kind) const { +ListResult AgentRegistry::application_assets(const std::string& application_id, AssetKind kind) const { if (kind != AssetKind::kAgents && kind != AssetKind::kMcpServers && kind != AssetKind::kEndpoints) throw GoogleError::invalid("applications hold agents, mcpServers and endpoints"); - return runner_.list(std::string("agentregistry.v1.aiApplications.") + asset_collection(kind) + ".list", + return runner_.list(rid("aiApplications.") + asset_collection(kind) + ".list", asset_collection(kind), with("parent", name_of(AssetKind::kAiApplications, application_id))); } @@ -285,12 +296,12 @@ Json AgentRegistry::application_assets(const std::string& application_id, AssetK Json AgentRegistry::operation(const std::string& id) const { Ids::require_segment(id, "operation id"); - return runner_.invoke("agentregistry.v1.operations.get", + return runner_.invoke(rid("operations.get"), with("name", context_.parent() + "/operations/" + id)); } -Json AgentRegistry::operations() const { - return runner_.list("agentregistry.v1.operations.list", "operations", +ListResult AgentRegistry::operations() const { + return runner_.list(rid("operations.list"), "operations", with("name", context_.parent())); } @@ -298,12 +309,12 @@ Json AgentRegistry::cancel_operation(const std::string& id) const { operation(id); CallArgs a = with("name", context_.parent() + "/operations/" + id); a.body = Json::object(); - return runner_.invoke("agentregistry.v1.operations.cancel", a); + return runner_.invoke(rid("operations.cancel"), a); } Json AgentRegistry::delete_operation(const std::string& id) const { operation(id); - return runner_.invoke("agentregistry.v1.operations.delete", + return runner_.invoke(rid("operations.delete"), with("name", context_.parent() + "/operations/" + id)); } @@ -318,8 +329,8 @@ void require_iam_kind(AssetKind k) { Json AgentRegistry::iam_policy(AssetKind kind, const std::string& id) const { require_iam_kind(kind); - std::string cid = kind == AssetKind::kSkills ? "agentregistry.v1.skills.getIamPolicy" - : "agentregistry.v1.aiApplications.getIamPolicy"; + std::string cid = kind == AssetKind::kSkills ? rid("skills.getIamPolicy") + : rid("aiApplications.getIamPolicy"); return runner_.invoke(cid, with("resource", name_of(kind, id))); } @@ -327,8 +338,8 @@ Json AgentRegistry::set_iam_policy(AssetKind kind, const std::string& id, const require_iam_kind(kind); if (!policy.is_object()) throw GoogleError::invalid("policy must be a JSON object"); iam_policy(kind, id); // Read first. - std::string cid = kind == AssetKind::kSkills ? "agentregistry.v1.skills.setIamPolicy" - : "agentregistry.v1.aiApplications.setIamPolicy"; + std::string cid = kind == AssetKind::kSkills ? rid("skills.setIamPolicy") + : rid("aiApplications.setIamPolicy"); CallArgs a = with("resource", name_of(kind, id)); a.body = Json::object(); a.body.set("policy", policy); @@ -340,8 +351,8 @@ Json AgentRegistry::test_iam_permissions(AssetKind kind, const std::string& id, require_iam_kind(kind); if (permissions.empty() || permissions.size() > RegistryCaps::kMaxPermissions) throw GoogleError::invalid("1..64 permissions"); - std::string cid = kind == AssetKind::kSkills ? "agentregistry.v1.skills.testIamPermissions" - : "agentregistry.v1.aiApplications.testIamPermissions"; + std::string cid = kind == AssetKind::kSkills ? rid("skills.testIamPermissions") + : rid("aiApplications.testIamPermissions"); CallArgs a = with("resource", name_of(kind, id)); a.body = Json::object(); Json arr = Json::array(); diff --git a/src/google/registry/agent_registry.h b/src/google/registry/agent_registry.h index db566f1..0f1ef1c 100644 --- a/src/google/registry/agent_registry.h +++ b/src/google/registry/agent_registry.h @@ -25,6 +25,11 @@ namespace molo::google::registry { +// Discovery lists every registry resource in v1 and in v1alpha (skills and +// publishers in v1alpha only). Both versions are in the call table; the +// version picks the rows. v1 is the default. +enum class RegistryVersion { kV1, kV1Alpha }; + enum class AssetKind { kAgents, kMcpServers, kEndpoints, kServices, kSkills, kBindings, kPublishers, kAiApplications }; const char* asset_collection(AssetKind kind); // "agents", "mcpServers", ... @@ -37,17 +42,22 @@ struct SearchPage { class AgentRegistry : public RegistryService { public: - AgentRegistry(std::shared_ptr client, CloudContext context); + AgentRegistry(std::shared_ptr client, CloudContext context, + RegistryVersion version = RegistryVersion::kV1); - static ServiceEndpoint default_endpoint(); // https://agentregistry.googleapis.com/v1 + // https://agentregistry.googleapis.com/v1 (or /v1alpha) + static ServiceEndpoint default_endpoint(RegistryVersion version = RegistryVersion::kV1); + const std::string& version() const { return version_; } // ---- Reads (always available before any mutation). - clients::body::Json locations() const; + ListResult locations() const; clients::body::Json location(const std::string& id) const; // `query` holds optional filter / orderBy, accepted only where the // discovery document lists them for that collection. - clients::body::Json list(AssetKind kind, - const std::map& query = {}) const; + // Paged up to RegistryCaps::kMaxPages / kMaxItems; ListResult::truncated + // says loudly when more existed. + ListResult list(AssetKind kind, + const std::map& query = {}) const; clients::body::Json get(AssetKind kind, const std::string& id) const; // agents:search and mcpServers:search (POST body), one page per call. SearchPage search(AssetKind kind, const std::string& query, int page_size = 20, @@ -56,7 +66,7 @@ class AgentRegistry : public RegistryService { SearchPage search_skills(const std::string& query, const std::string& search_type = "", int page_size = 20, const std::string& page_token = "") const; clients::body::Json tools(const std::string& mcp_server_id) const; // McpServer.tools - clients::body::Json list_approved(AssetKind kind, const ApprovalLedger& ledger) const; + ListResult list_approved(AssetKind kind, const ApprovalLedger& ledger) const; // ---- Services (agents, MCP servers, endpoints are registered through // these). Each mutation reads first and waits for the operation. @@ -74,7 +84,7 @@ class AgentRegistry : public RegistryService { clients::body::Json set_skill_state(const std::string& id, SkillState state) const; // force=false unless the caller says otherwise; never defaulted to true. clients::body::Json delete_skill(const std::string& id, bool force = false) const; - clients::body::Json list_revisions(const std::string& skill_id) const; + ListResult list_revisions(const std::string& skill_id) const; clients::body::Json get_revision(const std::string& skill_id, const std::string& revision) const; clients::body::Json create_revision(const std::string& skill_id, const std::string& revision_id, const SkillRevisionRecord& revision) const; @@ -85,16 +95,15 @@ class AgentRegistry : public RegistryService { clients::body::Json update_binding(const std::string& id, const BindingRecord& binding, const std::vector& mask) const; clients::body::Json delete_binding(const std::string& id) const; - clients::body::Json fetch_available_bindings(const std::string& source_id, - const std::string& target_id) const; + ListResult fetch_available_bindings(const std::string& source_id, + const std::string& target_id) const; // ---- AI applications (read only) and their agents, servers, endpoints. - clients::body::Json application_assets(const std::string& application_id, - AssetKind kind) const; + ListResult application_assets(const std::string& application_id, AssetKind kind) const; // ---- Operations (the registry's own long-running operations). clients::body::Json operation(const std::string& operation_id) const; - clients::body::Json operations() const; + ListResult operations() const; clients::body::Json cancel_operation(const std::string& operation_id) const; clients::body::Json delete_operation(const std::string& operation_id) const; @@ -107,8 +116,12 @@ class AgentRegistry : public RegistryService { private: std::string call_id(AssetKind kind, const char* verb) const; + // "agentregistry..": the table id for the chosen version. + std::string rid(const std::string& tail) const; std::string name_of(AssetKind kind, const std::string& id) const; CallArgs parent_args() const; + + std::string version_; }; } // namespace molo::google::registry diff --git a/src/google/registry/call_runner.cpp b/src/google/registry/call_runner.cpp index 8a82961..18fa228 100644 --- a/src/google/registry/call_runner.cpp +++ b/src/google/registry/call_runner.cpp @@ -89,14 +89,70 @@ Json CallRunner::invoke(const std::string& call_id, const CallArgs& args) const Calls::has_body(call) ? &args.body : nullptr); } -Json CallRunner::list(const std::string& call_id, const std::string& items_key, - const CallArgs& args) const { +Json ListResult::to_json() const { + Json out = Json::object(); + out.set("items", items); + out.set("count", Json(static_cast(size()))); + out.set("pages", Json(static_cast(pages))); + out.set("truncated", Json(truncated)); + out.set("next_page_token", Json(next_page_token)); + out.set("cap_items", Json(static_cast(RegistryCaps::kMaxItems))); + return out; +} + +ListPager::ListPager(std::string items_key, std::size_t max_pages, std::size_t max_items) + : items_key_(std::move(items_key)), max_pages_(max_pages), max_items_(max_items) {} + +bool ListPager::feed(const Json& page) { + ++result_.pages; + if (const Json* arr = page.is_object() ? page.find(items_key_) : nullptr; + arr && arr->is_array()) + for (const Json& item : arr->as_array()) { + if (result_.size() >= max_items_) { + result_.truncated = true; // This page alone overran the item cap. + break; + } + result_.items.push(item); + } + const Json* next = page.is_object() ? page.find("nextPageToken") : nullptr; + token_ = next && next->is_string() ? next->as_string() : ""; + if (result_.truncated) { + token_.clear(); // Mid-page overrun: no exact continuation exists. + return false; + } + if (token_.empty()) return false; + if (result_.pages >= max_pages_ || result_.size() >= max_items_) { + result_.truncated = true; + result_.next_page_token = token_; + return false; + } + return true; +} + +ListResult CallRunner::list(const std::string& call_id, const std::string& items_key, + const CallArgs& args) const { const CallSpec& call = Calls::get(call_id); if (Calls::method_of(call) != clients::Method::kGet) throw GoogleError::invalid("list needs a GET call"); + if (args.query.count("pageToken")) + throw GoogleError::invalid("pageToken is managed by the list call"); std::string path = build_path(call, args, true); - return list_all(*client_, Calls::endpoint(call, location_), path, items_key, - RegistryCaps::kMaxPages); + ServiceEndpoint ep = Calls::endpoint(call, location_); + bool pageable = Calls::allows_query(call, "pageToken"); + ListPager pager(items_key, RegistryCaps::kMaxPages, RegistryCaps::kMaxItems); + std::string url = path; + while (true) { + Json reply = client_->request(clients::Method::kGet, ep, url, nullptr); + bool more = pager.feed(reply); + if (!more || !pageable) { + ListResult out = pager.result(); + // A reply that names a next page the call cannot request is still loud. + if (!pageable && !pager.token().empty()) out.truncated = true; + return out; + } + url = path + (path.find('?') == std::string::npos ? '?' : '&') + + std::string("pageToken=") + clients::Url::encode_component(pager.token()); + } } Json CallRunner::mutate(const std::string& call_id, const CallArgs& args) const { diff --git a/src/google/registry/call_runner.h b/src/google/registry/call_runner.h index f5c010a..76616b5 100644 --- a/src/google/registry/call_runner.h +++ b/src/google/registry/call_runner.h @@ -16,6 +16,41 @@ namespace molo::google::registry { +// Result of a paged list. A list never truncates silently: when pages or +// items remain after the caps, `truncated` is true and `next_page_token` +// continues where the cap stopped. +struct ListResult { + clients::body::Json items = clients::body::Json::array(); + bool truncated = false; + std::string next_page_token; // Empty unless truncated. + std::size_t pages = 0; // Pages fetched. + + std::size_t size() const { return items.as_array().size(); } + // {"items":[...],"count":n,"pages":n,"truncated":bool,"next_page_token":"", + // "cap_items":RegistryCaps::kMaxItems} + clients::body::Json to_json() const; +}; + +// The paging state machine behind CallRunner::list, separate from any network +// so the cap logic is testable on its own. Feed it one page document at a +// time; it says whether another page is wanted and keeps the loud +// `truncated` flag. +class ListPager { + public: + ListPager(std::string items_key, std::size_t max_pages, std::size_t max_items); + // Takes the next page document. Returns true when the caller should fetch + // the page named by token(). + bool feed(const clients::body::Json& page); + const std::string& token() const { return token_; } + const ListResult& result() const { return result_; } + + private: + std::string items_key_; + std::size_t max_pages_, max_items_; + std::string token_; + ListResult result_; +}; + struct CallArgs { std::map vars; // {+parent}, {+name}, ... std::map query; // updateMask, filter, ... @@ -28,10 +63,12 @@ class CallRunner { // One request, one reply. clients::body::Json invoke(const std::string& call_id, const CallArgs& args) const; - // Follows nextPageToken (RegistryCaps::kMaxPages pages of kPageSize) and - // returns the concatenated `items_key` array. - clients::body::Json list(const std::string& call_id, const std::string& items_key, - const CallArgs& args) const; + // Follows nextPageToken up to RegistryCaps::kMaxPages pages and kMaxItems + // items and returns the concatenated `items_key` array. Stopping at a cap + // with pages left is reported (ListResult::truncated), never hidden. The + // caller cannot pass its own pageToken: the runner owns paging. + ListResult list(const std::string& call_id, const std::string& items_key, + const CallArgs& args) const; // invoke + wait for the operation the reply describes; returns // {"operation": name}. clients::body::Json mutate(const std::string& call_id, const CallArgs& args) const; diff --git a/src/google/registry/calls.cpp b/src/google/registry/calls.cpp index 41b4db6..1a62645 100644 --- a/src/google/registry/calls.cpp +++ b/src/google/registry/calls.cpp @@ -51,8 +51,44 @@ const std::vector kCalls = { {"agentregistry.v1.skills.getIamPolicy", "GET", "/{+resource}:getIamPolicy", "agentregistry", "v1", "agentregistry.projects.locations.skills.getIamPolicy", "-", "options.requestedPolicyVersion", "-"}, {"agentregistry.v1.skills.setIamPolicy", "POST", "/{+resource}:setIamPolicy", "agentregistry", "v1", "agentregistry.projects.locations.skills.setIamPolicy", "GoogleIamV1SetIamPolicyRequest", "-", "policy,updateMask"}, {"agentregistry.v1.skills.testIamPermissions", "POST", "/{+resource}:testIamPermissions", "agentregistry", "v1", "agentregistry.projects.locations.skills.testIamPermissions", "GoogleIamV1TestIamPermissionsRequest", "-", "permissions"}, + {"agentregistry.v1alpha.agents.get", "GET", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.agents.get", "-", "-", "-"}, + {"agentregistry.v1alpha.agents.list", "GET", "/{+parent}/agents", "agentregistry", "v1alpha", "agentregistry.projects.locations.agents.list", "-", "filter,orderBy,pageSize,pageToken", "-"}, + {"agentregistry.v1alpha.agents.search", "POST", "/{+parent}/agents:search", "agentregistry", "v1alpha", "agentregistry.projects.locations.agents.search", "SearchAgentsRequest", "-", "searchString,pageSize,pageToken"}, + {"agentregistry.v1alpha.aiApplications.agents.get", "GET", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.aiApplications.agents.get", "-", "-", "-"}, + {"agentregistry.v1alpha.aiApplications.agents.list", "GET", "/{+parent}/agents", "agentregistry", "v1alpha", "agentregistry.projects.locations.aiApplications.agents.list", "-", "pageSize,pageToken", "-"}, + {"agentregistry.v1alpha.aiApplications.endpoints.get", "GET", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.aiApplications.endpoints.get", "-", "-", "-"}, + {"agentregistry.v1alpha.aiApplications.endpoints.list", "GET", "/{+parent}/endpoints", "agentregistry", "v1alpha", "agentregistry.projects.locations.aiApplications.endpoints.list", "-", "pageSize,pageToken", "-"}, + {"agentregistry.v1alpha.aiApplications.get", "GET", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.aiApplications.get", "-", "-", "-"}, + {"agentregistry.v1alpha.aiApplications.getIamPolicy", "GET", "/{+resource}:getIamPolicy", "agentregistry", "v1alpha", "agentregistry.projects.locations.aiApplications.getIamPolicy", "-", "options.requestedPolicyVersion", "-"}, + {"agentregistry.v1alpha.aiApplications.list", "GET", "/{+parent}/aiApplications", "agentregistry", "v1alpha", "agentregistry.projects.locations.aiApplications.list", "-", "filter,orderBy,pageSize,pageToken", "-"}, + {"agentregistry.v1alpha.aiApplications.mcpServers.get", "GET", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.aiApplications.mcpServers.get", "-", "-", "-"}, + {"agentregistry.v1alpha.aiApplications.mcpServers.list", "GET", "/{+parent}/mcpServers", "agentregistry", "v1alpha", "agentregistry.projects.locations.aiApplications.mcpServers.list", "-", "pageSize,pageToken", "-"}, + {"agentregistry.v1alpha.aiApplications.setIamPolicy", "POST", "/{+resource}:setIamPolicy", "agentregistry", "v1alpha", "agentregistry.projects.locations.aiApplications.setIamPolicy", "GoogleIamV1SetIamPolicyRequest", "-", "policy,updateMask"}, + {"agentregistry.v1alpha.aiApplications.testIamPermissions", "POST", "/{+resource}:testIamPermissions", "agentregistry", "v1alpha", "agentregistry.projects.locations.aiApplications.testIamPermissions", "GoogleIamV1TestIamPermissionsRequest", "-", "permissions"}, + {"agentregistry.v1alpha.bindings.create", "POST", "/{+parent}/bindings", "agentregistry", "v1alpha", "agentregistry.projects.locations.bindings.create", "Binding", "bindingId,requestId", "displayName,description,source,target,authProviderBinding"}, + {"agentregistry.v1alpha.bindings.delete", "DELETE", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.bindings.delete", "-", "requestId", "-"}, + {"agentregistry.v1alpha.bindings.fetchAvailable", "GET", "/{+parent}/bindings:fetchAvailable", "agentregistry", "v1alpha", "agentregistry.projects.locations.bindings.fetchAvailable", "-", "pageSize,pageToken,sourceIdentifier,targetIdentifier", "-"}, + {"agentregistry.v1alpha.bindings.get", "GET", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.bindings.get", "-", "-", "-"}, + {"agentregistry.v1alpha.bindings.list", "GET", "/{+parent}/bindings", "agentregistry", "v1alpha", "agentregistry.projects.locations.bindings.list", "-", "filter,orderBy,pageSize,pageToken", "-"}, + {"agentregistry.v1alpha.bindings.patch", "PATCH", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.bindings.patch", "Binding", "requestId,updateMask", "displayName,description,source,target,authProviderBinding"}, + {"agentregistry.v1alpha.endpoints.get", "GET", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.endpoints.get", "-", "-", "-"}, + {"agentregistry.v1alpha.endpoints.list", "GET", "/{+parent}/endpoints", "agentregistry", "v1alpha", "agentregistry.projects.locations.endpoints.list", "-", "filter,pageSize,pageToken", "-"}, + {"agentregistry.v1alpha.locations.get", "GET", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.get", "-", "-", "-"}, + {"agentregistry.v1alpha.locations.list", "GET", "/{+name}/locations", "agentregistry", "v1alpha", "agentregistry.projects.locations.list", "-", "extraLocationTypes,filter,pageSize,pageToken", "-"}, + {"agentregistry.v1alpha.mcpServers.get", "GET", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.mcpServers.get", "-", "-", "-"}, + {"agentregistry.v1alpha.mcpServers.list", "GET", "/{+parent}/mcpServers", "agentregistry", "v1alpha", "agentregistry.projects.locations.mcpServers.list", "-", "filter,orderBy,pageSize,pageToken", "-"}, + {"agentregistry.v1alpha.mcpServers.search", "POST", "/{+parent}/mcpServers:search", "agentregistry", "v1alpha", "agentregistry.projects.locations.mcpServers.search", "SearchMcpServersRequest", "-", "searchString,pageSize,pageToken"}, + {"agentregistry.v1alpha.operations.cancel", "POST", "/{+name}:cancel", "agentregistry", "v1alpha", "agentregistry.projects.locations.operations.cancel", "CancelOperationRequest", "-", ""}, + {"agentregistry.v1alpha.operations.delete", "DELETE", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.operations.delete", "-", "-", "-"}, + {"agentregistry.v1alpha.operations.get", "GET", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.operations.get", "-", "-", "-"}, + {"agentregistry.v1alpha.operations.list", "GET", "/{+name}/operations", "agentregistry", "v1alpha", "agentregistry.projects.locations.operations.list", "-", "filter,pageSize,pageToken,returnPartialSuccess", "-"}, {"agentregistry.v1alpha.publishers.get", "GET", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.publishers.get", "-", "-", "-"}, {"agentregistry.v1alpha.publishers.list", "GET", "/{+parent}/publishers", "agentregistry", "v1alpha", "agentregistry.projects.locations.publishers.list", "-", "pageSize,pageToken", "-"}, + {"agentregistry.v1alpha.services.create", "POST", "/{+parent}/services", "agentregistry", "v1alpha", "agentregistry.projects.locations.services.create", "Service", "requestId,serviceId", "displayName,description,interfaces,agentSpec,mcpServerSpec,endpointSpec"}, + {"agentregistry.v1alpha.services.delete", "DELETE", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.services.delete", "-", "requestId", "-"}, + {"agentregistry.v1alpha.services.get", "GET", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.services.get", "-", "-", "-"}, + {"agentregistry.v1alpha.services.list", "GET", "/{+parent}/services", "agentregistry", "v1alpha", "agentregistry.projects.locations.services.list", "-", "filter,pageSize,pageToken", "-"}, + {"agentregistry.v1alpha.services.patch", "PATCH", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.services.patch", "Service", "requestId,updateMask", "displayName,description,interfaces,agentSpec,mcpServerSpec,endpointSpec"}, {"agentregistry.v1alpha.skills.create", "POST", "/{+parent}/skills", "agentregistry", "v1alpha", "agentregistry.projects.locations.skills.create", "Skill", "requestId,skillId", "displayName,description,type,targetState,publisher,defaultRevision,initialRevision"}, {"agentregistry.v1alpha.skills.delete", "DELETE", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.skills.delete", "-", "force,requestId", "-"}, {"agentregistry.v1alpha.skills.get", "GET", "/{+name}", "agentregistry", "v1alpha", "agentregistry.projects.locations.skills.get", "-", "-", "-"}, @@ -91,13 +127,19 @@ const std::vector kCalls = { {"networksecurity.v1.authzPolicies.patch", "PATCH", "/{+name}", "networksecurity", "v1", "networksecurity.projects.locations.authzPolicies.patch", "AuthzPolicy", "requestId,updateMask", "description,labels,target,httpRules,action,customProvider,policyProfile"}, {"networksecurity.v1.authzPolicies.setIamPolicy", "POST", "/{+resource}:setIamPolicy", "networksecurity", "v1", "networksecurity.projects.locations.authzPolicies.setIamPolicy", "GoogleIamV1SetIamPolicyRequest", "-", "policy,updateMask"}, {"networksecurity.v1.authzPolicies.testIamPermissions", "POST", "/{+resource}:testIamPermissions", "networksecurity", "v1", "networksecurity.projects.locations.authzPolicies.testIamPermissions", "GoogleIamV1TestIamPermissionsRequest", "-", "permissions"}, + {"networksecurity.v1.operations.cancel", "POST", "/{+name}:cancel", "networksecurity", "v1", "networksecurity.projects.locations.operations.cancel", "CancelOperationRequest", "-", ""}, + {"networksecurity.v1.operations.delete", "DELETE", "/{+name}", "networksecurity", "v1", "networksecurity.projects.locations.operations.delete", "-", "-", "-"}, {"networksecurity.v1.operations.get", "GET", "/{+name}", "networksecurity", "v1", "networksecurity.projects.locations.operations.get", "-", "-", "-"}, + {"networksecurity.v1.operations.list", "GET", "/{+name}/operations", "networksecurity", "v1", "networksecurity.projects.locations.operations.list", "-", "filter,pageSize,pageToken,returnPartialSuccess", "-"}, {"networkservices.v1.agentGateways.create", "POST", "/{+parent}/agentGateways", "networkservices", "v1", "networkservices.projects.locations.agentGateways.create", "AgentGateway", "agentGatewayId", "description,labels,registries,networkConfig,googleManaged,selfManaged,agentConnectivityTemplate,etag"}, {"networkservices.v1.agentGateways.delete", "DELETE", "/{+name}", "networkservices", "v1", "networkservices.projects.locations.agentGateways.delete", "-", "etag", "-"}, {"networkservices.v1.agentGateways.get", "GET", "/{+name}", "networkservices", "v1", "networkservices.projects.locations.agentGateways.get", "-", "-", "-"}, {"networkservices.v1.agentGateways.list", "GET", "/{+parent}/agentGateways", "networkservices", "v1", "networkservices.projects.locations.agentGateways.list", "-", "filter,pageSize,pageToken,returnPartialSuccess", "-"}, {"networkservices.v1.agentGateways.patch", "PATCH", "/{+name}", "networkservices", "v1", "networkservices.projects.locations.agentGateways.patch", "AgentGateway", "updateMask", "description,labels,registries,networkConfig,googleManaged,selfManaged,agentConnectivityTemplate,etag"}, + {"networkservices.v1.operations.cancel", "POST", "/{+name}:cancel", "networkservices", "v1", "networkservices.projects.locations.operations.cancel", "CancelOperationRequest", "-", ""}, + {"networkservices.v1.operations.delete", "DELETE", "/{+name}", "networkservices", "v1", "networkservices.projects.locations.operations.delete", "-", "-", "-"}, {"networkservices.v1.operations.get", "GET", "/{+name}", "networkservices", "v1", "networkservices.projects.locations.operations.get", "-", "-", "-"}, + {"networkservices.v1.operations.list", "GET", "/{+name}/operations", "networkservices", "v1", "networkservices.projects.locations.operations.list", "-", "filter,pageSize,pageToken,returnPartialSuccess", "-"}, }; } // namespace diff --git a/src/google/registry/gateway_policy.cpp b/src/google/registry/gateway_policy.cpp index 5a35d28..c4daaae 100644 --- a/src/google/registry/gateway_policy.cpp +++ b/src/google/registry/gateway_policy.cpp @@ -126,7 +126,7 @@ ServiceEndpoint GatewayPolicies::default_endpoint() { GatewayPolicies::GatewayPolicies(std::shared_ptr client, CloudContext context) : RegistryService(std::move(client), default_endpoint(), std::move(context)) {} -Json GatewayPolicies::list() const { +ListResult GatewayPolicies::list() const { return runner_.list("networksecurity.v1.authzPolicies.list", "authzPolicies", with("parent", context_.parent())); } @@ -184,4 +184,46 @@ Json GatewayPolicies::operation(const std::string& operation_id) const { with("name", context_.parent() + "/operations/" + operation_id)); } +Json GatewayPolicies::set_iam_policy(const std::string& id, const Json& policy) const { + if (!policy.is_object()) throw GoogleError::invalid("policy must be a JSON object"); + get(id); // Read first: a missing policy is a clear 404. + CallArgs a = with("resource", context_.child("authzPolicies", id)); + a.body = Json::object(); + a.body.set("policy", policy); + return runner_.invoke("networksecurity.v1.authzPolicies.setIamPolicy", a); +} + +Json GatewayPolicies::test_iam_permissions(const std::string& id, + const std::vector& permissions) const { + if (permissions.empty() || permissions.size() > RegistryCaps::kMaxPermissions) + throw GoogleError::invalid("1..64 permissions"); + CallArgs a = with("resource", context_.child("authzPolicies", id)); + a.body = Json::object(); + Json arr = Json::array(); + for (const auto& p : permissions) { + Ids::require_text(p, 256, "permission"); + arr.push(Json(p)); + } + a.body.set("permissions", arr); + return runner_.invoke("networksecurity.v1.authzPolicies.testIamPermissions", a); +} + +ListResult GatewayPolicies::operations() const { + return runner_.list("networksecurity.v1.operations.list", "operations", + with("name", context_.parent())); +} + +Json GatewayPolicies::cancel_operation(const std::string& operation_id) const { + operation(operation_id); + CallArgs a = with("name", context_.parent() + "/operations/" + operation_id); + a.body = Json::object(); + return runner_.invoke("networksecurity.v1.operations.cancel", a); +} + +Json GatewayPolicies::delete_operation(const std::string& operation_id) const { + operation(operation_id); + return runner_.invoke("networksecurity.v1.operations.delete", + with("name", context_.parent() + "/operations/" + operation_id)); +} + } // namespace molo::google::registry diff --git a/src/google/registry/gateway_policy.h b/src/google/registry/gateway_policy.h index 6af0233..e29bbe3 100644 --- a/src/google/registry/gateway_policy.h +++ b/src/google/registry/gateway_policy.h @@ -54,7 +54,8 @@ class GatewayPolicies : public RegistryService { static ServiceEndpoint default_endpoint(); // networksecurity .../v1 - clients::body::Json list() const; + // Paged up to the registry caps; ListResult::truncated is loud. + ListResult list() const; clients::body::Json get(const std::string& id) const; // Explicit policy changes; each reads first and waits for the operation. @@ -68,7 +69,18 @@ class GatewayPolicies : public RegistryService { clients::body::Json remove_policy(const std::string& id, const std::string& request_id = "") const; clients::body::Json iam_policy(const std::string& id) const; + // setIamPolicy reads the policy first; the body is {"policy": policy}. + clients::body::Json set_iam_policy(const std::string& id, + const clients::body::Json& policy) const; + clients::body::Json test_iam_permissions(const std::string& id, + const std::vector& permissions) const; + + // ---- Operations of the networksecurity location (get, list, cancel, delete). clients::body::Json operation(const std::string& operation_id) const; + ListResult operations() const; + clients::body::Json cancel_operation(const std::string& operation_id) const; + // Destructive: removes the operation record, never the policy. + clients::body::Json delete_operation(const std::string& operation_id) const; }; } // namespace molo::google::registry diff --git a/src/google/registry/tools/check_discovery.py b/src/google/registry/tools/check_discovery.py index a537f58..8f4bcad 100644 --- a/src/google/registry/tools/check_discovery.py +++ b/src/google/registry/tools/check_discovery.py @@ -36,6 +36,23 @@ def rows(): return out +# Coverage: every discovery method whose id starts with one of these prefixes +# must be a table row (or be listed in EXCLUDED with a reason). Narrow subsets +# of aiplatform and logging are used on purpose and are not covered here. +COVERED = [ + ("agentregistry", "v1", "agentregistry."), + ("agentregistry", "v1alpha", "agentregistry."), + ("networkservices", "v1", "networkservices.projects.locations.agentGateways."), + ("networkservices", "v1", "networkservices.projects.locations.operations."), + ("networksecurity", "v1", "networksecurity.projects.locations.authzPolicies."), + ("networksecurity", "v1", "networksecurity.projects.locations.operations."), + ("iam", "v2", "iam."), + ("iam", "v3", "iam.organizations.locations.principalAccessBoundaryPolicies."), +] +# rpc id -> reason it is not wrapped (none today). +EXCLUDED = {} + + def fetch(service, version, cache): name = "%s_%s.json" % (service, version) if cache: @@ -106,7 +123,20 @@ def main(): for f in row["fields"].split(","): if f and f not in props: failures.append("%s: field %s not in schema %s" % (row["id"], f, req)) - print("checked %d calls across %d discovery documents" % (len(table), len(docs))) + wrapped = {(r["service"], r["version"], r["rpc"]) for r in table} + missing = 0 + for service, version, prefix in COVERED: + if (service, version) not in docs: + doc = fetch(service, version, cache) + docs[(service, version)] = (doc, methods(doc)) + for rpc in sorted(docs[(service, version)][1]): + if rpc.startswith(prefix) and (service, version, rpc) not in wrapped \ + and rpc not in EXCLUDED: + failures.append("%s %s: discovery method %s is not in the table" % + (service, version, rpc)) + missing += 1 + print("checked %d calls across %d discovery documents, %d discovery methods unwrapped" % + (len(table), len(docs), missing)) for f in failures: print("FAIL", f) if failures: diff --git a/src/google/sessions/SPEC.md b/src/google/sessions/SPEC.md index 4207bb9..9047291 100644 --- a/src/google/sessions/SPEC.md +++ b/src/google/sessions/SPEC.md @@ -17,8 +17,16 @@ Gemini Enterprise Agent Platform, Agent Engine (`reasoningEngines`) children: create, get, list, patch, delete, purge (explicit, guarded), revisions (list, get), rollback, topics (managed and custom), scope maps, metadata. - Long-running operations: every mutating call returns an Operation; the module - waits with `wait_operation` (hard caps in `OperationCaps`). `operation()`, - `cancel_operation()` and `wait()` expose them for one engine only. + waits with `wait_operation` (hard caps in `OperationCaps`). All five + Discovery operations methods (get, list, cancel, delete, wait) are wrapped + for the four places Discovery puts them: under the engine, under a session, + under a memory and under a serving profile (20 table rows). `operation()`, + `list_operations()`, `cancel_operation()`, `delete_operation()`, `wait()` + (client polling) and `wait_server()` (one POST `:wait?timeout=Ns`, 1..60 s) + work for one root only: the engine, or the location for serving profiles. +- Serving profiles (`projects.locations.servingProfiles`, location level, not + engine scoped): create (waits), get, list, patch, delete, with scope + `GEMINI_LIVE`, `INTERACTIONS_API` or `RESPONSE_API`, optional CMEK key. Everything is against one API surface: `aiplatform` **v1beta1**. See section 6 for why not v1. @@ -29,15 +37,18 @@ for why not v1. |---|---| | `caps.h` | Static ceilings (`SessionsCaps`, `MemoryCaps`) | | `calls.h` | The ONE table of every REST call (verb, path, discovery method, request fields, query params) | -| `names.h/.cpp` | `EngineName` (validated resource name), `Check` (input validators), wire helpers | +| `names.h/.cpp` | `EngineName`, `LocationName` (validated resource names), `Check` (input validators), wire helpers | +| `operation_names.h/.cpp` | `OperationName`: validates an operation name against its root, picks the table row for each verb by owner (engine, session, memory, serving profile) | +| `operation_service.h/.cpp` | `OperationService : Service`: table-driven `send`, `settle`, loud capped `list_items` (`list_over_limit`), the operations surface | +| `serving_profile.h/.cpp` | `ServingProfileSpec`, `ServingProfilePatch`, `ServingProfile`, `ServingScope`, `ServingProfiles : OperationService` | | `session_types.h/.cpp` | `SessionSpec`, `Session`, `SessionPatch`, `CompactRequest`, `SessionEvent`, list options | | `memory_types.h/.cpp` | `Memory`, `MemorySpec`, `MemoryPatch`, `Topic`, `MetadataValue`, `MetadataFilter`, `MemoryGenerateRequest`, `IngestRequest`, `RetrieveRequest`, `PurgeRequest`, `MemoryRevision`, `Profile` | -| `engine_service.h/.cpp` | `EngineService : Service`: table-driven `send`, `settle`, capped `list_items`, operation helpers | +| `engine_service.h/.cpp` | `EngineService : OperationService`: one validated engine and its v1beta1 endpoint | | `sessions.h/.cpp` | `Sessions : EngineService` | | `memory_bank.h/.cpp` | `MemoryBank : EngineService` | | `tools/check_discovery.py` | Verifies `calls.h` and the typed bodies against Google's Discovery document | -Tests: `tests/src/google/sessions/test_sessions.cpp`. Example: +Tests: `tests/src/google/sessions/test_sessions.cpp` and `test_sessions_gaps.cpp`. Example: `example/app/agent/SessionsDemo.cpp` (routes under `/agent/sessions`), tests in `tests/example/agent/sessions/`. @@ -50,7 +61,9 @@ Tests: `tests/src/google/sessions/test_sessions.cpp`. Example: | Session TTL | `Ns` form, 24 h minimum (Google), 3650 days maximum (ours) | | session_state / event | 64 KiB / 256 KiB serialized | | Labels | 20, key and value 64 chars | -| List pages / items | 10 pages of 100, 1000 items; more is `RESULT_TOO_LARGE`, narrow the filter | +| List pages / items | 10 pages of 100, 1000 items; more items OR a further page after the last allowed one is `RESULT_TOO_LARGE` (502), never a silent cut, narrow the filter | +| Server-side operation wait | 1..60 s per call (`kMaxServerWaitSeconds`) | +| Serving profile | id 1..63 `[a-z0-9-]`, display name required 128, description 4000, KMS key name 512 | | Scope | 1..10 entries, 128 chars, no `*` | | Fact / description / context | 4000 chars | | Generate sources | 100 events, 64 KiB; direct memories max 5 (Google) | @@ -83,10 +96,14 @@ Tests: `tests/src/google/sessions/test_sessions.cpp`. Example: credential) and fails if a declared call, verb, path, request field, query parameter, enum value, response key or nested request field is absent. -Result 2026-10-24: `OK 2026-10-24: 23 calls, 7 enums, 25 response keys, 18 -nested schemas match aiplatform v1beta1 discovery`. A deliberately misspelled -field made it exit 1 (checked). Rerun it after any change to `calls.h` or the -typed bodies. +Result 2026-10-24: `OK 2026-10-24: 46 calls, 8 enums, 34 response keys, 20 +nested schemas match aiplatform v1beta1 discovery`. The script also checks +coverage: every Discovery method under `reasoningEngines.operations`, +`reasoningEngines.sessions`, `reasoningEngines.memories` and `servingProfiles` +must be a table row or listed in `EXCLUDED` (empty: nothing is excluded), so a +method Google adds later fails the check. A deliberately misspelled field made +it exit 1 (checked). Rerun it after any change to `calls.h` or the typed +bodies. From discovery (v1beta1, `projects.locations.reasoningEngines`): sessions create (POST, `sessionId` query, returns Operation), get, list @@ -95,9 +112,12 @@ Operation), appendEvent (POST `:appendEvent`, body `SessionEvent`), compact (POST `:compact`, returns Operation), events.list (`sessionEvents`); memories generate / retrieve / retrieveProfiles / create (`memoryId`) / get / list / patch (`updateMask`) / delete / purge / rollback / ingestEvents, -memories.revisions list / get; operations get / cancel. Discovery also lists -`wait`, `list`, `delete` on operations, which this module does not wrap -(`wait_operation` polls with GET). +memories.revisions list / get; operations get / cancel / list / delete / wait +under the engine, `sessions.operations.*` and `memories.operations.*` (same +five), `locations.servingProfiles` create (`servingProfileId` query, returns +Operation) / get / list / patch (`updateMask`) / delete (returns Empty) and +`servingProfiles.operations.*`. The Method column of a `calls.h` row is +relative to `kDiscoveryPrefix` unless it starts with `aiplatform.`. ## 6. Verified live @@ -127,6 +147,20 @@ an express key and a token; v1 refuses keys. The opt-in test `sessions_live` the real client (sessions list / get / events, memories list / get / revisions) and passed with the key and with the token on 2026-10-24. +Gap-closing round, same day, service-account token, read-only: + +| Call | Result | +|---|---| +| GET `v1beta1/.../reasoningEngines/1/operations?pageSize=5`, `.../sessions/abc/operations`, `.../memories/abc/operations` | 200 `{}` (empty list, even though engine 1 does not exist) | +| GET `v1beta1/.../locations/us-central1/servingProfiles?pageSize=5` (also `global` host and `europe-west4`) | 501 `UNIMPLEMENTED` "Operation is not implemented, or supported, or enabled." (not available to this project) | +| GET `v1beta1/.../servingProfiles/probe` | 501 UNIMPLEMENTED | +| GET `v1beta1/.../servingProfiles/probe/operations` | 400 INVALID_ARGUMENT: "the idType servingProfiles is not a documented LRO Parent ID, the associated value must be a Long, but was instead: probe" | +| GET `v1beta1/.../servingProfiles/123/operations?pageSize=1` | 200 `{}` | + +The opt-in test `sessions_live.operations_and_serving_profiles_reads` repeats +the operations lists and the servingProfiles list through the real client; it +accepts 200 or 501/403/404 for servingProfiles and prints which one it saw. + No Agent Engine exists in the project, so **no happy path was verified live**: create, patch, delete, append, compact, generate, ingest, retrieve, retrieveProfiles, purge, rollback are verified by discovery only. Nothing that @@ -151,18 +185,25 @@ Neither discovery nor the docs pages read state these; they are inferred: search model (docs); the error body when it is not is unread. - Memory Profiles need schema configs on the engine (`MemoryBankConfig` in the runtime module); the shape of `profile` is free-form and kept raw. +- Serving profile id charset (`[a-z0-9-]`, 1..63 is our choice) and the real + id rule: Google's LRO layer wants a numeric id for operations under a + serving profile (400 above), so Google may assign or require numeric ids. + The create/patch responses (Operation `response` is `Any`) are parsed + tolerantly with a GET fallback. `servingProfiles` itself answers 501 on this + project, so no body of it was ever seen. +- Whether `operations.wait` answers `done: false` at the timeout or an error + (the docs line says "at most the time permitted"): both are handled, a + not-done answer is returned to the caller, not retried. - Managed topic names beyond the four in discovery, and `thought`/`audio` parts inside event `content` (typed by `gemini_types.h`, unknown parts dropped). ## 8. Gaps -- `servingProfiles` (location-level create/get/list/patch for Gemini Live, - Interactions and Response APIs) is listed by discovery next to this product - but is not engine-scoped; not implemented here. Owner: the `live` or - `models` module. - Engine-level Memory Bank configuration (`contextSpec.memoryBankConfig`, customization, TTL config) is part of the Agent Engine resource: runtime module. -- Operations `list`, `delete`, `wait` (server-side) are not wrapped. +- `servingProfiles` answers 501 for the test project in three locations, so + the happy path is Discovery-verified only. Owner: a project that has the + feature; test `sessions_live.operations_and_serving_profiles_reads`. - Session and memory `labels` / `metadata` filters beyond a passthrough string. - Token refresh is the caller's job (foundation gap). diff --git a/src/google/sessions/calls.h b/src/google/sessions/calls.h index 153ae5c..a4d56e4 100644 --- a/src/google/sessions/calls.h +++ b/src/google/sessions/calls.h @@ -9,6 +9,9 @@ // "API keys are not supported" for sessions and memories, v1beta1 accepts an // express API key (404 for a missing engine). See SPEC.md. // +// Method column: relative to kDiscoveryPrefix, except rows that start with +// "aiplatform." (location-level resources such as servingProfiles) which are +// absolute Discovery method ids. // Row: {Call, verb, path template, discovery method (under // aiplatform.projects.locations.reasoningEngines.), request schema suffix // (after GoogleCloudAiplatformV1beta1), request fields used, query params used}. @@ -30,7 +33,16 @@ enum class Call { kMemoryGenerate, kMemoryRetrieve, kMemoryRetrieveProfiles, kMemoryCreate, kMemoryGet, kMemoryList, kMemoryPatch, kMemoryDelete, kMemoryPurge, kMemoryRollback, kMemoryIngestEvents, kRevisionList, kRevisionGet, - kOperationGet, kOperationCancel, + kOperationGet, kOperationCancel, kOperationList, kOperationDelete, kOperationWait, + kSessionOperationGet, kSessionOperationCancel, kSessionOperationList, + kSessionOperationDelete, kSessionOperationWait, + kMemoryOperationGet, kMemoryOperationCancel, kMemoryOperationList, + kMemoryOperationDelete, kMemoryOperationWait, + kServingProfileCreate, kServingProfileGet, kServingProfileList, + kServingProfilePatch, kServingProfileDelete, + kServingProfileOperationGet, kServingProfileOperationCancel, + kServingProfileOperationList, kServingProfileOperationDelete, + kServingProfileOperationWait, kCount }; @@ -69,6 +81,29 @@ inline constexpr CallSpec kCalls[] = { {Call::kRevisionGet, "GET", "/{name}", "memories.revisions.get", "", "", ""}, {Call::kOperationGet, "GET", "/{name}", "operations.get", "", "", ""}, {Call::kOperationCancel, "POST", "/{name}:cancel", "operations.cancel", "", "", ""}, +{Call::kOperationList, "GET", "/{name}/operations", "operations.list", "", "", "filter pageSize pageToken"}, +{Call::kOperationDelete, "DELETE", "/{name}", "operations.delete", "", "", ""}, +{Call::kOperationWait, "POST", "/{name}:wait", "operations.wait", "", "", "timeout"}, +{Call::kSessionOperationGet, "GET", "/{name}", "sessions.operations.get", "", "", ""}, +{Call::kSessionOperationCancel, "POST", "/{name}:cancel", "sessions.operations.cancel", "", "", ""}, +{Call::kSessionOperationList, "GET", "/{name}/operations", "sessions.operations.list", "", "", "filter pageSize pageToken"}, +{Call::kSessionOperationDelete, "DELETE", "/{name}", "sessions.operations.delete", "", "", ""}, +{Call::kSessionOperationWait, "POST", "/{name}:wait", "sessions.operations.wait", "", "", "timeout"}, +{Call::kMemoryOperationGet, "GET", "/{name}", "memories.operations.get", "", "", ""}, +{Call::kMemoryOperationCancel, "POST", "/{name}:cancel", "memories.operations.cancel", "", "", ""}, +{Call::kMemoryOperationList, "GET", "/{name}/operations", "memories.operations.list", "", "", "filter pageSize pageToken"}, +{Call::kMemoryOperationDelete, "DELETE", "/{name}", "memories.operations.delete", "", "", ""}, +{Call::kMemoryOperationWait, "POST", "/{name}:wait", "memories.operations.wait", "", "", "timeout"}, +{Call::kServingProfileCreate, "POST", "/{parent}/servingProfiles", "aiplatform.projects.locations.servingProfiles.create", "ServingProfile", "displayName description scope cmekConfig", "servingProfileId"}, +{Call::kServingProfileGet, "GET", "/{name}", "aiplatform.projects.locations.servingProfiles.get", "", "", ""}, +{Call::kServingProfileList, "GET", "/{parent}/servingProfiles", "aiplatform.projects.locations.servingProfiles.list", "", "", "pageSize pageToken"}, +{Call::kServingProfilePatch, "PATCH", "/{name}", "aiplatform.projects.locations.servingProfiles.patch", "ServingProfile", "displayName description cmekConfig", "updateMask"}, +{Call::kServingProfileDelete, "DELETE", "/{name}", "aiplatform.projects.locations.servingProfiles.delete", "", "", ""}, +{Call::kServingProfileOperationGet, "GET", "/{name}", "aiplatform.projects.locations.servingProfiles.operations.get", "", "", ""}, +{Call::kServingProfileOperationCancel, "POST", "/{name}:cancel", "aiplatform.projects.locations.servingProfiles.operations.cancel", "", "", ""}, +{Call::kServingProfileOperationList, "GET", "/{name}/operations", "aiplatform.projects.locations.servingProfiles.operations.list", "", "", "filter pageSize pageToken"}, +{Call::kServingProfileOperationDelete, "DELETE", "/{name}", "aiplatform.projects.locations.servingProfiles.operations.delete", "", "", ""}, +{Call::kServingProfileOperationWait, "POST", "/{name}:wait", "aiplatform.projects.locations.servingProfiles.operations.wait", "", "", "timeout"}, }; // clang-format on diff --git a/src/google/sessions/caps.h b/src/google/sessions/caps.h index ac7cb56..7ad72ec 100644 --- a/src/google/sessions/caps.h +++ b/src/google/sessions/caps.h @@ -22,6 +22,9 @@ struct SessionsCaps { static constexpr int kPageSize = 100; // Google ceiling static constexpr std::size_t kMaxListPages = 10; static constexpr std::size_t kMaxListItems = 1000; + static constexpr std::size_t kMaxServerWaitSeconds = 60; // one POST :wait + static constexpr std::size_t kMaxDescriptionChars = 4000; + static constexpr std::size_t kMaxKmsKeyChars = 512; }; struct MemoryCaps { diff --git a/src/google/sessions/engine_service.cpp b/src/google/sessions/engine_service.cpp index c43d965..1359b68 100644 --- a/src/google/sessions/engine_service.cpp +++ b/src/google/sessions/engine_service.cpp @@ -4,70 +4,10 @@ namespace molo::google::sessions { -using clients::body::Json; - EngineService::EngineService(std::shared_ptr client, EngineName engine, Sleeper sleeper) - : Service(std::move(client), endpoints::aiplatform_beta(engine.location())), - engine_name_(std::move(engine)), - sleeper_(std::move(sleeper)) {} - -Json EngineService::send(Call call, const std::string& resource, const Json* body, - const std::string& query) const { - const CallSpec& row = spec_of(call); - return client_->request(verb_of(row), endpoint_, expand(call, resource, query), body); -} - -Operation EngineService::settle(const Json& reply) const { - Operation op = Operation::from_json(reply); - if (op.done) { - if (op.failed()) - throw GoogleError(502, "OPERATION_FAILED", op.error_message, endpoint_.name); - return op; - } - if (op.name.empty()) - throw GoogleError(502, "EMPTY_RESPONSE", "no operation returned", endpoint_.name); - return wait(op.name); -} - -namespace { -void check_operation_name(const EngineName& engine, const std::string& name) { - const std::string prefix = engine.str() + "/"; - bool ok = name.size() > prefix.size() && name.compare(0, prefix.size(), prefix) == 0 && - name.find("/operations/") != std::string::npos && - name.find("..") == std::string::npos && name.find('?') == std::string::npos && - name.size() <= 512; - if (!ok) throw GoogleError::invalid("operation name must belong to this engine"); -} -} // namespace - -Json EngineService::operation(const std::string& operation_name) const { - check_operation_name(engine_name_, operation_name); - return send(Call::kOperationGet, operation_name); -} - -void EngineService::cancel_operation(const std::string& operation_name) const { - check_operation_name(engine_name_, operation_name); - Json empty = Json::object(); - send(Call::kOperationCancel, operation_name, &empty); -} - -Operation EngineService::wait(const std::string& operation_name) const { - check_operation_name(engine_name_, operation_name); - return wait_operation(*client_, endpoint_, operation_name, sleeper_); -} - -Json EngineService::list_items(Call call, const std::string& resource, - const std::string& query, const std::string& items_key, - std::size_t max_pages, std::size_t max_items) const { - Json items = list_all(*client_, endpoint_, expand(call, resource, query), items_key, - max_pages); - if (items.as_array().size() > max_items) - throw GoogleError(502, "RESULT_TOO_LARGE", - "more than " + std::to_string(max_items) + - " items; narrow the filter", - endpoint_.name); - return items; -} + : OperationService(std::move(client), endpoints::aiplatform_beta(engine.location()), + engine.str(), std::move(sleeper)), + engine_name_(std::move(engine)) {} } // namespace molo::google::sessions diff --git a/src/google/sessions/engine_service.h b/src/google/sessions/engine_service.h index a7679fd..5632a6c 100644 --- a/src/google/sessions/engine_service.h +++ b/src/google/sessions/engine_service.h @@ -1,48 +1,26 @@ #pragma once -// Base of Sessions and MemoryBank: one validated Agent Engine, the v1beta1 -// aiplatform endpoint of its location, table-driven requests (calls.h) and -// long-running operation handling shared by both. +// Base of Sessions and MemoryBank: one validated Agent Engine and the v1beta1 +// aiplatform endpoint of its location. Table-driven requests and the +// operations surface come from OperationService. #include #include -#include "google/operation.h" -#include "google/service.h" -#include "google/sessions/calls.h" #include "google/sessions/names.h" +#include "google/sessions/operation_service.h" namespace molo::google::sessions { -class EngineService : public Service { +class EngineService : public OperationService { public: const EngineName& engine() const { return engine_name_; } - // Raw operation document (GET) and cancel (POST :cancel). The name must be - // an operation of this engine; nothing else is reachable through here. - clients::body::Json operation(const std::string& operation_name) const; - void cancel_operation(const std::string& operation_name) const; - // Polls until done (hard caps in OperationCaps); throws on failure. - Operation wait(const std::string& operation_name) const; - protected: EngineService(std::shared_ptr client, EngineName engine, Sleeper sleeper); - // One table-driven request. `resource` fills the {parent} / {name} slot. - clients::body::Json send(Call call, const std::string& resource, - const clients::body::Json* body = nullptr, - const std::string& query = "") const; - // Follows an Operation reply to completion (done replies are checked). - Operation settle(const clients::body::Json& reply) const; - // Paged list through the table; caps pages and total items. - clients::body::Json list_items(Call call, const std::string& resource, - const std::string& query, - const std::string& items_key, - std::size_t max_pages, std::size_t max_items) const; - EngineName engine_name_; - Sleeper sleeper_; }; } // namespace molo::google::sessions diff --git a/src/google/sessions/memory_bank.cpp b/src/google/sessions/memory_bank.cpp index 72f8abe..cb90b7d 100644 --- a/src/google/sessions/memory_bank.cpp +++ b/src/google/sessions/memory_bank.cpp @@ -175,4 +175,9 @@ void MemoryBank::rollback(const std::string& memory_id, settle(send(Call::kMemoryRollback, engine_name_.memory(memory_id), &body)); } +std::vector MemoryBank::list_memory_operations( + const std::string& memory_id, const OperationListOptions& options) const { + return list_operations(engine_name_.memory(memory_id), options); +} + } // namespace molo::google::sessions diff --git a/src/google/sessions/memory_bank.h b/src/google/sessions/memory_bank.h index d2f2756..f16f096 100644 --- a/src/google/sessions/memory_bank.h +++ b/src/google/sessions/memory_bank.h @@ -41,6 +41,10 @@ class MemoryBank : public EngineService { MemoryRevision get_revision(const std::string& memory_id, const std::string& revision_id) const; void rollback(const std::string& memory_id, const std::string& revision_id) const; + + // Operations that belong to one memory (update, rollback runs). + std::vector list_memory_operations( + const std::string& memory_id, const OperationListOptions& options = {}) const; }; } // namespace molo::google::sessions diff --git a/src/google/sessions/names.cpp b/src/google/sessions/names.cpp index a87731f..4924393 100644 --- a/src/google/sessions/names.cpp +++ b/src/google/sessions/names.cpp @@ -59,6 +59,35 @@ EngineName::EngineName(std::string full) : full_(std::move(full)) { if (!valid_segment(engine_)) bad("engine name: bad engine id"); } +LocationName::LocationName(std::string full) : full_(std::move(full)) { + if (full_.empty() || full_.size() > SessionsCaps::kMaxNameChars) + bad("location name missing or too long"); + std::vector parts; + std::string cur; + for (char c : full_) { + if (c == '/') { + parts.push_back(cur); + cur.clear(); + } else { + cur.push_back(c); + } + } + parts.push_back(cur); + if (parts.size() != 4 || parts[0] != "projects" || parts[2] != "locations") + bad("location must look like projects/{p}/locations/{l}"); + project_ = parts[1]; + location_ = parts[3]; + if (!project_ok(project_)) bad("location name: bad project"); + if (location_.empty() || location_.size() > 64) bad("location name: bad location"); + for (char c : location_) + if (!id_char(c)) bad("location name: bad location"); +} + +std::string LocationName::serving_profile(const std::string& profile_id) const { + Check::serving_profile_id(profile_id); + return full_ + "/servingProfiles/" + profile_id; +} + std::string EngineName::session(const std::string& session_id) const { Check::session_id(session_id); return full_ + "/sessions/" + session_id; @@ -77,6 +106,14 @@ void Check::session_id(const std::string& id) { if (!id_char(c)) bad("session id allows only a-z 0-9 -"); } +void Check::serving_profile_id(const std::string& id) { + if (id.empty() || id.size() > SessionsCaps::kMaxIdChars) + bad("serving profile id must be 1.." + std::to_string(SessionsCaps::kMaxIdChars) + + " characters"); + for (char c : id) + if (!id_char(c)) bad("serving profile id allows only a-z 0-9 -"); +} + void Check::memory_id(const std::string& id) { if (id.empty() || id.size() > SessionsCaps::kMaxIdChars) bad("memory id must be 1.." + std::to_string(SessionsCaps::kMaxIdChars) + diff --git a/src/google/sessions/names.h b/src/google/sessions/names.h index 24df42a..c85ec34 100644 --- a/src/google/sessions/names.h +++ b/src/google/sessions/names.h @@ -29,6 +29,22 @@ class EngineName { std::string full_, project_, location_, engine_; }; +// "projects/{project}/locations/{location}": the root of location-level +// resources (serving profiles). +class LocationName { + public: + explicit LocationName(std::string full); // throws when malformed + + const std::string& str() const { return full_; } + const std::string& project() const { return project_; } + const std::string& location() const { return location_; } + + std::string serving_profile(const std::string& profile_id) const; + + private: + std::string full_, project_, location_; +}; + using StringMap = std::map; // Input validators. @@ -36,6 +52,9 @@ class Check { public: // Session id: 1..63 of [a-z0-9-]. static void session_id(const std::string& id); + // Serving profile id: same charset as a session id (Google's exact rule is + // not published, see SPEC Unverified). + static void serving_profile_id(const std::string& id); // Memory id: 1..63 of [a-z0-9-], starts with a letter, ends alphanumeric. static void memory_id(const std::string& id); static void text(const std::string& field, const std::string& value, diff --git a/src/google/sessions/operation_names.cpp b/src/google/sessions/operation_names.cpp new file mode 100644 index 0000000..fec0fca --- /dev/null +++ b/src/google/sessions/operation_names.cpp @@ -0,0 +1,92 @@ +#include "google/sessions/operation_names.h" + +#include + +#include "google/errors.h" +#include "google/resource.h" +#include "google/sessions/caps.h" + +namespace molo::google::sessions { + +namespace { + +[[noreturn]] void bad(const std::string& message) { throw GoogleError::invalid(message); } + +std::vector parts_of(const std::string& s) { + std::vector out; + std::string cur; + for (char c : s) { + if (c == '/') { + out.push_back(cur); + cur.clear(); + } else { + cur.push_back(c); + } + } + out.push_back(cur); + return out; +} + +void check_root_prefix(const std::string& root, const std::string& name) { + if (name.size() > SessionsCaps::kMaxNameChars) bad("operation name too long"); + if (name.size() <= root.size() + 1 || name.compare(0, root.size() + 1, root + "/") != 0) + bad("operation name must belong to " + root); +} + +} // namespace + +OperationName OperationName::classify(const std::string& root, const std::string& rest, + const std::string& full, bool want_operation) { + bool location_root = root.find("/reasoningEngines/") == std::string::npos; + std::vector p; + if (!rest.empty()) p = parts_of(rest); + for (const std::string& seg : p) + if (!valid_segment(seg)) bad("operation name has an empty or invalid segment"); + if (want_operation) { + std::size_t n = p.size(); + if (n < 2 || p[n - 2] != "operations") bad("not an operation name"); + p.resize(n - 2); + } + std::size_t n = p.size(); + if (n == 0) { + if (location_root) bad("a location has no operations here; name a serving profile"); + return OperationName(full, OperationOwner::kEngine); + } + if (n == 2 && !location_root && p[0] == "sessions") + return OperationName(full, OperationOwner::kSession); + if (n == 2 && !location_root && p[0] == "memories") + return OperationName(full, OperationOwner::kMemory); + if (n == 2 && location_root && p[0] == "servingProfiles") + return OperationName(full, OperationOwner::kServingProfile); + bad("operation parent must be the engine, a session, a memory or a serving profile"); +} + +OperationName OperationName::parse(const std::string& root, const std::string& name) { + check_root_prefix(root, name); + return classify(root, name.substr(root.size() + 1), name, true); +} + +OperationName OperationName::parse_parent(const std::string& root, + const std::string& parent) { + if (parent == root) return classify(root, "", parent, false); + check_root_prefix(root, parent); + return classify(root, parent.substr(root.size() + 1), parent, false); +} + +Call OperationName::call(OperationVerb verb) const { + static constexpr Call kTable[4][5] = { + {Call::kOperationGet, Call::kOperationCancel, Call::kOperationList, + Call::kOperationDelete, Call::kOperationWait}, + {Call::kSessionOperationGet, Call::kSessionOperationCancel, + Call::kSessionOperationList, Call::kSessionOperationDelete, + Call::kSessionOperationWait}, + {Call::kMemoryOperationGet, Call::kMemoryOperationCancel, + Call::kMemoryOperationList, Call::kMemoryOperationDelete, + Call::kMemoryOperationWait}, + {Call::kServingProfileOperationGet, Call::kServingProfileOperationCancel, + Call::kServingProfileOperationList, Call::kServingProfileOperationDelete, + Call::kServingProfileOperationWait}}; + return kTable[static_cast(owner_)][static_cast(verb)]; +} + +} // namespace molo::google::sessions diff --git a/src/google/sessions/operation_names.h b/src/google/sessions/operation_names.h new file mode 100644 index 0000000..7e11d6b --- /dev/null +++ b/src/google/sessions/operation_names.h @@ -0,0 +1,44 @@ +#pragma once + +// A validated long-running operation name and the table row that serves each +// verb for it. Discovery lists the same five operations methods (get, list, +// cancel, delete, wait) under the Agent Engine, under each session, under +// each memory and under each serving profile; the owner is read from the +// name so a caller can never reach an operation outside the root it passed. + +#include + +#include "google/sessions/calls.h" + +namespace molo::google::sessions { + +enum class OperationOwner { kEngine, kSession, kMemory, kServingProfile }; +enum class OperationVerb { kGet, kCancel, kList, kDelete, kWait }; + +class OperationName { + public: + // `root` is an engine name (owners engine, session, memory) or a location + // name "projects/{p}/locations/{l}" (owner serving profile). `name` must be + // "{root}[/{sessions|memories|servingProfiles}/{id}]/operations/{op}". + // Throws GoogleError::invalid otherwise. + static OperationName parse(const std::string& root, const std::string& name); + // A list parent: "{root}", "{root}/sessions/{id}", "{root}/memories/{id}" or + // "{root}/servingProfiles/{id}". Same validation, no trailing operation. + static OperationName parse_parent(const std::string& root, const std::string& parent); + + const std::string& str() const { return full_; } + OperationOwner owner() const { return owner_; } + // The table row serving a verb for this owner. + Call call(OperationVerb verb) const; + + private: + OperationName(std::string full, OperationOwner owner) + : full_(std::move(full)), owner_(owner) {} + static OperationName classify(const std::string& root, const std::string& rest, + const std::string& full, bool want_operation); + + std::string full_; + OperationOwner owner_; +}; + +} // namespace molo::google::sessions diff --git a/src/google/sessions/operation_service.cpp b/src/google/sessions/operation_service.cpp new file mode 100644 index 0000000..1207aa9 --- /dev/null +++ b/src/google/sessions/operation_service.cpp @@ -0,0 +1,123 @@ +#include "google/sessions/operation_service.h" + +#include "clients/url.h" +#include "google/errors.h" +#include "google/sessions/caps.h" +#include "google/sessions/names.h" + +namespace molo::google::sessions { + +using clients::body::Json; + +OperationService::OperationService(std::shared_ptr client, + ServiceEndpoint endpoint, std::string root, + Sleeper sleeper) + : Service(std::move(client), std::move(endpoint)), + root_(std::move(root)), + sleeper_(std::move(sleeper)) {} + +Json OperationService::send(Call call, const std::string& resource, const Json* body, + const std::string& query) const { + const CallSpec& row = spec_of(call); + return client_->request(verb_of(row), endpoint_, expand(call, resource, query), body); +} + +Operation OperationService::settle(const Json& reply) const { + Operation op = Operation::from_json(reply); + if (op.done) { + if (op.failed()) + throw GoogleError(502, "OPERATION_FAILED", op.error_message, endpoint_.name); + return op; + } + if (op.name.empty()) + throw GoogleError(502, "EMPTY_RESPONSE", "no operation returned", endpoint_.name); + return wait(op.name); +} + +bool list_over_limit(std::size_t item_count, bool more_pages, std::size_t page_index, + std::size_t max_pages, std::size_t max_items) { + return item_count > max_items || (more_pages && page_index + 1 >= max_pages); +} + +Json OperationService::operation(const std::string& operation_name) const { + OperationName name = OperationName::parse(root_, operation_name); + return send(name.call(OperationVerb::kGet), name.str()); +} + +void OperationService::cancel_operation(const std::string& operation_name) const { + OperationName name = OperationName::parse(root_, operation_name); + Json empty = Json::object(); + send(name.call(OperationVerb::kCancel), name.str(), &empty); +} + +void OperationService::delete_operation(const std::string& operation_name) const { + OperationName name = OperationName::parse(root_, operation_name); + send(name.call(OperationVerb::kDelete), name.str()); +} + +Operation OperationService::wait(const std::string& operation_name) const { + OperationName name = OperationName::parse(root_, operation_name); + return wait_operation(*client_, endpoint_, name.str(), sleeper_); +} + +Operation OperationService::wait_server(const std::string& operation_name, + int timeout_seconds) const { + OperationName name = OperationName::parse(root_, operation_name); + if (timeout_seconds < 1 || + timeout_seconds > static_cast(SessionsCaps::kMaxServerWaitSeconds)) + throw GoogleError::invalid("timeout_seconds must be 1.." + + std::to_string(SessionsCaps::kMaxServerWaitSeconds)); + Json empty = Json::object(); + Operation op = Operation::from_json( + send(name.call(OperationVerb::kWait), name.str(), &empty, + query_of({{"timeout", std::to_string(timeout_seconds) + "s"}}))); + if (op.done && op.failed()) + throw GoogleError(502, "OPERATION_FAILED", op.error_message, endpoint_.name); + return op; +} + +std::vector OperationService::list_operations( + const std::string& parent, const OperationListOptions& options) const { + OperationName where = OperationName::parse_parent(root_, parent.empty() ? root_ : parent); + Check::expression("filter", options.filter); + Check::page_size(options.page_size, SessionsCaps::kPageSize); + Json items = list_items( + where.call(OperationVerb::kList), where.str(), + query_of({{"filter", options.filter}, + {"pageSize", options.page_size > 0 ? std::to_string(options.page_size) : ""}}), + "operations", SessionsCaps::kMaxListPages, SessionsCaps::kMaxListItems); + std::vector out; + for (const Json& item : items.as_array()) out.push_back(Operation::from_json(item)); + return out; +} + +Json OperationService::list_items(Call call, const std::string& resource, + const std::string& query, const std::string& items_key, + std::size_t max_pages, std::size_t max_items) const { + // Own loop instead of list_all: list_all stops silently at max_pages, and a + // short page size would hide that. Anything left over is a loud error. + std::string path = expand(call, resource, query); + Json items = Json::array(); + std::string token; + for (std::size_t page = 0;; ++page) { + std::string url = path; + if (!token.empty()) + url += (url.find('?') == std::string::npos ? "?" : "&") + + std::string("pageToken=") + clients::Url::encode_component(token); + Json reply = client_->request(clients::Method::kGet, endpoint_, url, nullptr); + if (const Json* arr = reply.is_object() ? reply.find(items_key) : nullptr; + arr && arr->is_array()) + for (const Json& item : arr->as_array()) items.push(item); + const Json* next = reply.is_object() ? reply.find("nextPageToken") : nullptr; + token = next && next->is_string() ? next->as_string() : ""; + if (list_over_limit(items.as_array().size(), !token.empty(), page, max_pages, + max_items)) + throw GoogleError(502, "RESULT_TOO_LARGE", + "more than " + std::to_string(max_items) + " items or " + + std::to_string(max_pages) + " pages; narrow the filter", + endpoint_.name); + if (token.empty()) return items; + } +} + +} // namespace molo::google::sessions diff --git a/src/google/sessions/operation_service.h b/src/google/sessions/operation_service.h new file mode 100644 index 0000000..7702590 --- /dev/null +++ b/src/google/sessions/operation_service.h @@ -0,0 +1,72 @@ +#pragma once + +// Table-driven requests and the whole operations surface (get, list, cancel, +// delete, wait on the server, wait by polling) for one validated root: an +// Agent Engine (Sessions, MemoryBank) or a location (ServingProfiles). The +// root bounds every operation name a caller can reach. + +#include +#include +#include + +#include "google/operation.h" +#include "google/service.h" +#include "google/sessions/calls.h" +#include "google/sessions/operation_names.h" + +namespace molo::google::sessions { + +struct OperationListOptions { + std::string filter; // AIP-160 on name, done, ... + int page_size = 0; // 0: Google default +}; + +// True when a list must stop loudly: more items than max_items, or a further +// page exists after page `page_index` (0 based) used up max_pages. Lists never +// truncate silently; the caller narrows the filter instead. +bool list_over_limit(std::size_t item_count, bool more_pages, std::size_t page_index, + std::size_t max_pages, std::size_t max_items); + +class OperationService : public Service { + public: + // Raw operation document (GET). + clients::body::Json operation(const std::string& operation_name) const; + // POST :cancel. Best effort at Google; the operation may still finish. + void cancel_operation(const std::string& operation_name) const; + // Destructive: removes the operation record (DELETE). Explicit method; it + // never cancels the work. + void delete_operation(const std::string& operation_name) const; + // Polls by GET until done (OperationCaps); throws on failure. + Operation wait(const std::string& operation_name) const; + // One server-side wait (POST :wait?timeout=Ns), 1..kMaxServerWaitSeconds. + // Returns the operation as Google answers it: done may still be false when + // the timeout passed first. A failed operation throws. + Operation wait_server(const std::string& operation_name, int timeout_seconds) const; + // Operations under `parent` (default: the root engine; for a location root + // a serving profile name is required). Capped by SessionsCaps list limits. + std::vector list_operations(const std::string& parent = "", + const OperationListOptions& options = {}) const; + + const std::string& root() const { return root_; } + + protected: + OperationService(std::shared_ptr client, ServiceEndpoint endpoint, + std::string root, Sleeper sleeper); + + // One table-driven request. `resource` fills the {parent} / {name} slot. + clients::body::Json send(Call call, const std::string& resource, + const clients::body::Json* body = nullptr, + const std::string& query = "") const; + // Follows an Operation reply to completion (done replies are checked). + Operation settle(const clients::body::Json& reply) const; + // Paged list through the table; caps pages and total items. + clients::body::Json list_items(Call call, const std::string& resource, + const std::string& query, + const std::string& items_key, std::size_t max_pages, + std::size_t max_items) const; + + std::string root_; + Sleeper sleeper_; +}; + +} // namespace molo::google::sessions diff --git a/src/google/sessions/serving_profile.cpp b/src/google/sessions/serving_profile.cpp new file mode 100644 index 0000000..c94289e --- /dev/null +++ b/src/google/sessions/serving_profile.cpp @@ -0,0 +1,165 @@ +#include "google/sessions/serving_profile.h" + +#include "google/errors.h" +#include "google/resource.h" +#include "google/sessions/caps.h" + +namespace molo::google::sessions { + +using clients::body::Json; + +const char* serving_scope_name(ServingScope scope) { + switch (scope) { + case ServingScope::kGeminiLive: return "GEMINI_LIVE"; + case ServingScope::kInteractionsApi: return "INTERACTIONS_API"; + case ServingScope::kResponseApi: return "RESPONSE_API"; + } + return "GEMINI_LIVE"; +} + +ServingScope serving_scope_from(const std::string& name) { + for (ServingScope s : {ServingScope::kGeminiLive, ServingScope::kInteractionsApi, + ServingScope::kResponseApi}) + if (name == serving_scope_name(s)) return s; + throw GoogleError::invalid( + "scope must be GEMINI_LIVE, INTERACTIONS_API or RESPONSE_API"); +} + +namespace { + +void check_kms(const std::string& key) { + if (key.empty()) return; + Check::text("kms_key_name", key, SessionsCaps::kMaxKmsKeyChars); + if (key.rfind("projects/", 0) != 0 || key.find("/cryptoKeys/") == std::string::npos || + key.find("..") != std::string::npos || key.find('?') != std::string::npos) + throw GoogleError::invalid( + "kms_key_name must be projects/{p}/locations/{l}/keyRings/{k}/cryptoKeys/{c}"); +} + +Json cmek(const std::string& key) { + Json spec = Json::object(); + spec.set("kmsKeyName", Json(key)); + Json out = Json::object(); + out.set("encryptionSpec", spec); + return out; +} + +} // namespace + +void ServingProfileSpec::validate() const { + Check::serving_profile_id(profile_id); + Check::text("display_name", display_name, SessionsCaps::kMaxDisplayChars, true); + Check::text("description", description, SessionsCaps::kMaxDescriptionChars); + check_kms(kms_key_name); +} + +Json ServingProfileSpec::to_json() const { + Json out = Json::object(); + out.set("displayName", Json(display_name)); + if (!description.empty()) out.set("description", Json(description)); + out.set("scope", Json(serving_scope_name(scope))); + if (!kms_key_name.empty()) out.set("cmekConfig", cmek(kms_key_name)); + return out; +} + +void ServingProfilePatch::validate() const { + Check::serving_profile_id(profile_id); + Check::text("update_mask", update_mask, 256, true); + for (char c : update_mask) { + bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == ',' || c == '.'; + if (!ok) throw GoogleError::invalid("update_mask allows letters, comma, dot"); + } + Check::text("display_name", display_name, SessionsCaps::kMaxDisplayChars); + Check::text("description", description, SessionsCaps::kMaxDescriptionChars); + check_kms(kms_key_name); +} + +Json ServingProfilePatch::to_json() const { + Json out = Json::object(); + if (!display_name.empty()) out.set("displayName", Json(display_name)); + if (!description.empty()) out.set("description", Json(description)); + if (!kms_key_name.empty()) out.set("cmekConfig", cmek(kms_key_name)); + return out; +} + +std::string ServingProfile::id() const { + std::size_t slash = name.rfind('/'); + return slash == std::string::npos ? name : name.substr(slash + 1); +} + +ServingProfile ServingProfile::from_json(const Json& doc) { + ServingProfile p; + p.name = str_at(doc, "name"); + p.display_name = str_at(doc, "displayName"); + p.description = str_at(doc, "description"); + p.scope = str_at(doc, "scope"); + p.create_time = str_at(doc, "createTime"); + p.update_time = str_at(doc, "updateTime"); + if (const Json* c = doc.is_object() ? doc.find("cmekConfig") : nullptr) + if (const Json* e = c->is_object() ? c->find("encryptionSpec") : nullptr) + p.kms_key_name = str_at(*e, "kmsKeyName"); + return p; +} + +Json ServingProfile::to_json() const { + Json out = Json::object(); + out.set("name", Json(name)); + out.set("display_name", Json(display_name)); + out.set("description", Json(description)); + out.set("scope", Json(scope)); + out.set("kms_key_name", Json(kms_key_name)); + out.set("create_time", Json(create_time)); + out.set("update_time", Json(update_time)); + return out; +} + +ServingProfiles::ServingProfiles(std::shared_ptr client, + LocationName location, Sleeper sleeper) + : OperationService(std::move(client), endpoints::aiplatform_beta(location.location()), + location.str(), std::move(sleeper)), + location_(std::move(location)) {} + +ServingProfile ServingProfiles::create(const ServingProfileSpec& spec) const { + spec.validate(); + Json body = spec.to_json(); + Operation op = settle(send(Call::kServingProfileCreate, location_.str(), &body, + query_of({{"servingProfileId", spec.profile_id}}))); + if (op.response.is_object() && !str_at(op.response, "name").empty()) + return ServingProfile::from_json(op.response); + return get(spec.profile_id); +} + +ServingProfile ServingProfiles::get(const std::string& profile_id) const { + return ServingProfile::from_json( + send(Call::kServingProfileGet, location_.serving_profile(profile_id))); +} + +std::vector ServingProfiles::list(int page_size) const { + Check::page_size(page_size, SessionsCaps::kPageSize); + Json items = list_items( + Call::kServingProfileList, location_.str(), + query_of({{"pageSize", page_size > 0 ? std::to_string(page_size) : ""}}), + "servingProfiles", SessionsCaps::kMaxListPages, SessionsCaps::kMaxListItems); + std::vector out; + for (const Json& item : items.as_array()) out.push_back(ServingProfile::from_json(item)); + return out; +} + +ServingProfile ServingProfiles::patch(const ServingProfilePatch& patch) const { + patch.validate(); + Json body = patch.to_json(); + return ServingProfile::from_json( + send(Call::kServingProfilePatch, location_.serving_profile(patch.profile_id), &body, + query_of({{"updateMask", patch.update_mask}}))); +} + +void ServingProfiles::remove(const std::string& profile_id) const { + send(Call::kServingProfileDelete, location_.serving_profile(profile_id)); +} + +std::vector ServingProfiles::list_profile_operations( + const std::string& profile_id, const OperationListOptions& options) const { + return list_operations(location_.serving_profile(profile_id), options); +} + +} // namespace molo::google::sessions diff --git a/src/google/sessions/serving_profile.h b/src/google/sessions/serving_profile.h new file mode 100644 index 0000000..2ead12c --- /dev/null +++ b/src/google/sessions/serving_profile.h @@ -0,0 +1,82 @@ +#pragma once + +// Serving profiles (aiplatform v1beta1 projects.locations.servingProfiles): +// location-level settings for resource-less GenAI serving (Gemini Live, +// Interactions API, Response API). Not engine scoped, so ServingProfiles +// takes a location, not an Agent Engine. + +#include +#include +#include + +#include "google/sessions/names.h" +#include "google/sessions/operation_service.h" + +namespace molo::google::sessions { + +// Discovery enum ServingProfile.scope (without the unused UNSPECIFIED). +enum class ServingScope { kGeminiLive, kInteractionsApi, kResponseApi }; + +const char* serving_scope_name(ServingScope scope); +// Throws GoogleError::invalid for any other string. +ServingScope serving_scope_from(const std::string& name); + +struct ServingProfileSpec { // Input to create. + std::string profile_id; // Required, becomes the servingProfileId query. + std::string display_name; // Required, up to 128 characters. + std::string description; + ServingScope scope = ServingScope::kGeminiLive; + std::string kms_key_name; // Optional CMEK: projects/.../cryptoKeys/... + + void validate() const; + clients::body::Json to_json() const; +}; + +// PATCH of one profile; `scope` is not patchable here (not in the patch row). +struct ServingProfilePatch { + std::string profile_id; + std::string display_name; + std::string description; + std::string kms_key_name; + std::string update_mask; // Required: displayName,description,cmekConfig + + void validate() const; + clients::body::Json to_json() const; +}; + +struct ServingProfile { + std::string name; + std::string display_name; + std::string description; + std::string scope; // As Google sent it, kept even when unknown to the enum. + std::string kms_key_name; + std::string create_time, update_time; + + std::string id() const; // Last component of name. + static ServingProfile from_json(const clients::body::Json& doc); + clients::body::Json to_json() const; +}; + +class ServingProfiles : public OperationService { + public: + ServingProfiles(std::shared_ptr client, LocationName location, + Sleeper sleeper = nullptr); + + const LocationName& location() const { return location_; } + + ServingProfile create(const ServingProfileSpec& spec) const; // waits + ServingProfile get(const std::string& profile_id) const; + std::vector list(int page_size = 0) const; + ServingProfile patch(const ServingProfilePatch& patch) const; + // Destructive: explicit method. + void remove(const std::string& profile_id) const; + + // Operations of one serving profile (name from list_operations or create). + std::vector list_profile_operations(const std::string& profile_id, + const OperationListOptions& o = {}) const; + + private: + LocationName location_; +}; + +} // namespace molo::google::sessions diff --git a/src/google/sessions/sessions.cpp b/src/google/sessions/sessions.cpp index 435094d..1e5a0a4 100644 --- a/src/google/sessions/sessions.cpp +++ b/src/google/sessions/sessions.cpp @@ -86,4 +86,9 @@ std::vector Sessions::list_events(const std::string& session_id, return out; } +std::vector Sessions::list_session_operations( + const std::string& session_id, const OperationListOptions& options) const { + return list_operations(engine_name_.session(session_id), options); +} + } // namespace molo::google::sessions diff --git a/src/google/sessions/sessions.h b/src/google/sessions/sessions.h index 98a0a84..75db228 100644 --- a/src/google/sessions/sessions.h +++ b/src/google/sessions/sessions.h @@ -30,6 +30,10 @@ class Sessions : public EngineService { void append_event(const std::string& session_id, const SessionEvent& event) const; std::vector list_events(const std::string& session_id, const EventListOptions& options = {}) const; + + // Operations that belong to one session (create, compact, delete runs). + std::vector list_session_operations( + const std::string& session_id, const OperationListOptions& options = {}) const; }; } // namespace molo::google::sessions diff --git a/src/google/sessions/tools/check_discovery.py b/src/google/sessions/tools/check_discovery.py index 8957738..1eff66a 100644 --- a/src/google/sessions/tools/check_discovery.py +++ b/src/google/sessions/tools/check_discovery.py @@ -34,6 +34,7 @@ ENUMS = [ ("Memory", "memoryType", ["NATURAL_LANGUAGE_COLLECTION", "STRUCTURED_PROFILE"]), ("CompactionConfigLlmSummarizationConfig", "mode", ["AUTO"]), ("CompactionConfigEventEditingConfig", "mode", ["AUTO"]), + ("ServingProfile", "scope", ["GEMINI_LIVE", "INTERACTIONS_API", "RESPONSE_API"]), ] # (schema suffix, property) read by the response parsers. @@ -51,6 +52,12 @@ RESPONSE_KEYS = [ ("MemoryRevision", "fact"), ("MemoryRevision", "context"), ("MemoryRevision", "labels"), ("MemoryRevision", "createTime"), ("MemoryRevision", "expireTime"), + ("ListServingProfilesResponse", "servingProfiles"), + ("ListServingProfilesResponse", "nextPageToken"), + ("ServingProfile", "name"), ("ServingProfile", "displayName"), + ("ServingProfile", "description"), ("ServingProfile", "scope"), + ("ServingProfile", "createTime"), ("ServingProfile", "updateTime"), + ("ServingProfile", "cmekConfig"), ("Session", "name"), ("Session", "createTime"), ("Session", "updateTime"), ("Session", "expireTime"), ("Memory", "name"), ("Memory", "expireTime"), ] @@ -73,12 +80,30 @@ NESTED = [ ("MemoryFilter", "key op negate value"), ("MemoryMetadataValue", "stringValue doubleValue boolValue timestampValue"), ("MemoryTopicId", "customMemoryTopicLabel managedMemoryTopic"), + ("ServingProfileCmekConfig", "encryptionSpec"), + ("EncryptionSpec", "kmsKeyName"), ("CompactionConfig", "summarization eventEditing"), ("CompactionConfigLlmSummarizationConfig", "mode"), ("CompactionConfigEventEditingConfig", "mode"), ] +# Discovery methods this module owns: every one must be a table row. +OWNED_PREFIXES = [ + "aiplatform.projects.locations.reasoningEngines.operations.", + "aiplatform.projects.locations.reasoningEngines.sessions.", + "aiplatform.projects.locations.reasoningEngines.memories.", + "aiplatform.projects.locations.servingProfiles.", +] +# method id -> reason it is not wrapped (none today). +EXCLUDED = {} + +def collect_methods(res, out): + out.extend(res.get("methods", {}).values()) + for r in res.get("resources", {}).values(): + collect_methods(r, out) + + def load(path): if path: with open(path, "r", encoding="utf-8") as f: @@ -144,7 +169,7 @@ def main(): errors.append("calls.h: enum Call and table rows differ") for row in rows: - mid = prefix + row["method"] + mid = row["method"] if row["method"].startswith("aiplatform.") else prefix + row["method"] m = find_method(doc, mid) if m is None: errors.append("%s: method %s absent" % (row["call"], mid)) @@ -171,6 +196,25 @@ def main(): errors.append("%s: discovery has a request body %s the table omits" % (row["call"], m["request"]["$ref"])) + # Coverage: every Discovery method of the resources this module owns must + # be a table row (or be listed, with a reason, in EXCLUDED). + declared_ids = set() + for row in rows: + declared_ids.add(row["method"] if row["method"].startswith("aiplatform.") + else prefix + row["method"]) + owned = [] + collect_methods(doc, owned) + covered = 0 + for m in owned: + if not any(m["id"].startswith(p) for p in OWNED_PREFIXES): + continue + if m["id"] in declared_ids: + covered += 1 + elif m["id"] in EXCLUDED: + continue + else: + errors.append("discovery method %s is not in the call table" % m["id"]) + for suffix, prop, values in ENUMS: s = schemas.get(schema_prefix + suffix, {}).get("properties", {}).get(prop) if s is None: @@ -182,6 +226,8 @@ def main(): for v in values: if v not in allowed: errors.append("enum value %s absent from %s.%s" % (v, suffix, prop)) + if "operations" not in schemas.get("GoogleLongrunningListOperationsResponse", {}).get("properties", {}): + errors.append("response key GoogleLongrunningListOperationsResponse.operations absent") for suffix, prop in RESPONSE_KEYS: if prop not in schemas.get(schema_prefix + suffix, {}).get("properties", {}): errors.append("response key %s.%s absent" % (suffix, prop)) diff --git a/tests/example/agent/armor/scc_test.cpp b/tests/example/agent/armor/scc_test.cpp new file mode 100644 index 0000000..3abb2fa --- /dev/null +++ b/tests/example/agent/armor/scc_test.cpp @@ -0,0 +1,96 @@ +// Routes: GET /agent/armor/categories (no credential) and the Security +// Command Center administration routes under /agent/armor/scc. Without a +// credential a valid request answers 503 google_not_configured; bad input is +// 422 before any traffic; a wrong verb is 405. Google replies are not scripted. +#include +#include + +#include "support/GoogleRig.hpp" + +using namespace molo::example::test; +using molo::example::fw::Method; + +int main() { + setenv("GOOGLE_CLOUD_PROJECT", "demo-project", 1); + const std::string json = "application/json"; + const std::string finding = "organizations/1/sources/2/findings/f1"; + struct Route { + std::string path, valid, invalid; + }; + const std::vector routes = { + {"/agent/armor/scc/availability", "{\"scope\":\"projects/demo-project\"}", + "{\"scope\":\"projects/demo-project/locations/global\"}"}, + {"/agent/armor/scc/configs/list", + "{\"kind\":\"muteConfigs\",\"parent\":\"projects/demo-project/locations/global\"}", + "{\"kind\":\"widgets\",\"parent\":\"projects/demo-project\"}"}, + {"/agent/armor/scc/configs/get", + "{\"kind\":\"muteConfigs\",\"name\":\"projects/demo-project/muteConfigs/m1\"}", + "{\"kind\":\"muteConfigs\",\"name\":\"projects/demo-project\"}"}, + {"/agent/armor/scc/finding/state", "{\"finding\":\"" + finding + "\",\"state\":\"INACTIVE\"}", + "{\"finding\":\"" + finding + "\",\"state\":\"MAYBE\"}"}, + {"/agent/armor/scc/finding/mute", "{\"finding\":\"" + finding + "\",\"mute\":\"MUTED\"}", + "{\"finding\":\"organizations/1/sources/2\",\"mute\":\"MUTED\"}"}, + {"/agent/armor/scc/bulk-mute", + "{\"parent\":\"projects/demo-project\",\"filter\":\"severity=\\\"LOW\\\"\",\"value\":\"MUTED\"," + "\"confirm\":\"bulk-mute\"}", + "{\"parent\":\"projects/demo-project\",\"filter\":\"\",\"value\":\"MUTED\",\"confirm\":\"bulk-mute\"}"}, + {"/agent/armor/scc/operations/list", "{\"organization\":\"organizations/1\"}", + "{\"organization\":\"projects/demo-project\"}"}, + }; + { + GoogleRig rig(false); + Fixture f(rig.client); + for (const auto& r : routes) { + auto res = f.client->post(r.path, r.valid, json); + check(res.status == 503 && contains(res.body, "google_not_configured"), + "503 google_not_configured: " + r.path); + } + auto del = f.client->request(Method::Delete, + "/agent/armor/scc/configs?kind=muteConfigs&name=projects/demo-project/muteConfigs/m1"); + check(del.status == 503 && contains(del.body, "google_not_configured"), "delete config: 503 without a credential"); + check(rig.mock->send_count() == 0, "no upstream traffic without a credential"); + } + { + GoogleRig rig; + Fixture f(rig.client); + for (const auto& r : routes) { + auto res = f.client->post(r.path, r.invalid, json); + check(res.status == 422 && contains(res.body, "validation_error"), "422 on bad input: " + r.path); + check(f.client->get(r.path).status == 405, "get is 405: " + r.path); + } + // Limit and guard cases. + check(f.client->post("/agent/armor/scc/bulk-mute", + "{\"parent\":\"projects/demo-project\",\"filter\":\"x\",\"value\":\"MUTED\"}", json) + .status == 422, + "bulk mute without confirm is 422"); + check(f.client->post("/agent/armor/scc/bulk-mute", + "{\"parent\":\"projects/demo-project\",\"filter\":\"x\",\"value\":\"UNMUTED\"," + "\"confirm\":\"bulk-mute\"}", json).status == 422, + "bulk mute value UNMUTED is not allowed (422)"); + check(f.client->request(Method::Delete, "/agent/armor/scc/configs?kind=x&name=y").status == 422, + "delete config: unknown kind is 422"); + check(f.client->request(Method::Delete, + "/agent/armor/scc/configs?kind=muteConfigs&name=projects/demo-project").status == 422, + "delete config: a parent is not a config name (422)"); + check(f.client->post("/agent/armor/scc/configs").status == 405, "post on the delete route is 405"); + check(f.client->post("/agent/armor/findings", "{\"scope\":\"projects/demo-project\",\"family\":\"bogus\"}", + json).status == 422, + "findings: unknown family is 422"); + check(rig.mock->send_count() == 0, "side effect: rejected before any upstream call"); + } + { + // The category table needs no credential and never calls Google. + GoogleRig rig(false); + Fixture f(rig.client); + auto res = f.client->get("/agent/armor/categories"); + check(res.status == 200 && contains(res.headers.get("Content-Type"), "application/json"), "categories 200 json"); + check(contains(res.body, "Execution: Container Escape") && contains(res.body, "AGENT_ENGINE_CONTAINER_ESCAPE"), + "lists a documented runtime category and its module"); + check(contains(res.body, "AGENT_ENGINE_REVERSE_SHELL") && contains(res.body, "control_plane"), + "lists both families"); + check(contains(res.body, "agentAnomaly"), "says how anomaly findings are recognised"); + check(f.client->post("/agent/armor/categories").status == 405, "post is 405"); + check(rig.mock->send_count() == 0, "side effect: GET never calls Google"); + } + return finish("agent/armor/scc"); +} diff --git a/tests/example/agent/optimize/gaps_test.cpp b/tests/example/agent/optimize/gaps_test.cpp new file mode 100644 index 0000000..5f5df72 --- /dev/null +++ b/tests/example/agent/optimize/gaps_test.cpp @@ -0,0 +1,71 @@ +// Routes: POST /agent/optimize/synthetic, /operations/{list,get,wait,cancel}, +// DELETE /agent/optimize/operations, POST /agent/optimize/logs/write +#include +#include + +#include "support/GoogleRig.hpp" + +using namespace molo::example::test; +using molo::example::fw::Method; + +namespace { +const std::string kJson = "application/json"; +const std::string kEval = "projects/test-proj/locations/us-central1/onlineEvaluators/1"; +const std::string kOp = kEval + "/operations/7"; + +struct Case { + std::string path, good, bad; +}; +} // namespace + +int main() { + setenv("GOOGLE_CLOUD_PROJECT", "test-proj", 1); + const Case cases[] = { + {"/agent/optimize/synthetic", "{\"request\":{\"taskDescription\":\"x\"}}", "{\"request\":{\"nope\":1}}"}, + {"/agent/optimize/operations/list", "{\"evaluator\":\"" + kEval + "\"}", + "{\"evaluator\":\"projects/p/locations/l/evaluationRuns/r\"}"}, + {"/agent/optimize/operations/get", "{\"name\":\"" + kOp + "\"}", "{\"name\":\"operations/7\"}"}, + {"/agent/optimize/operations/wait", "{\"name\":\"" + kOp + "\",\"timeout_seconds\":\"60\"}", + "{\"name\":\"" + kOp + "\",\"timeout_seconds\":\"61\"}"}, + {"/agent/optimize/operations/cancel", "{\"name\":\"" + kOp + "\"}", "{\"name\":\"\"}"}, + {"/agent/optimize/logs/write", "{\"log_id\":\"agent-check\",\"text\":\"hello\"}", + "{\"log_id\":\"a/b\",\"text\":\"hello\"}"}, + }; + for (const Case& c : cases) { + { + GoogleRig none(false); + Fixture f(none.client); + auto res = f.client->post(c.path, c.good, kJson); + check(res.status == 503 && contains(res.body, "google_not_configured"), + c.path + ": no credential is a loud 503"); + check(none.mock->send_count() == 0, c.path + ": no network without a credential"); + } + { + GoogleRig keyed; + Fixture f(keyed.client); + auto res = f.client->post(c.path, c.bad, kJson); + check(res.status == 422 && contains(res.body, "validation_error"), c.path + ": bad input is 422"); + check(f.client->get(c.path).status == 405, c.path + ": get is 405"); + check(keyed.mock->send_count() == 0, c.path + ": rejected before any upstream call"); + } + } + { + GoogleRig none(false); + Fixture f(none.client); + auto del = f.client->request(Method::Delete, "/agent/optimize/operations?name=" + kOp); + check(del.status == 503 && contains(del.body, "google_not_configured"), + "delete operation: no credential is 503"); + check(none.mock->send_count() == 0, "no network without a credential"); + } + { + GoogleRig keyed; + Fixture f(keyed.client); + check(f.client->request(Method::Delete, "/agent/optimize/operations").status == 422, + "delete: missing name is 422"); + check(f.client->request(Method::Delete, "/agent/optimize/operations?name=x").status == 422, + "delete: bad name is 422"); + check(f.client->post("/agent/optimize/operations").status == 405, "post on the delete route is 405"); + check(keyed.mock->send_count() == 0, "side effect: rejected before any upstream call"); + } + return finish("agent/optimize/gaps"); +} diff --git a/tests/example/agent/registry/iam_test.cpp b/tests/example/agent/registry/iam_test.cpp new file mode 100644 index 0000000..9230792 --- /dev/null +++ b/tests/example/agent/registry/iam_test.cpp @@ -0,0 +1,27 @@ +// Routes: POST /agent/registry/deny-policies/list, POST /agent/registry/boundaries/bindings +#include "agent/registry/cases.hpp" + +using namespace molo::example::test; + +int main() { + check_google_post_route("/agent/registry/deny-policies/list", "{\"project_number\":\"123456\"}", + "{\"project_number\":\"my-project\"}", "non numeric project number"); + check_google_post_route("/agent/registry/boundaries/bindings", + "{\"org_id\":\"42\",\"id\":\"pab-1\"}", + "{\"org_id\":\"42\",\"id\":\"a/b\"}", "bad policy id"); + { + GoogleRig rig; + Fixture f(rig.client); + check(f.client->post("/agent/registry/deny-policies/list", "{}", "application/json").status == 422, + "limit: a missing project number is 422"); + check(f.client->post("/agent/registry/deny-policies/list", + "{\"project_number\":\"" + std::string(21, '1') + "\"}", "application/json") + .status == 422, + "limit: 21 digits is 422"); + check(f.client->post("/agent/registry/boundaries/bindings", "{\"org_id\":\"x\",\"id\":\"pab-1\"}", + "application/json").status == 422, + "boundary bindings: bad org id is 422"); + check(rig.mock->send_count() == 0, "side effect: rejected before any upstream call"); + } + return finish("agent/registry/iam"); +} diff --git a/tests/example/agent/registry/list_test.cpp b/tests/example/agent/registry/list_test.cpp index 9d00445..2ab3f74 100644 --- a/tests/example/agent/registry/list_test.cpp +++ b/tests/example/agent/registry/list_test.cpp @@ -14,6 +14,18 @@ int main() { check(f.client->post("/agent/registry/list", "{\"kind\":\"aiApplications\"}", "application/json").status == 422, "limit: a collection that is not listable here is 422"); check(f.client->post("/agent/registry/list", "", "application/json").status == 422, "empty body is 422"); + check(f.client->post("/agent/registry/list", "{\"kind\":\"agents\",\"version\":\"v2\"}", + "application/json").status == 422, + "limit: an unknown API version is 422"); + } + { + // v1alpha is accepted as input and still needs a credential. + GoogleRig rig(false); + Fixture f(rig.client); + auto res = f.client->post("/agent/registry/list", "{\"kind\":\"agents\",\"version\":\"v1alpha\"}", + "application/json"); + check(res.status == 503 && contains(res.body, "google_not_configured"), "v1alpha without a credential is 503"); + check(rig.mock->send_count() == 0, "side effect: no upstream traffic"); } return finish("agent/registry/list"); } diff --git a/tests/example/agent/registry/operations_test.cpp b/tests/example/agent/registry/operations_test.cpp new file mode 100644 index 0000000..12b578e --- /dev/null +++ b/tests/example/agent/registry/operations_test.cpp @@ -0,0 +1,39 @@ +// Routes: POST /agent/registry/operations/list, POST /agent/registry/operations/cancel, +// DELETE /agent/registry/operations (explicit destructive route) +#include "agent/registry/cases.hpp" + +using namespace molo::example::test; +using molo::example::fw::Method; + +int main() { + for (const char* scope : {"registry", "gateways", "policies"}) { + std::string s = scope; + check_google_post_route("/agent/registry/operations/list", "{\"scope\":\"" + s + "\"}", + "{\"scope\":\"nowhere\"}", "unknown scope"); + check_google_post_route("/agent/registry/operations/cancel", + "{\"scope\":\"" + s + "\",\"id\":\"op-1\"}", + "{\"scope\":\"" + s + "\",\"id\":\"a/b\"}", "bad operation id"); + } + { + GoogleRig rig(false); + Fixture f(rig.client); + auto del = f.client->request(Method::Delete, "/agent/registry/operations?scope=gateways&id=op-1"); + check(del.status == 503 && contains(del.body, "google_not_configured"), + "delete operation: 503 without a credential"); + check(rig.mock->send_count() == 0, "side effect: no upstream traffic"); + } + { + GoogleRig rig; + Fixture f(rig.client); + check(f.client->request(Method::Delete, "/agent/registry/operations?scope=x&id=op-1").status == 422, + "delete: unknown scope is 422"); + check(f.client->request(Method::Delete, "/agent/registry/operations?scope=policies&id=a..b").status == 422, + "delete: bad id is 422"); + check(f.client->request(Method::Delete, "/agent/registry/operations").status == 422, + "delete: missing parameters are 422"); + check(f.client->get("/agent/registry/operations").status == 405, "get never deletes"); + check(f.client->post("/agent/registry/operations").status == 405, "post never deletes"); + check(rig.mock->send_count() == 0, "side effect: rejected before any upstream call"); + } + return finish("agent/registry/operations"); +} diff --git a/tests/example/agent/registry/page_test.cpp b/tests/example/agent/registry/page_test.cpp index 099d8f2..8798255 100644 --- a/tests/example/agent/registry/page_test.cpp +++ b/tests/example/agent/registry/page_test.cpp @@ -13,6 +13,9 @@ int main() { check(res.status == 200, "status 200 without a credential"); check(contains(res.headers.get("Content-Type"), "text/html"), "html"); check(contains(res.body, "Agent registry") && contains(res.body, "agent_registry.js"), "page and script"); + check(contains(res.body, "id=\"r-truncated\""), "page has the loud truncation banner"); + check(contains(res.body, "id=\"r-version\"") && contains(res.body, "v1alpha"), "page offers v1alpha"); + check(contains(res.body, "ops-gateways"), "page offers the operations lists"); check(f.client->post("/agent/registry").status == 405, "post is 405"); check(f.client->get("/agent/registry/nope").status == 404, "unknown sub route is 404"); check(rig.mock->send_count() == 0, "side effect: no upstream traffic"); diff --git a/tests/example/agent/registry/status_test.cpp b/tests/example/agent/registry/status_test.cpp index 895b4fc..eabc44f 100644 --- a/tests/example/agent/registry/status_test.cpp +++ b/tests/example/agent/registry/status_test.cpp @@ -15,6 +15,7 @@ int main() { check(contains(res.headers.get("Content-Type"), "application/json"), "json"); check(contains(res.body, "\"configured\":false") && contains(res.body, "\"project_set\":true"), "state shown"); check(contains(res.body, "\"max_ledger_entries\":5000"), "caps shown"); + check(contains(res.body, "\"max_items\":1000"), "the list item cap is stated"); check(rig.mock->send_count() == 0, "side effect: no upstream traffic"); } { diff --git a/tests/example/agent/sessions/calls_test.cpp b/tests/example/agent/sessions/calls_test.cpp index ac93129..a512c4e 100644 --- a/tests/example/agent/sessions/calls_test.cpp +++ b/tests/example/agent/sessions/calls_test.cpp @@ -57,6 +57,26 @@ int main() { body("\"scope_key\":\"user_id\",\"scope_value\":\"1\""), body("\"scope_key\":\"\"")}, {"/agent/sessions/memories/purge", body("\"filter\":\"scope.user_id=\\\"1\\\"\""), body("\"filter\":\"\"")}, + {"/agent/sessions/operations/list", body("\"parent\":\"" + kEngine + "/sessions/chat-1\""), + body("\"parent\":\"" + kEngine + "/other/x\"")}, + {"/agent/sessions/operations/get", body("\"name\":\"" + kEngine + "/operations/7\""), + body("\"name\":\"" + kEngine + "/operations/..\"")}, + {"/agent/sessions/operations/wait", + body("\"name\":\"" + kEngine + "/operations/7\",\"timeout_seconds\":\"60\""), + body("\"name\":\"" + kEngine + "/operations/7\",\"timeout_seconds\":\"61\"")}, + {"/agent/sessions/operations/cancel", + body("\"name\":\"" + kEngine + "/memories/mem-1/operations/7\""), + body("\"name\":\"operations/7\"")}, + {"/agent/sessions/serving-profiles/list", "{\"location\":\"us-central1\"}", + "{\"location\":\"Bad Place\"}"}, + {"/agent/sessions/serving-profiles/get", + "{\"location\":\"us-central1\",\"profile_id\":\"sp-1\"}", + "{\"location\":\"us-central1\",\"profile_id\":\"Bad_Id\"}"}, + {"/agent/sessions/serving-profiles/create", + "{\"location\":\"us-central1\",\"profile_id\":\"sp-1\",\"display_name\":\"Live\"," + "\"scope\":\"GEMINI_LIVE\"}", + "{\"location\":\"us-central1\",\"profile_id\":\"sp-1\",\"display_name\":\"Live\"," + "\"scope\":\"EVERYTHING\"}"}, }; GoogleRig rig(false); @@ -119,6 +139,30 @@ int main() { 422, "delete memory: bad id is 422"); + auto del_op = f.client->request( + Method::Delete, "/agent/sessions/operations/operation?engine=" + kEngine + + "&name=" + kEngine + "/operations/7"); + check(del_op.status == 503 && contains(del_op.body, "google_not_configured"), + "delete operation: no credential is 503"); + check(f.client->request(Method::Delete, + "/agent/sessions/operations/operation?engine=" + kEngine + + "&name=" + kEngine + "/sessions").status == 422, + "delete operation: a name that is no operation is 422"); + check(f.client->post("/agent/sessions/operations/operation").status == 405, + "delete operation route: post is 405"); + auto del_profile = f.client->request( + Method::Delete, "/agent/sessions/serving-profiles/profile?location=us-central1&id=sp-1"); + check(del_profile.status == 503 && contains(del_profile.body, "google_not_configured"), + "delete serving profile: no credential is 503"); + check(f.client->request(Method::Delete, + "/agent/sessions/serving-profiles/profile?location=us-central1&id=A_B") + .status == 422, + "delete serving profile: bad id is 422"); + check(f.client->post("/agent/sessions/operations/wait", + body("\"name\":\"" + kEngine + "/operations/7\",\"timeout_seconds\":\"0\""), + kJson).status == 422, + "server-side wait of 0 seconds is 422"); + check(rig.mock->send_count() == 0, "side effect: nothing reached the network"); // With a credential, validation still runs first and still stops traffic. diff --git a/tests/example/agent/sessions/page_test.cpp b/tests/example/agent/sessions/page_test.cpp index cc0ff51..a38aee2 100644 --- a/tests/example/agent/sessions/page_test.cpp +++ b/tests/example/agent/sessions/page_test.cpp @@ -12,6 +12,8 @@ int main() { check(contains(page.headers.get("Content-Type"), "text/html"), "html"); check(contains(page.body, "Agent sessions and memory"), "heading"); check(contains(page.body, "/agent/sessions/memories/purge"), "lists the purge route"); + check(contains(page.body, "/agent/sessions/operations/wait"), "lists the server-side wait route"); + check(contains(page.body, "/agent/sessions/serving-profiles/list"), "lists serving profiles"); check(contains(page.body, "viewport"), "mobile viewport meta from the base template"); auto status = f.client->get("/agent/sessions/status"); @@ -20,6 +22,7 @@ int main() { check(contains(status.body, "\"configured\":true"), "configured with a credential"); check(contains(status.body, "aiplatform v1beta1"), "names the API surface"); check(contains(status.body, "\"max_top_k\":50"), "shows the top_k cap"); + check(contains(status.body, "\"max_server_wait_seconds\":60"), "shows the wait cap"); check(!contains(status.body, kTestKey), "credential never shown"); check(rig.mock->send_count() == 0, "side effect: GET pages never call Google"); } diff --git a/tests/src/google/armor/test_armor.cpp b/tests/src/google/armor/test_armor.cpp index 487b088..e9b4a3e 100644 --- a/tests/src/google/armor/test_armor.cpp +++ b/tests/src/google/armor/test_armor.cpp @@ -19,6 +19,7 @@ #include "google/armor/findings.h" #include "google/armor/guarded_model.h" #include "google/armor/model_armor.h" +#include "google/armor/security_admin.h" #include "google/credentials.h" using namespace molo::google; @@ -252,13 +253,13 @@ TEST(armor_model_armor, requests_match_discovery) { "modelarmor.projects.locations.templates.sanitizeModelResponse", std::string("/v1/") + p + "/templates/t1:sanitizeModelResponse"); FloorSetting f; check(on_fresh([&](ModelArmor& armor) { armor.get_floor_setting(); }), - "modelarmor.projects.locations.getFloorSetting", "/v1/projects/demo-project/locations/us-central1/floorSetting"); + "modelarmor.projects.locations.getFloorSetting", "/v1/projects/demo-project/locations/global/floorSetting"); check(on_fresh([&](ModelArmor& armor) { armor.update_floor_setting(f, {"enableFloorSettingEnforcement"}); }), - "modelarmor.projects.locations.updateFloorSetting", "/v1/projects/demo-project/locations/us-central1/floorSetting?updateMask="); + "modelarmor.projects.locations.updateFloorSetting", "/v1/projects/demo-project/locations/global/floorSetting?updateMask="); check(on_fresh([&](ModelArmor& armor) { armor.get_floor_setting("folders/42"); }), - "modelarmor.folders.locations.getFloorSetting", "/v1/folders/42/locations/us-central1/floorSetting"); + "modelarmor.folders.locations.getFloorSetting", "/v1/folders/42/locations/global/floorSetting"); check(on_fresh([&](ModelArmor& armor) { armor.get_floor_setting("organizations/7"); }), - "modelarmor.organizations.locations.getFloorSetting", "/v1/organizations/7/locations/us-central1/floorSetting"); + "modelarmor.organizations.locations.getFloorSetting", "/v1/organizations/7/locations/global/floorSetting"); check(on_fresh([&](ModelArmor& armor) { armor.list_locations(); }), "modelarmor.projects.locations.list", "/v1/projects/demo-project/locations"); check(on_fresh([&](ModelArmor& armor) { armor.get_location(); }), @@ -836,15 +837,51 @@ TEST(google_live_armor, oauth_token_reads_templates_and_findings) { } catch (const GoogleError& e) { if (!disabled(e)) throw; } + // Model Armor locations: list works on the global host only, get on the + // location's own host (observed live 2026-10-24). try { + ASSERT_TRUE(armor.list_locations().size() > 3); + ArmorLocation here = armor.get_location(); + ASSERT_EQ(here.location_id, ctx.location); + } catch (const GoogleError& e) { + if (!disabled(e)) throw; + } + // sanitize needs a template and this test never creates one. When none + // exists, a call against an absent template id proves routing and request + // shape: Google answers 404 NOT_FOUND for the template (observed live). + try { + if (armor.list_templates(1).templates.empty()) { + try { + armor.sanitize_user_prompt("molo-live-check-absent", "hello"); + FAIL("an absent template cannot sanitize"); + } catch (const GoogleError& e) { + std::cout << " sanitize against an absent template: " << e.http_status() << " " + << e.status() << ": " << e.message().substr(0, 160) << "\n"; + ASSERT_TRUE(e.http_status() == 404 || e.http_status() == 400); + } + } + } catch (const GoogleError& e) { + if (!disabled(e)) throw; + } + // Security Command Center. The API can be enabled while SCC is not + // activated for the project (no organization or tier): sources answers 404 + // and the config lists 400 FAILED_PRECONDITION. availability() classifies + // that instead of throwing; findings are listed only when it is available. + SecurityAdmin admin(client); + SccAvailability avail = admin.availability("projects/" + ctx.project); + std::cout << " security command center: " << availability_name(avail) << "\n"; + if (avail == SccAvailability::kAvailable) { SecurityFindings scc(client); FindingQuery q; q.scope = "projects/" + ctx.project; q.page_size = 5; FindingPage found = scc.list(q); std::cout << " findings listed: " << found.findings.size() << "\n"; - } catch (const GoogleError& e) { - if (!disabled(e)) throw; + SccList mutes = admin.list_configs(SccCollection::kMuteConfigs, q.scope + "/locations/global"); + std::cout << " mute configs: " << mutes.items.as_array().size() + << (mutes.truncated ? " (TRUNCATED)" : "") << "\n"; + } else { + std::cout << " findings and configs NOT exercised: SCC is not usable on this project\n"; } } diff --git a/tests/src/google/armor/test_armor_gaps.cpp b/tests/src/google/armor/test_armor_gaps.cpp new file mode 100644 index 0000000..50baf2a --- /dev/null +++ b/tests/src/google/armor/test_armor_gaps.cpp @@ -0,0 +1,330 @@ +// Armor gap closing: every Security Command Center v2 method outside the +// attack exposure engine (SecurityAdmin), the Agent Threat Detection category +// table, the floor setting location fix. Local only: requests are read back +// from the mock (a fresh client per probe, the circuit breaker opens after a +// run of failures); nothing answers (no scripted Google replies). +#include +#include +#include +#include +#include +#include + +#include "framework.h" +#include "clients/transports/mock_transport.h" +#include "google/armor/agent_categories.h" +#include "google/armor/calls.h" +#include "google/armor/caps.h" +#include "google/armor/findings.h" +#include "google/armor/model_armor.h" +#include "google/armor/scc_name.h" +#include "google/armor/security_admin.h" +#include "google/credentials.h" +#include "google/errors.h" + +using namespace molo::google; +using namespace molo::google::armor; +using molo::clients::body::Json; +using molo::clients::transports::MockTransport; + +namespace { + +struct Probe { + std::string method, target, body; + std::size_t sent = 0; + bool invalid = false; + std::string error; +}; + +// Runs f on a fresh client and reports what reached the wire. +Probe probe(const std::function& f, bool credential = true) { + auto t = std::make_unique(); + MockTransport* mock = t.get(); + auto client = std::make_shared( + std::move(t), credential ? Credentials::bearer("tok") : nullptr); + SecurityAdmin admin(client); + Probe p; + try { + f(admin); + } catch (const GoogleError& e) { + p.error = e.what(); + p.invalid = e.status() == "INVALID_ARGUMENT" && e.service().empty(); + } + p.sent = mock->send_count(); + if (p.sent) { + auto s = mock->seen().front(); + p.method = s.method; + p.target = s.target; + p.body = s.body; + } + return p; +} + +bool has(const std::string& hay, const std::string& needle) { + return hay.find(needle) != std::string::npos; +} + +const char* kProj = "projects/p"; +const char* kProjLoc = "projects/p/locations/global"; +const char* kFinding = "organizations/1/sources/2/findings/f1"; +const char* kFindingLoc = "organizations/1/sources/2/locations/global/findings/f1"; + +} // namespace + +TEST(armor_table, rows_are_well_formed_and_every_body_has_fields) { + std::set ids; + for (const CallSpec& c : kCalls) { + ASSERT_TRUE(ids.insert(c.rpc).second); + std::string http = c.http; + ASSERT_TRUE(http == "GET" || http == "POST" || http == "PATCH" || http == "DELETE"); + if (http == "GET" || http == "DELETE") ASSERT_EQ(std::string(c.request), std::string()); + ASSERT_EQ(std::string(c.request).empty(), std::string(c.fields).empty()); + } + ASSERT_TRUE(kCallCount >= 140); + // Families that must exist for each scope, with and without a location. + for (const char* scope : {"organizations", "folders", "projects"}) { + for (const char* tail : {"muteConfigs.list", "muteConfigs.create", "findings.bulkMute"}) { + ASSERT_TRUE(find_rpc(std::string("securitycenter.") + scope + "." + tail) != nullptr); + std::string loc = std::string(tail).rfind("findings", 0) == 0 + ? std::string("securitycenter.") + scope + ".locations." + tail + : std::string("securitycenter.") + scope + ".locations." + tail; + ASSERT_TRUE(find_rpc(loc) != nullptr); + } + } +} + +TEST(armor_names, rows_follow_the_name) { + SccName a = SccName::parse("projects/p/locations/global/muteConfigs/m1"); + ASSERT_TRUE(a.has_location()); + ASSERT_EQ(a.rpc_for("muteConfigs.get"), std::string("securitycenter.projects.locations.muteConfigs.get")); + ASSERT_TRUE(a.row("muteConfigs.get") != nullptr); + SccName b = SccName::parse("projects/p/muteConfigs/m1"); + ASSERT_FALSE(b.has_location()); + ASSERT_EQ(b.rpc_for("muteConfigs.get"), std::string("securitycenter.projects.muteConfigs.get")); + SccName f = SccName::parse(kFindingLoc); + ASSERT_TRUE(f.has_location()); + ASSERT_EQ(f.rpc_for("sources.findings.setState"), + std::string("securitycenter.organizations.sources.locations.findings.setState")); + ASSERT_EQ(SccName::parse(kFinding).rpc_for("sources.findings.setState"), + std::string("securitycenter.organizations.sources.findings.setState")); + // Notification configs and BigQuery exports exist only with a location. + ASSERT_TRUE(SccName::parse("projects/p/notificationConfigs/n").row("notificationConfigs.get") == nullptr); + ASSERT_TRUE(SccName::parse("projects/p/locations/global/notificationConfigs/n").row("notificationConfigs.get") != nullptr); + ASSERT_TRUE(SccName::parse("folders/9/locations/global/bigQueryExports/b").row("bigQueryExports.get") != nullptr); + for (const char* bad : {"", "x/1", "projects", "projects/", "/projects/p", "projects/p/../q", + "projects/p//q", "projects/a b", "sources/1/x"}) + ASSERT_THROW(SccName::parse(bad), GoogleError); +} + +TEST(armor_admin, config_calls_use_the_declared_rows) { + for (SccCollection c : {SccCollection::kMuteConfigs, SccCollection::kNotificationConfigs, + SccCollection::kBigQueryExports}) { + std::string coll = scc_collection_name(c); + std::string parent = kProjLoc; + std::string name = parent + "/" + coll + "/c1"; + Probe p = probe([&](SecurityAdmin& a) { a.list_configs(c, parent); }); + ASSERT_EQ(p.method, std::string("GET")); + ASSERT_TRUE(has(p.target, "/v2/" + parent + "/" + coll + "?pageSize=100")); + p = probe([&](SecurityAdmin& a) { a.get_config(c, name); }); + ASSERT_TRUE(has(p.target, "/v2/" + name)); + p = probe([&](SecurityAdmin& a) { a.delete_config(c, name); }); + ASSERT_EQ(p.method, std::string("DELETE")); + Json body = Json::object(); + body.set("description", "d"); + p = probe([&](SecurityAdmin& a) { a.create_config(c, parent, "c1", body); }); + ASSERT_EQ(p.method, std::string("POST")); + ASSERT_TRUE(has(p.target, "/v2/" + parent + "/" + coll + "?")); + ASSERT_TRUE(has(p.target, std::string(c == SccCollection::kMuteConfigs ? "muteConfigId=c1" + : c == SccCollection::kNotificationConfigs ? "configId=c1" + : "bigQueryExportId=c1"))); + p = probe([&](SecurityAdmin& a) { a.update_config(c, name, body, "description"); }); + ASSERT_EQ(p.method, std::string("PATCH")); + ASSERT_TRUE(has(p.target, "updateMask=description")); + } + // Without a location the mute configs also exist; the other two do not. + Probe p = probe([&](SecurityAdmin& a) { a.list_configs(SccCollection::kMuteConfigs, kProj); }); + ASSERT_TRUE(has(p.target, "/v2/projects/p/muteConfigs?pageSize=100")); + p = probe([&](SecurityAdmin& a) { a.list_configs(SccCollection::kBigQueryExports, kProj); }); + ASSERT_TRUE(p.invalid); + ASSERT_EQ(p.sent, std::size_t(0)); +} + +TEST(armor_admin, bad_input_is_refused_before_traffic) { + Json body = Json::object(); + body.set("description", "d"); + Json bogus = Json::object(); + bogus.set("notAField", 1); + std::vector> bad = { + [&](SecurityAdmin& a) { a.list_configs(SccCollection::kMuteConfigs, std::string(kProjLoc) + "/muteConfigs"); }, + [&](SecurityAdmin& a) { a.get_config(SccCollection::kMuteConfigs, kProjLoc); }, + [&](SecurityAdmin& a) { a.create_config(SccCollection::kMuteConfigs, kProj, "a/b", body); }, + [&](SecurityAdmin& a) { a.create_config(SccCollection::kMuteConfigs, kProj, "ok", bogus); }, + [&](SecurityAdmin& a) { a.create_config(SccCollection::kMuteConfigs, kProj, "ok", Json::array()); }, + [&](SecurityAdmin& a) { a.update_config(SccCollection::kMuteConfigs, "projects/p/muteConfigs/m", body, ""); }, + [&](SecurityAdmin& a) { a.update_config(SccCollection::kMuteConfigs, "projects/p/muteConfigs/m", body, "nope"); }, + [&](SecurityAdmin& a) { a.update_config(SccCollection::kMuteConfigs, "projects/p/muteConfigs/m", body, "descr iption"); }, + [&](SecurityAdmin& a) { a.delete_config(SccCollection::kMuteConfigs, "projects/p"); }, + [&](SecurityAdmin& a) { a.set_finding_state("organizations/1/sources/2", FindingState::kActive); }, + [&](SecurityAdmin& a) { a.set_finding_mute("../x/findings/f", MuteValue::kMuted); }, + [&](SecurityAdmin& a) { a.patch_finding(kFinding, bogus, "category"); }, + [&](SecurityAdmin& a) { a.patch_finding(kFinding, body, ""); }, + [&](SecurityAdmin& a) { a.create_finding("organizations/1/sources/2", "a/b", body); }, + [&](SecurityAdmin& a) { a.bulk_mute(kProj, "", BulkMuteValue::kMuted); }, + [&](SecurityAdmin& a) { a.bulk_mute(std::string(kProj) + "/findings", "x", BulkMuteValue::kMuted); }, + [&](SecurityAdmin& a) { a.export_findings("organizations/1/sources/2", Json::object()); }, + [&](SecurityAdmin& a) { a.create_source("folders/1", body); }, + [&](SecurityAdmin& a) { a.create_source("organizations/1/x", body); }, + [&](SecurityAdmin& a) { a.update_source("organizations/1", body, "displayName"); }, + [&](SecurityAdmin& a) { a.test_source_iam("organizations/1/sources/2", {}); }, + [&](SecurityAdmin& a) { a.set_source_iam_policy("organizations/1/sources/2", Json::array()); }, + [&](SecurityAdmin& a) { a.update_asset_marks("projects/p/assets/a1", body, "marks.k"); }, + [&](SecurityAdmin& a) { a.get_operation("organizations/1"); }, + [&](SecurityAdmin& a) { a.list_operations("projects/p"); }, + [&](SecurityAdmin& a) { a.cancel_operation("organizations/1/sources/2"); }, + [&](SecurityAdmin& a) { a.delete_operation("nope"); }, + [&](SecurityAdmin& a) { a.availability("projects/p/locations/global"); }, + }; + int n = 0; + for (auto& f : bad) { + Probe p = probe(f); + if (!p.invalid || p.sent != 0) FAIL("case " << n << " reached the network or was not invalid: " << p.error); + ++n; + } +} + +TEST(armor_admin, finding_and_source_calls_use_the_declared_rows) { + Probe p = probe([&](SecurityAdmin& a) { a.set_finding_state(kFinding, FindingState::kInactive); }); + ASSERT_EQ(p.method, std::string("POST")); + ASSERT_TRUE(has(p.target, std::string("/v2/") + kFinding + ":setState")); + ASSERT_TRUE(has(p.body, "\"state\":\"INACTIVE\"")); + p = probe([&](SecurityAdmin& a) { a.set_finding_mute(kFindingLoc, MuteValue::kUnmuted); }); + ASSERT_TRUE(has(p.target, std::string("/v2/") + kFindingLoc + ":setMute")); + ASSERT_TRUE(has(p.body, "\"mute\":\"UNMUTED\"")); + Json f = Json::object(); + f.set("category", "X"); + p = probe([&](SecurityAdmin& a) { a.patch_finding(kFinding, f, "category"); }); + ASSERT_EQ(p.method, std::string("PATCH")); + ASSERT_TRUE(has(p.target, std::string("/v2/") + kFinding + "?updateMask=category")); + p = probe([&](SecurityAdmin& a) { a.create_finding("organizations/1/sources/2", "f9", f); }); + ASSERT_TRUE(has(p.target, "/v2/organizations/1/sources/2/findings?findingId=f9")); + Json marks = Json::object(); + marks.set("marks", Json::parse("{\"k\":\"v\"}")); + p = probe([&](SecurityAdmin& a) { a.update_finding_marks(kFinding, marks, "marks.k"); }); + ASSERT_TRUE(has(p.target, std::string("/v2/") + kFinding + "/securityMarks?updateMask=marks.k")); + p = probe([&](SecurityAdmin& a) { + a.update_asset_marks("projects/p/assets/a1/securityMarks", marks, "marks.k"); + }); + ASSERT_TRUE(has(p.target, "/v2/projects/p/assets/a1/securityMarks?updateMask=marks.k")); + p = probe([&](SecurityAdmin& a) { a.bulk_mute(kProjLoc, "severity=\"LOW\"", BulkMuteValue::kMuted); }); + ASSERT_TRUE(has(p.target, std::string("/v2/") + kProjLoc + "/findings:bulkMute")); + ASSERT_TRUE(has(p.body, "\"muteState\":\"MUTED\"")); + p = probe([&](SecurityAdmin& a) { + a.export_findings("organizations/1/sources/2/locations/global", + Json::parse("{\"bigQueryDestination\":{}}")); + }); + ASSERT_TRUE(has(p.target, "/v2/organizations/1/sources/2/locations/global/findings:export")); + p = probe([&](SecurityAdmin& a) { a.patch_external_system(std::string(kFinding) + "/externalSystems/x", Json::parse("{\"status\":\"OPEN\"}"), "status"); }); + ASSERT_TRUE(has(p.target, "/externalSystems/x?updateMask=status")); + p = probe([&](SecurityAdmin& a) { a.create_source("organizations/1", Json::parse("{\"displayName\":\"s\"}")); }); + ASSERT_TRUE(has(p.target, "/v2/organizations/1/sources")); + p = probe([&](SecurityAdmin& a) { a.update_source("organizations/1/sources/2", Json::parse("{\"displayName\":\"s\"}"), "displayName"); }); + ASSERT_EQ(p.method, std::string("PATCH")); + p = probe([&](SecurityAdmin& a) { a.source_iam_policy("organizations/1/sources/2"); }); + ASSERT_EQ(p.method, std::string("POST")); + ASSERT_TRUE(has(p.target, "organizations/1/sources/2:getIamPolicy")); + p = probe([&](SecurityAdmin& a) { a.set_source_iam_policy("organizations/1/sources/2", Json::object()); }); + ASSERT_TRUE(has(p.target, ":getIamPolicy")); // read first + p = probe([&](SecurityAdmin& a) { a.test_source_iam("organizations/1/sources/2", {"securitycenter.sources.get"}); }); + ASSERT_TRUE(has(p.target, ":testIamPermissions")); + ASSERT_TRUE(has(p.body, "securitycenter.sources.get")); + p = probe([&](SecurityAdmin& a) { a.get_operation("organizations/1/operations/op1"); }); + ASSERT_TRUE(has(p.target, "/v2/organizations/1/operations/op1")); + p = probe([&](SecurityAdmin& a) { a.list_operations("organizations/1"); }); + ASSERT_TRUE(has(p.target, "/v2/organizations/1/operations?pageSize=100")); + p = probe([&](SecurityAdmin& a) { a.cancel_operation("organizations/1/operations/op1"); }); + ASSERT_EQ(p.method, std::string("GET")); // read first + p = probe([&](SecurityAdmin& a) { a.delete_operation("organizations/1/operations/op1"); }); + ASSERT_EQ(p.method, std::string("GET")); + p = probe([&](SecurityAdmin& a) { a.availability("projects/p"); }); + ASSERT_TRUE(has(p.target, "/v2/projects/p/sources?pageSize=1")); +} + +TEST(armor_admin, no_credential_is_loud_and_offline) { + auto t = std::make_unique(); + MockTransport* mock = t.get(); + SecurityAdmin admin(std::make_shared(std::move(t), nullptr)); + ASSERT_THROW(admin.list_configs(SccCollection::kMuteConfigs, kProj), NotConfiguredError); + ASSERT_THROW(admin.availability("projects/p"), NotConfiguredError); + ASSERT_THROW(admin.delete_config(SccCollection::kMuteConfigs, "projects/p/muteConfigs/m"), NotConfiguredError); + ASSERT_EQ(mock->send_count(), std::size_t(0)); +} + +TEST(armor_admin, availability_names_are_stable) { + ASSERT_EQ(std::string(availability_name(SccAvailability::kNotActivated)), std::string("not_activated")); + ASSERT_EQ(std::string(availability_name(SccAvailability::kApiDisabled)), std::string("api_disabled")); + ASSERT_EQ(std::string(availability_name(SccAvailability::kAvailable)), std::string("available")); +} + +TEST(armor_categories, table_is_complete_and_consistent) { + const auto& all = AgentCategories::all(); + ASSERT_TRUE(all.size() >= 55); + std::set seen; + std::size_t runtime = 0, control = 0; + for (const AgentCategory& c : all) { + ASSERT_TRUE(seen.insert(c.category).second); + if (c.alt[0]) ASSERT_TRUE(seen.insert(c.alt).second); + (c.family == AgentFamily::kRuntime ? runtime : control)++; + if (c.family == AgentFamily::kRuntime) { + ASSERT_EQ(std::string(c.module).rfind("AGENT_ENGINE_", 0), std::size_t(0)); + } + } + ASSERT_TRUE(runtime >= 38); + ASSERT_TRUE(control >= 14); + const AgentCategory* c = AgentCategories::find("Execution: Container Escape"); + ASSERT_TRUE(c != nullptr); + ASSERT_EQ(std::string(c->module), std::string("AGENT_ENGINE_CONTAINER_ESCAPE")); + ASSERT_TRUE(AgentCategories::find("Exfiltration: AI Agent Initiated Cloud SQL Exfiltration to Public Bucket") == + AgentCategories::find("Exfiltration: AI Agent Initiated CloudSQL Exfiltration to Public Bucket")); + ASSERT_TRUE(AgentCategories::find("Persistence: New Geography for AI Service") == nullptr); + ASSERT_TRUE(AgentCategories::find_module("AGENT_ENGINE_REVERSE_SHELL") != nullptr); + ASSERT_TRUE(AgentCategories::find_module("") == nullptr); + ASSERT_TRUE(AgentCategories::is_agent_threat("Reverse Shell")); +} + +TEST(armor_categories, every_category_is_accepted_by_the_finding_filter) { + FindingQuery q; + q.scope = "projects/p"; + q.categories = AgentCategories::strings(AgentFamily::kRuntime); + std::string filter = q.filter(); // validates every string + ASSERT_TRUE(has(filter, "category=\"Execution: Container Escape\"")); + ASSERT_TRUE(has(filter, "(CVE-2024-47177)")); + q.categories = AgentCategories::strings(AgentFamily::kControlPlane); + q.filter(); + q.categories = AgentCategories::strings(); + ASSERT_TRUE(q.categories.size() <= ArmorCaps::kMaxFilterCategories); + q.filter(); + q.categories = {"bad\"quote"}; + ASSERT_THROW(q.filter(), GoogleError); + q.categories = {"bad\\slash"}; + ASSERT_THROW(q.filter(), GoogleError); + q.categories = {std::string("tab\there")}; + ASSERT_THROW(q.filter(), GoogleError); +} + +TEST(armor_model_armor, floor_settings_live_at_global_only) { + auto t = std::make_unique(); + MockTransport* mock = t.get(); + auto client = std::make_shared(std::move(t), Credentials::bearer("tok")); + CloudContext c; + c.project = "p"; + c.location = "us-central1"; + ModelArmor armor(client, c); + try { + armor.get_floor_setting(); + } catch (const GoogleError&) { + } + auto seen = mock->seen(); + ASSERT_EQ(seen.size(), std::size_t(1)); + ASSERT_TRUE(has(seen[0].target, "/v1/projects/p/locations/global/floorSetting")); + ASSERT_EQ(armor.floor_path(""), std::string("/projects/p/locations/global/floorSetting")); +} diff --git a/tests/src/google/optimize/test_gaps.cpp b/tests/src/google/optimize/test_gaps.cpp new file mode 100644 index 0000000..14a63e4 --- /dev/null +++ b/tests/src/google/optimize/test_gaps.cpp @@ -0,0 +1,284 @@ +// Optimize gap closing: the shared Gemini adapter (models::VertexModel) behind +// make_gemini_model, online evaluator operations, generateSyntheticData and +// the telemetry writers. Local only: requests are read back from the mock, +// nothing answers (no scripted Google replies). +#include +#include +#include +#include +#include +#include + +#include "framework.h" +#include "clients/transports/mock_transport.h" +#include "google/credentials.h" +#include "google/errors.h" +#include "google/models/vertex_model.h" +#include "google/optimize/calls.h" +#include "google/optimize/evaluation_api.h" +#include "google/optimize/model_factory.h" +#include "google/optimize/operations_api.h" +#include "google/optimize/telemetry_writer.h" + +using namespace molo::google; +using namespace molo::google::optimize; +using molo::clients::body::Json; +using molo::clients::transports::MockTransport; + +namespace { + +struct Rig { + MockTransport* mock; + std::shared_ptr client; + explicit Rig(bool with_credential = true) { + auto t = std::make_unique(); + mock = t.get(); + client = std::make_shared( + std::move(t), with_credential ? Credentials::bearer("tok") : nullptr); + } +}; + +CloudContext ctx() { + CloudContext c; + c.project = "p"; + c.location = "us-central1"; + return c; +} + +bool has(const std::string& hay, const std::string& needle) { + return hay.find(needle) != std::string::npos; +} + +MockTransport::SeenRequest first_sent(Rig& rig, const std::function& fn) { + std::size_t before = rig.mock->seen().size(); + try { + fn(); + } catch (...) { + } + auto all = rig.mock->seen(); + if (all.size() <= before) FAIL("no request reached the transport"); + return all[before]; +} + +bool invalid_without_traffic(Rig& rig, const std::function& fn) { + std::size_t before = rig.mock->send_count(); + try { + fn(); + } catch (const GoogleError& e) { + return e.status() == "INVALID_ARGUMENT" && rig.mock->send_count() == before; + } + return false; +} + +// Sets or clears environment variables for one test and restores them, so +// later (live) tests still see the caller's credentials. +class EnvGuard { + public: + EnvGuard() { + for (const char* k : {"GOOGLE_ACCESS_TOKEN", "GOOGLE_CLOUD_PROJECT"}) { + const char* v = std::getenv(k); + saved_.push_back({k, v != nullptr, v ? v : ""}); + } + } + ~EnvGuard() { + for (const auto& s : saved_) { + if (s.had) setenv(s.key.c_str(), s.value.c_str(), 1); + else unsetenv(s.key.c_str()); + } + } + + private: + struct Saved { + std::string key; + bool had; + std::string value; + }; + std::vector saved_; +}; + +const char* kEval = "projects/p/locations/us-central1/onlineEvaluators/oe1"; +const std::string kOp = std::string(kEval) + "/operations/op1"; + +} // namespace + +TEST(optimize_model, factory_returns_the_shared_vertex_adapter) { + Rig rig; + auto model = make_gemini_model(rig.client, "gemini-2.5-flash"); + auto* vertex = dynamic_cast(model.get()); + ASSERT_TRUE(vertex != nullptr); + ASSERT_EQ(vertex->default_model(), std::string("gemini-2.5-flash")); + ASSERT_TRUE(dynamic_cast(make_gemini_model(rig.client).get()) != nullptr); +} + +TEST(optimize_model, oauth_with_a_project_uses_the_global_project_path) { + EnvGuard guard; + setenv("GOOGLE_ACCESS_TOKEN", "tok", 1); + setenv("GOOGLE_CLOUD_PROJECT", "p1", 1); + Rig rig; + auto model = make_gemini_model(rig.client, "m1"); + GenerateRequest req; + req.contents.push_back(Content::user("hi")); + auto r = first_sent(rig, [&] { model->generate(req); }); + ASSERT_EQ(r.method, std::string("POST")); + ASSERT_TRUE(has(r.target, "/projects/p1/locations/global/publishers/google/models/m1:generateContent")); +} + +TEST(optimize_model, without_a_token_it_is_vertex_express) { + EnvGuard guard; + unsetenv("GOOGLE_ACCESS_TOKEN"); + setenv("GOOGLE_CLOUD_PROJECT", "p1", 1); + Rig rig; + auto model = make_gemini_model(rig.client, "m1"); + GenerateRequest req; + req.contents.push_back(Content::user("hi")); + auto r = first_sent(rig, [&] { model->generate(req); }); + ASSERT_TRUE(has(r.target, "/publishers/google/models/m1:generateContent")); + ASSERT_FALSE(has(r.target, "/projects/")); + // A bad model name is refused before any traffic. + ASSERT_TRUE(invalid_without_traffic(rig, [&] { make_gemini_model(rig.client, "a/b"); })); +} + +TEST(optimize_operations, names_and_bounds) { + ASSERT_TRUE(OnlineEvaluatorOperations::is_operation_name(kOp)); + for (const char* bad : {"", "operations/1", "projects/p/locations/l/onlineEvaluators/e", + "projects/p/locations/l/evaluationRuns/e/operations/o", + "projects/p/locations/l/onlineEvaluators/e/operations/", + "projects/p/locations/l/onlineEvaluators/e/operations/..", + "projects/p/locations/l/onlineEvaluators/e/operations/o/x"}) + ASSERT_FALSE(OnlineEvaluatorOperations::is_operation_name(bad)); + Rig rig; + OnlineEvaluatorOperations ops(rig.client, ctx()); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.get("nope"); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.cancel("nope"); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.remove("nope"); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.wait_server(kOp, 0); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.wait_server(kOp, 61); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.list("projects/p/locations/l/evaluationRuns/r"); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.list(kEval, "", 101); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { ops.list(kEval, std::string(1001, 'x')); })); + Rig off(false); + OnlineEvaluatorOperations n(off.client, ctx()); + ASSERT_THROW(n.get(kOp), NotConfiguredError); + ASSERT_EQ(off.mock->send_count(), std::size_t(0)); +} + +TEST(optimize_operations, requests_use_the_declared_rows) { + Rig rig; + OnlineEvaluatorOperations ops(rig.client, ctx()); + auto r = first_sent(rig, [&] { ops.get(kOp); }); + ASSERT_EQ(r.method, std::string("GET")); + ASSERT_TRUE(has(r.target, "/v1/" + kOp)); + r = first_sent(rig, [&] { ops.cancel(kOp); }); + ASSERT_EQ(r.method, std::string("POST")); + ASSERT_TRUE(has(r.target, kOp + ":cancel")); + r = first_sent(rig, [&] { ops.remove(kOp); }); + ASSERT_EQ(r.method, std::string("DELETE")); + r = first_sent(rig, [&] { ops.wait_server(kOp, 30); }); + ASSERT_EQ(r.method, std::string("POST")); + ASSERT_TRUE(has(r.target, kOp + ":wait?timeout=30s")); + r = first_sent(rig, [&] { ops.list(kEval, "done=true", 20); }); + ASSERT_TRUE(has(r.target, std::string(kEval) + "/operations?")); + ASSERT_TRUE(has(r.target, "pageSize=20")); +} + +TEST(optimize_synthetic_data, request_is_checked_against_the_table) { + Rig rig; + EvaluationApi api(rig.client, ctx()); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { api.generate_synthetic_data(Json::object()); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { + api.generate_synthetic_data(Json::parse("{\"notAField\":1}")); + })); + const CallSpec& spec = call_spec(rpc::kGenerateSyntheticData); + std::string field = split_list(spec.body_fields).front(); + Json body = Json::object(); + body.set(field, Json::object()); + auto r = first_sent(rig, [&] { api.generate_synthetic_data(body); }); + ASSERT_EQ(r.method, std::string("POST")); + ASSERT_TRUE(has(r.target, "/v1/projects/p/locations/us-central1:generateSyntheticData")); +} + +TEST(optimize_writers, trace_writer_validates_then_uses_v1_and_v2) { + Rig rig; + TraceWriter w(rig.client); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.patch_traces("BAD ID", Json::object()); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.patch_traces("p", Json::parse("{\"traces\":{}}")); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.batch_write("p", Json::parse("{\"spans\":[]}")); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.batch_write("p", Json::parse("{\"other\":[1]}")); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.create_span("x", Json::object()); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { + w.create_span("projects/p/traces/abc/spans/s1", Json::parse("{\"name\":\"other\"}")); + })); + auto r = first_sent(rig, [&] { + w.patch_traces("p", Json::parse("{\"traces\":{\"traces\":[{\"traceId\":\"a\"}]}}")); + }); + ASSERT_EQ(r.method, std::string("PATCH")); + ASSERT_TRUE(has(r.target, "/v1/projects/p/traces")); + r = first_sent(rig, [&] { w.batch_write("p", Json::parse("{\"spans\":[{\"name\":\"n\"}]}")); }); + ASSERT_EQ(r.method, std::string("POST")); + ASSERT_TRUE(has(r.target, "/v2/projects/p/traces:batchWrite")); + r = first_sent(rig, [&] { + w.create_span("projects/p/traces/abc/spans/s1", Json::parse("{\"name\":\"projects/p/traces/abc/spans/s1\"}")); + }); + ASSERT_TRUE(has(r.target, "/v2/projects/p/traces/abc/spans/s1")); +} + +TEST(optimize_writers, log_writer_validates_then_uses_the_table) { + Rig rig; + LogWriter w(rig.client); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.write(Json::object()); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.write(Json::parse("{\"entries\":[]}")); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.write(Json::parse("{\"bogus\":[1]}")); })); + Json many = Json::array(); + for (int i = 0; i < 1001; ++i) many.push(Json::object()); + Json big = Json::object(); + big.set("entries", many); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.write(big); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { w.copy(Json::parse("{\"name\":\"x\"}")); })); + auto r = first_sent(rig, [&] { + w.write(Json::parse("{\"dryRun\":true,\"logName\":\"projects/p/logs/l\",\"entries\":[{\"textPayload\":\"x\"}]}")); + }); + ASSERT_EQ(r.method, std::string("POST")); + ASSERT_TRUE(has(r.target, "/v2/entries:write")); + ASSERT_TRUE(has(r.body, "\"dryRun\":true")); + r = first_sent(rig, [&] { w.copy(Json::parse("{\"name\":\"projects/p/locations/global/buckets/b\",\"destination\":\"storage.googleapis.com/b2\"}")); }); + ASSERT_TRUE(has(r.target, "/v2/entries:copy")); +} + +// Read-only live probe (MOLO_LIVE_NET=1, GOOGLE_ACCESS_TOKEN, GOOGLE_CLOUD_PROJECT; +// loud SKIP otherwise). Observed 2026-10-24: an operations list under a +// numeric, absent online evaluator id answers 200 with no operations; a +// non numeric evaluator id is 400 INVALID_ARGUMENT from Google's LRO layer. +TEST(optimize_live_gaps, online_evaluator_operations_list_and_shared_model) { + std::string project = Credentials::env("GOOGLE_CLOUD_PROJECT"); + if (!std::getenv("MOLO_LIVE_NET") || Credentials::env("GOOGLE_ACCESS_TOKEN").empty() || + project.empty()) { + std::cout << "SKIP optimize_live_gaps (set MOLO_LIVE_NET=1, GOOGLE_ACCESS_TOKEN and " + "GOOGLE_CLOUD_PROJECT)\n"; + return; + } + auto client = GoogleClient::from_env(); + CloudContext c; + c.project = project; + c.location = "us-central1"; + OnlineEvaluatorOperations ops(client, c); + OperationPage page = ops.list("projects/" + project + "/locations/us-central1/onlineEvaluators/1"); + ASSERT_EQ(page.operations.size(), std::size_t(0)); + ASSERT_FALSE(page.truncated); + // The shared adapter answers a real generateContent (a few tokens). + auto model = make_gemini_model(client); + GenerateRequest req; + req.contents.push_back(Content::user("Reply with the single word: ok")); + GenerateResponse res = model->generate(req); + ASSERT_FALSE(res.text().empty()); + std::cout << " shared VertexModel reply chars: " << res.text().size() << "\n"; +} + +TEST(optimize_table, new_calls_are_rows) { + for (const char* id : {rpc::kGenerateSyntheticData, rpc::kOnlineEvaluatorOpGet, + rpc::kOnlineEvaluatorOpList, rpc::kOnlineEvaluatorOpCancel, + rpc::kOnlineEvaluatorOpDelete, rpc::kOnlineEvaluatorOpWait, + rpc::kTracePatch, rpc::kTraceBatchWrite, rpc::kTraceCreateSpan, + rpc::kLogEntriesWrite, rpc::kLogEntriesCopy}) + ASSERT_TRUE(find_call(id) != nullptr); + ASSERT_TRUE(all_calls().size() >= 46); +} diff --git a/tests/src/google/optimize/test_live.cpp b/tests/src/google/optimize/test_live.cpp index 9e10812..851b2cc 100644 --- a/tests/src/google/optimize/test_live.cpp +++ b/tests/src/google/optimize/test_live.cpp @@ -7,7 +7,7 @@ #include "framework.h" #include "google/credentials.h" #include "google/optimize/autorater.h" -#include "google/optimize/direct_model.h" +#include "google/optimize/model_factory.h" #include "google/optimize/eval_resources.h" #include "google/optimize/evaluation_api.h" #include "google/optimize/observability.h" @@ -105,7 +105,7 @@ TEST(optimize_live, llm_judged_evaluate_instances) { TEST(optimize_live, autorater_parses_a_real_model_reply) { if (!live("optimize_live.autorater_parses_a_real_model_reply", false)) return; - auto model = DirectModel::from_env(GoogleClient::from_env()); + auto model = make_gemini_model(GoogleClient::from_env()); Autorater rater(model); Conversation c = Conversation::parse_transcript( "user: What is 2+2?\nagent: 4.", "live"); diff --git a/tests/src/google/registry/test_registry_gaps.cpp b/tests/src/google/registry/test_registry_gaps.cpp new file mode 100644 index 0000000..8c60f47 --- /dev/null +++ b/tests/src/google/registry/test_registry_gaps.cpp @@ -0,0 +1,299 @@ +// Registry gap closing: lists that report truncation loudly, v1alpha twins of +// every v1 collection, operations of the gateway and policy hosts, IAM on +// authz policies, deny policy update, boundary patch and bindings search. +// Local only: the paging state machine is fed synthetic documents (it is local +// logic, not a claim about Google's replies) and requests are inspected as the +// mock saw them. Nothing answers. +#include +#include +#include +#include + +#include "framework.h" +#include "clients/transports/mock_transport.h" +#include "google/credentials.h" +#include "google/registry/agent_gateway.h" +#include "google/registry/agent_identity.h" +#include "google/registry/agent_registry.h" +#include "google/registry/approvals.h" +#include "google/registry/call_runner.h" +#include "google/registry/caps.h" +#include "google/registry/gateway_policy.h" + +using namespace molo::google; +using namespace molo::google::registry; +using molo::clients::body::Json; +using molo::clients::transports::MockTransport; + +namespace { + +struct Rig { + MockTransport* mock; + std::shared_ptr client; + explicit Rig(bool with_credential = true) { + auto t = std::make_unique(); + mock = t.get(); + client = std::make_shared( + std::move(t), with_credential ? Credentials::bearer("tok") : nullptr); + } + CloudContext ctx() const { + CloudContext c; + c.project = "p"; + c.location = "us-central1"; + return c; + } +}; + +bool has(const std::string& hay, const std::string& needle) { + return hay.find(needle) != std::string::npos; +} + +// Runs fn (which fails: the mock has no expectation) and returns the first +// request that reached the transport during it. +MockTransport::SeenRequest first_sent(Rig& rig, const std::function& fn) { + std::size_t before = rig.mock->seen().size(); + try { + fn(); + } catch (...) { + } + auto all = rig.mock->seen(); + if (all.size() <= before) FAIL("no request reached the transport"); + return all[before]; +} + +template +bool invalid_without_traffic(Rig& r, F&& f) { + std::size_t before = r.mock->send_count(); + try { + f(); + } catch (const GoogleError& e) { + return e.status() == "INVALID_ARGUMENT" && r.mock->send_count() == before; + } + return false; +} + +Json page(int items, const std::string& next) { + Json arr = Json::array(); + for (int i = 0; i < items; ++i) { + Json o = Json::object(); + o.set("name", Json("x/" + std::to_string(i))); + arr.push(o); + } + Json doc = Json::object(); + doc.set("agents", arr); + if (!next.empty()) doc.set("nextPageToken", Json(next)); + return doc; +} + +} // namespace + +TEST(registry_lists, pager_follows_pages_until_the_server_stops) { + ListPager p("agents", 10, 1000); + ASSERT_TRUE(p.feed(page(100, "t1"))); + ASSERT_EQ(p.token(), std::string("t1")); + ASSERT_FALSE(p.feed(page(7, ""))); + ASSERT_EQ(p.result().size(), std::size_t(107)); + ASSERT_EQ(p.result().pages, std::size_t(2)); + ASSERT_FALSE(p.result().truncated); + ASSERT_EQ(p.result().next_page_token, std::string()); +} + +TEST(registry_lists, pager_reports_truncation_at_the_page_cap_loudly) { + ListPager p("agents", 3, 1000); + ASSERT_TRUE(p.feed(page(100, "t1"))); + ASSERT_TRUE(p.feed(page(100, "t2"))); + ASSERT_FALSE(p.feed(page(100, "t3"))); // third page used, a fourth exists + ASSERT_TRUE(p.result().truncated); + ASSERT_EQ(p.result().next_page_token, std::string("t3")); + ASSERT_EQ(p.result().size(), std::size_t(300)); + Json j = p.result().to_json(); + ASSERT_TRUE(j.find("truncated")->as_bool()); + ASSERT_EQ(j.find("next_page_token")->as_string(), std::string("t3")); + ASSERT_EQ(j.find("count")->as_int(), 300); + ASSERT_EQ(j.find("cap_items")->as_int(), 1000); +} + +TEST(registry_lists, pager_reports_truncation_at_the_item_cap_loudly) { + ListPager p("agents", 100, 250); + ASSERT_TRUE(p.feed(page(100, "t1"))); + ASSERT_TRUE(p.feed(page(100, "t2"))); + ASSERT_FALSE(p.feed(page(100, "t3"))); // 300 returned, 250 allowed + ASSERT_TRUE(p.result().truncated); + ASSERT_EQ(p.result().size(), std::size_t(250)); + ASSERT_EQ(p.result().next_page_token, std::string()); // no exact continuation +} + +TEST(registry_lists, a_list_that_ends_exactly_at_the_cap_is_not_truncated) { + ListPager p("agents", 2, 200); + ASSERT_TRUE(p.feed(page(100, "t1"))); + ASSERT_FALSE(p.feed(page(100, ""))); + ASSERT_FALSE(p.result().truncated); + ASSERT_EQ(p.result().size(), std::size_t(200)); +} + +TEST(registry_lists, caps_are_the_stated_ceiling) { + ASSERT_EQ(RegistryCaps::kMaxItems, std::size_t(1000)); + ASSERT_EQ(RegistryCaps::kMaxPages, std::size_t(10)); + ListPager p("agents", RegistryCaps::kMaxPages, RegistryCaps::kMaxItems); + for (std::size_t i = 0; i + 1 < RegistryCaps::kMaxPages; ++i) + ASSERT_TRUE(p.feed(page(100, "t" + std::to_string(i)))); + ASSERT_FALSE(p.feed(page(100, "more"))); + ASSERT_TRUE(p.result().truncated); // 1000 items and more existed +} + +TEST(registry_lists, the_caller_cannot_page_by_hand) { + Rig rig; + AgentRegistry reg(rig.client, rig.ctx()); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { + CallArgs a; + a.vars["parent"] = "projects/p/locations/us-central1"; + a.query["pageToken"] = "x"; + reg.runner().list("agentregistry.v1.agents.list", "agents", a); + })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { reg.list(AssetKind::kAgents, {{"pageToken", "x"}}); })); +} + +TEST(registry_lists, a_first_page_request_carries_the_stated_page_size) { + Rig rig; + AgentRegistry reg(rig.client, rig.ctx()); + auto r = first_sent(rig, [&] { reg.list(AssetKind::kAgents); }); + ASSERT_EQ(r.method, std::string("GET")); + ASSERT_TRUE(has(r.target, "/v1/projects/p/locations/us-central1/agents?pageSize=100")); +} + +TEST(registry_lists, approved_listing_keeps_the_truncation_flag_in_its_type) { + // list_approved returns ListResult: the flag cannot be dropped silently. + static_assert(std::is_same_v().list_approved( + AssetKind::kAgents, std::declval())), + ListResult>); + static_assert(std::is_same_v().list()), ListResult>); + static_assert(std::is_same_v().list()), ListResult>); + static_assert(std::is_same_v().deny_policies("a%2Fb")), + ListResult>); + ASSERT_TRUE(true); +} + +TEST(registry_versions, v1alpha_twins_exist_for_every_v1_collection) { + std::set ids; + for (const CallSpec& c : Calls::all()) ids.insert(c.id); + std::size_t twins = 0; + for (const CallSpec& c : Calls::all()) { + std::string id = c.id; + const std::string v1 = "agentregistry.v1."; + if (id.rfind(v1, 0) != 0) continue; + ASSERT_TRUE(ids.count("agentregistry.v1alpha." + id.substr(v1.size())) == 1); + ++twins; + } + ASSERT_TRUE(twins >= 35); +} + +TEST(registry_versions, version_picks_the_host_path_and_rows) { + Rig rig; + AgentRegistry v1(rig.client, rig.ctx()); + AgentRegistry alpha(rig.client, rig.ctx(), RegistryVersion::kV1Alpha); + ASSERT_EQ(v1.version(), std::string("v1")); + ASSERT_EQ(alpha.version(), std::string("v1alpha")); + ASSERT_EQ(alpha.default_endpoint(RegistryVersion::kV1Alpha).base_url, + std::string("https://agentregistry.googleapis.com/v1alpha")); + auto r = first_sent(rig, [&] { alpha.list(AssetKind::kMcpServers); }); + ASSERT_TRUE(has(r.target, "/v1alpha/projects/p/locations/us-central1/mcpServers?pageSize=100")); + r = first_sent(rig, [&] { alpha.get(AssetKind::kAgents, "a1"); }); + ASSERT_TRUE(has(r.target, "/v1alpha/projects/p/locations/us-central1/agents/a1")); + r = first_sent(rig, [&] { alpha.operations(); }); + ASSERT_TRUE(has(r.target, "/v1alpha/projects/p/locations/us-central1/operations")); + r = first_sent(rig, [&] { v1.operations(); }); + ASSERT_TRUE(has(r.target, "/v1/projects/p/locations/us-central1/operations")); + // Skills exist in v1alpha only for CRUD, whichever version is chosen. + r = first_sent(rig, [&] { v1.get(AssetKind::kSkills, "s1"); }); + ASSERT_TRUE(has(r.target, "/v1alpha/projects/p/locations/us-central1/skills/s1")); + // Validation still happens before traffic in v1alpha. + ASSERT_TRUE(invalid_without_traffic(rig, [&] { alpha.get(AssetKind::kAgents, "a/b"); })); +} + +TEST(registry_operations, gateway_and_policy_hosts_wrap_all_four_operation_methods) { + Rig rig; + AgentGateway gw(rig.client, rig.ctx()); + GatewayPolicies pol(rig.client, rig.ctx()); + auto r = first_sent(rig, [&] { gw.operations(); }); + ASSERT_TRUE(has(r.target, "/v1/projects/p/locations/us-central1/operations")); + r = first_sent(rig, [&] { pol.operations(); }); + ASSERT_TRUE(has(r.target, "/v1/projects/p/locations/us-central1/operations")); + // cancel and delete read the operation first. + r = first_sent(rig, [&] { gw.cancel_operation("op-1"); }); + ASSERT_EQ(r.method, std::string("GET")); + ASSERT_TRUE(has(r.target, "/operations/op-1")); + r = first_sent(rig, [&] { pol.delete_operation("op-1"); }); + ASSERT_EQ(r.method, std::string("GET")); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { gw.cancel_operation("a/b"); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { gw.delete_operation(".."); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { pol.cancel_operation(""); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { pol.delete_operation("x?y"); })); + Rig off(false); + AgentGateway n(off.client, off.ctx()); + ASSERT_THROW(n.operations(), NotConfiguredError); + ASSERT_EQ(off.mock->send_count(), std::size_t(0)); +} + +TEST(registry_policies, iam_on_authz_policies_is_validated_and_read_first) { + Rig rig; + GatewayPolicies pol(rig.client, rig.ctx()); + auto r = first_sent(rig, [&] { pol.set_iam_policy("pol-1", Json::parse("{\"bindings\":[]}")); }); + ASSERT_EQ(r.method, std::string("GET")); // read before write + ASSERT_TRUE(has(r.target, "/authzPolicies/pol-1")); + r = first_sent(rig, [&] { pol.test_iam_permissions("pol-1", {"networksecurity.authzPolicies.get"}); }); + ASSERT_EQ(r.method, std::string("POST")); + ASSERT_TRUE(has(r.target, "/authzPolicies/pol-1:testIamPermissions")); + ASSERT_TRUE(has(r.body, "\"permissions\":[\"networksecurity.authzPolicies.get\"]")); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { pol.set_iam_policy("pol-1", Json::array()); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { pol.set_iam_policy("a/b", Json::object()); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { pol.test_iam_permissions("pol-1", {}); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { + pol.test_iam_permissions("pol-1", std::vector(65, "p.q")); + })); +} + +TEST(registry_identity, deny_policy_update_boundary_patch_and_operations) { + Rig rig; + AgentIdentity id(rig.client, rig.ctx()); + std::string attach = AgentIdentity::attachment_for_project("123"); + DenyPolicySpec spec; + spec.display_name = "deny"; + DenyRuleSpec rule; + rule.denied_principals = {"principalSet://goog/group/a@b.c"}; + rule.denied_permissions = {"storage.googleapis.com/buckets.delete"}; + spec.rules = {rule}; + spec.etag = "abc"; + ASSERT_TRUE(has(spec.to_json().dump(), "\"etag\":\"abc\"")); + spec.etag.clear(); + ASSERT_FALSE(has(spec.to_json().dump(), "etag")); + // update reads first (needs the etag and a clear 404). + auto r = first_sent(rig, [&] { id.update_deny_policy(attach, "p1", spec); }); + ASSERT_EQ(r.method, std::string("GET")); + ASSERT_TRUE(has(r.target, "/v2/policies/" + attach + "/denypolicies/p1")); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { id.update_deny_policy("plain", "p1", spec); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { id.update_deny_policy(attach, "a/b", spec); })); + + r = first_sent(rig, [&] { id.deny_policy_operation("policies/" + attach + "/operations/o1"); }); + ASSERT_TRUE(has(r.target, "/v2/policies/" + attach + "/operations/o1")); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { id.deny_policy_operation("operations/o1"); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { id.deny_policy_operation("policies/x"); })); + + r = first_sent(rig, [&] { id.engine_operation("op9"); }); + ASSERT_TRUE(has(r.target, "/v1/projects/p/locations/us-central1/operations/op9")); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { id.engine_operation("a/b"); })); + + AccessBoundarySpec b; + b.display_name = "pab"; + b.resources = {"//cloudresourcemanager.googleapis.com/projects/p"}; + r = first_sent(rig, [&] { id.update_boundary("42", "pab1", b, {"displayName"}); }); + ASSERT_EQ(r.method, std::string("GET")); // exists first + ASSERT_TRUE(has(r.target, "/v3/organizations/42/locations/global/principalAccessBoundaryPolicies/pab1")); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { id.update_boundary("42", "pab1", b, {}); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { id.update_boundary("4x", "pab1", b, {"displayName"}); })); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { id.update_boundary("42", "pab1", b, {"name"}); })); + + r = first_sent(rig, [&] { id.boundary_bindings("42", "pab1"); }); + ASSERT_TRUE(has(r.target, + "/v3/organizations/42/locations/global/principalAccessBoundaryPolicies/pab1:searchPolicyBindings?pageSize=100")); + ASSERT_TRUE(invalid_without_traffic(rig, [&] { id.boundary_bindings("42", "a/b"); })); +} diff --git a/tests/src/google/registry/test_registry_live.cpp b/tests/src/google/registry/test_registry_live.cpp index 3caf248..a5dc100 100644 --- a/tests/src/google/registry/test_registry_live.cpp +++ b/tests/src/google/registry/test_registry_live.cpp @@ -62,9 +62,9 @@ void expect_read(F&& f) { TEST(registry_live, reasoning_engines_list_with_key_or_token) { if (!live_ready("registry_live.reasoning_engines_list_with_key_or_token")) return; AgentIdentity identity(GoogleClient::from_env(), CloudContext::from_env()); - molo::clients::body::Json engines = identity.engines(); - ASSERT_TRUE(engines.is_array()); - std::cout << " engines: " << engines.as_array().size() << "\n"; + ListResult engines = identity.engines(); + ASSERT_TRUE(engines.items.is_array()); + std::cout << " engines: " << engines.size() << (engines.truncated ? " (TRUNCATED)" : "") << "\n"; } TEST(registry_live, audit_trail_read_needs_oauth) { @@ -89,6 +89,34 @@ TEST(registry_live, registry_reads) { expect_read([&] { reg.list(k); }); } +// Gap-closing reads: v1alpha twins of every v1 collection, the operations lists +// of all three registry-side hosts, deny policies (iam v2). Read-only. Each +// answers 200 once its API is enabled; a disabled API is reported loudly. +TEST(registry_live, v1alpha_operations_and_deny_policy_reads) { + if (!live_ready("registry_live.v1alpha_operations_and_deny_policy_reads")) return; + AgentRegistry alpha(GoogleClient::from_env(), CloudContext::from_env(), + RegistryVersion::kV1Alpha); + ASSERT_EQ(alpha.version(), std::string("v1alpha")); + for (AssetKind k : {AssetKind::kServices, AssetKind::kAgents, AssetKind::kMcpServers, + AssetKind::kEndpoints, AssetKind::kBindings, AssetKind::kAiApplications}) + expect_read([&] { alpha.list(k); }); + AgentRegistry reg(GoogleClient::from_env(), CloudContext::from_env()); + expect_read([&] { reg.operations(); }); + AgentGateway gw(GoogleClient::from_env(), CloudContext::from_env()); + expect_read([&] { gw.operations(); }); + GatewayPolicies pol(GoogleClient::from_env(), CloudContext::from_env()); + expect_read([&] { pol.operations(); }); + // The project number is not known here; the deny policy list needs it. The + // attachment is built from GOOGLE_CLOUD_PROJECT_NUMBER when the caller sets it. + std::string number = Credentials::env("GOOGLE_CLOUD_PROJECT_NUMBER"); + if (number.empty()) { + std::cout << "SKIP deny policy list (set GOOGLE_CLOUD_PROJECT_NUMBER)\n"; + return; + } + AgentIdentity identity(GoogleClient::from_env(), CloudContext::from_env()); + expect_read([&] { identity.deny_policies(AgentIdentity::attachment_for_project(number)); }); +} + TEST(registry_live, gateway_and_policy_reads) { if (!live_ready("registry_live.gateway_and_policy_reads")) return; AgentGateway gw(GoogleClient::from_env(), CloudContext::from_env()); diff --git a/tests/src/google/sessions/test_sessions_gaps.cpp b/tests/src/google/sessions/test_sessions_gaps.cpp new file mode 100644 index 0000000..2ca73bb --- /dev/null +++ b/tests/src/google/sessions/test_sessions_gaps.cpp @@ -0,0 +1,353 @@ +// Gap closing for Agent Sessions and Memory Bank: operations (get, list, +// cancel, delete, wait) under the engine, a session, a memory and a serving +// profile; servingProfiles; lists that stop loudly. Requests are inspected as +// the mock saw them; nothing answers (no scripted Google replies). +#include +#include +#include +#include +#include +#include + +#include "framework.h" +#include "clients/transports/mock_transport.h" +#include "google/credentials.h" +#include "google/sessions/calls.h" +#include "google/sessions/memory_bank.h" +#include "google/sessions/operation_names.h" +#include "google/sessions/serving_profile.h" +#include "google/sessions/sessions.h" + +using namespace molo::google; +using namespace molo::google::sessions; +using molo::clients::body::Json; +using molo::clients::transports::MockTransport; + +namespace { + +const char* kEngine = "projects/p1/locations/us-central1/reasoningEngines/e1"; +const char* kLocation = "projects/p1/locations/us-central1"; + +struct Rig { + MockTransport* mock; + std::shared_ptr client; + explicit Rig(bool with_credential = true) { + auto t = std::make_unique(); + mock = t.get(); + client = std::make_shared( + std::move(t), with_credential ? Credentials::bearer("tok") : nullptr); + } +}; + +bool has(const std::string& hay, const std::string& needle) { + return hay.find(needle) != std::string::npos; +} + +std::set words(const std::string& s) { + std::set out; + std::istringstream in(s); + for (std::string w; in >> w;) out.insert(w); + return out; +} + +void keys_declared(const Json& body, Call call) { + std::set allowed = words(spec_of(call).fields); + ASSERT_TRUE(body.is_object()); + for (const auto& [key, value] : body.as_object()) { + (void)value; + if (!allowed.count(key)) FAIL("body key '" << key << "' not in the table row"); + } +} + +void invalid(const std::function& fn) { + try { + fn(); + } catch (const GoogleError& e) { + ASSERT_EQ(e.http_status(), 400); + ASSERT_EQ(e.status(), std::string("INVALID_ARGUMENT")); + ASSERT_TRUE(e.service().empty()); // local validation, not upstream + return; + } + FAIL("expected INVALID_ARGUMENT"); +} + +// Runs fn (which fails: the mock has no expectation) and returns the one +// request that reached the transport. +MockTransport::SeenRequest sent(Rig& rig, const std::function& fn) { + std::size_t before = rig.mock->seen().size(); + try { + fn(); + } catch (...) { + } + auto all = rig.mock->seen(); + if (all.size() != before + 1) + FAIL("expected exactly one request, saw " << all.size() - before); + return all.back(); +} + +std::string engine_op(const std::string& middle = "") { + return std::string(kEngine) + middle + "/operations/op1"; +} + +} // namespace + +TEST(sessions_operations, every_owner_has_all_five_verbs_in_the_table) { + const std::string e = kEngine; + const std::string sp = std::string(kLocation) + "/servingProfiles/sp1/operations/op1"; + struct Case { + std::string root, name; + OperationOwner owner; + }; + std::vector cases = {{e, engine_op(), OperationOwner::kEngine}, + {e, engine_op("/sessions/s1"), OperationOwner::kSession}, + {e, engine_op("/memories/m1"), OperationOwner::kMemory}, + {kLocation, sp, OperationOwner::kServingProfile}}; + std::set rows; + for (const Case& c : cases) { + OperationName n = OperationName::parse(c.root, c.name); + ASSERT_EQ(static_cast(n.owner()), static_cast(c.owner)); + ASSERT_EQ(n.str(), c.name); + for (OperationVerb v : {OperationVerb::kGet, OperationVerb::kCancel, OperationVerb::kList, + OperationVerb::kDelete, OperationVerb::kWait}) { + ASSERT_TRUE(rows.insert(static_cast(n.call(v))).second); // all distinct + ASSERT_TRUE(has(spec_of(n.call(v)).method, "operations.")); + } + } + ASSERT_EQ(rows.size(), std::size_t(20)); +} + +TEST(sessions_operations, names_are_confined_to_their_root) { + const std::string e = kEngine; + for (const char* bad : + {"", "operations/1", + "projects/p1/locations/us-central1/reasoningEngines/other/operations/1", + "projects/p1/locations/us-central1/reasoningEngines/e1/operations/..", + "projects/p1/locations/us-central1/reasoningEngines/e1/operations//x", + "projects/p1/locations/us-central1/reasoningEngines/e1/jobs/j/operations/1", + "projects/p1/locations/us-central1/reasoningEngines/e1/sessions/s/events/e/operations/1", + "projects/p1/locations/us-central1/reasoningEngines/e1/servingProfiles/p/operations/1", + "projects/p1/locations/us-central1/reasoningEngines/e1/operations/1?x=1"}) + invalid([&] { OperationName::parse(e, bad); }); + // A location root only serves serving profiles. + invalid([&] { OperationName::parse(kLocation, std::string(kLocation) + "/operations/1"); }); + invalid([&] { + OperationName::parse(kLocation, std::string(kLocation) + "/sessions/s/operations/1"); + }); + invalid([&] { OperationName::parse_parent(kLocation, kLocation); }); + OperationName::parse_parent(e, e); + OperationName::parse_parent(e, e + "/sessions/s1"); + invalid([&] { OperationName::parse_parent(e, e + "/sessions"); }); + invalid([&] { OperationName::parse_parent(e, e + "/other/x"); }); +} + +TEST(sessions_operations, requests_use_the_declared_rows) { + Rig rig; + Sessions s(rig.client, EngineName(kEngine)); + MemoryBank b(rig.client, EngineName(kEngine)); + auto r = sent(rig, [&] { s.operation(engine_op()); }); + ASSERT_EQ(r.method, std::string("GET")); + ASSERT_TRUE(has(r.target, "/v1beta1/" + engine_op())); + r = sent(rig, [&] { s.delete_operation(engine_op()); }); + ASSERT_EQ(r.method, std::string("DELETE")); + ASSERT_TRUE(has(r.target, "/v1beta1/" + engine_op())); + r = sent(rig, [&] { s.cancel_operation(engine_op("/sessions/s1")); }); + ASSERT_EQ(r.method, std::string("POST")); + ASSERT_TRUE(has(r.target, engine_op("/sessions/s1") + ":cancel")); + r = sent(rig, [&] { b.wait_server(engine_op("/memories/m1"), 30); }); + ASSERT_EQ(r.method, std::string("POST")); + ASSERT_TRUE(has(r.target, engine_op("/memories/m1") + ":wait?timeout=30s")); + r = sent(rig, [&] { s.list_operations("", {"done=true", 10}); }); + ASSERT_EQ(r.method, std::string("GET")); + ASSERT_TRUE(has(r.target, std::string(kEngine) + "/operations?")); + ASSERT_TRUE(has(r.target, "pageSize=10")); + ASSERT_TRUE(has(r.target, "filter=done%3Dtrue")); + r = sent(rig, [&] { s.list_session_operations("s1"); }); + ASSERT_TRUE(has(r.target, std::string(kEngine) + "/sessions/s1/operations")); + r = sent(rig, [&] { b.list_memory_operations("m1"); }); + ASSERT_TRUE(has(r.target, std::string(kEngine) + "/memories/m1/operations")); +} + +TEST(sessions_operations, bad_input_never_reaches_the_network) { + Rig rig; + Sessions s(rig.client, EngineName(kEngine)); + invalid([&] { s.wait_server(engine_op(), 0); }); + invalid([&] { s.wait_server(engine_op(), 61); }); + invalid([&] { s.wait_server("nope", 5); }); + invalid([&] { s.delete_operation(engine_op() + "?x"); }); + invalid([&] { s.list_operations("", {"a\nb", 0}); }); + invalid([&] { s.list_operations("", {"", 101}); }); + invalid([&] { s.list_operations(std::string(kEngine) + "/sessions/BAD ID"); }); + invalid([&] { s.list_session_operations("Bad_Id"); }); + ASSERT_EQ(rig.mock->send_count(), std::size_t(0)); + Rig off(false); + Sessions n(off.client, EngineName(kEngine)); + ASSERT_THROW(n.delete_operation(engine_op()), NotConfiguredError); + ASSERT_THROW(n.list_operations(), NotConfiguredError); + ASSERT_THROW(n.wait_server(engine_op(), 5), NotConfiguredError); + ASSERT_EQ(off.mock->send_count(), std::size_t(0)); +} + +TEST(sessions_operations, lists_stop_loudly_instead_of_truncating) { + ASSERT_FALSE(list_over_limit(100, false, 0, 10, 1000)); + ASSERT_FALSE(list_over_limit(1000, false, 9, 10, 1000)); + ASSERT_TRUE(list_over_limit(1001, false, 3, 10, 1000)); + ASSERT_TRUE(list_over_limit(10, true, 9, 10, 1000)); // page 10 used, more left + ASSERT_FALSE(list_over_limit(10, true, 8, 10, 1000)); // page 9, one more allowed +} + +TEST(sessions_serving_profiles, scope_enum_and_names) { + ASSERT_EQ(std::string(serving_scope_name(ServingScope::kGeminiLive)), + std::string("GEMINI_LIVE")); + ASSERT_EQ(std::string(serving_scope_name(ServingScope::kInteractionsApi)), + std::string("INTERACTIONS_API")); + ASSERT_EQ(std::string(serving_scope_name(serving_scope_from("RESPONSE_API"))), + std::string("RESPONSE_API")); + for (const char* bad : {"", "gemini_live", "SERVING_PROFILE_SCOPE_UNSPECIFIED", "ALL"}) + invalid([&] { serving_scope_from(bad); }); + LocationName loc(kLocation); + ASSERT_EQ(loc.location(), std::string("us-central1")); + ASSERT_EQ(loc.serving_profile("sp-1"), std::string(kLocation) + "/servingProfiles/sp-1"); + for (const char* bad : {"", "projects/p1", "projects/p1/locations", "projects/P/locations/l", + "projects/p1/locations/us-central1/x", "projects/p/locations/L", + "projects/p/locations/"}) + invalid([&] { LocationName n{std::string(bad)}; }); + invalid([&] { loc.serving_profile("Bad_Id"); }); + invalid([&] { loc.serving_profile(""); }); +} + +TEST(sessions_serving_profiles, bodies_use_only_declared_fields) { + ServingProfileSpec spec; + spec.profile_id = "sp-1"; + spec.display_name = "Live profile"; + spec.description = "d"; + spec.scope = ServingScope::kResponseApi; + spec.kms_key_name = "projects/p1/locations/us-central1/keyRings/k/cryptoKeys/c"; + spec.validate(); + Json body = spec.to_json(); + keys_declared(body, Call::kServingProfileCreate); + ASSERT_EQ(body.find("scope")->as_string(), std::string("RESPONSE_API")); + ASSERT_EQ(body.find("cmekConfig")->find("encryptionSpec")->find("kmsKeyName")->as_string(), + spec.kms_key_name); + ServingProfilePatch patch; + patch.profile_id = "sp-1"; + patch.display_name = "x"; + patch.description = "y"; + patch.kms_key_name = spec.kms_key_name; + patch.update_mask = "displayName,description,cmekConfig"; + patch.validate(); + keys_declared(patch.to_json(), Call::kServingProfilePatch); + + ServingProfileSpec bad = spec; + bad.display_name = ""; + invalid([&] { bad.validate(); }); + bad = spec; + bad.display_name = std::string(129, 'x'); + invalid([&] { bad.validate(); }); + bad = spec; + bad.profile_id = "A"; + invalid([&] { bad.validate(); }); + bad = spec; + bad.kms_key_name = "keys/1"; + invalid([&] { bad.validate(); }); + bad = spec; + bad.description = std::string(4001, 'x'); + invalid([&] { bad.validate(); }); + ServingProfilePatch bp = patch; + bp.update_mask = ""; + invalid([&] { bp.validate(); }); + bp.update_mask = "display name"; + invalid([&] { bp.validate(); }); + + ServingProfile p = ServingProfile::from_json(Json::parse( + "{\"name\":\"projects/p1/locations/us-central1/servingProfiles/sp-1\"," + "\"displayName\":\"n\",\"scope\":\"GEMINI_LIVE\"," + "\"cmekConfig\":{\"encryptionSpec\":{\"kmsKeyName\":\"k\"}}}")); + ASSERT_EQ(p.id(), std::string("sp-1")); + ASSERT_EQ(p.kms_key_name, std::string("k")); + ASSERT_EQ(p.scope, std::string("GEMINI_LIVE")); +} + +TEST(sessions_serving_profiles, requests_use_the_declared_rows) { + Rig rig; + ServingProfiles sp(rig.client, LocationName(kLocation)); + ASSERT_EQ(sp.endpoint().base_url, + std::string("https://us-central1-aiplatform.googleapis.com/v1beta1")); + ServingProfileSpec spec; + spec.profile_id = "sp-1"; + spec.display_name = "Live"; + spec.scope = ServingScope::kGeminiLive; + auto r = sent(rig, [&] { sp.create(spec); }); + ASSERT_EQ(r.method, std::string("POST")); + ASSERT_TRUE(has(r.target, std::string(kLocation) + "/servingProfiles?servingProfileId=sp-1")); + ASSERT_TRUE(has(r.body, "\"scope\":\"GEMINI_LIVE\"")); + r = sent(rig, [&] { sp.get("sp-1"); }); + ASSERT_EQ(r.method, std::string("GET")); + ASSERT_TRUE(has(r.target, std::string(kLocation) + "/servingProfiles/sp-1")); + r = sent(rig, [&] { sp.list(25); }); + ASSERT_TRUE(has(r.target, "/servingProfiles?pageSize=25")); + ServingProfilePatch patch; + patch.profile_id = "sp-1"; + patch.display_name = "new"; + patch.update_mask = "displayName"; + r = sent(rig, [&] { sp.patch(patch); }); + ASSERT_EQ(r.method, std::string("PATCH")); + ASSERT_TRUE(has(r.target, "/servingProfiles/sp-1?updateMask=displayName")); + r = sent(rig, [&] { sp.remove("sp-1"); }); + ASSERT_EQ(r.method, std::string("DELETE")); + r = sent(rig, [&] { sp.list_profile_operations("sp-1"); }); + ASSERT_TRUE(has(r.target, "/servingProfiles/sp-1/operations")); + r = sent(rig, [&] { + sp.wait_server(std::string(kLocation) + "/servingProfiles/sp-1/operations/o1", 5); + }); + ASSERT_TRUE(has(r.target, "/servingProfiles/sp-1/operations/o1:wait?timeout=5s")); + // The location root has no operations of its own. + invalid([&] { sp.list_operations(); }); + invalid([&] { sp.operation(engine_op()); }); +} + +TEST(sessions_serving_profiles, validation_and_credentials_run_before_traffic) { + Rig rig; + ServingProfiles sp(rig.client, LocationName(kLocation)); + invalid([&] { sp.get("Bad_Id"); }); + invalid([&] { sp.remove(""); }); + invalid([&] { sp.create(ServingProfileSpec{}); }); + invalid([&] { sp.list(101); }); + ASSERT_EQ(rig.mock->send_count(), std::size_t(0)); + Rig off(false); + ServingProfiles n(off.client, LocationName(kLocation)); + ASSERT_THROW(n.get("sp-1"), NotConfiguredError); + ASSERT_THROW(n.list(), NotConfiguredError); + ASSERT_EQ(off.mock->send_count(), std::size_t(0)); +} + +// Read-only live probe of the gap-closing calls (MOLO_LIVE_NET=1, a +// credential, GOOGLE_CLOUD_PROJECT; loud SKIP otherwise). Observed +// 2026-10-24 with a service-account token: operations lists under any engine, +// session or memory answer 200 with an empty list even for a missing engine; +// servingProfiles list answers 501 UNIMPLEMENTED (not available to the +// project). Either answer is accepted for servingProfiles and reported. +TEST(sessions_live, operations_and_serving_profiles_reads) { + std::string project = Credentials::env("GOOGLE_CLOUD_PROJECT"); + bool has_credential = + !Credentials::env().empty() || !Credentials::env("GOOGLE_ACCESS_TOKEN").empty(); + if (!std::getenv("MOLO_LIVE_NET") || !has_credential || project.empty()) { + std::cout << "SKIP sessions_live gaps (set MOLO_LIVE_NET=1, a credential and " + "GOOGLE_CLOUD_PROJECT)\n"; + return; + } + auto client = GoogleClient::from_env(); + EngineName engine("projects/" + project + "/locations/us-central1/reasoningEngines/1"); + Sessions s(client, engine); + MemoryBank b(client, engine); + ASSERT_EQ(s.list_operations().size(), std::size_t(0)); + ASSERT_EQ(s.list_session_operations("probe").size(), std::size_t(0)); + ASSERT_EQ(b.list_memory_operations("probe").size(), std::size_t(0)); + ServingProfiles sp(client, LocationName("projects/" + project + "/locations/us-central1")); + try { + std::size_t n = sp.list(5).size(); + std::cout << "servingProfiles list: 200, " << n << " profiles\n"; + } catch (const GoogleError& e) { + std::cout << "servingProfiles list: " << e.http_status() << " " << e.status() << "\n"; + ASSERT_TRUE(e.http_status() == 501 || e.http_status() == 403 || e.http_status() == 404); + } +}