diff --git a/example/app/google/DataDemo.cpp b/example/app/google/DataDemo.cpp index a48f2a8..89a993f 100644 --- a/example/app/google/DataDemo.cpp +++ b/example/app/google/DataDemo.cpp @@ -1,13 +1,757 @@ #include "../GoogleDemos.hpp" +#include +#include + #include "../../fw/Router.hpp" +#include "../../fw/Templates.hpp" +#include "google/data/data.h" +#include "google/data/caps.h" namespace molo::example::app { -// BigQuery, Cloud Storage, Pub/Sub, Firestore, Spanner, Bigtable, Secret Manager, Dataplex, DLP, KMS. Stub: the package agent fills this in. +// Google data package: BigQuery, Cloud Storage, Pub/Sub, Firestore, Spanner, +// Bigtable admin, Secret Manager, Dataplex, DLP, Cloud KMS, Datastore over the +// real src/google/data part. GET page, /status and /calls answer without a +// credential. Every other route is a POST: 422 on bad input, 503 +// google_not_configured without a credential. Reads run directly; writes, +// secret reads and paid operations are dry runs unless the body has +// "confirm":true. + +namespace { + +namespace dt = google::data; +using Json = clients::body::Json; + +constexpr std::size_t kMaxBody = 3 * 1024 * 1024; +constexpr std::size_t kMaxShown = 256 * 1024; // response bytes returned to the page +constexpr std::size_t kMaxDemoUpload = 2 * 1024 * 1024; +constexpr std::size_t kMaxDownloadShown = 64 * 1024; + +const char kPage[] = R"DATA({% extends "base.html" %} +{% block content %} +

Google data

+

BigQuery, Cloud Storage, Pub/Sub, Firestore, Spanner, Bigtable admin, +Secret Manager, Dataplex, DLP, Cloud KMS and Datastore. Reads run at once; writes, +secret reads and paid calls stay a dry run until the body says "confirm":true.

+

Checking Google credentials...

+
+ + + +
+ +

+
+ + +{% endblock %} +)DATA"; + +fw::Response json_out(const Json& j, int status = 200) { + fw::Response res = fw::JsonResponse(j.dump(), status); + res.headers.set("Cache-Control", "no-store"); + return res; +} + +[[noreturn]] void bad(const std::string& why) { + throw fw::HttpError(422, "validation_error", why); +} + +[[noreturn]] void rethrow(const google::GoogleError& e) { + if (e.http_status() == 503 && e.status() == "NOT_CONFIGURED") + throw fw::HttpError(503, "google_not_configured", e.message()); + if (e.service().empty() && e.status() == "INVALID_ARGUMENT") + throw fw::HttpError(422, "validation_error", e.message()); + throw fw::HttpError(502, "google_error", e.status() + ": " + e.message()); +} + +Json parse_body(const fw::Request& req) { + if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large"); + if (req.body.empty()) return Json::object(); + try { + Json j = Json::parse(req.body); + if (!j.is_object()) bad("body must be a JSON object"); + return j; + } catch (const clients::body::JsonError&) { + bad("body is not valid JSON"); + } +} + +std::string str(const Json& b, const char* key, bool required = false) { + const Json* v = b.find(key); + if (!v || v->is_null()) { + if (required) bad(std::string(key) + " is required"); + return ""; + } + if (!v->is_string()) bad(std::string(key) + " must be a string"); + return v->as_string(); +} + +int num(const Json& b, const char* key, int fallback) { + const Json* v = b.find(key); + if (!v || v->is_null()) return fallback; + if (!v->is_number()) bad(std::string(key) + " must be a number"); + return static_cast(v->as_int()); +} + +bool flag(const Json& b, const char* key) { + const Json* v = b.find(key); + if (!v || v->is_null()) return false; + if (!v->is_bool()) bad(std::string(key) + " must be true or false"); + return v->as_bool(); +} + +Json member(const Json& b, const char* key, bool object, bool required = false) { + const Json* v = b.find(key); + if (!v || v->is_null()) { + if (required) bad(std::string(key) + " is required"); + return Json(); + } + if (object ? !v->is_object() : !v->is_array()) + bad(std::string(key) + (object ? " must be a JSON object" : " must be a JSON array")); + return *v; +} + +std::vector strings(const Json& b, const char* key) { + std::vector out; + Json a = member(b, key, false); + if (a.is_null()) return out; + for (const Json& x : a.as_array()) { + if (!x.is_string()) bad(std::string(key) + " must hold strings"); + out.push_back(x.as_string()); + } + return out; +} + +// A response that fits the page; bigger ones are cut and say so. +Json shown(const Json& j) { + std::string text = j.dump(); + if (text.size() <= kMaxShown) return j; + Json o = Json::object(); + o.set("truncated", Json(true)); + o.set("bytes", Json(static_cast(text.size()))); + o.set("head", Json(text.substr(0, kMaxShown / 4))); + return o; +} + +Json dry(const std::string& group, const std::string& action) { + Json o = Json::object(); + o.set("dry_run", Json(true)); + o.set("service", Json(group)); + o.set("action", Json(action)); + o.set("hint", Json("send confirm:true to run this action")); + return o; +} + +// One action of a service: runs against Google, or is a dry run until confirmed. +struct Action { + bool confirm_required; + std::function run; +}; +using ActionMap = std::map; + +class DataRoutes { + public: + explicit DataRoutes(std::shared_ptr client) : client_(std::move(client)) { + build_bigquery(); + build_storage(); + build_pubsub(); + build_firestore(); + build_spanner(); + build_bigtable(); + build_secrets(); + build_dataplex(); + build_dlp(); + build_kms(); + build_datastore(); + } + + fw::Response handle(const std::string& group, const fw::Request& req) const { + Json body = parse_body(req); + std::string action = str(body, "action", true); + const ActionMap& actions = groups_.at(group); + auto it = actions.find(action); + if (it == actions.end()) { + std::string known; + for (const auto& kv : actions) known += (known.empty() ? "" : ", ") + kv.first; + bad("unknown action; use one of: " + known); + } + if (it->second.confirm_required && !flag(body, "confirm")) return json_out(dry(group, action)); + try { + return json_out(shown(it->second.run(body))); + } catch (const google::GoogleError& e) { + rethrow(e); + } + } + + fw::Response sign(const fw::Request& req) const { + Json body = parse_body(req); + dt::SignedUrlRequest r; + r.method = str(body, "method"); + if (r.method.empty()) r.method = "GET"; + r.bucket = str(body, "bucket", true); + r.object = str(body, "object"); + r.expires_seconds = num(body, "expires_seconds", 900); + Json headers = member(body, "headers", true); + if (!headers.is_null()) + for (const auto& kv : headers.as_object()) { + if (!kv.second.is_string()) bad("header values must be strings"); + r.headers[kv.first] = kv.second.as_string(); + } + try { + r.validate(); + // A signed URL is a bearer capability: dry run unless confirmed. + if (!flag(body, "confirm")) { + Json o = dry("storage/sign", "sign"); + o.set("expires_seconds", Json(static_cast(r.expires_seconds))); + o.set("max_expires_seconds", Json(static_cast(dt::DataCaps::kMaxSignedUrlSeconds))); + return json_out(o); + } + dt::SignedUrl signer = dt::SignedUrl::from_env(); + Json o = Json::object(); + o.set("url", Json(signer.sign(r))); + o.set("method", Json(r.method)); + o.set("expires_seconds", Json(static_cast(r.expires_seconds))); + o.set("signer", Json(signer.email())); + return json_out(o); + } catch (const google::GoogleError& e) { + rethrow(e); + } + } + + // Any row of the call table by id. Non-GET rows are dry runs unless confirmed. + fw::Response call(const fw::Request& req) const { + Json body = parse_body(req); + std::string rpc = str(body, "rpc", true); + try { + const dt::CallSpec& spec = dt::Catalog::get(rpc); + dt::CallArgs args; + Json path = member(body, "path", true); + if (!path.is_null()) + for (const auto& kv : path.as_object()) { + if (!kv.second.is_string()) bad("path values must be strings"); + args.p(kv.first, kv.second.as_string()); + } + Json query = member(body, "query", true); + if (!query.is_null()) args.query = query; + Json b = member(body, "body", true); + if (!b.is_null()) args.body = b; + dt::ApiService svc(client_, dt::Catalog::api_of(spec)); + dt::Prepared prepared = svc.prepare(rpc, args); // validates: 422 + bool read = std::string(spec.http) == "GET"; + if (!read && !flag(body, "confirm")) { + Json o = dry("call", rpc); + o.set("method", Json(spec.http)); + o.set("url", Json(prepared.url())); + return json_out(o); + } + return json_out(shown(svc.invoke(rpc, args))); + } catch (const google::GoogleError& e) { + rethrow(e); + } + } + + const std::map& groups() const { return groups_; } + + private: + using PF = std::shared_ptr; + + PF platform() const { + return std::make_shared(client_, google::CloudContext::from_env()); + } + void add(const std::string& group, const std::string& action, bool confirm, + std::function fn) { + groups_[group][action] = Action{confirm, [this, fn](const Json& b) { + PF p = platform(); + return fn(b, *p); + }}; + } + + void build_bigquery() { + const char* g = "bigquery"; + auto request = [](const Json& b) { + dt::QueryRequest q; + q.sql = str(b, "sql", true); + q.use_legacy_sql = flag(b, "legacy"); + q.max_results = num(b, "max_results", 100); + q.timeout_ms = num(b, "timeout_ms", 10000); + q.location = str(b, "location"); + q.default_dataset = str(b, "default_dataset"); + q.maximum_bytes_billed = str(b, "maximum_bytes_billed"); + q.query_parameters = member(b, "parameters", false); + q.parameter_mode = str(b, "parameter_mode"); + return q; + }; + add(g, "dry_run", false, [request](const Json& b, dt::DataPlatform& p) { + dt::QueryRequest q = request(b); + q.dry_run = true; + return p.bigquery.query(q); + }); + add(g, "query", true, [request](const Json& b, dt::DataPlatform& p) { + dt::QueryRequest q = request(b); + q.maximum_bytes_billed = q.maximum_bytes_billed.empty() ? "1000000000" : q.maximum_bytes_billed; + return p.bigquery.query(q); // billed bytes capped at 1 GB unless the body says otherwise + }); + add(g, "results", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigquery.query_results(str(b, "job_id", true), str(b, "location"), + num(b, "max_results", 100), str(b, "page_token")); + }); + add(g, "datasets", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigquery.list_datasets(num(b, "max_results", 50)); + }); + add(g, "dataset", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigquery.get_dataset(str(b, "dataset", true)); + }); + add(g, "tables", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigquery.list_tables(str(b, "dataset", true), num(b, "max_results", 50)); + }); + add(g, "table", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigquery.get_table(str(b, "dataset", true), str(b, "table", true)); + }); + add(g, "rows", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigquery.list_rows(str(b, "dataset", true), str(b, "table", true), + num(b, "max_results", 20), str(b, "page_token")); + }); + add(g, "jobs", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigquery.list_jobs(num(b, "max_results", 20), str(b, "state")); + }); + add(g, "job", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigquery.get_job(str(b, "job_id", true), str(b, "location")); + }); + add(g, "models", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigquery.list_models(str(b, "dataset", true), num(b, "max_results", 50)); + }); + add(g, "routines", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigquery.list_routines(str(b, "dataset", true), num(b, "max_results", 50)); + }); + add(g, "insert", true, [](const Json& b, dt::DataPlatform& p) { + return p.bigquery.insert_all(str(b, "dataset", true), str(b, "table", true), + member(b, "rows", false, true), flag(b, "skip_invalid_rows"), + flag(b, "ignore_unknown_values")); + }); + add(g, "create_dataset", true, [](const Json& b, dt::DataPlatform& p) { + return p.bigquery.create_dataset(str(b, "dataset", true), str(b, "location"), + str(b, "description")); + }); + add(g, "delete_table", true, [](const Json& b, dt::DataPlatform& p) { + return p.bigquery.delete_table(str(b, "dataset", true), str(b, "table", true)); + }); + add(g, "delete_dataset", true, [](const Json& b, dt::DataPlatform& p) { + return p.bigquery.delete_dataset(str(b, "dataset", true), flag(b, "delete_contents")); + }); + } + + static std::string bytes_of(const Json& b) { + std::string text = str(b, "text"); + std::string b64 = str(b, "base64"); + if (!text.empty() == !b64.empty()) bad("give exactly one of text and base64"); + std::string data = text; + if (!b64.empty() && !security::crypto::Base64::decode(b64, data)) bad("base64 is not valid"); + if (data.size() > kMaxDemoUpload) bad("data over the demo cap of 2 MiB"); + return data; + } + + void build_storage() { + const char* g = "storage"; + add(g, "buckets", false, [](const Json& b, dt::DataPlatform& p) { + return p.storage.list_buckets(num(b, "max_results", 50), str(b, "prefix"), str(b, "page_token")); + }); + add(g, "bucket", false, [](const Json& b, dt::DataPlatform& p) { + return p.storage.get_bucket(str(b, "bucket", true)); + }); + add(g, "objects", false, [](const Json& b, dt::DataPlatform& p) { + return p.storage.list_objects(str(b, "bucket", true), str(b, "prefix"), str(b, "delimiter"), + num(b, "max_results", 50), str(b, "page_token")); + }); + add(g, "object", false, [](const Json& b, dt::DataPlatform& p) { + return p.storage.get_object(str(b, "bucket", true), str(b, "object", true)); + }); + add(g, "download", false, [](const Json& b, dt::DataPlatform&) { + auto media = dt::StorageMedia::from_env(); + std::string bucket = str(b, "bucket", true), object = str(b, "object", true); + dt::Downloaded d = media->download_range( + bucket, object, 0, static_cast(kMaxDownloadShown) - 1); + Json o = Json::object(); + o.set("bytes", Json(static_cast(d.data.size()))); + o.set("content_type", Json(d.content_type)); + o.set("generation", Json(d.generation)); + o.set("base64", Json(security::crypto::Base64::encode(d.data))); + o.set("first_bytes_only", Json(true)); + return o; + }); + add(g, "upload", true, [](const Json& b, dt::DataPlatform&) { + auto media = dt::StorageMedia::from_env(); + std::string type = str(b, "content_type"); + return media->upload_simple(str(b, "bucket", true), str(b, "object", true), bytes_of(b), + type.empty() ? "application/octet-stream" : type); + }); + add(g, "resumable", true, [](const Json& b, dt::DataPlatform&) { + auto media = dt::StorageMedia::from_env(); + std::string bucket = str(b, "bucket", true), object = str(b, "object", true); + std::string type = str(b, "content_type"); + if (type.empty()) type = "application/octet-stream"; + std::string data = bytes_of(b); + long long total = static_cast(data.size()); + std::string session = media->resumable_start(bucket, object, type, total, Json()); + dt::ResumableProgress progress; + long long offset = 0; + const long long step = static_cast(dt::DataCaps::kMaxResumableChunk); + while (offset < total || (total == 0 && offset == 0)) { + long long n = std::min(step, total - offset); + if (total == 0) break; + progress = media->resumable_put(session, offset, data.substr(offset, n), total); + offset = progress.next_offset; + if (progress.done) break; + } + Json o = Json::object(); + o.set("done", Json(progress.done)); + o.set("object", progress.object); + return o; + }); + add(g, "compose", true, [](const Json& b, dt::DataPlatform& p) { + std::vector sources; + for (const std::string& s : strings(b, "sources")) sources.push_back({s, 0}); + return p.storage.compose(str(b, "bucket", true), sources, str(b, "destination", true), + str(b, "content_type")); + }); + add(g, "copy", true, [](const Json& b, dt::DataPlatform& p) { + return p.storage.copy_object(str(b, "bucket", true), str(b, "object", true), + str(b, "dest_bucket", true), str(b, "dest_object", true)); + }); + add(g, "delete", true, [](const Json& b, dt::DataPlatform& p) { + return p.storage.delete_object(str(b, "bucket", true), str(b, "object", true)); + }); + } + + void build_pubsub() { + const char* g = "pubsub"; + add(g, "topics", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.list_topics(num(b, "max_results", 50)); }); + add(g, "topic", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.get_topic(str(b, "topic", true)); }); + add(g, "subscriptions", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.list_subscriptions(num(b, "max_results", 50)); }); + add(g, "subscription", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.get_subscription(str(b, "subscription", true)); }); + add(g, "snapshots", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.list_snapshots(num(b, "max_results", 50)); }); + add(g, "publish", true, [](const Json& b, dt::DataPlatform& p) { + std::vector messages; + for (const std::string& m : strings(b, "messages")) { + dt::PubsubMessage pm; + pm.data = m; + messages.push_back(pm); + } + return p.pubsub.publish(str(b, "topic", true), messages); + }); + add(g, "pull", true, [](const Json& b, dt::DataPlatform& p) { + return p.pubsub.pull(str(b, "subscription", true), num(b, "max_messages", 10)); + }); + add(g, "ack", true, [](const Json& b, dt::DataPlatform& p) { + return p.pubsub.acknowledge(str(b, "subscription", true), strings(b, "ack_ids")); + }); + add(g, "create_topic", true, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.create_topic(str(b, "topic", true)); }); + add(g, "delete_topic", true, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.delete_topic(str(b, "topic", true)); }); + } + + void build_firestore() { + const char* g = "firestore"; + add(g, "databases", false, [](const Json&, dt::DataPlatform& p) { return p.firestore.list_databases(); }); + add(g, "database", false, [](const Json&, dt::DataPlatform& p) { return p.firestore.get_database(); }); + add(g, "get", false, [](const Json& b, dt::DataPlatform& p) { return p.firestore.get_document(str(b, "path", true)); }); + add(g, "list", false, [](const Json& b, dt::DataPlatform& p) { + return p.firestore.list_documents(str(b, "parent"), str(b, "collection", true), + num(b, "max_results", 20), str(b, "page_token")); + }); + add(g, "collections", false, [](const Json& b, dt::DataPlatform& p) { + return p.firestore.list_collection_ids(str(b, "parent"), num(b, "max_results", 50)); + }); + add(g, "query", false, [](const Json& b, dt::DataPlatform& p) { + return p.firestore.run_query(str(b, "parent"), member(b, "query", true, true)); + }); + add(g, "aggregate", false, [](const Json& b, dt::DataPlatform& p) { + return p.firestore.run_aggregation_query(str(b, "parent"), member(b, "query", true, true)); + }); + add(g, "batch_get", false, [](const Json& b, dt::DataPlatform& p) { + return p.firestore.batch_get(strings(b, "paths")); + }); + add(g, "create", true, [](const Json& b, dt::DataPlatform& p) { + return p.firestore.create_document(str(b, "parent"), str(b, "collection", true), + str(b, "document_id"), member(b, "fields", true, true)); + }); + add(g, "commit", true, [](const Json& b, dt::DataPlatform& p) { + return p.firestore.commit(member(b, "writes", false, true)); + }); + add(g, "delete", true, [](const Json& b, dt::DataPlatform& p) { + return p.firestore.delete_document(str(b, "path", true)); + }); + } + + void build_spanner() { + const char* g = "spanner"; + add(g, "instances", false, [](const Json& b, dt::DataPlatform& p) { return p.spanner.list_instances(num(b, "max_results", 50)); }); + add(g, "instance", false, [](const Json& b, dt::DataPlatform& p) { return p.spanner.get_instance(str(b, "instance", true)); }); + add(g, "databases", false, [](const Json& b, dt::DataPlatform& p) { + return p.spanner.list_databases(str(b, "instance", true), num(b, "max_results", 50)); + }); + add(g, "database", false, [](const Json& b, dt::DataPlatform& p) { + return p.spanner.get_database(str(b, "instance", true), str(b, "database", true)); + }); + add(g, "ddl", false, [](const Json& b, dt::DataPlatform& p) { + return p.spanner.get_ddl(str(b, "instance", true), str(b, "database", true)); + }); + // Creates a session, runs one read-only statement, deletes the session. + add(g, "query", true, [](const Json& b, dt::DataPlatform& p) { + dt::SpannerSql s; + s.sql = str(b, "sql", true); + s.params = member(b, "params", true); + s.param_types = member(b, "param_types", true); + std::string mode = str(b, "query_mode"); + if (!mode.empty()) s.query_mode = mode; + return p.spanner.query(str(b, "instance", true), str(b, "database", true), s); + }); + } + + void build_bigtable() { + const char* g = "bigtable"; + add(g, "instances", false, [](const Json&, dt::DataPlatform& p) { return p.bigtable.list_instances(); }); + add(g, "instance", false, [](const Json& b, dt::DataPlatform& p) { return p.bigtable.get_instance(str(b, "instance", true)); }); + add(g, "clusters", false, [](const Json& b, dt::DataPlatform& p) { return p.bigtable.list_clusters(str(b, "instance", true)); }); + add(g, "cluster", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigtable.get_cluster(str(b, "instance", true), str(b, "cluster", true)); + }); + add(g, "tables", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigtable.list_tables(str(b, "instance", true), num(b, "max_results", 50)); + }); + add(g, "table", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigtable.get_table(str(b, "instance", true), str(b, "table", true)); + }); + add(g, "profiles", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigtable.list_app_profiles(str(b, "instance", true), num(b, "max_results", 50)); + }); + add(g, "iam", false, [](const Json& b, dt::DataPlatform& p) { + return p.bigtable.table_iam_policy(str(b, "instance", true), str(b, "table", true)); + }); + } + + void build_secrets() { + const char* g = "secrets"; + add(g, "list", false, [](const Json& b, dt::DataPlatform& p) { return p.secrets.list_secrets(num(b, "max_results", 50), str(b, "filter")); }); + add(g, "get", false, [](const Json& b, dt::DataPlatform& p) { return p.secrets.get_secret(str(b, "secret", true)); }); + add(g, "versions", false, [](const Json& b, dt::DataPlatform& p) { return p.secrets.list_versions(str(b, "secret", true), num(b, "max_results", 50)); }); + add(g, "version", false, [](const Json& b, dt::DataPlatform& p) { + return p.secrets.get_version(str(b, "secret", true), str(b, "version", true)); + }); + add(g, "access", true, [](const Json& b, dt::DataPlatform& p) { + return p.secrets.access_version(str(b, "secret", true), str(b, "version", true)); + }); + add(g, "create", true, [](const Json& b, dt::DataPlatform& p) { return p.secrets.create_secret(str(b, "secret", true)); }); + add(g, "add_version", true, [](const Json& b, dt::DataPlatform& p) { + return p.secrets.add_version(str(b, "secret", true), str(b, "payload", true)); + }); + add(g, "disable", true, [](const Json& b, dt::DataPlatform& p) { + return p.secrets.disable_version(str(b, "secret", true), str(b, "version", true)); + }); + add(g, "destroy", true, [](const Json& b, dt::DataPlatform& p) { + return p.secrets.destroy_version(str(b, "secret", true), str(b, "version", true)); + }); + add(g, "delete", true, [](const Json& b, dt::DataPlatform& p) { return p.secrets.delete_secret(str(b, "secret", true)); }); + } + + void build_dataplex() { + const char* g = "dataplex"; + add(g, "locations", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_locations(num(b, "max_results", 50)); }); + add(g, "lakes", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_lakes(str(b, "location"), num(b, "max_results", 50)); }); + add(g, "lake", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.get_lake(str(b, "location"), str(b, "lake", true)); }); + add(g, "entry_groups", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_entry_groups(str(b, "location"), num(b, "max_results", 50)); }); + add(g, "entry_group", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.get_entry_group(str(b, "location"), str(b, "group", true)); }); + add(g, "entry_types", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_entry_types(str(b, "location"), num(b, "max_results", 50)); }); + add(g, "aspect_types", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_aspect_types(str(b, "location"), num(b, "max_results", 50)); }); + add(g, "scans", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_data_scans(str(b, "location"), num(b, "max_results", 50)); }); + add(g, "scan", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.get_data_scan(str(b, "location"), str(b, "scan", true)); }); + add(g, "search", false, [](const Json& b, dt::DataPlatform& p) { + return p.dataplex.search_entries(str(b, "location"), str(b, "query", true), num(b, "max_results", 20)); + }); + add(g, "lookup", false, [](const Json& b, dt::DataPlatform& p) { + return p.dataplex.lookup_entry(str(b, "location"), str(b, "entry", true)); + }); + } + + void build_dlp() { + const char* g = "dlp"; + add(g, "info_types", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_info_types(str(b, "language")); }); + add(g, "inspect_templates", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_inspect_templates(num(b, "max_results", 50)); }); + add(g, "deidentify_templates", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_deidentify_templates(num(b, "max_results", 50)); }); + add(g, "triggers", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_job_triggers(num(b, "max_results", 50)); }); + add(g, "jobs", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_jobs(num(b, "max_results", 50)); }); + add(g, "inspect", true, [](const Json& b, dt::DataPlatform& p) { // paid per byte inspected + dt::InspectRequest r; + r.text = str(b, "text", true); + r.info_types = strings(b, "info_types"); + std::string likelihood = str(b, "min_likelihood"); + if (!likelihood.empty()) r.min_likelihood = likelihood; + r.include_quote = flag(b, "include_quote"); + r.max_findings = num(b, "max_findings", 100); + r.location = str(b, "location"); + return p.dlp.inspect_content(r); + }); + add(g, "deidentify", true, [](const Json& b, dt::DataPlatform& p) { + return p.dlp.deidentify_content(str(b, "text", true), member(b, "config", true, true), + str(b, "location")); + }); + } + + void build_kms() { + const char* g = "kms"; + add(g, "locations", false, [](const Json& b, dt::DataPlatform& p) { return p.kms.list_locations(num(b, "max_results", 50)); }); + add(g, "rings", false, [](const Json& b, dt::DataPlatform& p) { return p.kms.list_key_rings(str(b, "location", true), num(b, "max_results", 50)); }); + add(g, "ring", false, [](const Json& b, dt::DataPlatform& p) { return p.kms.get_key_ring(str(b, "location", true), str(b, "ring", true)); }); + add(g, "keys", false, [](const Json& b, dt::DataPlatform& p) { + return p.kms.list_keys(str(b, "location", true), str(b, "ring", true), num(b, "max_results", 50)); + }); + add(g, "key", false, [](const Json& b, dt::DataPlatform& p) { + return p.kms.get_key(str(b, "location", true), str(b, "ring", true), str(b, "key", true)); + }); + add(g, "versions", false, [](const Json& b, dt::DataPlatform& p) { + return p.kms.list_versions(str(b, "location", true), str(b, "ring", true), str(b, "key", true), + num(b, "max_results", 50)); + }); + add(g, "public_key", false, [](const Json& b, dt::DataPlatform& p) { + return p.kms.get_public_key(str(b, "location", true), str(b, "ring", true), str(b, "key", true), + str(b, "version", true)); + }); + add(g, "encrypt", true, [](const Json& b, dt::DataPlatform& p) { + return p.kms.encrypt(str(b, "location", true), str(b, "ring", true), str(b, "key", true), + str(b, "plaintext", true), str(b, "aad")); + }); + add(g, "decrypt", true, [](const Json& b, dt::DataPlatform& p) { + return p.kms.decrypt(str(b, "location", true), str(b, "ring", true), str(b, "key", true), + str(b, "ciphertext", true), str(b, "aad")); + }); + } + + void build_datastore() { + const char* g = "datastore"; + add(g, "lookup", false, [](const Json& b, dt::DataPlatform& p) { return p.datastore.lookup(member(b, "keys", false, true)); }); + add(g, "query", false, [](const Json& b, dt::DataPlatform& p) { + return p.datastore.run_query(member(b, "query", true), member(b, "gql", true)); + }); + add(g, "aggregate", false, [](const Json& b, dt::DataPlatform& p) { + return p.datastore.run_aggregation_query(member(b, "query", true, true)); + }); + add(g, "begin", true, [](const Json& b, dt::DataPlatform& p) { return p.datastore.begin_transaction(flag(b, "read_only")); }); + add(g, "rollback", true, [](const Json& b, dt::DataPlatform& p) { return p.datastore.rollback(str(b, "transaction", true)); }); + add(g, "allocate", true, [](const Json& b, dt::DataPlatform& p) { return p.datastore.allocate_ids(member(b, "keys", false, true)); }); + add(g, "reserve", true, [](const Json& b, dt::DataPlatform& p) { return p.datastore.reserve_ids(member(b, "keys", false, true)); }); + add(g, "commit", true, [](const Json& b, dt::DataPlatform& p) { + std::string mode = str(b, "mode"); + return p.datastore.commit(member(b, "mutations", false, true), + mode.empty() ? "NON_TRANSACTIONAL" : mode, str(b, "transaction")); + }); + } + + std::shared_ptr client_; + std::map groups_; +}; + +} // namespace + void register_google_data_demo(fw::Router& router, Services& services) { - (void)router; - (void)services; + if (!services.google) services.google = google::GoogleClient::from_env(); + auto client = services.google; + fw::Environment* env = services.templates.get(); + services.templates->add_template("google_data.html", kPage); + services.add_feature("google-data", "/google/data"); + + auto routes = std::make_shared(client); + + router.get("/google/data", "google-data-page", [env](fw::Request req) { + (void)req; + fw::TemplateCtx ctx; + ctx["title"] = std::string("Google data"); + return fw::Response(200, env->render("google_data.html", ctx), "text/html; charset=utf-8"); + }); + + router.get("/google/data/status", "google-data-status", [client, routes](fw::Request req) { + (void)req; + google::CloudContext ctx = google::CloudContext::from_env(); + Json o = Json::object(); + o.set("configured", Json(client->configured())); + o.set("project_set", Json(!ctx.project.empty())); + o.set("location", Json(ctx.location)); + o.set("calls", Json(static_cast(dt::Catalog::size()))); + Json apis = Json::array(); + for (std::size_t i = 0; i < dt::Catalog::api_count(); ++i) { + const dt::ApiInfo& info = dt::Catalog::info_at(i); + Json a = Json::object(); + a.set("name", Json(info.name)); + a.set("version", Json(info.version)); + a.set("service", Json(info.service)); + a.set("calls", Json(static_cast(dt::Catalog::count_of(info.api)))); + apis.push(a); + } + o.set("apis", apis); + Json groups = Json::object(); + for (const auto& g : routes->groups()) { + Json names = Json::array(); + for (const auto& a : g.second) { + Json n = Json::object(); + n.set("action", Json(a.first)); + n.set("confirm", Json(a.second.confirm_required)); + names.push(n); + } + groups.set(g.first, names); + } + o.set("actions", groups); + o.set("max_signed_url_seconds", Json(static_cast(dt::DataCaps::kMaxSignedUrlSeconds))); + return json_out(o); + }); + + // The discovery-checked call table, optionally one API: /calls?api=bigquery + router.get("/google/data/calls", "google-data-calls", [](fw::Request req) { + std::string only = req.query_or("api", ""); + Json arr = Json::array(); + for (std::size_t i = 0; i < dt::Catalog::size(); ++i) { + const dt::CallSpec& s = dt::Catalog::at(i); + if (!only.empty() && dt::Catalog::info(dt::Catalog::api_of(s)).name != only) continue; + Json row = Json::object(); + row.set("rpc", Json(s.rpc)); + row.set("http", Json(s.http)); + row.set("path", Json(s.path)); + arr.push(row); + } + Json o = Json::object(); + o.set("calls", arr); + return json_out(o); + }); + + static const char* const kGroups[] = {"bigquery", "storage", "pubsub", "firestore", "spanner", + "bigtable", "secrets", "dataplex", "dlp", "kms", "datastore"}; + for (const char* g : kGroups) { + std::string group = g; + router.post("/google/data/" + group, "google-data-" + group, + [routes, group](fw::Request req) { return routes->handle(group, req); }); + } + router.post("/google/data/storage/sign", "google-data-storage-sign", + [routes](fw::Request req) { return routes->sign(req); }); + router.post("/google/data/call", "google-data-call", + [routes](fw::Request req) { return routes->call(req); }); } } // namespace molo::example::app diff --git a/example/assets/google_data.js b/example/assets/google_data.js new file mode 100644 index 0000000..4f5baf0 --- /dev/null +++ b/example/assets/google_data.js @@ -0,0 +1,97 @@ +// Google data page: thin client over /google/data/* (all limits server side). +(function () { + "use strict"; + var $ = function (id) { return document.getElementById(id); }; + var out = $("gd-out"), err = $("gd-error"); + var actions = {}; + + // One example body per action; the server validates everything. + var examples = { + "bigquery": { + dry_run: { sql: "SELECT 1 AS one" }, + query: { sql: "SELECT 1 AS one", confirm: false }, + datasets: {}, jobs: {}, + tables: { dataset: "DATASET" }, table: { dataset: "DATASET", table: "TABLE" }, + rows: { dataset: "DATASET", table: "TABLE", max_results: 10 }, + insert: { dataset: "DATASET", table: "TABLE", rows: [{ json: { a: 1 } }], confirm: false } + }, + "storage": { + buckets: {}, objects: { bucket: "BUCKET", prefix: "" }, + object: { bucket: "BUCKET", object: "path/file.txt" }, + download: { bucket: "BUCKET", object: "path/file.txt" }, + upload: { bucket: "BUCKET", object: "demo.txt", text: "hello", content_type: "text/plain", confirm: false }, + compose: { bucket: "BUCKET", sources: ["a.txt", "b.txt"], destination: "ab.txt", confirm: false } + }, + "pubsub": { topics: {}, subscriptions: {}, publish: { topic: "TOPIC", messages: ["hello"], confirm: false } }, + "firestore": { databases: {}, collections: {}, get: { path: "users/alice" } }, + "spanner": { instances: {}, databases: { instance: "INSTANCE" } }, + "bigtable": { instances: {}, tables: { instance: "INSTANCE" } }, + "secrets": { list: {}, versions: { secret: "SECRET" }, access: { secret: "SECRET", version: "latest", confirm: false } }, + "dataplex": { locations: {}, search: { location: "global", query: "type=table" } }, + "dlp": { info_types: {}, inspect: { text: "mail jane@example.com", info_types: ["EMAIL_ADDRESS"], confirm: false } }, + "kms": { locations: {}, rings: { location: "global" } }, + "datastore": { query: { gql: { queryString: "SELECT * FROM __kind__ LIMIT 1" } } }, + "storage/sign": { sign: { bucket: "BUCKET", object: "path/file.txt", method: "GET", expires_seconds: 900, confirm: false } }, + "call": { call: { rpc: "bigquery.datasets.list", path: { projectId: "PROJECT" } } } + }; + + function post(path, body) { + err.textContent = ""; + out.textContent = "..."; + return fetch(path, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body) + }).then(function (res) { + return res.text().then(function (t) { + var j; + try { j = JSON.parse(t); } catch (e) { j = { message: t }; } + if (!res.ok) { err.textContent = j.message || ("HTTP " + res.status); out.textContent = ""; } + else { out.textContent = JSON.stringify(j, null, 2); } + }); + }).catch(function (e) { err.textContent = String(e); out.textContent = ""; }); + } + + function fillActions() { + var svc = $("gd-service").value, sel = $("gd-action"); + sel.innerHTML = ""; + var names = Object.keys(examples[svc] || {}); + var known = (actions[svc] || []).map(function (a) { return a.action; }); + known.forEach(function (n) { if (names.indexOf(n) < 0) names.push(n); }); + names.forEach(function (n) { + var o = document.createElement("option"); + o.value = n; + var meta = (actions[svc] || []).filter(function (a) { return a.action === n; })[0]; + o.textContent = n + (meta && meta.confirm ? " (confirm)" : ""); + sel.appendChild(o); + }); + fillBody(); + } + + function fillBody() { + var svc = $("gd-service").value, action = $("gd-action").value; + var ex = (examples[svc] || {})[action] || {}; + var b = {}; + if (svc !== "call" && svc !== "storage/sign") b.action = action; + Object.keys(ex).forEach(function (k) { b[k] = ex[k]; }); + $("gd-body").value = JSON.stringify(b, null, 1); + } + + fetch("/google/data/status").then(function (r) { return r.json(); }).then(function (s) { + actions = s.actions || {}; + $("gd-config").textContent = s.configured + ? "Google credential found." + (s.project_set ? "" : " Set GOOGLE_CLOUD_PROJECT.") + : "No Google credential: calls answer 503 until one is configured."; + fillActions(); + }); + + $("gd-service").onchange = fillActions; + $("gd-action").onchange = fillBody; + $("gd-send").onclick = function () { + var body; + try { body = JSON.parse($("gd-body").value || "{}"); } + catch (e) { err.textContent = "Body is not valid JSON"; return; } + post("/google/data/" + $("gd-service").value, body); + }; + fillActions(); +})(); diff --git a/src/google/data/SPEC.md b/src/google/data/SPEC.md new file mode 100644 index 0000000..04250c5 --- /dev/null +++ b/src/google/data/SPEC.md @@ -0,0 +1,186 @@ +# Google data package - Specification + +Owned directory: `src/google/data/`. Namespace `molo::google::data`. Depends on +the `src/google` foundation (`GoogleClient`, `Service`, `CloudContext`, +`list_all`), `molo::clients` and wolfCrypt. Plan: `plans/18-breadth.md`. + +## 1. Scope + +Eleven Google data APIs, every method of each one reachable through one +Discovery-generated call table (918 rows), the common ones through typed +classes: + +| API (Discovery) | Class | Typed calls | +|---|---|---| +| BigQuery v2 | `BigQuery` | query, dry run, results, jobs (list, get, insert, cancel), datasets, tables, rows, streaming insert (`tabledata.insertAll`), models, routines | +| Cloud Storage JSON v1 | `Storage`, `StorageMedia`, `SignedUrl` | buckets, objects, compose, copy, patch, delete; download and range download; simple and resumable upload; V4 signed URLs | +| Pub/Sub v1 | `PubSub` | topics, subscriptions, snapshots, publish, pull, acknowledge | +| Firestore v1 | `Firestore` | databases, get, list, collection ids, run query, aggregation, batch get, create, commit, delete | +| Spanner v1 | `Spanner` | instances, databases, DDL, sessions, read-only SQL | +| Bigtable Admin v2 | `BigtableAdmin` | instances, clusters, tables, app profiles, table IAM (admin plane only; row access is gRPC) | +| Secret Manager v1 | `SecretManager` | secrets, versions, access, create, add version, disable, destroy, delete | +| Dataplex v1 | `Dataplex` | locations, lakes, entry groups, entry and aspect types, data scans, entry search and lookup | +| DLP v2 | `Dlp` | inspect, de-identify, infoTypes, templates, triggers, jobs | +| Cloud KMS v1 | `Kms` | locations, key rings, keys, versions, public key, encrypt, decrypt, asymmetric sign | +| Datastore v1 | `Datastore` | lookup, query, aggregation, transactions, commit, allocate and reserve ids | + +Anything else (for example `bigquery.rowAccessPolicies.*`, `dataplex.*.glossaries.*`) +is `ApiService::invoke("", args)` with the same validation. `DataPlatform` +holds all typed services over one client and project. + +Not here: Bigtable data plane (gRPC), Spanner partitioned/streaming reads, BigQuery +Storage Read/Write API (gRPC), Pub/Sub streaming pull, Firestore listen. They have no +REST surface in Discovery. + +## 2. Module map + +| File | Concern | +|---|---| +| `caps.h` | Static ceilings (`DataCaps`) | +| `calls.h/.cpp`, `calls_table.inc` | `Catalog`: the ONE table of REST calls (generated), api info, endpoints | +| `ids.h` | `Ids`: project, location, dataset, bucket, object, pubsub and generic id rules | +| `api_service.h/.cpp` | `ApiService : Service`, `CallArgs`, `Prepared`: validate, render, send, page | +| `project_service.h` | `ProjectService`: context bound base with name and paging helpers | +| `bigquery`, `storage`, `pubsub`, `firestore`, `spanner`, `bigtable_admin`, `secret_manager`, `dataplex`, `dlp`, `kms`, `datastore` | one typed class per API | +| `storage_media.h/.cpp` | `StorageMedia`: raw bytes (GoogleClient is JSON only) over its own `clients::Client` | +| `signed_url.h/.cpp` | `SignedUrl`: GOOG4-RSA-SHA256 signing with the service-account key, no network | +| `data.h` | `DataPlatform` facade | +| `tools/gen_calls.py` | regenerates `calls_table.inc` from Discovery | +| `tools/check_discovery.py` | conformance check against Discovery | + +## 3. Static caps (`caps.h`) + +| Knob | Value | +|---|---| +| path value / resource name | 1024 chars | +| query parameters per call / value chars / array items | 40 / 2048 / 100 | +| JSON request body | 10 MiB | +| list page size / page walk | 1000 / 20 pages | +| BigQuery sql / result rows / query wait | 1 MiB / 10000 / 60000 ms | +| BigQuery insertAll rows / bytes | 10000 / 10 MiB | +| Storage download / simple upload | 16 MiB / 8 MiB | +| Storage resumable chunk / total / granularity | 8 MiB / 64 MiB / 256 KiB | +| Storage compose sources | 32 | +| signed URL lifetime | 1 s to 604800 s (7 days, the V4 maximum) | +| signed headers / extra query | 20 / 20 | +| Pub/Sub publish messages / bytes / pull / ack ids | 1000 / 10 MiB / 100 / 1000 | +| Secret payload / KMS plaintext | 64 KiB / 64 KiB | +| DLP content item | 512 KiB | +| Datastore keys per lookup / writes per commit | 1000 / 500 | +| example route body / response shown / upload / download shown | 3 MiB / 256 KiB / 2 MiB / 64 KiB | + +## 4. Contracts + +- Every URL is `endpoint base + the table row's path`; no path literal elsewhere. + `{x}` placeholders are one URL-encoded segment (object names with `/` and spaces + are encoded); `{+x}` placeholders are resource names (`[A-Za-z0-9._~:@=,$()*/-]`, + no empty, `.` or `..` segment). +- `prepare()` rejects before any traffic (GoogleError 400 INVALID_ARGUMENT): a call + of another API, unknown or missing path value, unknown query parameter, missing + required query parameter, body on a call without request message, body field not + in the request schema, control characters, caps. Then `invoke()` needs a + credential (`NotConfiguredError`, 503). +- Base URL overrides (regional hosts) must be `https://*.googleapis.com`. +- Destructive and write calls are explicit methods without defaults. The example + routes run them only with `"confirm":true`; reading a secret payload, publishing, + pulling (leases messages), billed BigQuery queries, DLP and KMS operations count as + gated too. A BigQuery `query` is billed at most 1 GB unless `maximum_bytes_billed` + says otherwise. +- `StorageMedia` sends the credential only to `https://storage.googleapis.com`; a + resumable session URL from anywhere else is rejected. Over-cap downloads fail + loudly, never truncate. A chunk that does not end the object must be a multiple of + 256 KiB. +- `Spanner::query` always deletes the session it created. +- Signed URL signing needs an RSA service-account key; the lifetime is capped at 7 + days. A bad key is rejected loudly. Anyone holding a URL can use it until it expires. + +## 5. Example routes (`/google/data`) + +GET `/google/data` (page), `/status` (credential state, APIs with call counts, actions), +`/calls[?api=name]` (the table). POST `/bigquery`, `/storage`, `/pubsub`, `/firestore`, +`/spanner`, `/bigtable`, `/secrets`, `/dataplex`, `/dlp`, `/kms`, `/datastore` +(`{"action":...}`; `/status` lists the actions and which need confirm), `/storage/sign` +(signed URL, dry run unless confirm), `/call` (any row by rpc id; non-GET rows are dry +runs unless confirm). Bad input 422, no credential 503 `google_not_configured`, body +over 3 MiB 413. Excluded from locust. + +## Required APIs + +Enable these on the project (placeholder project `example-214512124`; full list and a one-line `gcloud` command in `src/google/APIS.md`). A disabled API answers 403 `SERVICE_DISABLED`. + +- `bigquery.googleapis.com`: https://console.developers.google.com/apis/api/bigquery.googleapis.com/overview?project=example-214512124 +- `storage.googleapis.com`: https://console.developers.google.com/apis/api/storage.googleapis.com/overview?project=example-214512124 +- `pubsub.googleapis.com`: https://console.developers.google.com/apis/api/pubsub.googleapis.com/overview?project=example-214512124 +- `firestore.googleapis.com`: https://console.developers.google.com/apis/api/firestore.googleapis.com/overview?project=example-214512124 +- `spanner.googleapis.com`: https://console.developers.google.com/apis/api/spanner.googleapis.com/overview?project=example-214512124 +- `bigtableadmin.googleapis.com`: https://console.developers.google.com/apis/api/bigtableadmin.googleapis.com/overview?project=example-214512124 +- `secretmanager.googleapis.com`: https://console.developers.google.com/apis/api/secretmanager.googleapis.com/overview?project=example-214512124 +- `dataplex.googleapis.com`: https://console.developers.google.com/apis/api/dataplex.googleapis.com/overview?project=example-214512124 +- `dlp.googleapis.com`: https://console.developers.google.com/apis/api/dlp.googleapis.com/overview?project=example-214512124 +- `cloudkms.googleapis.com`: https://console.developers.google.com/apis/api/cloudkms.googleapis.com/overview?project=example-214512124 +- `datastore.googleapis.com`: https://console.developers.google.com/apis/api/datastore.googleapis.com/overview?project=example-214512124 + +Scope: all calls use `cloud-platform` (the service-account default). Signed URLs need +only the key file. + +## Sources + +- Discovery documents, fetched 2026-10-25: `https://.googleapis.com/$discovery/rest?version=` + for bigquery v2 (rev 20260922), storage v1 (20261020), pubsub v1 (20261013), firestore v1 + (20261013), spanner v1 (20261006), bigtableadmin v2 (20261017), secretmanager v1 + (20260925), dataplex v1 (20261013), dlp v2 (20261020), cloudkms v1 (20261016), datastore v1 + (20261013). +- Cloud Storage "Signatures V4" and "Resumable uploads" documentation (canonical request, + string to sign, session protocol). + +## Verified against discovery + +`python3 -I src/google/data/tools/check_discovery.py`, run 2026-10-25: +`discovery ok: 918 rows, 120 call ids used by sources, 36 nested schemas, storage media contract`. +Checks per row: HTTP method, path, request body fields (exactly the request schema's +top level), query parameters, required query parameters, path placeholders; total +coverage (every Discovery method of the 11 APIs is a row and vice versa); every call id +named in the C++ sources is a row; nested schema fields the typed classes write; the +Storage media upload and download contract. + +## Verified live + +Project `lisa-448004`, service account, 2026-10-25, read only (`make google-live`): + +- BigQuery: dry-run `SELECT 1` (200, totalBytesProcessed), datasets.list, jobs.list, + projects.getServiceAccount: 200. +- Storage: buckets.list 200 (the project has no bucket), projects.serviceAccount.get 200. + Download and signed URL against the Google-owned public object + `gcp-public-data-landsat/index.csv.gz`: ranged download of 100 bytes (bearer token) 200; + V4 signed HEAD URL 200; the same URL with one changed signature character 403. +- Secret Manager secrets.list 200. Dataplex locations.list and searchEntries 200. +- Datastore runQuery: 404 NOT_FOUND (no Datastore database in the project; route and auth + accepted). +- API NOT ENABLED on the project (403, "has not been used ... or it is disabled"): Pub/Sub, + Firestore, Spanner, Bigtable Admin, DLP, Cloud KMS. Their typed calls are verified by + Discovery only. + +## Unverified + +- Every write (create, insert, publish, upload, compose, delete, commit, addVersion): + Discovery only; the live rules forbid writes. +- Simple and resumable upload request shapes and the resumable protocol (308 handling, + `Content-Range`). The resumable session is started on the documented + `/upload/storage/v1/b/{bucket}/o?uploadType=resumable`; Discovery lists + `/resumable/upload/storage/v1/b/{bucket}/o` for the same protocol (both hosts accepted + as session URLs). Not exercised against Google. +- Response shapes: typed methods return Google's JSON unchanged. +- Spanner `query` (session create, `executeSql`, session delete), DLP inspect and + de-identify, KMS encrypt, decrypt and sign, Pub/Sub publish and pull, Firestore queries: + APIs not enabled or no resources; request bodies checked against Discovery schemas only. +- Regional hosts (DLP, Spanner regional endpoints): override accepted, never used live. + +## Gaps + +- `GoogleClient` is JSON only, so `StorageMedia` owns a second `clients::Client`; a raw + `request_bytes` in the foundation would remove it. +- The google build links `Base64.cpp` only, so `SignedUrl` hashes with wolfCrypt instead + of `security/crypto/Sha256`. +- `api_catalog.*` and `APIS.md` do not list the 11 services yet (owner adds them, see the + Required APIs list). +- Streaming (Pub/Sub streamingPull, Firestore listen, BigQuery Storage API) is gRPC only. diff --git a/src/google/data/api_service.cpp b/src/google/data/api_service.cpp new file mode 100644 index 0000000..5a861c6 --- /dev/null +++ b/src/google/data/api_service.cpp @@ -0,0 +1,197 @@ +#include "google/data/api_service.h" + +#include + +#include "clients/url.h" +#include "google/data/caps.h" +#include "google/errors.h" +#include "google/operation.h" + +namespace molo::google::data { + +using clients::body::Json; + +CallArgs& CallArgs::p(const std::string& name, const std::string& value) { + path[name] = value; + return *this; +} +CallArgs& CallArgs::q(const std::string& name, Json value) { + query.set(name, std::move(value)); + return *this; +} +CallArgs& CallArgs::with_body(Json b) { + body = std::move(b); + return *this; +} + +namespace { + +bool control(char c) { return static_cast(c) < 0x20 || c == 0x7f; } + +clients::Method method_of(const char* m) { + if (!std::strcmp(m, "GET")) return clients::Method::kGet; + if (!std::strcmp(m, "POST")) return clients::Method::kPost; + if (!std::strcmp(m, "PUT")) return clients::Method::kPut; + if (!std::strcmp(m, "PATCH")) return clients::Method::kPatch; + if (!std::strcmp(m, "DELETE")) return clients::Method::kDelete; + throw GoogleError::invalid("unsupported http method in table"); +} + +} // namespace + +void ApiService::check_segment(const std::string& name, const std::string& v) { + if (v.empty() || v.size() > DataCaps::kMaxPathValueChars) + throw GoogleError::invalid(name + " must be 1.." + + std::to_string(DataCaps::kMaxPathValueChars) + " characters"); + if (v == "." || v == "..") throw GoogleError::invalid(name + " must not be . or .."); + for (char c : v) + if (control(c)) throw GoogleError::invalid(name + " has control characters"); +} + +void ApiService::check_resource(const std::string& name, const std::string& v) { + if (v.empty() || v.size() > DataCaps::kMaxResourceNameChars) + throw GoogleError::invalid(name + " must be 1.." + + std::to_string(DataCaps::kMaxResourceNameChars) + " characters"); + if (v.front() == '/' || v.back() == '/' || v.find("//") != std::string::npos || + v.find("/../") != std::string::npos || v.rfind("../", 0) == 0 || + (v.size() >= 3 && v.compare(v.size() - 3, 3, "/..") == 0) || v == "..") + throw GoogleError::invalid(name + " is not a well formed resource name"); + for (char c : v) { + bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + std::strchr("._~:@=,$()*/-", c) != nullptr; + if (!ok) throw GoogleError::invalid(name + " has characters outside A-Za-z0-9._~:@=,$()*/-"); + } +} + +std::string ApiService::encode_query_value(const std::string& name, const Json& v) { + auto one = [&](const Json& x) -> std::string { + if (x.is_string()) { + if (x.as_string().size() > DataCaps::kMaxQueryValueChars) + throw GoogleError::invalid("query " + name + " too long"); + for (char c : x.as_string()) + if (control(c)) throw GoogleError::invalid("query " + name + " has control characters"); + return clients::Url::encode_component(x.as_string()); + } + if (x.is_bool()) return x.as_bool() ? "true" : "false"; + if (x.is_number()) { + double d = x.as_number(); + if (d == static_cast(static_cast(d))) + return std::to_string(static_cast(d)); + return std::to_string(d); + } + throw GoogleError::invalid("query " + name + " must be a string, number or boolean"); + }; + return one(v); +} + +Prepared ApiService::prepare(const std::string& rpc, const CallArgs& args) const { + const CallSpec& spec = Catalog::get(rpc); + if (Catalog::api_of(spec) != api_) + throw GoogleError::invalid("call " + rpc + " belongs to another api"); + Prepared out; + out.method = method_of(spec.http); + out.endpoint = endpoint_; + + // Path. + std::string path = spec.path; + auto names = Catalog::path_params(spec); + for (const auto& kv : args.path) { + bool known = false; + for (const auto& n : names) known = known || n == kv.first; + if (!known) throw GoogleError::invalid("unknown path value: " + kv.first.substr(0, 64)); + } + std::string rendered; + for (std::size_t i = 0; path[i];) { + if (path[i] != '{') { + rendered.push_back(path[i++]); + continue; + } + std::size_t close = path.find('}', i); + std::string key = path.substr(i + 1, close - i - 1); + bool plus = !key.empty() && key[0] == '+'; + if (plus) key.erase(0, 1); + auto it = args.path.find(key); + if (it == args.path.end()) throw GoogleError::invalid("missing path value: " + key); + if (plus) { + check_resource(key, it->second); + rendered += it->second; + } else { + check_segment(key, it->second); + rendered += clients::Url::encode_component(it->second); + } + i = close + 1; + } + out.path = "/" + rendered; + + // Query. + std::string qs; + std::size_t count = 0; + if (!args.query.is_object()) throw GoogleError::invalid("query must be an object"); + for (const auto& kv : args.query.as_object()) { + if (kv.second.is_null()) continue; + if (!Catalog::in_list(spec.query, kv.first)) + throw GoogleError::invalid("unknown query parameter for " + rpc + ": " + + kv.first.substr(0, 64)); + std::vector values; + if (kv.second.is_array()) { + if (kv.second.as_array().size() > DataCaps::kMaxQueryArrayItems) + throw GoogleError::invalid("query " + kv.first + " has too many items"); + for (const Json& x : kv.second.as_array()) values.push_back(&x); + } else { + values.push_back(&kv.second); + } + for (const Json* x : values) { + if (++count > DataCaps::kMaxQueryParams) + throw GoogleError::invalid("too many query parameters"); + qs += qs.empty() ? "?" : "&"; + qs += kv.first + "=" + encode_query_value(kv.first, *x); + } + } + for (const auto& r : Catalog::split(spec.required)) { + const Json* v = args.query.find(r); + if (!v || v->is_null()) throw GoogleError::invalid("missing required query parameter " + r); + } + out.path += qs; + + // Body. + bool body_allowed = spec.body[0] != '\0'; + if (!args.body.is_null()) { + if (!body_allowed) throw GoogleError::invalid(rpc + " takes no request body"); + if (!args.body.is_object()) throw GoogleError::invalid("body must be a JSON object"); + for (const auto& kv : args.body.as_object()) + if (!Catalog::in_list(spec.body, kv.first)) + throw GoogleError::invalid("unknown body field for " + rpc + ": " + + kv.first.substr(0, 64)); + if (args.body.dump().size() > DataCaps::kMaxBodyBytes) + throw GoogleError::invalid("request body too large"); + out.body = args.body; + out.has_body = true; + } else if (body_allowed && (out.method == clients::Method::kPost || + out.method == clients::Method::kPut || + out.method == clients::Method::kPatch)) { + out.body = Json::object(); // POST with an empty request message + out.has_body = true; + } + return out; +} + +ApiService::ApiService(std::shared_ptr client, Api api, + const std::string& base_url_override) + : Service(std::move(client), Catalog::endpoint(api, base_url_override)), api_(api) {} + +Json ApiService::invoke(const std::string& rpc, const CallArgs& args) const { + Prepared p = prepare(rpc, args); + return client_->request(p.method, p.endpoint, p.path, p.has_body ? &p.body : nullptr); +} + +Json ApiService::invoke_paged(const std::string& rpc, const CallArgs& args, + const std::string& items_key, int max_pages) const { + if (max_pages < 1 || max_pages > DataCaps::kMaxListPages) + throw GoogleError::invalid("max_pages must be 1.." + std::to_string(DataCaps::kMaxListPages)); + Prepared p = prepare(rpc, args); + if (p.method != clients::Method::kGet) + throw GoogleError::invalid("paged calls must be GET"); + return list_all(*client_, p.endpoint, p.path, items_key, static_cast(max_pages)); +} + +} // namespace molo::google::data diff --git a/src/google/data/api_service.h b/src/google/data/api_service.h new file mode 100644 index 0000000..6e84ae6 --- /dev/null +++ b/src/google/data/api_service.h @@ -0,0 +1,67 @@ +#pragma once + +// ApiService: the generic, table-driven call engine every typed service of the +// data package derives from. A call is an rpc id from the Catalog plus +// CallArgs; path values, query parameters and body fields are validated +// against the table row before any network traffic. + +#include +#include +#include + +#include "clients/body/json.h" +#include "clients/request.h" +#include "google/data/calls.h" +#include "google/service.h" + +namespace molo::google::data { + +// Arguments of one call. +struct CallArgs { + std::map path; // {x} and {+x} values + clients::body::Json query = clients::body::Json::object(); // string|number|bool|array + clients::body::Json body; // null or object + + CallArgs& p(const std::string& name, const std::string& value); + CallArgs& q(const std::string& name, clients::body::Json value); + CallArgs& with_body(clients::body::Json b); +}; + +// A validated, rendered request (no network yet). +struct Prepared { + clients::Method method = clients::Method::kGet; + ServiceEndpoint endpoint; + std::string path; // "/v1/projects/..." plus "?query" + clients::body::Json body; + bool has_body = false; + std::string url() const { return endpoint.base_url + path; } +}; + +class ApiService : public Service { + public: + ApiService(std::shared_ptr client, Api api, + const std::string& base_url_override = ""); + + Api api() const { return api_; } + + // Validation + rendering only. Throws GoogleError 400 INVALID_ARGUMENT. + Prepared prepare(const std::string& rpc, const CallArgs& args = {}) const; + // prepare + send. NotConfiguredError when no credential. + clients::body::Json invoke(const std::string& rpc, const CallArgs& args = {}) const; + // Follows nextPageToken (pageSize is the caller's query arg) and collects the + // `items_key` arrays. Page walk capped. + clients::body::Json invoke_paged(const std::string& rpc, const CallArgs& args, + const std::string& items_key, + int max_pages = 20) const; + + // Value checks shared by the engine and the typed services. + static void check_segment(const std::string& name, const std::string& value); + static void check_resource(const std::string& name, const std::string& value); + static std::string encode_query_value(const std::string& name, + const clients::body::Json& value); + + private: + Api api_; +}; + +} // namespace molo::google::data diff --git a/src/google/data/bigquery.cpp b/src/google/data/bigquery.cpp new file mode 100644 index 0000000..8d3796b --- /dev/null +++ b/src/google/data/bigquery.cpp @@ -0,0 +1,257 @@ +#include "google/data/bigquery.h" + +#include "google/data/caps.h" +#include "google/data/ids.h" + +namespace molo::google::data { + +using clients::body::Json; + +void QueryRequest::validate() const { + if (sql.empty() || sql.size() > DataCaps::kMaxSqlChars) + throw GoogleError::invalid("sql must be 1.." + std::to_string(DataCaps::kMaxSqlChars) + + " characters"); + if (max_results < 1 || max_results > DataCaps::kMaxResultRows) + throw GoogleError::invalid("max_results must be 1.." + + std::to_string(DataCaps::kMaxResultRows)); + if (timeout_ms < 0 || timeout_ms > DataCaps::kMaxQueryTimeoutMs) + throw GoogleError::invalid("timeout_ms must be 0.." + + std::to_string(DataCaps::kMaxQueryTimeoutMs)); + if (!location.empty()) Ids::location(location); + if (!default_dataset.empty()) Ids::dataset(default_dataset); + if (!maximum_bytes_billed.empty()) { + if (maximum_bytes_billed.size() > 18) + throw GoogleError::invalid("maximum_bytes_billed must be a decimal string"); + for (char c : maximum_bytes_billed) + if (c < '0' || c > '9') throw GoogleError::invalid("maximum_bytes_billed must be digits"); + } + if (!query_parameters.is_null()) { + if (!query_parameters.is_array()) throw GoogleError::invalid("query_parameters must be an array"); + if (parameter_mode != "NAMED" && parameter_mode != "POSITIONAL") + throw GoogleError::invalid("parameter_mode must be NAMED or POSITIONAL"); + } +} + +Json QueryRequest::to_json(const std::string& project) const { + validate(); + Json b = Json::object(); + b.set("query", Json(sql)); + b.set("useLegacySql", Json(use_legacy_sql)); + b.set("useQueryCache", Json(use_query_cache)); + b.set("dryRun", Json(dry_run)); + b.set("maxResults", Json(max_results)); + b.set("timeoutMs", Json(timeout_ms)); + if (!location.empty()) b.set("location", Json(location)); + if (!default_dataset.empty()) { + Json d = Json::object(); + d.set("projectId", Json(project)); + d.set("datasetId", Json(default_dataset)); + b.set("defaultDataset", d); + } + if (!maximum_bytes_billed.empty()) b.set("maximumBytesBilled", Json(maximum_bytes_billed)); + if (!query_parameters.is_null()) { + b.set("queryParameters", query_parameters); + b.set("parameterMode", Json(parameter_mode)); + } + return b; +} + +BigQuery::BigQuery(std::shared_ptr client, CloudContext context) + : ApiService(std::move(client), Api::kBigquery), context_(std::move(context)) {} + +CallArgs BigQuery::base() const { + CallArgs a; + a.p("projectId", Ids::project(context_.project)); + return a; +} + +CallArgs BigQuery::in_dataset(const std::string& dataset) const { + CallArgs a = base(); + a.p("datasetId", Ids::dataset(dataset)); + return a; +} + +Json BigQuery::page(int max_results) { + if (max_results < 1 || max_results > DataCaps::kMaxPageSize) + throw GoogleError::invalid("max_results must be 1.." + std::to_string(DataCaps::kMaxPageSize)); + return Json(max_results); +} + +Json BigQuery::query(const QueryRequest& request) const { + CallArgs a = base(); + a.with_body(request.to_json(context_.project)); + return invoke("bigquery.jobs.query", a); +} + +Json BigQuery::dry_run(const std::string& sql) const { + QueryRequest r; + r.sql = sql; + r.dry_run = true; + return query(r); +} + +Json BigQuery::query_results(const std::string& job_id, const std::string& location, + int max_results, const std::string& page_token) const { + CallArgs a = base(); + a.p("jobId", Ids::bq_id(job_id, "job id")); + a.q("maxResults", page(max_results)); + if (!location.empty()) a.q("location", Json(Ids::location(location))); + if (!page_token.empty()) a.q("pageToken", Json(page_token)); + return invoke("bigquery.jobs.getQueryResults", a); +} + +Json BigQuery::list_jobs(int max_results, const std::string& state_filter) const { + CallArgs a = base(); + a.q("maxResults", page(max_results)); + if (!state_filter.empty()) { + if (state_filter != "done" && state_filter != "pending" && state_filter != "running") + throw GoogleError::invalid("state_filter must be done, pending or running"); + a.q("stateFilter", Json(state_filter)); + } + return invoke("bigquery.jobs.list", a); +} + +Json BigQuery::get_job(const std::string& job_id, const std::string& location) const { + CallArgs a = base(); + a.p("jobId", Ids::bq_id(job_id, "job id")); + if (!location.empty()) a.q("location", Json(Ids::location(location))); + return invoke("bigquery.jobs.get", a); +} + +Json BigQuery::insert_job(const Json& job) const { + CallArgs a = base(); + a.with_body(job); + return invoke("bigquery.jobs.insert", a); +} + +Json BigQuery::cancel_job(const std::string& job_id, const std::string& location) const { + CallArgs a = base(); + a.p("jobId", Ids::bq_id(job_id, "job id")); + if (!location.empty()) a.q("location", Json(Ids::location(location))); + return invoke("bigquery.jobs.cancel", a); +} + +Json BigQuery::list_datasets(int max_results) const { + CallArgs a = base(); + a.q("maxResults", page(max_results)); + return invoke("bigquery.datasets.list", a); +} + +Json BigQuery::get_dataset(const std::string& dataset) const { + return invoke("bigquery.datasets.get", in_dataset(dataset)); +} + +Json BigQuery::create_dataset(const std::string& dataset, const std::string& location, + const std::string& description) const { + CallArgs a = base(); + Json ref = Json::object(); + ref.set("projectId", Json(Ids::project(context_.project))); + ref.set("datasetId", Json(Ids::dataset(dataset))); + Json b = Json::object(); + b.set("datasetReference", ref); + if (!location.empty()) b.set("location", Json(Ids::location(location))); + if (description.size() > 1024) throw GoogleError::invalid("description over 1024 characters"); + if (!description.empty()) b.set("description", Json(description)); + a.with_body(b); + return invoke("bigquery.datasets.insert", a); +} + +Json BigQuery::delete_dataset(const std::string& dataset, bool delete_contents) const { + CallArgs a = in_dataset(dataset); + a.q("deleteContents", Json(delete_contents)); + return invoke("bigquery.datasets.delete", a); +} + +Json BigQuery::list_tables(const std::string& dataset, int max_results) const { + CallArgs a = in_dataset(dataset); + a.q("maxResults", page(max_results)); + return invoke("bigquery.tables.list", a); +} + +Json BigQuery::get_table(const std::string& dataset, const std::string& table) const { + CallArgs a = in_dataset(dataset); + a.p("tableId", Ids::bq_id(table)); + return invoke("bigquery.tables.get", a); +} + +Json BigQuery::create_table(const std::string& dataset, const Json& table) const { + CallArgs a = in_dataset(dataset); + a.with_body(table); + return invoke("bigquery.tables.insert", a); +} + +Json BigQuery::delete_table(const std::string& dataset, const std::string& table) const { + CallArgs a = in_dataset(dataset); + a.p("tableId", Ids::bq_id(table)); + return invoke("bigquery.tables.delete", a); +} + +Json BigQuery::list_rows(const std::string& dataset, const std::string& table, int max_results, + const std::string& page_token) const { + CallArgs a = in_dataset(dataset); + a.p("tableId", Ids::bq_id(table)); + a.q("maxResults", page(max_results)); + if (!page_token.empty()) a.q("pageToken", Json(page_token)); + return invoke("bigquery.tabledata.list", a); +} + +Json BigQuery::insert_all(const std::string& dataset, const std::string& table, const Json& rows, + bool skip_invalid_rows, bool ignore_unknown_values) const { + if (!rows.is_array() || rows.as_array().empty() || + rows.as_array().size() > DataCaps::kMaxInsertRows) + throw GoogleError::invalid("rows must be an array of 1.." + + std::to_string(DataCaps::kMaxInsertRows) + " items"); + for (const Json& r : rows.as_array()) + if (!r.is_object() || !r.find("json") || !r.find("json")->is_object()) + throw GoogleError::invalid("every row must be an object with a json object"); + if (rows.dump().size() > DataCaps::kMaxInsertBytes) + throw GoogleError::invalid("rows over the insert size cap"); + CallArgs a = in_dataset(dataset); + a.p("tableId", Ids::bq_id(table)); + Json b = Json::object(); + b.set("rows", rows); + b.set("skipInvalidRows", Json(skip_invalid_rows)); + b.set("ignoreUnknownValues", Json(ignore_unknown_values)); + a.with_body(b); + return invoke("bigquery.tabledata.insertAll", a); +} + +Json BigQuery::list_models(const std::string& dataset, int max_results) const { + CallArgs a = in_dataset(dataset); + a.q("maxResults", page(max_results)); + return invoke("bigquery.models.list", a); +} + +Json BigQuery::get_model(const std::string& dataset, const std::string& model) const { + CallArgs a = in_dataset(dataset); + a.p("modelId", Ids::bq_id(model, "model")); + return invoke("bigquery.models.get", a); +} + +Json BigQuery::delete_model(const std::string& dataset, const std::string& model) const { + CallArgs a = in_dataset(dataset); + a.p("modelId", Ids::bq_id(model, "model")); + return invoke("bigquery.models.delete", a); +} + +Json BigQuery::list_routines(const std::string& dataset, int max_results) const { + CallArgs a = in_dataset(dataset); + a.q("maxResults", page(max_results)); + return invoke("bigquery.routines.list", a); +} + +Json BigQuery::get_routine(const std::string& dataset, const std::string& routine) const { + CallArgs a = in_dataset(dataset); + a.p("routineId", Ids::bq_id(routine, "routine")); + return invoke("bigquery.routines.get", a); +} + +Json BigQuery::delete_routine(const std::string& dataset, const std::string& routine) const { + CallArgs a = in_dataset(dataset); + a.p("routineId", Ids::bq_id(routine, "routine")); + return invoke("bigquery.routines.delete", a); +} + +Json BigQuery::service_account() const { return invoke("bigquery.projects.getServiceAccount", base()); } + +} // namespace molo::google::data diff --git a/src/google/data/bigquery.h b/src/google/data/bigquery.h new file mode 100644 index 0000000..645ca4c --- /dev/null +++ b/src/google/data/bigquery.h @@ -0,0 +1,85 @@ +#pragma once + +// BigQuery REST API v2: jobs, queries, datasets, tables, routines, models, +// streaming insert (tabledata.insertAll). Every other method of the API is +// reachable with ApiService::invoke("bigquery...."). + +#include + +#include "google/data/api_service.h" +#include "google/resource.h" + +namespace molo::google::data { + +struct QueryRequest { + std::string sql; + bool dry_run = false; // validate and estimate bytes, run nothing + bool use_legacy_sql = false; + bool use_query_cache = true; + int max_results = 100; // rows in the first response, 1..10000 + int timeout_ms = 10000; // wait for completion, 0..60000 + std::string location; // processing location, "" = Google decides + std::string default_dataset; // dataset id inside the client project + std::string maximum_bytes_billed; // decimal string, "" = no limit set + clients::body::Json query_parameters; // array of QueryParameter, or null + std::string parameter_mode; // "NAMED" or "POSITIONAL" with parameters + + void validate() const; + clients::body::Json to_json(const std::string& project) const; +}; + +class BigQuery : public ApiService { + public: + explicit BigQuery(std::shared_ptr client, CloudContext context); + + const CloudContext& context() const { return context_; } + + // jobs.query. dry_run=true is free and returns totalBytesProcessed. + clients::body::Json query(const QueryRequest& request) const; + clients::body::Json dry_run(const std::string& sql) const; + clients::body::Json query_results(const std::string& job_id, const std::string& location, + int max_results, const std::string& page_token) const; + + clients::body::Json list_jobs(int max_results, const std::string& state_filter) const; + clients::body::Json get_job(const std::string& job_id, const std::string& location) const; + clients::body::Json insert_job(const clients::body::Json& job) const; + clients::body::Json cancel_job(const std::string& job_id, const std::string& location) const; + + clients::body::Json list_datasets(int max_results) const; + clients::body::Json get_dataset(const std::string& dataset) const; + clients::body::Json create_dataset(const std::string& dataset, const std::string& location, + const std::string& description) const; + // Destructive, no defaults. + clients::body::Json delete_dataset(const std::string& dataset, bool delete_contents) const; + + clients::body::Json list_tables(const std::string& dataset, int max_results) const; + clients::body::Json get_table(const std::string& dataset, const std::string& table) const; + clients::body::Json create_table(const std::string& dataset, + const clients::body::Json& table) const; + clients::body::Json delete_table(const std::string& dataset, const std::string& table) const; + clients::body::Json list_rows(const std::string& dataset, const std::string& table, + int max_results, const std::string& page_token) const; + + // tabledata.insertAll: rows is an array of {"insertId"?, "json":{...}}. + clients::body::Json insert_all(const std::string& dataset, const std::string& table, + const clients::body::Json& rows, bool skip_invalid_rows, + bool ignore_unknown_values) const; + + clients::body::Json list_models(const std::string& dataset, int max_results) const; + clients::body::Json get_model(const std::string& dataset, const std::string& model) const; + clients::body::Json delete_model(const std::string& dataset, const std::string& model) const; + clients::body::Json list_routines(const std::string& dataset, int max_results) const; + clients::body::Json get_routine(const std::string& dataset, const std::string& routine) const; + clients::body::Json delete_routine(const std::string& dataset, + const std::string& routine) const; + clients::body::Json service_account() const; + + private: + CallArgs base() const; + CallArgs in_dataset(const std::string& dataset) const; + static clients::body::Json page(int max_results); + + CloudContext context_; +}; + +} // namespace molo::google::data diff --git a/src/google/data/bigtable_admin.cpp b/src/google/data/bigtable_admin.cpp new file mode 100644 index 0000000..12490d3 --- /dev/null +++ b/src/google/data/bigtable_admin.cpp @@ -0,0 +1,46 @@ +#include "google/data/bigtable_admin.h" + +namespace molo::google::data { + +using clients::body::Json; + +std::string BigtableAdmin::instance_name(const std::string& i) const { + return project_path() + "/instances/" + Ids::id(i, "instance"); +} +std::string BigtableAdmin::table_name(const std::string& i, const std::string& t) const { + return instance_name(i) + "/tables/" + Ids::id(t, "table"); +} + +Json BigtableAdmin::list_instances() const { + return invoke("bigtableadmin.projects.instances.list", on("parent", project_path())); +} +Json BigtableAdmin::get_instance(const std::string& i) const { + return invoke("bigtableadmin.projects.instances.get", on("name", instance_name(i))); +} +Json BigtableAdmin::list_clusters(const std::string& i) const { + return invoke("bigtableadmin.projects.instances.clusters.list", on("parent", instance_name(i))); +} +Json BigtableAdmin::get_cluster(const std::string& i, const std::string& c) const { + return invoke("bigtableadmin.projects.instances.clusters.get", + on("name", instance_name(i) + "/clusters/" + Ids::id(c, "cluster"))); +} +Json BigtableAdmin::list_tables(const std::string& i, int n) const { + CallArgs a = on("parent", instance_name(i)); + a.q("pageSize", page(n)); + return invoke("bigtableadmin.projects.instances.tables.list", a); +} +Json BigtableAdmin::get_table(const std::string& i, const std::string& t) const { + return invoke("bigtableadmin.projects.instances.tables.get", on("name", table_name(i, t))); +} +Json BigtableAdmin::list_app_profiles(const std::string& i, int n) const { + CallArgs a = on("parent", instance_name(i)); + a.q("pageSize", page(n)); + return invoke("bigtableadmin.projects.instances.appProfiles.list", a); +} +Json BigtableAdmin::table_iam_policy(const std::string& i, const std::string& t) const { + CallArgs a = on("resource", table_name(i, t)); + a.with_body(Json::object()); + return invoke("bigtableadmin.projects.instances.tables.getIamPolicy", a); +} + +} // namespace molo::google::data diff --git a/src/google/data/bigtable_admin.h b/src/google/data/bigtable_admin.h new file mode 100644 index 0000000..efeaf59 --- /dev/null +++ b/src/google/data/bigtable_admin.h @@ -0,0 +1,32 @@ +#pragma once + +// Cloud Bigtable Admin API v2: instances, clusters, tables, app profiles +// (admin plane only; row reads and writes are the Bigtable data API, which is +// gRPC and not part of this package). Other methods: +// ApiService::invoke("bigtableadmin...."). + +#include + +#include "google/data/project_service.h" + +namespace molo::google::data { + +class BigtableAdmin : public ProjectService { + public: + explicit BigtableAdmin(std::shared_ptr client, CloudContext context) + : ProjectService(std::move(client), Api::kBigtableAdmin, std::move(context)) {} + + std::string instance_name(const std::string& instance) const; + std::string table_name(const std::string& instance, const std::string& table) const; + + clients::body::Json list_instances() const; + clients::body::Json get_instance(const std::string& instance) const; + clients::body::Json list_clusters(const std::string& instance) const; + clients::body::Json get_cluster(const std::string& instance, const std::string& cluster) const; + clients::body::Json list_tables(const std::string& instance, int page_size) const; + clients::body::Json get_table(const std::string& instance, const std::string& table) const; + clients::body::Json list_app_profiles(const std::string& instance, int page_size) const; + clients::body::Json table_iam_policy(const std::string& instance, const std::string& table) const; +}; + +} // namespace molo::google::data diff --git a/src/google/data/calls.cpp b/src/google/data/calls.cpp new file mode 100644 index 0000000..efcffd1 --- /dev/null +++ b/src/google/data/calls.cpp @@ -0,0 +1,154 @@ +#include "google/data/calls.h" + +#include +#include + +#include "google/errors.h" + +namespace molo::google::data { + +namespace { + +const ApiInfo kApis[] = { + {Api::kBigquery, "bigquery", "v2", "https://bigquery.googleapis.com/bigquery/v2", + "bigquery.googleapis.com"}, + {Api::kStorage, "storage", "v1", "https://storage.googleapis.com/storage/v1", + "storage.googleapis.com"}, + {Api::kPubsub, "pubsub", "v1", "https://pubsub.googleapis.com", "pubsub.googleapis.com"}, + {Api::kFirestore, "firestore", "v1", "https://firestore.googleapis.com", + "firestore.googleapis.com"}, + {Api::kSpanner, "spanner", "v1", "https://spanner.googleapis.com", "spanner.googleapis.com"}, + {Api::kBigtableAdmin, "bigtableadmin", "v2", "https://bigtableadmin.googleapis.com", + "bigtableadmin.googleapis.com"}, + {Api::kSecretManager, "secretmanager", "v1", "https://secretmanager.googleapis.com", + "secretmanager.googleapis.com"}, + {Api::kDataplex, "dataplex", "v1", "https://dataplex.googleapis.com", + "dataplex.googleapis.com"}, + {Api::kDlp, "dlp", "v2", "https://dlp.googleapis.com", "dlp.googleapis.com"}, + {Api::kKms, "cloudkms", "v1", "https://cloudkms.googleapis.com", "cloudkms.googleapis.com"}, + {Api::kDatastore, "datastore", "v1", "https://datastore.googleapis.com", + "datastore.googleapis.com"}, +}; + +// CALLS-BEGIN (parsed by tools/check_discovery.py; one row per line) +const CallSpec kCalls[] = { +#include "google/data/calls_table.inc" +}; +// CALLS-END + +const std::unordered_map& index() { + static const auto map = [] { + std::unordered_map m; + for (const CallSpec& s : kCalls) m.emplace(s.rpc, &s); + return m; + }(); + return map; +} + +} // namespace + +std::size_t Catalog::size() { return sizeof(kCalls) / sizeof(kCalls[0]); } + +const CallSpec& Catalog::at(std::size_t i) { + if (i >= size()) throw GoogleError::invalid("call index out of range"); + return kCalls[i]; +} + +const CallSpec* Catalog::find(const std::string& rpc) { + auto it = index().find(rpc); + return it == index().end() ? nullptr : it->second; +} + +const CallSpec& Catalog::get(const std::string& rpc) { + const CallSpec* s = find(rpc); + if (!s) throw GoogleError::invalid("unknown call: " + rpc.substr(0, 128)); + return *s; +} + +const ApiInfo& Catalog::info(Api api) { + for (const ApiInfo& i : kApis) + if (i.api == api) return i; + throw GoogleError::invalid("unknown api"); +} + +std::size_t Catalog::api_count() { return sizeof(kApis) / sizeof(kApis[0]); } + +const ApiInfo& Catalog::info_at(std::size_t i) { + if (i >= api_count()) throw GoogleError::invalid("api index out of range"); + return kApis[i]; +} + +Api Catalog::api_of(const CallSpec& spec) { + const char* dot = std::strchr(spec.rpc, '.'); + std::size_t n = dot ? static_cast(dot - spec.rpc) : std::strlen(spec.rpc); + for (const ApiInfo& i : kApis) + if (std::strlen(i.name) == n && std::strncmp(i.name, spec.rpc, n) == 0) return i.api; + throw GoogleError::invalid("call has no known api prefix"); +} + +std::size_t Catalog::count_of(Api api) { + std::size_t n = 0; + for (const CallSpec& s : kCalls) + if (api_of(s) == api) ++n; + return n; +} + +ServiceEndpoint Catalog::endpoint(Api api, const std::string& base_url_override) { + const ApiInfo& i = info(api); + ServiceEndpoint e; + e.name = i.name; + if (base_url_override.empty()) { + e.base_url = i.base_url; + return e; + } + if (base_url_override.rfind("https://", 0) != 0 || base_url_override.size() > 256 || + base_url_override.back() == '/') + throw GoogleError::invalid("base url override must be https:// without trailing slash"); + // Only Google hosts: a typo must not send a bearer token elsewhere. + auto host_end = base_url_override.find('/', 8); + std::string host = base_url_override.substr( + 8, host_end == std::string::npos ? std::string::npos : host_end - 8); + const std::string tail = ".googleapis.com"; + if (host.size() <= tail.size() || + host.compare(host.size() - tail.size(), tail.size(), tail) != 0) + throw GoogleError::invalid("base url override must be on googleapis.com"); + e.base_url = base_url_override; + return e; +} + +std::vector Catalog::split(const char* list) { + std::vector out; + std::string cur; + for (const char* p = list; *p; ++p) { + if (*p == ',') { + if (!cur.empty()) out.push_back(cur); + cur.clear(); + } else { + cur.push_back(*p); + } + } + if (!cur.empty()) out.push_back(cur); + return out; +} + +bool Catalog::in_list(const char* list, const std::string& item) { + for (const std::string& s : split(list)) + if (s == item) return true; + return false; +} + +std::vector Catalog::path_params(const CallSpec& spec) { + std::vector out; + const char* p = spec.path; + while ((p = std::strchr(p, '{')) != nullptr) { + const char* e = std::strchr(p, '}'); + if (!e) break; + std::string name(p + 1, e); + if (!name.empty() && name[0] == '+') name.erase(0, 1); + out.push_back(name); + p = e + 1; + } + return out; +} + +} // namespace molo::google::data diff --git a/src/google/data/calls.h b/src/google/data/calls.h new file mode 100644 index 0000000..962a555 --- /dev/null +++ b/src/google/data/calls.h @@ -0,0 +1,64 @@ +#pragma once + +// The ONE table of REST calls of the data package. Every row is generated from +// Google's Discovery documents (tools/gen_calls.py) and checked by +// tools/check_discovery.py. Code builds every URL from this table, never from +// a literal path. + +#include +#include +#include + +#include "google/endpoints.h" + +namespace molo::google::data { + +enum class Api { + kBigquery, kStorage, kPubsub, kFirestore, kSpanner, kBigtableAdmin, + kSecretManager, kDataplex, kDlp, kKms, kDatastore, +}; + +struct ApiInfo { + Api api; + const char* name; // Discovery name and rpc id prefix: "bigquery" + const char* version; // "v2" + const char* base_url; // rootUrl + servicePath, no trailing slash + const char* service; // service to enable: "bigquery.googleapis.com" +}; + +struct CallSpec { + const char* rpc; // discovery method id, e.g. "bigquery.jobs.query" + const char* http; // GET POST PUT PATCH DELETE + const char* path; // discovery "path" relative to base_url + const char* body; // comma list: every top-level field of the request schema + const char* query; // comma list: every query parameter of the method + const char* required; // comma list: required query parameters +}; + +class Catalog { + public: + static std::size_t size(); + static const CallSpec& at(std::size_t index); + // nullptr when absent. + static const CallSpec* find(const std::string& rpc); + // Throws GoogleError (invalid) when absent. + static const CallSpec& get(const std::string& rpc); + + static const ApiInfo& info(Api api); + static std::size_t api_count(); + static const ApiInfo& info_at(std::size_t index); + static Api api_of(const CallSpec& spec); // from the rpc id prefix + static std::size_t count_of(Api api); + + // base_url_override replaces the discovery base (regional hosts); it must + // be https on googleapis.com. + static ServiceEndpoint endpoint(Api api, const std::string& base_url_override = ""); + + // Comma list helpers. + static std::vector split(const char* list); + static bool in_list(const char* list, const std::string& item); + // Names of the {x} and {+x} placeholders of a path, in order, without "+". + static std::vector path_params(const CallSpec& spec); +}; + +} // namespace molo::google::data diff --git a/src/google/data/calls_table.inc b/src/google/data/calls_table.inc new file mode 100644 index 0000000..1e44afc --- /dev/null +++ b/src/google/data/calls_table.inc @@ -0,0 +1,930 @@ +// GENERATED by tools/gen_calls.py from Google Discovery; do not edit by hand. +// bigquery v2 revision 20261009 + {"bigquery.datasets.delete", "DELETE", "projects/{+projectId}/datasets/{+datasetId}", "", "deleteContents", ""}, + {"bigquery.datasets.get", "GET", "projects/{+projectId}/datasets/{+datasetId}", "", "accessPolicyVersion,datasetView", ""}, + {"bigquery.datasets.insert", "POST", "projects/{+projectId}/datasets", "access,catalogSource,creationTime,datasetReference,defaultCollation,defaultEncryptionConfiguration,defaultPartitionExpirationMs,defaultRoundingMode,defaultTableExpirationMs,description,etag,externalCatalogDatasetOptions,externalDatasetReference,friendlyName,id,isCaseInsensitive,kind,labels,lastModifiedTime,linkedDatasetMetadata,linkedDatasetSource,location,maxTimeTravelHours,resourceTags,restrictions,satisfiesPzi,satisfiesPzs,selfLink,storageBillingModel,tags,type", "accessPolicyVersion", ""}, + {"bigquery.datasets.list", "GET", "projects/{+projectId}/datasets", "", "all,filter,maxResults,pageToken", ""}, + {"bigquery.datasets.patch", "PATCH", "projects/{+projectId}/datasets/{+datasetId}", "access,catalogSource,creationTime,datasetReference,defaultCollation,defaultEncryptionConfiguration,defaultPartitionExpirationMs,defaultRoundingMode,defaultTableExpirationMs,description,etag,externalCatalogDatasetOptions,externalDatasetReference,friendlyName,id,isCaseInsensitive,kind,labels,lastModifiedTime,linkedDatasetMetadata,linkedDatasetSource,location,maxTimeTravelHours,resourceTags,restrictions,satisfiesPzi,satisfiesPzs,selfLink,storageBillingModel,tags,type", "accessPolicyVersion,updateMode", ""}, + {"bigquery.datasets.undelete", "POST", "projects/{+projectId}/datasets/{+datasetId}:undelete", "deletionTime", "", ""}, + {"bigquery.datasets.update", "PUT", "projects/{+projectId}/datasets/{+datasetId}", "access,catalogSource,creationTime,datasetReference,defaultCollation,defaultEncryptionConfiguration,defaultPartitionExpirationMs,defaultRoundingMode,defaultTableExpirationMs,description,etag,externalCatalogDatasetOptions,externalDatasetReference,friendlyName,id,isCaseInsensitive,kind,labels,lastModifiedTime,linkedDatasetMetadata,linkedDatasetSource,location,maxTimeTravelHours,resourceTags,restrictions,satisfiesPzi,satisfiesPzs,selfLink,storageBillingModel,tags,type", "accessPolicyVersion,updateMode", ""}, + {"bigquery.jobs.cancel", "POST", "projects/{+projectId}/jobs/{+jobId}/cancel", "", "location", ""}, + {"bigquery.jobs.delete", "DELETE", "projects/{+projectId}/jobs/{+jobId}/delete", "", "location", ""}, + {"bigquery.jobs.get", "GET", "projects/{+projectId}/jobs/{+jobId}", "", "location", ""}, + {"bigquery.jobs.getQueryResults", "GET", "projects/{+projectId}/queries/{+jobId}", "", "formatOptions.timestampOutputFormat,formatOptions.useInt64Timestamp,location,maxResults,pageToken,startIndex,timeoutMs", ""}, + {"bigquery.jobs.insert", "POST", "projects/{+projectId}/jobs", "configuration,etag,id,jobCreationReason,jobReference,kind,principal_subject,selfLink,statistics,status,user_email", "", ""}, + {"bigquery.jobs.list", "GET", "projects/{+projectId}/jobs", "", "allUsers,maxCreationTime,maxResults,minCreationTime,pageToken,parentJobId,projection,stateFilter", ""}, + {"bigquery.jobs.query", "POST", "projects/{+projectId}/queries", "arrowSerializationOptions,connectionProperties,continuous,createSession,defaultDataset,destinationEncryptionConfiguration,dryRun,formatOptions,jobCreationMode,jobTimeoutMs,kind,labels,location,maxResults,maxSlots,maximumBytesBilled,parameterMode,preserveNulls,query,queryParameters,queryResultsFormat,requestId,reservation,secureContext,timeoutMs,useLegacySql,useQueryCache,writeIncrementalResults", "", ""}, + {"bigquery.models.delete", "DELETE", "projects/{+projectId}/datasets/{+datasetId}/models/{+modelId}", "", "", ""}, + {"bigquery.models.get", "GET", "projects/{+projectId}/datasets/{+datasetId}/models/{+modelId}", "", "", ""}, + {"bigquery.models.list", "GET", "projects/{+projectId}/datasets/{+datasetId}/models", "", "maxResults,pageToken", ""}, + {"bigquery.models.patch", "PATCH", "projects/{+projectId}/datasets/{+datasetId}/models/{+modelId}", "bestTrialId,creationTime,defaultTrialId,description,encryptionConfiguration,etag,expirationTime,featureColumns,friendlyName,hparamSearchSpaces,hparamTrials,labelColumns,labels,lastModifiedTime,location,modelReference,modelType,optimalTrialIds,remoteModelInfo,trainingRuns,transformColumns", "", ""}, + {"bigquery.projects.getServiceAccount", "GET", "projects/{+projectId}/serviceAccount", "", "", ""}, + {"bigquery.projects.list", "GET", "projects", "", "maxResults,pageToken", ""}, + {"bigquery.routines.delete", "DELETE", "projects/{+projectId}/datasets/{+datasetId}/routines/{+routineId}", "", "", ""}, + {"bigquery.routines.get", "GET", "projects/{+projectId}/datasets/{+datasetId}/routines/{+routineId}", "", "readMask", ""}, + {"bigquery.routines.getIamPolicy", "POST", "{+resource}:getIamPolicy", "options", "", ""}, + {"bigquery.routines.insert", "POST", "projects/{+projectId}/datasets/{+datasetId}/routines", "arguments,buildStatus,creationTime,dataGovernanceType,definitionBody,description,determinismLevel,etag,externalRuntimeOptions,importedLibraries,language,lastModifiedTime,pythonOptions,remoteFunctionOptions,returnTableType,returnType,routineReference,routineType,securityMode,sparkOptions,strictMode", "", ""}, + {"bigquery.routines.list", "GET", "projects/{+projectId}/datasets/{+datasetId}/routines", "", "filter,maxResults,pageToken,readMask", ""}, + {"bigquery.routines.setIamPolicy", "POST", "{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"bigquery.routines.testIamPermissions", "POST", "{+resource}:testIamPermissions", "permissions", "", ""}, + {"bigquery.routines.update", "PUT", "projects/{+projectId}/datasets/{+datasetId}/routines/{+routineId}", "arguments,buildStatus,creationTime,dataGovernanceType,definitionBody,description,determinismLevel,etag,externalRuntimeOptions,importedLibraries,language,lastModifiedTime,pythonOptions,remoteFunctionOptions,returnTableType,returnType,routineReference,routineType,securityMode,sparkOptions,strictMode", "", ""}, + {"bigquery.rowAccessPolicies.batchDelete", "POST", "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}/rowAccessPolicies:batchDelete", "force,policyIds", "", ""}, + {"bigquery.rowAccessPolicies.delete", "DELETE", "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}/rowAccessPolicies/{+policyId}", "", "force", ""}, + {"bigquery.rowAccessPolicies.get", "GET", "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}/rowAccessPolicies/{+policyId}", "", "", ""}, + {"bigquery.rowAccessPolicies.getIamPolicy", "POST", "{+resource}:getIamPolicy", "options", "", ""}, + {"bigquery.rowAccessPolicies.insert", "POST", "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}/rowAccessPolicies", "creationTime,etag,filterPredicate,grantees,lastModifiedTime,rowAccessPolicyReference", "", ""}, + {"bigquery.rowAccessPolicies.list", "GET", "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}/rowAccessPolicies", "", "pageSize,pageToken", ""}, + {"bigquery.rowAccessPolicies.testIamPermissions", "POST", "{+resource}:testIamPermissions", "permissions", "", ""}, + {"bigquery.rowAccessPolicies.update", "PUT", "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}/rowAccessPolicies/{+policyId}", "creationTime,etag,filterPredicate,grantees,lastModifiedTime,rowAccessPolicyReference", "", ""}, + {"bigquery.tabledata.insertAll", "POST", "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}/insertAll", "ignoreUnknownValues,kind,rows,skipInvalidRows,templateSuffix,traceId", "", ""}, + {"bigquery.tabledata.list", "GET", "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}/data", "", "formatOptions.timestampOutputFormat,formatOptions.useInt64Timestamp,maxResults,pageToken,selectedFields,startIndex", ""}, + {"bigquery.tables.delete", "DELETE", "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}", "", "", ""}, + {"bigquery.tables.get", "GET", "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}", "", "selectedFields,view", ""}, + {"bigquery.tables.getIamPolicy", "POST", "{+resource}:getIamPolicy", "options", "", ""}, + {"bigquery.tables.insert", "POST", "projects/{+projectId}/datasets/{+datasetId}/tables", "biglakeConfiguration,cloneDefinition,clustering,creationTime,defaultCollation,defaultRoundingMode,description,encryptionConfiguration,etag,expirationTime,externalCatalogTableOptions,externalDataConfiguration,friendlyName,id,kind,labels,lastModifiedTime,location,managedTableType,materializedView,materializedViewStatus,maxStaleness,model,numActiveLogicalBytes,numActivePhysicalBytes,numBytes,numCurrentPhysicalBytes,numLongTermBytes,numLongTermLogicalBytes,numLongTermPhysicalBytes,numPartitions,numPhysicalBytes,numRows,numTimeTravelPhysicalBytes,numTotalLogicalBytes,numTotalPhysicalBytes,partitionDefinition,rangePartitioning,replicas,requirePartitionFilter,resourceTags,restrictions,schema,selfLink,snapshotDefinition,streamingBuffer,tableConstraints,tableReference,tableReplicationInfo,timePartitioning,type,view", "", ""}, + {"bigquery.tables.list", "GET", "projects/{+projectId}/datasets/{+datasetId}/tables", "", "maxResults,pageToken", ""}, + {"bigquery.tables.patch", "PATCH", "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}", "biglakeConfiguration,cloneDefinition,clustering,creationTime,defaultCollation,defaultRoundingMode,description,encryptionConfiguration,etag,expirationTime,externalCatalogTableOptions,externalDataConfiguration,friendlyName,id,kind,labels,lastModifiedTime,location,managedTableType,materializedView,materializedViewStatus,maxStaleness,model,numActiveLogicalBytes,numActivePhysicalBytes,numBytes,numCurrentPhysicalBytes,numLongTermBytes,numLongTermLogicalBytes,numLongTermPhysicalBytes,numPartitions,numPhysicalBytes,numRows,numTimeTravelPhysicalBytes,numTotalLogicalBytes,numTotalPhysicalBytes,partitionDefinition,rangePartitioning,replicas,requirePartitionFilter,resourceTags,restrictions,schema,selfLink,snapshotDefinition,streamingBuffer,tableConstraints,tableReference,tableReplicationInfo,timePartitioning,type,view", "autodetect_schema", ""}, + {"bigquery.tables.setIamPolicy", "POST", "{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"bigquery.tables.testIamPermissions", "POST", "{+resource}:testIamPermissions", "permissions", "", ""}, + {"bigquery.tables.update", "PUT", "projects/{+projectId}/datasets/{+datasetId}/tables/{+tableId}", "biglakeConfiguration,cloneDefinition,clustering,creationTime,defaultCollation,defaultRoundingMode,description,encryptionConfiguration,etag,expirationTime,externalCatalogTableOptions,externalDataConfiguration,friendlyName,id,kind,labels,lastModifiedTime,location,managedTableType,materializedView,materializedViewStatus,maxStaleness,model,numActiveLogicalBytes,numActivePhysicalBytes,numBytes,numCurrentPhysicalBytes,numLongTermBytes,numLongTermLogicalBytes,numLongTermPhysicalBytes,numPartitions,numPhysicalBytes,numRows,numTimeTravelPhysicalBytes,numTotalLogicalBytes,numTotalPhysicalBytes,partitionDefinition,rangePartitioning,replicas,requirePartitionFilter,resourceTags,restrictions,schema,selfLink,snapshotDefinition,streamingBuffer,tableConstraints,tableReference,tableReplicationInfo,timePartitioning,type,view", "autodetect_schema", ""}, +// storage v1 revision 20261020 + {"storage.anywhereCaches.disable", "POST", "b/{bucket}/anywhereCaches/{anywhereCacheId}/disable", "", "", ""}, + {"storage.anywhereCaches.get", "GET", "b/{bucket}/anywhereCaches/{anywhereCacheId}", "", "", ""}, + {"storage.anywhereCaches.insert", "POST", "b/{bucket}/anywhereCaches", "admissionPolicy,anywhereCacheId,bucket,createTime,id,ingestOnWrite,kind,pendingUpdate,selfLink,state,ttl,updateTime,zone", "", ""}, + {"storage.anywhereCaches.list", "GET", "b/{bucket}/anywhereCaches", "", "pageSize,pageToken", ""}, + {"storage.anywhereCaches.pause", "POST", "b/{bucket}/anywhereCaches/{anywhereCacheId}/pause", "", "", ""}, + {"storage.anywhereCaches.resume", "POST", "b/{bucket}/anywhereCaches/{anywhereCacheId}/resume", "", "", ""}, + {"storage.anywhereCaches.update", "PATCH", "b/{bucket}/anywhereCaches/{anywhereCacheId}", "admissionPolicy,anywhereCacheId,bucket,createTime,id,ingestOnWrite,kind,pendingUpdate,selfLink,state,ttl,updateTime,zone", "", ""}, + {"storage.bucketAccessControls.delete", "DELETE", "b/{bucket}/acl/{entity}", "", "userProject", ""}, + {"storage.bucketAccessControls.get", "GET", "b/{bucket}/acl/{entity}", "", "userProject", ""}, + {"storage.bucketAccessControls.insert", "POST", "b/{bucket}/acl", "bucket,domain,email,entity,entityId,etag,id,kind,projectTeam,role,selfLink", "userProject", ""}, + {"storage.bucketAccessControls.list", "GET", "b/{bucket}/acl", "", "userProject", ""}, + {"storage.bucketAccessControls.patch", "PATCH", "b/{bucket}/acl/{entity}", "bucket,domain,email,entity,entityId,etag,id,kind,projectTeam,role,selfLink", "userProject", ""}, + {"storage.bucketAccessControls.update", "PUT", "b/{bucket}/acl/{entity}", "bucket,domain,email,entity,entityId,etag,id,kind,projectTeam,role,selfLink", "userProject", ""}, + {"storage.buckets.delete", "DELETE", "b/{bucket}", "", "ifMetagenerationMatch,ifMetagenerationNotMatch,userProject", ""}, + {"storage.buckets.get", "GET", "b/{bucket}", "", "generation,ifMetagenerationMatch,ifMetagenerationNotMatch,projection,softDeleted,userProject", ""}, + {"storage.buckets.getIamPolicy", "GET", "b/{bucket}/iam", "", "optionsRequestedPolicyVersion,userProject", ""}, + {"storage.buckets.getStorageLayout", "GET", "b/{bucket}/storageLayout", "", "prefix", ""}, + {"storage.buckets.insert", "POST", "b", "acl,autoclass,billing,cors,customPlacementConfig,defaultEventBasedHold,defaultObjectAcl,encryption,etag,generation,hardDeleteTime,hierarchicalNamespace,iamConfiguration,id,ipFilter,kind,labels,lifecycle,location,locationType,logging,metageneration,name,objectRetention,owner,projectNumber,retentionPolicy,rpo,satisfiesPZI,satisfiesPZS,selfLink,softDeletePolicy,softDeleteTime,storageClass,timeCreated,updated,versioning,website", "enableObjectRetention,predefinedAcl,predefinedDefaultObjectAcl,project,projection,userProject", "project"}, + {"storage.buckets.list", "GET", "b", "", "maxResults,pageToken,prefix,project,projection,returnPartialSuccess,softDeleted,userProject", "project"}, + {"storage.buckets.lockRetentionPolicy", "POST", "b/{bucket}/lockRetentionPolicy", "", "ifMetagenerationMatch,userProject", "ifMetagenerationMatch"}, + {"storage.buckets.operations.advanceRelocateBucket", "POST", "b/{bucket}/operations/{operationId}/advanceRelocateBucket", "expireTime,ttl", "", ""}, + {"storage.buckets.operations.cancel", "POST", "b/{bucket}/operations/{operationId}/cancel", "", "", ""}, + {"storage.buckets.operations.get", "GET", "b/{bucket}/operations/{operationId}", "", "", ""}, + {"storage.buckets.operations.list", "GET", "b/{bucket}/operations", "", "filter,pageSize,pageToken", ""}, + {"storage.buckets.patch", "PATCH", "b/{bucket}", "acl,autoclass,billing,cors,customPlacementConfig,defaultEventBasedHold,defaultObjectAcl,encryption,etag,generation,hardDeleteTime,hierarchicalNamespace,iamConfiguration,id,ipFilter,kind,labels,lifecycle,location,locationType,logging,metageneration,name,objectRetention,owner,projectNumber,retentionPolicy,rpo,satisfiesPZI,satisfiesPZS,selfLink,softDeletePolicy,softDeleteTime,storageClass,timeCreated,updated,versioning,website", "ifMetagenerationMatch,ifMetagenerationNotMatch,predefinedAcl,predefinedDefaultObjectAcl,projection,userProject", ""}, + {"storage.buckets.relocate", "POST", "b/{bucket}/relocate", "destinationCustomPlacementConfig,destinationKmsKeyName,destinationLocation,validateOnly", "", ""}, + {"storage.buckets.restore", "POST", "b/{bucket}/restore", "", "generation,projection,userProject", "generation"}, + {"storage.buckets.setIamPolicy", "PUT", "b/{bucket}/iam", "bindings,etag,kind,resourceId,version", "userProject", ""}, + {"storage.buckets.testIamPermissions", "GET", "b/{bucket}/iam/testPermissions", "", "permissions,userProject", "permissions"}, + {"storage.buckets.update", "PUT", "b/{bucket}", "acl,autoclass,billing,cors,customPlacementConfig,defaultEventBasedHold,defaultObjectAcl,encryption,etag,generation,hardDeleteTime,hierarchicalNamespace,iamConfiguration,id,ipFilter,kind,labels,lifecycle,location,locationType,logging,metageneration,name,objectRetention,owner,projectNumber,retentionPolicy,rpo,satisfiesPZI,satisfiesPZS,selfLink,softDeletePolicy,softDeleteTime,storageClass,timeCreated,updated,versioning,website", "ifMetagenerationMatch,ifMetagenerationNotMatch,predefinedAcl,predefinedDefaultObjectAcl,projection,userProject", ""}, + {"storage.defaultObjectAccessControls.delete", "DELETE", "b/{bucket}/defaultObjectAcl/{entity}", "", "userProject", ""}, + {"storage.defaultObjectAccessControls.get", "GET", "b/{bucket}/defaultObjectAcl/{entity}", "", "userProject", ""}, + {"storage.defaultObjectAccessControls.insert", "POST", "b/{bucket}/defaultObjectAcl", "bucket,domain,email,entity,entityId,etag,generation,id,kind,object,projectTeam,role,selfLink", "userProject", ""}, + {"storage.defaultObjectAccessControls.list", "GET", "b/{bucket}/defaultObjectAcl", "", "ifMetagenerationMatch,ifMetagenerationNotMatch,userProject", ""}, + {"storage.defaultObjectAccessControls.patch", "PATCH", "b/{bucket}/defaultObjectAcl/{entity}", "bucket,domain,email,entity,entityId,etag,generation,id,kind,object,projectTeam,role,selfLink", "userProject", ""}, + {"storage.defaultObjectAccessControls.update", "PUT", "b/{bucket}/defaultObjectAcl/{entity}", "bucket,domain,email,entity,entityId,etag,generation,id,kind,object,projectTeam,role,selfLink", "userProject", ""}, + {"storage.folders.delete", "DELETE", "b/{bucket}/folders/{folder}", "", "ifMetagenerationMatch,ifMetagenerationNotMatch", ""}, + {"storage.folders.deleteRecursive", "POST", "b/{bucket}/folders/{folder}/deleteRecursive", "", "ifMetagenerationMatch,ifMetagenerationNotMatch", ""}, + {"storage.folders.get", "GET", "b/{bucket}/folders/{folder}", "", "ifMetagenerationMatch,ifMetagenerationNotMatch", ""}, + {"storage.folders.insert", "POST", "b/{bucket}/folders", "bucket,createTime,id,kind,metageneration,name,pendingRenameInfo,selfLink,updateTime", "recursive", ""}, + {"storage.folders.list", "GET", "b/{bucket}/folders", "", "delimiter,endOffset,pageSize,pageToken,prefix,startOffset", ""}, + {"storage.folders.rename", "POST", "b/{bucket}/folders/{sourceFolder}/renameTo/folders/{destinationFolder}", "", "ifSourceMetagenerationMatch,ifSourceMetagenerationNotMatch", ""}, + {"storage.managedFolders.delete", "DELETE", "b/{bucket}/managedFolders/{managedFolder}", "", "allowNonEmpty,ifMetagenerationMatch,ifMetagenerationNotMatch", ""}, + {"storage.managedFolders.get", "GET", "b/{bucket}/managedFolders/{managedFolder}", "", "ifMetagenerationMatch,ifMetagenerationNotMatch", ""}, + {"storage.managedFolders.getIamPolicy", "GET", "b/{bucket}/managedFolders/{managedFolder}/iam", "", "optionsRequestedPolicyVersion,userProject", ""}, + {"storage.managedFolders.insert", "POST", "b/{bucket}/managedFolders", "bucket,createTime,id,kind,metageneration,name,rapidCacheConfig,selfLink,updateTime", "", ""}, + {"storage.managedFolders.list", "GET", "b/{bucket}/managedFolders", "", "pageSize,pageToken,prefix", ""}, + {"storage.managedFolders.setIamPolicy", "PUT", "b/{bucket}/managedFolders/{managedFolder}/iam", "bindings,etag,kind,resourceId,version", "userProject", ""}, + {"storage.managedFolders.testIamPermissions", "GET", "b/{bucket}/managedFolders/{managedFolder}/iam/testPermissions", "", "permissions,userProject", "permissions"}, + {"storage.managedFolders.update", "PATCH", "b/{bucket}/managedFolders/{managedFolder}", "bucket,createTime,id,kind,metageneration,name,rapidCacheConfig,selfLink,updateTime", "ifMetagenerationMatch,ifMetagenerationNotMatch", ""}, + {"storage.notifications.delete", "DELETE", "b/{bucket}/notificationConfigs/{notification}", "", "userProject", ""}, + {"storage.notifications.get", "GET", "b/{bucket}/notificationConfigs/{notification}", "", "userProject", ""}, + {"storage.notifications.insert", "POST", "b/{bucket}/notificationConfigs", "custom_attributes,etag,event_types,id,kind,object_name_prefix,payload_format,selfLink,topic", "userProject", ""}, + {"storage.notifications.list", "GET", "b/{bucket}/notificationConfigs", "", "userProject", ""}, + {"storage.objectAccessControls.delete", "DELETE", "b/{bucket}/o/{object}/acl/{entity}", "", "generation,userProject", ""}, + {"storage.objectAccessControls.get", "GET", "b/{bucket}/o/{object}/acl/{entity}", "", "generation,userProject", ""}, + {"storage.objectAccessControls.insert", "POST", "b/{bucket}/o/{object}/acl", "bucket,domain,email,entity,entityId,etag,generation,id,kind,object,projectTeam,role,selfLink", "generation,userProject", ""}, + {"storage.objectAccessControls.list", "GET", "b/{bucket}/o/{object}/acl", "", "generation,userProject", ""}, + {"storage.objectAccessControls.patch", "PATCH", "b/{bucket}/o/{object}/acl/{entity}", "bucket,domain,email,entity,entityId,etag,generation,id,kind,object,projectTeam,role,selfLink", "generation,userProject", ""}, + {"storage.objectAccessControls.update", "PUT", "b/{bucket}/o/{object}/acl/{entity}", "bucket,domain,email,entity,entityId,etag,generation,id,kind,object,projectTeam,role,selfLink", "generation,userProject", ""}, + {"storage.objects.bulkRestore", "POST", "b/{bucket}/o/bulkRestore", "allowOverwrite,copySourceAcl,createdAfterTime,createdBeforeTime,matchGlobs,softDeletedAfterTime,softDeletedBeforeTime", "", ""}, + {"storage.objects.compose", "POST", "b/{destinationBucket}/o/{destinationObject}/compose", "deleteSourceObjects,destination,kind,sourceObjects", "destinationPredefinedAcl,dropContextGroups,ifGenerationMatch,ifMetagenerationMatch,kmsKeyName,userProject", ""}, + {"storage.objects.copy", "POST", "b/{sourceBucket}/o/{sourceObject}/copyTo/b/{destinationBucket}/o/{destinationObject}", "acl,bucket,cacheControl,componentCount,contentDisposition,contentEncoding,contentLanguage,contentType,contexts,crc32c,customTime,customerEncryption,etag,eventBasedHold,generation,hardDeleteTime,id,kind,kmsKeyName,md5Hash,mediaLink,metadata,metageneration,name,owner,restoreToken,retention,retentionExpirationTime,selfLink,size,softDeleteTime,storageClass,temporaryHold,timeCreated,timeDeleted,timeFinalized,timeStorageClassUpdated,updated", "destinationKmsKeyName,destinationPredefinedAcl,ifGenerationMatch,ifGenerationNotMatch,ifMetagenerationMatch,ifMetagenerationNotMatch,ifSourceGenerationMatch,ifSourceGenerationNotMatch,ifSourceMetagenerationMatch,ifSourceMetagenerationNotMatch,projection,sourceGeneration,userProject", ""}, + {"storage.objects.delete", "DELETE", "b/{bucket}/o/{object}", "", "generation,ifGenerationMatch,ifGenerationNotMatch,ifMetagenerationMatch,ifMetagenerationNotMatch,userProject", ""}, + {"storage.objects.get", "GET", "b/{bucket}/o/{object}", "", "generation,ifGenerationMatch,ifGenerationNotMatch,ifMetagenerationMatch,ifMetagenerationNotMatch,projection,restoreToken,softDeleted,userProject", ""}, + {"storage.objects.getIamPolicy", "GET", "b/{bucket}/o/{object}/iam", "", "generation,userProject", ""}, + {"storage.objects.insert", "POST", "b/{bucket}/o", "acl,bucket,cacheControl,componentCount,contentDisposition,contentEncoding,contentLanguage,contentType,contexts,crc32c,customTime,customerEncryption,etag,eventBasedHold,generation,hardDeleteTime,id,kind,kmsKeyName,md5Hash,mediaLink,metadata,metageneration,name,owner,restoreToken,retention,retentionExpirationTime,selfLink,size,softDeleteTime,storageClass,temporaryHold,timeCreated,timeDeleted,timeFinalized,timeStorageClassUpdated,updated", "contentEncoding,ifGenerationMatch,ifGenerationNotMatch,ifMetagenerationMatch,ifMetagenerationNotMatch,kmsKeyName,name,predefinedAcl,projection,userProject", ""}, + {"storage.objects.list", "GET", "b/{bucket}/o", "", "delimiter,endOffset,filter,includeFoldersAsPrefixes,includeTrailingDelimiter,matchGlob,maxResults,pageToken,prefix,projection,softDeleted,startOffset,userProject,versions", ""}, + {"storage.objects.move", "POST", "b/{bucket}/o/{sourceObject}/moveTo/o/{destinationObject}", "", "ifGenerationMatch,ifGenerationNotMatch,ifMetagenerationMatch,ifMetagenerationNotMatch,ifSourceGenerationMatch,ifSourceGenerationNotMatch,ifSourceMetagenerationMatch,ifSourceMetagenerationNotMatch,projection,userProject", ""}, + {"storage.objects.patch", "PATCH", "b/{bucket}/o/{object}", "acl,bucket,cacheControl,componentCount,contentDisposition,contentEncoding,contentLanguage,contentType,contexts,crc32c,customTime,customerEncryption,etag,eventBasedHold,generation,hardDeleteTime,id,kind,kmsKeyName,md5Hash,mediaLink,metadata,metageneration,name,owner,restoreToken,retention,retentionExpirationTime,selfLink,size,softDeleteTime,storageClass,temporaryHold,timeCreated,timeDeleted,timeFinalized,timeStorageClassUpdated,updated", "generation,ifGenerationMatch,ifGenerationNotMatch,ifMetagenerationMatch,ifMetagenerationNotMatch,overrideUnlockedRetention,predefinedAcl,projection,userProject", ""}, + {"storage.objects.restore", "POST", "b/{bucket}/o/{object}/restore", "", "copySourceAcl,generation,ifGenerationMatch,ifGenerationNotMatch,ifMetagenerationMatch,ifMetagenerationNotMatch,projection,restoreToken,userProject", "generation"}, + {"storage.objects.rewrite", "POST", "b/{sourceBucket}/o/{sourceObject}/rewriteTo/b/{destinationBucket}/o/{destinationObject}", "acl,bucket,cacheControl,componentCount,contentDisposition,contentEncoding,contentLanguage,contentType,contexts,crc32c,customTime,customerEncryption,etag,eventBasedHold,generation,hardDeleteTime,id,kind,kmsKeyName,md5Hash,mediaLink,metadata,metageneration,name,owner,restoreToken,retention,retentionExpirationTime,selfLink,size,softDeleteTime,storageClass,temporaryHold,timeCreated,timeDeleted,timeFinalized,timeStorageClassUpdated,updated", "destinationKmsKeyName,destinationPredefinedAcl,dropContextGroups,ifGenerationMatch,ifGenerationNotMatch,ifMetagenerationMatch,ifMetagenerationNotMatch,ifSourceGenerationMatch,ifSourceGenerationNotMatch,ifSourceMetagenerationMatch,ifSourceMetagenerationNotMatch,maxBytesRewrittenPerCall,projection,rewriteToken,sourceGeneration,userProject", ""}, + {"storage.objects.setIamPolicy", "PUT", "b/{bucket}/o/{object}/iam", "bindings,etag,kind,resourceId,version", "generation,userProject", ""}, + {"storage.objects.testIamPermissions", "GET", "b/{bucket}/o/{object}/iam/testPermissions", "", "generation,permissions,userProject", "permissions"}, + {"storage.objects.update", "PUT", "b/{bucket}/o/{object}", "acl,bucket,cacheControl,componentCount,contentDisposition,contentEncoding,contentLanguage,contentType,contexts,crc32c,customTime,customerEncryption,etag,eventBasedHold,generation,hardDeleteTime,id,kind,kmsKeyName,md5Hash,mediaLink,metadata,metageneration,name,owner,restoreToken,retention,retentionExpirationTime,selfLink,size,softDeleteTime,storageClass,temporaryHold,timeCreated,timeDeleted,timeFinalized,timeStorageClassUpdated,updated", "generation,ifGenerationMatch,ifGenerationNotMatch,ifMetagenerationMatch,ifMetagenerationNotMatch,overrideUnlockedRetention,predefinedAcl,projection,userProject", ""}, + {"storage.objects.viewFullContext", "GET", "b/{bucket}/o/{object}/viewFullContext", "", "contextKey,generation,userProject", "contextKey"}, + {"storage.projects.hmacKeys.create", "POST", "projects/{projectId}/hmacKeys", "", "serviceAccountEmail,userProject", "serviceAccountEmail"}, + {"storage.projects.hmacKeys.delete", "DELETE", "projects/{projectId}/hmacKeys/{accessId}", "", "userProject", ""}, + {"storage.projects.hmacKeys.get", "GET", "projects/{projectId}/hmacKeys/{accessId}", "", "userProject", ""}, + {"storage.projects.hmacKeys.list", "GET", "projects/{projectId}/hmacKeys", "", "maxResults,pageToken,serviceAccountEmail,showDeletedKeys,userProject", ""}, + {"storage.projects.hmacKeys.update", "PUT", "projects/{projectId}/hmacKeys/{accessId}", "accessId,etag,id,kind,projectId,selfLink,serviceAccountEmail,state,timeCreated,updated", "userProject", ""}, + {"storage.projects.serviceAccount.get", "GET", "projects/{projectId}/serviceAccount", "", "userProject", ""}, + {"storage.rapidCaches.disable", "POST", "b/{bucket}/rapidCaches/{rapidCacheId}/disable", "", "", ""}, + {"storage.rapidCaches.get", "GET", "b/{bucket}/rapidCaches/{rapidCacheId}", "", "", ""}, + {"storage.rapidCaches.insert", "POST", "b/{bucket}/rapidCaches", "admissionPolicy,bucket,cacheType,createTime,id,ingestOnWrite,kind,pendingUpdate,rapidCacheId,selfLink,state,ttl,updateTime,zone", "", ""}, + {"storage.rapidCaches.list", "GET", "b/{bucket}/rapidCaches", "", "pageSize,pageToken", ""}, + {"storage.rapidCaches.update", "PATCH", "b/{bucket}/rapidCaches/{rapidCacheId}", "admissionPolicy,bucket,cacheType,createTime,id,ingestOnWrite,kind,pendingUpdate,rapidCacheId,selfLink,state,ttl,updateTime,zone", "", ""}, +// pubsub v1 revision 20261013 + {"pubsub.projects.schemas.commit", "POST", "v1/{+name}:commit", "schema", "", ""}, + {"pubsub.projects.schemas.create", "POST", "v1/{+parent}/schemas", "compiledProtoSchema,definition,name,revisionCreateTime,revisionId,type", "schemaId", ""}, + {"pubsub.projects.schemas.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"pubsub.projects.schemas.deleteRevision", "DELETE", "v1/{+name}:deleteRevision", "", "revisionId", ""}, + {"pubsub.projects.schemas.get", "GET", "v1/{+name}", "", "view", ""}, + {"pubsub.projects.schemas.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"pubsub.projects.schemas.list", "GET", "v1/{+parent}/schemas", "", "pageSize,pageToken,view", ""}, + {"pubsub.projects.schemas.listRevisions", "GET", "v1/{+name}:listRevisions", "", "pageSize,pageToken,view", ""}, + {"pubsub.projects.schemas.rollback", "POST", "v1/{+name}:rollback", "revisionId", "", ""}, + {"pubsub.projects.schemas.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy", "", ""}, + {"pubsub.projects.schemas.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"pubsub.projects.schemas.validate", "POST", "v1/{+parent}/schemas:validate", "schema", "", ""}, + {"pubsub.projects.schemas.validateMessage", "POST", "v1/{+parent}/schemas:validateMessage", "encoding,message,name,schema", "", ""}, + {"pubsub.projects.snapshots.create", "PUT", "v1/{+name}", "labels,subscription,tags", "", ""}, + {"pubsub.projects.snapshots.delete", "DELETE", "v1/{+snapshot}", "", "", ""}, + {"pubsub.projects.snapshots.get", "GET", "v1/{+snapshot}", "", "", ""}, + {"pubsub.projects.snapshots.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"pubsub.projects.snapshots.list", "GET", "v1/{+project}/snapshots", "", "pageSize,pageToken", ""}, + {"pubsub.projects.snapshots.patch", "PATCH", "v1/{+name}", "snapshot,updateMask", "", ""}, + {"pubsub.projects.snapshots.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy", "", ""}, + {"pubsub.projects.snapshots.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"pubsub.projects.subscriptions.acknowledge", "POST", "v1/{+subscription}:acknowledge", "ackIds", "", ""}, + {"pubsub.projects.subscriptions.create", "PUT", "v1/{+name}", "ackDeadlineSeconds,analyticsHubSubscriptionInfo,bigqueryConfig,bigtableConfig,cloudStorageConfig,deadLetterPolicy,detached,enableExactlyOnceDelivery,enableMessageOrdering,expirationPolicy,filter,labels,messageRetentionDuration,messageTransforms,name,pushConfig,retainAckedMessages,retryPolicy,state,tags,topic,topicMessageRetentionDuration", "", ""}, + {"pubsub.projects.subscriptions.delete", "DELETE", "v1/{+subscription}", "", "", ""}, + {"pubsub.projects.subscriptions.detach", "POST", "v1/{+subscription}:detach", "", "", ""}, + {"pubsub.projects.subscriptions.get", "GET", "v1/{+subscription}", "", "", ""}, + {"pubsub.projects.subscriptions.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"pubsub.projects.subscriptions.list", "GET", "v1/{+project}/subscriptions", "", "pageSize,pageToken", ""}, + {"pubsub.projects.subscriptions.modifyAckDeadline", "POST", "v1/{+subscription}:modifyAckDeadline", "ackDeadlineSeconds,ackIds", "", ""}, + {"pubsub.projects.subscriptions.modifyPushConfig", "POST", "v1/{+subscription}:modifyPushConfig", "pushConfig", "", ""}, + {"pubsub.projects.subscriptions.patch", "PATCH", "v1/{+name}", "subscription,updateMask", "", ""}, + {"pubsub.projects.subscriptions.pull", "POST", "v1/{+subscription}:pull", "maxMessages,returnImmediately", "", ""}, + {"pubsub.projects.subscriptions.seek", "POST", "v1/{+subscription}:seek", "snapshot,time", "", ""}, + {"pubsub.projects.subscriptions.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy", "", ""}, + {"pubsub.projects.subscriptions.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"pubsub.projects.topics.create", "PUT", "v1/{+name}", "ingestionDataSourceSettings,kmsKeyName,labels,messageRetentionDuration,messageStoragePolicy,messageTransforms,name,satisfiesPzs,schemaSettings,state,tags", "", ""}, + {"pubsub.projects.topics.delete", "DELETE", "v1/{+topic}", "", "", ""}, + {"pubsub.projects.topics.get", "GET", "v1/{+topic}", "", "", ""}, + {"pubsub.projects.topics.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"pubsub.projects.topics.list", "GET", "v1/{+project}/topics", "", "pageSize,pageToken", ""}, + {"pubsub.projects.topics.patch", "PATCH", "v1/{+name}", "topic,updateMask", "", ""}, + {"pubsub.projects.topics.publish", "POST", "v1/{+topic}:publish", "messages", "", ""}, + {"pubsub.projects.topics.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy", "", ""}, + {"pubsub.projects.topics.snapshots.list", "GET", "v1/{+topic}/snapshots", "", "pageSize,pageToken", ""}, + {"pubsub.projects.topics.subscriptions.list", "GET", "v1/{+topic}/subscriptions", "", "pageSize,pageToken", ""}, + {"pubsub.projects.topics.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, +// firestore v1 revision 20261013 + {"firestore.projects.databases.backupSchedules.create", "POST", "v1/{+parent}/backupSchedules", "createTime,dailyRecurrence,name,retention,updateTime,weeklyRecurrence", "", ""}, + {"firestore.projects.databases.backupSchedules.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"firestore.projects.databases.backupSchedules.get", "GET", "v1/{+name}", "", "", ""}, + {"firestore.projects.databases.backupSchedules.list", "GET", "v1/{+parent}/backupSchedules", "", "", ""}, + {"firestore.projects.databases.backupSchedules.patch", "PATCH", "v1/{+name}", "createTime,dailyRecurrence,name,retention,updateTime,weeklyRecurrence", "updateMask", ""}, + {"firestore.projects.databases.bulkDeleteDocuments", "POST", "v1/{+name}:bulkDeleteDocuments", "collectionIds,namespaceIds", "", ""}, + {"firestore.projects.databases.changeStreams.create", "POST", "v1/{+parent}/changeStreams", "collectionGroupScope,createTime,databaseScope,etag,name,retentionPeriod,startTime,updateTime", "changeStreamId", ""}, + {"firestore.projects.databases.changeStreams.delete", "DELETE", "v1/{+name}", "", "etag", ""}, + {"firestore.projects.databases.changeStreams.get", "GET", "v1/{+name}", "", "", ""}, + {"firestore.projects.databases.changeStreams.list", "GET", "v1/{+parent}/changeStreams", "", "", ""}, + {"firestore.projects.databases.clone", "POST", "v1/{+parent}/databases:clone", "databaseId,encryptionConfig,pitrSnapshot,tags", "", ""}, + {"firestore.projects.databases.collectionGroups.fields.get", "GET", "v1/{+name}", "", "", ""}, + {"firestore.projects.databases.collectionGroups.fields.list", "GET", "v1/{+parent}/fields", "", "filter,pageSize,pageToken", ""}, + {"firestore.projects.databases.collectionGroups.fields.patch", "PATCH", "v1/{+name}", "indexConfig,name,ttlConfig", "updateMask", ""}, + {"firestore.projects.databases.collectionGroups.indexes.create", "POST", "v1/{+parent}/indexes", "apiScope,density,fields,multikey,name,queryScope,searchIndexOptions,shardCount,state,unique", "", ""}, + {"firestore.projects.databases.collectionGroups.indexes.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"firestore.projects.databases.collectionGroups.indexes.get", "GET", "v1/{+name}", "", "", ""}, + {"firestore.projects.databases.collectionGroups.indexes.list", "GET", "v1/{+parent}/indexes", "", "filter,pageSize,pageToken", ""}, + {"firestore.projects.databases.create", "POST", "v1/{+parent}/databases", "appEngineIntegrationMode,cmekConfig,concurrencyMode,createTime,databaseEdition,deleteProtectionState,deleteTime,earliestVersionTime,etag,firestoreDataAccessMode,freeTier,keyPrefix,locationId,mongodbCompatibleDataAccessMode,name,pointInTimeRecoveryEnablement,previousId,realtimeUpdatesMode,sourceInfo,tags,type,uid,updateTime,versionRetentionPeriod", "databaseId", ""}, + {"firestore.projects.databases.delete", "DELETE", "v1/{+name}", "", "etag", ""}, + {"firestore.projects.databases.documents.batchGet", "POST", "v1/{+database}/documents:batchGet", "documents,mask,newTransaction,readTime,requestOptions,transaction", "", ""}, + {"firestore.projects.databases.documents.batchWrite", "POST", "v1/{+database}/documents:batchWrite", "labels,requestOptions,writes", "", ""}, + {"firestore.projects.databases.documents.beginTransaction", "POST", "v1/{+database}/documents:beginTransaction", "options,requestOptions", "", ""}, + {"firestore.projects.databases.documents.commit", "POST", "v1/{+database}/documents:commit", "requestOptions,transaction,writes", "", ""}, + {"firestore.projects.databases.documents.createDocument", "POST", "v1/{+parent}/{collectionId}", "createTime,fields,name,updateTime", "documentId,mask.fieldPaths,requestOptions.requestTags", ""}, + {"firestore.projects.databases.documents.delete", "DELETE", "v1/{+name}", "", "currentDocument.exists,currentDocument.updateTime,requestOptions.requestTags", ""}, + {"firestore.projects.databases.documents.executePipeline", "POST", "v1/{+database}/documents:executePipeline", "autoCommitTransaction,newTransaction,readTime,requestOptions,structuredPipeline,transaction", "", ""}, + {"firestore.projects.databases.documents.get", "GET", "v1/{+name}", "", "mask.fieldPaths,readTime,requestOptions.requestTags,transaction", ""}, + {"firestore.projects.databases.documents.list", "GET", "v1/{+parent}/{collectionId}", "", "mask.fieldPaths,orderBy,pageSize,pageToken,readTime,recursive,requestOptions.requestTags,showMissing,transaction", ""}, + {"firestore.projects.databases.documents.listCollectionIds", "POST", "v1/{+parent}:listCollectionIds", "pageSize,pageToken,readTime,requestOptions", "", ""}, + {"firestore.projects.databases.documents.listDocuments", "GET", "v1/{+parent}/{collectionId}", "", "mask.fieldPaths,orderBy,pageSize,pageToken,readTime,recursive,requestOptions.requestTags,showMissing,transaction", ""}, + {"firestore.projects.databases.documents.listen", "POST", "v1/{+database}/documents:listen", "addTarget,labels,removeTarget,requestOptions", "", ""}, + {"firestore.projects.databases.documents.partitionQuery", "POST", "v1/{+parent}:partitionQuery", "pageSize,pageToken,partitionCount,readTime,requestOptions,structuredQuery", "", ""}, + {"firestore.projects.databases.documents.patch", "PATCH", "v1/{+name}", "createTime,fields,name,updateTime", "currentDocument.exists,currentDocument.updateTime,mask.fieldPaths,requestOptions.requestTags,updateMask.fieldPaths", ""}, + {"firestore.projects.databases.documents.rollback", "POST", "v1/{+database}/documents:rollback", "requestOptions,transaction", "", ""}, + {"firestore.projects.databases.documents.runAggregationQuery", "POST", "v1/{+parent}:runAggregationQuery", "explainOptions,newTransaction,readTime,requestOptions,structuredAggregationQuery,transaction", "", ""}, + {"firestore.projects.databases.documents.runQuery", "POST", "v1/{+parent}:runQuery", "explainOptions,newTransaction,readTime,requestOptions,structuredQuery,transaction", "", ""}, + {"firestore.projects.databases.documents.write", "POST", "v1/{+database}/documents:write", "labels,requestOptions,streamId,streamToken,writes", "", ""}, + {"firestore.projects.databases.exportDocuments", "POST", "v1/{+name}:exportDocuments", "collectionIds,namespaceIds,outputUriPrefix,snapshotTime", "", ""}, + {"firestore.projects.databases.get", "GET", "v1/{+name}", "", "", ""}, + {"firestore.projects.databases.importDocuments", "POST", "v1/{+name}:importDocuments", "collectionIds,inputUriPrefix,namespaceIds", "", ""}, + {"firestore.projects.databases.list", "GET", "v1/{+parent}/databases", "", "showDeleted", ""}, + {"firestore.projects.databases.operations.cancel", "POST", "v1/{+name}:cancel", "", "", ""}, + {"firestore.projects.databases.operations.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"firestore.projects.databases.operations.get", "GET", "v1/{+name}", "", "", ""}, + {"firestore.projects.databases.operations.list", "GET", "v1/{+name}/operations", "", "filter,pageSize,pageToken,returnPartialSuccess", ""}, + {"firestore.projects.databases.patch", "PATCH", "v1/{+name}", "appEngineIntegrationMode,cmekConfig,concurrencyMode,createTime,databaseEdition,deleteProtectionState,deleteTime,earliestVersionTime,etag,firestoreDataAccessMode,freeTier,keyPrefix,locationId,mongodbCompatibleDataAccessMode,name,pointInTimeRecoveryEnablement,previousId,realtimeUpdatesMode,sourceInfo,tags,type,uid,updateTime,versionRetentionPeriod", "updateMask", ""}, + {"firestore.projects.databases.restore", "POST", "v1/{+parent}/databases:restore", "backup,databaseId,encryptionConfig,tags", "", ""}, + {"firestore.projects.databases.userCreds.create", "POST", "v1/{+parent}/userCreds", "createTime,name,resourceIdentity,securePassword,state,updateTime", "userCredsId", ""}, + {"firestore.projects.databases.userCreds.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"firestore.projects.databases.userCreds.disable", "POST", "v1/{+name}:disable", "", "", ""}, + {"firestore.projects.databases.userCreds.enable", "POST", "v1/{+name}:enable", "", "", ""}, + {"firestore.projects.databases.userCreds.get", "GET", "v1/{+name}", "", "", ""}, + {"firestore.projects.databases.userCreds.list", "GET", "v1/{+parent}/userCreds", "", "", ""}, + {"firestore.projects.databases.userCreds.resetPassword", "POST", "v1/{+name}:resetPassword", "", "", ""}, + {"firestore.projects.locations.backups.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"firestore.projects.locations.backups.get", "GET", "v1/{+name}", "", "", ""}, + {"firestore.projects.locations.backups.list", "GET", "v1/{+parent}/backups", "", "filter", ""}, + {"firestore.projects.locations.get", "GET", "v1/{+name}", "", "", ""}, + {"firestore.projects.locations.list", "GET", "v1/{+name}/locations", "", "extraLocationTypes,filter,pageSize,pageToken", ""}, +// spanner v1 revision 20261006 + {"spanner.projects.instanceConfigOperations.list", "GET", "v1/{+parent}/instanceConfigOperations", "", "filter,pageSize,pageToken", ""}, + {"spanner.projects.instanceConfigs.create", "POST", "v1/{+parent}/instanceConfigs", "instanceConfig,instanceConfigId,validateOnly", "", ""}, + {"spanner.projects.instanceConfigs.delete", "DELETE", "v1/{+name}", "", "etag,validateOnly", ""}, + {"spanner.projects.instanceConfigs.get", "GET", "v1/{+name}", "", "", ""}, + {"spanner.projects.instanceConfigs.list", "GET", "v1/{+parent}/instanceConfigs", "", "pageSize,pageToken", ""}, + {"spanner.projects.instanceConfigs.operations.cancel", "POST", "v1/{+name}:cancel", "", "", ""}, + {"spanner.projects.instanceConfigs.operations.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"spanner.projects.instanceConfigs.operations.get", "GET", "v1/{+name}", "", "", ""}, + {"spanner.projects.instanceConfigs.operations.list", "GET", "v1/{+name}", "", "filter,pageSize,pageToken,returnPartialSuccess", ""}, + {"spanner.projects.instanceConfigs.patch", "PATCH", "v1/{+name}", "instanceConfig,updateMask,validateOnly", "", ""}, + {"spanner.projects.instanceConfigs.ssdCaches.operations.cancel", "POST", "v1/{+name}:cancel", "", "", ""}, + {"spanner.projects.instanceConfigs.ssdCaches.operations.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"spanner.projects.instanceConfigs.ssdCaches.operations.get", "GET", "v1/{+name}", "", "", ""}, + {"spanner.projects.instanceConfigs.ssdCaches.operations.list", "GET", "v1/{+name}", "", "filter,pageSize,pageToken,returnPartialSuccess", ""}, + {"spanner.projects.instances.backupOperations.list", "GET", "v1/{+parent}/backupOperations", "", "filter,pageSize,pageToken", ""}, + {"spanner.projects.instances.backups.copy", "POST", "v1/{+parent}/backups:copy", "backupId,encryptionConfig,expireTime,sourceBackup", "", ""}, + {"spanner.projects.instances.backups.create", "POST", "v1/{+parent}/backups", "backupSchedules,createTime,database,databaseDialect,encryptionInfo,encryptionInformation,exclusiveSizeBytes,expireTime,freeableSizeBytes,incrementalBackupChainId,instancePartitions,maxExpireTime,minimumRestorableEdition,name,oldestVersionTime,referencingBackups,referencingDatabases,sizeBytes,state,versionTime", "backupId,encryptionConfig.encryptionType,encryptionConfig.kmsKeyName,encryptionConfig.kmsKeyNames", ""}, + {"spanner.projects.instances.backups.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.backups.get", "GET", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.backups.getIamPolicy", "POST", "v1/{+resource}:getIamPolicy", "options", "", ""}, + {"spanner.projects.instances.backups.list", "GET", "v1/{+parent}/backups", "", "filter,pageSize,pageToken", ""}, + {"spanner.projects.instances.backups.operations.cancel", "POST", "v1/{+name}:cancel", "", "", ""}, + {"spanner.projects.instances.backups.operations.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.backups.operations.get", "GET", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.backups.operations.list", "GET", "v1/{+name}", "", "filter,pageSize,pageToken,returnPartialSuccess", ""}, + {"spanner.projects.instances.backups.patch", "PATCH", "v1/{+name}", "backupSchedules,createTime,database,databaseDialect,encryptionInfo,encryptionInformation,exclusiveSizeBytes,expireTime,freeableSizeBytes,incrementalBackupChainId,instancePartitions,maxExpireTime,minimumRestorableEdition,name,oldestVersionTime,referencingBackups,referencingDatabases,sizeBytes,state,versionTime", "updateMask", ""}, + {"spanner.projects.instances.backups.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy", "", ""}, + {"spanner.projects.instances.backups.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"spanner.projects.instances.create", "POST", "v1/{+parent}/instances", "instance,instanceId", "", ""}, + {"spanner.projects.instances.databaseOperations.list", "GET", "v1/{+parent}/databaseOperations", "", "filter,pageSize,pageToken", ""}, + {"spanner.projects.instances.databases.addSplitPoints", "POST", "v1/{+database}:addSplitPoints", "initiator,splitPoints", "", ""}, + {"spanner.projects.instances.databases.backupSchedules.create", "POST", "v1/{+parent}/backupSchedules", "encryptionConfig,fullBackupSpec,incrementalBackupSpec,name,retentionDuration,spec,updateTime", "backupScheduleId", ""}, + {"spanner.projects.instances.databases.backupSchedules.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.databases.backupSchedules.get", "GET", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.databases.backupSchedules.getIamPolicy", "POST", "v1/{+resource}:getIamPolicy", "options", "", ""}, + {"spanner.projects.instances.databases.backupSchedules.list", "GET", "v1/{+parent}/backupSchedules", "", "pageSize,pageToken", ""}, + {"spanner.projects.instances.databases.backupSchedules.patch", "PATCH", "v1/{+name}", "encryptionConfig,fullBackupSpec,incrementalBackupSpec,name,retentionDuration,spec,updateTime", "updateMask", ""}, + {"spanner.projects.instances.databases.backupSchedules.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy", "", ""}, + {"spanner.projects.instances.databases.backupSchedules.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"spanner.projects.instances.databases.changequorum", "POST", "v1/{+name}:changequorum", "etag,name,quorumType", "", ""}, + {"spanner.projects.instances.databases.create", "POST", "v1/{+parent}/databases", "createStatement,databaseDialect,encryptionConfig,extraStatements,protoDescriptors", "", ""}, + {"spanner.projects.instances.databases.databaseRoles.list", "GET", "v1/{+parent}/databaseRoles", "", "pageSize,pageToken", ""}, + {"spanner.projects.instances.databases.databaseRoles.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"spanner.projects.instances.databases.dropDatabase", "DELETE", "v1/{+database}", "", "", ""}, + {"spanner.projects.instances.databases.get", "GET", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.databases.getDdl", "GET", "v1/{+database}/ddl", "", "", ""}, + {"spanner.projects.instances.databases.getIamPolicy", "POST", "v1/{+resource}:getIamPolicy", "options", "", ""}, + {"spanner.projects.instances.databases.getScans", "GET", "v1/{+name}/scans", "", "endTime,startTime,view", ""}, + {"spanner.projects.instances.databases.list", "GET", "v1/{+parent}/databases", "", "pageSize,pageToken", ""}, + {"spanner.projects.instances.databases.operations.cancel", "POST", "v1/{+name}:cancel", "", "", ""}, + {"spanner.projects.instances.databases.operations.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.databases.operations.get", "GET", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.databases.operations.list", "GET", "v1/{+name}", "", "filter,pageSize,pageToken,returnPartialSuccess", ""}, + {"spanner.projects.instances.databases.patch", "PATCH", "v1/{+name}", "createTime,databaseDialect,defaultLeader,earliestVersionTime,enableDropProtection,encryptionConfig,encryptionInfo,name,quorumInfo,reconciling,restoreInfo,state,versionRetentionPeriod", "updateMask", ""}, + {"spanner.projects.instances.databases.restore", "POST", "v1/{+parent}/databases:restore", "backup,databaseId,encryptionConfig", "", ""}, + {"spanner.projects.instances.databases.sessions.adaptMessage", "POST", "v1/{+name}:adaptMessage", "attachments,payload,protocol", "", ""}, + {"spanner.projects.instances.databases.sessions.adapter", "POST", "v1/{+parent}/sessions:adapter", "name", "", ""}, + {"spanner.projects.instances.databases.sessions.batchCreate", "POST", "v1/{+database}/sessions:batchCreate", "sessionCount,sessionTemplate", "", ""}, + {"spanner.projects.instances.databases.sessions.batchWrite", "POST", "v1/{+session}:batchWrite", "excludeTxnFromChangeStreams,mutationGroups,requestOptions", "", ""}, + {"spanner.projects.instances.databases.sessions.beginTransaction", "POST", "v1/{+session}:beginTransaction", "mutationKey,options,requestOptions", "", ""}, + {"spanner.projects.instances.databases.sessions.commit", "POST", "v1/{+session}:commit", "maxCommitDelay,mutations,precommitToken,requestOptions,returnCommitStats,singleUseTransaction,transactionId", "", ""}, + {"spanner.projects.instances.databases.sessions.create", "POST", "v1/{+database}/sessions", "session", "", ""}, + {"spanner.projects.instances.databases.sessions.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.databases.sessions.executeBatchDml", "POST", "v1/{+session}:executeBatchDml", "lastStatements,requestOptions,seqno,statements,transaction", "", ""}, + {"spanner.projects.instances.databases.sessions.executeSql", "POST", "v1/{+session}:executeSql", "dataBoostEnabled,directedReadOptions,lastStatement,paramTypes,params,partitionToken,queryMode,queryOptions,requestOptions,resumeToken,seqno,sql,transaction", "", ""}, + {"spanner.projects.instances.databases.sessions.executeStreamingSql", "POST", "v1/{+session}:executeStreamingSql", "dataBoostEnabled,directedReadOptions,lastStatement,paramTypes,params,partitionToken,queryMode,queryOptions,requestOptions,resumeToken,seqno,sql,transaction", "", ""}, + {"spanner.projects.instances.databases.sessions.get", "GET", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.databases.sessions.list", "GET", "v1/{+database}/sessions", "", "filter,pageSize,pageToken", ""}, + {"spanner.projects.instances.databases.sessions.partitionQuery", "POST", "v1/{+session}:partitionQuery", "paramTypes,params,partitionOptions,sql,transaction", "", ""}, + {"spanner.projects.instances.databases.sessions.partitionRead", "POST", "v1/{+session}:partitionRead", "columns,index,keySet,partitionOptions,table,transaction", "", ""}, + {"spanner.projects.instances.databases.sessions.read", "POST", "v1/{+session}:read", "columns,dataBoostEnabled,directedReadOptions,index,keySet,limit,lockHint,orderBy,partitionToken,requestOptions,resumeToken,table,transaction", "", ""}, + {"spanner.projects.instances.databases.sessions.rollback", "POST", "v1/{+session}:rollback", "transactionId", "", ""}, + {"spanner.projects.instances.databases.sessions.streamingRead", "POST", "v1/{+session}:streamingRead", "columns,dataBoostEnabled,directedReadOptions,index,keySet,limit,lockHint,orderBy,partitionToken,requestOptions,resumeToken,table,transaction", "", ""}, + {"spanner.projects.instances.databases.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy", "", ""}, + {"spanner.projects.instances.databases.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"spanner.projects.instances.databases.updateDdl", "PATCH", "v1/{+database}/ddl", "operationId,protoDescriptors,statements", "", ""}, + {"spanner.projects.instances.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.get", "GET", "v1/{+name}", "", "fieldMask", ""}, + {"spanner.projects.instances.getIamPolicy", "POST", "v1/{+resource}:getIamPolicy", "options", "", ""}, + {"spanner.projects.instances.instancePartitionOperations.list", "GET", "v1/{+parent}/instancePartitionOperations", "", "filter,instancePartitionDeadline,pageSize,pageToken", ""}, + {"spanner.projects.instances.instancePartitions.create", "POST", "v1/{+parent}/instancePartitions", "instancePartition,instancePartitionId", "", ""}, + {"spanner.projects.instances.instancePartitions.delete", "DELETE", "v1/{+name}", "", "etag", ""}, + {"spanner.projects.instances.instancePartitions.get", "GET", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.instancePartitions.list", "GET", "v1/{+parent}/instancePartitions", "", "instancePartitionDeadline,pageSize,pageToken", ""}, + {"spanner.projects.instances.instancePartitions.operations.cancel", "POST", "v1/{+name}:cancel", "", "", ""}, + {"spanner.projects.instances.instancePartitions.operations.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.instancePartitions.operations.get", "GET", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.instancePartitions.operations.list", "GET", "v1/{+name}", "", "filter,pageSize,pageToken,returnPartialSuccess", ""}, + {"spanner.projects.instances.instancePartitions.patch", "PATCH", "v1/{+name}", "fieldMask,instancePartition", "", ""}, + {"spanner.projects.instances.list", "GET", "v1/{+parent}/instances", "", "filter,instanceDeadline,pageSize,pageToken", ""}, + {"spanner.projects.instances.move", "POST", "v1/{+name}:move", "targetConfig,targetDatabaseMoveConfigs", "", ""}, + {"spanner.projects.instances.operations.cancel", "POST", "v1/{+name}:cancel", "", "", ""}, + {"spanner.projects.instances.operations.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.operations.get", "GET", "v1/{+name}", "", "", ""}, + {"spanner.projects.instances.operations.list", "GET", "v1/{+name}", "", "filter,pageSize,pageToken,returnPartialSuccess", ""}, + {"spanner.projects.instances.patch", "PATCH", "v1/{+name}", "fieldMask,instance", "", ""}, + {"spanner.projects.instances.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy", "", ""}, + {"spanner.projects.instances.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"spanner.scans.list", "GET", "v1/{+parent}", "", "filter,pageSize,pageToken,view", ""}, +// bigtableadmin v2 revision 20261017 + {"bigtableadmin.operations.get", "GET", "v2/{+name}", "", "", ""}, + {"bigtableadmin.operations.projects.operations.list", "GET", "v2/{+name}/operations", "", "filter,pageSize,pageToken,returnPartialSuccess", ""}, + {"bigtableadmin.projects.instances.appProfiles.create", "POST", "v2/{+parent}/appProfiles", "dataBoostIsolationReadOnly,description,etag,multiClusterRoutingUseAny,name,priority,singleClusterRouting,standardIsolation", "appProfileId,ignoreWarnings", ""}, + {"bigtableadmin.projects.instances.appProfiles.delete", "DELETE", "v2/{+name}", "", "ignoreWarnings", ""}, + {"bigtableadmin.projects.instances.appProfiles.get", "GET", "v2/{+name}", "", "", ""}, + {"bigtableadmin.projects.instances.appProfiles.list", "GET", "v2/{+parent}/appProfiles", "", "pageSize,pageToken", ""}, + {"bigtableadmin.projects.instances.appProfiles.patch", "PATCH", "v2/{+name}", "dataBoostIsolationReadOnly,description,etag,multiClusterRoutingUseAny,name,priority,singleClusterRouting,standardIsolation", "ignoreWarnings,updateMask", ""}, + {"bigtableadmin.projects.instances.clusters.backups.copy", "POST", "v2/{+parent}/backups:copy", "backupId,expireTime,sourceBackup", "", ""}, + {"bigtableadmin.projects.instances.clusters.backups.create", "POST", "v2/{+parent}/backups", "backupType,encryptionInfo,endTime,expireTime,hotToStandardTime,name,sizeBytes,sourceBackup,sourceTable,startTime,state", "backupId", ""}, + {"bigtableadmin.projects.instances.clusters.backups.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"bigtableadmin.projects.instances.clusters.backups.get", "GET", "v2/{+name}", "", "", ""}, + {"bigtableadmin.projects.instances.clusters.backups.getIamPolicy", "POST", "v2/{+resource}:getIamPolicy", "options", "", ""}, + {"bigtableadmin.projects.instances.clusters.backups.list", "GET", "v2/{+parent}/backups", "", "filter,orderBy,pageSize,pageToken", ""}, + {"bigtableadmin.projects.instances.clusters.backups.patch", "PATCH", "v2/{+name}", "backupType,encryptionInfo,endTime,expireTime,hotToStandardTime,name,sizeBytes,sourceBackup,sourceTable,startTime,state", "updateMask", ""}, + {"bigtableadmin.projects.instances.clusters.backups.setIamPolicy", "POST", "v2/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"bigtableadmin.projects.instances.clusters.backups.testIamPermissions", "POST", "v2/{+resource}:testIamPermissions", "permissions", "", ""}, + {"bigtableadmin.projects.instances.clusters.create", "POST", "v2/{+parent}/clusters", "clusterConfig,defaultStorageType,encryptionConfig,location,name,nodeScalingFactor,serveNodes,state", "clusterId", ""}, + {"bigtableadmin.projects.instances.clusters.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"bigtableadmin.projects.instances.clusters.get", "GET", "v2/{+name}", "", "", ""}, + {"bigtableadmin.projects.instances.clusters.getMemoryLayer", "GET", "v2/{+name}", "", "", ""}, + {"bigtableadmin.projects.instances.clusters.hotTablets.list", "GET", "v2/{+parent}/hotTablets", "", "endTime,pageSize,pageToken,startTime", ""}, + {"bigtableadmin.projects.instances.clusters.list", "GET", "v2/{+parent}/clusters", "", "pageToken", ""}, + {"bigtableadmin.projects.instances.clusters.memoryLayers.list", "GET", "v2/{+parent}/memoryLayers", "", "pageSize,pageToken", ""}, + {"bigtableadmin.projects.instances.clusters.partialUpdateCluster", "PATCH", "v2/{+name}", "clusterConfig,defaultStorageType,encryptionConfig,location,name,nodeScalingFactor,serveNodes,state", "updateMask", ""}, + {"bigtableadmin.projects.instances.clusters.update", "PUT", "v2/{+name}", "clusterConfig,defaultStorageType,encryptionConfig,location,name,nodeScalingFactor,serveNodes,state", "", ""}, + {"bigtableadmin.projects.instances.clusters.updateMemoryLayer", "PATCH", "v2/{+name}", "etag,memoryConfig,name,state", "updateMask", ""}, + {"bigtableadmin.projects.instances.create", "POST", "v2/{+parent}/instances", "clusters,instance,instanceId,parent", "", ""}, + {"bigtableadmin.projects.instances.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"bigtableadmin.projects.instances.get", "GET", "v2/{+name}", "", "", ""}, + {"bigtableadmin.projects.instances.getIamPolicy", "POST", "v2/{+resource}:getIamPolicy", "options", "", ""}, + {"bigtableadmin.projects.instances.list", "GET", "v2/{+parent}/instances", "", "pageToken", ""}, + {"bigtableadmin.projects.instances.logicalViews.create", "POST", "v2/{+parent}/logicalViews", "deletionProtection,etag,name,query", "logicalViewId", ""}, + {"bigtableadmin.projects.instances.logicalViews.delete", "DELETE", "v2/{+name}", "", "etag", ""}, + {"bigtableadmin.projects.instances.logicalViews.get", "GET", "v2/{+name}", "", "", ""}, + {"bigtableadmin.projects.instances.logicalViews.getIamPolicy", "POST", "v2/{+resource}:getIamPolicy", "options", "", ""}, + {"bigtableadmin.projects.instances.logicalViews.list", "GET", "v2/{+parent}/logicalViews", "", "pageSize,pageToken", ""}, + {"bigtableadmin.projects.instances.logicalViews.patch", "PATCH", "v2/{+name}", "deletionProtection,etag,name,query", "updateMask", ""}, + {"bigtableadmin.projects.instances.logicalViews.setIamPolicy", "POST", "v2/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"bigtableadmin.projects.instances.logicalViews.testIamPermissions", "POST", "v2/{+resource}:testIamPermissions", "permissions", "", ""}, + {"bigtableadmin.projects.instances.materializedViews.create", "POST", "v2/{+parent}/materializedViews", "clusterStates,deletionProtection,etag,name,query", "ignoreWarnings,materializedViewId", ""}, + {"bigtableadmin.projects.instances.materializedViews.delete", "DELETE", "v2/{+name}", "", "etag", ""}, + {"bigtableadmin.projects.instances.materializedViews.get", "GET", "v2/{+name}", "", "view", ""}, + {"bigtableadmin.projects.instances.materializedViews.getIamPolicy", "POST", "v2/{+resource}:getIamPolicy", "options", "", ""}, + {"bigtableadmin.projects.instances.materializedViews.list", "GET", "v2/{+parent}/materializedViews", "", "pageSize,pageToken,view", ""}, + {"bigtableadmin.projects.instances.materializedViews.patch", "PATCH", "v2/{+name}", "clusterStates,deletionProtection,etag,name,query", "updateMask", ""}, + {"bigtableadmin.projects.instances.materializedViews.setIamPolicy", "POST", "v2/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"bigtableadmin.projects.instances.materializedViews.testIamPermissions", "POST", "v2/{+resource}:testIamPermissions", "permissions", "", ""}, + {"bigtableadmin.projects.instances.partialUpdateInstance", "PATCH", "v2/{+name}", "createTime,displayName,edition,knowledgeCatalogRegion,labels,name,satisfiesPzi,satisfiesPzs,state,tags,type", "updateMask", ""}, + {"bigtableadmin.projects.instances.setIamPolicy", "POST", "v2/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"bigtableadmin.projects.instances.tables.authorizedViews.create", "POST", "v2/{+parent}/authorizedViews", "deletionProtection,etag,name,subsetView", "authorizedViewId", ""}, + {"bigtableadmin.projects.instances.tables.authorizedViews.delete", "DELETE", "v2/{+name}", "", "etag", ""}, + {"bigtableadmin.projects.instances.tables.authorizedViews.get", "GET", "v2/{+name}", "", "view", ""}, + {"bigtableadmin.projects.instances.tables.authorizedViews.getIamPolicy", "POST", "v2/{+resource}:getIamPolicy", "options", "", ""}, + {"bigtableadmin.projects.instances.tables.authorizedViews.list", "GET", "v2/{+parent}/authorizedViews", "", "pageSize,pageToken,view", ""}, + {"bigtableadmin.projects.instances.tables.authorizedViews.patch", "PATCH", "v2/{+name}", "deletionProtection,etag,name,subsetView", "ignoreWarnings,updateMask", ""}, + {"bigtableadmin.projects.instances.tables.authorizedViews.setIamPolicy", "POST", "v2/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"bigtableadmin.projects.instances.tables.authorizedViews.testIamPermissions", "POST", "v2/{+resource}:testIamPermissions", "permissions", "", ""}, + {"bigtableadmin.projects.instances.tables.checkConsistency", "POST", "v2/{+name}:checkConsistency", "consistencyToken,dataBoostReadLocalWrites,standardReadRemoteWrites", "", ""}, + {"bigtableadmin.projects.instances.tables.create", "POST", "v2/{+parent}/tables", "initialSplits,table,tableId", "", ""}, + {"bigtableadmin.projects.instances.tables.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"bigtableadmin.projects.instances.tables.dropRowRange", "POST", "v2/{+name}:dropRowRange", "deleteAllDataFromTable,rowKeyPrefix", "", ""}, + {"bigtableadmin.projects.instances.tables.generateConsistencyToken", "POST", "v2/{+name}:generateConsistencyToken", "", "", ""}, + {"bigtableadmin.projects.instances.tables.get", "GET", "v2/{+name}", "", "view", ""}, + {"bigtableadmin.projects.instances.tables.getIamPolicy", "POST", "v2/{+resource}:getIamPolicy", "options", "", ""}, + {"bigtableadmin.projects.instances.tables.list", "GET", "v2/{+parent}/tables", "", "pageSize,pageToken,view", ""}, + {"bigtableadmin.projects.instances.tables.modifyColumnFamilies", "POST", "v2/{+name}:modifyColumnFamilies", "ignoreWarnings,modifications", "", ""}, + {"bigtableadmin.projects.instances.tables.patch", "PATCH", "v2/{+name}", "automatedBackupPolicy,changeStreamConfig,clusterStates,columnFamilies,deletionProtection,effectiveAutomatedBackupPolicy,granularity,name,restoreInfo,rowKeySchema,stats,tieredStorageConfig", "ignoreWarnings,updateMask", ""}, + {"bigtableadmin.projects.instances.tables.restore", "POST", "v2/{+parent}/tables:restore", "backup,tableId", "", ""}, + {"bigtableadmin.projects.instances.tables.schemaBundles.create", "POST", "v2/{+parent}/schemaBundles", "avroSchema,etag,name,protoSchema", "schemaBundleId", ""}, + {"bigtableadmin.projects.instances.tables.schemaBundles.delete", "DELETE", "v2/{+name}", "", "etag", ""}, + {"bigtableadmin.projects.instances.tables.schemaBundles.get", "GET", "v2/{+name}", "", "", ""}, + {"bigtableadmin.projects.instances.tables.schemaBundles.getIamPolicy", "POST", "v2/{+resource}:getIamPolicy", "options", "", ""}, + {"bigtableadmin.projects.instances.tables.schemaBundles.list", "GET", "v2/{+parent}/schemaBundles", "", "pageSize,pageToken,view", ""}, + {"bigtableadmin.projects.instances.tables.schemaBundles.patch", "PATCH", "v2/{+name}", "avroSchema,etag,name,protoSchema", "ignoreWarnings,updateMask", ""}, + {"bigtableadmin.projects.instances.tables.schemaBundles.setIamPolicy", "POST", "v2/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"bigtableadmin.projects.instances.tables.schemaBundles.testIamPermissions", "POST", "v2/{+resource}:testIamPermissions", "permissions", "", ""}, + {"bigtableadmin.projects.instances.tables.setIamPolicy", "POST", "v2/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"bigtableadmin.projects.instances.tables.testIamPermissions", "POST", "v2/{+resource}:testIamPermissions", "permissions", "", ""}, + {"bigtableadmin.projects.instances.tables.undelete", "POST", "v2/{+name}:undelete", "", "", ""}, + {"bigtableadmin.projects.instances.testIamPermissions", "POST", "v2/{+resource}:testIamPermissions", "permissions", "", ""}, + {"bigtableadmin.projects.instances.update", "PUT", "v2/{+name}", "createTime,displayName,edition,knowledgeCatalogRegion,labels,name,satisfiesPzi,satisfiesPzs,state,tags,type", "", ""}, + {"bigtableadmin.projects.locations.list", "GET", "v2/{+name}/locations", "", "extraLocationTypes,filter,pageSize,pageToken", ""}, +// secretmanager v1 revision 20260925 + {"secretmanager.projects.locations.get", "GET", "v1/{+name}", "", "", ""}, + {"secretmanager.projects.locations.list", "GET", "v1/{+name}/locations", "", "extraLocationTypes,filter,pageSize,pageToken", ""}, + {"secretmanager.projects.locations.secrets.addVersion", "POST", "v1/{+parent}:addVersion", "payload", "", ""}, + {"secretmanager.projects.locations.secrets.create", "POST", "v1/{+parent}/secrets", "annotations,createTime,customerManagedEncryption,etag,expireTime,labels,name,policyMember,replication,rotation,secretType,tags,topics,ttl,versionAliases,versionDestroyTtl", "secretId", ""}, + {"secretmanager.projects.locations.secrets.delete", "DELETE", "v1/{+name}", "", "etag", ""}, + {"secretmanager.projects.locations.secrets.enableManagedRotation", "POST", "v1/{+parent}:enableManagedRotation", "cloudSqlSingleUserCredentials", "", ""}, + {"secretmanager.projects.locations.secrets.get", "GET", "v1/{+name}", "", "", ""}, + {"secretmanager.projects.locations.secrets.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"secretmanager.projects.locations.secrets.list", "GET", "v1/{+parent}/secrets", "", "filter,pageSize,pageToken", ""}, + {"secretmanager.projects.locations.secrets.patch", "PATCH", "v1/{+name}", "annotations,createTime,customerManagedEncryption,etag,expireTime,labels,name,policyMember,replication,rotation,secretType,tags,topics,ttl,versionAliases,versionDestroyTtl", "updateMask", ""}, + {"secretmanager.projects.locations.secrets.rotateSecret", "POST", "v1/{+parent}:rotateSecret", "", "", ""}, + {"secretmanager.projects.locations.secrets.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"secretmanager.projects.locations.secrets.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"secretmanager.projects.locations.secrets.versions.access", "GET", "v1/{+name}:access", "", "", ""}, + {"secretmanager.projects.locations.secrets.versions.destroy", "POST", "v1/{+name}:destroy", "etag", "", ""}, + {"secretmanager.projects.locations.secrets.versions.disable", "POST", "v1/{+name}:disable", "etag", "", ""}, + {"secretmanager.projects.locations.secrets.versions.enable", "POST", "v1/{+name}:enable", "etag", "", ""}, + {"secretmanager.projects.locations.secrets.versions.get", "GET", "v1/{+name}", "", "", ""}, + {"secretmanager.projects.locations.secrets.versions.list", "GET", "v1/{+parent}/versions", "", "filter,pageSize,pageToken", ""}, + {"secretmanager.projects.secrets.addVersion", "POST", "v1/{+parent}:addVersion", "payload", "", ""}, + {"secretmanager.projects.secrets.create", "POST", "v1/{+parent}/secrets", "annotations,createTime,customerManagedEncryption,etag,expireTime,labels,name,policyMember,replication,rotation,secretType,tags,topics,ttl,versionAliases,versionDestroyTtl", "secretId", ""}, + {"secretmanager.projects.secrets.delete", "DELETE", "v1/{+name}", "", "etag", ""}, + {"secretmanager.projects.secrets.enableManagedRotation", "POST", "v1/{+parent}:enableManagedRotation", "cloudSqlSingleUserCredentials", "", ""}, + {"secretmanager.projects.secrets.get", "GET", "v1/{+name}", "", "", ""}, + {"secretmanager.projects.secrets.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"secretmanager.projects.secrets.list", "GET", "v1/{+parent}/secrets", "", "filter,pageSize,pageToken", ""}, + {"secretmanager.projects.secrets.patch", "PATCH", "v1/{+name}", "annotations,createTime,customerManagedEncryption,etag,expireTime,labels,name,policyMember,replication,rotation,secretType,tags,topics,ttl,versionAliases,versionDestroyTtl", "updateMask", ""}, + {"secretmanager.projects.secrets.rotateSecret", "POST", "v1/{+parent}:rotateSecret", "", "", ""}, + {"secretmanager.projects.secrets.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"secretmanager.projects.secrets.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"secretmanager.projects.secrets.versions.access", "GET", "v1/{+name}:access", "", "", ""}, + {"secretmanager.projects.secrets.versions.destroy", "POST", "v1/{+name}:destroy", "etag", "", ""}, + {"secretmanager.projects.secrets.versions.disable", "POST", "v1/{+name}:disable", "etag", "", ""}, + {"secretmanager.projects.secrets.versions.enable", "POST", "v1/{+name}:enable", "etag", "", ""}, + {"secretmanager.projects.secrets.versions.get", "GET", "v1/{+name}", "", "", ""}, + {"secretmanager.projects.secrets.versions.list", "GET", "v1/{+parent}/versions", "", "filter,pageSize,pageToken", ""}, +// dataplex v1 revision 20261013 + {"dataplex.organizations.locations.encryptionConfigs.create", "POST", "v1/{+parent}/encryptionConfigs", "createTime,enableMetastoreEncryption,encryptionState,etag,failureDetails,key,name,updateTime", "encryptionConfigId", ""}, + {"dataplex.organizations.locations.encryptionConfigs.delete", "DELETE", "v1/{+name}", "", "etag", ""}, + {"dataplex.organizations.locations.encryptionConfigs.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.organizations.locations.encryptionConfigs.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.organizations.locations.encryptionConfigs.list", "GET", "v1/{+parent}/encryptionConfigs", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.organizations.locations.encryptionConfigs.patch", "PATCH", "v1/{+name}", "createTime,enableMetastoreEncryption,encryptionState,etag,failureDetails,key,name,updateTime", "updateMask", ""}, + {"dataplex.organizations.locations.encryptionConfigs.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.organizations.locations.encryptionConfigs.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.organizations.locations.operations.cancel", "POST", "v1/{+name}:cancel", "", "", ""}, + {"dataplex.organizations.locations.operations.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"dataplex.organizations.locations.operations.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.organizations.locations.operations.list", "GET", "v1/{+name}/operations", "", "filter,pageSize,pageToken,returnPartialSuccess", ""}, + {"dataplex.projects.locations.agentTasks.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.agentTasks.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.agentTasks.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.aspectTypes.create", "POST", "v1/{+parent}/aspectTypes", "authorization,createTime,dataClassification,description,displayName,etag,labels,metadataTemplate,name,transferStatus,uid,updateTime", "aspectTypeId,validateOnly", ""}, + {"dataplex.projects.locations.aspectTypes.delete", "DELETE", "v1/{+name}", "", "etag", ""}, + {"dataplex.projects.locations.aspectTypes.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.aspectTypes.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.aspectTypes.list", "GET", "v1/{+parent}/aspectTypes", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.aspectTypes.patch", "PATCH", "v1/{+name}", "authorization,createTime,dataClassification,description,displayName,etag,labels,metadataTemplate,name,transferStatus,uid,updateTime", "updateMask,validateOnly", ""}, + {"dataplex.projects.locations.aspectTypes.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.aspectTypes.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.changeRequests.approve", "POST", "v1/{+name}:approve", "comment,etag", "", ""}, + {"dataplex.projects.locations.changeRequests.delete", "DELETE", "v1/{+name}", "", "etag", ""}, + {"dataplex.projects.locations.changeRequests.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.changeRequests.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.changeRequests.list", "GET", "v1/{+parent}/changeRequests", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.changeRequests.patch", "PATCH", "v1/{+name}", "approver,author,batchChangeMetadataRequest,batchChangeMetadataResponse,changeType,createEntry,createEntryLink,createGlossary,createGlossaryCategory,createGlossaryTerm,createTime,dataProductAccessRequest,deleteEntry,deleteEntryLink,deleteGlossary,deleteGlossaryCategory,deleteGlossaryTerm,etag,justification,labels,name,rejectionComment,resource,reviewerComment,state,uid,updateEntry,updateGlossary,updateGlossaryCategory,updateGlossaryTerm,updateTime", "updateMask", ""}, + {"dataplex.projects.locations.changeRequests.reject", "POST", "v1/{+name}:reject", "comment,etag", "", ""}, + {"dataplex.projects.locations.changeRequests.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.changeRequests.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.dataAttributeBindings.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.dataAttributeBindings.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.dataAttributeBindings.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.dataDomains.bindings.create", "POST", "v1/{+parent}/bindings", "createTime,name,resource,uid", "dataDomainBindingId,validateOnly", ""}, + {"dataplex.projects.locations.dataDomains.bindings.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.dataDomains.bindings.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.dataDomains.bindings.list", "GET", "v1/{+parent}/bindings", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.dataDomains.create", "POST", "v1/{+parent}/dataDomains", "contacts,createTime,description,displayName,labels,name,parentDataDomain,policyMember,uid,updateTime", "dataDomainId,validateOnly", ""}, + {"dataplex.projects.locations.dataDomains.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.dataDomains.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.dataDomains.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.dataDomains.list", "GET", "v1/{+parent}/dataDomains", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.dataDomains.patch", "PATCH", "v1/{+name}", "contacts,createTime,description,displayName,labels,name,parentDataDomain,policyMember,uid,updateTime", "updateMask,validateOnly", ""}, + {"dataplex.projects.locations.dataDomains.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.dataDomains.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.dataProducts.create", "POST", "v1/{+parent}/dataProducts", "accessApprovalConfig,accessGroups,assetCount,createTime,description,displayName,etag,icon,labels,name,ownerEmails,uid,updateTime", "dataProductId,validateOnly", ""}, + {"dataplex.projects.locations.dataProducts.dataAssets.create", "POST", "v1/{+parent}/dataAssets", "accessGroupConfigs,createTime,etag,labels,name,resource,uid,updateTime", "dataAssetId,validateOnly", ""}, + {"dataplex.projects.locations.dataProducts.dataAssets.delete", "DELETE", "v1/{+name}", "", "etag,validateOnly", ""}, + {"dataplex.projects.locations.dataProducts.dataAssets.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.dataProducts.dataAssets.list", "GET", "v1/{+parent}/dataAssets", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.dataProducts.dataAssets.patch", "PATCH", "v1/{+name}", "accessGroupConfigs,createTime,etag,labels,name,resource,uid,updateTime", "updateMask,validateOnly", ""}, + {"dataplex.projects.locations.dataProducts.delete", "DELETE", "v1/{+name}", "", "etag,validateOnly", ""}, + {"dataplex.projects.locations.dataProducts.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.dataProducts.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.dataProducts.list", "GET", "v1/{+parent}/dataProducts", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.dataProducts.patch", "PATCH", "v1/{+name}", "accessApprovalConfig,accessGroups,assetCount,createTime,description,displayName,etag,icon,labels,name,ownerEmails,uid,updateTime", "updateMask,validateOnly", ""}, + {"dataplex.projects.locations.dataProducts.requestAccess", "POST", "v1/{+parent}:requestAccess", "changeRequest,validateOnly", "", ""}, + {"dataplex.projects.locations.dataProducts.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.dataProducts.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.dataScans.create", "POST", "v1/{+parent}/dataScans", "createTime,data,dataDiscoveryResult,dataDiscoverySpec,dataDocumentationResult,dataDocumentationSpec,dataProfileResult,dataProfileSpec,dataQualityResult,dataQualitySpec,description,displayName,executionIdentity,executionSpec,executionStatus,labels,name,state,type,uid,updateTime", "dataScanId,validateOnly", ""}, + {"dataplex.projects.locations.dataScans.delete", "DELETE", "v1/{+name}", "", "force", ""}, + {"dataplex.projects.locations.dataScans.generateDataQualityRules", "POST", "v1/{+name}:generateDataQualityRules", "", "", ""}, + {"dataplex.projects.locations.dataScans.get", "GET", "v1/{+name}", "", "view", ""}, + {"dataplex.projects.locations.dataScans.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.dataScans.jobs.cancel", "POST", "v1/{+name}:cancel", "", "", ""}, + {"dataplex.projects.locations.dataScans.jobs.generateDataQualityRules", "POST", "v1/{+name}:generateDataQualityRules", "", "", ""}, + {"dataplex.projects.locations.dataScans.jobs.get", "GET", "v1/{+name}", "", "view", ""}, + {"dataplex.projects.locations.dataScans.jobs.list", "GET", "v1/{+parent}/jobs", "", "filter,pageSize,pageToken", ""}, + {"dataplex.projects.locations.dataScans.list", "GET", "v1/{+parent}/dataScans", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.dataScans.patch", "PATCH", "v1/{+name}", "createTime,data,dataDiscoveryResult,dataDiscoverySpec,dataDocumentationResult,dataDocumentationSpec,dataProfileResult,dataProfileSpec,dataQualityResult,dataQualitySpec,description,displayName,executionIdentity,executionSpec,executionStatus,labels,name,state,type,uid,updateTime", "updateMask,validateOnly", ""}, + {"dataplex.projects.locations.dataScans.run", "POST", "v1/{+name}:run", "", "", ""}, + {"dataplex.projects.locations.dataScans.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.dataScans.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.dataTaxonomies.attributes.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.dataTaxonomies.attributes.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.dataTaxonomies.attributes.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.dataTaxonomies.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.dataTaxonomies.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.dataTaxonomies.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.entryGroups.batchChangeMetadata", "POST", "v1/{+name}:batchChangeMetadata", "changes,description,name", "", ""}, + {"dataplex.projects.locations.entryGroups.create", "POST", "v1/{+parent}/entryGroups", "changeRequestConfig,createTime,description,displayName,etag,labels,name,transferStatus,uid,updateTime", "entryGroupId,validateOnly", ""}, + {"dataplex.projects.locations.entryGroups.delete", "DELETE", "v1/{+name}", "", "etag", ""}, + {"dataplex.projects.locations.entryGroups.entries.create", "POST", "v1/{+parent}/entries", "aspects,createTime,entrySource,entryType,fullyQualifiedName,name,parentEntry,updateTime", "entryId", ""}, + {"dataplex.projects.locations.entryGroups.entries.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.entryGroups.entries.get", "GET", "v1/{+name}", "", "aspectTypes,paths,view", ""}, + {"dataplex.projects.locations.entryGroups.entries.list", "GET", "v1/{+parent}/entries", "", "filter,pageSize,pageToken", ""}, + {"dataplex.projects.locations.entryGroups.entries.patch", "PATCH", "v1/{+name}", "aspects,createTime,entrySource,entryType,fullyQualifiedName,name,parentEntry,updateTime", "allowMissing,aspectKeys,deleteMissingAspects,updateMask", ""}, + {"dataplex.projects.locations.entryGroups.entryLinks.create", "POST", "v1/{+parent}/entryLinks", "aspects,createTime,entryLinkType,entryReferences,name,updateTime", "entryLinkId", ""}, + {"dataplex.projects.locations.entryGroups.entryLinks.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.entryGroups.entryLinks.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.entryGroups.entryLinks.patch", "PATCH", "v1/{+name}", "aspects,createTime,entryLinkType,entryReferences,name,updateTime", "allowMissing,aspectKeys", ""}, + {"dataplex.projects.locations.entryGroups.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.entryGroups.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.entryGroups.list", "GET", "v1/{+parent}/entryGroups", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.entryGroups.patch", "PATCH", "v1/{+name}", "changeRequestConfig,createTime,description,displayName,etag,labels,name,transferStatus,uid,updateTime", "updateMask,validateOnly", ""}, + {"dataplex.projects.locations.entryGroups.requestBatchMetadataChange", "POST", "v1/{+name}:requestBatchMetadataChange", "changePayload,justification,labels", "", ""}, + {"dataplex.projects.locations.entryGroups.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.entryGroups.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.entryLinkTypes.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.entryLinkTypes.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.entryLinkTypes.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.entryTypes.create", "POST", "v1/{+parent}/entryTypes", "authorization,createTime,description,displayName,etag,labels,name,platform,requiredAspects,system,typeAliases,uid,updateTime", "entryTypeId,validateOnly", ""}, + {"dataplex.projects.locations.entryTypes.delete", "DELETE", "v1/{+name}", "", "etag", ""}, + {"dataplex.projects.locations.entryTypes.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.entryTypes.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.entryTypes.list", "GET", "v1/{+parent}/entryTypes", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.entryTypes.patch", "PATCH", "v1/{+name}", "authorization,createTime,description,displayName,etag,labels,name,platform,requiredAspects,system,typeAliases,uid,updateTime", "updateMask,validateOnly", ""}, + {"dataplex.projects.locations.entryTypes.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.entryTypes.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.glossaries.categories.create", "POST", "v1/{+parent}/categories", "createTime,description,displayName,labels,name,parent,uid,updateTime", "categoryId", ""}, + {"dataplex.projects.locations.glossaries.categories.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.glossaries.categories.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.glossaries.categories.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.glossaries.categories.list", "GET", "v1/{+parent}/categories", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.glossaries.categories.patch", "PATCH", "v1/{+name}", "createTime,description,displayName,labels,name,parent,uid,updateTime", "updateMask", ""}, + {"dataplex.projects.locations.glossaries.categories.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.glossaries.categories.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.glossaries.create", "POST", "v1/{+parent}/glossaries", "categoryCount,createTime,description,displayName,etag,labels,name,termCount,uid,updateTime", "glossaryId,validateOnly", ""}, + {"dataplex.projects.locations.glossaries.delete", "DELETE", "v1/{+name}", "", "etag", ""}, + {"dataplex.projects.locations.glossaries.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.glossaries.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.glossaries.list", "GET", "v1/{+parent}/glossaries", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.glossaries.patch", "PATCH", "v1/{+name}", "categoryCount,createTime,description,displayName,etag,labels,name,termCount,uid,updateTime", "updateMask,validateOnly", ""}, + {"dataplex.projects.locations.glossaries.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.glossaries.terms.create", "POST", "v1/{+parent}/terms", "createTime,description,displayName,labels,name,parent,uid,updateTime", "termId", ""}, + {"dataplex.projects.locations.glossaries.terms.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.glossaries.terms.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.glossaries.terms.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.glossaries.terms.list", "GET", "v1/{+parent}/terms", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.glossaries.terms.patch", "PATCH", "v1/{+name}", "createTime,description,displayName,labels,name,parent,uid,updateTime", "updateMask", ""}, + {"dataplex.projects.locations.glossaries.terms.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.glossaries.terms.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.glossaries.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.governanceRules.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.governanceRules.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.governanceRules.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.lakes.actions.list", "GET", "v1/{+parent}/actions", "", "pageSize,pageToken", ""}, + {"dataplex.projects.locations.lakes.create", "POST", "v1/{+parent}/lakes", "assetStatus,createTime,description,displayName,labels,metastore,metastoreStatus,name,serviceAccount,state,uid,updateTime", "lakeId,validateOnly", ""}, + {"dataplex.projects.locations.lakes.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.lakes.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.lakes.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.lakes.list", "GET", "v1/{+parent}/lakes", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.lakes.patch", "PATCH", "v1/{+name}", "assetStatus,createTime,description,displayName,labels,metastore,metastoreStatus,name,serviceAccount,state,uid,updateTime", "updateMask,validateOnly", ""}, + {"dataplex.projects.locations.lakes.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.lakes.tasks.create", "POST", "v1/{+parent}/tasks", "createTime,description,displayName,executionSpec,executionStatus,labels,name,notebook,spark,state,triggerSpec,uid,updateTime", "taskId,validateOnly", ""}, + {"dataplex.projects.locations.lakes.tasks.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.lakes.tasks.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.lakes.tasks.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.lakes.tasks.jobs.cancel", "POST", "v1/{+name}:cancel", "", "", ""}, + {"dataplex.projects.locations.lakes.tasks.jobs.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.lakes.tasks.jobs.list", "GET", "v1/{+parent}/jobs", "", "pageSize,pageToken", ""}, + {"dataplex.projects.locations.lakes.tasks.list", "GET", "v1/{+parent}/tasks", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.lakes.tasks.patch", "PATCH", "v1/{+name}", "createTime,description,displayName,executionSpec,executionStatus,labels,name,notebook,spark,state,triggerSpec,uid,updateTime", "updateMask,validateOnly", ""}, + {"dataplex.projects.locations.lakes.tasks.run", "POST", "v1/{+name}:run", "args,labels", "", ""}, + {"dataplex.projects.locations.lakes.tasks.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.lakes.tasks.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.lakes.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.lakes.zones.actions.list", "GET", "v1/{+parent}/actions", "", "pageSize,pageToken", ""}, + {"dataplex.projects.locations.lakes.zones.assets.actions.list", "GET", "v1/{+parent}/actions", "", "pageSize,pageToken", ""}, + {"dataplex.projects.locations.lakes.zones.assets.create", "POST", "v1/{+parent}/assets", "createTime,description,discoverySpec,discoveryStatus,displayName,labels,name,resourceSpec,resourceStatus,securityStatus,state,uid,updateTime", "assetId,validateOnly", ""}, + {"dataplex.projects.locations.lakes.zones.assets.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.lakes.zones.assets.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.lakes.zones.assets.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.lakes.zones.assets.list", "GET", "v1/{+parent}/assets", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.lakes.zones.assets.patch", "PATCH", "v1/{+name}", "createTime,description,discoverySpec,discoveryStatus,displayName,labels,name,resourceSpec,resourceStatus,securityStatus,state,uid,updateTime", "updateMask,validateOnly", ""}, + {"dataplex.projects.locations.lakes.zones.assets.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.lakes.zones.assets.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.lakes.zones.create", "POST", "v1/{+parent}/zones", "assetStatus,createTime,description,discoverySpec,displayName,labels,name,resourceSpec,state,type,uid,updateTime", "validateOnly,zoneId", ""}, + {"dataplex.projects.locations.lakes.zones.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.lakes.zones.entities.create", "POST", "v1/{+parent}/entities", "access,asset,catalogEntry,compatibility,createTime,dataPath,dataPathPattern,description,displayName,etag,format,id,name,schema,system,type,uid,updateTime", "validateOnly", ""}, + {"dataplex.projects.locations.lakes.zones.entities.delete", "DELETE", "v1/{+name}", "", "etag", ""}, + {"dataplex.projects.locations.lakes.zones.entities.get", "GET", "v1/{+name}", "", "view", ""}, + {"dataplex.projects.locations.lakes.zones.entities.list", "GET", "v1/{+parent}/entities", "", "filter,pageSize,pageToken,view", ""}, + {"dataplex.projects.locations.lakes.zones.entities.partitions.create", "POST", "v1/{+parent}/partitions", "etag,location,name,values", "validateOnly", ""}, + {"dataplex.projects.locations.lakes.zones.entities.partitions.delete", "DELETE", "v1/{+name}", "", "etag", ""}, + {"dataplex.projects.locations.lakes.zones.entities.partitions.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.lakes.zones.entities.partitions.list", "GET", "v1/{+parent}/partitions", "", "filter,pageSize,pageToken", ""}, + {"dataplex.projects.locations.lakes.zones.entities.update", "PUT", "v1/{+name}", "access,asset,catalogEntry,compatibility,createTime,dataPath,dataPathPattern,description,displayName,etag,format,id,name,schema,system,type,uid,updateTime", "validateOnly", ""}, + {"dataplex.projects.locations.lakes.zones.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.lakes.zones.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"dataplex.projects.locations.lakes.zones.list", "GET", "v1/{+parent}/zones", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.lakes.zones.patch", "PATCH", "v1/{+name}", "assetStatus,createTime,description,discoverySpec,displayName,labels,name,resourceSpec,state,type,uid,updateTime", "updateMask,validateOnly", ""}, + {"dataplex.projects.locations.lakes.zones.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"dataplex.projects.locations.lakes.zones.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"dataplex.projects.locations.list", "GET", "v1/{+name}/locations", "", "extraLocationTypes,filter,pageSize,pageToken", ""}, + {"dataplex.projects.locations.lookupContext", "POST", "v1/{+name}:lookupContext", "options,resources", "", ""}, + {"dataplex.projects.locations.lookupEntry", "GET", "v1/{+name}:lookupEntry", "", "aspectTypes,entry,paths,view", ""}, + {"dataplex.projects.locations.lookupEntryLinks", "GET", "v1/{+name}:lookupEntryLinks", "", "entry,entryLinkTypes,entryMode,pageSize,pageToken", ""}, + {"dataplex.projects.locations.metadataFeeds.create", "POST", "v1/{+parent}/metadataFeeds", "createTime,filters,labels,name,pubsubTopic,scope,uid,updateTime", "metadataFeedId,validateOnly", ""}, + {"dataplex.projects.locations.metadataFeeds.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.metadataFeeds.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.metadataFeeds.list", "GET", "v1/{+parent}/metadataFeeds", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.metadataFeeds.patch", "PATCH", "v1/{+name}", "createTime,filters,labels,name,pubsubTopic,scope,uid,updateTime", "updateMask,validateOnly", ""}, + {"dataplex.projects.locations.metadataJobs.cancel", "POST", "v1/{+name}:cancel", "", "", ""}, + {"dataplex.projects.locations.metadataJobs.create", "POST", "v1/{+parent}/metadataJobs", "createTime,exportResult,exportSpec,importResult,importSpec,labels,name,status,type,uid,updateTime", "metadataJobId,validateOnly", ""}, + {"dataplex.projects.locations.metadataJobs.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.metadataJobs.list", "GET", "v1/{+parent}/metadataJobs", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dataplex.projects.locations.modifyEntry", "POST", "v1/{+name}:modifyEntry", "aspectKeys,deleteMissingAspects,entry,updateMask", "", ""}, + {"dataplex.projects.locations.operations.cancel", "POST", "v1/{+name}:cancel", "", "", ""}, + {"dataplex.projects.locations.operations.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.operations.get", "GET", "v1/{+name}", "", "", ""}, + {"dataplex.projects.locations.operations.list", "GET", "v1/{+name}/operations", "", "filter,pageSize,pageToken,returnPartialSuccess", ""}, + {"dataplex.projects.locations.searchEntries", "POST", "v1/{+name}:searchEntries", "", "contexts,orderBy,pageSize,pageToken,query,scope,semanticSearch", ""}, +// dlp v2 revision 20261020 + {"dlp.infoTypes.list", "GET", "v2/infoTypes", "", "filter,languageCode,locationId,parent", ""}, + {"dlp.locations.infoTypes.list", "GET", "v2/{+parent}/infoTypes", "", "filter,languageCode,locationId", ""}, + {"dlp.organizations.deidentifyTemplates.create", "POST", "v2/{+parent}/deidentifyTemplates", "deidentifyTemplate,locationId,templateId", "", ""}, + {"dlp.organizations.deidentifyTemplates.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.organizations.deidentifyTemplates.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.organizations.deidentifyTemplates.list", "GET", "v2/{+parent}/deidentifyTemplates", "", "locationId,orderBy,pageSize,pageToken", ""}, + {"dlp.organizations.deidentifyTemplates.patch", "PATCH", "v2/{+name}", "deidentifyTemplate,updateMask", "", ""}, + {"dlp.organizations.inspectTemplates.create", "POST", "v2/{+parent}/inspectTemplates", "inspectTemplate,locationId,templateId", "", ""}, + {"dlp.organizations.inspectTemplates.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.organizations.inspectTemplates.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.organizations.inspectTemplates.list", "GET", "v2/{+parent}/inspectTemplates", "", "locationId,orderBy,pageSize,pageToken", ""}, + {"dlp.organizations.inspectTemplates.patch", "PATCH", "v2/{+name}", "inspectTemplate,updateMask", "", ""}, + {"dlp.organizations.locations.columnDataProfiles.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.columnDataProfiles.list", "GET", "v2/{+parent}/columnDataProfiles", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dlp.organizations.locations.connections.create", "POST", "v2/{+parent}/connections", "connection", "", ""}, + {"dlp.organizations.locations.connections.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.connections.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.connections.list", "GET", "v2/{+parent}/connections", "", "filter,pageSize,pageToken", ""}, + {"dlp.organizations.locations.connections.patch", "PATCH", "v2/{+name}", "connection,updateMask", "", ""}, + {"dlp.organizations.locations.connections.search", "GET", "v2/{+parent}/connections:search", "", "filter,pageSize,pageToken", ""}, + {"dlp.organizations.locations.deidentifyTemplates.create", "POST", "v2/{+parent}/deidentifyTemplates", "deidentifyTemplate,locationId,templateId", "", ""}, + {"dlp.organizations.locations.deidentifyTemplates.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.deidentifyTemplates.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.deidentifyTemplates.list", "GET", "v2/{+parent}/deidentifyTemplates", "", "locationId,orderBy,pageSize,pageToken", ""}, + {"dlp.organizations.locations.deidentifyTemplates.patch", "PATCH", "v2/{+name}", "deidentifyTemplate,updateMask", "", ""}, + {"dlp.organizations.locations.discoveryConfigs.create", "POST", "v2/{+parent}/discoveryConfigs", "configId,discoveryConfig", "", ""}, + {"dlp.organizations.locations.discoveryConfigs.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.discoveryConfigs.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.discoveryConfigs.list", "GET", "v2/{+parent}/discoveryConfigs", "", "orderBy,pageSize,pageToken", ""}, + {"dlp.organizations.locations.discoveryConfigs.patch", "PATCH", "v2/{+name}", "discoveryConfig,updateMask", "", ""}, + {"dlp.organizations.locations.dlpJobs.list", "GET", "v2/{+parent}/dlpJobs", "", "filter,locationId,orderBy,pageSize,pageToken,type", ""}, + {"dlp.organizations.locations.fileStoreDataProfiles.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.fileStoreDataProfiles.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.fileStoreDataProfiles.list", "GET", "v2/{+parent}/fileStoreDataProfiles", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dlp.organizations.locations.infoTypes.list", "GET", "v2/{+parent}/infoTypes", "", "filter,languageCode,locationId", ""}, + {"dlp.organizations.locations.inspectTemplates.create", "POST", "v2/{+parent}/inspectTemplates", "inspectTemplate,locationId,templateId", "", ""}, + {"dlp.organizations.locations.inspectTemplates.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.inspectTemplates.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.inspectTemplates.list", "GET", "v2/{+parent}/inspectTemplates", "", "locationId,orderBy,pageSize,pageToken", ""}, + {"dlp.organizations.locations.inspectTemplates.patch", "PATCH", "v2/{+name}", "inspectTemplate,updateMask", "", ""}, + {"dlp.organizations.locations.jobTriggers.create", "POST", "v2/{+parent}/jobTriggers", "jobTrigger,locationId,triggerId", "", ""}, + {"dlp.organizations.locations.jobTriggers.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.jobTriggers.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.jobTriggers.list", "GET", "v2/{+parent}/jobTriggers", "", "filter,locationId,orderBy,pageSize,pageToken,type", ""}, + {"dlp.organizations.locations.jobTriggers.patch", "PATCH", "v2/{+name}", "jobTrigger,updateMask", "", ""}, + {"dlp.organizations.locations.projectDataProfiles.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.projectDataProfiles.list", "GET", "v2/{+parent}/projectDataProfiles", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dlp.organizations.locations.storedInfoTypes.create", "POST", "v2/{+parent}/storedInfoTypes", "config,locationId,storedInfoTypeId", "", ""}, + {"dlp.organizations.locations.storedInfoTypes.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.storedInfoTypes.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.storedInfoTypes.list", "GET", "v2/{+parent}/storedInfoTypes", "", "locationId,orderBy,pageSize,pageToken", ""}, + {"dlp.organizations.locations.storedInfoTypes.patch", "PATCH", "v2/{+name}", "config,updateMask", "", ""}, + {"dlp.organizations.locations.tableDataProfiles.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.tableDataProfiles.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.organizations.locations.tableDataProfiles.list", "GET", "v2/{+parent}/tableDataProfiles", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dlp.organizations.storedInfoTypes.create", "POST", "v2/{+parent}/storedInfoTypes", "config,locationId,storedInfoTypeId", "", ""}, + {"dlp.organizations.storedInfoTypes.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.organizations.storedInfoTypes.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.organizations.storedInfoTypes.list", "GET", "v2/{+parent}/storedInfoTypes", "", "locationId,orderBy,pageSize,pageToken", ""}, + {"dlp.organizations.storedInfoTypes.patch", "PATCH", "v2/{+name}", "config,updateMask", "", ""}, + {"dlp.projects.content.deidentify", "POST", "v2/{+parent}/content:deidentify", "deidentifyConfig,deidentifyTemplateName,inspectConfig,inspectTemplateName,item,locationId", "", ""}, + {"dlp.projects.content.inspect", "POST", "v2/{+parent}/content:inspect", "inspectConfig,inspectTemplateName,item,locationId", "", ""}, + {"dlp.projects.content.reidentify", "POST", "v2/{+parent}/content:reidentify", "inspectConfig,inspectTemplateName,item,locationId,reidentifyConfig,reidentifyTemplateName", "", ""}, + {"dlp.projects.deidentifyTemplates.create", "POST", "v2/{+parent}/deidentifyTemplates", "deidentifyTemplate,locationId,templateId", "", ""}, + {"dlp.projects.deidentifyTemplates.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.deidentifyTemplates.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.deidentifyTemplates.list", "GET", "v2/{+parent}/deidentifyTemplates", "", "locationId,orderBy,pageSize,pageToken", ""}, + {"dlp.projects.deidentifyTemplates.patch", "PATCH", "v2/{+name}", "deidentifyTemplate,updateMask", "", ""}, + {"dlp.projects.dlpJobs.cancel", "POST", "v2/{+name}:cancel", "", "", ""}, + {"dlp.projects.dlpJobs.create", "POST", "v2/{+parent}/dlpJobs", "inspectJob,jobId,locationId,riskJob", "", ""}, + {"dlp.projects.dlpJobs.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.dlpJobs.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.dlpJobs.list", "GET", "v2/{+parent}/dlpJobs", "", "filter,locationId,orderBy,pageSize,pageToken,type", ""}, + {"dlp.projects.image.redact", "POST", "v2/{+parent}/image:redact", "byteItem,deidentifyTemplate,imageRedactionConfigs,includeFindings,inspectConfig,inspectTemplate,locationId", "", ""}, + {"dlp.projects.inspectTemplates.create", "POST", "v2/{+parent}/inspectTemplates", "inspectTemplate,locationId,templateId", "", ""}, + {"dlp.projects.inspectTemplates.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.inspectTemplates.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.inspectTemplates.list", "GET", "v2/{+parent}/inspectTemplates", "", "locationId,orderBy,pageSize,pageToken", ""}, + {"dlp.projects.inspectTemplates.patch", "PATCH", "v2/{+name}", "inspectTemplate,updateMask", "", ""}, + {"dlp.projects.jobTriggers.activate", "POST", "v2/{+name}:activate", "", "", ""}, + {"dlp.projects.jobTriggers.create", "POST", "v2/{+parent}/jobTriggers", "jobTrigger,locationId,triggerId", "", ""}, + {"dlp.projects.jobTriggers.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.jobTriggers.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.jobTriggers.list", "GET", "v2/{+parent}/jobTriggers", "", "filter,locationId,orderBy,pageSize,pageToken,type", ""}, + {"dlp.projects.jobTriggers.patch", "PATCH", "v2/{+name}", "jobTrigger,updateMask", "", ""}, + {"dlp.projects.locations.columnDataProfiles.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.columnDataProfiles.list", "GET", "v2/{+parent}/columnDataProfiles", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dlp.projects.locations.connections.create", "POST", "v2/{+parent}/connections", "connection", "", ""}, + {"dlp.projects.locations.connections.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.connections.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.connections.list", "GET", "v2/{+parent}/connections", "", "filter,pageSize,pageToken", ""}, + {"dlp.projects.locations.connections.patch", "PATCH", "v2/{+name}", "connection,updateMask", "", ""}, + {"dlp.projects.locations.connections.search", "GET", "v2/{+parent}/connections:search", "", "filter,pageSize,pageToken", ""}, + {"dlp.projects.locations.content.deidentify", "POST", "v2/{+parent}/content:deidentify", "deidentifyConfig,deidentifyTemplateName,inspectConfig,inspectTemplateName,item,locationId", "", ""}, + {"dlp.projects.locations.content.inspect", "POST", "v2/{+parent}/content:inspect", "inspectConfig,inspectTemplateName,item,locationId", "", ""}, + {"dlp.projects.locations.content.reidentify", "POST", "v2/{+parent}/content:reidentify", "inspectConfig,inspectTemplateName,item,locationId,reidentifyConfig,reidentifyTemplateName", "", ""}, + {"dlp.projects.locations.contentPolicies.create", "POST", "v2/{+parent}/contentPolicies", "contentPolicy,contentPolicyId", "", ""}, + {"dlp.projects.locations.contentPolicies.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.contentPolicies.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.contentPolicies.list", "GET", "v2/{+parent}/contentPolicies", "", "pageSize,pageToken", ""}, + {"dlp.projects.locations.contentPolicies.patch", "PATCH", "v2/{+name}", "contentPolicy,updateMask", "", ""}, + {"dlp.projects.locations.deidentifyTemplates.create", "POST", "v2/{+parent}/deidentifyTemplates", "deidentifyTemplate,locationId,templateId", "", ""}, + {"dlp.projects.locations.deidentifyTemplates.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.deidentifyTemplates.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.deidentifyTemplates.list", "GET", "v2/{+parent}/deidentifyTemplates", "", "locationId,orderBy,pageSize,pageToken", ""}, + {"dlp.projects.locations.deidentifyTemplates.patch", "PATCH", "v2/{+name}", "deidentifyTemplate,updateMask", "", ""}, + {"dlp.projects.locations.discoveryConfigs.create", "POST", "v2/{+parent}/discoveryConfigs", "configId,discoveryConfig", "", ""}, + {"dlp.projects.locations.discoveryConfigs.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.discoveryConfigs.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.discoveryConfigs.list", "GET", "v2/{+parent}/discoveryConfigs", "", "orderBy,pageSize,pageToken", ""}, + {"dlp.projects.locations.discoveryConfigs.patch", "PATCH", "v2/{+name}", "discoveryConfig,updateMask", "", ""}, + {"dlp.projects.locations.dlpJobs.cancel", "POST", "v2/{+name}:cancel", "", "", ""}, + {"dlp.projects.locations.dlpJobs.create", "POST", "v2/{+parent}/dlpJobs", "inspectJob,jobId,locationId,riskJob", "", ""}, + {"dlp.projects.locations.dlpJobs.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.dlpJobs.finish", "POST", "v2/{+name}:finish", "", "", ""}, + {"dlp.projects.locations.dlpJobs.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.dlpJobs.hybridInspect", "POST", "v2/{+name}:hybridInspect", "hybridItem", "", ""}, + {"dlp.projects.locations.dlpJobs.list", "GET", "v2/{+parent}/dlpJobs", "", "filter,locationId,orderBy,pageSize,pageToken,type", ""}, + {"dlp.projects.locations.fileStoreDataProfiles.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.fileStoreDataProfiles.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.fileStoreDataProfiles.list", "GET", "v2/{+parent}/fileStoreDataProfiles", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dlp.projects.locations.image.redact", "POST", "v2/{+parent}/image:redact", "byteItem,deidentifyTemplate,imageRedactionConfigs,includeFindings,inspectConfig,inspectTemplate,locationId", "", ""}, + {"dlp.projects.locations.infoTypes.list", "GET", "v2/{+parent}/infoTypes", "", "filter,languageCode,locationId", ""}, + {"dlp.projects.locations.inspectTemplates.create", "POST", "v2/{+parent}/inspectTemplates", "inspectTemplate,locationId,templateId", "", ""}, + {"dlp.projects.locations.inspectTemplates.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.inspectTemplates.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.inspectTemplates.list", "GET", "v2/{+parent}/inspectTemplates", "", "locationId,orderBy,pageSize,pageToken", ""}, + {"dlp.projects.locations.inspectTemplates.patch", "PATCH", "v2/{+name}", "inspectTemplate,updateMask", "", ""}, + {"dlp.projects.locations.jobTriggers.activate", "POST", "v2/{+name}:activate", "", "", ""}, + {"dlp.projects.locations.jobTriggers.create", "POST", "v2/{+parent}/jobTriggers", "jobTrigger,locationId,triggerId", "", ""}, + {"dlp.projects.locations.jobTriggers.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.jobTriggers.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.jobTriggers.hybridInspect", "POST", "v2/{+name}:hybridInspect", "hybridItem", "", ""}, + {"dlp.projects.locations.jobTriggers.list", "GET", "v2/{+parent}/jobTriggers", "", "filter,locationId,orderBy,pageSize,pageToken,type", ""}, + {"dlp.projects.locations.jobTriggers.patch", "PATCH", "v2/{+name}", "jobTrigger,updateMask", "", ""}, + {"dlp.projects.locations.projectDataProfiles.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.projectDataProfiles.list", "GET", "v2/{+parent}/projectDataProfiles", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dlp.projects.locations.storedInfoTypes.create", "POST", "v2/{+parent}/storedInfoTypes", "config,locationId,storedInfoTypeId", "", ""}, + {"dlp.projects.locations.storedInfoTypes.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.storedInfoTypes.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.storedInfoTypes.list", "GET", "v2/{+parent}/storedInfoTypes", "", "locationId,orderBy,pageSize,pageToken", ""}, + {"dlp.projects.locations.storedInfoTypes.patch", "PATCH", "v2/{+name}", "config,updateMask", "", ""}, + {"dlp.projects.locations.tableDataProfiles.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.tableDataProfiles.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.locations.tableDataProfiles.list", "GET", "v2/{+parent}/tableDataProfiles", "", "filter,orderBy,pageSize,pageToken", ""}, + {"dlp.projects.storedInfoTypes.create", "POST", "v2/{+parent}/storedInfoTypes", "config,locationId,storedInfoTypeId", "", ""}, + {"dlp.projects.storedInfoTypes.delete", "DELETE", "v2/{+name}", "", "", ""}, + {"dlp.projects.storedInfoTypes.get", "GET", "v2/{+name}", "", "", ""}, + {"dlp.projects.storedInfoTypes.list", "GET", "v2/{+parent}/storedInfoTypes", "", "locationId,orderBy,pageSize,pageToken", ""}, + {"dlp.projects.storedInfoTypes.patch", "PATCH", "v2/{+name}", "config,updateMask", "", ""}, +// cloudkms v1 revision 20261016 + {"cloudkms.folders.getAutokeyConfig", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.folders.getKajPolicyConfig", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.folders.showEffectiveAutokeyConfig", "GET", "v1/{+parent}:showEffectiveAutokeyConfig", "", "", ""}, + {"cloudkms.folders.updateAutokeyConfig", "PATCH", "v1/{+name}", "etag,keyProject,keyProjectResolutionMode,name,state", "updateMask", ""}, + {"cloudkms.folders.updateKajPolicyConfig", "PATCH", "v1/{+name}", "defaultKeyAccessJustificationPolicy,defaultPolicyAvailable,name", "updateMask", ""}, + {"cloudkms.organizations.getKajPolicyConfig", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.organizations.updateKajPolicyConfig", "PATCH", "v1/{+name}", "defaultKeyAccessJustificationPolicy,defaultPolicyAvailable,name", "updateMask", ""}, + {"cloudkms.projects.getAutokeyConfig", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.getKajPolicyConfig", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.ekmConfig.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"cloudkms.projects.locations.ekmConfig.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"cloudkms.projects.locations.ekmConfig.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"cloudkms.projects.locations.ekmConnections.create", "POST", "v1/{+parent}/ekmConnections", "createTime,cryptoSpacePath,etag,keyManagementMode,name,serviceResolvers", "ekmConnectionId", ""}, + {"cloudkms.projects.locations.ekmConnections.get", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.ekmConnections.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"cloudkms.projects.locations.ekmConnections.list", "GET", "v1/{+parent}/ekmConnections", "", "filter,orderBy,pageSize,pageToken", ""}, + {"cloudkms.projects.locations.ekmConnections.patch", "PATCH", "v1/{+name}", "createTime,cryptoSpacePath,etag,keyManagementMode,name,serviceResolvers", "updateMask", ""}, + {"cloudkms.projects.locations.ekmConnections.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"cloudkms.projects.locations.ekmConnections.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"cloudkms.projects.locations.ekmConnections.verifyConnectivity", "GET", "v1/{+name}:verifyConnectivity", "", "", ""}, + {"cloudkms.projects.locations.generateRandomBytes", "POST", "v1/{+location}:generateRandomBytes", "lengthBytes,protectionLevel", "", ""}, + {"cloudkms.projects.locations.get", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.getEkmConfig", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.keyHandles.create", "POST", "v1/{+parent}/keyHandles", "kmsKey,name,resourceTypeSelector", "keyHandleId", ""}, + {"cloudkms.projects.locations.keyHandles.get", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.keyHandles.list", "GET", "v1/{+parent}/keyHandles", "", "filter,pageSize,pageToken", ""}, + {"cloudkms.projects.locations.keyRings.create", "POST", "v1/{+parent}/keyRings", "createTime,name", "keyRingId", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.create", "POST", "v1/{+parent}/cryptoKeys", "createTime,cryptoKeyBackend,destroyScheduledDuration,importOnly,keyAccessJustificationsPolicy,labels,name,nextRotationTime,primary,purpose,rotationPeriod,versionTemplate", "cryptoKeyId,skipInitialVersionCreation,trustedWrappingEnabled", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.asymmetricDecrypt", "POST", "v1/{+name}:asymmetricDecrypt", "ciphertext,ciphertextCrc32c", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.asymmetricSign", "POST", "v1/{+name}:asymmetricSign", "data,dataCrc32c,digest,digestCrc32c", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.create", "POST", "v1/{+parent}/cryptoKeyVersions", "algorithm,attestation,createTime,destroyEventTime,destroyTime,externalDestructionFailureReason,externalProtectionLevelOptions,generateTime,generationFailureReason,hsmTrusted,importFailureReason,importJob,importTime,name,protectionLevel,reimportEligible,state,trustedWrappingEnabled", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.decapsulate", "POST", "v1/{+name}:decapsulate", "ciphertext,ciphertextCrc32c", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.destroy", "POST", "v1/{+name}:destroy", "", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.exportTrustedKeyWrappedCryptoKeyVersion", "GET", "v1/{+name}:exportTrustedKeyWrappedCryptoKeyVersion", "", "wrappingKey", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.get", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.getPublicKey", "GET", "v1/{+name}/publicKey", "", "publicKeyFormat", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.import", "POST", "v1/{+parent}/cryptoKeyVersions:import", "algorithm,cryptoKeyVersion,importJob,rsaAesWrappedKey,trustedWrappingEnabled,wrappedKey", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.importTrustedKeyWrappedCryptoKeyVersion", "POST", "v1/{+parent}/cryptoKeyVersions:importTrustedKeyWrappedCryptoKeyVersion", "algorithm,cryptoKeyVersion,importingKey,wrappedKey", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.list", "GET", "v1/{+parent}/cryptoKeyVersions", "", "filter,orderBy,pageSize,pageToken,view", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.macSign", "POST", "v1/{+name}:macSign", "data,dataCrc32c", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.macVerify", "POST", "v1/{+name}:macVerify", "data,dataCrc32c,mac,macCrc32c", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.patch", "PATCH", "v1/{+name}", "algorithm,attestation,createTime,destroyEventTime,destroyTime,externalDestructionFailureReason,externalProtectionLevelOptions,generateTime,generationFailureReason,hsmTrusted,importFailureReason,importJob,importTime,name,protectionLevel,reimportEligible,state,trustedWrappingEnabled", "updateMask", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.rawDecrypt", "POST", "v1/{+name}:rawDecrypt", "additionalAuthenticatedData,additionalAuthenticatedDataCrc32c,ciphertext,ciphertextCrc32c,initializationVector,initializationVectorCrc32c,tagLength", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.rawEncrypt", "POST", "v1/{+name}:rawEncrypt", "additionalAuthenticatedData,additionalAuthenticatedDataCrc32c,initializationVector,initializationVectorCrc32c,plaintext,plaintextCrc32c", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.restore", "POST", "v1/{+name}:restore", "", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.decrypt", "POST", "v1/{+name}:decrypt", "additionalAuthenticatedData,additionalAuthenticatedDataCrc32c,ciphertext,ciphertextCrc32c", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.encrypt", "POST", "v1/{+name}:encrypt", "additionalAuthenticatedData,additionalAuthenticatedDataCrc32c,plaintext,plaintextCrc32c", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.get", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.list", "GET", "v1/{+parent}/cryptoKeys", "", "filter,orderBy,pageSize,pageToken,versionView", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.patch", "PATCH", "v1/{+name}", "createTime,cryptoKeyBackend,destroyScheduledDuration,importOnly,keyAccessJustificationsPolicy,labels,name,nextRotationTime,primary,purpose,rotationPeriod,versionTemplate", "updateMask", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"cloudkms.projects.locations.keyRings.cryptoKeys.updatePrimaryVersion", "POST", "v1/{+name}:updatePrimaryVersion", "cryptoKeyVersionId", "", ""}, + {"cloudkms.projects.locations.keyRings.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.keyRings.get", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.keyRings.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"cloudkms.projects.locations.keyRings.importJobs.create", "POST", "v1/{+parent}/importJobs", "attestation,createTime,cryptoKeyBackend,expireEventTime,expireTime,generateTime,importMethod,name,protectionLevel,publicKey,publicKeyFormat,state", "importJobId", ""}, + {"cloudkms.projects.locations.keyRings.importJobs.get", "GET", "v1/{+name}", "", "publicKeyFormat", ""}, + {"cloudkms.projects.locations.keyRings.importJobs.getIamPolicy", "GET", "v1/{+resource}:getIamPolicy", "", "options.requestedPolicyVersion", ""}, + {"cloudkms.projects.locations.keyRings.importJobs.list", "GET", "v1/{+parent}/importJobs", "", "filter,orderBy,pageSize,pageToken", ""}, + {"cloudkms.projects.locations.keyRings.importJobs.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"cloudkms.projects.locations.keyRings.importJobs.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"cloudkms.projects.locations.keyRings.list", "GET", "v1/{+parent}/keyRings", "", "filter,orderBy,pageSize,pageToken", ""}, + {"cloudkms.projects.locations.keyRings.setIamPolicy", "POST", "v1/{+resource}:setIamPolicy", "policy,updateMask", "", ""}, + {"cloudkms.projects.locations.keyRings.testIamPermissions", "POST", "v1/{+resource}:testIamPermissions", "permissions", "", ""}, + {"cloudkms.projects.locations.list", "GET", "v1/{+name}/locations", "", "extraLocationTypes,filter,pageSize,pageToken", ""}, + {"cloudkms.projects.locations.operations.get", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.retiredResources.get", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.retiredResources.list", "GET", "v1/{+parent}/retiredResources", "", "pageSize,pageToken", ""}, + {"cloudkms.projects.locations.singleTenantHsmInstances.create", "POST", "v1/{+parent}/singleTenantHsmInstances", "createTime,deleteTime,disableTime,keyPortabilityEnabled,name,quorumAuth,state,unrefreshedDurationUntilDisable", "singleTenantHsmInstanceId", ""}, + {"cloudkms.projects.locations.singleTenantHsmInstances.get", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.singleTenantHsmInstances.list", "GET", "v1/{+parent}/singleTenantHsmInstances", "", "filter,orderBy,pageSize,pageToken,showDeleted", ""}, + {"cloudkms.projects.locations.singleTenantHsmInstances.proposals.approve", "POST", "v1/{+name}:approve", "quorumReply,requiredActionQuorumReply", "", ""}, + {"cloudkms.projects.locations.singleTenantHsmInstances.proposals.create", "POST", "v1/{+parent}/proposals", "addQuorumMember,createTime,deleteSingleTenantHsmInstance,deleteTime,disableSingleTenantHsmInstance,enableSingleTenantHsmInstance,expireTime,failureReason,name,purgeTime,quorumParameters,refreshSingleTenantHsmInstance,registerTwoFactorAuthKeys,removeQuorumMember,requiredActionQuorumParameters,state,ttl,upgradeKeyTrust", "singleTenantHsmInstanceProposalId", ""}, + {"cloudkms.projects.locations.singleTenantHsmInstances.proposals.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.singleTenantHsmInstances.proposals.execute", "POST", "v1/{+name}:execute", "", "", ""}, + {"cloudkms.projects.locations.singleTenantHsmInstances.proposals.get", "GET", "v1/{+name}", "", "", ""}, + {"cloudkms.projects.locations.singleTenantHsmInstances.proposals.list", "GET", "v1/{+parent}/proposals", "", "filter,orderBy,pageSize,pageToken,showDeleted", ""}, + {"cloudkms.projects.locations.updateEkmConfig", "PATCH", "v1/{+name}", "defaultEkmConnection,name", "updateMask", ""}, + {"cloudkms.projects.showEffectiveAutokeyConfig", "GET", "v1/{+parent}:showEffectiveAutokeyConfig", "", "", ""}, + {"cloudkms.projects.showEffectiveKeyAccessJustificationsEnrollmentConfig", "GET", "v1/{+project}:showEffectiveKeyAccessJustificationsEnrollmentConfig", "", "", ""}, + {"cloudkms.projects.showEffectiveKeyAccessJustificationsPolicyConfig", "GET", "v1/{+project}:showEffectiveKeyAccessJustificationsPolicyConfig", "", "", ""}, + {"cloudkms.projects.updateAutokeyConfig", "PATCH", "v1/{+name}", "etag,keyProject,keyProjectResolutionMode,name,state", "updateMask", ""}, + {"cloudkms.projects.updateKajPolicyConfig", "PATCH", "v1/{+name}", "defaultKeyAccessJustificationPolicy,defaultPolicyAvailable,name", "updateMask", ""}, +// datastore v1 revision 20261013 + {"datastore.projects.allocateIds", "POST", "v1/projects/{projectId}:allocateIds", "databaseId,keys,requestOptions", "", ""}, + {"datastore.projects.beginTransaction", "POST", "v1/projects/{projectId}:beginTransaction", "databaseId,requestOptions,transactionOptions", "", ""}, + {"datastore.projects.commit", "POST", "v1/projects/{projectId}:commit", "databaseId,mode,mutations,requestOptions,singleUseTransaction,transaction", "", ""}, + {"datastore.projects.export", "POST", "v1/projects/{projectId}:export", "entityFilter,labels,outputUrlPrefix", "", ""}, + {"datastore.projects.import", "POST", "v1/projects/{projectId}:import", "entityFilter,inputUrl,labels", "", ""}, + {"datastore.projects.indexes.create", "POST", "v1/projects/{projectId}/indexes", "ancestor,indexId,kind,projectId,properties,state", "", ""}, + {"datastore.projects.indexes.delete", "DELETE", "v1/projects/{projectId}/indexes/{indexId}", "", "", ""}, + {"datastore.projects.indexes.get", "GET", "v1/projects/{projectId}/indexes/{indexId}", "", "", ""}, + {"datastore.projects.indexes.list", "GET", "v1/projects/{projectId}/indexes", "", "filter,pageSize,pageToken", ""}, + {"datastore.projects.lookup", "POST", "v1/projects/{projectId}:lookup", "databaseId,keys,propertyMask,readOptions,requestOptions", "", ""}, + {"datastore.projects.operations.cancel", "POST", "v1/{+name}:cancel", "", "", ""}, + {"datastore.projects.operations.delete", "DELETE", "v1/{+name}", "", "", ""}, + {"datastore.projects.operations.get", "GET", "v1/{+name}", "", "", ""}, + {"datastore.projects.operations.list", "GET", "v1/{+name}/operations", "", "filter,pageSize,pageToken,returnPartialSuccess", ""}, + {"datastore.projects.reserveIds", "POST", "v1/projects/{projectId}:reserveIds", "databaseId,keys,requestOptions", "", ""}, + {"datastore.projects.rollback", "POST", "v1/projects/{projectId}:rollback", "databaseId,requestOptions,transaction", "", ""}, + {"datastore.projects.runAggregationQuery", "POST", "v1/projects/{projectId}:runAggregationQuery", "aggregationQuery,databaseId,explainOptions,gqlQuery,partitionId,readOptions,requestOptions", "", ""}, + {"datastore.projects.runQuery", "POST", "v1/projects/{projectId}:runQuery", "databaseId,explainOptions,gqlQuery,partitionId,propertyMask,query,readOptions,requestOptions", "", ""}, diff --git a/src/google/data/caps.h b/src/google/data/caps.h new file mode 100644 index 0000000..3035fc4 --- /dev/null +++ b/src/google/data/caps.h @@ -0,0 +1,50 @@ +#pragma once + +// Static ceilings for the data package. Never autotuned. + +#include + +namespace molo::google::data { + +struct DataCaps { + // Generic call engine. + static constexpr std::size_t kMaxPathValueChars = 1024; // object name limit + static constexpr std::size_t kMaxResourceNameChars = 1024; // {+name} values + static constexpr std::size_t kMaxQueryParams = 40; + static constexpr std::size_t kMaxQueryValueChars = 2048; + static constexpr std::size_t kMaxQueryArrayItems = 100; + static constexpr std::size_t kMaxBodyBytes = 10u * 1024u * 1024u; // JSON request body + static constexpr int kMaxListPages = 20; + static constexpr int kMaxPageSize = 1000; + // BigQuery. + static constexpr std::size_t kMaxSqlChars = 1024u * 1024u; // Google: 1 MB query text + static constexpr int kMaxQueryTimeoutMs = 60000; + static constexpr int kMaxResultRows = 10000; + static constexpr std::size_t kMaxInsertRows = 10000; // Google: 10,000 rows per insertAll + static constexpr std::size_t kMaxInsertBytes = 10u * 1024u * 1024u; + // Cloud Storage media. + static constexpr std::size_t kMaxDownloadBytes = 16u * 1024u * 1024u; + static constexpr std::size_t kMaxSimpleUploadBytes = 8u * 1024u * 1024u; + static constexpr std::size_t kChunkGranularity = 256u * 1024u; // resumable chunk multiple + static constexpr std::size_t kMaxResumableChunk = 8u * 1024u * 1024u; + static constexpr std::size_t kMaxResumableTotal = 64u * 1024u * 1024u; + static constexpr std::size_t kMaxComposeSources = 32; // Google: 32 per compose + static constexpr long kMinSignedUrlSeconds = 1; + static constexpr long kMaxSignedUrlSeconds = 604800; // V4 maximum: 7 days + static constexpr std::size_t kMaxSignedHeaders = 20; + // Pub/Sub. + static constexpr std::size_t kMaxPublishMessages = 1000; // Google: 1000 per publish + static constexpr std::size_t kMaxMessageBytes = 10u * 1024u * 1024u; + static constexpr int kMaxPullMessages = 100; + static constexpr std::size_t kMaxAckIds = 1000; + // Secret Manager / KMS / DLP / Datastore / Firestore. + static constexpr std::size_t kMaxSecretBytes = 64u * 1024u; // Google: 64 KiB payload + static constexpr std::size_t kMaxCryptoBytes = 64u * 1024u; // KMS plaintext limit + static constexpr std::size_t kMaxDlpItemBytes = 512u * 1024u; // DLP content item + static constexpr std::size_t kMaxKeysPerLookup = 1000; // Datastore lookup + static constexpr std::size_t kMaxWritesPerCommit = 500; // Datastore / Firestore commit + static constexpr std::size_t kMaxFilterChars = 1024; + static constexpr std::size_t kMaxIdChars = 1024; +}; + +} // namespace molo::google::data diff --git a/src/google/data/data.h b/src/google/data/data.h new file mode 100644 index 0000000..efc679e --- /dev/null +++ b/src/google/data/data.h @@ -0,0 +1,52 @@ +#pragma once + +// DataPlatform: one object holding every typed service of the data package +// over a shared client and project context. + +#include + +#include "google/data/bigquery.h" +#include "google/data/bigtable_admin.h" +#include "google/data/dataplex.h" +#include "google/data/datastore.h" +#include "google/data/dlp.h" +#include "google/data/firestore.h" +#include "google/data/kms.h" +#include "google/data/pubsub.h" +#include "google/data/secret_manager.h" +#include "google/data/signed_url.h" +#include "google/data/spanner.h" +#include "google/data/storage.h" +#include "google/data/storage_media.h" + +namespace molo::google::data { + +class DataPlatform { + public: + DataPlatform(std::shared_ptr client, CloudContext context) + : bigquery(client, context), + storage(client, context), + pubsub(client, context), + firestore(client, context), + spanner(client, context), + bigtable(client, context), + secrets(client, context), + dataplex(client, context), + dlp(client, context), + kms(client, context), + datastore(client, context) {} + + BigQuery bigquery; + Storage storage; + PubSub pubsub; + Firestore firestore; + Spanner spanner; + BigtableAdmin bigtable; + SecretManager secrets; + Dataplex dataplex; + Dlp dlp; + Kms kms; + Datastore datastore; +}; + +} // namespace molo::google::data diff --git a/src/google/data/dataplex.cpp b/src/google/data/dataplex.cpp new file mode 100644 index 0000000..45fe168 --- /dev/null +++ b/src/google/data/dataplex.cpp @@ -0,0 +1,64 @@ +#include "google/data/dataplex.h" + +namespace molo::google::data { + +using clients::body::Json; + +Json Dataplex::list_locations(int n) const { + CallArgs a = on("name", project_path()); + a.q("pageSize", page(n)); + return invoke("dataplex.projects.locations.list", a); +} +Json Dataplex::list_lakes(const std::string& l, int n) const { + CallArgs a = on("parent", location_path(l)); + a.q("pageSize", page(n)); + return invoke("dataplex.projects.locations.lakes.list", a); +} +Json Dataplex::get_lake(const std::string& l, const std::string& lake) const { + return invoke("dataplex.projects.locations.lakes.get", + on("name", location_path(l) + "/lakes/" + Ids::id(lake, "lake"))); +} +Json Dataplex::list_entry_groups(const std::string& l, int n) const { + CallArgs a = on("parent", location_path(l)); + a.q("pageSize", page(n)); + return invoke("dataplex.projects.locations.entryGroups.list", a); +} +Json Dataplex::get_entry_group(const std::string& l, const std::string& g) const { + return invoke("dataplex.projects.locations.entryGroups.get", + on("name", location_path(l) + "/entryGroups/" + Ids::id(g, "entry group"))); +} +Json Dataplex::list_entry_types(const std::string& l, int n) const { + CallArgs a = on("parent", location_path(l)); + a.q("pageSize", page(n)); + return invoke("dataplex.projects.locations.entryTypes.list", a); +} +Json Dataplex::list_aspect_types(const std::string& l, int n) const { + CallArgs a = on("parent", location_path(l)); + a.q("pageSize", page(n)); + return invoke("dataplex.projects.locations.aspectTypes.list", a); +} +Json Dataplex::list_data_scans(const std::string& l, int n) const { + CallArgs a = on("parent", location_path(l)); + a.q("pageSize", page(n)); + return invoke("dataplex.projects.locations.dataScans.list", a); +} +Json Dataplex::get_data_scan(const std::string& l, const std::string& s) const { + return invoke("dataplex.projects.locations.dataScans.get", + on("name", location_path(l) + "/dataScans/" + Ids::id(s, "data scan"))); +} +Json Dataplex::search_entries(const std::string& l, const std::string& query, int n) const { + need(!query.empty() && query.size() <= DataCaps::kMaxFilterChars, + "query must be 1.." + std::to_string(DataCaps::kMaxFilterChars) + " characters"); + CallArgs a = on("name", location_path(l)); + a.q("query", Json(query)); + a.q("pageSize", page(n)); + return invoke("dataplex.projects.locations.searchEntries", a); +} +Json Dataplex::lookup_entry(const std::string& l, const std::string& entry) const { + need(entry.rfind("projects/", 0) == 0, "entry must be a full name starting with projects/"); + CallArgs a = on("name", location_path(l)); + a.q("entry", Json(entry)); + return invoke("dataplex.projects.locations.lookupEntry", a); +} + +} // namespace molo::google::data diff --git a/src/google/data/dataplex.h b/src/google/data/dataplex.h new file mode 100644 index 0000000..86214ff --- /dev/null +++ b/src/google/data/dataplex.h @@ -0,0 +1,34 @@ +#pragma once + +// Dataplex Universal Catalog v1: lakes, entry groups, entry types, aspect +// types, data scans, entry search and lookup. Other methods (206 in total): +// ApiService::invoke("dataplex...."). + +#include + +#include "google/data/project_service.h" + +namespace molo::google::data { + +class Dataplex : public ProjectService { + public: + explicit Dataplex(std::shared_ptr client, CloudContext context) + : ProjectService(std::move(client), Api::kDataplex, std::move(context)) {} + + clients::body::Json list_locations(int page_size) const; + clients::body::Json list_lakes(const std::string& location, int page_size) const; + clients::body::Json get_lake(const std::string& location, const std::string& lake) const; + clients::body::Json list_entry_groups(const std::string& location, int page_size) const; + clients::body::Json get_entry_group(const std::string& location, const std::string& group) const; + clients::body::Json list_entry_types(const std::string& location, int page_size) const; + clients::body::Json list_aspect_types(const std::string& location, int page_size) const; + clients::body::Json list_data_scans(const std::string& location, int page_size) const; + clients::body::Json get_data_scan(const std::string& location, const std::string& scan) const; + // Catalog search over the project (POST, read only). + clients::body::Json search_entries(const std::string& location, const std::string& query, + int page_size) const; + // entry: full entry name, e.g. projects/p/locations/l/entryGroups/g/entries/e. + clients::body::Json lookup_entry(const std::string& location, const std::string& entry) const; +}; + +} // namespace molo::google::data diff --git a/src/google/data/datastore.cpp b/src/google/data/datastore.cpp new file mode 100644 index 0000000..54ab2b0 --- /dev/null +++ b/src/google/data/datastore.cpp @@ -0,0 +1,115 @@ +#include "google/data/datastore.h" + +namespace molo::google::data { + +using clients::body::Json; + +Datastore::Datastore(std::shared_ptr client, CloudContext context, + std::string database_id) + : ProjectService(std::move(client), Api::kDatastore, std::move(context)), + database_(std::move(database_id)) { + if (!database_.empty()) Ids::id(database_, "database"); +} + +CallArgs Datastore::base() const { return on("projectId", Ids::project(context_.project)); } + +Json Datastore::body() const { + Json b = Json::object(); + if (!database_.empty()) b.set("databaseId", Json(database_)); + return b; +} + +Json Datastore::lookup(const Json& keys) const { + need(keys.is_array() && !keys.as_array().empty() && + keys.as_array().size() <= DataCaps::kMaxKeysPerLookup, + "keys must be an array of 1.." + std::to_string(DataCaps::kMaxKeysPerLookup)); + Json b = body(); + b.set("keys", keys); + CallArgs a = base(); + a.with_body(b); + return invoke("datastore.projects.lookup", a); +} + +Json Datastore::run_query(const Json& query, const Json& gql) const { + need(query.is_null() != gql.is_null(), "give exactly one of query and gql"); + Json b = body(); + if (!query.is_null()) { + need(query.is_object(), "query must be an object"); + b.set("query", query); + } else { + need(gql.is_object(), "gql must be an object"); + b.set("gqlQuery", gql); + } + CallArgs a = base(); + a.with_body(b); + return invoke("datastore.projects.runQuery", a); +} + +Json Datastore::run_aggregation_query(const Json& agg) const { + need(agg.is_object() && !agg.as_object().empty(), "aggregation query must be an object"); + Json b = body(); + b.set("aggregationQuery", agg); + CallArgs a = base(); + a.with_body(b); + return invoke("datastore.projects.runAggregationQuery", a); +} + +Json Datastore::begin_transaction(bool read_only) const { + Json b = body(); + Json opts = Json::object(); + if (read_only) opts.set("readOnly", Json::object()); + else opts.set("readWrite", Json::object()); + b.set("transactionOptions", opts); + CallArgs a = base(); + a.with_body(b); + return invoke("datastore.projects.beginTransaction", a); +} + +Json Datastore::rollback(const std::string& transaction) const { + need(!transaction.empty() && transaction.size() <= 4096, "transaction handle required"); + Json b = body(); + b.set("transaction", Json(transaction)); + CallArgs a = base(); + a.with_body(b); + return invoke("datastore.projects.rollback", a); +} + +Json Datastore::allocate_ids(const Json& keys) const { + need(keys.is_array() && !keys.as_array().empty() && + keys.as_array().size() <= DataCaps::kMaxKeysPerLookup, + "keys must be an array of 1.." + std::to_string(DataCaps::kMaxKeysPerLookup)); + Json b = body(); + b.set("keys", keys); + CallArgs a = base(); + a.with_body(b); + return invoke("datastore.projects.allocateIds", a); +} + +Json Datastore::reserve_ids(const Json& keys) const { + need(keys.is_array() && !keys.as_array().empty() && + keys.as_array().size() <= DataCaps::kMaxKeysPerLookup, + "keys must be an array of 1.." + std::to_string(DataCaps::kMaxKeysPerLookup)); + Json b = body(); + b.set("keys", keys); + CallArgs a = base(); + a.with_body(b); + return invoke("datastore.projects.reserveIds", a); +} + +Json Datastore::commit(const Json& mutations, const std::string& mode, + const std::string& transaction) const { + need(mutations.is_array() && !mutations.as_array().empty() && + mutations.as_array().size() <= DataCaps::kMaxWritesPerCommit, + "mutations must be an array of 1.." + std::to_string(DataCaps::kMaxWritesPerCommit)); + need(mode == "TRANSACTIONAL" || mode == "NON_TRANSACTIONAL", + "mode must be TRANSACTIONAL or NON_TRANSACTIONAL"); + Json b = body(); + b.set("mutations", mutations); + b.set("mode", Json(mode)); + if (!transaction.empty()) b.set("transaction", Json(transaction)); + CallArgs a = base(); + a.with_body(b); + return invoke("datastore.projects.commit", a); +} + +} // namespace molo::google::data diff --git a/src/google/data/datastore.h b/src/google/data/datastore.h new file mode 100644 index 0000000..572c33c --- /dev/null +++ b/src/google/data/datastore.h @@ -0,0 +1,38 @@ +#pragma once + +// Cloud Datastore v1 (also Firestore in Datastore mode): lookup, queries, +// aggregation, transactions, commits, id allocation. Entities and keys are +// the Datastore JSON shapes and pass through as Json. + +#include + +#include "google/data/project_service.h" + +namespace molo::google::data { + +class Datastore : public ProjectService { + public: + // database_id "" = the default database. + Datastore(std::shared_ptr client, CloudContext context, + std::string database_id = ""); + + clients::body::Json lookup(const clients::body::Json& keys) const; + // query: a Datastore Query object; gql: a GqlQuery object. Exactly one. + clients::body::Json run_query(const clients::body::Json& query, + const clients::body::Json& gql) const; + clients::body::Json run_aggregation_query(const clients::body::Json& aggregation) const; + clients::body::Json begin_transaction(bool read_only) const; + clients::body::Json rollback(const std::string& transaction) const; + clients::body::Json allocate_ids(const clients::body::Json& incomplete_keys) const; + clients::body::Json reserve_ids(const clients::body::Json& keys) const; + // Writes (explicit). mode: TRANSACTIONAL or NON_TRANSACTIONAL. + clients::body::Json commit(const clients::body::Json& mutations, const std::string& mode, + const std::string& transaction) const; + + private: + CallArgs base() const; + clients::body::Json body() const; + std::string database_; +}; + +} // namespace molo::google::data diff --git a/src/google/data/dlp.cpp b/src/google/data/dlp.cpp new file mode 100644 index 0000000..414df03 --- /dev/null +++ b/src/google/data/dlp.cpp @@ -0,0 +1,110 @@ +#include "google/data/dlp.h" + +namespace molo::google::data { + +using clients::body::Json; + +namespace { +Json item(const std::string& text) { + if (text.empty() || text.size() > DataCaps::kMaxDlpItemBytes) + throw GoogleError::invalid("text must be 1.." + std::to_string(DataCaps::kMaxDlpItemBytes) + + " bytes"); + Json i = Json::object(); + i.set("value", Json(text)); + return i; +} +} // namespace + +void InspectRequest::validate() const { + item(text); + static const char* const likelihood[] = {"VERY_UNLIKELY", "UNLIKELY", "POSSIBLE", "LIKELY", + "VERY_LIKELY"}; + bool ok = false; + for (const char* l : likelihood) ok = ok || min_likelihood == l; + if (!ok) throw GoogleError::invalid("min_likelihood is not a DLP likelihood"); + if (max_findings < 1 || max_findings > 1000) + throw GoogleError::invalid("max_findings must be 1..1000"); + if (info_types.size() > 100) throw GoogleError::invalid("at most 100 info types"); + for (const auto& t : info_types) Ids::id(t, "info type"); + if (!location.empty()) Ids::location(location); +} + +Json InspectRequest::to_json() const { + validate(); + Json config = Json::object(); + if (!info_types.empty()) { + Json arr = Json::array(); + for (const auto& t : info_types) { + Json o = Json::object(); + o.set("name", Json(t)); + arr.push(o); + } + config.set("infoTypes", arr); + } + config.set("minLikelihood", Json(min_likelihood)); + config.set("includeQuote", Json(include_quote)); + Json limits = Json::object(); + limits.set("maxFindingsPerRequest", Json(max_findings)); + config.set("limits", limits); + Json b = Json::object(); + b.set("item", item(text)); + b.set("inspectConfig", config); + return b; +} + +std::string Dlp::parent(const std::string& location) const { + return location.empty() ? project_path() : location_path(location); +} + +Json Dlp::inspect_content(const InspectRequest& r) const { + CallArgs a = on("parent", parent(r.location)); + a.with_body(r.to_json()); + return invoke(r.location.empty() ? "dlp.projects.content.inspect" + : "dlp.projects.locations.content.inspect", + a); +} + +Json Dlp::deidentify_content(const std::string& text, const Json& config, + const std::string& location) const { + need(config.is_object() && !config.as_object().empty(), "deidentify config must be an object"); + Json b = Json::object(); + b.set("item", item(text)); + b.set("deidentifyConfig", config); + CallArgs a = on("parent", parent(location)); + a.with_body(b); + return invoke(location.empty() ? "dlp.projects.content.deidentify" + : "dlp.projects.locations.content.deidentify", + a); +} + +Json Dlp::list_info_types(const std::string& language) const { + CallArgs a; + if (!language.empty()) { + need(language.size() <= 16, "language code too long"); + a.q("languageCode", Json(language)); + } + return invoke("dlp.infoTypes.list", a); +} + +Json Dlp::list_inspect_templates(int n) const { + CallArgs a = on("parent", project_path()); + a.q("pageSize", page(n)); + return invoke("dlp.projects.inspectTemplates.list", a); +} +Json Dlp::list_deidentify_templates(int n) const { + CallArgs a = on("parent", project_path()); + a.q("pageSize", page(n)); + return invoke("dlp.projects.deidentifyTemplates.list", a); +} +Json Dlp::list_job_triggers(int n) const { + CallArgs a = on("parent", project_path()); + a.q("pageSize", page(n)); + return invoke("dlp.projects.jobTriggers.list", a); +} +Json Dlp::list_jobs(int n) const { + CallArgs a = on("parent", project_path()); + a.q("pageSize", page(n)); + return invoke("dlp.projects.dlpJobs.list", a); +} + +} // namespace molo::google::data diff --git a/src/google/data/dlp.h b/src/google/data/dlp.h new file mode 100644 index 0000000..7a26112 --- /dev/null +++ b/src/google/data/dlp.h @@ -0,0 +1,46 @@ +#pragma once + +// Cloud Data Loss Prevention v2: inspect and de-identify content, infoTypes, +// templates, job triggers, jobs. Other methods (150 in total): +// ApiService::invoke("dlp...."). + +#include +#include + +#include "google/data/project_service.h" + +namespace molo::google::data { + +struct InspectRequest { + std::string text; // 1..512 KiB + std::vector info_types; // e.g. EMAIL_ADDRESS; empty = Google's default set + std::string min_likelihood = "POSSIBLE"; + bool include_quote = false; + int max_findings = 100; // 1..1000 + std::string location; // "" = global endpoint path projects/{p} + void validate() const; + clients::body::Json to_json() const; +}; + +class Dlp : public ProjectService { + public: + explicit Dlp(std::shared_ptr client, CloudContext context) + : ProjectService(std::move(client), Api::kDlp, std::move(context)) {} + + // content:inspect is a read operation (it returns findings, stores nothing). + clients::body::Json inspect_content(const InspectRequest& request) const; + // content:deidentify: config is a DeidentifyConfig object. + clients::body::Json deidentify_content(const std::string& text, + const clients::body::Json& deidentify_config, + const std::string& location) const; + clients::body::Json list_info_types(const std::string& language_code) const; + clients::body::Json list_inspect_templates(int page_size) const; + clients::body::Json list_deidentify_templates(int page_size) const; + clients::body::Json list_job_triggers(int page_size) const; + clients::body::Json list_jobs(int page_size) const; + + private: + std::string parent(const std::string& location) const; +}; + +} // namespace molo::google::data diff --git a/src/google/data/firestore.cpp b/src/google/data/firestore.cpp new file mode 100644 index 0000000..021cb66 --- /dev/null +++ b/src/google/data/firestore.cpp @@ -0,0 +1,120 @@ +#include "google/data/firestore.h" + +namespace molo::google::data { + +using clients::body::Json; + +Firestore::Firestore(std::shared_ptr client, CloudContext context, + std::string database) + : ProjectService(std::move(client), Api::kFirestore, std::move(context)), + database_(std::move(database)) { + if (database_ != "(default)") Ids::id(database_, "database"); +} + +std::string Firestore::database_name() const { + return project_path() + "/databases/" + database_; +} +std::string Firestore::documents_root() const { return database_name() + "/documents"; } + +std::string Firestore::document_name(const std::string& path) const { + if (path.empty()) return documents_root(); + need(path.size() <= DataCaps::kMaxResourceNameChars, "document path too long"); + std::size_t segments = 1; + for (char c : path) segments += c == '/'; + need(segments <= 100, "document path over 100 segments"); + return documents_root() + "/" + path; // check_resource validates at render time +} + +Json Firestore::list_databases() const { + return invoke("firestore.projects.databases.list", on("parent", project_path())); +} +Json Firestore::get_database() const { + return invoke("firestore.projects.databases.get", on("name", database_name())); +} +Json Firestore::get_document(const std::string& path) const { + need(!path.empty(), "document path is required"); + return invoke("firestore.projects.databases.documents.get", on("name", document_name(path))); +} + +Json Firestore::list_documents(const std::string& parent, const std::string& collection, + int page_size, const std::string& page_token) const { + need(!collection.empty(), "collection is required"); + CallArgs a = on("parent", document_name(parent)); + a.p("collectionId", Ids::id(collection, "collection")); + a.q("pageSize", page(page_size)); + if (!page_token.empty()) a.q("pageToken", Json(page_token)); + return invoke("firestore.projects.databases.documents.list", a); +} + +Json Firestore::list_collection_ids(const std::string& parent, int page_size) const { + CallArgs a = on("parent", document_name(parent)); + Json b = Json::object(); + b.set("pageSize", page(page_size)); + a.with_body(b); + return invoke("firestore.projects.databases.documents.listCollectionIds", a); +} + +Json Firestore::run_query(const std::string& parent, const Json& q) const { + need(q.is_object() && !q.as_object().empty(), "structured query must be a non-empty object"); + CallArgs a = on("parent", document_name(parent)); + Json b = Json::object(); + b.set("structuredQuery", q); + a.with_body(b); + return invoke("firestore.projects.databases.documents.runQuery", a); +} + +Json Firestore::run_aggregation_query(const std::string& parent, const Json& agg) const { + need(agg.is_object() && !agg.as_object().empty(), "aggregation query must be a non-empty object"); + CallArgs a = on("parent", document_name(parent)); + Json b = Json::object(); + b.set("structuredAggregationQuery", agg); + a.with_body(b); + return invoke("firestore.projects.databases.documents.runAggregationQuery", a); +} + +Json Firestore::batch_get(const std::vector& paths) const { + need(!paths.empty() && paths.size() <= 100, "batch_get takes 1..100 documents"); + Json arr = Json::array(); + for (const auto& p : paths) { + need(!p.empty(), "empty document path"); + std::string name = document_name(p); + check_resource("document", name); + arr.push(Json(name)); + } + Json b = Json::object(); + b.set("documents", arr); + CallArgs a = on("database", database_name()); + a.with_body(b); + return invoke("firestore.projects.databases.documents.batchGet", a); +} + +Json Firestore::create_document(const std::string& parent, const std::string& collection, + const std::string& document_id, const Json& fields) const { + need(!collection.empty(), "collection is required"); + need(fields.is_object(), "fields must be an object"); + CallArgs a = on("parent", document_name(parent)); + a.p("collectionId", Ids::id(collection, "collection")); + if (!document_id.empty()) a.q("documentId", Json(Ids::id(document_id, "document id"))); + Json b = Json::object(); + b.set("fields", fields); + a.with_body(b); + return invoke("firestore.projects.databases.documents.createDocument", a); +} + +Json Firestore::commit(const Json& writes) const { + need(writes.is_array() && !writes.as_array().empty() && + writes.as_array().size() <= DataCaps::kMaxWritesPerCommit, + "writes must be an array of 1.." + std::to_string(DataCaps::kMaxWritesPerCommit)); + Json b = Json::object(); + b.set("writes", writes); + CallArgs a = on("database", database_name()); + a.with_body(b); + return invoke("firestore.projects.databases.documents.commit", a); +} + +Json Firestore::delete_document(const std::string& path) const { + need(!path.empty(), "document path is required"); + return invoke("firestore.projects.databases.documents.delete", on("name", document_name(path))); +} + +} // namespace molo::google::data diff --git a/src/google/data/firestore.h b/src/google/data/firestore.h new file mode 100644 index 0000000..b216ae5 --- /dev/null +++ b/src/google/data/firestore.h @@ -0,0 +1,50 @@ +#pragma once + +// Cloud Firestore v1 (native mode): documents, queries, collection ids, +// commits, databases. Other methods: ApiService::invoke("firestore...."). +// Paths are relative to the database's documents root ("users/alice"). + +#include + +#include "google/data/project_service.h" + +namespace molo::google::data { + +class Firestore : public ProjectService { + public: + Firestore(std::shared_ptr client, CloudContext context, + std::string database = "(default)"); + + const std::string& database_id() const { return database_; } + std::string database_name() const; // projects/p/databases/(default) + std::string documents_root() const; // .../documents + // Relative path -> full name; segments validated, 1..100 segments. + std::string document_name(const std::string& path) const; + + clients::body::Json list_databases() const; + clients::body::Json get_database() const; + + clients::body::Json get_document(const std::string& path) const; + // parent "" = documents root; collection is a collection id. + clients::body::Json list_documents(const std::string& parent, const std::string& collection, + int page_size, const std::string& page_token) const; + clients::body::Json list_collection_ids(const std::string& parent, int page_size) const; + // structuredQuery per the Firestore StructuredQuery schema. + clients::body::Json run_query(const std::string& parent, + const clients::body::Json& structured_query) const; + clients::body::Json run_aggregation_query(const std::string& parent, + const clients::body::Json& aggregation) const; + clients::body::Json batch_get(const std::vector& paths) const; + + // Writes (explicit). + clients::body::Json create_document(const std::string& parent, const std::string& collection, + const std::string& document_id, + const clients::body::Json& fields) const; + clients::body::Json commit(const clients::body::Json& writes) const; + clients::body::Json delete_document(const std::string& path) const; + + private: + std::string database_; +}; + +} // namespace molo::google::data diff --git a/src/google/data/ids.h b/src/google/data/ids.h new file mode 100644 index 0000000..b7a08a9 --- /dev/null +++ b/src/google/data/ids.h @@ -0,0 +1,77 @@ +#pragma once + +// Identifier rules of the data package. Every id is validated here before any +// network traffic; a failure is GoogleError 400 INVALID_ARGUMENT. + +#include + +#include "google/errors.h" + +namespace molo::google::data { + +class Ids { + public: + // [a-z0-9.:-], 1..100 (project ids, with domain-scoped "example.com:proj"). + static const std::string& project(const std::string& v) { + return check(v, "project", 100, "abcdefghijklmnopqrstuvwxyz0123456789.:-"); + } + // [a-z0-9-], 1..40 ("us", "us-central1", "global", "eu"). + static const std::string& location(const std::string& v) { + return check(v, "location", 40, "abcdefghijklmnopqrstuvwxyz0123456789-"); + } + // BigQuery dataset: letters, digits, underscore, 1..1024. + static const std::string& dataset(const std::string& v) { + return check(v, "dataset", 1024, alnum() + "_"); + } + // BigQuery table / model / routine / job id (ASCII subset): letters, digits, _ $ -. + static const std::string& bq_id(const std::string& v, const char* what = "table") { + return check(v, what, 1024, alnum() + "_$-"); + } + // Cloud Storage bucket: 3..222 of a-z 0-9 . _ -, alnum at both ends. + static const std::string& bucket(const std::string& v) { + check(v, "bucket", 222, "abcdefghijklmnopqrstuvwxyz0123456789._-"); + auto edge = [](char c) { return (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'); }; + if (v.size() < 3 || !edge(v.front()) || !edge(v.back())) + throw GoogleError::invalid("bucket must be 3..222 characters, alphanumeric at both ends"); + return v; + } + // Cloud Storage object name: 1..1024 bytes, no control characters, not . or .. + static const std::string& object(const std::string& v) { + if (v.empty() || v.size() > 1024) throw GoogleError::invalid("object must be 1..1024 bytes"); + if (v == "." || v == ".." || v.front() == '\r' || v.front() == '\n') + throw GoogleError::invalid("bad object name"); + for (char c : v) + if (static_cast(c) < 0x20 || c == 0x7f) + throw GoogleError::invalid("object has control characters"); + return v; + } + // Pub/Sub topic / subscription / snapshot id: 3..255, starts with a letter. + static const std::string& pubsub(const std::string& v) { + check(v, "pubsub id", 255, alnum() + "-_.~"); + if (v.size() < 3 || !((v[0] >= 'a' && v[0] <= 'z') || (v[0] >= 'A' && v[0] <= 'Z'))) + throw GoogleError::invalid("pubsub id must be 3..255 characters and start with a letter"); + return v; + } + // Generic single id (secret, key ring, crypto key, instance, ...): 1..255. + static const std::string& id(const std::string& v, const char* what = "id") { + return check(v, what, 255, alnum() + "_-."); + } + + static std::string alnum() { + return "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + } + + private: + static const std::string& check(const std::string& v, const char* what, std::size_t max, + const std::string& allowed) { + if (v.empty() || v.size() > max) + throw GoogleError::invalid(std::string(what) + " must be 1.." + std::to_string(max) + + " characters"); + for (char c : v) + if (allowed.find(c) == std::string::npos) + throw GoogleError::invalid(std::string(what) + " has characters outside " + allowed); + return v; + } +}; + +} // namespace molo::google::data diff --git a/src/google/data/kms.cpp b/src/google/data/kms.cpp new file mode 100644 index 0000000..9afe804 --- /dev/null +++ b/src/google/data/kms.cpp @@ -0,0 +1,92 @@ +#include "google/data/kms.h" + +namespace molo::google::data { + +using clients::body::Json; + +std::string Kms::key_ring_name(const std::string& l, const std::string& r) const { + return project_path() + "/locations/" + Ids::location(l) + "/keyRings/" + Ids::id(r, "key ring"); +} +std::string Kms::key_name(const std::string& l, const std::string& r, const std::string& k) const { + return key_ring_name(l, r) + "/cryptoKeys/" + Ids::id(k, "key"); +} +std::string Kms::version_name(const std::string& l, const std::string& r, const std::string& k, + const std::string& v) const { + need(!v.empty() && v.size() <= 10 && v.find_first_not_of("0123456789") == std::string::npos, + "key version must be a number"); + return key_name(l, r, k) + "/cryptoKeyVersions/" + v; +} + +Json Kms::list_locations(int n) const { + CallArgs a = on("name", project_path()); + a.q("pageSize", page(n)); + return invoke("cloudkms.projects.locations.list", a); +} +Json Kms::list_key_rings(const std::string& l, int n) const { + CallArgs a = on("parent", project_path() + "/locations/" + Ids::location(l)); + a.q("pageSize", page(n)); + return invoke("cloudkms.projects.locations.keyRings.list", a); +} +Json Kms::get_key_ring(const std::string& l, const std::string& r) const { + return invoke("cloudkms.projects.locations.keyRings.get", on("name", key_ring_name(l, r))); +} +Json Kms::list_keys(const std::string& l, const std::string& r, int n) const { + CallArgs a = on("parent", key_ring_name(l, r)); + a.q("pageSize", page(n)); + return invoke("cloudkms.projects.locations.keyRings.cryptoKeys.list", a); +} +Json Kms::get_key(const std::string& l, const std::string& r, const std::string& k) const { + return invoke("cloudkms.projects.locations.keyRings.cryptoKeys.get", on("name", key_name(l, r, k))); +} +Json Kms::list_versions(const std::string& l, const std::string& r, const std::string& k, + int n) const { + CallArgs a = on("parent", key_name(l, r, k)); + a.q("pageSize", page(n)); + return invoke("cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.list", a); +} +Json Kms::get_public_key(const std::string& l, const std::string& r, const std::string& k, + const std::string& v) const { + return invoke("cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.getPublicKey", + on("name", version_name(l, r, k, v))); +} + +Json Kms::encrypt(const std::string& l, const std::string& r, const std::string& k, + const std::string& plaintext, const std::string& aad) const { + need(!plaintext.empty() && plaintext.size() <= DataCaps::kMaxCryptoBytes, + "plaintext must be 1.." + std::to_string(DataCaps::kMaxCryptoBytes) + " bytes"); + need(aad.size() <= 64u * 1024u, "aad over 64 KiB"); + Json b = Json::object(); + b.set("plaintext", Json(b64(plaintext))); + if (!aad.empty()) b.set("additionalAuthenticatedData", Json(b64(aad))); + CallArgs a = on("name", key_name(l, r, k)); + a.with_body(b); + return invoke("cloudkms.projects.locations.keyRings.cryptoKeys.encrypt", a); +} + +Json Kms::decrypt(const std::string& l, const std::string& r, const std::string& k, + const std::string& ct, const std::string& aad) const { + need(!ct.empty() && ct.size() <= 2 * DataCaps::kMaxCryptoBytes, "ciphertext size out of range"); + std::string raw; + need(security::crypto::Base64::decode(ct, raw), "ciphertext must be base64"); + need(aad.size() <= 64u * 1024u, "aad over 64 KiB"); + Json b = Json::object(); + b.set("ciphertext", Json(ct)); + if (!aad.empty()) b.set("additionalAuthenticatedData", Json(b64(aad))); + CallArgs a = on("name", key_name(l, r, k)); + a.with_body(b); + return invoke("cloudkms.projects.locations.keyRings.cryptoKeys.decrypt", a); +} + +Json Kms::sign_digest(const std::string& l, const std::string& r, const std::string& k, + const std::string& v, const std::string& digest) const { + need(digest.size() == 32, "digest must be 32 raw bytes (SHA-256)"); + Json d = Json::object(); + d.set("sha256", Json(b64(digest))); + Json b = Json::object(); + b.set("digest", d); + CallArgs a = on("name", version_name(l, r, k, v)); + a.with_body(b); + return invoke("cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.asymmetricSign", a); +} + +} // namespace molo::google::data diff --git a/src/google/data/kms.h b/src/google/data/kms.h new file mode 100644 index 0000000..35063bd --- /dev/null +++ b/src/google/data/kms.h @@ -0,0 +1,50 @@ +#pragma once + +// Cloud KMS v1: key rings, crypto keys, versions, public keys, encrypt, +// decrypt, asymmetric sign, MAC sign. Key creation, rotation and destruction +// are not typed here: use ApiService::invoke("cloudkms....") deliberately. + +#include + +#include "google/data/project_service.h" + +namespace molo::google::data { + +class Kms : public ProjectService { + public: + explicit Kms(std::shared_ptr client, CloudContext context) + : ProjectService(std::move(client), Api::kKms, std::move(context)) {} + + std::string key_ring_name(const std::string& location, const std::string& ring) const; + std::string key_name(const std::string& location, const std::string& ring, + const std::string& key) const; + std::string version_name(const std::string& location, const std::string& ring, + const std::string& key, const std::string& version) const; + + clients::body::Json list_locations(int page_size) const; + clients::body::Json list_key_rings(const std::string& location, int page_size) const; + clients::body::Json get_key_ring(const std::string& location, const std::string& ring) const; + clients::body::Json list_keys(const std::string& location, const std::string& ring, + int page_size) const; + clients::body::Json get_key(const std::string& location, const std::string& ring, + const std::string& key) const; + clients::body::Json list_versions(const std::string& location, const std::string& ring, + const std::string& key, int page_size) const; + clients::body::Json get_public_key(const std::string& location, const std::string& ring, + const std::string& key, const std::string& version) const; + + // Symmetric: plaintext and ciphertext are raw bytes here, base64 on the wire. + // The ciphertext in the response stays base64 (field "ciphertext"). + clients::body::Json encrypt(const std::string& location, const std::string& ring, + const std::string& key, const std::string& plaintext, + const std::string& aad) const; + clients::body::Json decrypt(const std::string& location, const std::string& ring, + const std::string& key, const std::string& ciphertext_base64, + const std::string& aad) const; + // Asymmetric sign over a SHA-256 digest (32 raw bytes). + clients::body::Json sign_digest(const std::string& location, const std::string& ring, + const std::string& key, const std::string& version, + const std::string& sha256_digest) const; +}; + +} // namespace molo::google::data diff --git a/src/google/data/project_service.h b/src/google/data/project_service.h new file mode 100644 index 0000000..7c48015 --- /dev/null +++ b/src/google/data/project_service.h @@ -0,0 +1,50 @@ +#pragma once + +// ProjectService: ApiService bound to a CloudContext, with the resource-name +// and paging helpers every project scoped service of the package shares. + +#include + +#include "google/data/api_service.h" +#include "google/data/caps.h" +#include "google/data/ids.h" +#include "google/resource.h" +#include "security/crypto/Base64.h" + +namespace molo::google::data { + +class ProjectService : public ApiService { + public: + ProjectService(std::shared_ptr client, Api api, CloudContext context, + const std::string& base_url_override = "") + : ApiService(std::move(client), api, base_url_override), context_(std::move(context)) {} + + const CloudContext& context() const { return context_; } + + protected: + // "projects/{project}" + std::string project_path() const { return "projects/" + Ids::project(context_.project); } + // "projects/{project}/locations/{location}"; empty location uses the context's. + std::string location_path(const std::string& location) const { + return project_path() + "/locations/" + + Ids::location(location.empty() ? context_.location : location); + } + static clients::body::Json page(int size) { + if (size < 1 || size > DataCaps::kMaxPageSize) + throw GoogleError::invalid("page size must be 1.." + std::to_string(DataCaps::kMaxPageSize)); + return clients::body::Json(size); + } + static std::string b64(const std::string& raw) { return security::crypto::Base64::encode(raw); } + static void need(bool ok, const std::string& why) { + if (!ok) throw GoogleError::invalid(why); + } + CallArgs on(const char* key, const std::string& value) const { + CallArgs a; + a.p(key, value); + return a; + } + + CloudContext context_; +}; + +} // namespace molo::google::data diff --git a/src/google/data/pubsub.cpp b/src/google/data/pubsub.cpp new file mode 100644 index 0000000..468777b --- /dev/null +++ b/src/google/data/pubsub.cpp @@ -0,0 +1,108 @@ +#include "google/data/pubsub.h" + +namespace molo::google::data { + +using clients::body::Json; + +std::string PubSub::topic_name(const std::string& t) const { + return project_path() + "/topics/" + Ids::pubsub(t); +} +std::string PubSub::subscription_name(const std::string& s) const { + return project_path() + "/subscriptions/" + Ids::pubsub(s); +} + +Json PubSub::list_topics(int n) const { + CallArgs a = on("project", project_path()); + a.q("pageSize", page(n)); + return invoke("pubsub.projects.topics.list", a); +} +Json PubSub::get_topic(const std::string& t) const { + return invoke("pubsub.projects.topics.get", on("topic", topic_name(t))); +} +Json PubSub::create_topic(const std::string& t) const { + CallArgs a = on("name", topic_name(t)); + a.with_body(Json::object()); + return invoke("pubsub.projects.topics.create", a); +} +Json PubSub::delete_topic(const std::string& t) const { + return invoke("pubsub.projects.topics.delete", on("topic", topic_name(t))); +} +Json PubSub::list_subscriptions(int n) const { + CallArgs a = on("project", project_path()); + a.q("pageSize", page(n)); + return invoke("pubsub.projects.subscriptions.list", a); +} +Json PubSub::get_subscription(const std::string& s) const { + return invoke("pubsub.projects.subscriptions.get", on("subscription", subscription_name(s))); +} +Json PubSub::list_snapshots(int n) const { + CallArgs a = on("project", project_path()); + a.q("pageSize", page(n)); + return invoke("pubsub.projects.snapshots.list", a); +} + +Json PubSub::publish_body(const std::vector& messages) { + need(!messages.empty() && messages.size() <= DataCaps::kMaxPublishMessages, + "publish takes 1.." + std::to_string(DataCaps::kMaxPublishMessages) + " messages"); + std::size_t total = 0; + Json arr = Json::array(); + for (const PubsubMessage& m : messages) { + need(!m.data.empty() || !m.attributes.empty(), "a message needs data or attributes"); + need(m.data.size() <= DataCaps::kMaxMessageBytes, "message over the size cap"); + total += m.data.size(); + need(total <= DataCaps::kMaxMessageBytes, "publish request over the size cap"); + Json o = Json::object(); + if (!m.data.empty()) o.set("data", Json(b64(m.data))); + if (!m.attributes.empty()) { + need(m.attributes.size() <= 100, "at most 100 attributes"); + Json at = Json::object(); + for (const auto& kv : m.attributes) { + need(!kv.first.empty() && kv.first.size() <= 256 && kv.second.size() <= 1024, + "attribute key 1..256, value <= 1024 chars"); + at.set(kv.first, Json(kv.second)); + } + o.set("attributes", at); + } + if (!m.ordering_key.empty()) { + need(m.ordering_key.size() <= 1024, "ordering key over 1024 bytes"); + o.set("orderingKey", Json(m.ordering_key)); + } + arr.push(o); + } + Json b = Json::object(); + b.set("messages", arr); + return b; +} + +Json PubSub::publish(const std::string& t, const std::vector& messages) const { + CallArgs a = on("topic", topic_name(t)); + a.with_body(publish_body(messages)); + return invoke("pubsub.projects.topics.publish", a); +} + +Json PubSub::pull(const std::string& s, int max_messages) const { + need(max_messages >= 1 && max_messages <= DataCaps::kMaxPullMessages, + "max_messages must be 1.." + std::to_string(DataCaps::kMaxPullMessages)); + CallArgs a = on("subscription", subscription_name(s)); + Json b = Json::object(); + b.set("maxMessages", Json(max_messages)); + a.with_body(b); + return invoke("pubsub.projects.subscriptions.pull", a); +} + +Json PubSub::acknowledge(const std::string& s, const std::vector& ack_ids) const { + need(!ack_ids.empty() && ack_ids.size() <= DataCaps::kMaxAckIds, + "ack ids: 1.." + std::to_string(DataCaps::kMaxAckIds)); + Json ids = Json::array(); + for (const auto& id : ack_ids) { + need(!id.empty() && id.size() <= 512, "ack id 1..512 chars"); + ids.push(Json(id)); + } + Json b = Json::object(); + b.set("ackIds", ids); + CallArgs a = on("subscription", subscription_name(s)); + a.with_body(b); + return invoke("pubsub.projects.subscriptions.acknowledge", a); +} + +} // namespace molo::google::data diff --git a/src/google/data/pubsub.h b/src/google/data/pubsub.h new file mode 100644 index 0000000..156cd52 --- /dev/null +++ b/src/google/data/pubsub.h @@ -0,0 +1,46 @@ +#pragma once + +// Cloud Pub/Sub v1: topics, subscriptions, snapshots, publish, pull, +// acknowledge. Other methods: ApiService::invoke("pubsub...."). + +#include +#include + +#include "google/data/project_service.h" + +namespace molo::google::data { + +struct PubsubMessage { + std::string data; // raw bytes, base64 encoded on the wire + std::map attributes; + std::string ordering_key; +}; + +class PubSub : public ProjectService { + public: + explicit PubSub(std::shared_ptr client, CloudContext context) + : ProjectService(std::move(client), Api::kPubsub, std::move(context)) {} + + std::string topic_name(const std::string& topic) const; + std::string subscription_name(const std::string& subscription) const; + + clients::body::Json list_topics(int page_size) const; + clients::body::Json get_topic(const std::string& topic) const; + clients::body::Json create_topic(const std::string& topic) const; + clients::body::Json delete_topic(const std::string& topic) const; + + clients::body::Json list_subscriptions(int page_size) const; + clients::body::Json get_subscription(const std::string& subscription) const; + clients::body::Json list_snapshots(int page_size) const; + + // publish: 1..1000 messages, each <= 10 MiB, 10 MiB total. + clients::body::Json publish(const std::string& topic, + const std::vector& messages) const; + static clients::body::Json publish_body(const std::vector& messages); + // pull leases messages: they come back again unless acknowledged. + clients::body::Json pull(const std::string& subscription, int max_messages) const; + clients::body::Json acknowledge(const std::string& subscription, + const std::vector& ack_ids) const; +}; + +} // namespace molo::google::data diff --git a/src/google/data/secret_manager.cpp b/src/google/data/secret_manager.cpp new file mode 100644 index 0000000..3aa0f54 --- /dev/null +++ b/src/google/data/secret_manager.cpp @@ -0,0 +1,79 @@ +#include "google/data/secret_manager.h" + +namespace molo::google::data { + +using clients::body::Json; + +std::string SecretManager::secret_name(const std::string& s) const { + return project_path() + "/secrets/" + Ids::id(s, "secret"); +} + +std::string SecretManager::version_name(const std::string& s, const std::string& v) const { + bool ok = v == "latest" || (!v.empty() && v.size() <= 10); + for (char c : v) + if (v != "latest" && (c < '0' || c > '9')) ok = false; + need(ok, "version must be \"latest\" or a number"); + return secret_name(s) + "/versions/" + v; +} + +Json SecretManager::list_secrets(int n, const std::string& filter) const { + CallArgs a = on("parent", project_path()); + a.q("pageSize", page(n)); + need(filter.size() <= DataCaps::kMaxFilterChars, "filter too long"); + if (!filter.empty()) a.q("filter", Json(filter)); + return invoke("secretmanager.projects.secrets.list", a); +} +Json SecretManager::get_secret(const std::string& s) const { + return invoke("secretmanager.projects.secrets.get", on("name", secret_name(s))); +} +Json SecretManager::list_versions(const std::string& s, int n) const { + CallArgs a = on("parent", secret_name(s)); + a.q("pageSize", page(n)); + return invoke("secretmanager.projects.secrets.versions.list", a); +} +Json SecretManager::get_version(const std::string& s, const std::string& v) const { + return invoke("secretmanager.projects.secrets.versions.get", on("name", version_name(s, v))); +} +Json SecretManager::access_version(const std::string& s, const std::string& v) const { + return invoke("secretmanager.projects.secrets.versions.access", on("name", version_name(s, v))); +} + +Json SecretManager::create_secret(const std::string& s) const { + CallArgs a = on("parent", project_path()); + a.q("secretId", Json(Ids::id(s, "secret"))); + Json automatic = Json::object(); + Json replication = Json::object(); + replication.set("automatic", automatic); + Json b = Json::object(); + b.set("replication", replication); + a.with_body(b); + return invoke("secretmanager.projects.secrets.create", a); +} + +Json SecretManager::add_version(const std::string& s, const std::string& payload) const { + need(!payload.empty() && payload.size() <= DataCaps::kMaxSecretBytes, + "payload must be 1.." + std::to_string(DataCaps::kMaxSecretBytes) + " bytes"); + Json p = Json::object(); + p.set("data", Json(b64(payload))); + Json b = Json::object(); + b.set("payload", p); + CallArgs a = on("parent", secret_name(s)); + a.with_body(b); + return invoke("secretmanager.projects.secrets.addVersion", a); +} + +Json SecretManager::disable_version(const std::string& s, const std::string& v) const { + CallArgs a = on("name", version_name(s, v)); + a.with_body(Json::object()); + return invoke("secretmanager.projects.secrets.versions.disable", a); +} +Json SecretManager::destroy_version(const std::string& s, const std::string& v) const { + CallArgs a = on("name", version_name(s, v)); + a.with_body(Json::object()); + return invoke("secretmanager.projects.secrets.versions.destroy", a); +} +Json SecretManager::delete_secret(const std::string& s) const { + return invoke("secretmanager.projects.secrets.delete", on("name", secret_name(s))); +} + +} // namespace molo::google::data diff --git a/src/google/data/secret_manager.h b/src/google/data/secret_manager.h new file mode 100644 index 0000000..711b155 --- /dev/null +++ b/src/google/data/secret_manager.h @@ -0,0 +1,37 @@ +#pragma once + +// Secret Manager v1: secrets and versions (global resources; regional secrets +// via ApiService::invoke("secretmanager.projects.locations....")). Reading a +// payload is access_version(); listing and metadata never return payloads. + +#include + +#include "google/data/project_service.h" + +namespace molo::google::data { + +class SecretManager : public ProjectService { + public: + explicit SecretManager(std::shared_ptr client, CloudContext context) + : ProjectService(std::move(client), Api::kSecretManager, std::move(context)) {} + + std::string secret_name(const std::string& secret) const; + // version: "latest" or a positive number. + std::string version_name(const std::string& secret, const std::string& version) const; + + clients::body::Json list_secrets(int page_size, const std::string& filter) const; + clients::body::Json get_secret(const std::string& secret) const; + clients::body::Json list_versions(const std::string& secret, int page_size) const; + clients::body::Json get_version(const std::string& secret, const std::string& version) const; + // Returns the payload (base64 in payload.data). Sensitive: callers must not log it. + clients::body::Json access_version(const std::string& secret, const std::string& version) const; + + // Writes (explicit). + clients::body::Json create_secret(const std::string& secret) const; // automatic replication + clients::body::Json add_version(const std::string& secret, const std::string& payload) const; + clients::body::Json disable_version(const std::string& secret, const std::string& version) const; + clients::body::Json destroy_version(const std::string& secret, const std::string& version) const; + clients::body::Json delete_secret(const std::string& secret) const; +}; + +} // namespace molo::google::data diff --git a/src/google/data/signed_url.cpp b/src/google/data/signed_url.cpp new file mode 100644 index 0000000..4d3e459 --- /dev/null +++ b/src/google/data/signed_url.cpp @@ -0,0 +1,223 @@ +#include "google/data/signed_url.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "clients/url.h" +#include "google/credentials.h" +#include "google/data/caps.h" +#include "google/data/ids.h" +#include "google/errors.h" +#include "security/crypto/Base64.h" + +namespace molo::google::data { + +using security::crypto::Base64; + +namespace { + +std::string enc(const std::string& s) { return clients::Url::encode_component(s); } + +std::string lower(std::string s) { + for (char& c : s) c = static_cast(std::tolower(static_cast(c))); + return s; +} + +std::string trim(const std::string& s) { + std::size_t a = s.find_first_not_of(" \t"), b = s.find_last_not_of(" \t"); + return a == std::string::npos ? "" : s.substr(a, b - a + 1); +} + +std::string hex(const std::string& raw) { + static const char* k = "0123456789abcdef"; + std::string out; + for (unsigned char c : raw) { + out.push_back(k[c >> 4]); + out.push_back(k[c & 15]); + } + return out; +} + +std::string pem_to_der(const std::string& pem) { + std::string body, line; + for (char c : pem + "\n") { + if (c != '\n' && c != '\r') { + line.push_back(c); + continue; + } + if (!line.empty() && line.rfind("-----", 0) != 0) body += line; + line.clear(); + } + std::string der; + if (!Base64::decode(body, der) || der.empty()) + throw GoogleError::invalid("private_key is not valid PEM"); + return der; +} + +struct RsaGuard { + RsaGuard() { wc_InitRsaKey(&key, nullptr); } + ~RsaGuard() { wc_FreeRsaKey(&key); } + RsaGuard(const RsaGuard&) = delete; + RsaGuard& operator=(const RsaGuard&) = delete; + RsaKey key; +}; + +} // namespace + +void SignedUrlRequest::validate() const { + static const char* const ok[] = {"GET", "HEAD", "PUT", "POST", "DELETE"}; + bool known = false; + for (const char* m : ok) known = known || method == m; + if (!known) throw GoogleError::invalid("method must be GET, HEAD, PUT, POST or DELETE"); + Ids::bucket(bucket); + if (!object.empty()) Ids::object(object); + if (expires_seconds < DataCaps::kMinSignedUrlSeconds || + expires_seconds > DataCaps::kMaxSignedUrlSeconds) + throw GoogleError::invalid("expires_seconds must be 1.." + + std::to_string(DataCaps::kMaxSignedUrlSeconds)); + if (headers.size() > DataCaps::kMaxSignedHeaders || query.size() > DataCaps::kMaxSignedHeaders) + throw GoogleError::invalid("too many signed headers or query parameters"); + for (const auto& kv : headers) { + if (kv.first.empty() || kv.first.size() > 128 || kv.second.size() > 512) + throw GoogleError::invalid("bad signed header"); + for (char c : kv.first + kv.second) + if (static_cast(c) < 0x20 || c == 0x7f) + throw GoogleError::invalid("signed header has control characters"); + if (lower(kv.first) == "host") throw GoogleError::invalid("host is always signed"); + } + for (const auto& kv : query) { + if (kv.first.empty() || kv.first.size() > 128 || kv.second.size() > 1024) + throw GoogleError::invalid("bad query parameter"); + if (lower(kv.first).rfind("x-goog-", 0) == 0) + throw GoogleError::invalid("x-goog-* query parameters are set by the signer"); + } +} + +SignedUrl::SignedUrl(ServiceAccountKey key) : key_(std::move(key)) {} + +SignedUrl SignedUrl::from_env() { + std::string path = Credentials::env("GOOGLE_APPLICATION_CREDENTIALS"); + if (path.empty()) throw NotConfiguredError("storage"); + return SignedUrl(ServiceAccountKey::load_file(path)); +} + +namespace { +struct Stamp { + std::string date, datetime; +}; +Stamp stamp(std::time_t t) { + std::tm tm{}; + gmtime_r(&t, &tm); + char d[16], dt[32]; + std::strftime(d, sizeof d, "%Y%m%d", &tm); + std::strftime(dt, sizeof dt, "%Y%m%dT%H%M%SZ", &tm); + return {d, dt}; +} +} // namespace + +std::string SignedUrl::canonical_request(const SignedUrlRequest& r, std::time_t now) const { + Stamp s = stamp(now); + std::string uri = "/" + r.bucket; + if (!r.object.empty()) { + uri += "/"; + std::string seg; + for (char c : r.object) { + if (c == '/') { + uri += enc(seg) + "/"; + seg.clear(); + } else { + seg.push_back(c); + } + } + uri += enc(seg); + } + std::map hdr; + hdr["host"] = "storage.googleapis.com"; + for (const auto& kv : r.headers) hdr[lower(kv.first)] = trim(kv.second); + std::string signed_headers, canon_headers; + for (const auto& kv : hdr) { + if (!signed_headers.empty()) signed_headers += ";"; + signed_headers += kv.first; + canon_headers += kv.first + ":" + kv.second + "\n"; + } + std::map q; + for (const auto& kv : r.query) q[enc(kv.first)] = enc(kv.second); + q["X-Goog-Algorithm"] = "GOOG4-RSA-SHA256"; + q["X-Goog-Credential"] = enc(key_.client_email + "/" + s.date + "/auto/storage/goog4_request"); + q["X-Goog-Date"] = s.datetime; + q["X-Goog-Expires"] = std::to_string(r.expires_seconds); + q["X-Goog-SignedHeaders"] = enc(signed_headers); + std::string canon_query; + for (const auto& kv : q) { + if (!canon_query.empty()) canon_query += "&"; + canon_query += kv.first + "=" + kv.second; + } + return r.method + "\n" + uri + "\n" + canon_query + "\n" + canon_headers + "\n" + + signed_headers + "\nUNSIGNED-PAYLOAD"; +} + +std::string SignedUrl::string_to_sign(const std::string& datetime, const std::string& scope, + const std::string& canonical) { + // wolfCrypt, not security/crypto/Sha256: the google build only links Base64. + wc_Sha256 h; + byte digest[WC_SHA256_DIGEST_SIZE]; + if (wc_InitSha256(&h) != 0 || + wc_Sha256Update(&h, reinterpret_cast(canonical.data()), + static_cast(canonical.size())) != 0 || + wc_Sha256Final(&h, digest) != 0) + throw GoogleError(500, "INTERNAL", "sha256 failed"); + wc_Sha256Free(&h); + return "GOOG4-RSA-SHA256\n" + datetime + "\n" + scope + "\n" + + hex(std::string(reinterpret_cast(digest), sizeof digest)); +} + +std::string SignedUrl::rsa_sha256_hex(const std::string& data) const { + std::string der = pem_to_der(key_.private_key_pem); + RsaGuard rsa; + word32 idx = 0; + int off = wc_GetPkcs8TraditionalOffset(reinterpret_cast(der.data()), &idx, + static_cast(der.size())); + if (off < 0) idx = 0; + if (wc_RsaPrivateKeyDecode(reinterpret_cast(der.data()), &idx, &rsa.key, + static_cast(der.size())) != 0) + throw GoogleError::invalid("private_key is not a usable RSA key"); + WC_RNG rng; + if (wc_InitRng(&rng) != 0) throw GoogleError(500, "INTERNAL", "rng init failed"); + word32 len = static_cast( + wc_SignatureGetSize(WC_SIGNATURE_TYPE_RSA_W_ENC, &rsa.key, sizeof(rsa.key))); + std::string sig(len, '\0'); + int rc = wc_SignatureGenerate(WC_HASH_TYPE_SHA256, WC_SIGNATURE_TYPE_RSA_W_ENC, + reinterpret_cast(data.data()), + static_cast(data.size()), + reinterpret_cast(sig.data()), &len, &rsa.key, + sizeof(rsa.key), &rng); + wc_FreeRng(&rng); + if (rc != 0) throw GoogleError(500, "INTERNAL", "signing failed"); + sig.resize(len); + return hex(sig); +} + +std::string SignedUrl::sign(const SignedUrlRequest& r) const { + r.validate(); + std::time_t now = r.now ? r.now : std::time(nullptr); + Stamp s = stamp(now); + std::string canon = canonical_request(r, now); + std::string scope = s.date + "/auto/storage/goog4_request"; + std::string signature = rsa_sha256_hex(string_to_sign(s.datetime, scope, canon)); + // Rebuild "path?query" from the canonical request (line 2 and 3). + std::size_t a = canon.find('\n'), b = canon.find('\n', a + 1), c = canon.find('\n', b + 1); + std::string uri = canon.substr(a + 1, b - a - 1); + std::string query = canon.substr(b + 1, c - b - 1); + return "https://storage.googleapis.com" + uri + "?" + query + "&X-Goog-Signature=" + signature; +} + +} // namespace molo::google::data diff --git a/src/google/data/signed_url.h b/src/google/data/signed_url.h new file mode 100644 index 0000000..8b64711 --- /dev/null +++ b/src/google/data/signed_url.h @@ -0,0 +1,48 @@ +#pragma once + +// Cloud Storage V4 signed URLs (GOOG4-RSA-SHA256), signed locally with the +// service-account private key. No network traffic. Anyone holding the URL can +// perform the signed method until it expires, so lifetimes are capped at the +// V4 maximum of 7 days. + +#include +#include +#include + +#include "google/service_account.h" + +namespace molo::google::data { + +struct SignedUrlRequest { + std::string method = "GET"; // GET, HEAD, PUT, POST, DELETE + std::string bucket; + std::string object; // "" signs a bucket-level URL + long expires_seconds = 900; // 1..604800 + std::map headers; // extra signed headers (lowercased on sign) + std::map query; // extra query parameters (e.g. response-content-type) + std::time_t now = 0; // 0: current time (tests inject) + + void validate() const; +}; + +class SignedUrl { + public: + explicit SignedUrl(ServiceAccountKey key); + // Key from GOOGLE_APPLICATION_CREDENTIALS; throws NotConfiguredError if unset. + static SignedUrl from_env(); + + std::string sign(const SignedUrlRequest& request) const; + + // The parts, for tests and for the example page. + std::string canonical_request(const SignedUrlRequest& request, std::time_t now) const; + static std::string string_to_sign(const std::string& datetime, const std::string& scope, + const std::string& canonical_request); + const std::string& email() const { return key_.client_email; } + + private: + std::string rsa_sha256_hex(const std::string& data) const; + + ServiceAccountKey key_; +}; + +} // namespace molo::google::data diff --git a/src/google/data/spanner.cpp b/src/google/data/spanner.cpp new file mode 100644 index 0000000..afccbef --- /dev/null +++ b/src/google/data/spanner.cpp @@ -0,0 +1,89 @@ +#include "google/data/spanner.h" + +namespace molo::google::data { + +using clients::body::Json; + +void SpannerSql::validate() const { + if (sql.empty() || sql.size() > DataCaps::kMaxSqlChars) + throw GoogleError::invalid("sql must be 1.." + std::to_string(DataCaps::kMaxSqlChars) + + " characters"); + if (query_mode != "NORMAL" && query_mode != "PLAN" && query_mode != "PROFILE") + throw GoogleError::invalid("query_mode must be NORMAL, PLAN or PROFILE"); + if (!params.is_null() && !params.is_object()) throw GoogleError::invalid("params must be an object"); + if (!param_types.is_null() && !param_types.is_object()) + throw GoogleError::invalid("param_types must be an object"); +} + +std::string Spanner::instance_name(const std::string& i) const { + return project_path() + "/instances/" + Ids::id(i, "instance"); +} +std::string Spanner::database_name(const std::string& i, const std::string& d) const { + return instance_name(i) + "/databases/" + Ids::id(d, "database"); +} + +Json Spanner::list_instances(int n) const { + CallArgs a = on("parent", project_path()); + a.q("pageSize", page(n)); + return invoke("spanner.projects.instances.list", a); +} +Json Spanner::get_instance(const std::string& i) const { + return invoke("spanner.projects.instances.get", on("name", instance_name(i))); +} +Json Spanner::list_databases(const std::string& i, int n) const { + CallArgs a = on("parent", instance_name(i)); + a.q("pageSize", page(n)); + return invoke("spanner.projects.instances.databases.list", a); +} +Json Spanner::get_database(const std::string& i, const std::string& d) const { + return invoke("spanner.projects.instances.databases.get", on("name", database_name(i, d))); +} +Json Spanner::get_ddl(const std::string& i, const std::string& d) const { + return invoke("spanner.projects.instances.databases.getDdl", on("database", database_name(i, d))); +} + +Json Spanner::create_session(const std::string& i, const std::string& d) const { + CallArgs a = on("database", database_name(i, d)); + a.with_body(Json::object()); + return invoke("spanner.projects.instances.databases.sessions.create", a); +} +Json Spanner::delete_session(const std::string& name) const { + return invoke("spanner.projects.instances.databases.sessions.delete", on("name", name)); +} + +Json Spanner::execute_sql(const std::string& session, const SpannerSql& s) const { + s.validate(); + Json read_only = Json::object(); + read_only.set("strong", Json(true)); + Json opts = Json::object(); + opts.set("readOnly", read_only); + Json tx = Json::object(); + tx.set("singleUse", opts); + Json b = Json::object(); + b.set("sql", Json(s.sql)); + b.set("queryMode", Json(s.query_mode)); + b.set("transaction", tx); + if (!s.params.is_null()) b.set("params", s.params); + if (!s.param_types.is_null()) b.set("paramTypes", s.param_types); + CallArgs a = on("session", session); + a.with_body(b); + return invoke("spanner.projects.instances.databases.sessions.executeSql", a); +} + +Json Spanner::query(const std::string& i, const std::string& d, const SpannerSql& s) const { + s.validate(); + Json session = create_session(i, d); + std::string name = string_at(session, "name"); + need(!name.empty(), "create session returned no name"); + Json result; + try { + result = execute_sql(name, s); + } catch (...) { + try { delete_session(name); } catch (...) {} + throw; + } + delete_session(name); + return result; +} + +} // namespace molo::google::data diff --git a/src/google/data/spanner.h b/src/google/data/spanner.h new file mode 100644 index 0000000..f389fe2 --- /dev/null +++ b/src/google/data/spanner.h @@ -0,0 +1,45 @@ +#pragma once + +// Cloud Spanner v1: instances, databases, DDL, sessions, SQL. Other methods: +// ApiService::invoke("spanner...."). + +#include + +#include "google/data/project_service.h" + +namespace molo::google::data { + +struct SpannerSql { + std::string sql; + clients::body::Json params; // object name -> value, or null + clients::body::Json param_types; // object name -> Type, or null + std::string query_mode = "NORMAL"; // NORMAL, PLAN (no rows, no cost), PROFILE + void validate() const; +}; + +class Spanner : public ProjectService { + public: + explicit Spanner(std::shared_ptr client, CloudContext context) + : ProjectService(std::move(client), Api::kSpanner, std::move(context)) {} + + std::string instance_name(const std::string& instance) const; + std::string database_name(const std::string& instance, const std::string& database) const; + + clients::body::Json list_instances(int page_size) const; + clients::body::Json get_instance(const std::string& instance) const; + clients::body::Json list_databases(const std::string& instance, int page_size) const; + clients::body::Json get_database(const std::string& instance, const std::string& database) const; + clients::body::Json get_ddl(const std::string& instance, const std::string& database) const; + + // Sessions are cheap server side handles; always delete what you create. + clients::body::Json create_session(const std::string& instance, + const std::string& database) const; + clients::body::Json delete_session(const std::string& session_name) const; + // Read-only, single use strong transaction on an existing session. + clients::body::Json execute_sql(const std::string& session_name, const SpannerSql& sql) const; + // create_session + execute_sql + delete_session (always deletes). + clients::body::Json query(const std::string& instance, const std::string& database, + const SpannerSql& sql) const; +}; + +} // namespace molo::google::data diff --git a/src/google/data/storage.cpp b/src/google/data/storage.cpp new file mode 100644 index 0000000..34a21b7 --- /dev/null +++ b/src/google/data/storage.cpp @@ -0,0 +1,132 @@ +#include "google/data/storage.h" + +#include "google/data/caps.h" +#include "google/data/ids.h" + +namespace molo::google::data { + +using clients::body::Json; + +namespace { +Json page(int max_results) { + if (max_results < 1 || max_results > DataCaps::kMaxPageSize) + throw GoogleError::invalid("max_results must be 1.." + std::to_string(DataCaps::kMaxPageSize)); + return Json(max_results); +} +} // namespace + +Storage::Storage(std::shared_ptr client, CloudContext context) + : ApiService(std::move(client), Api::kStorage), context_(std::move(context)) {} + +Json Storage::list_buckets(int max_results, const std::string& prefix, + const std::string& page_token) const { + CallArgs a; + a.q("project", Json(Ids::project(context_.project))); + a.q("maxResults", page(max_results)); + if (!prefix.empty()) a.q("prefix", Json(prefix)); + if (!page_token.empty()) a.q("pageToken", Json(page_token)); + return invoke("storage.buckets.list", a); +} + +Json Storage::get_bucket(const std::string& bucket) const { + return invoke("storage.buckets.get", CallArgs().p("bucket", Ids::bucket(bucket))); +} + +Json Storage::create_bucket(const std::string& bucket, const std::string& location, + const std::string& storage_class) const { + Json b = Json::object(); + b.set("name", Json(Ids::bucket(bucket))); + if (!location.empty()) b.set("location", Json(location)); + if (!storage_class.empty()) { + if (storage_class != "STANDARD" && storage_class != "NEARLINE" && + storage_class != "COLDLINE" && storage_class != "ARCHIVE") + throw GoogleError::invalid("storage_class must be STANDARD, NEARLINE, COLDLINE or ARCHIVE"); + b.set("storageClass", Json(storage_class)); + } + CallArgs a; + a.q("project", Json(Ids::project(context_.project))); + a.with_body(b); + return invoke("storage.buckets.insert", a); +} + +Json Storage::delete_bucket(const std::string& bucket) const { + return invoke("storage.buckets.delete", CallArgs().p("bucket", Ids::bucket(bucket))); +} + +Json Storage::list_objects(const std::string& bucket, const std::string& prefix, + const std::string& delimiter, int max_results, + const std::string& page_token) const { + CallArgs a; + a.p("bucket", Ids::bucket(bucket)); + a.q("maxResults", page(max_results)); + if (!prefix.empty()) a.q("prefix", Json(prefix)); + if (!delimiter.empty()) a.q("delimiter", Json(delimiter)); + if (!page_token.empty()) a.q("pageToken", Json(page_token)); + return invoke("storage.objects.list", a); +} + +Json Storage::get_object(const std::string& bucket, const std::string& object) const { + return invoke("storage.objects.get", + CallArgs().p("bucket", Ids::bucket(bucket)).p("object", Ids::object(object))); +} + +Json Storage::patch_object(const std::string& bucket, const std::string& object, + const Json& fields) const { + CallArgs a; + a.p("bucket", Ids::bucket(bucket)).p("object", Ids::object(object)); + a.with_body(fields); + return invoke("storage.objects.patch", a); +} + +Json Storage::delete_object(const std::string& bucket, const std::string& object) const { + return invoke("storage.objects.delete", + CallArgs().p("bucket", Ids::bucket(bucket)).p("object", Ids::object(object))); +} + +Json Storage::copy_object(const std::string& sb, const std::string& so, const std::string& db, + const std::string& d) const { + CallArgs a; + a.p("sourceBucket", Ids::bucket(sb)).p("sourceObject", Ids::object(so)); + a.p("destinationBucket", Ids::bucket(db)).p("destinationObject", Ids::object(d)); + return invoke("storage.objects.copy", a); +} + +Json Storage::compose_body(const std::vector& sources, + const std::string& content_type) { + if (sources.empty() || sources.size() > DataCaps::kMaxComposeSources) + throw GoogleError::invalid("compose takes 1.." + std::to_string(DataCaps::kMaxComposeSources) + + " sources"); + Json arr = Json::array(); + for (const ComposeSource& s : sources) { + Json o = Json::object(); + o.set("name", Json(Ids::object(s.name))); + if (s.generation < 0) throw GoogleError::invalid("generation must be >= 0"); + if (s.generation > 0) o.set("generation", Json(std::to_string(s.generation))); + arr.push(o); + } + Json dest = Json::object(); + if (!content_type.empty()) dest.set("contentType", Json(content_type)); + Json b = Json::object(); + b.set("sourceObjects", arr); + b.set("destination", dest); + return b; +} + +Json Storage::compose(const std::string& bucket, const std::vector& sources, + const std::string& destination, const std::string& content_type) const { + CallArgs a; + a.p("destinationBucket", Ids::bucket(bucket)).p("destinationObject", Ids::object(destination)); + a.with_body(compose_body(sources, content_type)); + return invoke("storage.objects.compose", a); +} + +Json Storage::bucket_iam_policy(const std::string& bucket) const { + return invoke("storage.buckets.getIamPolicy", CallArgs().p("bucket", Ids::bucket(bucket))); +} + +Json Storage::service_account() const { + return invoke("storage.projects.serviceAccount.get", + CallArgs().p("projectId", Ids::project(context_.project))); +} + +} // namespace molo::google::data diff --git a/src/google/data/storage.h b/src/google/data/storage.h new file mode 100644 index 0000000..181b716 --- /dev/null +++ b/src/google/data/storage.h @@ -0,0 +1,58 @@ +#pragma once + +// Cloud Storage JSON API v1: buckets and objects (metadata, list, compose, +// copy, delete). Media (download, simple and resumable upload) is in +// storage_media.h, signed URLs in signed_url.h. Every other method is +// reachable with ApiService::invoke("storage...."). + +#include +#include + +#include "google/data/api_service.h" +#include "google/resource.h" + +namespace molo::google::data { + +struct ComposeSource { + std::string name; + long long generation = 0; // 0: current generation +}; + +class Storage : public ApiService { + public: + explicit Storage(std::shared_ptr client, CloudContext context); + + const CloudContext& context() const { return context_; } + + clients::body::Json list_buckets(int max_results, const std::string& prefix, + const std::string& page_token) const; + clients::body::Json get_bucket(const std::string& bucket) const; + clients::body::Json create_bucket(const std::string& bucket, const std::string& location, + const std::string& storage_class) const; + clients::body::Json delete_bucket(const std::string& bucket) const; + + clients::body::Json list_objects(const std::string& bucket, const std::string& prefix, + const std::string& delimiter, int max_results, + const std::string& page_token) const; + clients::body::Json get_object(const std::string& bucket, const std::string& object) const; + clients::body::Json patch_object(const std::string& bucket, const std::string& object, + const clients::body::Json& fields) const; + clients::body::Json delete_object(const std::string& bucket, const std::string& object) const; + clients::body::Json copy_object(const std::string& source_bucket, const std::string& source, + const std::string& dest_bucket, const std::string& dest) const; + + // objects.compose: 1..32 sources in one bucket into `destination`. + clients::body::Json compose(const std::string& bucket, const std::vector& sources, + const std::string& destination, + const std::string& content_type) const; + static clients::body::Json compose_body(const std::vector& sources, + const std::string& content_type); + + clients::body::Json bucket_iam_policy(const std::string& bucket) const; + clients::body::Json service_account() const; // projects.serviceAccount.get + + private: + CloudContext context_; +}; + +} // namespace molo::google::data diff --git a/src/google/data/storage_media.cpp b/src/google/data/storage_media.cpp new file mode 100644 index 0000000..a8a339c --- /dev/null +++ b/src/google/data/storage_media.cpp @@ -0,0 +1,231 @@ +#include "google/data/storage_media.h" + +#include "clients/transports/http_transport.h" +#include "clients/url.h" +#include "google/credentials.h" +#include "google/data/caps.h" +#include "google/data/ids.h" + +namespace molo::google::data { + +using clients::body::Json; + +namespace { + +clients::ClientConfig config() { + clients::ClientConfig c; + c.persist_cookies = false; + return c; +} + +std::string enc(const std::string& s) { return clients::Url::encode_component(s); } + +void check_type(const std::string& t) { + if (t.empty() || t.size() > 128) throw GoogleError::invalid("content type must be 1..128 chars"); + for (char c : t) + if (static_cast(c) < 0x20 || c == 0x7f) + throw GoogleError::invalid("content type has control characters"); +} + +Json parse(const clients::ClientResponse& r) { + try { + return r.json(); + } catch (const std::exception& e) { + throw GoogleError(502, "BAD_RESPONSE", e.what(), "storage"); + } +} + +} // namespace + +StorageMedia::StorageMedia(std::unique_ptr transport, + std::shared_ptr auth, std::string host_url) + : client_(std::make_unique(std::move(transport), config())), + auth_(std::move(auth)), + host_(std::move(host_url)) {} + +std::shared_ptr StorageMedia::from_env() { + std::shared_ptr auth = Credentials::oauth_from_env(); + if (!auth) auth = Credentials::api_key_from_env(); + return std::make_shared(std::make_unique(), + auth); +} + +std::string StorageMedia::download_url(const std::string& bucket, const std::string& object, + long long generation) const { + if (generation < 0) throw GoogleError::invalid("generation must be >= 0"); + std::string u = host_ + "/storage/v1/b/" + enc(Ids::bucket(bucket)) + "/o/" + + enc(Ids::object(object)) + "?alt=media"; + if (generation > 0) u += "&generation=" + std::to_string(generation); + return u; +} + +std::string StorageMedia::upload_url(const std::string& bucket, const std::string& object, + const char* upload_type) const { + return host_ + "/upload/storage/v1/b/" + enc(Ids::bucket(bucket)) + "/o?uploadType=" + + upload_type + "&name=" + enc(Ids::object(object)); +} + +std::string StorageMedia::check_session(const std::string& url) { + const std::string prefix = std::string(kHost) + "/upload/storage/v1/b/"; + const std::string alt = std::string(kHost) + "/resumable/upload/storage/v1/b/"; + if ((url.rfind(prefix, 0) != 0 && url.rfind(alt, 0) != 0) || url.size() > 2048) + throw GoogleError::invalid("session url must be on " + std::string(kHost) + + "/upload/storage/v1/b/"); + for (char c : url) + if (static_cast(c) <= 0x20 || c == 0x7f) + throw GoogleError::invalid("session url has control characters or spaces"); + return url; +} + +clients::ClientResponse StorageMedia::send(clients::ClientRequest request) const { + if (!auth_) throw NotConfiguredError("storage"); + try { + auth_->apply(request); + return client_->send_blocking(std::move(request)); + } catch (const GoogleError&) { + throw; + } catch (const std::exception& e) { + throw GoogleError(502, "UNAVAILABLE", e.what(), "storage"); + } +} + +void StorageMedia::fail(const clients::ClientResponse& r) const { + throw error_from_response("storage", r); +} + +Downloaded StorageMedia::download(const std::string& bucket, const std::string& object, + long long generation) const { + clients::ClientRequest req(clients::Method::kGet, + clients::Url::parse(download_url(bucket, object, generation))); + clients::ClientResponse r = send(std::move(req)); + if (!r.is_success()) fail(r); + if (r.body().size() > DataCaps::kMaxDownloadBytes) + throw GoogleError(502, "RESPONSE_TOO_LARGE", + "object is larger than the " + std::to_string(DataCaps::kMaxDownloadBytes) + + " byte download cap; use download_range", + "storage"); + Downloaded d; + d.data = r.body(); + d.content_type = r.content_type(); + d.generation = r.headers().get("x-goog-generation").value_or(""); + return d; +} + +Downloaded StorageMedia::download_range(const std::string& bucket, const std::string& object, + long long first, long long last) const { + if (first < 0 || last < first || + last - first + 1 > static_cast(DataCaps::kMaxDownloadBytes)) + throw GoogleError::invalid("range must satisfy 0 <= first <= last and span <= " + + std::to_string(DataCaps::kMaxDownloadBytes) + " bytes"); + clients::ClientRequest req(clients::Method::kGet, + clients::Url::parse(download_url(bucket, object, 0))); + req.header("Range", "bytes=" + std::to_string(first) + "-" + std::to_string(last)); + clients::ClientResponse r = send(std::move(req)); + if (!r.is_success()) fail(r); + if (r.body().size() > DataCaps::kMaxDownloadBytes) + throw GoogleError(502, "RESPONSE_TOO_LARGE", "range response over the cap", "storage"); + Downloaded d; + d.data = r.body(); + d.content_type = r.content_type(); + d.generation = r.headers().get("x-goog-generation").value_or(""); + return d; +} + +Json StorageMedia::upload_simple(const std::string& bucket, const std::string& object, + const std::string& data, const std::string& content_type) const { + if (data.size() > DataCaps::kMaxSimpleUploadBytes) + throw GoogleError::invalid("simple upload over " + + std::to_string(DataCaps::kMaxSimpleUploadBytes) + + " bytes; use the resumable upload"); + check_type(content_type); + clients::ClientRequest req(clients::Method::kPost, + clients::Url::parse(upload_url(bucket, object, "media"))); + req.bytes(data, content_type); + clients::ClientResponse r = send(std::move(req)); + if (!r.is_success()) fail(r); + return parse(r); +} + +std::string StorageMedia::resumable_start(const std::string& bucket, const std::string& object, + const std::string& content_type, long long total, + const Json& metadata) const { + if (total < 0 || total > static_cast(DataCaps::kMaxResumableTotal)) + throw GoogleError::invalid("total must be 0.." + std::to_string(DataCaps::kMaxResumableTotal)); + check_type(content_type); + if (!metadata.is_null() && !metadata.is_object()) + throw GoogleError::invalid("metadata must be a JSON object"); + clients::ClientRequest req(clients::Method::kPost, + clients::Url::parse(upload_url(bucket, object, "resumable"))); + req.header("X-Upload-Content-Type", content_type); + req.header("X-Upload-Content-Length", std::to_string(total)); + req.json(metadata.is_object() ? metadata : Json::object()); + clients::ClientResponse r = send(std::move(req)); + if (!r.is_success()) fail(r); + std::string loc = r.headers().get("location").value_or(""); + if (loc.empty()) + throw GoogleError(502, "BAD_RESPONSE", "resumable start returned no Location", "storage"); + return check_session(loc); +} + +ResumableProgress StorageMedia::resumable_put(const std::string& session_url, long long offset, + const std::string& chunk, long long total) const { + check_session(session_url); + if (total < 0 || total > static_cast(DataCaps::kMaxResumableTotal)) + throw GoogleError::invalid("total out of range"); + if (offset < 0 || chunk.empty() || chunk.size() > DataCaps::kMaxResumableChunk) + throw GoogleError::invalid("chunk must be 1.." + std::to_string(DataCaps::kMaxResumableChunk) + + " bytes at a non-negative offset"); + long long end = offset + static_cast(chunk.size()); + if (end > total) throw GoogleError::invalid("chunk runs past total"); + if (end < total && chunk.size() % DataCaps::kChunkGranularity != 0) + throw GoogleError::invalid("a chunk that does not end the object must be a multiple of 256 KiB"); + clients::ClientRequest req(clients::Method::kPut, clients::Url::parse(session_url)); + req.header("Content-Range", "bytes " + std::to_string(offset) + "-" + std::to_string(end - 1) + + "/" + std::to_string(total)); + req.bytes(chunk, "application/octet-stream"); + clients::ClientResponse r = send(std::move(req)); + ResumableProgress p; + if (r.status() == 308) { + std::string range = r.headers().get("range").value_or(""); // "bytes=0-N" + auto dash = range.find('-'); + p.next_offset = dash == std::string::npos ? 0 : std::stoll(range.substr(dash + 1)) + 1; + return p; + } + if (!r.is_success()) fail(r); + p.done = true; + p.next_offset = total; + p.object = parse(r); + return p; +} + +ResumableProgress StorageMedia::resumable_status(const std::string& session_url, + long long total) const { + check_session(session_url); + if (total < 0 || total > static_cast(DataCaps::kMaxResumableTotal)) + throw GoogleError::invalid("total out of range"); + clients::ClientRequest req(clients::Method::kPut, clients::Url::parse(session_url)); + req.header("Content-Range", "bytes */" + std::to_string(total)); + req.bytes("", "application/octet-stream"); + clients::ClientResponse r = send(std::move(req)); + ResumableProgress p; + if (r.status() == 308) { + std::string range = r.headers().get("range").value_or(""); + auto dash = range.find('-'); + p.next_offset = dash == std::string::npos ? 0 : std::stoll(range.substr(dash + 1)) + 1; + return p; + } + if (!r.is_success()) fail(r); + p.done = true; + p.next_offset = total; + p.object = parse(r); + return p; +} + +void StorageMedia::resumable_cancel(const std::string& session_url) const { + check_session(session_url); + clients::ClientRequest req(clients::Method::kDelete, clients::Url::parse(session_url)); + clients::ClientResponse r = send(std::move(req)); + if (r.status() != 499 && !r.is_success()) fail(r); +} + +} // namespace molo::google::data diff --git a/src/google/data/storage_media.h b/src/google/data/storage_media.h new file mode 100644 index 0000000..5cc832b --- /dev/null +++ b/src/google/data/storage_media.h @@ -0,0 +1,83 @@ +#pragma once + +// Cloud Storage media: object download (alt=media), simple upload +// (uploadType=media) and resumable upload (uploadType=resumable). These carry +// raw bytes, which GoogleClient (JSON only) cannot, so this class owns a +// clients::Client with the same credential. The credential is only ever sent to +// storage.googleapis.com: a resumable session URL is checked against that host. + +#include +#include + +#include "clients/auth.h" +#include "clients/body/json.h" +#include "clients/client.h" +#include "google/errors.h" + +namespace molo::google::data { + +struct Downloaded { + std::string data; + std::string content_type; + std::string generation; // x-goog-generation header +}; + +struct ResumableProgress { + bool done = false; + long long next_offset = 0; // first byte Google does not have yet + clients::body::Json object; // object resource when done +}; + +class StorageMedia { + public: + // auth may be null: every call then throws NotConfiguredError. + StorageMedia(std::unique_ptr transport, + std::shared_ptr auth, std::string host_url = kHost); + // HTTPS transport + GOOGLE_ACCESS_TOKEN / service account / API key from env. + static std::shared_ptr from_env(); + + static constexpr const char* kHost = "https://storage.googleapis.com"; + + bool configured() const { return auth_ != nullptr; } + + // GET .../o/{object}?alt=media, optional generation (0 = current). Capped at + // DataCaps::kMaxDownloadBytes; over the cap is an error, never a truncation. + Downloaded download(const std::string& bucket, const std::string& object, + long long generation = 0) const; + // Byte range [first, last] (inclusive), one request. + Downloaded download_range(const std::string& bucket, const std::string& object, + long long first, long long last) const; + + // POST /upload/storage/v1/b/{bucket}/o?uploadType=media&name=... (<= 8 MiB). + clients::body::Json upload_simple(const std::string& bucket, const std::string& object, + const std::string& data, + const std::string& content_type) const; + + // Resumable: start returns the session URL; put sends one chunk. + // total is the full object size (known up front, <= 64 MiB). + std::string resumable_start(const std::string& bucket, const std::string& object, + const std::string& content_type, long long total, + const clients::body::Json& metadata) const; + // chunk size: multiple of 256 KiB unless it ends the object; <= 8 MiB. + ResumableProgress resumable_put(const std::string& session_url, long long offset, + const std::string& chunk, long long total) const; + ResumableProgress resumable_status(const std::string& session_url, long long total) const; + void resumable_cancel(const std::string& session_url) const; + + // Request builders (pure, used by the methods above and by tests). + std::string download_url(const std::string& bucket, const std::string& object, + long long generation) const; + std::string upload_url(const std::string& bucket, const std::string& object, + const char* upload_type) const; + static std::string check_session(const std::string& url); + + private: + clients::ClientResponse send(clients::ClientRequest request) const; + [[noreturn]] void fail(const clients::ClientResponse& r) const; + + mutable std::unique_ptr client_; + std::shared_ptr auth_; + std::string host_; +}; + +} // namespace molo::google::data diff --git a/src/google/data/tools/check_discovery.py b/src/google/data/tools/check_discovery.py new file mode 100644 index 0000000..d15dd48 --- /dev/null +++ b/src/google/data/tools/check_discovery.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Check src/google/data against Google's Discovery documents. + +Fails (exit 1) when + * a row of calls_table.inc differs from Discovery (HTTP method, path, request + body fields, query parameters, required query parameters, path placeholders), + * a Discovery method of an owned API has no row, or a row has no method + (coverage is total: every method of the 11 APIs is a row), + * a call id used by the C++ sources (string literal "api.resource.method") is + not a row, + * a nested schema field the typed code writes (SCHEMAS below) is absent, + * the Cloud Storage media contract (upload path, protocols, alt=media) changed. + +Python 3 stdlib only, no credentials: + + python3 -I src/google/data/tools/check_discovery.py +""" +import json +import os +import re +import sys +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +DATA = os.path.join(HERE, "..") +TABLE = os.path.join(DATA, "calls_table.inc") +APIS = [("bigquery", "v2"), ("storage", "v1"), ("pubsub", "v1"), ("firestore", "v1"), + ("spanner", "v1"), ("bigtableadmin", "v2"), ("secretmanager", "v1"), + ("dataplex", "v1"), ("dlp", "v2"), ("cloudkms", "v1"), ("datastore", "v1")] +PREFIXES = tuple(n + "." for n, _ in APIS) + +# Nested schema fields the typed classes write: (api, schema, fields) +SCHEMAS = [ + ("bigquery", "QueryRequest", "query,useLegacySql,useQueryCache,dryRun,maxResults,timeoutMs," + "location,defaultDataset,maximumBytesBilled,queryParameters,parameterMode"), + ("bigquery", "DatasetReference", "projectId,datasetId"), + ("bigquery", "Dataset", "datasetReference,location,description"), + ("bigquery", "TableDataInsertAllRequest", "rows,skipInvalidRows,ignoreUnknownValues"), + ("pubsub", "PubsubMessage", "data,attributes,orderingKey"), + ("pubsub", "PublishRequest", "messages"), + ("pubsub", "PullRequest", "maxMessages"), + ("pubsub", "AcknowledgeRequest", "ackIds"), + ("dlp", "GooglePrivacyDlpV2InspectConfig", "infoTypes,minLikelihood,includeQuote,limits"), + ("dlp", "GooglePrivacyDlpV2FindingLimits", "maxFindingsPerRequest"), + ("dlp", "GooglePrivacyDlpV2ContentItem", "value"), + ("dlp", "GooglePrivacyDlpV2InfoType", "name"), + ("dlp", "GooglePrivacyDlpV2InspectContentRequest", "item,inspectConfig"), + ("dlp", "GooglePrivacyDlpV2DeidentifyContentRequest", "item,deidentifyConfig"), + ("spanner", "ExecuteSqlRequest", "sql,queryMode,transaction,params,paramTypes"), + ("spanner", "TransactionSelector", "singleUse"), + ("spanner", "TransactionOptions", "readOnly"), + ("spanner", "ReadOnly", "strong"), + ("secretmanager", "Secret", "replication"), + ("secretmanager", "Replication", "automatic"), + ("secretmanager", "SecretPayload", "data"), + ("cloudkms", "EncryptRequest", "plaintext,additionalAuthenticatedData"), + ("cloudkms", "DecryptRequest", "ciphertext,additionalAuthenticatedData"), + ("cloudkms", "AsymmetricSignRequest", "digest"), + ("cloudkms", "Digest", "sha256"), + ("datastore", "TransactionOptions", "readOnly,readWrite"), + ("datastore", "CommitRequest", "mutations,mode,transaction,databaseId"), + ("datastore", "LookupRequest", "keys,databaseId"), + ("firestore", "BatchGetDocumentsRequest", "documents"), + ("firestore", "ListCollectionIdsRequest", "pageSize"), + ("firestore", "RunQueryRequest", "structuredQuery"), + ("firestore", "RunAggregationQueryRequest", "structuredAggregationQuery"), + ("firestore", "CommitRequest", "writes"), + ("firestore", "Document", "fields"), + ("storage", "ComposeRequest", "sourceObjects,destination"), + ("storage", "Bucket", "name,location,storageClass"), +] + +ROW = re.compile(r'^\s*\{("(?:[^"\\]|\\.)*"(?:,\s*"(?:[^"\\]|\\.)*"){5})\},\s*$') +STR = re.compile(r'"((?:[^"\\]|\\.)*)"') + + +def fetch(name, version): + url = "https://%s.googleapis.com/$discovery/rest?version=%s" % (name, version) + with urllib.request.urlopen(url, timeout=90) as r: + return json.load(r) + + +def methods(node, out): + for m in node.get("methods", {}).values(): + out[m["id"]] = m + for child in node.get("resources", {}).values(): + methods(child, out) + + +def main(): + rows = [] + for line in open(TABLE): + m = ROW.match(line) + if m: + rows.append(STR.findall(m.group(1))) + bad = [] + docs, index = {}, {} + for name, version in APIS: + docs[name] = fetch(name, version) + index[name] = {} + methods(docs[name], index[name]) + seen = set() + for rpc, http, path, body, query, required in rows: + api = rpc.split(".")[0] + m = index.get(api, {}).get(rpc) + seen.add(rpc) + if not m: + bad.append("missing rpc " + rpc) + continue + if m["httpMethod"] != http: + bad.append("%s: http %s != %s" % (rpc, http, m["httpMethod"])) + if m["path"] != path: + bad.append("%s: path %s != %s" % (rpc, path, m["path"])) + params = m.get("parameters", {}) + want_q = sorted(k for k, v in params.items() if v.get("location") == "query") + if sorted(filter(None, query.split(","))) != want_q: + bad.append("%s: query params differ" % rpc) + want_r = sorted(k for k in want_q if params[k].get("required")) + if sorted(filter(None, required.split(","))) != want_r: + bad.append("%s: required query params differ" % rpc) + for p in re.findall(r"\{\+?([A-Za-z0-9_]+)\}", path): + if p not in params or params[p].get("location") != "path": + bad.append("%s: path placeholder %s is not a path parameter" % (rpc, p)) + ref = m.get("request", {}).get("$ref") + props = sorted(docs[api]["schemas"].get(ref, {}).get("properties", {})) if ref else [] + if sorted(filter(None, body.split(","))) != props: + bad.append("%s: body fields differ from %s" % (rpc, ref)) + for api, idx in index.items(): + for rpc in idx: + if rpc not in seen: + bad.append("coverage: Discovery method %s has no row" % rpc) + # call ids used by the sources + lit = re.compile(r'"((?:%s)[A-Za-z0-9.]+)"' % "|".join(re.escape(p) for p in PREFIXES)) + used = 0 + for fn in sorted(os.listdir(DATA)): + if not fn.endswith(".cpp") or fn == "calls.cpp": + continue + for rpc in lit.findall(open(os.path.join(DATA, fn)).read()): + if rpc.endswith(".googleapis.com"): + continue # a host name, not a call id + used += 1 + if rpc not in seen: + bad.append("%s uses call id %s which is not a row" % (fn, rpc)) + # nested schemas + for api, schema, fields in SCHEMAS: + props = docs[api]["schemas"].get(schema, {}).get("properties") + if props is None: + bad.append("missing schema %s.%s" % (api, schema)) + continue + for f in fields.split(","): + if f not in props: + bad.append("schema %s.%s lacks field %s" % (api, schema, f)) + # Cloud Storage media contract + st = index["storage"] + ins, get = st["storage.objects.insert"], st["storage.objects.get"] + mu = ins.get("mediaUpload", {}) + # simple: Discovery path used verbatim. resumable: Discovery lists + # /resumable/upload/...; the code starts sessions on the documented + # /upload/...?uploadType=resumable (same host, see SPEC Unverified). + proto = mu.get("protocols", {}) + if not ins.get("supportsMediaUpload") or \ + proto.get("simple", {}).get("path") != "/upload/storage/v1/b/{bucket}/o" or \ + proto.get("resumable", {}).get("path") != "/resumable/upload/storage/v1/b/{bucket}/o": + bad.append("storage media upload contract changed") + if mu.get("maxSize") and int(re.sub(r"\D", "", mu["maxSize"]) or 0) < 1: + bad.append("storage upload maxSize unreadable") + alt = get["parameters"].get("alt") or docs["storage"]["parameters"].get("alt", {}) + if not get.get("supportsMediaDownload") or "media" not in alt.get("enum", []): + bad.append("storage objects.get media download contract changed") + for q in ("name", "uploadType"): + if q not in ins["parameters"] and q not in docs["storage"]["parameters"]: + bad.append("storage objects.insert lacks parameter %s" % q) + for api, version in APIS: + print("%s %s revision %s: %d methods" % (api, version, docs[api]["revision"], len(index[api]))) + if bad: + print("\n".join(bad[:60])) + print("FAILED: %d problems" % len(bad)) + return 1 + print("discovery ok: %d rows, %d call ids used by sources, %d nested schemas, storage media contract" + % (len(rows), used, len(SCHEMAS))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/google/data/tools/gen_calls.py b/src/google/data/tools/gen_calls.py new file mode 100644 index 0000000..b230eb6 --- /dev/null +++ b/src/google/data/tools/gen_calls.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Generate the call table of src/google/data/calls.cpp from Discovery. + + python3 -I src/google/data/tools/gen_calls.py > src/google/data/calls_table.inc + +Every method of the 11 owned APIs becomes one row: + {rpc id, HTTP method, discovery path, body fields, query params, required query params} +body fields are all top-level properties of the request schema; query params +are all parameters with location=query. Python 3 stdlib only, no credentials. +""" +import json +import sys +import urllib.request + +APIS = [("bigquery", "v2"), ("storage", "v1"), ("pubsub", "v1"), ("firestore", "v1"), + ("spanner", "v1"), ("bigtableadmin", "v2"), ("secretmanager", "v1"), + ("dataplex", "v1"), ("dlp", "v2"), ("cloudkms", "v1"), ("datastore", "v1")] + + +def fetch(name, version): + url = "https://%s.googleapis.com/$discovery/rest?version=%s" % (name, version) + with urllib.request.urlopen(url, timeout=60) as r: + return json.load(r) + + +def methods(node, out): + for m in node.get("methods", {}).values(): + out.append(m) + for child in node.get("resources", {}).values(): + methods(child, out) + + +def main(): + print("// GENERATED by tools/gen_calls.py from Google Discovery; do not edit by hand.") + for name, version in APIS: + doc = fetch(name, version) + print("// %s %s revision %s" % (name, version, doc["revision"])) + found = [] + methods(doc, found) + for m in sorted(found, key=lambda x: x["id"]): + params = m.get("parameters", {}) + query = sorted(k for k, v in params.items() if v.get("location") == "query") + req = sorted(k for k in query if params[k].get("required")) + ref = m.get("request", {}).get("$ref") + body = sorted(doc["schemas"].get(ref, {}).get("properties", {})) if ref else [] + print(' {"%s", "%s", "%s", "%s", "%s", "%s"},' % ( + m["id"], m["httpMethod"], m["path"], ",".join(body), ",".join(query), ",".join(req))) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/example/google/data/call_test.cpp b/tests/example/google/data/call_test.cpp new file mode 100644 index 0000000..c77603c --- /dev/null +++ b/tests/example/google/data/call_test.cpp @@ -0,0 +1,48 @@ +// Routes: POST /google/data/call and /google/data/storage/sign +// The generic call route validates against the Discovery table; writes are +// dry runs unless confirm:true; the signer is a dry run unless confirm:true. +#include + +#include "support/GoogleRig.hpp" + +using namespace molo::example::test; + +int main() { + setenv("GOOGLE_CLOUD_PROJECT", "test-project", 1); + GoogleRig rig(false); + Fixture f(rig.client); + auto j = [&](const char* body) { return f.client->post("/google/data/call", body, "application/json"); }; + + auto read = j("{\"rpc\":\"bigquery.datasets.list\",\"path\":{\"projectId\":\"p1\"},\"query\":{\"maxResults\":5}}"); + check(read.status == 503 && contains(read.body, "google_not_configured"), "valid GET without credential is 503"); + auto post = j("{\"rpc\":\"pubsub.projects.topics.publish\",\"path\":{\"topic\":\"projects/p1/topics/t1\"},\"body\":{\"messages\":[]}}"); + check(post.status == 200 && contains(post.body, "\"dry_run\":true") && + contains(post.body, "https://pubsub.googleapis.com/v1/projects/p1/topics/t1:publish"), + "non-GET is a dry run that shows the URL"); + auto confirmed = j("{\"rpc\":\"pubsub.projects.topics.publish\",\"path\":{\"topic\":\"projects/p1/topics/t1\"},\"body\":{\"messages\":[]},\"confirm\":true}"); + check(confirmed.status == 503, "confirmed non-GET without credential is 503"); + check(j("{}").status == 422, "missing rpc is 422"); + check(j("{\"rpc\":\"nope.x\"}").status == 422, "unknown rpc is 422"); + check(j("{\"rpc\":\"bigquery.datasets.list\"}").status == 422, "missing path value is 422"); + check(j("{\"rpc\":\"bigquery.datasets.list\",\"path\":{\"projectId\":\"../x\"}}").status == 422, "path traversal is 422"); + check(j("{\"rpc\":\"bigquery.datasets.list\",\"path\":{\"projectId\":\"p\"},\"query\":{\"zzz\":1}}").status == 422, "unknown query parameter is 422"); + check(j("{\"rpc\":\"bigquery.jobs.query\",\"path\":{\"projectId\":\"p\"},\"body\":{\"zzz\":1}}").status == 422, "unknown body field is 422"); + check(j("{\"rpc\":\"bigquery.datasets.list\",\"path\":{\"projectId\":5}}").status == 422, "non-string path value is 422"); + check(f.client->get("/google/data/call").status == 405, "get is 405"); + + auto sign = [&](const char* body) { return f.client->post("/google/data/storage/sign", body, "application/json"); }; + auto dry = sign("{\"bucket\":\"my-bucket\",\"object\":\"a/b.txt\",\"expires_seconds\":600}"); + check(dry.status == 200 && contains(dry.body, "\"dry_run\":true") && !contains(dry.body, "X-Goog-Signature"), + "signing is a dry run by default"); + check(sign("{\"bucket\":\"my-bucket\",\"expires_seconds\":604801}").status == 422, "expiry over 7 days is 422"); + check(sign("{\"bucket\":\"my-bucket\",\"expires_seconds\":0}").status == 422, "expiry 0 is 422"); + check(sign("{\"bucket\":\"Bad\"}").status == 422, "bad bucket is 422"); + check(sign("{\"bucket\":\"my-bucket\",\"method\":\"TRACE\"}").status == 422, "bad method is 422"); + check(sign("{}").status == 422, "missing bucket is 422"); + check(sign("{\"bucket\":\"my-bucket\",\"headers\":{\"host\":\"x\"}}").status == 422, "host header is 422"); + auto real = sign("{\"bucket\":\"my-bucket\",\"object\":\"o\",\"confirm\":true}"); + check(real.status == 503 && contains(real.body, "google_not_configured"), "confirmed signing without a key is 503"); + check(f.client->get("/google/data/storage/sign").status == 405, "sign get is 405"); + check(rig.mock->send_count() == 0, "nothing reached Google"); + return finish("google/data/call"); +} diff --git a/tests/example/google/data/groups_test.cpp b/tests/example/google/data/groups_test.cpp new file mode 100644 index 0000000..41ec7c9 --- /dev/null +++ b/tests/example/google/data/groups_test.cpp @@ -0,0 +1,123 @@ +// Routes: POST /google/data/{bigquery,storage,pubsub,firestore,spanner,bigtable, +// secrets,dataplex,dlp,kms,datastore} +// Without a credential a well formed read is 503 google_not_configured; bad +// input is 422 and never reaches Google; writes, secret reads and paid calls +// are dry runs unless confirm:true; GET is 405. Nothing reaches Google. +#include + +#include "support/GoogleRig.hpp" + +using namespace molo::example::test; + +namespace { +struct Case { + const char* path; + const char* ok; // valid read, expected 503 without credential + const char* bad; // invalid input, expected 422 +}; + +const Case kReads[] = { + {"/google/data/bigquery", "{\"action\":\"dry_run\",\"sql\":\"SELECT 1\"}", "{\"action\":\"dry_run\"}"}, + {"/google/data/bigquery", "{\"action\":\"datasets\"}", "{\"action\":\"datasets\",\"max_results\":99999}"}, + {"/google/data/bigquery", "{\"action\":\"table\",\"dataset\":\"d\",\"table\":\"t\"}", "{\"action\":\"table\",\"dataset\":\"bad-id\",\"table\":\"t\"}"}, + {"/google/data/bigquery", "{\"action\":\"rows\",\"dataset\":\"d\",\"table\":\"t\"}", "{\"action\":\"rows\",\"dataset\":\"d\"}"}, + {"/google/data/bigquery", "{\"action\":\"dry_run\",\"sql\":\"SELECT 1\",\"timeout_ms\":1000}", "{\"action\":\"dry_run\",\"sql\":\"x\",\"timeout_ms\":999999}"}, + {"/google/data/storage", "{\"action\":\"buckets\"}", "{\"action\":\"buckets\",\"max_results\":0}"}, + {"/google/data/storage", "{\"action\":\"objects\",\"bucket\":\"my-bucket\"}", "{\"action\":\"objects\",\"bucket\":\"Bad_Bucket\"}"}, + {"/google/data/storage", "{\"action\":\"object\",\"bucket\":\"my-bucket\",\"object\":\"a/b.txt\"}", "{\"action\":\"object\",\"bucket\":\"my-bucket\",\"object\":\"..\"}"}, + {"/google/data/storage", "{\"action\":\"download\",\"bucket\":\"my-bucket\",\"object\":\"a.txt\"}", "{\"action\":\"download\",\"bucket\":\"my-bucket\"}"}, + {"/google/data/pubsub", "{\"action\":\"topics\"}", "{\"action\":\"topic\",\"topic\":\"1bad\"}"}, + {"/google/data/pubsub", "{\"action\":\"subscription\",\"subscription\":\"sub-1\"}", "{\"action\":\"subscription\"}"}, + {"/google/data/firestore", "{\"action\":\"get\",\"path\":\"users/alice\"}", "{\"action\":\"get\",\"path\":\"users//alice\"}"}, + {"/google/data/firestore", "{\"action\":\"collections\"}", "{\"action\":\"list\",\"collection\":\"\"}"}, + {"/google/data/firestore", "{\"action\":\"query\",\"query\":{\"from\":[{\"collectionId\":\"c\"}]}}", "{\"action\":\"query\"}"}, + {"/google/data/spanner", "{\"action\":\"instances\"}", "{\"action\":\"instance\",\"instance\":\"a/b\"}"}, + {"/google/data/spanner", "{\"action\":\"ddl\",\"instance\":\"i1\",\"database\":\"d1\"}", "{\"action\":\"ddl\",\"instance\":\"i1\"}"}, + {"/google/data/bigtable", "{\"action\":\"instances\"}", "{\"action\":\"instance\",\"instance\":\"a b\"}"}, + {"/google/data/bigtable", "{\"action\":\"tables\",\"instance\":\"i1\"}", "{\"action\":\"tables\"}"}, + {"/google/data/secrets", "{\"action\":\"list\"}", "{\"action\":\"version\",\"secret\":\"s\",\"version\":\"../x\"}"}, + {"/google/data/secrets", "{\"action\":\"versions\",\"secret\":\"s1\"}", "{\"action\":\"versions\"}"}, + {"/google/data/dataplex", "{\"action\":\"locations\"}", "{\"action\":\"search\",\"query\":\"\"}"}, + {"/google/data/dataplex", "{\"action\":\"search\",\"location\":\"global\",\"query\":\"type=table\"}", "{\"action\":\"lookup\",\"entry\":\"nope\"}"}, + {"/google/data/dlp", "{\"action\":\"info_types\"}", "{\"action\":\"info_types\",\"language\":\"aaaaaaaaaaaaaaaaaaaaaaaa\"}"}, + {"/google/data/kms", "{\"action\":\"rings\",\"location\":\"global\"}", "{\"action\":\"rings\"}"}, + {"/google/data/kms", "{\"action\":\"public_key\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"version\":\"1\"}", "{\"action\":\"public_key\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"version\":\"x\"}"}, + {"/google/data/datastore", "{\"action\":\"query\",\"gql\":{\"queryString\":\"SELECT * FROM k\"}}", "{\"action\":\"query\"}"}, + {"/google/data/datastore", "{\"action\":\"lookup\",\"keys\":[{\"path\":[{\"kind\":\"k\",\"id\":\"1\"}]}]}", "{\"action\":\"lookup\",\"keys\":[]}"}, +}; + +// Confirm-required: dry run without confirm, 503 with it, 422 on bad input when confirmed. +struct Gated { + const char* path; + const char* body; // no confirm + const char* confirmed; // with confirm:true + const char* bad; // confirm:true plus invalid input +}; +const Gated kGated[] = { + {"/google/data/bigquery", "{\"action\":\"query\",\"sql\":\"SELECT 1\"}", "{\"action\":\"query\",\"sql\":\"SELECT 1\",\"confirm\":true}", "{\"action\":\"query\",\"sql\":\"\",\"confirm\":true}"}, + {"/google/data/bigquery", "{\"action\":\"insert\",\"dataset\":\"d\",\"table\":\"t\",\"rows\":[{\"json\":{}}]}", "{\"action\":\"insert\",\"dataset\":\"d\",\"table\":\"t\",\"rows\":[{\"json\":{}}],\"confirm\":true}", "{\"action\":\"insert\",\"dataset\":\"d\",\"table\":\"t\",\"rows\":[],\"confirm\":true}"}, + {"/google/data/bigquery", "{\"action\":\"delete_table\",\"dataset\":\"d\",\"table\":\"t\"}", "{\"action\":\"delete_table\",\"dataset\":\"d\",\"table\":\"t\",\"confirm\":true}", "{\"action\":\"delete_table\",\"dataset\":\"d\",\"table\":\"a/b\",\"confirm\":true}"}, + {"/google/data/bigquery", "{\"action\":\"delete_dataset\",\"dataset\":\"d\"}", "{\"action\":\"delete_dataset\",\"dataset\":\"d\",\"confirm\":true}", "{\"action\":\"delete_dataset\",\"dataset\":\"d-d\",\"confirm\":true}"}, + {"/google/data/storage", "{\"action\":\"upload\",\"bucket\":\"my-bucket\",\"object\":\"o\",\"text\":\"hi\"}", "{\"action\":\"upload\",\"bucket\":\"my-bucket\",\"object\":\"o\",\"text\":\"hi\",\"confirm\":true}", "{\"action\":\"upload\",\"bucket\":\"my-bucket\",\"object\":\"o\",\"confirm\":true}"}, + {"/google/data/storage", "{\"action\":\"resumable\",\"bucket\":\"my-bucket\",\"object\":\"o\",\"text\":\"hi\"}", "{\"action\":\"resumable\",\"bucket\":\"my-bucket\",\"object\":\"o\",\"text\":\"hi\",\"confirm\":true}", "{\"action\":\"resumable\",\"bucket\":\"my-bucket\",\"object\":\"o\",\"text\":\"a\",\"base64\":\"YQ==\",\"confirm\":true}"}, + {"/google/data/storage", "{\"action\":\"compose\",\"bucket\":\"my-bucket\",\"sources\":[\"a\"],\"destination\":\"d\"}", "{\"action\":\"compose\",\"bucket\":\"my-bucket\",\"sources\":[\"a\"],\"destination\":\"d\",\"confirm\":true}", "{\"action\":\"compose\",\"bucket\":\"my-bucket\",\"sources\":[],\"destination\":\"d\",\"confirm\":true}"}, + {"/google/data/storage", "{\"action\":\"delete\",\"bucket\":\"my-bucket\",\"object\":\"o\"}", "{\"action\":\"delete\",\"bucket\":\"my-bucket\",\"object\":\"o\",\"confirm\":true}", "{\"action\":\"delete\",\"bucket\":\"my-bucket\",\"object\":\"\",\"confirm\":true}"}, + {"/google/data/pubsub", "{\"action\":\"publish\",\"topic\":\"topic-1\",\"messages\":[\"x\"]}", "{\"action\":\"publish\",\"topic\":\"topic-1\",\"messages\":[\"x\"],\"confirm\":true}", "{\"action\":\"publish\",\"topic\":\"topic-1\",\"messages\":[],\"confirm\":true}"}, + {"/google/data/pubsub", "{\"action\":\"pull\",\"subscription\":\"sub-1\"}", "{\"action\":\"pull\",\"subscription\":\"sub-1\",\"confirm\":true}", "{\"action\":\"pull\",\"subscription\":\"sub-1\",\"max_messages\":500,\"confirm\":true}"}, + {"/google/data/pubsub", "{\"action\":\"delete_topic\",\"topic\":\"topic-1\"}", "{\"action\":\"delete_topic\",\"topic\":\"topic-1\",\"confirm\":true}", "{\"action\":\"delete_topic\",\"topic\":\"1\",\"confirm\":true}"}, + {"/google/data/firestore", "{\"action\":\"delete\",\"path\":\"a/b\"}", "{\"action\":\"delete\",\"path\":\"a/b\",\"confirm\":true}", "{\"action\":\"delete\",\"path\":\"\",\"confirm\":true}"}, + {"/google/data/firestore", "{\"action\":\"commit\",\"writes\":[{}]}", "{\"action\":\"commit\",\"writes\":[{}],\"confirm\":true}", "{\"action\":\"commit\",\"writes\":[],\"confirm\":true}"}, + {"/google/data/spanner", "{\"action\":\"query\",\"instance\":\"i\",\"database\":\"d\",\"sql\":\"SELECT 1\"}", "{\"action\":\"query\",\"instance\":\"i\",\"database\":\"d\",\"sql\":\"SELECT 1\",\"confirm\":true}", "{\"action\":\"query\",\"instance\":\"i\",\"database\":\"d\",\"sql\":\"SELECT 1\",\"query_mode\":\"BOOM\",\"confirm\":true}"}, + {"/google/data/secrets", "{\"action\":\"access\",\"secret\":\"s\",\"version\":\"latest\"}", "{\"action\":\"access\",\"secret\":\"s\",\"version\":\"latest\",\"confirm\":true}", "{\"action\":\"access\",\"secret\":\"s\",\"version\":\"abc\",\"confirm\":true}"}, + {"/google/data/secrets", "{\"action\":\"add_version\",\"secret\":\"s\",\"payload\":\"v\"}", "{\"action\":\"add_version\",\"secret\":\"s\",\"payload\":\"v\",\"confirm\":true}", "{\"action\":\"add_version\",\"secret\":\"s\",\"confirm\":true}"}, + {"/google/data/secrets", "{\"action\":\"delete\",\"secret\":\"s\"}", "{\"action\":\"delete\",\"secret\":\"s\",\"confirm\":true}", "{\"action\":\"delete\",\"secret\":\"a/b\",\"confirm\":true}"}, + {"/google/data/dlp", "{\"action\":\"inspect\",\"text\":\"a@b.com\"}", "{\"action\":\"inspect\",\"text\":\"a@b.com\",\"confirm\":true}", "{\"action\":\"inspect\",\"text\":\"a\",\"min_likelihood\":\"MAYBE\",\"confirm\":true}"}, + {"/google/data/kms", "{\"action\":\"encrypt\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"plaintext\":\"x\"}", "{\"action\":\"encrypt\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"plaintext\":\"x\",\"confirm\":true}", "{\"action\":\"encrypt\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"plaintext\":\"\",\"confirm\":true}"}, + {"/google/data/kms", "{\"action\":\"decrypt\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"ciphertext\":\"YQ==\"}", "{\"action\":\"decrypt\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"ciphertext\":\"YQ==\",\"confirm\":true}", "{\"action\":\"decrypt\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"ciphertext\":\"!!\",\"confirm\":true}"}, + {"/google/data/datastore", "{\"action\":\"commit\",\"mutations\":[{}]}", "{\"action\":\"commit\",\"mutations\":[{}],\"confirm\":true}", "{\"action\":\"commit\",\"mutations\":[{}],\"mode\":\"MAYBE\",\"confirm\":true}"}, +}; +} // namespace + +int main() { + setenv("GOOGLE_CLOUD_PROJECT", "test-project", 1); + GoogleRig rig(false); + Fixture f(rig.client); + for (const Case& c : kReads) { + auto ok = f.client->post(c.path, c.ok, "application/json"); + check(ok.status == 503 && contains(ok.body, "google_not_configured"), + std::string(c.path) + " read without credential is 503: " + c.ok); + auto bad = f.client->post(c.path, c.bad, "application/json"); + check(bad.status == 422, std::string(c.path) + " bad input is 422: " + c.bad); + } + for (const Gated& g : kGated) { + auto dry = f.client->post(g.path, g.body, "application/json"); + check(dry.status == 200 && contains(dry.body, "\"dry_run\":true"), + std::string(g.path) + " default is a dry run: " + g.body); + auto real = f.client->post(g.path, g.confirmed, "application/json"); + check(real.status == 503 && contains(real.body, "google_not_configured"), + std::string(g.path) + " confirmed without credential is 503: " + g.confirmed); + check(f.client->post(g.path, g.bad, "application/json").status == 422, + std::string(g.path) + " confirmed bad input is 422: " + g.bad); + std::string odd = g.body; + odd.pop_back(); + odd += ",\"confirm\":\"yes\"}"; + check(f.client->post(g.path, odd, "application/json").status == 422, + std::string(g.path) + " non-bool confirm is 422"); + } + for (const char* path : {"/google/data/bigquery", "/google/data/storage", "/google/data/pubsub", + "/google/data/firestore", "/google/data/spanner", "/google/data/bigtable", + "/google/data/secrets", "/google/data/dataplex", "/google/data/dlp", + "/google/data/kms", "/google/data/datastore"}) { + check(f.client->get(path).status == 405, std::string(path) + " get is 405"); + check(f.client->post(path, "{}", "application/json").status == 422, std::string(path) + " missing action is 422"); + check(f.client->post(path, "{\"action\":\"explode\"}", "application/json").status == 422, + std::string(path) + " unknown action is 422"); + check(f.client->post(path, "not json", "application/json").status == 422, std::string(path) + " invalid json is 422"); + check(f.client->post(path, "[]", "application/json").status == 422, std::string(path) + " non-object body is 422"); + check(f.client->post(path, std::string("{\"action\":\"") + std::string(3 * 1024 * 1024, 'x') + "\"}", + "application/json").status == 413, + std::string(path) + " oversize body is 413"); + } + check(rig.mock->send_count() == 0, "nothing reached Google"); + return finish("google/data/groups"); +} diff --git a/tests/example/google/data/page_test.cpp b/tests/example/google/data/page_test.cpp new file mode 100644 index 0000000..61bbab0 --- /dev/null +++ b/tests/example/google/data/page_test.cpp @@ -0,0 +1,20 @@ +// Route: GET /google/data (page, no credential needed) +#include "support/GoogleRig.hpp" + +using namespace molo::example::test; + +int main() { + setenv("GOOGLE_CLOUD_PROJECT", "test-project", 1); + GoogleRig rig(false); + Fixture f(rig.client); + auto res = f.client->get("/google/data"); + check(res.status == 200, "status 200"); + check(contains(res.headers.get("Content-Type"), "text/html"), "html"); + check(contains(res.body, "Google data") && contains(res.body, "viewport"), "page and mobile viewport"); + check(contains(res.body, "/embedded/google_data.js"), "script wired"); + check(contains(res.body, "