feat(clients): data: BigQuery, Storage media and signed URLs, Pub/Sub, Firestore, Spanner, Bigtable admin, Secret Manager, Dataplex, DLP, KMS, Datastore

Discovery generated call table (918 rows), typed services, /google/data demo.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-25 15:03:43 +01:00
co-authored by Claude Sonnet 5.5
parent 3db67fcf1f
commit 1af14b5d74
47 changed files with 6315 additions and 3 deletions
+424
View File
@@ -0,0 +1,424 @@
// data package: deterministic local tests (call table, request rendering,
// validation, caps, typed request bodies, no-credential behavior). No scripted
// Google replies: shapes are verified against Discovery by
// src/google/data/tools/check_discovery.py and live (test_data_live.cpp).
#include <iostream>
#include <memory>
#include "framework.h"
#include "clients/transports/mock_transport.h"
#include "google/data/data.h"
#include "google/data/caps.h"
using namespace molo::google;
using namespace molo::google::data;
using molo::clients::Method;
using molo::clients::body::Json;
using molo::clients::transports::MockTransport;
namespace {
struct Rig {
MockTransport* mock;
std::shared_ptr<GoogleClient> client;
std::unique_ptr<DataPlatform> d;
Rig() {
auto t = std::make_unique<MockTransport>();
mock = t.get();
client = std::make_shared<GoogleClient>(std::move(t), nullptr);
CloudContext ctx;
ctx.project = "p1";
ctx.location = "us-central1";
d = std::make_unique<DataPlatform>(client, ctx);
}
};
// passes validation (stops at the credential check) or is invalid.
template <typename F>
bool invalid(F&& f) {
try {
f();
} catch (const NotConfiguredError&) {
return false;
} catch (const GoogleError& e) {
return e.http_status() == 400 && e.status() == "INVALID_ARGUMENT";
}
return false;
}
template <typename F>
bool valid(F&& f) {
try {
f();
} catch (const NotConfiguredError&) {
return true;
} catch (const GoogleError&) {
return false;
}
return true;
}
} // namespace
TEST(data, catalog_covers_all_eleven_apis) {
ASSERT_EQ(Catalog::api_count(), 11u);
std::size_t total = 0;
for (std::size_t i = 0; i < Catalog::api_count(); ++i) {
std::size_t n = Catalog::count_of(Catalog::info_at(i).api);
ASSERT_TRUE(n >= 18);
total += n;
}
ASSERT_EQ(total, Catalog::size());
ASSERT_TRUE(Catalog::size() > 900);
ASSERT_TRUE(Catalog::find("bigquery.jobs.query") != nullptr);
ASSERT_TRUE(Catalog::find("storage.objects.compose") != nullptr);
ASSERT_TRUE(Catalog::find("nope.x") == nullptr);
ASSERT_THROW(Catalog::get("nope.x"), GoogleError);
ASSERT_EQ(std::string(Catalog::get("pubsub.projects.topics.publish").http), "POST");
ASSERT_EQ(Catalog::endpoint(Api::kBigquery).base_url,
"https://bigquery.googleapis.com/bigquery/v2");
}
TEST(data, path_params_are_read_from_the_template) {
auto p = Catalog::path_params(Catalog::get("bigquery.tabledata.insertAll"));
ASSERT_EQ(p.size(), 3u);
ASSERT_EQ(p[0], "projectId");
ASSERT_EQ(p[2], "tableId");
auto q = Catalog::path_params(Catalog::get("storage.objects.compose"));
ASSERT_EQ(q[0], "destinationBucket");
}
TEST(data, base_url_override_is_google_only) {
ASSERT_TRUE(valid([] { Catalog::endpoint(Api::kDlp, "https://dlp.europe-west1.rep.googleapis.com"); }));
ASSERT_TRUE(invalid([] { Catalog::endpoint(Api::kDlp, "http://dlp.googleapis.com"); }));
ASSERT_TRUE(invalid([] { Catalog::endpoint(Api::kDlp, "https://evil.example.com"); }));
ASSERT_TRUE(invalid([] { Catalog::endpoint(Api::kDlp, "https://googleapis.com.evil.com"); }));
ASSERT_TRUE(invalid([] { Catalog::endpoint(Api::kDlp, "https://dlp.googleapis.com/"); }));
}
TEST(data, prepare_renders_paths_queries_and_bodies) {
Rig r;
Prepared p = r.d->bigquery.prepare(
"bigquery.datasets.list", CallArgs().p("projectId", "p1").q("maxResults", Json(5)));
ASSERT_EQ(p.url(), "https://bigquery.googleapis.com/bigquery/v2/projects/p1/datasets?maxResults=5");
ASSERT_TRUE(p.method == Method::kGet);
ASSERT_FALSE(p.has_body);
Prepared s = r.d->storage.prepare(
"storage.objects.get",
CallArgs().p("bucket", "my-bucket").p("object", "dir/my file+1.txt"));
ASSERT_EQ(s.path, "/b/my-bucket/o/dir%2Fmy%20file%2B1.txt");
Prepared pub = r.d->pubsub.prepare(
"pubsub.projects.topics.publish",
CallArgs().p("topic", "projects/p1/topics/t1").with_body(Json::parse("{\"messages\":[]}")));
ASSERT_EQ(pub.path, "/v1/projects/p1/topics/t1:publish");
ASSERT_TRUE(pub.method == Method::kPost && pub.has_body);
// POST whose request has fields gets an empty object when no body is given.
Prepared sess = r.d->spanner.prepare(
"spanner.projects.instances.databases.sessions.create",
CallArgs().p("database", "projects/p/instances/i/databases/d"));
ASSERT_TRUE(sess.has_body);
ASSERT_EQ(sess.body.dump(), "{}");
// Arrays repeat the parameter.
Prepared rep = r.d->bigquery.prepare(
"bigquery.tabledata.list",
CallArgs().p("projectId", "p1").p("datasetId", "d").p("tableId", "t")
.q("selectedFields", Json::parse("[\"a\",\"b\"]")));
ASSERT_TRUE(rep.path.find("selectedFields=a&selectedFields=b") != std::string::npos);
}
TEST(data, prepare_rejects_bad_input_before_any_traffic) {
Rig r;
auto& bq = r.d->bigquery;
ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.datasets.list", CallArgs()); })); // path missing
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "p1").p("extra", "x"));
}));
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.datasets.list",
CallArgs().p("projectId", "p1").q("notAParam", Json(1)));
}));
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "../x"));
}));
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "p1/../../x"));
}));
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "p1?x=1"));
}));
ASSERT_TRUE(invalid([&] { bq.prepare("storage.buckets.list", CallArgs()); })); // other api
ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.no.such", CallArgs()); }));
// no body allowed on a GET
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.datasets.list",
CallArgs().p("projectId", "p1").with_body(Json::object()));
}));
// unknown body field
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.jobs.query",
CallArgs().p("projectId", "p1").with_body(Json::parse("{\"bogus\":1}")));
}));
// required query parameter (storage.buckets.list needs project)
ASSERT_TRUE(invalid([&] { r.d->storage.prepare("storage.buckets.list", CallArgs()); }));
// control characters in a query value
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.datasets.list",
CallArgs().p("projectId", "p1").q("filter", Json("a\nb")));
}));
// too many query parameters via array
Json many = Json::array();
for (int i = 0; i < 101; ++i) many.push(Json(i));
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.tabledata.list",
CallArgs().p("projectId", "p1").p("datasetId", "d").p("tableId", "t")
.q("selectedFields", many));
}));
}
TEST(data, body_size_is_capped) {
Rig r;
Json big = Json::object();
big.set("query", Json(std::string(DataCaps::kMaxBodyBytes + 10, 'x')));
ASSERT_TRUE(invalid([&] {
r.d->bigquery.prepare("bigquery.jobs.query", CallArgs().p("projectId", "p1").with_body(big));
}));
}
TEST(data, no_credential_is_not_configured_after_validation) {
Rig r;
bool threw = false;
try {
r.d->bigquery.list_datasets(10);
} catch (const NotConfiguredError& e) {
threw = true;
ASSERT_EQ(e.http_status(), 503);
}
ASSERT_TRUE(threw);
ASSERT_EQ(r.mock->send_count(), 0u);
}
TEST(data, ids_follow_google_rules) {
ASSERT_TRUE(valid([] { Ids::bucket("my-bucket_1.x"); }));
ASSERT_TRUE(invalid([] { Ids::bucket("ab"); }));
ASSERT_TRUE(invalid([] { Ids::bucket("-bad-"); }));
ASSERT_TRUE(invalid([] { Ids::bucket("Upper"); }));
ASSERT_TRUE(valid([] { Ids::object("a/b/c d.txt"); }));
ASSERT_TRUE(invalid([] { Ids::object(""); }));
ASSERT_TRUE(invalid([] { Ids::object(".."); }));
ASSERT_TRUE(invalid([] { Ids::object(std::string(1025, 'a')); }));
ASSERT_TRUE(valid([] { Ids::pubsub("my-topic.v1"); }));
ASSERT_TRUE(invalid([] { Ids::pubsub("1abc"); }));
ASSERT_TRUE(invalid([] { Ids::pubsub("ab"); }));
ASSERT_TRUE(valid([] { Ids::project("example.com:proj-1"); }));
ASSERT_TRUE(invalid([] { Ids::project("Proj"); }));
ASSERT_TRUE(valid([] { Ids::dataset("my_data_1"); }));
ASSERT_TRUE(invalid([] { Ids::dataset("my-data"); }));
ASSERT_TRUE(invalid([] { Ids::dataset("a.b"); }));
}
TEST(data, bigquery_query_body_and_caps) {
QueryRequest q;
q.sql = "SELECT 1";
q.dry_run = true;
q.default_dataset = "ds";
Json b = q.to_json("p1");
ASSERT_EQ(b.find("query")->as_string(), "SELECT 1");
ASSERT_TRUE(b.find("dryRun")->as_bool());
ASSERT_FALSE(b.find("useLegacySql")->as_bool());
ASSERT_EQ(b.find("defaultDataset")->find("projectId")->as_string(), "p1");
ASSERT_EQ(b.find("timeoutMs")->as_int(), 10000);
Rig r;
QueryRequest bad;
ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); })); // empty sql
bad.sql = std::string(DataCaps::kMaxSqlChars + 1, 'x');
ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); }));
bad.sql = "SELECT 1";
bad.max_results = 10001;
ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); }));
bad.max_results = 10;
bad.timeout_ms = 60001;
ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); }));
bad.timeout_ms = 1;
bad.maximum_bytes_billed = "12a";
ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); }));
bad.maximum_bytes_billed = "";
bad.query_parameters = Json::array();
ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); })); // parameter_mode missing
bad.parameter_mode = "NAMED";
ASSERT_TRUE(valid([&] { r.d->bigquery.query(bad); }));
}
TEST(data, bigquery_insert_all_validation) {
Rig r;
auto& bq = r.d->bigquery;
Json rows = Json::parse("[{\"insertId\":\"1\",\"json\":{\"a\":1}}]");
ASSERT_TRUE(valid([&] { bq.insert_all("ds", "t1", rows, false, false); }));
ASSERT_TRUE(invalid([&] { bq.insert_all("ds", "t1", Json::array(), false, false); }));
ASSERT_TRUE(invalid([&] { bq.insert_all("ds", "t1", Json::parse("[{\"a\":1}]"), false, false); }));
Json many = Json::array();
for (std::size_t i = 0; i <= DataCaps::kMaxInsertRows; ++i) many.push(Json::parse("{\"json\":{}}"));
ASSERT_TRUE(invalid([&] { bq.insert_all("ds", "t1", many, false, false); }));
ASSERT_TRUE(invalid([&] { bq.insert_all("bad-ds", "t1", rows, false, false); }));
ASSERT_TRUE(invalid([&] { bq.insert_all("ds", "t/1", rows, false, false); }));
}
TEST(data, storage_compose_body_and_caps) {
std::vector<ComposeSource> src = {{"a.txt", 0}, {"b.txt", 12345}};
Json b = Storage::compose_body(src, "text/plain");
ASSERT_EQ(b.find("sourceObjects")->as_array().size(), 2u);
ASSERT_EQ(b.find("sourceObjects")->as_array()[1].find("generation")->as_string(), "12345");
ASSERT_EQ(b.find("destination")->find("contentType")->as_string(), "text/plain");
ASSERT_TRUE(invalid([] { Storage::compose_body({}, ""); }));
std::vector<ComposeSource> too_many(DataCaps::kMaxComposeSources + 1, ComposeSource{"x", 0});
ASSERT_TRUE(invalid([&] { Storage::compose_body(too_many, ""); }));
std::vector<ComposeSource> max(DataCaps::kMaxComposeSources, ComposeSource{"x", 0});
ASSERT_TRUE(valid([&] { Storage::compose_body(max, ""); }));
Rig r;
ASSERT_TRUE(invalid([&] { r.d->storage.compose("Bad", src, "out", ""); }));
ASSERT_TRUE(valid([&] { r.d->storage.compose("good-bucket", src, "out", ""); }));
}
TEST(data, storage_create_bucket_class_is_checked) {
Rig r;
ASSERT_TRUE(invalid([&] { r.d->storage.create_bucket("good-bucket", "US", "FAST"); }));
ASSERT_TRUE(valid([&] { r.d->storage.create_bucket("good-bucket", "US", "STANDARD"); }));
}
TEST(data, pubsub_publish_body_is_base64_and_capped) {
PubsubMessage m;
m.data = "hello";
m.attributes["k"] = "v";
m.ordering_key = "o1";
Json b = PubSub::publish_body({m});
const Json& msg = b.find("messages")->as_array()[0];
ASSERT_EQ(msg.find("data")->as_string(), "aGVsbG8=");
ASSERT_EQ(msg.find("attributes")->find("k")->as_string(), "v");
ASSERT_EQ(msg.find("orderingKey")->as_string(), "o1");
ASSERT_TRUE(invalid([] { PubSub::publish_body({}); }));
ASSERT_TRUE(invalid([] { PubSub::publish_body({PubsubMessage{}}); }));
std::vector<PubsubMessage> many(DataCaps::kMaxPublishMessages + 1, m);
ASSERT_TRUE(invalid([&] { PubSub::publish_body(many); }));
PubsubMessage big;
big.data = std::string(DataCaps::kMaxMessageBytes + 1, 'x');
ASSERT_TRUE(invalid([&] { PubSub::publish_body({big}); }));
Rig r;
ASSERT_TRUE(invalid([&] { r.d->pubsub.pull("sub-1", 0); }));
ASSERT_TRUE(invalid([&] { r.d->pubsub.pull("sub-1", 101); }));
ASSERT_TRUE(valid([&] { r.d->pubsub.pull("sub-1", 10); }));
ASSERT_TRUE(invalid([&] { r.d->pubsub.acknowledge("sub-1", {}); }));
ASSERT_TRUE(invalid([&] { r.d->pubsub.get_topic("1bad"); }));
}
TEST(data, firestore_paths) {
Rig r;
Firestore& f = r.d->firestore;
ASSERT_EQ(f.documents_root(), "projects/p1/databases/(default)/documents");
ASSERT_EQ(f.document_name("users/alice"), "projects/p1/databases/(default)/documents/users/alice");
Prepared p = f.prepare("firestore.projects.databases.documents.get",
CallArgs().p("name", f.document_name("users/alice")));
ASSERT_EQ(p.path, "/v1/projects/p1/databases/(default)/documents/users/alice");
ASSERT_TRUE(invalid([&] { f.get_document(""); }));
ASSERT_TRUE(invalid([&] { f.get_document("users//alice"); }));
ASSERT_TRUE(invalid([&] { f.get_document("users/../x"); }));
ASSERT_TRUE(invalid([&] { f.get_document("users/a b"); }));
ASSERT_TRUE(invalid([&] { f.commit(Json::array()); }));
ASSERT_TRUE(invalid([&] { f.run_query("", Json::object()); }));
}
TEST(data, spanner_sql_validation) {
SpannerSql s;
ASSERT_TRUE(invalid([&] { s.validate(); }));
s.sql = "SELECT 1";
ASSERT_TRUE(valid([&] { s.validate(); }));
s.query_mode = "EXPLODE";
ASSERT_TRUE(invalid([&] { s.validate(); }));
s.query_mode = "PLAN";
s.params = Json("x");
ASSERT_TRUE(invalid([&] { s.validate(); }));
Rig r;
ASSERT_EQ(r.d->spanner.database_name("i1", "d1"), "projects/p1/instances/i1/databases/d1");
ASSERT_TRUE(invalid([&] { r.d->spanner.get_instance("a/b"); }));
}
TEST(data, secret_versions_and_payload_caps) {
Rig r;
SecretManager& s = r.d->secrets;
ASSERT_EQ(s.version_name("s1", "latest"), "projects/p1/secrets/s1/versions/latest");
ASSERT_EQ(s.version_name("s1", "3"), "projects/p1/secrets/s1/versions/3");
ASSERT_TRUE(invalid([&] { s.version_name("s1", "../x"); }));
ASSERT_TRUE(invalid([&] { s.version_name("s1", ""); }));
ASSERT_TRUE(invalid([&] { s.add_version("s1", ""); }));
ASSERT_TRUE(invalid([&] { s.add_version("s1", std::string(DataCaps::kMaxSecretBytes + 1, 'x')); }));
ASSERT_TRUE(valid([&] { s.add_version("s1", "value"); }));
ASSERT_TRUE(invalid([&] { s.create_secret("bad/id"); }));
}
TEST(data, dlp_request_validation) {
InspectRequest q;
ASSERT_TRUE(invalid([&] { q.validate(); }));
q.text = "mail me at a@b.com";
q.info_types = {"EMAIL_ADDRESS"};
Json b = q.to_json();
ASSERT_EQ(b.find("item")->find("value")->as_string(), "mail me at a@b.com");
ASSERT_EQ(b.find("inspectConfig")->find("infoTypes")->as_array()[0].find("name")->as_string(),
"EMAIL_ADDRESS");
ASSERT_EQ(b.find("inspectConfig")->find("limits")->find("maxFindingsPerRequest")->as_int(), 100);
q.min_likelihood = "MAYBE";
ASSERT_TRUE(invalid([&] { q.validate(); }));
q.min_likelihood = "LIKELY";
q.text = std::string(DataCaps::kMaxDlpItemBytes + 1, 'x');
ASSERT_TRUE(invalid([&] { q.validate(); }));
Rig r;
q.text = "x";
ASSERT_TRUE(valid([&] { r.d->dlp.inspect_content(q); }));
ASSERT_TRUE(invalid([&] { r.d->dlp.deidentify_content("x", Json::object(), ""); }));
}
TEST(data, kms_names_and_caps) {
Rig r;
Kms& k = r.d->kms;
ASSERT_EQ(k.version_name("global", "ring", "key", "2"),
"projects/p1/locations/global/keyRings/ring/cryptoKeys/key/cryptoKeyVersions/2");
ASSERT_TRUE(invalid([&] { k.version_name("global", "ring", "key", "x"); }));
ASSERT_TRUE(invalid([&] { k.encrypt("global", "ring", "key", "", ""); }));
ASSERT_TRUE(invalid([&] { k.encrypt("global", "ring", "key", std::string(DataCaps::kMaxCryptoBytes + 1, 'x'), ""); }));
ASSERT_TRUE(valid([&] { k.encrypt("global", "ring", "key", "secret", ""); }));
ASSERT_TRUE(invalid([&] { k.decrypt("global", "ring", "key", "not base64!!", ""); }));
ASSERT_TRUE(invalid([&] { k.sign_digest("global", "ring", "key", "1", "short"); }));
ASSERT_TRUE(valid([&] { k.sign_digest("global", "ring", "key", "1", std::string(32, 'a')); }));
}
TEST(data, datastore_and_dataplex_validation) {
Rig r;
ASSERT_TRUE(invalid([&] { r.d->datastore.lookup(Json::array()); }));
ASSERT_TRUE(invalid([&] { r.d->datastore.run_query(Json::object(), Json::object()); }));
ASSERT_TRUE(invalid([&] { r.d->datastore.run_query(Json(), Json()); }));
ASSERT_TRUE(invalid([&] { r.d->datastore.commit(Json::array(), "TRANSACTIONAL", ""); }));
Json m = Json::array();
m.push(Json::object());
ASSERT_TRUE(invalid([&] { r.d->datastore.commit(m, "MAYBE", ""); }));
ASSERT_TRUE(valid([&] { r.d->datastore.commit(m, "NON_TRANSACTIONAL", ""); }));
ASSERT_TRUE(invalid([&] { r.d->dataplex.search_entries("global", "", 10); }));
ASSERT_TRUE(invalid([&] { r.d->dataplex.lookup_entry("global", "entries/x"); }));
ASSERT_TRUE(invalid([&] { r.d->bigtable.get_instance("a b"); }));
}
TEST(data, every_typed_call_id_exists_in_the_table) {
// The typed classes name their calls as string literals; check_discovery.py
// compares those with Discovery. Here: a spot check of the rows they hit.
for (const char* id : {"bigquery.jobs.query", "bigquery.tabledata.insertAll",
"storage.objects.compose", "pubsub.projects.subscriptions.pull",
"firestore.projects.databases.documents.runQuery",
"spanner.projects.instances.databases.sessions.executeSql",
"bigtableadmin.projects.instances.tables.list",
"secretmanager.projects.secrets.versions.access",
"dataplex.projects.locations.searchEntries",
"dlp.projects.content.inspect",
"cloudkms.projects.locations.keyRings.cryptoKeys.encrypt",
"datastore.projects.runQuery"})
ASSERT_TRUE(Catalog::find(id) != nullptr);
}
+218
View File
@@ -0,0 +1,218 @@
// Opt-in live checks (MOLO_LIVE_NET=1, GOOGLE_ACCESS_TOKEN, GOOGLE_CLOUD_PROJECT).
// Read-only: lists, gets, dry-run queries, content inspection of a fixed
// sample. Nothing is created, changed, published or deleted. A disabled API is
// reported loudly ("API NOT ENABLED") and is not a failure: enabling an API is
// a human decision. Credentials are never printed.
#include <cstdlib>
#include <iostream>
#include <string>
#include "framework.h"
#include "clients/client.h"
#include "clients/transports/http_transport.h"
#include "google/credentials.h"
#include "google/data/data.h"
using namespace molo::google;
using namespace molo::google::data;
using molo::clients::body::Json;
namespace {
bool ready(const char* test) {
if (!std::getenv("MOLO_LIVE_NET") || Credentials::env("GOOGLE_CLOUD_PROJECT").empty() ||
Credentials::env("GOOGLE_ACCESS_TOKEN").empty()) {
std::cout << "SKIP " << test
<< " (set MOLO_LIVE_NET=1, GOOGLE_CLOUD_PROJECT and GOOGLE_ACCESS_TOKEN)\n";
return false;
}
return true;
}
// Runs one read. Success prints the size; a disabled API, a denied role or a
// missing resource prints loudly and counts as a verified route (the request
// shape and auth were accepted); anything else fails the test.
template <typename F>
bool probe(const char* what, F&& f) {
try {
Json j = f();
std::cout << " OK " << what << " (" << j.dump().size() << " bytes)\n";
return true;
} catch (const GoogleError& e) {
const std::string& msg = e.message();
if (e.http_status() == 403 && (msg.find("has not been used") != std::string::npos ||
msg.find("is disabled") != std::string::npos ||
e.status() == "SERVICE_DISABLED")) {
std::cout << " API NOT ENABLED " << what << ": " << msg.substr(0, 140) << "\n";
return false;
}
if (e.http_status() == 403) {
std::cout << " 403 " << e.status() << " " << what << " (route and auth accepted; role or API): "
<< msg.substr(0, 140) << "\n";
return false;
}
if (false) {
std::cout << " API NOT ENABLED " << what << "\n";
return false;
}
if (e.http_status() == 403 || e.http_status() == 404) {
std::cout << " " << e.http_status() << " " << e.status() << " " << what << " (route and auth accepted)\n";
return false;
}
std::cout << " FAIL " << what << ": " << e.http_status() << " " << e.status() << " "
<< e.message().substr(0, 200) << "\n";
FAIL(std::string(what) + " failed upstream");
}
return false;
}
DataPlatform platform() {
CloudContext ctx = CloudContext::from_env();
return DataPlatform(GoogleClient::from_env(), ctx);
}
} // namespace
TEST(data_live, bigquery_reads_and_dry_run) {
if (!ready("data_live.bigquery_reads_and_dry_run")) return;
DataPlatform d = platform();
probe("bigquery dry-run SELECT 1", [&] {
Json r = d.bigquery.dry_run("SELECT 1 AS one");
ASSERT_TRUE(r.find("totalBytesProcessed") != nullptr);
return r;
});
probe("bigquery datasets.list", [&] { return d.bigquery.list_datasets(10); });
probe("bigquery jobs.list", [&] { return d.bigquery.list_jobs(5, ""); });
probe("bigquery projects.getServiceAccount", [&] { return d.bigquery.service_account(); });
}
TEST(data_live, storage_reads_download_and_signed_url) {
if (!ready("data_live.storage_reads_download_and_signed_url")) return;
DataPlatform d = platform();
Json buckets;
bool ok = probe("storage buckets.list", [&] { return buckets = d.storage.list_buckets(10, "", ""); });
probe("storage projects.serviceAccount.get", [&] { return d.storage.service_account(); });
if (!ok) return;
const Json* items = buckets.find("items");
if (!items || items->as_array().empty()) {
std::cout << " NOTE no buckets in the project: object, download and signed URL checks have nothing to read\n";
return;
}
std::string bucket = items->as_array()[0].find("name")->as_string();
std::cout << " using bucket " << bucket << "\n";
Json objects;
probe("storage buckets.get", [&] { return d.storage.get_bucket(bucket); });
probe("storage objects.list", [&] { return objects = d.storage.list_objects(bucket, "", "", 20, ""); });
const Json* objs = objects.find("items");
if (!objs) return;
for (const Json& o : objs->as_array()) {
long size = std::stol(o.find("size")->as_string());
if (size == 0 || size > 1024 * 1024) continue;
std::string name = o.find("name")->as_string();
probe("storage objects.get", [&] { return d.storage.get_object(bucket, name); });
auto media = StorageMedia::from_env();
Downloaded got = media->download(bucket, name);
std::cout << " OK storage download " << name << " (" << got.data.size() << " bytes)\n";
ASSERT_EQ(static_cast<long>(got.data.size()), size);
// Signed URL: fetched with no Authorization header at all.
std::string path = Credentials::env("GOOGLE_APPLICATION_CREDENTIALS");
if (path.empty()) {
std::cout << " SKIP signed URL fetch (GOOGLE_APPLICATION_CREDENTIALS not set)\n";
return;
}
SignedUrl signer(ServiceAccountKey::load_file(path));
SignedUrlRequest sr;
sr.bucket = bucket;
sr.object = name;
sr.expires_seconds = 60;
std::string url = signer.sign(sr);
molo::clients::Client plain(std::make_unique<molo::clients::transports::HttpTransport>());
auto resp = plain.get_blocking(url);
std::cout << " signed URL GET -> HTTP " << resp.status() << " (" << resp.body().size() << " bytes)\n";
ASSERT_EQ(resp.status(), 200);
ASSERT_EQ(resp.body(), got.data);
return;
}
std::cout << " NOTE no object of 1 B..1 MiB found: download and signed URL not exercised\n";
}
// Media and signed URLs against a Google-owned public dataset (read only):
// a ranged download with the bearer token, and a V4 signed HEAD URL that
// Google must accept (a wrong signature is 403 SignatureDoesNotMatch).
TEST(data_live, public_object_range_download_and_signed_head) {
if (!ready("data_live.public_object_range_download_and_signed_head")) return;
const std::string bucket = "gcp-public-data-landsat";
const std::string object = "index.csv.gz";
auto media = StorageMedia::from_env();
try {
Downloaded part = media->download_range(bucket, object, 0, 99);
std::cout << " OK storage download_range " << bucket << "/" << object << " (" << part.data.size()
<< " bytes, generation " << part.generation << ")\n";
ASSERT_EQ(part.data.size(), 100u);
ASSERT_EQ(static_cast<unsigned char>(part.data[0]), 0x1f); // gzip magic
} catch (const GoogleError& e) {
std::cout << " " << e.http_status() << " " << e.status() << " download_range of the public object: "
<< e.message().substr(0, 140) << "\n";
}
std::string path = Credentials::env("GOOGLE_APPLICATION_CREDENTIALS");
if (path.empty()) {
std::cout << " SKIP signed URL (GOOGLE_APPLICATION_CREDENTIALS not set)\n";
return;
}
SignedUrl signer(ServiceAccountKey::load_file(path));
SignedUrlRequest sr;
sr.method = "HEAD";
sr.bucket = bucket;
sr.object = object;
sr.expires_seconds = 60;
molo::clients::Client plain(std::make_unique<molo::clients::transports::HttpTransport>());
auto good = plain.head_blocking(signer.sign(sr));
std::cout << " signed HEAD -> HTTP " << good.status() << "\n";
std::string tampered = signer.sign(sr);
tampered.back() = tampered.back() == '0' ? '1' : '0';
auto bad = plain.head_blocking(tampered);
std::cout << " tampered signature HEAD -> HTTP " << bad.status() << "\n";
ASSERT_EQ(good.status(), 200);
ASSERT_EQ(bad.status(), 403);
}
TEST(data_live, pubsub_firestore_datastore_reads) {
if (!ready("data_live.pubsub_firestore_datastore_reads")) return;
DataPlatform d = platform();
probe("pubsub topics.list", [&] { return d.pubsub.list_topics(10); });
probe("pubsub subscriptions.list", [&] { return d.pubsub.list_subscriptions(10); });
probe("pubsub snapshots.list", [&] { return d.pubsub.list_snapshots(10); });
probe("firestore databases.list", [&] { return d.firestore.list_databases(); });
probe("firestore listCollectionIds (root)", [&] { return d.firestore.list_collection_ids("", 10); });
probe("datastore runQuery (GQL kinds)", [&] {
Json gql = Json::object();
gql.set("queryString", Json("SELECT * FROM __kind__ LIMIT 1"));
return d.datastore.run_query(Json(), gql);
});
}
TEST(data_live, spanner_bigtable_reads) {
if (!ready("data_live.spanner_bigtable_reads")) return;
DataPlatform d = platform();
probe("spanner instances.list", [&] { return d.spanner.list_instances(10); });
probe("bigtable instances.list", [&] { return d.bigtable.list_instances(); });
}
TEST(data_live, secrets_dataplex_dlp_kms_reads) {
if (!ready("data_live.secrets_dataplex_dlp_kms_reads")) return;
DataPlatform d = platform();
probe("secretmanager secrets.list (metadata only)", [&] { return d.secrets.list_secrets(10, ""); });
probe("dataplex locations.list", [&] { return d.dataplex.list_locations(10); });
probe("dataplex searchEntries", [&] { return d.dataplex.search_entries("global", "type=table", 5); });
probe("dlp infoTypes.list", [&] { return d.dlp.list_info_types("en"); });
probe("dlp content.inspect (fixed sample)", [&] {
InspectRequest q;
q.text = "Contact jane.doe@example.com";
q.info_types = {"EMAIL_ADDRESS"};
q.include_quote = true;
return d.dlp.inspect_content(q);
});
probe("dlp inspectTemplates.list", [&] { return d.dlp.list_inspect_templates(10); });
probe("kms locations.list", [&] { return d.kms.list_locations(5); });
probe("kms keyRings.list (global)", [&] { return d.kms.list_key_rings("global", 10); });
}
+227
View File
@@ -0,0 +1,227 @@
// Cloud Storage media (download, simple and resumable upload request shape,
// session URL guard) and V4 signed URLs with a real throwaway RSA key.
#include <wolfssl/options.h>
#include <wolfssl/wolfcrypt/asn_public.h>
#include <wolfssl/wolfcrypt/random.h>
#include <wolfssl/wolfcrypt/rsa.h>
#include <wolfssl/wolfcrypt/signature.h>
#include <ctime>
#include <memory>
#include "framework.h"
#include "clients/transports/mock_transport.h"
#include "google/data/caps.h"
#include "google/data/signed_url.h"
#include "google/data/storage_media.h"
#include "google/credentials.h"
using namespace molo::google;
using namespace molo::google::data;
using molo::clients::transports::MockTransport;
using molo::clients::body::Json;
namespace {
struct KeyPair {
RsaKey key;
WC_RNG rng;
std::string pem;
KeyPair() {
wc_InitRng(&rng);
wc_InitRsaKey(&key, nullptr);
if (wc_MakeRsaKey(&key, 2048, 65537, &rng) != 0) FAIL("key generation failed");
std::string der(4096, '\0');
int n = wc_RsaKeyToDer(&key, reinterpret_cast<byte*>(der.data()), der.size());
if (n <= 0) FAIL("key export failed");
der.resize(n);
pem.assign(8192, '\0');
int p = wc_DerToPem(reinterpret_cast<const byte*>(der.data()), der.size(),
reinterpret_cast<byte*>(pem.data()), pem.size(), PRIVATEKEY_TYPE);
if (p <= 0) FAIL("pem export failed");
pem.resize(p);
}
~KeyPair() {
wc_FreeRsaKey(&key);
wc_FreeRng(&rng);
}
};
ServiceAccountKey account(const std::string& pem) {
ServiceAccountKey k;
k.client_email = "sa@p-1.iam.gserviceaccount.com";
k.private_key_pem = pem;
return k;
}
std::string unhex(const std::string& h) {
std::string out;
for (std::size_t i = 0; i + 1 < h.size(); i += 2) out.push_back(static_cast<char>(std::stoi(h.substr(i, 2), nullptr, 16)));
return out;
}
std::string between(const std::string& s, const std::string& a, const std::string& b) {
auto i = s.find(a);
if (i == std::string::npos) return "";
i += a.size();
auto j = s.find(b, i);
return s.substr(i, j == std::string::npos ? std::string::npos : j - i);
}
template <typename F>
bool invalid(F&& f) {
try {
f();
} catch (const NotConfiguredError&) {
return false;
} catch (const GoogleError& e) {
return e.http_status() == 400;
}
return false;
}
StorageMedia media(bool with_auth) {
return StorageMedia(std::make_unique<MockTransport>(),
with_auth ? Credentials::bearer("tok") : nullptr);
}
} // namespace
TEST(data_media, urls_are_built_from_validated_ids) {
StorageMedia m = media(false);
ASSERT_EQ(m.download_url("my-bucket", "a/b c.txt", 0),
"https://storage.googleapis.com/storage/v1/b/my-bucket/o/a%2Fb%20c.txt?alt=media");
ASSERT_EQ(m.download_url("my-bucket", "x", 77),
"https://storage.googleapis.com/storage/v1/b/my-bucket/o/x?alt=media&generation=77");
ASSERT_EQ(m.upload_url("my-bucket", "x y", "media"),
"https://storage.googleapis.com/upload/storage/v1/b/my-bucket/o?uploadType=media&name=x%20y");
ASSERT_TRUE(invalid([&] { m.download_url("Bad", "x", 0); }));
ASSERT_TRUE(invalid([&] { m.download_url("good", "..", 0); }));
ASSERT_TRUE(invalid([&] { m.download_url("good", "x", -1); }));
}
TEST(data_media, no_credential_is_not_configured_and_sends_nothing) {
auto t = std::make_unique<MockTransport>();
MockTransport* mock = t.get();
StorageMedia m(std::move(t), nullptr);
ASSERT_FALSE(m.configured());
bool threw = false;
try {
m.download("good-bucket", "x");
} catch (const NotConfiguredError&) {
threw = true;
}
ASSERT_TRUE(threw);
ASSERT_EQ(mock->send_count(), 0u);
}
TEST(data_media, upload_caps_and_chunk_rules) {
StorageMedia m = media(true);
ASSERT_TRUE(invalid([&] { m.upload_simple("good-bucket", "x", std::string(DataCaps::kMaxSimpleUploadBytes + 1, 'a'), "text/plain"); }));
ASSERT_TRUE(invalid([&] { m.upload_simple("good-bucket", "x", "a", ""); }));
ASSERT_TRUE(invalid([&] { m.upload_simple("good-bucket", "x", "a", "text/plain\r\nX: y"); }));
ASSERT_TRUE(invalid([&] { m.resumable_start("good-bucket", "x", "text/plain", DataCaps::kMaxResumableTotal + 1, Json()); }));
ASSERT_TRUE(invalid([&] { m.resumable_start("good-bucket", "x", "text/plain", 10, Json("str")); }));
const std::string session = "https://storage.googleapis.com/upload/storage/v1/b/good-bucket/o?uploadType=resumable&upload_id=abc";
// a chunk that does not end the object must be a multiple of 256 KiB
ASSERT_TRUE(invalid([&] { m.resumable_put(session, 0, std::string(1000, 'a'), 5000); }));
ASSERT_TRUE(invalid([&] { m.resumable_put(session, 0, "", 10); }));
ASSERT_TRUE(invalid([&] { m.resumable_put(session, 0, std::string(DataCaps::kMaxResumableChunk + 1, 'a'), DataCaps::kMaxResumableTotal); }));
ASSERT_TRUE(invalid([&] { m.resumable_put(session, 5, std::string(10, 'a'), 10); })); // past total
ASSERT_TRUE(invalid([&] { m.download_range("good-bucket", "x", 5, 2); }));
ASSERT_TRUE(invalid([&] { m.download_range("good-bucket", "x", 0, DataCaps::kMaxDownloadBytes); }));
}
TEST(data_media, session_url_never_leaves_storage_host) {
ASSERT_TRUE(invalid([] { StorageMedia::check_session("https://evil.example.com/upload/storage/v1/b/b/o?x=1"); }));
ASSERT_TRUE(invalid([] { StorageMedia::check_session("http://storage.googleapis.com/upload/storage/v1/b/b/o"); }));
ASSERT_TRUE(invalid([] { StorageMedia::check_session("https://storage.googleapis.com.evil.com/upload/storage/v1/b/b/o"); }));
ASSERT_TRUE(invalid([] { StorageMedia::check_session("https://storage.googleapis.com/storage/v1/b/b/o"); }));
ASSERT_TRUE(invalid([] { StorageMedia::check_session("https://storage.googleapis.com/upload/storage/v1/b/b/o\r\nX: y"); }));
StorageMedia::check_session("https://storage.googleapis.com/upload/storage/v1/b/b/o?uploadType=resumable&upload_id=1");
StorageMedia m = media(true);
ASSERT_TRUE(invalid([&] { m.resumable_cancel("https://evil.example.com/x"); }));
ASSERT_TRUE(invalid([&] { m.resumable_status("https://evil.example.com/x", 10); }));
}
TEST(data_signed_url, signature_verifies_with_the_public_key) {
KeyPair kp;
SignedUrl signer(account(kp.pem));
SignedUrlRequest r;
r.bucket = "my-bucket";
r.object = "dir/file name.txt";
r.expires_seconds = 3600;
r.now = 1893456000; // 2030-01-01T00:00:00Z
std::string url = signer.sign(r);
ASSERT_TRUE(url.rfind("https://storage.googleapis.com/my-bucket/dir/file%20name.txt?", 0) == 0);
ASSERT_TRUE(url.find("X-Goog-Algorithm=GOOG4-RSA-SHA256") != std::string::npos);
ASSERT_TRUE(url.find("X-Goog-Credential=sa%40p-1.iam.gserviceaccount.com%2F20300101%2Fauto%2Fstorage%2Fgoog4_request") != std::string::npos);
ASSERT_TRUE(url.find("X-Goog-Date=20300101T000000Z") != std::string::npos);
ASSERT_TRUE(url.find("X-Goog-Expires=3600") != std::string::npos);
ASSERT_TRUE(url.find("X-Goog-SignedHeaders=host") != std::string::npos);
// Recompute the string to sign and verify the RSA signature with the key.
std::string canon = signer.canonical_request(r, r.now);
ASSERT_TRUE(canon.rfind("GET\n/my-bucket/dir/file%20name.txt\n", 0) == 0);
ASSERT_TRUE(canon.find("\nhost:storage.googleapis.com\n\nhost\nUNSIGNED-PAYLOAD") != std::string::npos);
std::string sts = SignedUrl::string_to_sign("20300101T000000Z", "20300101/auto/storage/goog4_request", canon);
ASSERT_TRUE(sts.rfind("GOOG4-RSA-SHA256\n20300101T000000Z\n20300101/auto/storage/goog4_request\n", 0) == 0);
std::string sig = unhex(between(url, "X-Goog-Signature=", "&"));
ASSERT_EQ(sig.size(), 256u);
int rc = wc_SignatureVerify(WC_HASH_TYPE_SHA256, WC_SIGNATURE_TYPE_RSA_W_ENC,
reinterpret_cast<const byte*>(sts.data()), sts.size(),
reinterpret_cast<const byte*>(sig.data()), sig.size(), &kp.key,
sizeof(kp.key));
ASSERT_EQ(rc, 0);
// Deterministic parts: the same input gives the same canonical request.
ASSERT_EQ(canon, signer.canonical_request(r, r.now));
// A tampered string does not verify.
std::string other = sts + "x";
rc = wc_SignatureVerify(WC_HASH_TYPE_SHA256, WC_SIGNATURE_TYPE_RSA_W_ENC,
reinterpret_cast<const byte*>(other.data()), other.size(),
reinterpret_cast<const byte*>(sig.data()), sig.size(), &kp.key,
sizeof(kp.key));
ASSERT_TRUE(rc != 0);
}
TEST(data_signed_url, extra_headers_and_query_are_signed_in_order) {
KeyPair kp;
SignedUrl signer(account(kp.pem));
SignedUrlRequest r;
r.method = "PUT";
r.bucket = "my-bucket";
r.object = "up.bin";
r.now = 1893456000;
r.headers["Content-Type"] = " application/octet-stream ";
r.headers["x-goog-meta-a"] = "1";
r.query["response-content-disposition"] = "attachment";
std::string canon = signer.canonical_request(r, r.now);
ASSERT_TRUE(canon.rfind("PUT\n/my-bucket/up.bin\n", 0) == 0);
// byte order: upper case X-Goog-* sorts before lower case response-*
ASSERT_TRUE(canon.find("&response-content-disposition=attachment\n") != std::string::npos);
ASSERT_TRUE(canon.find("\nX-Goog-Algorithm=") != std::string::npos);
ASSERT_TRUE(canon.find("\ncontent-type:application/octet-stream\nhost:storage.googleapis.com\nx-goog-meta-a:1\n\ncontent-type;host;x-goog-meta-a\nUNSIGNED-PAYLOAD") != std::string::npos);
}
TEST(data_signed_url, caps_and_validation) {
KeyPair kp;
SignedUrl signer(account(kp.pem));
SignedUrlRequest r;
r.bucket = "my-bucket";
r.object = "x";
r.now = 1893456000;
ASSERT_TRUE(invalid([&] { SignedUrlRequest b = r; b.expires_seconds = 0; signer.sign(b); }));
ASSERT_TRUE(invalid([&] { SignedUrlRequest b = r; b.expires_seconds = DataCaps::kMaxSignedUrlSeconds + 1; signer.sign(b); }));
ASSERT_TRUE(invalid([&] { SignedUrlRequest b = r; b.method = "TRACE"; signer.sign(b); }));
ASSERT_TRUE(invalid([&] { SignedUrlRequest b = r; b.bucket = "Bad"; signer.sign(b); }));
ASSERT_TRUE(invalid([&] { SignedUrlRequest b = r; b.headers["host"] = "evil"; signer.sign(b); }));
ASSERT_TRUE(invalid([&] { SignedUrlRequest b = r; b.query["X-Goog-Expires"] = "9"; signer.sign(b); }));
ASSERT_TRUE(invalid([&] { SignedUrlRequest b = r; b.headers["a"] = "x\ny"; signer.sign(b); }));
r.expires_seconds = DataCaps::kMaxSignedUrlSeconds;
signer.sign(r); // the 7 day maximum is allowed
// A key that is not RSA PEM is rejected loudly.
SignedUrl bad(account("-----BEGIN PRIVATE KEY-----\nAAAA\n-----END PRIVATE KEY-----\n"));
ASSERT_TRUE(invalid([&] { bad.sign(r); }));
}