Files
moloapp/tests/src/google/data/test_data.cpp
T
2026-10-25 15:03:43 +01:00

425 lines
18 KiB
C++

// data package: deterministic local tests (call table, request rendering,
// validation, caps, typed request bodies, no-credential behavior). No scripted
// Google replies: shapes are verified against Discovery by
// src/google/data/tools/check_discovery.py and live (test_data_live.cpp).
#include <iostream>
#include <memory>
#include "framework.h"
#include "clients/transports/mock_transport.h"
#include "google/data/data.h"
#include "google/data/caps.h"
using namespace molo::google;
using namespace molo::google::data;
using molo::clients::Method;
using molo::clients::body::Json;
using molo::clients::transports::MockTransport;
namespace {
struct Rig {
MockTransport* mock;
std::shared_ptr<GoogleClient> client;
std::unique_ptr<DataPlatform> d;
Rig() {
auto t = std::make_unique<MockTransport>();
mock = t.get();
client = std::make_shared<GoogleClient>(std::move(t), nullptr);
CloudContext ctx;
ctx.project = "p1";
ctx.location = "us-central1";
d = std::make_unique<DataPlatform>(client, ctx);
}
};
// passes validation (stops at the credential check) or is invalid.
template <typename F>
bool invalid(F&& f) {
try {
f();
} catch (const NotConfiguredError&) {
return false;
} catch (const GoogleError& e) {
return e.http_status() == 400 && e.status() == "INVALID_ARGUMENT";
}
return false;
}
template <typename F>
bool valid(F&& f) {
try {
f();
} catch (const NotConfiguredError&) {
return true;
} catch (const GoogleError&) {
return false;
}
return true;
}
} // namespace
TEST(data, catalog_covers_all_eleven_apis) {
ASSERT_EQ(Catalog::api_count(), 11u);
std::size_t total = 0;
for (std::size_t i = 0; i < Catalog::api_count(); ++i) {
std::size_t n = Catalog::count_of(Catalog::info_at(i).api);
ASSERT_TRUE(n >= 18);
total += n;
}
ASSERT_EQ(total, Catalog::size());
ASSERT_TRUE(Catalog::size() > 900);
ASSERT_TRUE(Catalog::find("bigquery.jobs.query") != nullptr);
ASSERT_TRUE(Catalog::find("storage.objects.compose") != nullptr);
ASSERT_TRUE(Catalog::find("nope.x") == nullptr);
ASSERT_THROW(Catalog::get("nope.x"), GoogleError);
ASSERT_EQ(std::string(Catalog::get("pubsub.projects.topics.publish").http), "POST");
ASSERT_EQ(Catalog::endpoint(Api::kBigquery).base_url,
"https://bigquery.googleapis.com/bigquery/v2");
}
TEST(data, path_params_are_read_from_the_template) {
auto p = Catalog::path_params(Catalog::get("bigquery.tabledata.insertAll"));
ASSERT_EQ(p.size(), 3u);
ASSERT_EQ(p[0], "projectId");
ASSERT_EQ(p[2], "tableId");
auto q = Catalog::path_params(Catalog::get("storage.objects.compose"));
ASSERT_EQ(q[0], "destinationBucket");
}
TEST(data, base_url_override_is_google_only) {
ASSERT_TRUE(valid([] { Catalog::endpoint(Api::kDlp, "https://dlp.europe-west1.rep.googleapis.com"); }));
ASSERT_TRUE(invalid([] { Catalog::endpoint(Api::kDlp, "http://dlp.googleapis.com"); }));
ASSERT_TRUE(invalid([] { Catalog::endpoint(Api::kDlp, "https://evil.example.com"); }));
ASSERT_TRUE(invalid([] { Catalog::endpoint(Api::kDlp, "https://googleapis.com.evil.com"); }));
ASSERT_TRUE(invalid([] { Catalog::endpoint(Api::kDlp, "https://dlp.googleapis.com/"); }));
}
TEST(data, prepare_renders_paths_queries_and_bodies) {
Rig r;
Prepared p = r.d->bigquery.prepare(
"bigquery.datasets.list", CallArgs().p("projectId", "p1").q("maxResults", Json(5)));
ASSERT_EQ(p.url(), "https://bigquery.googleapis.com/bigquery/v2/projects/p1/datasets?maxResults=5");
ASSERT_TRUE(p.method == Method::kGet);
ASSERT_FALSE(p.has_body);
Prepared s = r.d->storage.prepare(
"storage.objects.get",
CallArgs().p("bucket", "my-bucket").p("object", "dir/my file+1.txt"));
ASSERT_EQ(s.path, "/b/my-bucket/o/dir%2Fmy%20file%2B1.txt");
Prepared pub = r.d->pubsub.prepare(
"pubsub.projects.topics.publish",
CallArgs().p("topic", "projects/p1/topics/t1").with_body(Json::parse("{\"messages\":[]}")));
ASSERT_EQ(pub.path, "/v1/projects/p1/topics/t1:publish");
ASSERT_TRUE(pub.method == Method::kPost && pub.has_body);
// POST whose request has fields gets an empty object when no body is given.
Prepared sess = r.d->spanner.prepare(
"spanner.projects.instances.databases.sessions.create",
CallArgs().p("database", "projects/p/instances/i/databases/d"));
ASSERT_TRUE(sess.has_body);
ASSERT_EQ(sess.body.dump(), "{}");
// Arrays repeat the parameter.
Prepared rep = r.d->bigquery.prepare(
"bigquery.tabledata.list",
CallArgs().p("projectId", "p1").p("datasetId", "d").p("tableId", "t")
.q("selectedFields", Json::parse("[\"a\",\"b\"]")));
ASSERT_TRUE(rep.path.find("selectedFields=a&selectedFields=b") != std::string::npos);
}
TEST(data, prepare_rejects_bad_input_before_any_traffic) {
Rig r;
auto& bq = r.d->bigquery;
ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.datasets.list", CallArgs()); })); // path missing
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "p1").p("extra", "x"));
}));
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.datasets.list",
CallArgs().p("projectId", "p1").q("notAParam", Json(1)));
}));
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "../x"));
}));
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "p1/../../x"));
}));
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "p1?x=1"));
}));
ASSERT_TRUE(invalid([&] { bq.prepare("storage.buckets.list", CallArgs()); })); // other api
ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.no.such", CallArgs()); }));
// no body allowed on a GET
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.datasets.list",
CallArgs().p("projectId", "p1").with_body(Json::object()));
}));
// unknown body field
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.jobs.query",
CallArgs().p("projectId", "p1").with_body(Json::parse("{\"bogus\":1}")));
}));
// required query parameter (storage.buckets.list needs project)
ASSERT_TRUE(invalid([&] { r.d->storage.prepare("storage.buckets.list", CallArgs()); }));
// control characters in a query value
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.datasets.list",
CallArgs().p("projectId", "p1").q("filter", Json("a\nb")));
}));
// too many query parameters via array
Json many = Json::array();
for (int i = 0; i < 101; ++i) many.push(Json(i));
ASSERT_TRUE(invalid([&] {
bq.prepare("bigquery.tabledata.list",
CallArgs().p("projectId", "p1").p("datasetId", "d").p("tableId", "t")
.q("selectedFields", many));
}));
}
TEST(data, body_size_is_capped) {
Rig r;
Json big = Json::object();
big.set("query", Json(std::string(DataCaps::kMaxBodyBytes + 10, 'x')));
ASSERT_TRUE(invalid([&] {
r.d->bigquery.prepare("bigquery.jobs.query", CallArgs().p("projectId", "p1").with_body(big));
}));
}
TEST(data, no_credential_is_not_configured_after_validation) {
Rig r;
bool threw = false;
try {
r.d->bigquery.list_datasets(10);
} catch (const NotConfiguredError& e) {
threw = true;
ASSERT_EQ(e.http_status(), 503);
}
ASSERT_TRUE(threw);
ASSERT_EQ(r.mock->send_count(), 0u);
}
TEST(data, ids_follow_google_rules) {
ASSERT_TRUE(valid([] { Ids::bucket("my-bucket_1.x"); }));
ASSERT_TRUE(invalid([] { Ids::bucket("ab"); }));
ASSERT_TRUE(invalid([] { Ids::bucket("-bad-"); }));
ASSERT_TRUE(invalid([] { Ids::bucket("Upper"); }));
ASSERT_TRUE(valid([] { Ids::object("a/b/c d.txt"); }));
ASSERT_TRUE(invalid([] { Ids::object(""); }));
ASSERT_TRUE(invalid([] { Ids::object(".."); }));
ASSERT_TRUE(invalid([] { Ids::object(std::string(1025, 'a')); }));
ASSERT_TRUE(valid([] { Ids::pubsub("my-topic.v1"); }));
ASSERT_TRUE(invalid([] { Ids::pubsub("1abc"); }));
ASSERT_TRUE(invalid([] { Ids::pubsub("ab"); }));
ASSERT_TRUE(valid([] { Ids::project("example.com:proj-1"); }));
ASSERT_TRUE(invalid([] { Ids::project("Proj"); }));
ASSERT_TRUE(valid([] { Ids::dataset("my_data_1"); }));
ASSERT_TRUE(invalid([] { Ids::dataset("my-data"); }));
ASSERT_TRUE(invalid([] { Ids::dataset("a.b"); }));
}
TEST(data, bigquery_query_body_and_caps) {
QueryRequest q;
q.sql = "SELECT 1";
q.dry_run = true;
q.default_dataset = "ds";
Json b = q.to_json("p1");
ASSERT_EQ(b.find("query")->as_string(), "SELECT 1");
ASSERT_TRUE(b.find("dryRun")->as_bool());
ASSERT_FALSE(b.find("useLegacySql")->as_bool());
ASSERT_EQ(b.find("defaultDataset")->find("projectId")->as_string(), "p1");
ASSERT_EQ(b.find("timeoutMs")->as_int(), 10000);
Rig r;
QueryRequest bad;
ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); })); // empty sql
bad.sql = std::string(DataCaps::kMaxSqlChars + 1, 'x');
ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); }));
bad.sql = "SELECT 1";
bad.max_results = 10001;
ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); }));
bad.max_results = 10;
bad.timeout_ms = 60001;
ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); }));
bad.timeout_ms = 1;
bad.maximum_bytes_billed = "12a";
ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); }));
bad.maximum_bytes_billed = "";
bad.query_parameters = Json::array();
ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); })); // parameter_mode missing
bad.parameter_mode = "NAMED";
ASSERT_TRUE(valid([&] { r.d->bigquery.query(bad); }));
}
TEST(data, bigquery_insert_all_validation) {
Rig r;
auto& bq = r.d->bigquery;
Json rows = Json::parse("[{\"insertId\":\"1\",\"json\":{\"a\":1}}]");
ASSERT_TRUE(valid([&] { bq.insert_all("ds", "t1", rows, false, false); }));
ASSERT_TRUE(invalid([&] { bq.insert_all("ds", "t1", Json::array(), false, false); }));
ASSERT_TRUE(invalid([&] { bq.insert_all("ds", "t1", Json::parse("[{\"a\":1}]"), false, false); }));
Json many = Json::array();
for (std::size_t i = 0; i <= DataCaps::kMaxInsertRows; ++i) many.push(Json::parse("{\"json\":{}}"));
ASSERT_TRUE(invalid([&] { bq.insert_all("ds", "t1", many, false, false); }));
ASSERT_TRUE(invalid([&] { bq.insert_all("bad-ds", "t1", rows, false, false); }));
ASSERT_TRUE(invalid([&] { bq.insert_all("ds", "t/1", rows, false, false); }));
}
TEST(data, storage_compose_body_and_caps) {
std::vector<ComposeSource> src = {{"a.txt", 0}, {"b.txt", 12345}};
Json b = Storage::compose_body(src, "text/plain");
ASSERT_EQ(b.find("sourceObjects")->as_array().size(), 2u);
ASSERT_EQ(b.find("sourceObjects")->as_array()[1].find("generation")->as_string(), "12345");
ASSERT_EQ(b.find("destination")->find("contentType")->as_string(), "text/plain");
ASSERT_TRUE(invalid([] { Storage::compose_body({}, ""); }));
std::vector<ComposeSource> too_many(DataCaps::kMaxComposeSources + 1, ComposeSource{"x", 0});
ASSERT_TRUE(invalid([&] { Storage::compose_body(too_many, ""); }));
std::vector<ComposeSource> max(DataCaps::kMaxComposeSources, ComposeSource{"x", 0});
ASSERT_TRUE(valid([&] { Storage::compose_body(max, ""); }));
Rig r;
ASSERT_TRUE(invalid([&] { r.d->storage.compose("Bad", src, "out", ""); }));
ASSERT_TRUE(valid([&] { r.d->storage.compose("good-bucket", src, "out", ""); }));
}
TEST(data, storage_create_bucket_class_is_checked) {
Rig r;
ASSERT_TRUE(invalid([&] { r.d->storage.create_bucket("good-bucket", "US", "FAST"); }));
ASSERT_TRUE(valid([&] { r.d->storage.create_bucket("good-bucket", "US", "STANDARD"); }));
}
TEST(data, pubsub_publish_body_is_base64_and_capped) {
PubsubMessage m;
m.data = "hello";
m.attributes["k"] = "v";
m.ordering_key = "o1";
Json b = PubSub::publish_body({m});
const Json& msg = b.find("messages")->as_array()[0];
ASSERT_EQ(msg.find("data")->as_string(), "aGVsbG8=");
ASSERT_EQ(msg.find("attributes")->find("k")->as_string(), "v");
ASSERT_EQ(msg.find("orderingKey")->as_string(), "o1");
ASSERT_TRUE(invalid([] { PubSub::publish_body({}); }));
ASSERT_TRUE(invalid([] { PubSub::publish_body({PubsubMessage{}}); }));
std::vector<PubsubMessage> many(DataCaps::kMaxPublishMessages + 1, m);
ASSERT_TRUE(invalid([&] { PubSub::publish_body(many); }));
PubsubMessage big;
big.data = std::string(DataCaps::kMaxMessageBytes + 1, 'x');
ASSERT_TRUE(invalid([&] { PubSub::publish_body({big}); }));
Rig r;
ASSERT_TRUE(invalid([&] { r.d->pubsub.pull("sub-1", 0); }));
ASSERT_TRUE(invalid([&] { r.d->pubsub.pull("sub-1", 101); }));
ASSERT_TRUE(valid([&] { r.d->pubsub.pull("sub-1", 10); }));
ASSERT_TRUE(invalid([&] { r.d->pubsub.acknowledge("sub-1", {}); }));
ASSERT_TRUE(invalid([&] { r.d->pubsub.get_topic("1bad"); }));
}
TEST(data, firestore_paths) {
Rig r;
Firestore& f = r.d->firestore;
ASSERT_EQ(f.documents_root(), "projects/p1/databases/(default)/documents");
ASSERT_EQ(f.document_name("users/alice"), "projects/p1/databases/(default)/documents/users/alice");
Prepared p = f.prepare("firestore.projects.databases.documents.get",
CallArgs().p("name", f.document_name("users/alice")));
ASSERT_EQ(p.path, "/v1/projects/p1/databases/(default)/documents/users/alice");
ASSERT_TRUE(invalid([&] { f.get_document(""); }));
ASSERT_TRUE(invalid([&] { f.get_document("users//alice"); }));
ASSERT_TRUE(invalid([&] { f.get_document("users/../x"); }));
ASSERT_TRUE(invalid([&] { f.get_document("users/a b"); }));
ASSERT_TRUE(invalid([&] { f.commit(Json::array()); }));
ASSERT_TRUE(invalid([&] { f.run_query("", Json::object()); }));
}
TEST(data, spanner_sql_validation) {
SpannerSql s;
ASSERT_TRUE(invalid([&] { s.validate(); }));
s.sql = "SELECT 1";
ASSERT_TRUE(valid([&] { s.validate(); }));
s.query_mode = "EXPLODE";
ASSERT_TRUE(invalid([&] { s.validate(); }));
s.query_mode = "PLAN";
s.params = Json("x");
ASSERT_TRUE(invalid([&] { s.validate(); }));
Rig r;
ASSERT_EQ(r.d->spanner.database_name("i1", "d1"), "projects/p1/instances/i1/databases/d1");
ASSERT_TRUE(invalid([&] { r.d->spanner.get_instance("a/b"); }));
}
TEST(data, secret_versions_and_payload_caps) {
Rig r;
SecretManager& s = r.d->secrets;
ASSERT_EQ(s.version_name("s1", "latest"), "projects/p1/secrets/s1/versions/latest");
ASSERT_EQ(s.version_name("s1", "3"), "projects/p1/secrets/s1/versions/3");
ASSERT_TRUE(invalid([&] { s.version_name("s1", "../x"); }));
ASSERT_TRUE(invalid([&] { s.version_name("s1", ""); }));
ASSERT_TRUE(invalid([&] { s.add_version("s1", ""); }));
ASSERT_TRUE(invalid([&] { s.add_version("s1", std::string(DataCaps::kMaxSecretBytes + 1, 'x')); }));
ASSERT_TRUE(valid([&] { s.add_version("s1", "value"); }));
ASSERT_TRUE(invalid([&] { s.create_secret("bad/id"); }));
}
TEST(data, dlp_request_validation) {
InspectRequest q;
ASSERT_TRUE(invalid([&] { q.validate(); }));
q.text = "mail me at a@b.com";
q.info_types = {"EMAIL_ADDRESS"};
Json b = q.to_json();
ASSERT_EQ(b.find("item")->find("value")->as_string(), "mail me at a@b.com");
ASSERT_EQ(b.find("inspectConfig")->find("infoTypes")->as_array()[0].find("name")->as_string(),
"EMAIL_ADDRESS");
ASSERT_EQ(b.find("inspectConfig")->find("limits")->find("maxFindingsPerRequest")->as_int(), 100);
q.min_likelihood = "MAYBE";
ASSERT_TRUE(invalid([&] { q.validate(); }));
q.min_likelihood = "LIKELY";
q.text = std::string(DataCaps::kMaxDlpItemBytes + 1, 'x');
ASSERT_TRUE(invalid([&] { q.validate(); }));
Rig r;
q.text = "x";
ASSERT_TRUE(valid([&] { r.d->dlp.inspect_content(q); }));
ASSERT_TRUE(invalid([&] { r.d->dlp.deidentify_content("x", Json::object(), ""); }));
}
TEST(data, kms_names_and_caps) {
Rig r;
Kms& k = r.d->kms;
ASSERT_EQ(k.version_name("global", "ring", "key", "2"),
"projects/p1/locations/global/keyRings/ring/cryptoKeys/key/cryptoKeyVersions/2");
ASSERT_TRUE(invalid([&] { k.version_name("global", "ring", "key", "x"); }));
ASSERT_TRUE(invalid([&] { k.encrypt("global", "ring", "key", "", ""); }));
ASSERT_TRUE(invalid([&] { k.encrypt("global", "ring", "key", std::string(DataCaps::kMaxCryptoBytes + 1, 'x'), ""); }));
ASSERT_TRUE(valid([&] { k.encrypt("global", "ring", "key", "secret", ""); }));
ASSERT_TRUE(invalid([&] { k.decrypt("global", "ring", "key", "not base64!!", ""); }));
ASSERT_TRUE(invalid([&] { k.sign_digest("global", "ring", "key", "1", "short"); }));
ASSERT_TRUE(valid([&] { k.sign_digest("global", "ring", "key", "1", std::string(32, 'a')); }));
}
TEST(data, datastore_and_dataplex_validation) {
Rig r;
ASSERT_TRUE(invalid([&] { r.d->datastore.lookup(Json::array()); }));
ASSERT_TRUE(invalid([&] { r.d->datastore.run_query(Json::object(), Json::object()); }));
ASSERT_TRUE(invalid([&] { r.d->datastore.run_query(Json(), Json()); }));
ASSERT_TRUE(invalid([&] { r.d->datastore.commit(Json::array(), "TRANSACTIONAL", ""); }));
Json m = Json::array();
m.push(Json::object());
ASSERT_TRUE(invalid([&] { r.d->datastore.commit(m, "MAYBE", ""); }));
ASSERT_TRUE(valid([&] { r.d->datastore.commit(m, "NON_TRANSACTIONAL", ""); }));
ASSERT_TRUE(invalid([&] { r.d->dataplex.search_entries("global", "", 10); }));
ASSERT_TRUE(invalid([&] { r.d->dataplex.lookup_entry("global", "entries/x"); }));
ASSERT_TRUE(invalid([&] { r.d->bigtable.get_instance("a b"); }));
}
TEST(data, every_typed_call_id_exists_in_the_table) {
// The typed classes name their calls as string literals; check_discovery.py
// compares those with Discovery. Here: a spot check of the rows they hit.
for (const char* id : {"bigquery.jobs.query", "bigquery.tabledata.insertAll",
"storage.objects.compose", "pubsub.projects.subscriptions.pull",
"firestore.projects.databases.documents.runQuery",
"spanner.projects.instances.databases.sessions.executeSql",
"bigtableadmin.projects.instances.tables.list",
"secretmanager.projects.secrets.versions.access",
"dataplex.projects.locations.searchEntries",
"dlp.projects.content.inspect",
"cloudkms.projects.locations.keyRings.cryptoKeys.encrypt",
"datastore.projects.runQuery"})
ASSERT_TRUE(Catalog::find(id) != nullptr);
}