// data package: deterministic local tests (call table, request rendering, // validation, caps, typed request bodies, no-credential behavior). No scripted // Google replies: shapes are verified against Discovery by // src/google/data/tools/check_discovery.py and live (test_data_live.cpp). #include #include #include "framework.h" #include "clients/transports/mock_transport.h" #include "google/data/data.h" #include "google/data/caps.h" using namespace molo::google; using namespace molo::google::data; using molo::clients::Method; using molo::clients::body::Json; using molo::clients::transports::MockTransport; namespace { struct Rig { MockTransport* mock; std::shared_ptr client; std::unique_ptr d; Rig() { auto t = std::make_unique(); mock = t.get(); client = std::make_shared(std::move(t), nullptr); CloudContext ctx; ctx.project = "p1"; ctx.location = "us-central1"; d = std::make_unique(client, ctx); } }; // passes validation (stops at the credential check) or is invalid. template bool invalid(F&& f) { try { f(); } catch (const NotConfiguredError&) { return false; } catch (const GoogleError& e) { return e.http_status() == 400 && e.status() == "INVALID_ARGUMENT"; } return false; } template bool valid(F&& f) { try { f(); } catch (const NotConfiguredError&) { return true; } catch (const GoogleError&) { return false; } return true; } } // namespace TEST(data, catalog_covers_all_eleven_apis) { ASSERT_EQ(Catalog::api_count(), 11u); std::size_t total = 0; for (std::size_t i = 0; i < Catalog::api_count(); ++i) { std::size_t n = Catalog::count_of(Catalog::info_at(i).api); ASSERT_TRUE(n >= 18); total += n; } ASSERT_EQ(total, Catalog::size()); ASSERT_TRUE(Catalog::size() > 900); ASSERT_TRUE(Catalog::find("bigquery.jobs.query") != nullptr); ASSERT_TRUE(Catalog::find("storage.objects.compose") != nullptr); ASSERT_TRUE(Catalog::find("nope.x") == nullptr); ASSERT_THROW(Catalog::get("nope.x"), GoogleError); ASSERT_EQ(std::string(Catalog::get("pubsub.projects.topics.publish").http), "POST"); ASSERT_EQ(Catalog::endpoint(Api::kBigquery).base_url, "https://bigquery.googleapis.com/bigquery/v2"); } TEST(data, path_params_are_read_from_the_template) { auto p = Catalog::path_params(Catalog::get("bigquery.tabledata.insertAll")); ASSERT_EQ(p.size(), 3u); ASSERT_EQ(p[0], "projectId"); ASSERT_EQ(p[2], "tableId"); auto q = Catalog::path_params(Catalog::get("storage.objects.compose")); ASSERT_EQ(q[0], "destinationBucket"); } TEST(data, base_url_override_is_google_only) { ASSERT_TRUE(valid([] { Catalog::endpoint(Api::kDlp, "https://dlp.europe-west1.rep.googleapis.com"); })); ASSERT_TRUE(invalid([] { Catalog::endpoint(Api::kDlp, "http://dlp.googleapis.com"); })); ASSERT_TRUE(invalid([] { Catalog::endpoint(Api::kDlp, "https://evil.example.com"); })); ASSERT_TRUE(invalid([] { Catalog::endpoint(Api::kDlp, "https://googleapis.com.evil.com"); })); ASSERT_TRUE(invalid([] { Catalog::endpoint(Api::kDlp, "https://dlp.googleapis.com/"); })); } TEST(data, prepare_renders_paths_queries_and_bodies) { Rig r; Prepared p = r.d->bigquery.prepare( "bigquery.datasets.list", CallArgs().p("projectId", "p1").q("maxResults", Json(5))); ASSERT_EQ(p.url(), "https://bigquery.googleapis.com/bigquery/v2/projects/p1/datasets?maxResults=5"); ASSERT_TRUE(p.method == Method::kGet); ASSERT_FALSE(p.has_body); Prepared s = r.d->storage.prepare( "storage.objects.get", CallArgs().p("bucket", "my-bucket").p("object", "dir/my file+1.txt")); ASSERT_EQ(s.path, "/b/my-bucket/o/dir%2Fmy%20file%2B1.txt"); Prepared pub = r.d->pubsub.prepare( "pubsub.projects.topics.publish", CallArgs().p("topic", "projects/p1/topics/t1").with_body(Json::parse("{\"messages\":[]}"))); ASSERT_EQ(pub.path, "/v1/projects/p1/topics/t1:publish"); ASSERT_TRUE(pub.method == Method::kPost && pub.has_body); // POST whose request has fields gets an empty object when no body is given. Prepared sess = r.d->spanner.prepare( "spanner.projects.instances.databases.sessions.create", CallArgs().p("database", "projects/p/instances/i/databases/d")); ASSERT_TRUE(sess.has_body); ASSERT_EQ(sess.body.dump(), "{}"); // Arrays repeat the parameter. Prepared rep = r.d->bigquery.prepare( "bigquery.tabledata.list", CallArgs().p("projectId", "p1").p("datasetId", "d").p("tableId", "t") .q("selectedFields", Json::parse("[\"a\",\"b\"]"))); ASSERT_TRUE(rep.path.find("selectedFields=a&selectedFields=b") != std::string::npos); } TEST(data, prepare_rejects_bad_input_before_any_traffic) { Rig r; auto& bq = r.d->bigquery; ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.datasets.list", CallArgs()); })); // path missing ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "p1").p("extra", "x")); })); ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "p1").q("notAParam", Json(1))); })); ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "../x")); })); ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "p1/../../x")); })); ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "p1?x=1")); })); ASSERT_TRUE(invalid([&] { bq.prepare("storage.buckets.list", CallArgs()); })); // other api ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.no.such", CallArgs()); })); // no body allowed on a GET ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "p1").with_body(Json::object())); })); // unknown body field ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.jobs.query", CallArgs().p("projectId", "p1").with_body(Json::parse("{\"bogus\":1}"))); })); // required query parameter (storage.buckets.list needs project) ASSERT_TRUE(invalid([&] { r.d->storage.prepare("storage.buckets.list", CallArgs()); })); // control characters in a query value ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.datasets.list", CallArgs().p("projectId", "p1").q("filter", Json("a\nb"))); })); // too many query parameters via array Json many = Json::array(); for (int i = 0; i < 101; ++i) many.push(Json(i)); ASSERT_TRUE(invalid([&] { bq.prepare("bigquery.tabledata.list", CallArgs().p("projectId", "p1").p("datasetId", "d").p("tableId", "t") .q("selectedFields", many)); })); } TEST(data, body_size_is_capped) { Rig r; Json big = Json::object(); big.set("query", Json(std::string(DataCaps::kMaxBodyBytes + 10, 'x'))); ASSERT_TRUE(invalid([&] { r.d->bigquery.prepare("bigquery.jobs.query", CallArgs().p("projectId", "p1").with_body(big)); })); } TEST(data, no_credential_is_not_configured_after_validation) { Rig r; bool threw = false; try { r.d->bigquery.list_datasets(10); } catch (const NotConfiguredError& e) { threw = true; ASSERT_EQ(e.http_status(), 503); } ASSERT_TRUE(threw); ASSERT_EQ(r.mock->send_count(), 0u); } TEST(data, ids_follow_google_rules) { ASSERT_TRUE(valid([] { Ids::bucket("my-bucket_1.x"); })); ASSERT_TRUE(invalid([] { Ids::bucket("ab"); })); ASSERT_TRUE(invalid([] { Ids::bucket("-bad-"); })); ASSERT_TRUE(invalid([] { Ids::bucket("Upper"); })); ASSERT_TRUE(valid([] { Ids::object("a/b/c d.txt"); })); ASSERT_TRUE(invalid([] { Ids::object(""); })); ASSERT_TRUE(invalid([] { Ids::object(".."); })); ASSERT_TRUE(invalid([] { Ids::object(std::string(1025, 'a')); })); ASSERT_TRUE(valid([] { Ids::pubsub("my-topic.v1"); })); ASSERT_TRUE(invalid([] { Ids::pubsub("1abc"); })); ASSERT_TRUE(invalid([] { Ids::pubsub("ab"); })); ASSERT_TRUE(valid([] { Ids::project("example.com:proj-1"); })); ASSERT_TRUE(invalid([] { Ids::project("Proj"); })); ASSERT_TRUE(valid([] { Ids::dataset("my_data_1"); })); ASSERT_TRUE(invalid([] { Ids::dataset("my-data"); })); ASSERT_TRUE(invalid([] { Ids::dataset("a.b"); })); } TEST(data, bigquery_query_body_and_caps) { QueryRequest q; q.sql = "SELECT 1"; q.dry_run = true; q.default_dataset = "ds"; Json b = q.to_json("p1"); ASSERT_EQ(b.find("query")->as_string(), "SELECT 1"); ASSERT_TRUE(b.find("dryRun")->as_bool()); ASSERT_FALSE(b.find("useLegacySql")->as_bool()); ASSERT_EQ(b.find("defaultDataset")->find("projectId")->as_string(), "p1"); ASSERT_EQ(b.find("timeoutMs")->as_int(), 10000); Rig r; QueryRequest bad; ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); })); // empty sql bad.sql = std::string(DataCaps::kMaxSqlChars + 1, 'x'); ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); })); bad.sql = "SELECT 1"; bad.max_results = 10001; ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); })); bad.max_results = 10; bad.timeout_ms = 60001; ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); })); bad.timeout_ms = 1; bad.maximum_bytes_billed = "12a"; ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); })); bad.maximum_bytes_billed = ""; bad.query_parameters = Json::array(); ASSERT_TRUE(invalid([&] { r.d->bigquery.query(bad); })); // parameter_mode missing bad.parameter_mode = "NAMED"; ASSERT_TRUE(valid([&] { r.d->bigquery.query(bad); })); } TEST(data, bigquery_insert_all_validation) { Rig r; auto& bq = r.d->bigquery; Json rows = Json::parse("[{\"insertId\":\"1\",\"json\":{\"a\":1}}]"); ASSERT_TRUE(valid([&] { bq.insert_all("ds", "t1", rows, false, false); })); ASSERT_TRUE(invalid([&] { bq.insert_all("ds", "t1", Json::array(), false, false); })); ASSERT_TRUE(invalid([&] { bq.insert_all("ds", "t1", Json::parse("[{\"a\":1}]"), false, false); })); Json many = Json::array(); for (std::size_t i = 0; i <= DataCaps::kMaxInsertRows; ++i) many.push(Json::parse("{\"json\":{}}")); ASSERT_TRUE(invalid([&] { bq.insert_all("ds", "t1", many, false, false); })); ASSERT_TRUE(invalid([&] { bq.insert_all("bad-ds", "t1", rows, false, false); })); ASSERT_TRUE(invalid([&] { bq.insert_all("ds", "t/1", rows, false, false); })); } TEST(data, storage_compose_body_and_caps) { std::vector src = {{"a.txt", 0}, {"b.txt", 12345}}; Json b = Storage::compose_body(src, "text/plain"); ASSERT_EQ(b.find("sourceObjects")->as_array().size(), 2u); ASSERT_EQ(b.find("sourceObjects")->as_array()[1].find("generation")->as_string(), "12345"); ASSERT_EQ(b.find("destination")->find("contentType")->as_string(), "text/plain"); ASSERT_TRUE(invalid([] { Storage::compose_body({}, ""); })); std::vector too_many(DataCaps::kMaxComposeSources + 1, ComposeSource{"x", 0}); ASSERT_TRUE(invalid([&] { Storage::compose_body(too_many, ""); })); std::vector max(DataCaps::kMaxComposeSources, ComposeSource{"x", 0}); ASSERT_TRUE(valid([&] { Storage::compose_body(max, ""); })); Rig r; ASSERT_TRUE(invalid([&] { r.d->storage.compose("Bad", src, "out", ""); })); ASSERT_TRUE(valid([&] { r.d->storage.compose("good-bucket", src, "out", ""); })); } TEST(data, storage_create_bucket_class_is_checked) { Rig r; ASSERT_TRUE(invalid([&] { r.d->storage.create_bucket("good-bucket", "US", "FAST"); })); ASSERT_TRUE(valid([&] { r.d->storage.create_bucket("good-bucket", "US", "STANDARD"); })); } TEST(data, pubsub_publish_body_is_base64_and_capped) { PubsubMessage m; m.data = "hello"; m.attributes["k"] = "v"; m.ordering_key = "o1"; Json b = PubSub::publish_body({m}); const Json& msg = b.find("messages")->as_array()[0]; ASSERT_EQ(msg.find("data")->as_string(), "aGVsbG8="); ASSERT_EQ(msg.find("attributes")->find("k")->as_string(), "v"); ASSERT_EQ(msg.find("orderingKey")->as_string(), "o1"); ASSERT_TRUE(invalid([] { PubSub::publish_body({}); })); ASSERT_TRUE(invalid([] { PubSub::publish_body({PubsubMessage{}}); })); std::vector many(DataCaps::kMaxPublishMessages + 1, m); ASSERT_TRUE(invalid([&] { PubSub::publish_body(many); })); PubsubMessage big; big.data = std::string(DataCaps::kMaxMessageBytes + 1, 'x'); ASSERT_TRUE(invalid([&] { PubSub::publish_body({big}); })); Rig r; ASSERT_TRUE(invalid([&] { r.d->pubsub.pull("sub-1", 0); })); ASSERT_TRUE(invalid([&] { r.d->pubsub.pull("sub-1", 101); })); ASSERT_TRUE(valid([&] { r.d->pubsub.pull("sub-1", 10); })); ASSERT_TRUE(invalid([&] { r.d->pubsub.acknowledge("sub-1", {}); })); ASSERT_TRUE(invalid([&] { r.d->pubsub.get_topic("1bad"); })); } TEST(data, firestore_paths) { Rig r; Firestore& f = r.d->firestore; ASSERT_EQ(f.documents_root(), "projects/p1/databases/(default)/documents"); ASSERT_EQ(f.document_name("users/alice"), "projects/p1/databases/(default)/documents/users/alice"); Prepared p = f.prepare("firestore.projects.databases.documents.get", CallArgs().p("name", f.document_name("users/alice"))); ASSERT_EQ(p.path, "/v1/projects/p1/databases/(default)/documents/users/alice"); ASSERT_TRUE(invalid([&] { f.get_document(""); })); ASSERT_TRUE(invalid([&] { f.get_document("users//alice"); })); ASSERT_TRUE(invalid([&] { f.get_document("users/../x"); })); ASSERT_TRUE(invalid([&] { f.get_document("users/a b"); })); ASSERT_TRUE(invalid([&] { f.commit(Json::array()); })); ASSERT_TRUE(invalid([&] { f.run_query("", Json::object()); })); } TEST(data, spanner_sql_validation) { SpannerSql s; ASSERT_TRUE(invalid([&] { s.validate(); })); s.sql = "SELECT 1"; ASSERT_TRUE(valid([&] { s.validate(); })); s.query_mode = "EXPLODE"; ASSERT_TRUE(invalid([&] { s.validate(); })); s.query_mode = "PLAN"; s.params = Json("x"); ASSERT_TRUE(invalid([&] { s.validate(); })); Rig r; ASSERT_EQ(r.d->spanner.database_name("i1", "d1"), "projects/p1/instances/i1/databases/d1"); ASSERT_TRUE(invalid([&] { r.d->spanner.get_instance("a/b"); })); } TEST(data, secret_versions_and_payload_caps) { Rig r; SecretManager& s = r.d->secrets; ASSERT_EQ(s.version_name("s1", "latest"), "projects/p1/secrets/s1/versions/latest"); ASSERT_EQ(s.version_name("s1", "3"), "projects/p1/secrets/s1/versions/3"); ASSERT_TRUE(invalid([&] { s.version_name("s1", "../x"); })); ASSERT_TRUE(invalid([&] { s.version_name("s1", ""); })); ASSERT_TRUE(invalid([&] { s.add_version("s1", ""); })); ASSERT_TRUE(invalid([&] { s.add_version("s1", std::string(DataCaps::kMaxSecretBytes + 1, 'x')); })); ASSERT_TRUE(valid([&] { s.add_version("s1", "value"); })); ASSERT_TRUE(invalid([&] { s.create_secret("bad/id"); })); } TEST(data, dlp_request_validation) { InspectRequest q; ASSERT_TRUE(invalid([&] { q.validate(); })); q.text = "mail me at a@b.com"; q.info_types = {"EMAIL_ADDRESS"}; Json b = q.to_json(); ASSERT_EQ(b.find("item")->find("value")->as_string(), "mail me at a@b.com"); ASSERT_EQ(b.find("inspectConfig")->find("infoTypes")->as_array()[0].find("name")->as_string(), "EMAIL_ADDRESS"); ASSERT_EQ(b.find("inspectConfig")->find("limits")->find("maxFindingsPerRequest")->as_int(), 100); q.min_likelihood = "MAYBE"; ASSERT_TRUE(invalid([&] { q.validate(); })); q.min_likelihood = "LIKELY"; q.text = std::string(DataCaps::kMaxDlpItemBytes + 1, 'x'); ASSERT_TRUE(invalid([&] { q.validate(); })); Rig r; q.text = "x"; ASSERT_TRUE(valid([&] { r.d->dlp.inspect_content(q); })); ASSERT_TRUE(invalid([&] { r.d->dlp.deidentify_content("x", Json::object(), ""); })); } TEST(data, kms_names_and_caps) { Rig r; Kms& k = r.d->kms; ASSERT_EQ(k.version_name("global", "ring", "key", "2"), "projects/p1/locations/global/keyRings/ring/cryptoKeys/key/cryptoKeyVersions/2"); ASSERT_TRUE(invalid([&] { k.version_name("global", "ring", "key", "x"); })); ASSERT_TRUE(invalid([&] { k.encrypt("global", "ring", "key", "", ""); })); ASSERT_TRUE(invalid([&] { k.encrypt("global", "ring", "key", std::string(DataCaps::kMaxCryptoBytes + 1, 'x'), ""); })); ASSERT_TRUE(valid([&] { k.encrypt("global", "ring", "key", "secret", ""); })); ASSERT_TRUE(invalid([&] { k.decrypt("global", "ring", "key", "not base64!!", ""); })); ASSERT_TRUE(invalid([&] { k.sign_digest("global", "ring", "key", "1", "short"); })); ASSERT_TRUE(valid([&] { k.sign_digest("global", "ring", "key", "1", std::string(32, 'a')); })); } TEST(data, datastore_and_dataplex_validation) { Rig r; ASSERT_TRUE(invalid([&] { r.d->datastore.lookup(Json::array()); })); ASSERT_TRUE(invalid([&] { r.d->datastore.run_query(Json::object(), Json::object()); })); ASSERT_TRUE(invalid([&] { r.d->datastore.run_query(Json(), Json()); })); ASSERT_TRUE(invalid([&] { r.d->datastore.commit(Json::array(), "TRANSACTIONAL", ""); })); Json m = Json::array(); m.push(Json::object()); ASSERT_TRUE(invalid([&] { r.d->datastore.commit(m, "MAYBE", ""); })); ASSERT_TRUE(valid([&] { r.d->datastore.commit(m, "NON_TRANSACTIONAL", ""); })); ASSERT_TRUE(invalid([&] { r.d->dataplex.search_entries("global", "", 10); })); ASSERT_TRUE(invalid([&] { r.d->dataplex.lookup_entry("global", "entries/x"); })); ASSERT_TRUE(invalid([&] { r.d->bigtable.get_instance("a b"); })); } TEST(data, every_typed_call_id_exists_in_the_table) { // The typed classes name their calls as string literals; check_discovery.py // compares those with Discovery. Here: a spot check of the rows they hit. for (const char* id : {"bigquery.jobs.query", "bigquery.tabledata.insertAll", "storage.objects.compose", "pubsub.projects.subscriptions.pull", "firestore.projects.databases.documents.runQuery", "spanner.projects.instances.databases.sessions.executeSql", "bigtableadmin.projects.instances.tables.list", "secretmanager.projects.secrets.versions.access", "dataplex.projects.locations.searchEntries", "dlp.projects.content.inspect", "cloudkms.projects.locations.keyRings.cryptoKeys.encrypt", "datastore.projects.runQuery"}) ASSERT_TRUE(Catalog::find(id) != nullptr); }