feat(clients): data: BigQuery, Storage media and signed URLs, Pub/Sub, Firestore, Spanner, Bigtable admin, Secret Manager, Dataplex, DLP, KMS, Datastore

Discovery generated call table (918 rows), typed services, /google/data demo.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-25 15:03:43 +01:00
co-authored by Claude Sonnet 5.5
parent 3db67fcf1f
commit 1af14b5d74
47 changed files with 6315 additions and 3 deletions
+48
View File
@@ -0,0 +1,48 @@
// Routes: POST /google/data/call and /google/data/storage/sign
// The generic call route validates against the Discovery table; writes are
// dry runs unless confirm:true; the signer is a dry run unless confirm:true.
#include <cstdlib>
#include "support/GoogleRig.hpp"
using namespace molo::example::test;
int main() {
setenv("GOOGLE_CLOUD_PROJECT", "test-project", 1);
GoogleRig rig(false);
Fixture f(rig.client);
auto j = [&](const char* body) { return f.client->post("/google/data/call", body, "application/json"); };
auto read = j("{\"rpc\":\"bigquery.datasets.list\",\"path\":{\"projectId\":\"p1\"},\"query\":{\"maxResults\":5}}");
check(read.status == 503 && contains(read.body, "google_not_configured"), "valid GET without credential is 503");
auto post = j("{\"rpc\":\"pubsub.projects.topics.publish\",\"path\":{\"topic\":\"projects/p1/topics/t1\"},\"body\":{\"messages\":[]}}");
check(post.status == 200 && contains(post.body, "\"dry_run\":true") &&
contains(post.body, "https://pubsub.googleapis.com/v1/projects/p1/topics/t1:publish"),
"non-GET is a dry run that shows the URL");
auto confirmed = j("{\"rpc\":\"pubsub.projects.topics.publish\",\"path\":{\"topic\":\"projects/p1/topics/t1\"},\"body\":{\"messages\":[]},\"confirm\":true}");
check(confirmed.status == 503, "confirmed non-GET without credential is 503");
check(j("{}").status == 422, "missing rpc is 422");
check(j("{\"rpc\":\"nope.x\"}").status == 422, "unknown rpc is 422");
check(j("{\"rpc\":\"bigquery.datasets.list\"}").status == 422, "missing path value is 422");
check(j("{\"rpc\":\"bigquery.datasets.list\",\"path\":{\"projectId\":\"../x\"}}").status == 422, "path traversal is 422");
check(j("{\"rpc\":\"bigquery.datasets.list\",\"path\":{\"projectId\":\"p\"},\"query\":{\"zzz\":1}}").status == 422, "unknown query parameter is 422");
check(j("{\"rpc\":\"bigquery.jobs.query\",\"path\":{\"projectId\":\"p\"},\"body\":{\"zzz\":1}}").status == 422, "unknown body field is 422");
check(j("{\"rpc\":\"bigquery.datasets.list\",\"path\":{\"projectId\":5}}").status == 422, "non-string path value is 422");
check(f.client->get("/google/data/call").status == 405, "get is 405");
auto sign = [&](const char* body) { return f.client->post("/google/data/storage/sign", body, "application/json"); };
auto dry = sign("{\"bucket\":\"my-bucket\",\"object\":\"a/b.txt\",\"expires_seconds\":600}");
check(dry.status == 200 && contains(dry.body, "\"dry_run\":true") && !contains(dry.body, "X-Goog-Signature"),
"signing is a dry run by default");
check(sign("{\"bucket\":\"my-bucket\",\"expires_seconds\":604801}").status == 422, "expiry over 7 days is 422");
check(sign("{\"bucket\":\"my-bucket\",\"expires_seconds\":0}").status == 422, "expiry 0 is 422");
check(sign("{\"bucket\":\"Bad\"}").status == 422, "bad bucket is 422");
check(sign("{\"bucket\":\"my-bucket\",\"method\":\"TRACE\"}").status == 422, "bad method is 422");
check(sign("{}").status == 422, "missing bucket is 422");
check(sign("{\"bucket\":\"my-bucket\",\"headers\":{\"host\":\"x\"}}").status == 422, "host header is 422");
auto real = sign("{\"bucket\":\"my-bucket\",\"object\":\"o\",\"confirm\":true}");
check(real.status == 503 && contains(real.body, "google_not_configured"), "confirmed signing without a key is 503");
check(f.client->get("/google/data/storage/sign").status == 405, "sign get is 405");
check(rig.mock->send_count() == 0, "nothing reached Google");
return finish("google/data/call");
}
+123
View File
@@ -0,0 +1,123 @@
// Routes: POST /google/data/{bigquery,storage,pubsub,firestore,spanner,bigtable,
// secrets,dataplex,dlp,kms,datastore}
// Without a credential a well formed read is 503 google_not_configured; bad
// input is 422 and never reaches Google; writes, secret reads and paid calls
// are dry runs unless confirm:true; GET is 405. Nothing reaches Google.
#include <cstdlib>
#include "support/GoogleRig.hpp"
using namespace molo::example::test;
namespace {
struct Case {
const char* path;
const char* ok; // valid read, expected 503 without credential
const char* bad; // invalid input, expected 422
};
const Case kReads[] = {
{"/google/data/bigquery", "{\"action\":\"dry_run\",\"sql\":\"SELECT 1\"}", "{\"action\":\"dry_run\"}"},
{"/google/data/bigquery", "{\"action\":\"datasets\"}", "{\"action\":\"datasets\",\"max_results\":99999}"},
{"/google/data/bigquery", "{\"action\":\"table\",\"dataset\":\"d\",\"table\":\"t\"}", "{\"action\":\"table\",\"dataset\":\"bad-id\",\"table\":\"t\"}"},
{"/google/data/bigquery", "{\"action\":\"rows\",\"dataset\":\"d\",\"table\":\"t\"}", "{\"action\":\"rows\",\"dataset\":\"d\"}"},
{"/google/data/bigquery", "{\"action\":\"dry_run\",\"sql\":\"SELECT 1\",\"timeout_ms\":1000}", "{\"action\":\"dry_run\",\"sql\":\"x\",\"timeout_ms\":999999}"},
{"/google/data/storage", "{\"action\":\"buckets\"}", "{\"action\":\"buckets\",\"max_results\":0}"},
{"/google/data/storage", "{\"action\":\"objects\",\"bucket\":\"my-bucket\"}", "{\"action\":\"objects\",\"bucket\":\"Bad_Bucket\"}"},
{"/google/data/storage", "{\"action\":\"object\",\"bucket\":\"my-bucket\",\"object\":\"a/b.txt\"}", "{\"action\":\"object\",\"bucket\":\"my-bucket\",\"object\":\"..\"}"},
{"/google/data/storage", "{\"action\":\"download\",\"bucket\":\"my-bucket\",\"object\":\"a.txt\"}", "{\"action\":\"download\",\"bucket\":\"my-bucket\"}"},
{"/google/data/pubsub", "{\"action\":\"topics\"}", "{\"action\":\"topic\",\"topic\":\"1bad\"}"},
{"/google/data/pubsub", "{\"action\":\"subscription\",\"subscription\":\"sub-1\"}", "{\"action\":\"subscription\"}"},
{"/google/data/firestore", "{\"action\":\"get\",\"path\":\"users/alice\"}", "{\"action\":\"get\",\"path\":\"users//alice\"}"},
{"/google/data/firestore", "{\"action\":\"collections\"}", "{\"action\":\"list\",\"collection\":\"\"}"},
{"/google/data/firestore", "{\"action\":\"query\",\"query\":{\"from\":[{\"collectionId\":\"c\"}]}}", "{\"action\":\"query\"}"},
{"/google/data/spanner", "{\"action\":\"instances\"}", "{\"action\":\"instance\",\"instance\":\"a/b\"}"},
{"/google/data/spanner", "{\"action\":\"ddl\",\"instance\":\"i1\",\"database\":\"d1\"}", "{\"action\":\"ddl\",\"instance\":\"i1\"}"},
{"/google/data/bigtable", "{\"action\":\"instances\"}", "{\"action\":\"instance\",\"instance\":\"a b\"}"},
{"/google/data/bigtable", "{\"action\":\"tables\",\"instance\":\"i1\"}", "{\"action\":\"tables\"}"},
{"/google/data/secrets", "{\"action\":\"list\"}", "{\"action\":\"version\",\"secret\":\"s\",\"version\":\"../x\"}"},
{"/google/data/secrets", "{\"action\":\"versions\",\"secret\":\"s1\"}", "{\"action\":\"versions\"}"},
{"/google/data/dataplex", "{\"action\":\"locations\"}", "{\"action\":\"search\",\"query\":\"\"}"},
{"/google/data/dataplex", "{\"action\":\"search\",\"location\":\"global\",\"query\":\"type=table\"}", "{\"action\":\"lookup\",\"entry\":\"nope\"}"},
{"/google/data/dlp", "{\"action\":\"info_types\"}", "{\"action\":\"info_types\",\"language\":\"aaaaaaaaaaaaaaaaaaaaaaaa\"}"},
{"/google/data/kms", "{\"action\":\"rings\",\"location\":\"global\"}", "{\"action\":\"rings\"}"},
{"/google/data/kms", "{\"action\":\"public_key\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"version\":\"1\"}", "{\"action\":\"public_key\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"version\":\"x\"}"},
{"/google/data/datastore", "{\"action\":\"query\",\"gql\":{\"queryString\":\"SELECT * FROM k\"}}", "{\"action\":\"query\"}"},
{"/google/data/datastore", "{\"action\":\"lookup\",\"keys\":[{\"path\":[{\"kind\":\"k\",\"id\":\"1\"}]}]}", "{\"action\":\"lookup\",\"keys\":[]}"},
};
// Confirm-required: dry run without confirm, 503 with it, 422 on bad input when confirmed.
struct Gated {
const char* path;
const char* body; // no confirm
const char* confirmed; // with confirm:true
const char* bad; // confirm:true plus invalid input
};
const Gated kGated[] = {
{"/google/data/bigquery", "{\"action\":\"query\",\"sql\":\"SELECT 1\"}", "{\"action\":\"query\",\"sql\":\"SELECT 1\",\"confirm\":true}", "{\"action\":\"query\",\"sql\":\"\",\"confirm\":true}"},
{"/google/data/bigquery", "{\"action\":\"insert\",\"dataset\":\"d\",\"table\":\"t\",\"rows\":[{\"json\":{}}]}", "{\"action\":\"insert\",\"dataset\":\"d\",\"table\":\"t\",\"rows\":[{\"json\":{}}],\"confirm\":true}", "{\"action\":\"insert\",\"dataset\":\"d\",\"table\":\"t\",\"rows\":[],\"confirm\":true}"},
{"/google/data/bigquery", "{\"action\":\"delete_table\",\"dataset\":\"d\",\"table\":\"t\"}", "{\"action\":\"delete_table\",\"dataset\":\"d\",\"table\":\"t\",\"confirm\":true}", "{\"action\":\"delete_table\",\"dataset\":\"d\",\"table\":\"a/b\",\"confirm\":true}"},
{"/google/data/bigquery", "{\"action\":\"delete_dataset\",\"dataset\":\"d\"}", "{\"action\":\"delete_dataset\",\"dataset\":\"d\",\"confirm\":true}", "{\"action\":\"delete_dataset\",\"dataset\":\"d-d\",\"confirm\":true}"},
{"/google/data/storage", "{\"action\":\"upload\",\"bucket\":\"my-bucket\",\"object\":\"o\",\"text\":\"hi\"}", "{\"action\":\"upload\",\"bucket\":\"my-bucket\",\"object\":\"o\",\"text\":\"hi\",\"confirm\":true}", "{\"action\":\"upload\",\"bucket\":\"my-bucket\",\"object\":\"o\",\"confirm\":true}"},
{"/google/data/storage", "{\"action\":\"resumable\",\"bucket\":\"my-bucket\",\"object\":\"o\",\"text\":\"hi\"}", "{\"action\":\"resumable\",\"bucket\":\"my-bucket\",\"object\":\"o\",\"text\":\"hi\",\"confirm\":true}", "{\"action\":\"resumable\",\"bucket\":\"my-bucket\",\"object\":\"o\",\"text\":\"a\",\"base64\":\"YQ==\",\"confirm\":true}"},
{"/google/data/storage", "{\"action\":\"compose\",\"bucket\":\"my-bucket\",\"sources\":[\"a\"],\"destination\":\"d\"}", "{\"action\":\"compose\",\"bucket\":\"my-bucket\",\"sources\":[\"a\"],\"destination\":\"d\",\"confirm\":true}", "{\"action\":\"compose\",\"bucket\":\"my-bucket\",\"sources\":[],\"destination\":\"d\",\"confirm\":true}"},
{"/google/data/storage", "{\"action\":\"delete\",\"bucket\":\"my-bucket\",\"object\":\"o\"}", "{\"action\":\"delete\",\"bucket\":\"my-bucket\",\"object\":\"o\",\"confirm\":true}", "{\"action\":\"delete\",\"bucket\":\"my-bucket\",\"object\":\"\",\"confirm\":true}"},
{"/google/data/pubsub", "{\"action\":\"publish\",\"topic\":\"topic-1\",\"messages\":[\"x\"]}", "{\"action\":\"publish\",\"topic\":\"topic-1\",\"messages\":[\"x\"],\"confirm\":true}", "{\"action\":\"publish\",\"topic\":\"topic-1\",\"messages\":[],\"confirm\":true}"},
{"/google/data/pubsub", "{\"action\":\"pull\",\"subscription\":\"sub-1\"}", "{\"action\":\"pull\",\"subscription\":\"sub-1\",\"confirm\":true}", "{\"action\":\"pull\",\"subscription\":\"sub-1\",\"max_messages\":500,\"confirm\":true}"},
{"/google/data/pubsub", "{\"action\":\"delete_topic\",\"topic\":\"topic-1\"}", "{\"action\":\"delete_topic\",\"topic\":\"topic-1\",\"confirm\":true}", "{\"action\":\"delete_topic\",\"topic\":\"1\",\"confirm\":true}"},
{"/google/data/firestore", "{\"action\":\"delete\",\"path\":\"a/b\"}", "{\"action\":\"delete\",\"path\":\"a/b\",\"confirm\":true}", "{\"action\":\"delete\",\"path\":\"\",\"confirm\":true}"},
{"/google/data/firestore", "{\"action\":\"commit\",\"writes\":[{}]}", "{\"action\":\"commit\",\"writes\":[{}],\"confirm\":true}", "{\"action\":\"commit\",\"writes\":[],\"confirm\":true}"},
{"/google/data/spanner", "{\"action\":\"query\",\"instance\":\"i\",\"database\":\"d\",\"sql\":\"SELECT 1\"}", "{\"action\":\"query\",\"instance\":\"i\",\"database\":\"d\",\"sql\":\"SELECT 1\",\"confirm\":true}", "{\"action\":\"query\",\"instance\":\"i\",\"database\":\"d\",\"sql\":\"SELECT 1\",\"query_mode\":\"BOOM\",\"confirm\":true}"},
{"/google/data/secrets", "{\"action\":\"access\",\"secret\":\"s\",\"version\":\"latest\"}", "{\"action\":\"access\",\"secret\":\"s\",\"version\":\"latest\",\"confirm\":true}", "{\"action\":\"access\",\"secret\":\"s\",\"version\":\"abc\",\"confirm\":true}"},
{"/google/data/secrets", "{\"action\":\"add_version\",\"secret\":\"s\",\"payload\":\"v\"}", "{\"action\":\"add_version\",\"secret\":\"s\",\"payload\":\"v\",\"confirm\":true}", "{\"action\":\"add_version\",\"secret\":\"s\",\"confirm\":true}"},
{"/google/data/secrets", "{\"action\":\"delete\",\"secret\":\"s\"}", "{\"action\":\"delete\",\"secret\":\"s\",\"confirm\":true}", "{\"action\":\"delete\",\"secret\":\"a/b\",\"confirm\":true}"},
{"/google/data/dlp", "{\"action\":\"inspect\",\"text\":\"a@b.com\"}", "{\"action\":\"inspect\",\"text\":\"a@b.com\",\"confirm\":true}", "{\"action\":\"inspect\",\"text\":\"a\",\"min_likelihood\":\"MAYBE\",\"confirm\":true}"},
{"/google/data/kms", "{\"action\":\"encrypt\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"plaintext\":\"x\"}", "{\"action\":\"encrypt\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"plaintext\":\"x\",\"confirm\":true}", "{\"action\":\"encrypt\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"plaintext\":\"\",\"confirm\":true}"},
{"/google/data/kms", "{\"action\":\"decrypt\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"ciphertext\":\"YQ==\"}", "{\"action\":\"decrypt\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"ciphertext\":\"YQ==\",\"confirm\":true}", "{\"action\":\"decrypt\",\"location\":\"global\",\"ring\":\"r\",\"key\":\"k\",\"ciphertext\":\"!!\",\"confirm\":true}"},
{"/google/data/datastore", "{\"action\":\"commit\",\"mutations\":[{}]}", "{\"action\":\"commit\",\"mutations\":[{}],\"confirm\":true}", "{\"action\":\"commit\",\"mutations\":[{}],\"mode\":\"MAYBE\",\"confirm\":true}"},
};
} // namespace
int main() {
setenv("GOOGLE_CLOUD_PROJECT", "test-project", 1);
GoogleRig rig(false);
Fixture f(rig.client);
for (const Case& c : kReads) {
auto ok = f.client->post(c.path, c.ok, "application/json");
check(ok.status == 503 && contains(ok.body, "google_not_configured"),
std::string(c.path) + " read without credential is 503: " + c.ok);
auto bad = f.client->post(c.path, c.bad, "application/json");
check(bad.status == 422, std::string(c.path) + " bad input is 422: " + c.bad);
}
for (const Gated& g : kGated) {
auto dry = f.client->post(g.path, g.body, "application/json");
check(dry.status == 200 && contains(dry.body, "\"dry_run\":true"),
std::string(g.path) + " default is a dry run: " + g.body);
auto real = f.client->post(g.path, g.confirmed, "application/json");
check(real.status == 503 && contains(real.body, "google_not_configured"),
std::string(g.path) + " confirmed without credential is 503: " + g.confirmed);
check(f.client->post(g.path, g.bad, "application/json").status == 422,
std::string(g.path) + " confirmed bad input is 422: " + g.bad);
std::string odd = g.body;
odd.pop_back();
odd += ",\"confirm\":\"yes\"}";
check(f.client->post(g.path, odd, "application/json").status == 422,
std::string(g.path) + " non-bool confirm is 422");
}
for (const char* path : {"/google/data/bigquery", "/google/data/storage", "/google/data/pubsub",
"/google/data/firestore", "/google/data/spanner", "/google/data/bigtable",
"/google/data/secrets", "/google/data/dataplex", "/google/data/dlp",
"/google/data/kms", "/google/data/datastore"}) {
check(f.client->get(path).status == 405, std::string(path) + " get is 405");
check(f.client->post(path, "{}", "application/json").status == 422, std::string(path) + " missing action is 422");
check(f.client->post(path, "{\"action\":\"explode\"}", "application/json").status == 422,
std::string(path) + " unknown action is 422");
check(f.client->post(path, "not json", "application/json").status == 422, std::string(path) + " invalid json is 422");
check(f.client->post(path, "[]", "application/json").status == 422, std::string(path) + " non-object body is 422");
check(f.client->post(path, std::string("{\"action\":\"") + std::string(3 * 1024 * 1024, 'x') + "\"}",
"application/json").status == 413,
std::string(path) + " oversize body is 413");
}
check(rig.mock->send_count() == 0, "nothing reached Google");
return finish("google/data/groups");
}
+20
View File
@@ -0,0 +1,20 @@
// Route: GET /google/data (page, no credential needed)
#include "support/GoogleRig.hpp"
using namespace molo::example::test;
int main() {
setenv("GOOGLE_CLOUD_PROJECT", "test-project", 1);
GoogleRig rig(false);
Fixture f(rig.client);
auto res = f.client->get("/google/data");
check(res.status == 200, "status 200");
check(contains(res.headers.get("Content-Type"), "text/html"), "html");
check(contains(res.body, "Google data") && contains(res.body, "viewport"), "page and mobile viewport");
check(contains(res.body, "/embedded/google_data.js"), "script wired");
check(contains(res.body, "<select id=\"gd-service\">"), "services are a dropdown, not a button group");
check(f.client->post("/google/data", "{}", "application/json").status == 405, "post is 405");
check(f.client->get("/embedded/google_data.js").status == 200, "script is served");
check(rig.mock->send_count() == 0, "no network");
return finish("google/data");
}
+30
View File
@@ -0,0 +1,30 @@
// Routes: GET /google/data/status and /google/data/calls
#include "support/GoogleRig.hpp"
using namespace molo::example::test;
int main() {
setenv("GOOGLE_CLOUD_PROJECT", "test-project", 1);
GoogleRig off(false);
Fixture f(off.client);
auto res = f.client->get("/google/data/status");
check(res.status == 200, "status 200");
check(contains(res.body, "\"configured\":false"), "reports no credential");
check(contains(res.body, "\"project_set\":true"), "project read from env");
for (const char* api : {"bigquery", "storage", "pubsub", "firestore", "spanner", "bigtableadmin",
"secretmanager", "dataplex", "dlp", "cloudkms", "datastore"})
check(contains(res.body, std::string("\"name\":\"") + api + "\""), std::string("lists ") + api);
check(contains(res.body, "\"service\":\"bigquery.googleapis.com\""), "shows the service to enable");
check(contains(res.body, "\"max_signed_url_seconds\":604800"), "signed URL cap shown");
auto all = f.client->get("/google/data/calls?api=pubsub");
check(all.status == 200 && contains(all.body, "pubsub.projects.topics.publish"), "table lists publish");
check(!contains(all.body, "bigquery.jobs.query"), "api filter applies");
check(contains(f.client->get("/google/data/calls?api=storage").body, "storage.objects.compose"), "table lists compose");
GoogleRig on;
Fixture g(on.client);
check(contains(g.client->get("/google/data/status").body, "\"configured\":true"), "reports credential");
check(f.client->post("/google/data/status", "{}", "application/json").status == 405, "post is 405");
check(f.client->post("/google/data/calls", "{}", "application/json").status == 405, "calls post is 405");
check(on.mock->send_count() == 0, "no network");
return finish("google/data/status");
}