feat(clients): data: BigQuery, Storage media and signed URLs, Pub/Sub, Firestore, Spanner, Bigtable admin, Secret Manager, Dataplex, DLP, KMS, Datastore
Discovery generated call table (918 rows), typed services, /google/data demo. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,13 +1,757 @@
|
||||
#include "../GoogleDemos.hpp"
|
||||
|
||||
#include <functional>
|
||||
#include <map>
|
||||
|
||||
#include "../../fw/Router.hpp"
|
||||
#include "../../fw/Templates.hpp"
|
||||
#include "google/data/data.h"
|
||||
#include "google/data/caps.h"
|
||||
|
||||
namespace molo::example::app {
|
||||
|
||||
// BigQuery, Cloud Storage, Pub/Sub, Firestore, Spanner, Bigtable, Secret Manager, Dataplex, DLP, KMS. Stub: the package agent fills this in.
|
||||
// Google data package: BigQuery, Cloud Storage, Pub/Sub, Firestore, Spanner,
|
||||
// Bigtable admin, Secret Manager, Dataplex, DLP, Cloud KMS, Datastore over the
|
||||
// real src/google/data part. GET page, /status and /calls answer without a
|
||||
// credential. Every other route is a POST: 422 on bad input, 503
|
||||
// google_not_configured without a credential. Reads run directly; writes,
|
||||
// secret reads and paid operations are dry runs unless the body has
|
||||
// "confirm":true.
|
||||
|
||||
namespace {
|
||||
|
||||
namespace dt = google::data;
|
||||
using Json = clients::body::Json;
|
||||
|
||||
constexpr std::size_t kMaxBody = 3 * 1024 * 1024;
|
||||
constexpr std::size_t kMaxShown = 256 * 1024; // response bytes returned to the page
|
||||
constexpr std::size_t kMaxDemoUpload = 2 * 1024 * 1024;
|
||||
constexpr std::size_t kMaxDownloadShown = 64 * 1024;
|
||||
|
||||
const char kPage[] = R"DATA({% extends "base.html" %}
|
||||
{% block content %}
|
||||
<h1>Google data</h1>
|
||||
<p class="hint">BigQuery, Cloud Storage, Pub/Sub, Firestore, Spanner, Bigtable admin,
|
||||
Secret Manager, Dataplex, DLP, Cloud KMS and Datastore. Reads run at once; writes,
|
||||
secret reads and paid calls stay a dry run until the body says <code>"confirm":true</code>.</p>
|
||||
<p id="gd-config" class="hint">Checking Google credentials...</p>
|
||||
<section class="gd">
|
||||
<label>Service
|
||||
<select id="gd-service">
|
||||
<option value="bigquery">BigQuery</option>
|
||||
<option value="storage">Cloud Storage</option>
|
||||
<option value="pubsub">Pub/Sub</option>
|
||||
<option value="firestore">Firestore</option>
|
||||
<option value="spanner">Spanner</option>
|
||||
<option value="bigtable">Bigtable admin</option>
|
||||
<option value="secrets">Secret Manager</option>
|
||||
<option value="dataplex">Dataplex</option>
|
||||
<option value="dlp">DLP</option>
|
||||
<option value="kms">Cloud KMS</option>
|
||||
<option value="datastore">Datastore</option>
|
||||
<option value="storage/sign">Storage signed URL</option>
|
||||
<option value="call">Any call by id</option>
|
||||
</select></label>
|
||||
<label>Action <select id="gd-action"></select></label>
|
||||
<label>Body (JSON object) <textarea id="gd-body" rows="7"></textarea></label>
|
||||
<div class="gd-actions"><button id="gd-send" type="button" class="primary">Send</button></div>
|
||||
<p id="gd-error" class="error" role="alert"></p>
|
||||
<pre id="gd-out" class="gd-out"></pre>
|
||||
</section>
|
||||
<style>
|
||||
.gd label{display:block;margin:.5rem 0}
|
||||
.gd select,.gd textarea{width:100%;box-sizing:border-box}
|
||||
.gd-actions{display:flex;flex-wrap:wrap;gap:.5rem;margin:.5rem 0}
|
||||
.gd-out{overflow:auto;max-height:26rem;white-space:pre-wrap;word-break:break-word}
|
||||
</style>
|
||||
<script src="/embedded/google_data.js"></script>
|
||||
{% endblock %}
|
||||
)DATA";
|
||||
|
||||
fw::Response json_out(const Json& j, int status = 200) {
|
||||
fw::Response res = fw::JsonResponse(j.dump(), status);
|
||||
res.headers.set("Cache-Control", "no-store");
|
||||
return res;
|
||||
}
|
||||
|
||||
[[noreturn]] void bad(const std::string& why) {
|
||||
throw fw::HttpError(422, "validation_error", why);
|
||||
}
|
||||
|
||||
[[noreturn]] void rethrow(const google::GoogleError& e) {
|
||||
if (e.http_status() == 503 && e.status() == "NOT_CONFIGURED")
|
||||
throw fw::HttpError(503, "google_not_configured", e.message());
|
||||
if (e.service().empty() && e.status() == "INVALID_ARGUMENT")
|
||||
throw fw::HttpError(422, "validation_error", e.message());
|
||||
throw fw::HttpError(502, "google_error", e.status() + ": " + e.message());
|
||||
}
|
||||
|
||||
Json parse_body(const fw::Request& req) {
|
||||
if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large");
|
||||
if (req.body.empty()) return Json::object();
|
||||
try {
|
||||
Json j = Json::parse(req.body);
|
||||
if (!j.is_object()) bad("body must be a JSON object");
|
||||
return j;
|
||||
} catch (const clients::body::JsonError&) {
|
||||
bad("body is not valid JSON");
|
||||
}
|
||||
}
|
||||
|
||||
std::string str(const Json& b, const char* key, bool required = false) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) {
|
||||
if (required) bad(std::string(key) + " is required");
|
||||
return "";
|
||||
}
|
||||
if (!v->is_string()) bad(std::string(key) + " must be a string");
|
||||
return v->as_string();
|
||||
}
|
||||
|
||||
int num(const Json& b, const char* key, int fallback) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return fallback;
|
||||
if (!v->is_number()) bad(std::string(key) + " must be a number");
|
||||
return static_cast<int>(v->as_int());
|
||||
}
|
||||
|
||||
bool flag(const Json& b, const char* key) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return false;
|
||||
if (!v->is_bool()) bad(std::string(key) + " must be true or false");
|
||||
return v->as_bool();
|
||||
}
|
||||
|
||||
Json member(const Json& b, const char* key, bool object, bool required = false) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) {
|
||||
if (required) bad(std::string(key) + " is required");
|
||||
return Json();
|
||||
}
|
||||
if (object ? !v->is_object() : !v->is_array())
|
||||
bad(std::string(key) + (object ? " must be a JSON object" : " must be a JSON array"));
|
||||
return *v;
|
||||
}
|
||||
|
||||
std::vector<std::string> strings(const Json& b, const char* key) {
|
||||
std::vector<std::string> out;
|
||||
Json a = member(b, key, false);
|
||||
if (a.is_null()) return out;
|
||||
for (const Json& x : a.as_array()) {
|
||||
if (!x.is_string()) bad(std::string(key) + " must hold strings");
|
||||
out.push_back(x.as_string());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// A response that fits the page; bigger ones are cut and say so.
|
||||
Json shown(const Json& j) {
|
||||
std::string text = j.dump();
|
||||
if (text.size() <= kMaxShown) return j;
|
||||
Json o = Json::object();
|
||||
o.set("truncated", Json(true));
|
||||
o.set("bytes", Json(static_cast<double>(text.size())));
|
||||
o.set("head", Json(text.substr(0, kMaxShown / 4)));
|
||||
return o;
|
||||
}
|
||||
|
||||
Json dry(const std::string& group, const std::string& action) {
|
||||
Json o = Json::object();
|
||||
o.set("dry_run", Json(true));
|
||||
o.set("service", Json(group));
|
||||
o.set("action", Json(action));
|
||||
o.set("hint", Json("send confirm:true to run this action"));
|
||||
return o;
|
||||
}
|
||||
|
||||
// One action of a service: runs against Google, or is a dry run until confirmed.
|
||||
struct Action {
|
||||
bool confirm_required;
|
||||
std::function<Json(const Json&)> run;
|
||||
};
|
||||
using ActionMap = std::map<std::string, Action>;
|
||||
|
||||
class DataRoutes {
|
||||
public:
|
||||
explicit DataRoutes(std::shared_ptr<google::GoogleClient> client) : client_(std::move(client)) {
|
||||
build_bigquery();
|
||||
build_storage();
|
||||
build_pubsub();
|
||||
build_firestore();
|
||||
build_spanner();
|
||||
build_bigtable();
|
||||
build_secrets();
|
||||
build_dataplex();
|
||||
build_dlp();
|
||||
build_kms();
|
||||
build_datastore();
|
||||
}
|
||||
|
||||
fw::Response handle(const std::string& group, const fw::Request& req) const {
|
||||
Json body = parse_body(req);
|
||||
std::string action = str(body, "action", true);
|
||||
const ActionMap& actions = groups_.at(group);
|
||||
auto it = actions.find(action);
|
||||
if (it == actions.end()) {
|
||||
std::string known;
|
||||
for (const auto& kv : actions) known += (known.empty() ? "" : ", ") + kv.first;
|
||||
bad("unknown action; use one of: " + known);
|
||||
}
|
||||
if (it->second.confirm_required && !flag(body, "confirm")) return json_out(dry(group, action));
|
||||
try {
|
||||
return json_out(shown(it->second.run(body)));
|
||||
} catch (const google::GoogleError& e) {
|
||||
rethrow(e);
|
||||
}
|
||||
}
|
||||
|
||||
fw::Response sign(const fw::Request& req) const {
|
||||
Json body = parse_body(req);
|
||||
dt::SignedUrlRequest r;
|
||||
r.method = str(body, "method");
|
||||
if (r.method.empty()) r.method = "GET";
|
||||
r.bucket = str(body, "bucket", true);
|
||||
r.object = str(body, "object");
|
||||
r.expires_seconds = num(body, "expires_seconds", 900);
|
||||
Json headers = member(body, "headers", true);
|
||||
if (!headers.is_null())
|
||||
for (const auto& kv : headers.as_object()) {
|
||||
if (!kv.second.is_string()) bad("header values must be strings");
|
||||
r.headers[kv.first] = kv.second.as_string();
|
||||
}
|
||||
try {
|
||||
r.validate();
|
||||
// A signed URL is a bearer capability: dry run unless confirmed.
|
||||
if (!flag(body, "confirm")) {
|
||||
Json o = dry("storage/sign", "sign");
|
||||
o.set("expires_seconds", Json(static_cast<double>(r.expires_seconds)));
|
||||
o.set("max_expires_seconds", Json(static_cast<double>(dt::DataCaps::kMaxSignedUrlSeconds)));
|
||||
return json_out(o);
|
||||
}
|
||||
dt::SignedUrl signer = dt::SignedUrl::from_env();
|
||||
Json o = Json::object();
|
||||
o.set("url", Json(signer.sign(r)));
|
||||
o.set("method", Json(r.method));
|
||||
o.set("expires_seconds", Json(static_cast<double>(r.expires_seconds)));
|
||||
o.set("signer", Json(signer.email()));
|
||||
return json_out(o);
|
||||
} catch (const google::GoogleError& e) {
|
||||
rethrow(e);
|
||||
}
|
||||
}
|
||||
|
||||
// Any row of the call table by id. Non-GET rows are dry runs unless confirmed.
|
||||
fw::Response call(const fw::Request& req) const {
|
||||
Json body = parse_body(req);
|
||||
std::string rpc = str(body, "rpc", true);
|
||||
try {
|
||||
const dt::CallSpec& spec = dt::Catalog::get(rpc);
|
||||
dt::CallArgs args;
|
||||
Json path = member(body, "path", true);
|
||||
if (!path.is_null())
|
||||
for (const auto& kv : path.as_object()) {
|
||||
if (!kv.second.is_string()) bad("path values must be strings");
|
||||
args.p(kv.first, kv.second.as_string());
|
||||
}
|
||||
Json query = member(body, "query", true);
|
||||
if (!query.is_null()) args.query = query;
|
||||
Json b = member(body, "body", true);
|
||||
if (!b.is_null()) args.body = b;
|
||||
dt::ApiService svc(client_, dt::Catalog::api_of(spec));
|
||||
dt::Prepared prepared = svc.prepare(rpc, args); // validates: 422
|
||||
bool read = std::string(spec.http) == "GET";
|
||||
if (!read && !flag(body, "confirm")) {
|
||||
Json o = dry("call", rpc);
|
||||
o.set("method", Json(spec.http));
|
||||
o.set("url", Json(prepared.url()));
|
||||
return json_out(o);
|
||||
}
|
||||
return json_out(shown(svc.invoke(rpc, args)));
|
||||
} catch (const google::GoogleError& e) {
|
||||
rethrow(e);
|
||||
}
|
||||
}
|
||||
|
||||
const std::map<std::string, ActionMap>& groups() const { return groups_; }
|
||||
|
||||
private:
|
||||
using PF = std::shared_ptr<dt::DataPlatform>;
|
||||
|
||||
PF platform() const {
|
||||
return std::make_shared<dt::DataPlatform>(client_, google::CloudContext::from_env());
|
||||
}
|
||||
void add(const std::string& group, const std::string& action, bool confirm,
|
||||
std::function<Json(const Json&, dt::DataPlatform&)> fn) {
|
||||
groups_[group][action] = Action{confirm, [this, fn](const Json& b) {
|
||||
PF p = platform();
|
||||
return fn(b, *p);
|
||||
}};
|
||||
}
|
||||
|
||||
void build_bigquery() {
|
||||
const char* g = "bigquery";
|
||||
auto request = [](const Json& b) {
|
||||
dt::QueryRequest q;
|
||||
q.sql = str(b, "sql", true);
|
||||
q.use_legacy_sql = flag(b, "legacy");
|
||||
q.max_results = num(b, "max_results", 100);
|
||||
q.timeout_ms = num(b, "timeout_ms", 10000);
|
||||
q.location = str(b, "location");
|
||||
q.default_dataset = str(b, "default_dataset");
|
||||
q.maximum_bytes_billed = str(b, "maximum_bytes_billed");
|
||||
q.query_parameters = member(b, "parameters", false);
|
||||
q.parameter_mode = str(b, "parameter_mode");
|
||||
return q;
|
||||
};
|
||||
add(g, "dry_run", false, [request](const Json& b, dt::DataPlatform& p) {
|
||||
dt::QueryRequest q = request(b);
|
||||
q.dry_run = true;
|
||||
return p.bigquery.query(q);
|
||||
});
|
||||
add(g, "query", true, [request](const Json& b, dt::DataPlatform& p) {
|
||||
dt::QueryRequest q = request(b);
|
||||
q.maximum_bytes_billed = q.maximum_bytes_billed.empty() ? "1000000000" : q.maximum_bytes_billed;
|
||||
return p.bigquery.query(q); // billed bytes capped at 1 GB unless the body says otherwise
|
||||
});
|
||||
add(g, "results", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigquery.query_results(str(b, "job_id", true), str(b, "location"),
|
||||
num(b, "max_results", 100), str(b, "page_token"));
|
||||
});
|
||||
add(g, "datasets", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigquery.list_datasets(num(b, "max_results", 50));
|
||||
});
|
||||
add(g, "dataset", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigquery.get_dataset(str(b, "dataset", true));
|
||||
});
|
||||
add(g, "tables", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigquery.list_tables(str(b, "dataset", true), num(b, "max_results", 50));
|
||||
});
|
||||
add(g, "table", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigquery.get_table(str(b, "dataset", true), str(b, "table", true));
|
||||
});
|
||||
add(g, "rows", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigquery.list_rows(str(b, "dataset", true), str(b, "table", true),
|
||||
num(b, "max_results", 20), str(b, "page_token"));
|
||||
});
|
||||
add(g, "jobs", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigquery.list_jobs(num(b, "max_results", 20), str(b, "state"));
|
||||
});
|
||||
add(g, "job", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigquery.get_job(str(b, "job_id", true), str(b, "location"));
|
||||
});
|
||||
add(g, "models", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigquery.list_models(str(b, "dataset", true), num(b, "max_results", 50));
|
||||
});
|
||||
add(g, "routines", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigquery.list_routines(str(b, "dataset", true), num(b, "max_results", 50));
|
||||
});
|
||||
add(g, "insert", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigquery.insert_all(str(b, "dataset", true), str(b, "table", true),
|
||||
member(b, "rows", false, true), flag(b, "skip_invalid_rows"),
|
||||
flag(b, "ignore_unknown_values"));
|
||||
});
|
||||
add(g, "create_dataset", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigquery.create_dataset(str(b, "dataset", true), str(b, "location"),
|
||||
str(b, "description"));
|
||||
});
|
||||
add(g, "delete_table", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigquery.delete_table(str(b, "dataset", true), str(b, "table", true));
|
||||
});
|
||||
add(g, "delete_dataset", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigquery.delete_dataset(str(b, "dataset", true), flag(b, "delete_contents"));
|
||||
});
|
||||
}
|
||||
|
||||
static std::string bytes_of(const Json& b) {
|
||||
std::string text = str(b, "text");
|
||||
std::string b64 = str(b, "base64");
|
||||
if (!text.empty() == !b64.empty()) bad("give exactly one of text and base64");
|
||||
std::string data = text;
|
||||
if (!b64.empty() && !security::crypto::Base64::decode(b64, data)) bad("base64 is not valid");
|
||||
if (data.size() > kMaxDemoUpload) bad("data over the demo cap of 2 MiB");
|
||||
return data;
|
||||
}
|
||||
|
||||
void build_storage() {
|
||||
const char* g = "storage";
|
||||
add(g, "buckets", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.storage.list_buckets(num(b, "max_results", 50), str(b, "prefix"), str(b, "page_token"));
|
||||
});
|
||||
add(g, "bucket", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.storage.get_bucket(str(b, "bucket", true));
|
||||
});
|
||||
add(g, "objects", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.storage.list_objects(str(b, "bucket", true), str(b, "prefix"), str(b, "delimiter"),
|
||||
num(b, "max_results", 50), str(b, "page_token"));
|
||||
});
|
||||
add(g, "object", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.storage.get_object(str(b, "bucket", true), str(b, "object", true));
|
||||
});
|
||||
add(g, "download", false, [](const Json& b, dt::DataPlatform&) {
|
||||
auto media = dt::StorageMedia::from_env();
|
||||
std::string bucket = str(b, "bucket", true), object = str(b, "object", true);
|
||||
dt::Downloaded d = media->download_range(
|
||||
bucket, object, 0, static_cast<long long>(kMaxDownloadShown) - 1);
|
||||
Json o = Json::object();
|
||||
o.set("bytes", Json(static_cast<double>(d.data.size())));
|
||||
o.set("content_type", Json(d.content_type));
|
||||
o.set("generation", Json(d.generation));
|
||||
o.set("base64", Json(security::crypto::Base64::encode(d.data)));
|
||||
o.set("first_bytes_only", Json(true));
|
||||
return o;
|
||||
});
|
||||
add(g, "upload", true, [](const Json& b, dt::DataPlatform&) {
|
||||
auto media = dt::StorageMedia::from_env();
|
||||
std::string type = str(b, "content_type");
|
||||
return media->upload_simple(str(b, "bucket", true), str(b, "object", true), bytes_of(b),
|
||||
type.empty() ? "application/octet-stream" : type);
|
||||
});
|
||||
add(g, "resumable", true, [](const Json& b, dt::DataPlatform&) {
|
||||
auto media = dt::StorageMedia::from_env();
|
||||
std::string bucket = str(b, "bucket", true), object = str(b, "object", true);
|
||||
std::string type = str(b, "content_type");
|
||||
if (type.empty()) type = "application/octet-stream";
|
||||
std::string data = bytes_of(b);
|
||||
long long total = static_cast<long long>(data.size());
|
||||
std::string session = media->resumable_start(bucket, object, type, total, Json());
|
||||
dt::ResumableProgress progress;
|
||||
long long offset = 0;
|
||||
const long long step = static_cast<long long>(dt::DataCaps::kMaxResumableChunk);
|
||||
while (offset < total || (total == 0 && offset == 0)) {
|
||||
long long n = std::min(step, total - offset);
|
||||
if (total == 0) break;
|
||||
progress = media->resumable_put(session, offset, data.substr(offset, n), total);
|
||||
offset = progress.next_offset;
|
||||
if (progress.done) break;
|
||||
}
|
||||
Json o = Json::object();
|
||||
o.set("done", Json(progress.done));
|
||||
o.set("object", progress.object);
|
||||
return o;
|
||||
});
|
||||
add(g, "compose", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
std::vector<dt::ComposeSource> sources;
|
||||
for (const std::string& s : strings(b, "sources")) sources.push_back({s, 0});
|
||||
return p.storage.compose(str(b, "bucket", true), sources, str(b, "destination", true),
|
||||
str(b, "content_type"));
|
||||
});
|
||||
add(g, "copy", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.storage.copy_object(str(b, "bucket", true), str(b, "object", true),
|
||||
str(b, "dest_bucket", true), str(b, "dest_object", true));
|
||||
});
|
||||
add(g, "delete", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.storage.delete_object(str(b, "bucket", true), str(b, "object", true));
|
||||
});
|
||||
}
|
||||
|
||||
void build_pubsub() {
|
||||
const char* g = "pubsub";
|
||||
add(g, "topics", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.list_topics(num(b, "max_results", 50)); });
|
||||
add(g, "topic", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.get_topic(str(b, "topic", true)); });
|
||||
add(g, "subscriptions", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.list_subscriptions(num(b, "max_results", 50)); });
|
||||
add(g, "subscription", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.get_subscription(str(b, "subscription", true)); });
|
||||
add(g, "snapshots", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.list_snapshots(num(b, "max_results", 50)); });
|
||||
add(g, "publish", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
std::vector<dt::PubsubMessage> messages;
|
||||
for (const std::string& m : strings(b, "messages")) {
|
||||
dt::PubsubMessage pm;
|
||||
pm.data = m;
|
||||
messages.push_back(pm);
|
||||
}
|
||||
return p.pubsub.publish(str(b, "topic", true), messages);
|
||||
});
|
||||
add(g, "pull", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.pubsub.pull(str(b, "subscription", true), num(b, "max_messages", 10));
|
||||
});
|
||||
add(g, "ack", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.pubsub.acknowledge(str(b, "subscription", true), strings(b, "ack_ids"));
|
||||
});
|
||||
add(g, "create_topic", true, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.create_topic(str(b, "topic", true)); });
|
||||
add(g, "delete_topic", true, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.delete_topic(str(b, "topic", true)); });
|
||||
}
|
||||
|
||||
void build_firestore() {
|
||||
const char* g = "firestore";
|
||||
add(g, "databases", false, [](const Json&, dt::DataPlatform& p) { return p.firestore.list_databases(); });
|
||||
add(g, "database", false, [](const Json&, dt::DataPlatform& p) { return p.firestore.get_database(); });
|
||||
add(g, "get", false, [](const Json& b, dt::DataPlatform& p) { return p.firestore.get_document(str(b, "path", true)); });
|
||||
add(g, "list", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.firestore.list_documents(str(b, "parent"), str(b, "collection", true),
|
||||
num(b, "max_results", 20), str(b, "page_token"));
|
||||
});
|
||||
add(g, "collections", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.firestore.list_collection_ids(str(b, "parent"), num(b, "max_results", 50));
|
||||
});
|
||||
add(g, "query", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.firestore.run_query(str(b, "parent"), member(b, "query", true, true));
|
||||
});
|
||||
add(g, "aggregate", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.firestore.run_aggregation_query(str(b, "parent"), member(b, "query", true, true));
|
||||
});
|
||||
add(g, "batch_get", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.firestore.batch_get(strings(b, "paths"));
|
||||
});
|
||||
add(g, "create", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.firestore.create_document(str(b, "parent"), str(b, "collection", true),
|
||||
str(b, "document_id"), member(b, "fields", true, true));
|
||||
});
|
||||
add(g, "commit", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.firestore.commit(member(b, "writes", false, true));
|
||||
});
|
||||
add(g, "delete", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.firestore.delete_document(str(b, "path", true));
|
||||
});
|
||||
}
|
||||
|
||||
void build_spanner() {
|
||||
const char* g = "spanner";
|
||||
add(g, "instances", false, [](const Json& b, dt::DataPlatform& p) { return p.spanner.list_instances(num(b, "max_results", 50)); });
|
||||
add(g, "instance", false, [](const Json& b, dt::DataPlatform& p) { return p.spanner.get_instance(str(b, "instance", true)); });
|
||||
add(g, "databases", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.spanner.list_databases(str(b, "instance", true), num(b, "max_results", 50));
|
||||
});
|
||||
add(g, "database", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.spanner.get_database(str(b, "instance", true), str(b, "database", true));
|
||||
});
|
||||
add(g, "ddl", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.spanner.get_ddl(str(b, "instance", true), str(b, "database", true));
|
||||
});
|
||||
// Creates a session, runs one read-only statement, deletes the session.
|
||||
add(g, "query", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
dt::SpannerSql s;
|
||||
s.sql = str(b, "sql", true);
|
||||
s.params = member(b, "params", true);
|
||||
s.param_types = member(b, "param_types", true);
|
||||
std::string mode = str(b, "query_mode");
|
||||
if (!mode.empty()) s.query_mode = mode;
|
||||
return p.spanner.query(str(b, "instance", true), str(b, "database", true), s);
|
||||
});
|
||||
}
|
||||
|
||||
void build_bigtable() {
|
||||
const char* g = "bigtable";
|
||||
add(g, "instances", false, [](const Json&, dt::DataPlatform& p) { return p.bigtable.list_instances(); });
|
||||
add(g, "instance", false, [](const Json& b, dt::DataPlatform& p) { return p.bigtable.get_instance(str(b, "instance", true)); });
|
||||
add(g, "clusters", false, [](const Json& b, dt::DataPlatform& p) { return p.bigtable.list_clusters(str(b, "instance", true)); });
|
||||
add(g, "cluster", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigtable.get_cluster(str(b, "instance", true), str(b, "cluster", true));
|
||||
});
|
||||
add(g, "tables", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigtable.list_tables(str(b, "instance", true), num(b, "max_results", 50));
|
||||
});
|
||||
add(g, "table", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigtable.get_table(str(b, "instance", true), str(b, "table", true));
|
||||
});
|
||||
add(g, "profiles", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigtable.list_app_profiles(str(b, "instance", true), num(b, "max_results", 50));
|
||||
});
|
||||
add(g, "iam", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.bigtable.table_iam_policy(str(b, "instance", true), str(b, "table", true));
|
||||
});
|
||||
}
|
||||
|
||||
void build_secrets() {
|
||||
const char* g = "secrets";
|
||||
add(g, "list", false, [](const Json& b, dt::DataPlatform& p) { return p.secrets.list_secrets(num(b, "max_results", 50), str(b, "filter")); });
|
||||
add(g, "get", false, [](const Json& b, dt::DataPlatform& p) { return p.secrets.get_secret(str(b, "secret", true)); });
|
||||
add(g, "versions", false, [](const Json& b, dt::DataPlatform& p) { return p.secrets.list_versions(str(b, "secret", true), num(b, "max_results", 50)); });
|
||||
add(g, "version", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.secrets.get_version(str(b, "secret", true), str(b, "version", true));
|
||||
});
|
||||
add(g, "access", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.secrets.access_version(str(b, "secret", true), str(b, "version", true));
|
||||
});
|
||||
add(g, "create", true, [](const Json& b, dt::DataPlatform& p) { return p.secrets.create_secret(str(b, "secret", true)); });
|
||||
add(g, "add_version", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.secrets.add_version(str(b, "secret", true), str(b, "payload", true));
|
||||
});
|
||||
add(g, "disable", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.secrets.disable_version(str(b, "secret", true), str(b, "version", true));
|
||||
});
|
||||
add(g, "destroy", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.secrets.destroy_version(str(b, "secret", true), str(b, "version", true));
|
||||
});
|
||||
add(g, "delete", true, [](const Json& b, dt::DataPlatform& p) { return p.secrets.delete_secret(str(b, "secret", true)); });
|
||||
}
|
||||
|
||||
void build_dataplex() {
|
||||
const char* g = "dataplex";
|
||||
add(g, "locations", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_locations(num(b, "max_results", 50)); });
|
||||
add(g, "lakes", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_lakes(str(b, "location"), num(b, "max_results", 50)); });
|
||||
add(g, "lake", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.get_lake(str(b, "location"), str(b, "lake", true)); });
|
||||
add(g, "entry_groups", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_entry_groups(str(b, "location"), num(b, "max_results", 50)); });
|
||||
add(g, "entry_group", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.get_entry_group(str(b, "location"), str(b, "group", true)); });
|
||||
add(g, "entry_types", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_entry_types(str(b, "location"), num(b, "max_results", 50)); });
|
||||
add(g, "aspect_types", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_aspect_types(str(b, "location"), num(b, "max_results", 50)); });
|
||||
add(g, "scans", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_data_scans(str(b, "location"), num(b, "max_results", 50)); });
|
||||
add(g, "scan", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.get_data_scan(str(b, "location"), str(b, "scan", true)); });
|
||||
add(g, "search", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.dataplex.search_entries(str(b, "location"), str(b, "query", true), num(b, "max_results", 20));
|
||||
});
|
||||
add(g, "lookup", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.dataplex.lookup_entry(str(b, "location"), str(b, "entry", true));
|
||||
});
|
||||
}
|
||||
|
||||
void build_dlp() {
|
||||
const char* g = "dlp";
|
||||
add(g, "info_types", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_info_types(str(b, "language")); });
|
||||
add(g, "inspect_templates", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_inspect_templates(num(b, "max_results", 50)); });
|
||||
add(g, "deidentify_templates", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_deidentify_templates(num(b, "max_results", 50)); });
|
||||
add(g, "triggers", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_job_triggers(num(b, "max_results", 50)); });
|
||||
add(g, "jobs", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_jobs(num(b, "max_results", 50)); });
|
||||
add(g, "inspect", true, [](const Json& b, dt::DataPlatform& p) { // paid per byte inspected
|
||||
dt::InspectRequest r;
|
||||
r.text = str(b, "text", true);
|
||||
r.info_types = strings(b, "info_types");
|
||||
std::string likelihood = str(b, "min_likelihood");
|
||||
if (!likelihood.empty()) r.min_likelihood = likelihood;
|
||||
r.include_quote = flag(b, "include_quote");
|
||||
r.max_findings = num(b, "max_findings", 100);
|
||||
r.location = str(b, "location");
|
||||
return p.dlp.inspect_content(r);
|
||||
});
|
||||
add(g, "deidentify", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.dlp.deidentify_content(str(b, "text", true), member(b, "config", true, true),
|
||||
str(b, "location"));
|
||||
});
|
||||
}
|
||||
|
||||
void build_kms() {
|
||||
const char* g = "kms";
|
||||
add(g, "locations", false, [](const Json& b, dt::DataPlatform& p) { return p.kms.list_locations(num(b, "max_results", 50)); });
|
||||
add(g, "rings", false, [](const Json& b, dt::DataPlatform& p) { return p.kms.list_key_rings(str(b, "location", true), num(b, "max_results", 50)); });
|
||||
add(g, "ring", false, [](const Json& b, dt::DataPlatform& p) { return p.kms.get_key_ring(str(b, "location", true), str(b, "ring", true)); });
|
||||
add(g, "keys", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.kms.list_keys(str(b, "location", true), str(b, "ring", true), num(b, "max_results", 50));
|
||||
});
|
||||
add(g, "key", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.kms.get_key(str(b, "location", true), str(b, "ring", true), str(b, "key", true));
|
||||
});
|
||||
add(g, "versions", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.kms.list_versions(str(b, "location", true), str(b, "ring", true), str(b, "key", true),
|
||||
num(b, "max_results", 50));
|
||||
});
|
||||
add(g, "public_key", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.kms.get_public_key(str(b, "location", true), str(b, "ring", true), str(b, "key", true),
|
||||
str(b, "version", true));
|
||||
});
|
||||
add(g, "encrypt", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.kms.encrypt(str(b, "location", true), str(b, "ring", true), str(b, "key", true),
|
||||
str(b, "plaintext", true), str(b, "aad"));
|
||||
});
|
||||
add(g, "decrypt", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.kms.decrypt(str(b, "location", true), str(b, "ring", true), str(b, "key", true),
|
||||
str(b, "ciphertext", true), str(b, "aad"));
|
||||
});
|
||||
}
|
||||
|
||||
void build_datastore() {
|
||||
const char* g = "datastore";
|
||||
add(g, "lookup", false, [](const Json& b, dt::DataPlatform& p) { return p.datastore.lookup(member(b, "keys", false, true)); });
|
||||
add(g, "query", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.datastore.run_query(member(b, "query", true), member(b, "gql", true));
|
||||
});
|
||||
add(g, "aggregate", false, [](const Json& b, dt::DataPlatform& p) {
|
||||
return p.datastore.run_aggregation_query(member(b, "query", true, true));
|
||||
});
|
||||
add(g, "begin", true, [](const Json& b, dt::DataPlatform& p) { return p.datastore.begin_transaction(flag(b, "read_only")); });
|
||||
add(g, "rollback", true, [](const Json& b, dt::DataPlatform& p) { return p.datastore.rollback(str(b, "transaction", true)); });
|
||||
add(g, "allocate", true, [](const Json& b, dt::DataPlatform& p) { return p.datastore.allocate_ids(member(b, "keys", false, true)); });
|
||||
add(g, "reserve", true, [](const Json& b, dt::DataPlatform& p) { return p.datastore.reserve_ids(member(b, "keys", false, true)); });
|
||||
add(g, "commit", true, [](const Json& b, dt::DataPlatform& p) {
|
||||
std::string mode = str(b, "mode");
|
||||
return p.datastore.commit(member(b, "mutations", false, true),
|
||||
mode.empty() ? "NON_TRANSACTIONAL" : mode, str(b, "transaction"));
|
||||
});
|
||||
}
|
||||
|
||||
std::shared_ptr<google::GoogleClient> client_;
|
||||
std::map<std::string, ActionMap> groups_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
void register_google_data_demo(fw::Router& router, Services& services) {
|
||||
(void)router;
|
||||
(void)services;
|
||||
if (!services.google) services.google = google::GoogleClient::from_env();
|
||||
auto client = services.google;
|
||||
fw::Environment* env = services.templates.get();
|
||||
services.templates->add_template("google_data.html", kPage);
|
||||
services.add_feature("google-data", "/google/data");
|
||||
|
||||
auto routes = std::make_shared<DataRoutes>(client);
|
||||
|
||||
router.get("/google/data", "google-data-page", [env](fw::Request req) {
|
||||
(void)req;
|
||||
fw::TemplateCtx ctx;
|
||||
ctx["title"] = std::string("Google data");
|
||||
return fw::Response(200, env->render("google_data.html", ctx), "text/html; charset=utf-8");
|
||||
});
|
||||
|
||||
router.get("/google/data/status", "google-data-status", [client, routes](fw::Request req) {
|
||||
(void)req;
|
||||
google::CloudContext ctx = google::CloudContext::from_env();
|
||||
Json o = Json::object();
|
||||
o.set("configured", Json(client->configured()));
|
||||
o.set("project_set", Json(!ctx.project.empty()));
|
||||
o.set("location", Json(ctx.location));
|
||||
o.set("calls", Json(static_cast<double>(dt::Catalog::size())));
|
||||
Json apis = Json::array();
|
||||
for (std::size_t i = 0; i < dt::Catalog::api_count(); ++i) {
|
||||
const dt::ApiInfo& info = dt::Catalog::info_at(i);
|
||||
Json a = Json::object();
|
||||
a.set("name", Json(info.name));
|
||||
a.set("version", Json(info.version));
|
||||
a.set("service", Json(info.service));
|
||||
a.set("calls", Json(static_cast<double>(dt::Catalog::count_of(info.api))));
|
||||
apis.push(a);
|
||||
}
|
||||
o.set("apis", apis);
|
||||
Json groups = Json::object();
|
||||
for (const auto& g : routes->groups()) {
|
||||
Json names = Json::array();
|
||||
for (const auto& a : g.second) {
|
||||
Json n = Json::object();
|
||||
n.set("action", Json(a.first));
|
||||
n.set("confirm", Json(a.second.confirm_required));
|
||||
names.push(n);
|
||||
}
|
||||
groups.set(g.first, names);
|
||||
}
|
||||
o.set("actions", groups);
|
||||
o.set("max_signed_url_seconds", Json(static_cast<double>(dt::DataCaps::kMaxSignedUrlSeconds)));
|
||||
return json_out(o);
|
||||
});
|
||||
|
||||
// The discovery-checked call table, optionally one API: /calls?api=bigquery
|
||||
router.get("/google/data/calls", "google-data-calls", [](fw::Request req) {
|
||||
std::string only = req.query_or("api", "");
|
||||
Json arr = Json::array();
|
||||
for (std::size_t i = 0; i < dt::Catalog::size(); ++i) {
|
||||
const dt::CallSpec& s = dt::Catalog::at(i);
|
||||
if (!only.empty() && dt::Catalog::info(dt::Catalog::api_of(s)).name != only) continue;
|
||||
Json row = Json::object();
|
||||
row.set("rpc", Json(s.rpc));
|
||||
row.set("http", Json(s.http));
|
||||
row.set("path", Json(s.path));
|
||||
arr.push(row);
|
||||
}
|
||||
Json o = Json::object();
|
||||
o.set("calls", arr);
|
||||
return json_out(o);
|
||||
});
|
||||
|
||||
static const char* const kGroups[] = {"bigquery", "storage", "pubsub", "firestore", "spanner",
|
||||
"bigtable", "secrets", "dataplex", "dlp", "kms", "datastore"};
|
||||
for (const char* g : kGroups) {
|
||||
std::string group = g;
|
||||
router.post("/google/data/" + group, "google-data-" + group,
|
||||
[routes, group](fw::Request req) { return routes->handle(group, req); });
|
||||
}
|
||||
router.post("/google/data/storage/sign", "google-data-storage-sign",
|
||||
[routes](fw::Request req) { return routes->sign(req); });
|
||||
router.post("/google/data/call", "google-data-call",
|
||||
[routes](fw::Request req) { return routes->call(req); });
|
||||
}
|
||||
|
||||
} // namespace molo::example::app
|
||||
|
||||
Reference in New Issue
Block a user