Merge branch 'worktree-agent-a2aaa8fe42eac35db'

This commit is contained in:
2026-10-25 15:24:27 +01:00
47 changed files with 6315 additions and 3 deletions
+747 -3
View File
@@ -1,13 +1,757 @@
#include "../GoogleDemos.hpp"
#include <functional>
#include <map>
#include "../../fw/Router.hpp"
#include "../../fw/Templates.hpp"
#include "google/data/data.h"
#include "google/data/caps.h"
namespace molo::example::app {
// BigQuery, Cloud Storage, Pub/Sub, Firestore, Spanner, Bigtable, Secret Manager, Dataplex, DLP, KMS. Stub: the package agent fills this in.
// Google data package: BigQuery, Cloud Storage, Pub/Sub, Firestore, Spanner,
// Bigtable admin, Secret Manager, Dataplex, DLP, Cloud KMS, Datastore over the
// real src/google/data part. GET page, /status and /calls answer without a
// credential. Every other route is a POST: 422 on bad input, 503
// google_not_configured without a credential. Reads run directly; writes,
// secret reads and paid operations are dry runs unless the body has
// "confirm":true.
namespace {
namespace dt = google::data;
using Json = clients::body::Json;
constexpr std::size_t kMaxBody = 3 * 1024 * 1024;
constexpr std::size_t kMaxShown = 256 * 1024; // response bytes returned to the page
constexpr std::size_t kMaxDemoUpload = 2 * 1024 * 1024;
constexpr std::size_t kMaxDownloadShown = 64 * 1024;
const char kPage[] = R"DATA({% extends "base.html" %}
{% block content %}
<h1>Google data</h1>
<p class="hint">BigQuery, Cloud Storage, Pub/Sub, Firestore, Spanner, Bigtable admin,
Secret Manager, Dataplex, DLP, Cloud KMS and Datastore. Reads run at once; writes,
secret reads and paid calls stay a dry run until the body says <code>"confirm":true</code>.</p>
<p id="gd-config" class="hint">Checking Google credentials...</p>
<section class="gd">
<label>Service
<select id="gd-service">
<option value="bigquery">BigQuery</option>
<option value="storage">Cloud Storage</option>
<option value="pubsub">Pub/Sub</option>
<option value="firestore">Firestore</option>
<option value="spanner">Spanner</option>
<option value="bigtable">Bigtable admin</option>
<option value="secrets">Secret Manager</option>
<option value="dataplex">Dataplex</option>
<option value="dlp">DLP</option>
<option value="kms">Cloud KMS</option>
<option value="datastore">Datastore</option>
<option value="storage/sign">Storage signed URL</option>
<option value="call">Any call by id</option>
</select></label>
<label>Action <select id="gd-action"></select></label>
<label>Body (JSON object) <textarea id="gd-body" rows="7"></textarea></label>
<div class="gd-actions"><button id="gd-send" type="button" class="primary">Send</button></div>
<p id="gd-error" class="error" role="alert"></p>
<pre id="gd-out" class="gd-out"></pre>
</section>
<style>
.gd label{display:block;margin:.5rem 0}
.gd select,.gd textarea{width:100%;box-sizing:border-box}
.gd-actions{display:flex;flex-wrap:wrap;gap:.5rem;margin:.5rem 0}
.gd-out{overflow:auto;max-height:26rem;white-space:pre-wrap;word-break:break-word}
</style>
<script src="/embedded/google_data.js"></script>
{% endblock %}
)DATA";
fw::Response json_out(const Json& j, int status = 200) {
fw::Response res = fw::JsonResponse(j.dump(), status);
res.headers.set("Cache-Control", "no-store");
return res;
}
[[noreturn]] void bad(const std::string& why) {
throw fw::HttpError(422, "validation_error", why);
}
[[noreturn]] void rethrow(const google::GoogleError& e) {
if (e.http_status() == 503 && e.status() == "NOT_CONFIGURED")
throw fw::HttpError(503, "google_not_configured", e.message());
if (e.service().empty() && e.status() == "INVALID_ARGUMENT")
throw fw::HttpError(422, "validation_error", e.message());
throw fw::HttpError(502, "google_error", e.status() + ": " + e.message());
}
Json parse_body(const fw::Request& req) {
if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large");
if (req.body.empty()) return Json::object();
try {
Json j = Json::parse(req.body);
if (!j.is_object()) bad("body must be a JSON object");
return j;
} catch (const clients::body::JsonError&) {
bad("body is not valid JSON");
}
}
std::string str(const Json& b, const char* key, bool required = false) {
const Json* v = b.find(key);
if (!v || v->is_null()) {
if (required) bad(std::string(key) + " is required");
return "";
}
if (!v->is_string()) bad(std::string(key) + " must be a string");
return v->as_string();
}
int num(const Json& b, const char* key, int fallback) {
const Json* v = b.find(key);
if (!v || v->is_null()) return fallback;
if (!v->is_number()) bad(std::string(key) + " must be a number");
return static_cast<int>(v->as_int());
}
bool flag(const Json& b, const char* key) {
const Json* v = b.find(key);
if (!v || v->is_null()) return false;
if (!v->is_bool()) bad(std::string(key) + " must be true or false");
return v->as_bool();
}
Json member(const Json& b, const char* key, bool object, bool required = false) {
const Json* v = b.find(key);
if (!v || v->is_null()) {
if (required) bad(std::string(key) + " is required");
return Json();
}
if (object ? !v->is_object() : !v->is_array())
bad(std::string(key) + (object ? " must be a JSON object" : " must be a JSON array"));
return *v;
}
std::vector<std::string> strings(const Json& b, const char* key) {
std::vector<std::string> out;
Json a = member(b, key, false);
if (a.is_null()) return out;
for (const Json& x : a.as_array()) {
if (!x.is_string()) bad(std::string(key) + " must hold strings");
out.push_back(x.as_string());
}
return out;
}
// A response that fits the page; bigger ones are cut and say so.
Json shown(const Json& j) {
std::string text = j.dump();
if (text.size() <= kMaxShown) return j;
Json o = Json::object();
o.set("truncated", Json(true));
o.set("bytes", Json(static_cast<double>(text.size())));
o.set("head", Json(text.substr(0, kMaxShown / 4)));
return o;
}
Json dry(const std::string& group, const std::string& action) {
Json o = Json::object();
o.set("dry_run", Json(true));
o.set("service", Json(group));
o.set("action", Json(action));
o.set("hint", Json("send confirm:true to run this action"));
return o;
}
// One action of a service: runs against Google, or is a dry run until confirmed.
struct Action {
bool confirm_required;
std::function<Json(const Json&)> run;
};
using ActionMap = std::map<std::string, Action>;
class DataRoutes {
public:
explicit DataRoutes(std::shared_ptr<google::GoogleClient> client) : client_(std::move(client)) {
build_bigquery();
build_storage();
build_pubsub();
build_firestore();
build_spanner();
build_bigtable();
build_secrets();
build_dataplex();
build_dlp();
build_kms();
build_datastore();
}
fw::Response handle(const std::string& group, const fw::Request& req) const {
Json body = parse_body(req);
std::string action = str(body, "action", true);
const ActionMap& actions = groups_.at(group);
auto it = actions.find(action);
if (it == actions.end()) {
std::string known;
for (const auto& kv : actions) known += (known.empty() ? "" : ", ") + kv.first;
bad("unknown action; use one of: " + known);
}
if (it->second.confirm_required && !flag(body, "confirm")) return json_out(dry(group, action));
try {
return json_out(shown(it->second.run(body)));
} catch (const google::GoogleError& e) {
rethrow(e);
}
}
fw::Response sign(const fw::Request& req) const {
Json body = parse_body(req);
dt::SignedUrlRequest r;
r.method = str(body, "method");
if (r.method.empty()) r.method = "GET";
r.bucket = str(body, "bucket", true);
r.object = str(body, "object");
r.expires_seconds = num(body, "expires_seconds", 900);
Json headers = member(body, "headers", true);
if (!headers.is_null())
for (const auto& kv : headers.as_object()) {
if (!kv.second.is_string()) bad("header values must be strings");
r.headers[kv.first] = kv.second.as_string();
}
try {
r.validate();
// A signed URL is a bearer capability: dry run unless confirmed.
if (!flag(body, "confirm")) {
Json o = dry("storage/sign", "sign");
o.set("expires_seconds", Json(static_cast<double>(r.expires_seconds)));
o.set("max_expires_seconds", Json(static_cast<double>(dt::DataCaps::kMaxSignedUrlSeconds)));
return json_out(o);
}
dt::SignedUrl signer = dt::SignedUrl::from_env();
Json o = Json::object();
o.set("url", Json(signer.sign(r)));
o.set("method", Json(r.method));
o.set("expires_seconds", Json(static_cast<double>(r.expires_seconds)));
o.set("signer", Json(signer.email()));
return json_out(o);
} catch (const google::GoogleError& e) {
rethrow(e);
}
}
// Any row of the call table by id. Non-GET rows are dry runs unless confirmed.
fw::Response call(const fw::Request& req) const {
Json body = parse_body(req);
std::string rpc = str(body, "rpc", true);
try {
const dt::CallSpec& spec = dt::Catalog::get(rpc);
dt::CallArgs args;
Json path = member(body, "path", true);
if (!path.is_null())
for (const auto& kv : path.as_object()) {
if (!kv.second.is_string()) bad("path values must be strings");
args.p(kv.first, kv.second.as_string());
}
Json query = member(body, "query", true);
if (!query.is_null()) args.query = query;
Json b = member(body, "body", true);
if (!b.is_null()) args.body = b;
dt::ApiService svc(client_, dt::Catalog::api_of(spec));
dt::Prepared prepared = svc.prepare(rpc, args); // validates: 422
bool read = std::string(spec.http) == "GET";
if (!read && !flag(body, "confirm")) {
Json o = dry("call", rpc);
o.set("method", Json(spec.http));
o.set("url", Json(prepared.url()));
return json_out(o);
}
return json_out(shown(svc.invoke(rpc, args)));
} catch (const google::GoogleError& e) {
rethrow(e);
}
}
const std::map<std::string, ActionMap>& groups() const { return groups_; }
private:
using PF = std::shared_ptr<dt::DataPlatform>;
PF platform() const {
return std::make_shared<dt::DataPlatform>(client_, google::CloudContext::from_env());
}
void add(const std::string& group, const std::string& action, bool confirm,
std::function<Json(const Json&, dt::DataPlatform&)> fn) {
groups_[group][action] = Action{confirm, [this, fn](const Json& b) {
PF p = platform();
return fn(b, *p);
}};
}
void build_bigquery() {
const char* g = "bigquery";
auto request = [](const Json& b) {
dt::QueryRequest q;
q.sql = str(b, "sql", true);
q.use_legacy_sql = flag(b, "legacy");
q.max_results = num(b, "max_results", 100);
q.timeout_ms = num(b, "timeout_ms", 10000);
q.location = str(b, "location");
q.default_dataset = str(b, "default_dataset");
q.maximum_bytes_billed = str(b, "maximum_bytes_billed");
q.query_parameters = member(b, "parameters", false);
q.parameter_mode = str(b, "parameter_mode");
return q;
};
add(g, "dry_run", false, [request](const Json& b, dt::DataPlatform& p) {
dt::QueryRequest q = request(b);
q.dry_run = true;
return p.bigquery.query(q);
});
add(g, "query", true, [request](const Json& b, dt::DataPlatform& p) {
dt::QueryRequest q = request(b);
q.maximum_bytes_billed = q.maximum_bytes_billed.empty() ? "1000000000" : q.maximum_bytes_billed;
return p.bigquery.query(q); // billed bytes capped at 1 GB unless the body says otherwise
});
add(g, "results", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigquery.query_results(str(b, "job_id", true), str(b, "location"),
num(b, "max_results", 100), str(b, "page_token"));
});
add(g, "datasets", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigquery.list_datasets(num(b, "max_results", 50));
});
add(g, "dataset", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigquery.get_dataset(str(b, "dataset", true));
});
add(g, "tables", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigquery.list_tables(str(b, "dataset", true), num(b, "max_results", 50));
});
add(g, "table", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigquery.get_table(str(b, "dataset", true), str(b, "table", true));
});
add(g, "rows", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigquery.list_rows(str(b, "dataset", true), str(b, "table", true),
num(b, "max_results", 20), str(b, "page_token"));
});
add(g, "jobs", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigquery.list_jobs(num(b, "max_results", 20), str(b, "state"));
});
add(g, "job", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigquery.get_job(str(b, "job_id", true), str(b, "location"));
});
add(g, "models", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigquery.list_models(str(b, "dataset", true), num(b, "max_results", 50));
});
add(g, "routines", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigquery.list_routines(str(b, "dataset", true), num(b, "max_results", 50));
});
add(g, "insert", true, [](const Json& b, dt::DataPlatform& p) {
return p.bigquery.insert_all(str(b, "dataset", true), str(b, "table", true),
member(b, "rows", false, true), flag(b, "skip_invalid_rows"),
flag(b, "ignore_unknown_values"));
});
add(g, "create_dataset", true, [](const Json& b, dt::DataPlatform& p) {
return p.bigquery.create_dataset(str(b, "dataset", true), str(b, "location"),
str(b, "description"));
});
add(g, "delete_table", true, [](const Json& b, dt::DataPlatform& p) {
return p.bigquery.delete_table(str(b, "dataset", true), str(b, "table", true));
});
add(g, "delete_dataset", true, [](const Json& b, dt::DataPlatform& p) {
return p.bigquery.delete_dataset(str(b, "dataset", true), flag(b, "delete_contents"));
});
}
static std::string bytes_of(const Json& b) {
std::string text = str(b, "text");
std::string b64 = str(b, "base64");
if (!text.empty() == !b64.empty()) bad("give exactly one of text and base64");
std::string data = text;
if (!b64.empty() && !security::crypto::Base64::decode(b64, data)) bad("base64 is not valid");
if (data.size() > kMaxDemoUpload) bad("data over the demo cap of 2 MiB");
return data;
}
void build_storage() {
const char* g = "storage";
add(g, "buckets", false, [](const Json& b, dt::DataPlatform& p) {
return p.storage.list_buckets(num(b, "max_results", 50), str(b, "prefix"), str(b, "page_token"));
});
add(g, "bucket", false, [](const Json& b, dt::DataPlatform& p) {
return p.storage.get_bucket(str(b, "bucket", true));
});
add(g, "objects", false, [](const Json& b, dt::DataPlatform& p) {
return p.storage.list_objects(str(b, "bucket", true), str(b, "prefix"), str(b, "delimiter"),
num(b, "max_results", 50), str(b, "page_token"));
});
add(g, "object", false, [](const Json& b, dt::DataPlatform& p) {
return p.storage.get_object(str(b, "bucket", true), str(b, "object", true));
});
add(g, "download", false, [](const Json& b, dt::DataPlatform&) {
auto media = dt::StorageMedia::from_env();
std::string bucket = str(b, "bucket", true), object = str(b, "object", true);
dt::Downloaded d = media->download_range(
bucket, object, 0, static_cast<long long>(kMaxDownloadShown) - 1);
Json o = Json::object();
o.set("bytes", Json(static_cast<double>(d.data.size())));
o.set("content_type", Json(d.content_type));
o.set("generation", Json(d.generation));
o.set("base64", Json(security::crypto::Base64::encode(d.data)));
o.set("first_bytes_only", Json(true));
return o;
});
add(g, "upload", true, [](const Json& b, dt::DataPlatform&) {
auto media = dt::StorageMedia::from_env();
std::string type = str(b, "content_type");
return media->upload_simple(str(b, "bucket", true), str(b, "object", true), bytes_of(b),
type.empty() ? "application/octet-stream" : type);
});
add(g, "resumable", true, [](const Json& b, dt::DataPlatform&) {
auto media = dt::StorageMedia::from_env();
std::string bucket = str(b, "bucket", true), object = str(b, "object", true);
std::string type = str(b, "content_type");
if (type.empty()) type = "application/octet-stream";
std::string data = bytes_of(b);
long long total = static_cast<long long>(data.size());
std::string session = media->resumable_start(bucket, object, type, total, Json());
dt::ResumableProgress progress;
long long offset = 0;
const long long step = static_cast<long long>(dt::DataCaps::kMaxResumableChunk);
while (offset < total || (total == 0 && offset == 0)) {
long long n = std::min(step, total - offset);
if (total == 0) break;
progress = media->resumable_put(session, offset, data.substr(offset, n), total);
offset = progress.next_offset;
if (progress.done) break;
}
Json o = Json::object();
o.set("done", Json(progress.done));
o.set("object", progress.object);
return o;
});
add(g, "compose", true, [](const Json& b, dt::DataPlatform& p) {
std::vector<dt::ComposeSource> sources;
for (const std::string& s : strings(b, "sources")) sources.push_back({s, 0});
return p.storage.compose(str(b, "bucket", true), sources, str(b, "destination", true),
str(b, "content_type"));
});
add(g, "copy", true, [](const Json& b, dt::DataPlatform& p) {
return p.storage.copy_object(str(b, "bucket", true), str(b, "object", true),
str(b, "dest_bucket", true), str(b, "dest_object", true));
});
add(g, "delete", true, [](const Json& b, dt::DataPlatform& p) {
return p.storage.delete_object(str(b, "bucket", true), str(b, "object", true));
});
}
void build_pubsub() {
const char* g = "pubsub";
add(g, "topics", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.list_topics(num(b, "max_results", 50)); });
add(g, "topic", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.get_topic(str(b, "topic", true)); });
add(g, "subscriptions", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.list_subscriptions(num(b, "max_results", 50)); });
add(g, "subscription", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.get_subscription(str(b, "subscription", true)); });
add(g, "snapshots", false, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.list_snapshots(num(b, "max_results", 50)); });
add(g, "publish", true, [](const Json& b, dt::DataPlatform& p) {
std::vector<dt::PubsubMessage> messages;
for (const std::string& m : strings(b, "messages")) {
dt::PubsubMessage pm;
pm.data = m;
messages.push_back(pm);
}
return p.pubsub.publish(str(b, "topic", true), messages);
});
add(g, "pull", true, [](const Json& b, dt::DataPlatform& p) {
return p.pubsub.pull(str(b, "subscription", true), num(b, "max_messages", 10));
});
add(g, "ack", true, [](const Json& b, dt::DataPlatform& p) {
return p.pubsub.acknowledge(str(b, "subscription", true), strings(b, "ack_ids"));
});
add(g, "create_topic", true, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.create_topic(str(b, "topic", true)); });
add(g, "delete_topic", true, [](const Json& b, dt::DataPlatform& p) { return p.pubsub.delete_topic(str(b, "topic", true)); });
}
void build_firestore() {
const char* g = "firestore";
add(g, "databases", false, [](const Json&, dt::DataPlatform& p) { return p.firestore.list_databases(); });
add(g, "database", false, [](const Json&, dt::DataPlatform& p) { return p.firestore.get_database(); });
add(g, "get", false, [](const Json& b, dt::DataPlatform& p) { return p.firestore.get_document(str(b, "path", true)); });
add(g, "list", false, [](const Json& b, dt::DataPlatform& p) {
return p.firestore.list_documents(str(b, "parent"), str(b, "collection", true),
num(b, "max_results", 20), str(b, "page_token"));
});
add(g, "collections", false, [](const Json& b, dt::DataPlatform& p) {
return p.firestore.list_collection_ids(str(b, "parent"), num(b, "max_results", 50));
});
add(g, "query", false, [](const Json& b, dt::DataPlatform& p) {
return p.firestore.run_query(str(b, "parent"), member(b, "query", true, true));
});
add(g, "aggregate", false, [](const Json& b, dt::DataPlatform& p) {
return p.firestore.run_aggregation_query(str(b, "parent"), member(b, "query", true, true));
});
add(g, "batch_get", false, [](const Json& b, dt::DataPlatform& p) {
return p.firestore.batch_get(strings(b, "paths"));
});
add(g, "create", true, [](const Json& b, dt::DataPlatform& p) {
return p.firestore.create_document(str(b, "parent"), str(b, "collection", true),
str(b, "document_id"), member(b, "fields", true, true));
});
add(g, "commit", true, [](const Json& b, dt::DataPlatform& p) {
return p.firestore.commit(member(b, "writes", false, true));
});
add(g, "delete", true, [](const Json& b, dt::DataPlatform& p) {
return p.firestore.delete_document(str(b, "path", true));
});
}
void build_spanner() {
const char* g = "spanner";
add(g, "instances", false, [](const Json& b, dt::DataPlatform& p) { return p.spanner.list_instances(num(b, "max_results", 50)); });
add(g, "instance", false, [](const Json& b, dt::DataPlatform& p) { return p.spanner.get_instance(str(b, "instance", true)); });
add(g, "databases", false, [](const Json& b, dt::DataPlatform& p) {
return p.spanner.list_databases(str(b, "instance", true), num(b, "max_results", 50));
});
add(g, "database", false, [](const Json& b, dt::DataPlatform& p) {
return p.spanner.get_database(str(b, "instance", true), str(b, "database", true));
});
add(g, "ddl", false, [](const Json& b, dt::DataPlatform& p) {
return p.spanner.get_ddl(str(b, "instance", true), str(b, "database", true));
});
// Creates a session, runs one read-only statement, deletes the session.
add(g, "query", true, [](const Json& b, dt::DataPlatform& p) {
dt::SpannerSql s;
s.sql = str(b, "sql", true);
s.params = member(b, "params", true);
s.param_types = member(b, "param_types", true);
std::string mode = str(b, "query_mode");
if (!mode.empty()) s.query_mode = mode;
return p.spanner.query(str(b, "instance", true), str(b, "database", true), s);
});
}
void build_bigtable() {
const char* g = "bigtable";
add(g, "instances", false, [](const Json&, dt::DataPlatform& p) { return p.bigtable.list_instances(); });
add(g, "instance", false, [](const Json& b, dt::DataPlatform& p) { return p.bigtable.get_instance(str(b, "instance", true)); });
add(g, "clusters", false, [](const Json& b, dt::DataPlatform& p) { return p.bigtable.list_clusters(str(b, "instance", true)); });
add(g, "cluster", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigtable.get_cluster(str(b, "instance", true), str(b, "cluster", true));
});
add(g, "tables", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigtable.list_tables(str(b, "instance", true), num(b, "max_results", 50));
});
add(g, "table", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigtable.get_table(str(b, "instance", true), str(b, "table", true));
});
add(g, "profiles", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigtable.list_app_profiles(str(b, "instance", true), num(b, "max_results", 50));
});
add(g, "iam", false, [](const Json& b, dt::DataPlatform& p) {
return p.bigtable.table_iam_policy(str(b, "instance", true), str(b, "table", true));
});
}
void build_secrets() {
const char* g = "secrets";
add(g, "list", false, [](const Json& b, dt::DataPlatform& p) { return p.secrets.list_secrets(num(b, "max_results", 50), str(b, "filter")); });
add(g, "get", false, [](const Json& b, dt::DataPlatform& p) { return p.secrets.get_secret(str(b, "secret", true)); });
add(g, "versions", false, [](const Json& b, dt::DataPlatform& p) { return p.secrets.list_versions(str(b, "secret", true), num(b, "max_results", 50)); });
add(g, "version", false, [](const Json& b, dt::DataPlatform& p) {
return p.secrets.get_version(str(b, "secret", true), str(b, "version", true));
});
add(g, "access", true, [](const Json& b, dt::DataPlatform& p) {
return p.secrets.access_version(str(b, "secret", true), str(b, "version", true));
});
add(g, "create", true, [](const Json& b, dt::DataPlatform& p) { return p.secrets.create_secret(str(b, "secret", true)); });
add(g, "add_version", true, [](const Json& b, dt::DataPlatform& p) {
return p.secrets.add_version(str(b, "secret", true), str(b, "payload", true));
});
add(g, "disable", true, [](const Json& b, dt::DataPlatform& p) {
return p.secrets.disable_version(str(b, "secret", true), str(b, "version", true));
});
add(g, "destroy", true, [](const Json& b, dt::DataPlatform& p) {
return p.secrets.destroy_version(str(b, "secret", true), str(b, "version", true));
});
add(g, "delete", true, [](const Json& b, dt::DataPlatform& p) { return p.secrets.delete_secret(str(b, "secret", true)); });
}
void build_dataplex() {
const char* g = "dataplex";
add(g, "locations", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_locations(num(b, "max_results", 50)); });
add(g, "lakes", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_lakes(str(b, "location"), num(b, "max_results", 50)); });
add(g, "lake", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.get_lake(str(b, "location"), str(b, "lake", true)); });
add(g, "entry_groups", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_entry_groups(str(b, "location"), num(b, "max_results", 50)); });
add(g, "entry_group", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.get_entry_group(str(b, "location"), str(b, "group", true)); });
add(g, "entry_types", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_entry_types(str(b, "location"), num(b, "max_results", 50)); });
add(g, "aspect_types", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_aspect_types(str(b, "location"), num(b, "max_results", 50)); });
add(g, "scans", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.list_data_scans(str(b, "location"), num(b, "max_results", 50)); });
add(g, "scan", false, [](const Json& b, dt::DataPlatform& p) { return p.dataplex.get_data_scan(str(b, "location"), str(b, "scan", true)); });
add(g, "search", false, [](const Json& b, dt::DataPlatform& p) {
return p.dataplex.search_entries(str(b, "location"), str(b, "query", true), num(b, "max_results", 20));
});
add(g, "lookup", false, [](const Json& b, dt::DataPlatform& p) {
return p.dataplex.lookup_entry(str(b, "location"), str(b, "entry", true));
});
}
void build_dlp() {
const char* g = "dlp";
add(g, "info_types", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_info_types(str(b, "language")); });
add(g, "inspect_templates", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_inspect_templates(num(b, "max_results", 50)); });
add(g, "deidentify_templates", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_deidentify_templates(num(b, "max_results", 50)); });
add(g, "triggers", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_job_triggers(num(b, "max_results", 50)); });
add(g, "jobs", false, [](const Json& b, dt::DataPlatform& p) { return p.dlp.list_jobs(num(b, "max_results", 50)); });
add(g, "inspect", true, [](const Json& b, dt::DataPlatform& p) { // paid per byte inspected
dt::InspectRequest r;
r.text = str(b, "text", true);
r.info_types = strings(b, "info_types");
std::string likelihood = str(b, "min_likelihood");
if (!likelihood.empty()) r.min_likelihood = likelihood;
r.include_quote = flag(b, "include_quote");
r.max_findings = num(b, "max_findings", 100);
r.location = str(b, "location");
return p.dlp.inspect_content(r);
});
add(g, "deidentify", true, [](const Json& b, dt::DataPlatform& p) {
return p.dlp.deidentify_content(str(b, "text", true), member(b, "config", true, true),
str(b, "location"));
});
}
void build_kms() {
const char* g = "kms";
add(g, "locations", false, [](const Json& b, dt::DataPlatform& p) { return p.kms.list_locations(num(b, "max_results", 50)); });
add(g, "rings", false, [](const Json& b, dt::DataPlatform& p) { return p.kms.list_key_rings(str(b, "location", true), num(b, "max_results", 50)); });
add(g, "ring", false, [](const Json& b, dt::DataPlatform& p) { return p.kms.get_key_ring(str(b, "location", true), str(b, "ring", true)); });
add(g, "keys", false, [](const Json& b, dt::DataPlatform& p) {
return p.kms.list_keys(str(b, "location", true), str(b, "ring", true), num(b, "max_results", 50));
});
add(g, "key", false, [](const Json& b, dt::DataPlatform& p) {
return p.kms.get_key(str(b, "location", true), str(b, "ring", true), str(b, "key", true));
});
add(g, "versions", false, [](const Json& b, dt::DataPlatform& p) {
return p.kms.list_versions(str(b, "location", true), str(b, "ring", true), str(b, "key", true),
num(b, "max_results", 50));
});
add(g, "public_key", false, [](const Json& b, dt::DataPlatform& p) {
return p.kms.get_public_key(str(b, "location", true), str(b, "ring", true), str(b, "key", true),
str(b, "version", true));
});
add(g, "encrypt", true, [](const Json& b, dt::DataPlatform& p) {
return p.kms.encrypt(str(b, "location", true), str(b, "ring", true), str(b, "key", true),
str(b, "plaintext", true), str(b, "aad"));
});
add(g, "decrypt", true, [](const Json& b, dt::DataPlatform& p) {
return p.kms.decrypt(str(b, "location", true), str(b, "ring", true), str(b, "key", true),
str(b, "ciphertext", true), str(b, "aad"));
});
}
void build_datastore() {
const char* g = "datastore";
add(g, "lookup", false, [](const Json& b, dt::DataPlatform& p) { return p.datastore.lookup(member(b, "keys", false, true)); });
add(g, "query", false, [](const Json& b, dt::DataPlatform& p) {
return p.datastore.run_query(member(b, "query", true), member(b, "gql", true));
});
add(g, "aggregate", false, [](const Json& b, dt::DataPlatform& p) {
return p.datastore.run_aggregation_query(member(b, "query", true, true));
});
add(g, "begin", true, [](const Json& b, dt::DataPlatform& p) { return p.datastore.begin_transaction(flag(b, "read_only")); });
add(g, "rollback", true, [](const Json& b, dt::DataPlatform& p) { return p.datastore.rollback(str(b, "transaction", true)); });
add(g, "allocate", true, [](const Json& b, dt::DataPlatform& p) { return p.datastore.allocate_ids(member(b, "keys", false, true)); });
add(g, "reserve", true, [](const Json& b, dt::DataPlatform& p) { return p.datastore.reserve_ids(member(b, "keys", false, true)); });
add(g, "commit", true, [](const Json& b, dt::DataPlatform& p) {
std::string mode = str(b, "mode");
return p.datastore.commit(member(b, "mutations", false, true),
mode.empty() ? "NON_TRANSACTIONAL" : mode, str(b, "transaction"));
});
}
std::shared_ptr<google::GoogleClient> client_;
std::map<std::string, ActionMap> groups_;
};
} // namespace
void register_google_data_demo(fw::Router& router, Services& services) {
(void)router;
(void)services;
if (!services.google) services.google = google::GoogleClient::from_env();
auto client = services.google;
fw::Environment* env = services.templates.get();
services.templates->add_template("google_data.html", kPage);
services.add_feature("google-data", "/google/data");
auto routes = std::make_shared<DataRoutes>(client);
router.get("/google/data", "google-data-page", [env](fw::Request req) {
(void)req;
fw::TemplateCtx ctx;
ctx["title"] = std::string("Google data");
return fw::Response(200, env->render("google_data.html", ctx), "text/html; charset=utf-8");
});
router.get("/google/data/status", "google-data-status", [client, routes](fw::Request req) {
(void)req;
google::CloudContext ctx = google::CloudContext::from_env();
Json o = Json::object();
o.set("configured", Json(client->configured()));
o.set("project_set", Json(!ctx.project.empty()));
o.set("location", Json(ctx.location));
o.set("calls", Json(static_cast<double>(dt::Catalog::size())));
Json apis = Json::array();
for (std::size_t i = 0; i < dt::Catalog::api_count(); ++i) {
const dt::ApiInfo& info = dt::Catalog::info_at(i);
Json a = Json::object();
a.set("name", Json(info.name));
a.set("version", Json(info.version));
a.set("service", Json(info.service));
a.set("calls", Json(static_cast<double>(dt::Catalog::count_of(info.api))));
apis.push(a);
}
o.set("apis", apis);
Json groups = Json::object();
for (const auto& g : routes->groups()) {
Json names = Json::array();
for (const auto& a : g.second) {
Json n = Json::object();
n.set("action", Json(a.first));
n.set("confirm", Json(a.second.confirm_required));
names.push(n);
}
groups.set(g.first, names);
}
o.set("actions", groups);
o.set("max_signed_url_seconds", Json(static_cast<double>(dt::DataCaps::kMaxSignedUrlSeconds)));
return json_out(o);
});
// The discovery-checked call table, optionally one API: /calls?api=bigquery
router.get("/google/data/calls", "google-data-calls", [](fw::Request req) {
std::string only = req.query_or("api", "");
Json arr = Json::array();
for (std::size_t i = 0; i < dt::Catalog::size(); ++i) {
const dt::CallSpec& s = dt::Catalog::at(i);
if (!only.empty() && dt::Catalog::info(dt::Catalog::api_of(s)).name != only) continue;
Json row = Json::object();
row.set("rpc", Json(s.rpc));
row.set("http", Json(s.http));
row.set("path", Json(s.path));
arr.push(row);
}
Json o = Json::object();
o.set("calls", arr);
return json_out(o);
});
static const char* const kGroups[] = {"bigquery", "storage", "pubsub", "firestore", "spanner",
"bigtable", "secrets", "dataplex", "dlp", "kms", "datastore"};
for (const char* g : kGroups) {
std::string group = g;
router.post("/google/data/" + group, "google-data-" + group,
[routes, group](fw::Request req) { return routes->handle(group, req); });
}
router.post("/google/data/storage/sign", "google-data-storage-sign",
[routes](fw::Request req) { return routes->sign(req); });
router.post("/google/data/call", "google-data-call",
[routes](fw::Request req) { return routes->call(req); });
}
} // namespace molo::example::app
+97
View File
@@ -0,0 +1,97 @@
// Google data page: thin client over /google/data/* (all limits server side).
(function () {
"use strict";
var $ = function (id) { return document.getElementById(id); };
var out = $("gd-out"), err = $("gd-error");
var actions = {};
// One example body per action; the server validates everything.
var examples = {
"bigquery": {
dry_run: { sql: "SELECT 1 AS one" },
query: { sql: "SELECT 1 AS one", confirm: false },
datasets: {}, jobs: {},
tables: { dataset: "DATASET" }, table: { dataset: "DATASET", table: "TABLE" },
rows: { dataset: "DATASET", table: "TABLE", max_results: 10 },
insert: { dataset: "DATASET", table: "TABLE", rows: [{ json: { a: 1 } }], confirm: false }
},
"storage": {
buckets: {}, objects: { bucket: "BUCKET", prefix: "" },
object: { bucket: "BUCKET", object: "path/file.txt" },
download: { bucket: "BUCKET", object: "path/file.txt" },
upload: { bucket: "BUCKET", object: "demo.txt", text: "hello", content_type: "text/plain", confirm: false },
compose: { bucket: "BUCKET", sources: ["a.txt", "b.txt"], destination: "ab.txt", confirm: false }
},
"pubsub": { topics: {}, subscriptions: {}, publish: { topic: "TOPIC", messages: ["hello"], confirm: false } },
"firestore": { databases: {}, collections: {}, get: { path: "users/alice" } },
"spanner": { instances: {}, databases: { instance: "INSTANCE" } },
"bigtable": { instances: {}, tables: { instance: "INSTANCE" } },
"secrets": { list: {}, versions: { secret: "SECRET" }, access: { secret: "SECRET", version: "latest", confirm: false } },
"dataplex": { locations: {}, search: { location: "global", query: "type=table" } },
"dlp": { info_types: {}, inspect: { text: "mail jane@example.com", info_types: ["EMAIL_ADDRESS"], confirm: false } },
"kms": { locations: {}, rings: { location: "global" } },
"datastore": { query: { gql: { queryString: "SELECT * FROM __kind__ LIMIT 1" } } },
"storage/sign": { sign: { bucket: "BUCKET", object: "path/file.txt", method: "GET", expires_seconds: 900, confirm: false } },
"call": { call: { rpc: "bigquery.datasets.list", path: { projectId: "PROJECT" } } }
};
function post(path, body) {
err.textContent = "";
out.textContent = "...";
return fetch(path, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body)
}).then(function (res) {
return res.text().then(function (t) {
var j;
try { j = JSON.parse(t); } catch (e) { j = { message: t }; }
if (!res.ok) { err.textContent = j.message || ("HTTP " + res.status); out.textContent = ""; }
else { out.textContent = JSON.stringify(j, null, 2); }
});
}).catch(function (e) { err.textContent = String(e); out.textContent = ""; });
}
function fillActions() {
var svc = $("gd-service").value, sel = $("gd-action");
sel.innerHTML = "";
var names = Object.keys(examples[svc] || {});
var known = (actions[svc] || []).map(function (a) { return a.action; });
known.forEach(function (n) { if (names.indexOf(n) < 0) names.push(n); });
names.forEach(function (n) {
var o = document.createElement("option");
o.value = n;
var meta = (actions[svc] || []).filter(function (a) { return a.action === n; })[0];
o.textContent = n + (meta && meta.confirm ? " (confirm)" : "");
sel.appendChild(o);
});
fillBody();
}
function fillBody() {
var svc = $("gd-service").value, action = $("gd-action").value;
var ex = (examples[svc] || {})[action] || {};
var b = {};
if (svc !== "call" && svc !== "storage/sign") b.action = action;
Object.keys(ex).forEach(function (k) { b[k] = ex[k]; });
$("gd-body").value = JSON.stringify(b, null, 1);
}
fetch("/google/data/status").then(function (r) { return r.json(); }).then(function (s) {
actions = s.actions || {};
$("gd-config").textContent = s.configured
? "Google credential found." + (s.project_set ? "" : " Set GOOGLE_CLOUD_PROJECT.")
: "No Google credential: calls answer 503 until one is configured.";
fillActions();
});
$("gd-service").onchange = fillActions;
$("gd-action").onchange = fillBody;
$("gd-send").onclick = function () {
var body;
try { body = JSON.parse($("gd-body").value || "{}"); }
catch (e) { err.textContent = "Body is not valid JSON"; return; }
post("/google/data/" + $("gd-service").value, body);
};
fillActions();
})();