feat(clients): discovery: generic engine for all Google Discovery APIs
Directory and document cache with caps, parameter and body validation before traffic, path/query/repeated/regional handling, key-or-OAuth choice, paging, long-running wait, whole-directory checker script, demo at /google/discovery. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,13 +1,287 @@
|
||||
#include "../GoogleDemos.hpp"
|
||||
|
||||
#include "../../fw/Router.hpp"
|
||||
#include "../../fw/Templates.hpp"
|
||||
#include "google/discovery/caps.h"
|
||||
#include "google/discovery/engine.h"
|
||||
|
||||
namespace molo::example::app {
|
||||
|
||||
// every Google API generically (all 316 Discovery documents). Stub: the package agent fills this in.
|
||||
// Every Google API generically: the Discovery engine over src/google/discovery.
|
||||
// GET routes need no credential. Every route that reaches Google (documents
|
||||
// and the API itself) is a POST: 422 on bad input first, then 503
|
||||
// google_not_configured without a credential. A call runs at once when the
|
||||
// method is a GET; any other verb is a dry run (the validated request) unless
|
||||
// the body says "confirm":true.
|
||||
|
||||
namespace {
|
||||
|
||||
namespace dc = google::discovery;
|
||||
using Json = clients::body::Json;
|
||||
|
||||
constexpr std::size_t kMaxBody = 512 * 1024;
|
||||
constexpr std::size_t kMaxText = 200;
|
||||
|
||||
const char kPage[] = R"DISCOVERY({% extends "base.html" %}
|
||||
{% block content %}
|
||||
<h1>Google Discovery</h1>
|
||||
<p class="hint">Find any Google API (the whole Discovery directory) and call any
|
||||
method by id. Requests are validated against the real Discovery document before
|
||||
any traffic. Only GET methods run at once; other verbs are dry runs until you
|
||||
tick confirm.</p>
|
||||
<p id="dc-config" class="hint">Checking Google credentials...</p>
|
||||
<section class="dc">
|
||||
<label>Find APIs <input id="dc-filter" type="text" maxlength="100" placeholder="e.g. bigquery"></label>
|
||||
<div class="dc-actions"><button id="dc-apis" type="button" class="primary">List APIs</button></div>
|
||||
<label>API <input id="dc-api" type="text" maxlength="100" placeholder="name or name:version"></label>
|
||||
<label>Method id <input id="dc-method" type="text" maxlength="200" placeholder="serviceusage.services.list"></label>
|
||||
<div class="dc-actions">
|
||||
<button id="dc-methods" type="button">Methods of API</button>
|
||||
<button id="dc-describe" type="button">Describe</button>
|
||||
<button id="dc-plan" type="button">Validate</button>
|
||||
</div>
|
||||
<label>Parameters (JSON object) <textarea id="dc-params" rows="3">{}</textarea></label>
|
||||
<label>Body (JSON object, optional) <textarea id="dc-body" rows="3"></textarea></label>
|
||||
<label>Regional location <input id="dc-location" type="text" maxlength="64" placeholder="optional, e.g. us-central1"></label>
|
||||
<label>Credential
|
||||
<select id="dc-auth"><option value="auto">auto</option><option value="key">API key</option><option value="oauth">OAuth</option></select></label>
|
||||
<label>Pages <input id="dc-pages" type="number" min="1" max="50" value="1"></label>
|
||||
<label><input id="dc-wait" type="checkbox"> wait for long-running operation</label>
|
||||
<label><input id="dc-confirm" type="checkbox"> confirm (run non-GET methods)</label>
|
||||
<div class="dc-actions"><button id="dc-call" type="button" class="primary">Call</button></div>
|
||||
<p id="dc-error" class="error" role="alert"></p>
|
||||
<pre id="dc-out" class="dc-out"></pre>
|
||||
</section>
|
||||
<style>
|
||||
.dc label{display:block;margin:.5rem 0}
|
||||
.dc input[type=text],.dc input[type=number],.dc textarea,.dc select{width:100%;box-sizing:border-box}
|
||||
.dc-actions{display:flex;flex-wrap:wrap;gap:.5rem;margin:.5rem 0}
|
||||
.dc-out{overflow:auto;max-height:24rem;white-space:pre-wrap;word-break:break-word}
|
||||
</style>
|
||||
<script src="/embedded/google_discovery.js"></script>
|
||||
{% endblock %}
|
||||
)DISCOVERY";
|
||||
|
||||
fw::Response json_out(const Json& j, int status = 200) {
|
||||
fw::Response res = fw::JsonResponse(j.dump(), status);
|
||||
res.headers.set("Cache-Control", "no-store");
|
||||
return res;
|
||||
}
|
||||
|
||||
[[noreturn]] void bad(const std::string& why) {
|
||||
throw fw::HttpError(422, "validation_error", why);
|
||||
}
|
||||
|
||||
[[noreturn]] void rethrow(const google::GoogleError& e) {
|
||||
if (e.http_status() == 503 && e.status() == "NOT_CONFIGURED")
|
||||
throw fw::HttpError(503, "google_not_configured", e.message());
|
||||
if (e.service().empty() && e.status() == "INVALID_ARGUMENT")
|
||||
throw fw::HttpError(422, "validation_error", e.message());
|
||||
if (e.service() == "discovery" && e.status() == "NOT_FOUND")
|
||||
throw fw::HttpError(404, "not_found", e.message());
|
||||
throw fw::HttpError(502, "google_error", e.status() + ": " + e.message());
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
auto guarded(F&& f) -> decltype(f()) {
|
||||
try {
|
||||
return f();
|
||||
} catch (const google::GoogleError& e) {
|
||||
rethrow(e);
|
||||
}
|
||||
}
|
||||
|
||||
Json parse_body(const fw::Request& req) {
|
||||
if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large");
|
||||
if (req.body.empty()) return Json::object();
|
||||
try {
|
||||
Json j = Json::parse(req.body);
|
||||
if (!j.is_object()) bad("body must be a JSON object");
|
||||
return j;
|
||||
} catch (const clients::body::JsonError&) {
|
||||
bad("body is not valid JSON");
|
||||
}
|
||||
}
|
||||
|
||||
std::string text(const Json& body, const char* key, bool required = false) {
|
||||
const Json* v = body.find(key);
|
||||
if (!v || v->is_null()) {
|
||||
if (required) bad(std::string(key) + " is required");
|
||||
return "";
|
||||
}
|
||||
if (!v->is_string()) bad(std::string(key) + " must be a string");
|
||||
if (v->as_string().size() > kMaxText) bad(std::string(key) + " is too long");
|
||||
return v->as_string();
|
||||
}
|
||||
|
||||
bool flag(const Json& body, const char* key) {
|
||||
const Json* v = body.find(key);
|
||||
if (!v || v->is_null()) return false;
|
||||
if (!v->is_bool()) bad(std::string(key) + " must be true or false");
|
||||
return v->as_bool();
|
||||
}
|
||||
|
||||
std::size_t bounded(const Json& body, const char* key, std::size_t lo, std::size_t hi,
|
||||
std::size_t fallback) {
|
||||
const Json* v = body.find(key);
|
||||
if (!v || v->is_null()) return fallback;
|
||||
if (!v->is_number()) bad(std::string(key) + " must be a number");
|
||||
long n = v->as_int();
|
||||
if (n < static_cast<long>(lo) || n > static_cast<long>(hi))
|
||||
bad(std::string(key) + " must be " + std::to_string(lo) + ".." + std::to_string(hi));
|
||||
return static_cast<std::size_t>(n);
|
||||
}
|
||||
|
||||
// "name" or "name:version", letters digits _ - . and one colon.
|
||||
std::string api_ref(const Json& body) {
|
||||
std::string api = text(body, "api", true);
|
||||
std::size_t colons = 0;
|
||||
for (char c : api) {
|
||||
if (c == ':') ++colons;
|
||||
else if (!(std::isalnum(static_cast<unsigned char>(c)) || c == '_' || c == '-' || c == '.'))
|
||||
bad("api must be name or name:version");
|
||||
}
|
||||
if (colons > 1) bad("api must be name or name:version");
|
||||
return api;
|
||||
}
|
||||
|
||||
std::string method_id(const Json& body) {
|
||||
std::string id = text(body, "method", true);
|
||||
for (char c : id)
|
||||
if (!(std::isalnum(static_cast<unsigned char>(c)) || c == '_' || c == '-' || c == '.'))
|
||||
bad("method must be a Discovery method id such as serviceusage.services.list");
|
||||
return id;
|
||||
}
|
||||
|
||||
dc::CallArgs args_from(const Json& body) {
|
||||
dc::CallArgs a;
|
||||
if (const Json* p = body.find("params"); p && !p->is_null()) {
|
||||
if (!p->is_object()) bad("params must be a JSON object");
|
||||
a.params = *p;
|
||||
}
|
||||
if (const Json* b = body.find("body"); b && !b->is_null()) {
|
||||
if (!b->is_object()) bad("body must be a JSON object");
|
||||
a.body = *b;
|
||||
}
|
||||
a.location = text(body, "location");
|
||||
try {
|
||||
a.auth = dc::auth_mode_from(text(body, "auth"));
|
||||
} catch (const google::GoogleError& e) {
|
||||
bad(e.message());
|
||||
}
|
||||
a.max_pages = bounded(body, "maxPages", 1, dc::DiscoveryCaps::kMaxPages, 1);
|
||||
a.items_key = text(body, "itemsKey");
|
||||
a.wait = flag(body, "wait");
|
||||
return a;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void register_google_discovery_demo(fw::Router& router, Services& services) {
|
||||
(void)router;
|
||||
(void)services;
|
||||
if (!services.google) services.google = google::GoogleClient::from_env();
|
||||
auto client = services.google;
|
||||
auto catalog = std::make_shared<dc::Catalog>(std::make_shared<dc::HttpDocumentSource>());
|
||||
auto engine = std::make_shared<dc::Engine>(client, catalog);
|
||||
fw::Environment* env = services.templates.get();
|
||||
services.templates->add_template("google_discovery.html", kPage);
|
||||
services.add_feature("google-discovery", "/google/discovery");
|
||||
|
||||
// Shape checks come first (422); then the credential (503).
|
||||
auto need_credential = [client]() {
|
||||
if (!client->configured()) throw google::NotConfiguredError("discovery");
|
||||
};
|
||||
|
||||
router.get("/google/discovery", "google-discovery-page", [env](fw::Request req) {
|
||||
(void)req;
|
||||
fw::TemplateCtx ctx;
|
||||
ctx["title"] = std::string("Google Discovery");
|
||||
return fw::Response(200, env->render("google_discovery.html", ctx), "text/html; charset=utf-8");
|
||||
});
|
||||
|
||||
router.get("/google/discovery/status", "google-discovery-status", [client, catalog](fw::Request req) {
|
||||
(void)req;
|
||||
dc::CatalogStats s = catalog->stats();
|
||||
Json caps = Json::object();
|
||||
caps.set("max_pages", Json(static_cast<long>(dc::DiscoveryCaps::kMaxPages)));
|
||||
caps.set("max_items", Json(static_cast<long>(dc::DiscoveryCaps::kMaxItems)));
|
||||
caps.set("max_doc_bytes", Json(static_cast<long>(dc::DiscoveryCaps::kMaxDocBytes)));
|
||||
caps.set("max_body_bytes", Json(static_cast<long>(dc::DiscoveryCaps::kMaxBodyBytes)));
|
||||
caps.set("max_cached_apis", Json(static_cast<long>(dc::DiscoveryCaps::kMaxCachedApis)));
|
||||
caps.set("max_age_seconds", Json(static_cast<long>(dc::DiscoveryCaps::kMaxAge.count())));
|
||||
caps.set("min_refetch_seconds", Json(static_cast<long>(dc::DiscoveryCaps::kMinRefetch.count())));
|
||||
Json o = Json::object();
|
||||
o.set("configured", Json(client->configured()));
|
||||
o.set("directory", Json(std::string(dc::kDirectoryUrl)));
|
||||
o.set("cached_apis", Json(static_cast<long>(s.cached)));
|
||||
o.set("cached_bytes", Json(static_cast<long>(s.cached_bytes)));
|
||||
o.set("fetches", Json(static_cast<long>(s.fetches)));
|
||||
o.set("caps", caps);
|
||||
return json_out(o);
|
||||
});
|
||||
|
||||
router.post("/google/discovery/apis", "google-discovery-apis", [engine, need_credential](fw::Request req) {
|
||||
Json b = parse_body(req);
|
||||
std::string filter = text(b, "filter");
|
||||
std::size_t limit = bounded(b, "limit", 1, dc::DiscoveryCaps::kMaxListed, 50);
|
||||
bool all = flag(b, "all");
|
||||
return guarded([&] {
|
||||
need_credential();
|
||||
Json o = Json::object();
|
||||
o.set("apis", engine->list_apis(filter, limit, !all));
|
||||
return json_out(o);
|
||||
});
|
||||
});
|
||||
|
||||
router.post("/google/discovery/methods", "google-discovery-methods", [engine, need_credential](fw::Request req) {
|
||||
Json b = parse_body(req);
|
||||
std::string api = api_ref(b);
|
||||
std::string filter = text(b, "filter");
|
||||
std::size_t limit = bounded(b, "limit", 1, dc::DiscoveryCaps::kMaxListed, 100);
|
||||
return guarded([&] {
|
||||
need_credential();
|
||||
return json_out(engine->list_methods(api, filter, limit));
|
||||
});
|
||||
});
|
||||
|
||||
router.post("/google/discovery/describe", "google-discovery-describe", [engine, need_credential](fw::Request req) {
|
||||
Json b = parse_body(req);
|
||||
std::string api = api_ref(b);
|
||||
std::string method = method_id(b);
|
||||
return guarded([&] {
|
||||
need_credential();
|
||||
return json_out(engine->describe(api, method));
|
||||
});
|
||||
});
|
||||
|
||||
router.post("/google/discovery/plan", "google-discovery-plan", [engine, need_credential](fw::Request req) {
|
||||
Json b = parse_body(req);
|
||||
std::string api = api_ref(b);
|
||||
std::string method = method_id(b);
|
||||
dc::CallArgs args = args_from(b);
|
||||
return guarded([&] {
|
||||
need_credential();
|
||||
return json_out(engine->plan(api, method, args).to_json());
|
||||
});
|
||||
});
|
||||
|
||||
router.post("/google/discovery/call", "google-discovery-call", [engine, need_credential](fw::Request req) {
|
||||
Json b = parse_body(req);
|
||||
std::string api = api_ref(b);
|
||||
std::string method = method_id(b);
|
||||
dc::CallArgs args = args_from(b);
|
||||
bool confirm = flag(b, "confirm");
|
||||
return guarded([&] {
|
||||
need_credential();
|
||||
dc::PreparedCall plan = engine->plan(api, method, args); // validates, no API traffic
|
||||
if (plan.http != clients::Method::kGet && !confirm) {
|
||||
Json o = plan.to_json();
|
||||
o.set("dry_run", Json(true));
|
||||
o.set("hint", Json("send confirm:true to run it"));
|
||||
return json_out(o);
|
||||
}
|
||||
return json_out(engine->call(api, method, args).to_json());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace molo::example::app
|
||||
|
||||
Reference in New Issue
Block a user