feat(clients): discovery: generic engine for all Google Discovery APIs
Directory and document cache with caps, parameter and body validation before traffic, path/query/repeated/regional handling, key-or-OAuth choice, paging, long-running wait, whole-directory checker script, demo at /google/discovery. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,228 @@
|
||||
#include "google/discovery/engine.h"
|
||||
|
||||
#include <regex>
|
||||
#include <thread>
|
||||
|
||||
#include "google/discovery/caps.h"
|
||||
#include "google/errors.h"
|
||||
|
||||
namespace molo::google::discovery {
|
||||
|
||||
using clients::body::Json;
|
||||
|
||||
namespace {
|
||||
[[noreturn]] void bad(const std::string& why) { throw GoogleError::invalid(why); }
|
||||
|
||||
std::string str(const Json& o, const char* key) {
|
||||
const Json* v = o.find(key);
|
||||
return v && v->is_string() ? v->as_string() : "";
|
||||
}
|
||||
} // namespace
|
||||
|
||||
Json CallResult::to_json() const {
|
||||
Json o = Json::object();
|
||||
o.set("body", body);
|
||||
o.set("pages", Json(static_cast<long>(pages)));
|
||||
o.set("truncated", Json(truncated));
|
||||
o.set("operationWaited", Json(operation_waited));
|
||||
o.set("itemsKey", Json(items_key));
|
||||
o.set("support", Json(support_name(support)));
|
||||
Json n = Json::array();
|
||||
for (const std::string& s : notes) n.push(Json(s));
|
||||
o.set("notes", n);
|
||||
return o;
|
||||
}
|
||||
|
||||
Engine::Engine(std::shared_ptr<GoogleClient> client, std::shared_ptr<Catalog> catalog,
|
||||
Sleeper sleeper)
|
||||
: client_(std::move(client)), catalog_(std::move(catalog)), sleeper_(std::move(sleeper)) {
|
||||
if (!sleeper_)
|
||||
sleeper_ = [](std::chrono::milliseconds d) { std::this_thread::sleep_for(d); };
|
||||
}
|
||||
|
||||
Json Engine::list_apis(const std::string& filter, std::size_t limit, bool preferred_only) {
|
||||
auto dir = catalog_->directory();
|
||||
Json out = Json::array();
|
||||
for (const DirectoryItem* i : dir->search(filter, limit, preferred_only)) out.push(i->to_json());
|
||||
return out;
|
||||
}
|
||||
|
||||
Json Engine::list_methods(const std::string& api_ref, const std::string& filter,
|
||||
std::size_t limit) {
|
||||
auto api = catalog_->api(api_ref);
|
||||
limit = std::min(limit, DiscoveryCaps::kMaxListed);
|
||||
Json out = Json::array();
|
||||
std::size_t matched = 0;
|
||||
for (const Method& m : api->methods()) {
|
||||
if (!filter.empty() && m.id.find(filter) == std::string::npos) continue;
|
||||
++matched;
|
||||
if (out.as_array().size() < limit) out.push(m.summary());
|
||||
}
|
||||
Json o = Json::object();
|
||||
o.set("api", Json(api->id()));
|
||||
o.set("total", Json(static_cast<long>(api->methods().size())));
|
||||
o.set("matched", Json(static_cast<long>(matched)));
|
||||
o.set("truncated", Json(matched > out.as_array().size()));
|
||||
o.set("methods", out);
|
||||
return o;
|
||||
}
|
||||
|
||||
const Method& Engine::method_of(const Api& api, const std::string& method_id) const {
|
||||
const Method* m = api.find(method_id);
|
||||
if (!m) throw GoogleError(404, "NOT_FOUND", "api " + api.id() + " has no method " + method_id, "discovery");
|
||||
return *m;
|
||||
}
|
||||
|
||||
Json Engine::describe(const std::string& api_ref, const std::string& method_id) {
|
||||
auto api = catalog_->api(api_ref);
|
||||
const Method& m = method_of(*api, method_id);
|
||||
Json o = m.summary();
|
||||
Json params = Json::array();
|
||||
for (const auto& [n, p] : m.params) {
|
||||
Json pj = Json::object();
|
||||
pj.set("name", Json(n));
|
||||
pj.set("in", Json(p.location));
|
||||
pj.set("type", Json(p.type));
|
||||
if (!p.format.empty()) pj.set("format", Json(p.format));
|
||||
pj.set("required", Json(p.required));
|
||||
pj.set("repeated", Json(p.repeated));
|
||||
if (!p.pattern.empty()) pj.set("pattern", Json(p.pattern));
|
||||
if (!p.enum_values.empty()) {
|
||||
Json e = Json::array();
|
||||
for (const std::string& v : p.enum_values) e.push(Json(v));
|
||||
pj.set("enum", e);
|
||||
}
|
||||
params.push(pj);
|
||||
}
|
||||
o.set("parameters", params);
|
||||
Json scopes = Json::array();
|
||||
for (const std::string& s : m.scopes) scopes.push(Json(s));
|
||||
o.set("scopes", scopes);
|
||||
o.set("request", Json(m.request_ref));
|
||||
o.set("response", Json(m.response_ref));
|
||||
o.set("pageable", Json(api->is_pageable(m)));
|
||||
o.set("itemsKey", Json(api->is_pageable(m) ? api->items_key(m) : ""));
|
||||
o.set("operation", Json(!m.response_ref.empty() && api->is_operation_schema(m.response_ref)));
|
||||
Json locs = Json::array();
|
||||
for (const EndpointRow& e : api->endpoints()) locs.push(Json(e.location));
|
||||
o.set("locations", locs);
|
||||
return o;
|
||||
}
|
||||
|
||||
PreparedCall Engine::plan(const std::string& api_ref, const std::string& method_id,
|
||||
const CallArgs& args) {
|
||||
auto api = catalog_->api(api_ref);
|
||||
return Planner::plan(*api, method_of(*api, method_id), args);
|
||||
}
|
||||
|
||||
Json Engine::send(const PreparedCall& p) {
|
||||
return client_->request(p.http, p.endpoint, p.path, p.has_body ? &p.body : nullptr);
|
||||
}
|
||||
|
||||
CallResult Engine::call(const std::string& api_ref, const std::string& method_id,
|
||||
const CallArgs& args) {
|
||||
if (args.max_pages == 0 || args.max_pages > DiscoveryCaps::kMaxPages)
|
||||
bad("max_pages must be 1 to " + std::to_string(DiscoveryCaps::kMaxPages));
|
||||
auto api = catalog_->api(api_ref);
|
||||
const Method& m = method_of(*api, method_id);
|
||||
PreparedCall first = Planner::plan(*api, m, args); // all validation, no traffic
|
||||
if (args.max_pages > 1) {
|
||||
if (!first.pageable) bad("method " + m.id + " is not pageable (needs a pageToken parameter and a nextPageToken reply)");
|
||||
if (args.items_key.empty() && api->items_key(m).empty())
|
||||
bad("method " + m.id + " has several list fields: name one with items_key");
|
||||
}
|
||||
if (!client_->configured()) throw NotConfiguredError(api->name());
|
||||
|
||||
CallResult out;
|
||||
out.support = first.support;
|
||||
out.notes = first.notes;
|
||||
if (args.max_pages > 1) {
|
||||
walk_pages(*api, m, args, first, out);
|
||||
return out;
|
||||
}
|
||||
out.body = send(first);
|
||||
if (args.wait && first.operation) {
|
||||
out.body = wait_for(*api, first, args, out.body);
|
||||
out.operation_waited = true;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
void Engine::walk_pages(const Api& api, const Method& m, const CallArgs& args,
|
||||
const PreparedCall& first, CallResult& out) {
|
||||
std::string key = !args.items_key.empty() ? args.items_key : api.items_key(m);
|
||||
out.items_key = key;
|
||||
Json items = Json::array();
|
||||
Json last = Json::object();
|
||||
PreparedCall call = first;
|
||||
std::size_t count = 0;
|
||||
for (std::size_t page = 0; page < args.max_pages; ++page) {
|
||||
last = send(call);
|
||||
out.pages = page + 1;
|
||||
if (const Json* arr = last.find(key); arr && arr->is_array())
|
||||
for (const Json& item : arr->as_array()) {
|
||||
if (count >= DiscoveryCaps::kMaxItems) {
|
||||
out.truncated = true;
|
||||
break;
|
||||
}
|
||||
items.push(item);
|
||||
++count;
|
||||
}
|
||||
const Json* next = last.find("nextPageToken");
|
||||
std::string token = next && next->is_string() ? next->as_string() : "";
|
||||
if (token.empty()) break;
|
||||
if (out.truncated || page + 1 == args.max_pages) {
|
||||
out.truncated = true;
|
||||
break;
|
||||
}
|
||||
CallArgs again = args;
|
||||
again.params.set("pageToken", Json(token));
|
||||
call = Planner::plan(api, m, again);
|
||||
}
|
||||
if (last.is_object()) {
|
||||
last.set(key, items);
|
||||
if (!out.truncated) last.as_object().erase("nextPageToken");
|
||||
}
|
||||
out.body = last;
|
||||
}
|
||||
|
||||
Json Engine::wait_for(const Api& api, const PreparedCall& origin, const CallArgs& args,
|
||||
const Json& op_in) {
|
||||
Json op = op_in;
|
||||
std::string name = str(op, "name");
|
||||
if (name.empty()) bad("operation reply has no name");
|
||||
const Method* getter = nullptr;
|
||||
for (const Method* g : api.operation_getters()) {
|
||||
const Param& p = g->params.at("name");
|
||||
if (p.pattern.empty() ? api.operation_getters().size() == 1
|
||||
: std::regex_search(name, std::regex(p.pattern))) {
|
||||
getter = g;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!getter)
|
||||
bad("no operations.get method of " + api.id() + " accepts the operation name " + name);
|
||||
CallArgs poll;
|
||||
poll.params.set("name", Json(name));
|
||||
poll.location = args.location;
|
||||
poll.auth = args.auth;
|
||||
PreparedCall get = Planner::plan(api, *getter, poll);
|
||||
(void)origin;
|
||||
std::chrono::milliseconds waited{0};
|
||||
for (std::size_t i = 0; i < OperationCaps::kMaxPolls; ++i) {
|
||||
Operation parsed = Operation::from_json(op);
|
||||
if (parsed.done) {
|
||||
if (parsed.failed())
|
||||
throw GoogleError(502, "OPERATION_FAILED", parsed.error_message, api.name());
|
||||
return op;
|
||||
}
|
||||
if (waited >= OperationCaps::kMaxWait) break;
|
||||
sleeper_(OperationCaps::kInterval);
|
||||
waited += OperationCaps::kInterval;
|
||||
op = send(get);
|
||||
}
|
||||
throw GoogleError(504, "DEADLINE_EXCEEDED",
|
||||
"operation still running at the wait cap: " + name, api.name());
|
||||
}
|
||||
|
||||
} // namespace molo::google::discovery
|
||||
Reference in New Issue
Block a user