feat(clients): discovery: generic engine for all Google Discovery APIs
Directory and document cache with caps, parameter and body validation before traffic, path/query/repeated/regional handling, key-or-OAuth choice, paging, long-running wait, whole-directory checker script, demo at /google/discovery. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,285 @@
|
||||
#include "google/discovery/catalog.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
|
||||
#include "clients/transports/http_transport.h"
|
||||
#include "google/discovery/caps.h"
|
||||
#include "google/errors.h"
|
||||
|
||||
namespace molo::google::discovery {
|
||||
|
||||
using clients::body::Json;
|
||||
|
||||
const char* const kDirectoryUrl = "https://discovery.googleapis.com/discovery/v1/apis";
|
||||
|
||||
namespace {
|
||||
|
||||
std::string str(const Json& o, const char* key) {
|
||||
const Json* v = o.find(key);
|
||||
return v && v->is_string() ? v->as_string() : "";
|
||||
}
|
||||
|
||||
std::string lower(std::string s) {
|
||||
for (char& c : s) c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
|
||||
return s;
|
||||
}
|
||||
|
||||
bool valid_name(const std::string& s) {
|
||||
if (s.empty() || s.size() > 64) return false;
|
||||
for (char c : s)
|
||||
if (!(std::isalnum(static_cast<unsigned char>(c)) || c == '_' || c == '-' || c == '.')) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Json DirectoryItem::to_json() const {
|
||||
Json o = Json::object();
|
||||
o.set("id", Json(id));
|
||||
o.set("name", Json(name));
|
||||
o.set("version", Json(version));
|
||||
o.set("title", Json(title));
|
||||
o.set("description", Json(description));
|
||||
o.set("discoveryRestUrl", Json(rest_url));
|
||||
o.set("documentationLink", Json(documentation));
|
||||
o.set("preferred", Json(preferred));
|
||||
return o;
|
||||
}
|
||||
|
||||
Directory Directory::parse(const std::string& text) {
|
||||
if (text.size() > DiscoveryCaps::kMaxDirectoryBytes)
|
||||
throw GoogleError(502, "BAD_DOCUMENT", "discovery directory is larger than the cap");
|
||||
Json doc;
|
||||
try {
|
||||
doc = Json::parse(text);
|
||||
} catch (const std::exception& e) {
|
||||
throw GoogleError(502, "BAD_DOCUMENT", std::string("discovery directory: ") + e.what());
|
||||
}
|
||||
const Json* items = doc.is_object() ? doc.find("items") : nullptr;
|
||||
if (!items || !items->is_array() || str(doc, "kind") != "discovery#directoryList")
|
||||
throw GoogleError(502, "BAD_DOCUMENT", "discovery directory: not a directoryList");
|
||||
if (items->as_array().size() > DiscoveryCaps::kMaxDirectoryItems)
|
||||
throw GoogleError(502, "BAD_DOCUMENT", "discovery directory: more items than the cap");
|
||||
Directory dir;
|
||||
for (const Json& it : items->as_array()) {
|
||||
DirectoryItem row;
|
||||
row.id = str(it, "id");
|
||||
row.name = str(it, "name");
|
||||
row.version = str(it, "version");
|
||||
row.title = str(it, "title");
|
||||
row.description = str(it, "description");
|
||||
row.rest_url = str(it, "discoveryRestUrl");
|
||||
row.documentation = str(it, "documentationLink");
|
||||
const Json* p = it.find("preferred");
|
||||
row.preferred = p && p->is_bool() && p->as_bool();
|
||||
if (row.id.empty() || row.name.empty() || row.version.empty() ||
|
||||
!is_google_host_url(row.rest_url))
|
||||
throw GoogleError(502, "BAD_DOCUMENT", "discovery directory: bad item " + row.id);
|
||||
dir.items_.push_back(std::move(row));
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
std::size_t Directory::preferred_count() const {
|
||||
return static_cast<std::size_t>(std::count_if(items_.begin(), items_.end(),
|
||||
[](const DirectoryItem& i) { return i.preferred; }));
|
||||
}
|
||||
|
||||
const DirectoryItem* Directory::find(const std::string& name, const std::string& version) const {
|
||||
const DirectoryItem* fallback = nullptr;
|
||||
for (const DirectoryItem& i : items_) {
|
||||
if (i.name != name) continue;
|
||||
if (!version.empty()) {
|
||||
if (i.version == version) return &i;
|
||||
} else if (i.preferred) {
|
||||
return &i;
|
||||
} else if (!fallback) {
|
||||
fallback = &i;
|
||||
}
|
||||
}
|
||||
return version.empty() ? fallback : nullptr;
|
||||
}
|
||||
|
||||
std::vector<const DirectoryItem*> Directory::search(const std::string& needle, std::size_t limit,
|
||||
bool preferred_only) const {
|
||||
std::vector<const DirectoryItem*> out;
|
||||
std::string n = lower(needle);
|
||||
limit = std::min(limit, DiscoveryCaps::kMaxListed);
|
||||
for (const DirectoryItem& i : items_) {
|
||||
if (out.size() >= limit) break;
|
||||
if (preferred_only && !i.preferred) continue;
|
||||
if (n.empty() || lower(i.name).find(n) != std::string::npos ||
|
||||
lower(i.title).find(n) != std::string::npos)
|
||||
out.push_back(&i);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
namespace {
|
||||
clients::ClientConfig source_config() {
|
||||
clients::ClientConfig c;
|
||||
c.auth = nullptr;
|
||||
c.persist_cookies = false;
|
||||
return c;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
HttpDocumentSource::HttpDocumentSource()
|
||||
: client_(std::make_unique<clients::transports::HttpTransport>(), source_config()) {}
|
||||
|
||||
HttpDocumentSource::HttpDocumentSource(std::unique_ptr<clients::Transport> transport)
|
||||
: client_(std::move(transport), source_config()) {}
|
||||
|
||||
std::string HttpDocumentSource::fetch(const std::string& url, std::size_t max_bytes) {
|
||||
if (!is_google_host_url(url))
|
||||
throw GoogleError::invalid("discovery url is not an https googleapis.com url: " + url);
|
||||
try {
|
||||
clients::ClientResponse res = client_.get_blocking(url);
|
||||
if (!res.is_success())
|
||||
throw GoogleError(res.status(), res.status() == 404 ? "NOT_FOUND" : "UNAVAILABLE",
|
||||
"discovery fetch failed with HTTP " + std::to_string(res.status()), "discovery");
|
||||
if (res.body().size() > max_bytes)
|
||||
throw GoogleError(502, "BAD_DOCUMENT", "discovery document is larger than the cap", "discovery");
|
||||
return res.body();
|
||||
} catch (const GoogleError&) {
|
||||
throw;
|
||||
} catch (const std::exception& e) {
|
||||
throw GoogleError(502, "UNAVAILABLE", e.what(), "discovery");
|
||||
}
|
||||
}
|
||||
|
||||
std::string FileDocumentSource::fetch(const std::string& url, std::size_t max_bytes) {
|
||||
std::string file;
|
||||
if (url.rfind(kDirectoryUrl, 0) == 0) {
|
||||
file = "directory.json";
|
||||
} else {
|
||||
if (index_.empty()) {
|
||||
std::ifstream din(dir_ + "/directory.json", std::ios::binary);
|
||||
if (!din) throw GoogleError(404, "NOT_FOUND", "saved directory.json missing", "discovery");
|
||||
std::ostringstream dss;
|
||||
dss << din.rdbuf();
|
||||
for (const DirectoryItem& i : Directory::parse(dss.str()).items())
|
||||
index_[i.rest_url] = i.name + "_" + i.version + ".json";
|
||||
}
|
||||
auto hit = index_.find(url);
|
||||
if (hit == index_.end()) throw GoogleError::invalid("no saved document for url: " + url);
|
||||
file = hit->second;
|
||||
}
|
||||
std::ifstream in(dir_ + "/" + file, std::ios::binary);
|
||||
if (!in) throw GoogleError(404, "NOT_FOUND", "saved discovery document missing: " + file, "discovery");
|
||||
std::ostringstream ss;
|
||||
ss << in.rdbuf();
|
||||
std::string text = ss.str();
|
||||
if (text.size() > max_bytes)
|
||||
throw GoogleError(502, "BAD_DOCUMENT", "discovery document is larger than the cap", "discovery");
|
||||
return text;
|
||||
}
|
||||
|
||||
Catalog::Catalog(std::shared_ptr<DocumentSource> source, Clock clock)
|
||||
: source_(std::move(source)), clock_(std::move(clock)) {}
|
||||
|
||||
std::chrono::steady_clock::time_point Catalog::now() const {
|
||||
return clock_ ? clock_() : std::chrono::steady_clock::now();
|
||||
}
|
||||
|
||||
std::shared_ptr<const Directory> Catalog::directory() {
|
||||
std::lock_guard<std::mutex> lock(mu_);
|
||||
auto t = now();
|
||||
if (directory_ && t - directory_at_ < DiscoveryCaps::kMaxAge) {
|
||||
++stats_.hits;
|
||||
return directory_;
|
||||
}
|
||||
++stats_.fetches;
|
||||
std::string text = source_->fetch(kDirectoryUrl, DiscoveryCaps::kMaxDirectoryBytes);
|
||||
directory_ = std::make_shared<const Directory>(Directory::parse(text));
|
||||
directory_at_ = t;
|
||||
return directory_;
|
||||
}
|
||||
|
||||
std::shared_ptr<const Api> Catalog::api(const std::string& ref) {
|
||||
std::string name = ref, version;
|
||||
if (std::size_t c = ref.find(':'); c != std::string::npos) {
|
||||
name = ref.substr(0, c);
|
||||
version = ref.substr(c + 1);
|
||||
}
|
||||
if (!valid_name(name) || (!version.empty() && !valid_name(version)))
|
||||
throw GoogleError::invalid("api must be name or name:version, got '" + ref + "'");
|
||||
std::shared_ptr<const Directory> dir = directory();
|
||||
const DirectoryItem* item = dir->find(name, version);
|
||||
if (!item) throw GoogleError(404, "NOT_FOUND", "no such API in the Discovery directory: " + ref, "discovery");
|
||||
std::lock_guard<std::mutex> lock(mu_);
|
||||
auto t = now();
|
||||
auto it = entries_.find(item->id);
|
||||
if (it != entries_.end()) {
|
||||
bool stale = t - it->second.fetched >= DiscoveryCaps::kMaxAge;
|
||||
auto lf = last_fetch_.find(item->id);
|
||||
bool floor = lf != last_fetch_.end() && t - lf->second < DiscoveryCaps::kMinRefetch;
|
||||
if (!stale || floor) {
|
||||
lru_.remove(item->id);
|
||||
lru_.push_front(item->id);
|
||||
++stats_.hits;
|
||||
return it->second.api;
|
||||
}
|
||||
}
|
||||
++stats_.fetches;
|
||||
std::string text = source_->fetch(item->rest_url, DiscoveryCaps::kMaxDocBytes);
|
||||
Json doc;
|
||||
try {
|
||||
doc = Json::parse(text);
|
||||
} catch (const std::exception& e) {
|
||||
throw GoogleError(502, "BAD_DOCUMENT", std::string("discovery document ") + item->id + ": " + e.what());
|
||||
}
|
||||
Api parsed = Api::parse(doc);
|
||||
parsed.raw_bytes = text.size();
|
||||
if (parsed.name() != item->name || parsed.version() != item->version)
|
||||
throw GoogleError(502, "BAD_DOCUMENT", "document " + parsed.id() + " is not the one requested: " + item->id);
|
||||
Entry e;
|
||||
e.api = std::make_shared<const Api>(std::move(parsed));
|
||||
e.fetched = t;
|
||||
e.bytes = text.size();
|
||||
entries_[item->id] = e;
|
||||
last_fetch_[item->id] = t;
|
||||
lru_.remove(item->id);
|
||||
lru_.push_front(item->id);
|
||||
evict_locked();
|
||||
return e.api;
|
||||
}
|
||||
|
||||
void Catalog::evict_locked() {
|
||||
auto total = [&] {
|
||||
std::size_t b = 0;
|
||||
for (const auto& [k, v] : entries_) b += v.bytes;
|
||||
return b;
|
||||
};
|
||||
while (entries_.size() > 1 &&
|
||||
(entries_.size() > DiscoveryCaps::kMaxCachedApis || total() > DiscoveryCaps::kMaxCachedBytes)) {
|
||||
entries_.erase(lru_.back());
|
||||
lru_.pop_back();
|
||||
++stats_.evictions;
|
||||
}
|
||||
}
|
||||
|
||||
void Catalog::forget(const std::string& ref) {
|
||||
std::lock_guard<std::mutex> lock(mu_);
|
||||
std::string name = ref, version;
|
||||
if (std::size_t c = ref.find(':'); c != std::string::npos) {
|
||||
name = ref.substr(0, c);
|
||||
version = ref.substr(c + 1);
|
||||
}
|
||||
for (auto& [id, e] : entries_)
|
||||
if (e.api->name() == name && (version.empty() || e.api->version() == version))
|
||||
e.fetched = std::chrono::steady_clock::time_point{};
|
||||
}
|
||||
|
||||
CatalogStats Catalog::stats() const {
|
||||
std::lock_guard<std::mutex> lock(mu_);
|
||||
CatalogStats s = stats_;
|
||||
s.cached = entries_.size();
|
||||
for (const auto& [k, v] : entries_) s.cached_bytes += v.bytes;
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace molo::google::discovery
|
||||
Reference in New Issue
Block a user