#include "google/discovery/catalog.h" #include #include #include #include #include "clients/transports/http_transport.h" #include "google/discovery/caps.h" #include "google/errors.h" namespace molo::google::discovery { using clients::body::Json; const char* const kDirectoryUrl = "https://discovery.googleapis.com/discovery/v1/apis"; namespace { std::string str(const Json& o, const char* key) { const Json* v = o.find(key); return v && v->is_string() ? v->as_string() : ""; } std::string lower(std::string s) { for (char& c : s) c = static_cast(std::tolower(static_cast(c))); return s; } bool valid_name(const std::string& s) { if (s.empty() || s.size() > 64) return false; for (char c : s) if (!(std::isalnum(static_cast(c)) || c == '_' || c == '-' || c == '.')) return false; return true; } } // namespace Json DirectoryItem::to_json() const { Json o = Json::object(); o.set("id", Json(id)); o.set("name", Json(name)); o.set("version", Json(version)); o.set("title", Json(title)); o.set("description", Json(description)); o.set("discoveryRestUrl", Json(rest_url)); o.set("documentationLink", Json(documentation)); o.set("preferred", Json(preferred)); return o; } Directory Directory::parse(const std::string& text) { if (text.size() > DiscoveryCaps::kMaxDirectoryBytes) throw GoogleError(502, "BAD_DOCUMENT", "discovery directory is larger than the cap"); Json doc; try { doc = Json::parse(text); } catch (const std::exception& e) { throw GoogleError(502, "BAD_DOCUMENT", std::string("discovery directory: ") + e.what()); } const Json* items = doc.is_object() ? doc.find("items") : nullptr; if (!items || !items->is_array() || str(doc, "kind") != "discovery#directoryList") throw GoogleError(502, "BAD_DOCUMENT", "discovery directory: not a directoryList"); if (items->as_array().size() > DiscoveryCaps::kMaxDirectoryItems) throw GoogleError(502, "BAD_DOCUMENT", "discovery directory: more items than the cap"); Directory dir; for (const Json& it : items->as_array()) { DirectoryItem row; row.id = str(it, "id"); row.name = str(it, "name"); row.version = str(it, "version"); row.title = str(it, "title"); row.description = str(it, "description"); row.rest_url = str(it, "discoveryRestUrl"); row.documentation = str(it, "documentationLink"); const Json* p = it.find("preferred"); row.preferred = p && p->is_bool() && p->as_bool(); if (row.id.empty() || row.name.empty() || row.version.empty() || !is_google_host_url(row.rest_url)) throw GoogleError(502, "BAD_DOCUMENT", "discovery directory: bad item " + row.id); dir.items_.push_back(std::move(row)); } return dir; } std::size_t Directory::preferred_count() const { return static_cast(std::count_if(items_.begin(), items_.end(), [](const DirectoryItem& i) { return i.preferred; })); } const DirectoryItem* Directory::find(const std::string& name, const std::string& version) const { const DirectoryItem* fallback = nullptr; for (const DirectoryItem& i : items_) { if (i.name != name) continue; if (!version.empty()) { if (i.version == version) return &i; } else if (i.preferred) { return &i; } else if (!fallback) { fallback = &i; } } return version.empty() ? fallback : nullptr; } std::vector Directory::search(const std::string& needle, std::size_t limit, bool preferred_only) const { std::vector out; std::string n = lower(needle); limit = std::min(limit, DiscoveryCaps::kMaxListed); for (const DirectoryItem& i : items_) { if (out.size() >= limit) break; if (preferred_only && !i.preferred) continue; if (n.empty() || lower(i.name).find(n) != std::string::npos || lower(i.title).find(n) != std::string::npos) out.push_back(&i); } return out; } namespace { clients::ClientConfig source_config() { clients::ClientConfig c; c.auth = nullptr; c.persist_cookies = false; return c; } } // namespace HttpDocumentSource::HttpDocumentSource() : client_(std::make_unique(), source_config()) {} HttpDocumentSource::HttpDocumentSource(std::unique_ptr transport) : client_(std::move(transport), source_config()) {} std::string HttpDocumentSource::fetch(const std::string& url, std::size_t max_bytes) { if (!is_google_host_url(url)) throw GoogleError::invalid("discovery url is not an https googleapis.com url: " + url); try { clients::ClientResponse res = client_.get_blocking(url); if (!res.is_success()) throw GoogleError(res.status(), res.status() == 404 ? "NOT_FOUND" : "UNAVAILABLE", "discovery fetch failed with HTTP " + std::to_string(res.status()), "discovery"); if (res.body().size() > max_bytes) throw GoogleError(502, "BAD_DOCUMENT", "discovery document is larger than the cap", "discovery"); return res.body(); } catch (const GoogleError&) { throw; } catch (const std::exception& e) { throw GoogleError(502, "UNAVAILABLE", e.what(), "discovery"); } } std::string FileDocumentSource::fetch(const std::string& url, std::size_t max_bytes) { std::string file; if (url.rfind(kDirectoryUrl, 0) == 0) { file = "directory.json"; } else { if (index_.empty()) { std::ifstream din(dir_ + "/directory.json", std::ios::binary); if (!din) throw GoogleError(404, "NOT_FOUND", "saved directory.json missing", "discovery"); std::ostringstream dss; dss << din.rdbuf(); for (const DirectoryItem& i : Directory::parse(dss.str()).items()) index_[i.rest_url] = i.name + "_" + i.version + ".json"; } auto hit = index_.find(url); if (hit == index_.end()) throw GoogleError::invalid("no saved document for url: " + url); file = hit->second; } std::ifstream in(dir_ + "/" + file, std::ios::binary); if (!in) throw GoogleError(404, "NOT_FOUND", "saved discovery document missing: " + file, "discovery"); std::ostringstream ss; ss << in.rdbuf(); std::string text = ss.str(); if (text.size() > max_bytes) throw GoogleError(502, "BAD_DOCUMENT", "discovery document is larger than the cap", "discovery"); return text; } Catalog::Catalog(std::shared_ptr source, Clock clock) : source_(std::move(source)), clock_(std::move(clock)) {} std::chrono::steady_clock::time_point Catalog::now() const { return clock_ ? clock_() : std::chrono::steady_clock::now(); } std::shared_ptr Catalog::directory() { std::lock_guard lock(mu_); auto t = now(); if (directory_ && t - directory_at_ < DiscoveryCaps::kMaxAge) { ++stats_.hits; return directory_; } ++stats_.fetches; std::string text = source_->fetch(kDirectoryUrl, DiscoveryCaps::kMaxDirectoryBytes); directory_ = std::make_shared(Directory::parse(text)); directory_at_ = t; return directory_; } std::shared_ptr Catalog::api(const std::string& ref) { std::string name = ref, version; if (std::size_t c = ref.find(':'); c != std::string::npos) { name = ref.substr(0, c); version = ref.substr(c + 1); } if (!valid_name(name) || (!version.empty() && !valid_name(version))) throw GoogleError::invalid("api must be name or name:version, got '" + ref + "'"); std::shared_ptr dir = directory(); const DirectoryItem* item = dir->find(name, version); if (!item) throw GoogleError(404, "NOT_FOUND", "no such API in the Discovery directory: " + ref, "discovery"); std::lock_guard lock(mu_); auto t = now(); auto it = entries_.find(item->id); if (it != entries_.end()) { bool stale = t - it->second.fetched >= DiscoveryCaps::kMaxAge; auto lf = last_fetch_.find(item->id); bool floor = lf != last_fetch_.end() && t - lf->second < DiscoveryCaps::kMinRefetch; if (!stale || floor) { lru_.remove(item->id); lru_.push_front(item->id); ++stats_.hits; return it->second.api; } } ++stats_.fetches; std::string text = source_->fetch(item->rest_url, DiscoveryCaps::kMaxDocBytes); Json doc; try { doc = Json::parse(text); } catch (const std::exception& e) { throw GoogleError(502, "BAD_DOCUMENT", std::string("discovery document ") + item->id + ": " + e.what()); } Api parsed = Api::parse(doc); parsed.raw_bytes = text.size(); if (parsed.name() != item->name || parsed.version() != item->version) throw GoogleError(502, "BAD_DOCUMENT", "document " + parsed.id() + " is not the one requested: " + item->id); Entry e; e.api = std::make_shared(std::move(parsed)); e.fetched = t; e.bytes = text.size(); entries_[item->id] = e; last_fetch_[item->id] = t; lru_.remove(item->id); lru_.push_front(item->id); evict_locked(); return e.api; } void Catalog::evict_locked() { auto total = [&] { std::size_t b = 0; for (const auto& [k, v] : entries_) b += v.bytes; return b; }; while (entries_.size() > 1 && (entries_.size() > DiscoveryCaps::kMaxCachedApis || total() > DiscoveryCaps::kMaxCachedBytes)) { entries_.erase(lru_.back()); lru_.pop_back(); ++stats_.evictions; } } void Catalog::forget(const std::string& ref) { std::lock_guard lock(mu_); std::string name = ref, version; if (std::size_t c = ref.find(':'); c != std::string::npos) { name = ref.substr(0, c); version = ref.substr(c + 1); } for (auto& [id, e] : entries_) if (e.api->name() == name && (version.empty() || e.api->version() == version)) e.fetched = std::chrono::steady_clock::time_point{}; } CatalogStats Catalog::stats() const { std::lock_guard lock(mu_); CatalogStats s = stats_; s.cached = entries_.size(); for (const auto& [k, v] : entries_) s.cached_bytes += v.bytes; return s; } } // namespace molo::google::discovery