Files
moloapp/src/google/discovery/catalog.cpp
T

286 lines
9.9 KiB
C++
Raw Normal View History

#include "google/discovery/catalog.h"
#include <algorithm>
#include <cctype>
#include <fstream>
#include <sstream>
#include "clients/transports/http_transport.h"
#include "google/discovery/caps.h"
#include "google/errors.h"
namespace molo::google::discovery {
using clients::body::Json;
const char* const kDirectoryUrl = "https://discovery.googleapis.com/discovery/v1/apis";
namespace {
std::string str(const Json& o, const char* key) {
const Json* v = o.find(key);
return v && v->is_string() ? v->as_string() : "";
}
std::string lower(std::string s) {
for (char& c : s) c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
return s;
}
bool valid_name(const std::string& s) {
if (s.empty() || s.size() > 64) return false;
for (char c : s)
if (!(std::isalnum(static_cast<unsigned char>(c)) || c == '_' || c == '-' || c == '.')) return false;
return true;
}
} // namespace
Json DirectoryItem::to_json() const {
Json o = Json::object();
o.set("id", Json(id));
o.set("name", Json(name));
o.set("version", Json(version));
o.set("title", Json(title));
o.set("description", Json(description));
o.set("discoveryRestUrl", Json(rest_url));
o.set("documentationLink", Json(documentation));
o.set("preferred", Json(preferred));
return o;
}
Directory Directory::parse(const std::string& text) {
if (text.size() > DiscoveryCaps::kMaxDirectoryBytes)
throw GoogleError(502, "BAD_DOCUMENT", "discovery directory is larger than the cap");
Json doc;
try {
doc = Json::parse(text);
} catch (const std::exception& e) {
throw GoogleError(502, "BAD_DOCUMENT", std::string("discovery directory: ") + e.what());
}
const Json* items = doc.is_object() ? doc.find("items") : nullptr;
if (!items || !items->is_array() || str(doc, "kind") != "discovery#directoryList")
throw GoogleError(502, "BAD_DOCUMENT", "discovery directory: not a directoryList");
if (items->as_array().size() > DiscoveryCaps::kMaxDirectoryItems)
throw GoogleError(502, "BAD_DOCUMENT", "discovery directory: more items than the cap");
Directory dir;
for (const Json& it : items->as_array()) {
DirectoryItem row;
row.id = str(it, "id");
row.name = str(it, "name");
row.version = str(it, "version");
row.title = str(it, "title");
row.description = str(it, "description");
row.rest_url = str(it, "discoveryRestUrl");
row.documentation = str(it, "documentationLink");
const Json* p = it.find("preferred");
row.preferred = p && p->is_bool() && p->as_bool();
if (row.id.empty() || row.name.empty() || row.version.empty() ||
!is_google_host_url(row.rest_url))
throw GoogleError(502, "BAD_DOCUMENT", "discovery directory: bad item " + row.id);
dir.items_.push_back(std::move(row));
}
return dir;
}
std::size_t Directory::preferred_count() const {
return static_cast<std::size_t>(std::count_if(items_.begin(), items_.end(),
[](const DirectoryItem& i) { return i.preferred; }));
}
const DirectoryItem* Directory::find(const std::string& name, const std::string& version) const {
const DirectoryItem* fallback = nullptr;
for (const DirectoryItem& i : items_) {
if (i.name != name) continue;
if (!version.empty()) {
if (i.version == version) return &i;
} else if (i.preferred) {
return &i;
} else if (!fallback) {
fallback = &i;
}
}
return version.empty() ? fallback : nullptr;
}
std::vector<const DirectoryItem*> Directory::search(const std::string& needle, std::size_t limit,
bool preferred_only) const {
std::vector<const DirectoryItem*> out;
std::string n = lower(needle);
limit = std::min(limit, DiscoveryCaps::kMaxListed);
for (const DirectoryItem& i : items_) {
if (out.size() >= limit) break;
if (preferred_only && !i.preferred) continue;
if (n.empty() || lower(i.name).find(n) != std::string::npos ||
lower(i.title).find(n) != std::string::npos)
out.push_back(&i);
}
return out;
}
namespace {
clients::ClientConfig source_config() {
clients::ClientConfig c;
c.auth = nullptr;
c.persist_cookies = false;
return c;
}
} // namespace
HttpDocumentSource::HttpDocumentSource()
: client_(std::make_unique<clients::transports::HttpTransport>(), source_config()) {}
HttpDocumentSource::HttpDocumentSource(std::unique_ptr<clients::Transport> transport)
: client_(std::move(transport), source_config()) {}
std::string HttpDocumentSource::fetch(const std::string& url, std::size_t max_bytes) {
if (!is_google_host_url(url))
throw GoogleError::invalid("discovery url is not an https googleapis.com url: " + url);
try {
clients::ClientResponse res = client_.get_blocking(url);
if (!res.is_success())
throw GoogleError(res.status(), res.status() == 404 ? "NOT_FOUND" : "UNAVAILABLE",
"discovery fetch failed with HTTP " + std::to_string(res.status()), "discovery");
if (res.body().size() > max_bytes)
throw GoogleError(502, "BAD_DOCUMENT", "discovery document is larger than the cap", "discovery");
return res.body();
} catch (const GoogleError&) {
throw;
} catch (const std::exception& e) {
throw GoogleError(502, "UNAVAILABLE", e.what(), "discovery");
}
}
std::string FileDocumentSource::fetch(const std::string& url, std::size_t max_bytes) {
std::string file;
if (url.rfind(kDirectoryUrl, 0) == 0) {
file = "directory.json";
} else {
if (index_.empty()) {
std::ifstream din(dir_ + "/directory.json", std::ios::binary);
if (!din) throw GoogleError(404, "NOT_FOUND", "saved directory.json missing", "discovery");
std::ostringstream dss;
dss << din.rdbuf();
for (const DirectoryItem& i : Directory::parse(dss.str()).items())
index_[i.rest_url] = i.name + "_" + i.version + ".json";
}
auto hit = index_.find(url);
if (hit == index_.end()) throw GoogleError::invalid("no saved document for url: " + url);
file = hit->second;
}
std::ifstream in(dir_ + "/" + file, std::ios::binary);
if (!in) throw GoogleError(404, "NOT_FOUND", "saved discovery document missing: " + file, "discovery");
std::ostringstream ss;
ss << in.rdbuf();
std::string text = ss.str();
if (text.size() > max_bytes)
throw GoogleError(502, "BAD_DOCUMENT", "discovery document is larger than the cap", "discovery");
return text;
}
Catalog::Catalog(std::shared_ptr<DocumentSource> source, Clock clock)
: source_(std::move(source)), clock_(std::move(clock)) {}
std::chrono::steady_clock::time_point Catalog::now() const {
return clock_ ? clock_() : std::chrono::steady_clock::now();
}
std::shared_ptr<const Directory> Catalog::directory() {
std::lock_guard<std::mutex> lock(mu_);
auto t = now();
if (directory_ && t - directory_at_ < DiscoveryCaps::kMaxAge) {
++stats_.hits;
return directory_;
}
++stats_.fetches;
std::string text = source_->fetch(kDirectoryUrl, DiscoveryCaps::kMaxDirectoryBytes);
directory_ = std::make_shared<const Directory>(Directory::parse(text));
directory_at_ = t;
return directory_;
}
std::shared_ptr<const Api> Catalog::api(const std::string& ref) {
std::string name = ref, version;
if (std::size_t c = ref.find(':'); c != std::string::npos) {
name = ref.substr(0, c);
version = ref.substr(c + 1);
}
if (!valid_name(name) || (!version.empty() && !valid_name(version)))
throw GoogleError::invalid("api must be name or name:version, got '" + ref + "'");
std::shared_ptr<const Directory> dir = directory();
const DirectoryItem* item = dir->find(name, version);
if (!item) throw GoogleError(404, "NOT_FOUND", "no such API in the Discovery directory: " + ref, "discovery");
std::lock_guard<std::mutex> lock(mu_);
auto t = now();
auto it = entries_.find(item->id);
if (it != entries_.end()) {
bool stale = t - it->second.fetched >= DiscoveryCaps::kMaxAge;
auto lf = last_fetch_.find(item->id);
bool floor = lf != last_fetch_.end() && t - lf->second < DiscoveryCaps::kMinRefetch;
if (!stale || floor) {
lru_.remove(item->id);
lru_.push_front(item->id);
++stats_.hits;
return it->second.api;
}
}
++stats_.fetches;
std::string text = source_->fetch(item->rest_url, DiscoveryCaps::kMaxDocBytes);
Json doc;
try {
doc = Json::parse(text);
} catch (const std::exception& e) {
throw GoogleError(502, "BAD_DOCUMENT", std::string("discovery document ") + item->id + ": " + e.what());
}
Api parsed = Api::parse(doc);
parsed.raw_bytes = text.size();
if (parsed.name() != item->name || parsed.version() != item->version)
throw GoogleError(502, "BAD_DOCUMENT", "document " + parsed.id() + " is not the one requested: " + item->id);
Entry e;
e.api = std::make_shared<const Api>(std::move(parsed));
e.fetched = t;
e.bytes = text.size();
entries_[item->id] = e;
last_fetch_[item->id] = t;
lru_.remove(item->id);
lru_.push_front(item->id);
evict_locked();
return e.api;
}
void Catalog::evict_locked() {
auto total = [&] {
std::size_t b = 0;
for (const auto& [k, v] : entries_) b += v.bytes;
return b;
};
while (entries_.size() > 1 &&
(entries_.size() > DiscoveryCaps::kMaxCachedApis || total() > DiscoveryCaps::kMaxCachedBytes)) {
entries_.erase(lru_.back());
lru_.pop_back();
++stats_.evictions;
}
}
void Catalog::forget(const std::string& ref) {
std::lock_guard<std::mutex> lock(mu_);
std::string name = ref, version;
if (std::size_t c = ref.find(':'); c != std::string::npos) {
name = ref.substr(0, c);
version = ref.substr(c + 1);
}
for (auto& [id, e] : entries_)
if (e.api->name() == name && (version.empty() || e.api->version() == version))
e.fetched = std::chrono::steady_clock::time_point{};
}
CatalogStats Catalog::stats() const {
std::lock_guard<std::mutex> lock(mu_);
CatalogStats s = stats_;
s.cached = entries_.size();
for (const auto& [k, v] : entries_) s.cached_bytes += v.bytes;
return s;
}
} // namespace molo::google::discovery