#include "google/data/storage_media.h" #include "clients/transports/http_transport.h" #include "clients/url.h" #include "google/credentials.h" #include "google/data/caps.h" #include "google/data/ids.h" namespace molo::google::data { using clients::body::Json; namespace { clients::ClientConfig config() { clients::ClientConfig c; c.persist_cookies = false; return c; } std::string enc(const std::string& s) { return clients::Url::encode_component(s); } void check_type(const std::string& t) { if (t.empty() || t.size() > 128) throw GoogleError::invalid("content type must be 1..128 chars"); for (char c : t) if (static_cast(c) < 0x20 || c == 0x7f) throw GoogleError::invalid("content type has control characters"); } Json parse(const clients::ClientResponse& r) { try { return r.json(); } catch (const std::exception& e) { throw GoogleError(502, "BAD_RESPONSE", e.what(), "storage"); } } } // namespace StorageMedia::StorageMedia(std::unique_ptr transport, std::shared_ptr auth, std::string host_url) : client_(std::make_unique(std::move(transport), config())), auth_(std::move(auth)), host_(std::move(host_url)) {} std::shared_ptr StorageMedia::from_env() { std::shared_ptr auth = Credentials::oauth_from_env(); if (!auth) auth = Credentials::api_key_from_env(); return std::make_shared(std::make_unique(), auth); } std::string StorageMedia::download_url(const std::string& bucket, const std::string& object, long long generation) const { if (generation < 0) throw GoogleError::invalid("generation must be >= 0"); std::string u = host_ + "/storage/v1/b/" + enc(Ids::bucket(bucket)) + "/o/" + enc(Ids::object(object)) + "?alt=media"; if (generation > 0) u += "&generation=" + std::to_string(generation); return u; } std::string StorageMedia::upload_url(const std::string& bucket, const std::string& object, const char* upload_type) const { return host_ + "/upload/storage/v1/b/" + enc(Ids::bucket(bucket)) + "/o?uploadType=" + upload_type + "&name=" + enc(Ids::object(object)); } std::string StorageMedia::check_session(const std::string& url) { const std::string prefix = std::string(kHost) + "/upload/storage/v1/b/"; const std::string alt = std::string(kHost) + "/resumable/upload/storage/v1/b/"; if ((url.rfind(prefix, 0) != 0 && url.rfind(alt, 0) != 0) || url.size() > 2048) throw GoogleError::invalid("session url must be on " + std::string(kHost) + "/upload/storage/v1/b/"); for (char c : url) if (static_cast(c) <= 0x20 || c == 0x7f) throw GoogleError::invalid("session url has control characters or spaces"); return url; } clients::ClientResponse StorageMedia::send(clients::ClientRequest request) const { if (!auth_) throw NotConfiguredError("storage"); try { auth_->apply(request); return client_->send_blocking(std::move(request)); } catch (const GoogleError&) { throw; } catch (const std::exception& e) { throw GoogleError(502, "UNAVAILABLE", e.what(), "storage"); } } void StorageMedia::fail(const clients::ClientResponse& r) const { throw error_from_response("storage", r); } Downloaded StorageMedia::download(const std::string& bucket, const std::string& object, long long generation) const { clients::ClientRequest req(clients::Method::kGet, clients::Url::parse(download_url(bucket, object, generation))); clients::ClientResponse r = send(std::move(req)); if (!r.is_success()) fail(r); if (r.body().size() > DataCaps::kMaxDownloadBytes) throw GoogleError(502, "RESPONSE_TOO_LARGE", "object is larger than the " + std::to_string(DataCaps::kMaxDownloadBytes) + " byte download cap; use download_range", "storage"); Downloaded d; d.data = r.body(); d.content_type = r.content_type(); d.generation = r.headers().get("x-goog-generation").value_or(""); return d; } Downloaded StorageMedia::download_range(const std::string& bucket, const std::string& object, long long first, long long last) const { if (first < 0 || last < first || last - first + 1 > static_cast(DataCaps::kMaxDownloadBytes)) throw GoogleError::invalid("range must satisfy 0 <= first <= last and span <= " + std::to_string(DataCaps::kMaxDownloadBytes) + " bytes"); clients::ClientRequest req(clients::Method::kGet, clients::Url::parse(download_url(bucket, object, 0))); req.header("Range", "bytes=" + std::to_string(first) + "-" + std::to_string(last)); clients::ClientResponse r = send(std::move(req)); if (!r.is_success()) fail(r); if (r.body().size() > DataCaps::kMaxDownloadBytes) throw GoogleError(502, "RESPONSE_TOO_LARGE", "range response over the cap", "storage"); Downloaded d; d.data = r.body(); d.content_type = r.content_type(); d.generation = r.headers().get("x-goog-generation").value_or(""); return d; } Json StorageMedia::upload_simple(const std::string& bucket, const std::string& object, const std::string& data, const std::string& content_type) const { if (data.size() > DataCaps::kMaxSimpleUploadBytes) throw GoogleError::invalid("simple upload over " + std::to_string(DataCaps::kMaxSimpleUploadBytes) + " bytes; use the resumable upload"); check_type(content_type); clients::ClientRequest req(clients::Method::kPost, clients::Url::parse(upload_url(bucket, object, "media"))); req.bytes(data, content_type); clients::ClientResponse r = send(std::move(req)); if (!r.is_success()) fail(r); return parse(r); } std::string StorageMedia::resumable_start(const std::string& bucket, const std::string& object, const std::string& content_type, long long total, const Json& metadata) const { if (total < 0 || total > static_cast(DataCaps::kMaxResumableTotal)) throw GoogleError::invalid("total must be 0.." + std::to_string(DataCaps::kMaxResumableTotal)); check_type(content_type); if (!metadata.is_null() && !metadata.is_object()) throw GoogleError::invalid("metadata must be a JSON object"); clients::ClientRequest req(clients::Method::kPost, clients::Url::parse(upload_url(bucket, object, "resumable"))); req.header("X-Upload-Content-Type", content_type); req.header("X-Upload-Content-Length", std::to_string(total)); req.json(metadata.is_object() ? metadata : Json::object()); clients::ClientResponse r = send(std::move(req)); if (!r.is_success()) fail(r); std::string loc = r.headers().get("location").value_or(""); if (loc.empty()) throw GoogleError(502, "BAD_RESPONSE", "resumable start returned no Location", "storage"); return check_session(loc); } ResumableProgress StorageMedia::resumable_put(const std::string& session_url, long long offset, const std::string& chunk, long long total) const { check_session(session_url); if (total < 0 || total > static_cast(DataCaps::kMaxResumableTotal)) throw GoogleError::invalid("total out of range"); if (offset < 0 || chunk.empty() || chunk.size() > DataCaps::kMaxResumableChunk) throw GoogleError::invalid("chunk must be 1.." + std::to_string(DataCaps::kMaxResumableChunk) + " bytes at a non-negative offset"); long long end = offset + static_cast(chunk.size()); if (end > total) throw GoogleError::invalid("chunk runs past total"); if (end < total && chunk.size() % DataCaps::kChunkGranularity != 0) throw GoogleError::invalid("a chunk that does not end the object must be a multiple of 256 KiB"); clients::ClientRequest req(clients::Method::kPut, clients::Url::parse(session_url)); req.header("Content-Range", "bytes " + std::to_string(offset) + "-" + std::to_string(end - 1) + "/" + std::to_string(total)); req.bytes(chunk, "application/octet-stream"); clients::ClientResponse r = send(std::move(req)); ResumableProgress p; if (r.status() == 308) { std::string range = r.headers().get("range").value_or(""); // "bytes=0-N" auto dash = range.find('-'); p.next_offset = dash == std::string::npos ? 0 : std::stoll(range.substr(dash + 1)) + 1; return p; } if (!r.is_success()) fail(r); p.done = true; p.next_offset = total; p.object = parse(r); return p; } ResumableProgress StorageMedia::resumable_status(const std::string& session_url, long long total) const { check_session(session_url); if (total < 0 || total > static_cast(DataCaps::kMaxResumableTotal)) throw GoogleError::invalid("total out of range"); clients::ClientRequest req(clients::Method::kPut, clients::Url::parse(session_url)); req.header("Content-Range", "bytes */" + std::to_string(total)); req.bytes("", "application/octet-stream"); clients::ClientResponse r = send(std::move(req)); ResumableProgress p; if (r.status() == 308) { std::string range = r.headers().get("range").value_or(""); auto dash = range.find('-'); p.next_offset = dash == std::string::npos ? 0 : std::stoll(range.substr(dash + 1)) + 1; return p; } if (!r.is_success()) fail(r); p.done = true; p.next_offset = total; p.object = parse(r); return p; } void StorageMedia::resumable_cancel(const std::string& session_url) const { check_session(session_url); clients::ClientRequest req(clients::Method::kDelete, clients::Url::parse(session_url)); clients::ClientResponse r = send(std::move(req)); if (r.status() != 499 && !r.is_success()) fail(r); } } // namespace molo::google::data