Files
moloapp/src/google/data/storage_media.cpp
T

232 lines
10 KiB
C++
Raw Normal View History

#include "google/data/storage_media.h"
#include "clients/transports/http_transport.h"
#include "clients/url.h"
#include "google/credentials.h"
#include "google/data/caps.h"
#include "google/data/ids.h"
namespace molo::google::data {
using clients::body::Json;
namespace {
clients::ClientConfig config() {
clients::ClientConfig c;
c.persist_cookies = false;
return c;
}
std::string enc(const std::string& s) { return clients::Url::encode_component(s); }
void check_type(const std::string& t) {
if (t.empty() || t.size() > 128) throw GoogleError::invalid("content type must be 1..128 chars");
for (char c : t)
if (static_cast<unsigned char>(c) < 0x20 || c == 0x7f)
throw GoogleError::invalid("content type has control characters");
}
Json parse(const clients::ClientResponse& r) {
try {
return r.json();
} catch (const std::exception& e) {
throw GoogleError(502, "BAD_RESPONSE", e.what(), "storage");
}
}
} // namespace
StorageMedia::StorageMedia(std::unique_ptr<clients::Transport> transport,
std::shared_ptr<clients::Auth> auth, std::string host_url)
: client_(std::make_unique<clients::Client>(std::move(transport), config())),
auth_(std::move(auth)),
host_(std::move(host_url)) {}
std::shared_ptr<StorageMedia> StorageMedia::from_env() {
std::shared_ptr<clients::Auth> auth = Credentials::oauth_from_env();
if (!auth) auth = Credentials::api_key_from_env();
return std::make_shared<StorageMedia>(std::make_unique<clients::transports::HttpTransport>(),
auth);
}
std::string StorageMedia::download_url(const std::string& bucket, const std::string& object,
long long generation) const {
if (generation < 0) throw GoogleError::invalid("generation must be >= 0");
std::string u = host_ + "/storage/v1/b/" + enc(Ids::bucket(bucket)) + "/o/" +
enc(Ids::object(object)) + "?alt=media";
if (generation > 0) u += "&generation=" + std::to_string(generation);
return u;
}
std::string StorageMedia::upload_url(const std::string& bucket, const std::string& object,
const char* upload_type) const {
return host_ + "/upload/storage/v1/b/" + enc(Ids::bucket(bucket)) + "/o?uploadType=" +
upload_type + "&name=" + enc(Ids::object(object));
}
std::string StorageMedia::check_session(const std::string& url) {
const std::string prefix = std::string(kHost) + "/upload/storage/v1/b/";
const std::string alt = std::string(kHost) + "/resumable/upload/storage/v1/b/";
if ((url.rfind(prefix, 0) != 0 && url.rfind(alt, 0) != 0) || url.size() > 2048)
throw GoogleError::invalid("session url must be on " + std::string(kHost) +
"/upload/storage/v1/b/");
for (char c : url)
if (static_cast<unsigned char>(c) <= 0x20 || c == 0x7f)
throw GoogleError::invalid("session url has control characters or spaces");
return url;
}
clients::ClientResponse StorageMedia::send(clients::ClientRequest request) const {
if (!auth_) throw NotConfiguredError("storage");
try {
auth_->apply(request);
return client_->send_blocking(std::move(request));
} catch (const GoogleError&) {
throw;
} catch (const std::exception& e) {
throw GoogleError(502, "UNAVAILABLE", e.what(), "storage");
}
}
void StorageMedia::fail(const clients::ClientResponse& r) const {
throw error_from_response("storage", r);
}
Downloaded StorageMedia::download(const std::string& bucket, const std::string& object,
long long generation) const {
clients::ClientRequest req(clients::Method::kGet,
clients::Url::parse(download_url(bucket, object, generation)));
clients::ClientResponse r = send(std::move(req));
if (!r.is_success()) fail(r);
if (r.body().size() > DataCaps::kMaxDownloadBytes)
throw GoogleError(502, "RESPONSE_TOO_LARGE",
"object is larger than the " + std::to_string(DataCaps::kMaxDownloadBytes) +
" byte download cap; use download_range",
"storage");
Downloaded d;
d.data = r.body();
d.content_type = r.content_type();
d.generation = r.headers().get("x-goog-generation").value_or("");
return d;
}
Downloaded StorageMedia::download_range(const std::string& bucket, const std::string& object,
long long first, long long last) const {
if (first < 0 || last < first ||
last - first + 1 > static_cast<long long>(DataCaps::kMaxDownloadBytes))
throw GoogleError::invalid("range must satisfy 0 <= first <= last and span <= " +
std::to_string(DataCaps::kMaxDownloadBytes) + " bytes");
clients::ClientRequest req(clients::Method::kGet,
clients::Url::parse(download_url(bucket, object, 0)));
req.header("Range", "bytes=" + std::to_string(first) + "-" + std::to_string(last));
clients::ClientResponse r = send(std::move(req));
if (!r.is_success()) fail(r);
if (r.body().size() > DataCaps::kMaxDownloadBytes)
throw GoogleError(502, "RESPONSE_TOO_LARGE", "range response over the cap", "storage");
Downloaded d;
d.data = r.body();
d.content_type = r.content_type();
d.generation = r.headers().get("x-goog-generation").value_or("");
return d;
}
Json StorageMedia::upload_simple(const std::string& bucket, const std::string& object,
const std::string& data, const std::string& content_type) const {
if (data.size() > DataCaps::kMaxSimpleUploadBytes)
throw GoogleError::invalid("simple upload over " +
std::to_string(DataCaps::kMaxSimpleUploadBytes) +
" bytes; use the resumable upload");
check_type(content_type);
clients::ClientRequest req(clients::Method::kPost,
clients::Url::parse(upload_url(bucket, object, "media")));
req.bytes(data, content_type);
clients::ClientResponse r = send(std::move(req));
if (!r.is_success()) fail(r);
return parse(r);
}
std::string StorageMedia::resumable_start(const std::string& bucket, const std::string& object,
const std::string& content_type, long long total,
const Json& metadata) const {
if (total < 0 || total > static_cast<long long>(DataCaps::kMaxResumableTotal))
throw GoogleError::invalid("total must be 0.." + std::to_string(DataCaps::kMaxResumableTotal));
check_type(content_type);
if (!metadata.is_null() && !metadata.is_object())
throw GoogleError::invalid("metadata must be a JSON object");
clients::ClientRequest req(clients::Method::kPost,
clients::Url::parse(upload_url(bucket, object, "resumable")));
req.header("X-Upload-Content-Type", content_type);
req.header("X-Upload-Content-Length", std::to_string(total));
req.json(metadata.is_object() ? metadata : Json::object());
clients::ClientResponse r = send(std::move(req));
if (!r.is_success()) fail(r);
std::string loc = r.headers().get("location").value_or("");
if (loc.empty())
throw GoogleError(502, "BAD_RESPONSE", "resumable start returned no Location", "storage");
return check_session(loc);
}
ResumableProgress StorageMedia::resumable_put(const std::string& session_url, long long offset,
const std::string& chunk, long long total) const {
check_session(session_url);
if (total < 0 || total > static_cast<long long>(DataCaps::kMaxResumableTotal))
throw GoogleError::invalid("total out of range");
if (offset < 0 || chunk.empty() || chunk.size() > DataCaps::kMaxResumableChunk)
throw GoogleError::invalid("chunk must be 1.." + std::to_string(DataCaps::kMaxResumableChunk) +
" bytes at a non-negative offset");
long long end = offset + static_cast<long long>(chunk.size());
if (end > total) throw GoogleError::invalid("chunk runs past total");
if (end < total && chunk.size() % DataCaps::kChunkGranularity != 0)
throw GoogleError::invalid("a chunk that does not end the object must be a multiple of 256 KiB");
clients::ClientRequest req(clients::Method::kPut, clients::Url::parse(session_url));
req.header("Content-Range", "bytes " + std::to_string(offset) + "-" + std::to_string(end - 1) +
"/" + std::to_string(total));
req.bytes(chunk, "application/octet-stream");
clients::ClientResponse r = send(std::move(req));
ResumableProgress p;
if (r.status() == 308) {
std::string range = r.headers().get("range").value_or(""); // "bytes=0-N"
auto dash = range.find('-');
p.next_offset = dash == std::string::npos ? 0 : std::stoll(range.substr(dash + 1)) + 1;
return p;
}
if (!r.is_success()) fail(r);
p.done = true;
p.next_offset = total;
p.object = parse(r);
return p;
}
ResumableProgress StorageMedia::resumable_status(const std::string& session_url,
long long total) const {
check_session(session_url);
if (total < 0 || total > static_cast<long long>(DataCaps::kMaxResumableTotal))
throw GoogleError::invalid("total out of range");
clients::ClientRequest req(clients::Method::kPut, clients::Url::parse(session_url));
req.header("Content-Range", "bytes */" + std::to_string(total));
req.bytes("", "application/octet-stream");
clients::ClientResponse r = send(std::move(req));
ResumableProgress p;
if (r.status() == 308) {
std::string range = r.headers().get("range").value_or("");
auto dash = range.find('-');
p.next_offset = dash == std::string::npos ? 0 : std::stoll(range.substr(dash + 1)) + 1;
return p;
}
if (!r.is_success()) fail(r);
p.done = true;
p.next_offset = total;
p.object = parse(r);
return p;
}
void StorageMedia::resumable_cancel(const std::string& session_url) const {
check_session(session_url);
clients::ClientRequest req(clients::Method::kDelete, clients::Url::parse(session_url));
clients::ClientResponse r = send(std::move(req));
if (r.status() != 499 && !r.is_success()) fail(r);
}
} // namespace molo::google::data