Discovery generated call table (918 rows), typed services, /google/data demo. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
232 lines
10 KiB
C++
232 lines
10 KiB
C++
#include "google/data/storage_media.h"
|
|
|
|
#include "clients/transports/http_transport.h"
|
|
#include "clients/url.h"
|
|
#include "google/credentials.h"
|
|
#include "google/data/caps.h"
|
|
#include "google/data/ids.h"
|
|
|
|
namespace molo::google::data {
|
|
|
|
using clients::body::Json;
|
|
|
|
namespace {
|
|
|
|
clients::ClientConfig config() {
|
|
clients::ClientConfig c;
|
|
c.persist_cookies = false;
|
|
return c;
|
|
}
|
|
|
|
std::string enc(const std::string& s) { return clients::Url::encode_component(s); }
|
|
|
|
void check_type(const std::string& t) {
|
|
if (t.empty() || t.size() > 128) throw GoogleError::invalid("content type must be 1..128 chars");
|
|
for (char c : t)
|
|
if (static_cast<unsigned char>(c) < 0x20 || c == 0x7f)
|
|
throw GoogleError::invalid("content type has control characters");
|
|
}
|
|
|
|
Json parse(const clients::ClientResponse& r) {
|
|
try {
|
|
return r.json();
|
|
} catch (const std::exception& e) {
|
|
throw GoogleError(502, "BAD_RESPONSE", e.what(), "storage");
|
|
}
|
|
}
|
|
|
|
} // namespace
|
|
|
|
StorageMedia::StorageMedia(std::unique_ptr<clients::Transport> transport,
|
|
std::shared_ptr<clients::Auth> auth, std::string host_url)
|
|
: client_(std::make_unique<clients::Client>(std::move(transport), config())),
|
|
auth_(std::move(auth)),
|
|
host_(std::move(host_url)) {}
|
|
|
|
std::shared_ptr<StorageMedia> StorageMedia::from_env() {
|
|
std::shared_ptr<clients::Auth> auth = Credentials::oauth_from_env();
|
|
if (!auth) auth = Credentials::api_key_from_env();
|
|
return std::make_shared<StorageMedia>(std::make_unique<clients::transports::HttpTransport>(),
|
|
auth);
|
|
}
|
|
|
|
std::string StorageMedia::download_url(const std::string& bucket, const std::string& object,
|
|
long long generation) const {
|
|
if (generation < 0) throw GoogleError::invalid("generation must be >= 0");
|
|
std::string u = host_ + "/storage/v1/b/" + enc(Ids::bucket(bucket)) + "/o/" +
|
|
enc(Ids::object(object)) + "?alt=media";
|
|
if (generation > 0) u += "&generation=" + std::to_string(generation);
|
|
return u;
|
|
}
|
|
|
|
std::string StorageMedia::upload_url(const std::string& bucket, const std::string& object,
|
|
const char* upload_type) const {
|
|
return host_ + "/upload/storage/v1/b/" + enc(Ids::bucket(bucket)) + "/o?uploadType=" +
|
|
upload_type + "&name=" + enc(Ids::object(object));
|
|
}
|
|
|
|
std::string StorageMedia::check_session(const std::string& url) {
|
|
const std::string prefix = std::string(kHost) + "/upload/storage/v1/b/";
|
|
const std::string alt = std::string(kHost) + "/resumable/upload/storage/v1/b/";
|
|
if ((url.rfind(prefix, 0) != 0 && url.rfind(alt, 0) != 0) || url.size() > 2048)
|
|
throw GoogleError::invalid("session url must be on " + std::string(kHost) +
|
|
"/upload/storage/v1/b/");
|
|
for (char c : url)
|
|
if (static_cast<unsigned char>(c) <= 0x20 || c == 0x7f)
|
|
throw GoogleError::invalid("session url has control characters or spaces");
|
|
return url;
|
|
}
|
|
|
|
clients::ClientResponse StorageMedia::send(clients::ClientRequest request) const {
|
|
if (!auth_) throw NotConfiguredError("storage");
|
|
try {
|
|
auth_->apply(request);
|
|
return client_->send_blocking(std::move(request));
|
|
} catch (const GoogleError&) {
|
|
throw;
|
|
} catch (const std::exception& e) {
|
|
throw GoogleError(502, "UNAVAILABLE", e.what(), "storage");
|
|
}
|
|
}
|
|
|
|
void StorageMedia::fail(const clients::ClientResponse& r) const {
|
|
throw error_from_response("storage", r);
|
|
}
|
|
|
|
Downloaded StorageMedia::download(const std::string& bucket, const std::string& object,
|
|
long long generation) const {
|
|
clients::ClientRequest req(clients::Method::kGet,
|
|
clients::Url::parse(download_url(bucket, object, generation)));
|
|
clients::ClientResponse r = send(std::move(req));
|
|
if (!r.is_success()) fail(r);
|
|
if (r.body().size() > DataCaps::kMaxDownloadBytes)
|
|
throw GoogleError(502, "RESPONSE_TOO_LARGE",
|
|
"object is larger than the " + std::to_string(DataCaps::kMaxDownloadBytes) +
|
|
" byte download cap; use download_range",
|
|
"storage");
|
|
Downloaded d;
|
|
d.data = r.body();
|
|
d.content_type = r.content_type();
|
|
d.generation = r.headers().get("x-goog-generation").value_or("");
|
|
return d;
|
|
}
|
|
|
|
Downloaded StorageMedia::download_range(const std::string& bucket, const std::string& object,
|
|
long long first, long long last) const {
|
|
if (first < 0 || last < first ||
|
|
last - first + 1 > static_cast<long long>(DataCaps::kMaxDownloadBytes))
|
|
throw GoogleError::invalid("range must satisfy 0 <= first <= last and span <= " +
|
|
std::to_string(DataCaps::kMaxDownloadBytes) + " bytes");
|
|
clients::ClientRequest req(clients::Method::kGet,
|
|
clients::Url::parse(download_url(bucket, object, 0)));
|
|
req.header("Range", "bytes=" + std::to_string(first) + "-" + std::to_string(last));
|
|
clients::ClientResponse r = send(std::move(req));
|
|
if (!r.is_success()) fail(r);
|
|
if (r.body().size() > DataCaps::kMaxDownloadBytes)
|
|
throw GoogleError(502, "RESPONSE_TOO_LARGE", "range response over the cap", "storage");
|
|
Downloaded d;
|
|
d.data = r.body();
|
|
d.content_type = r.content_type();
|
|
d.generation = r.headers().get("x-goog-generation").value_or("");
|
|
return d;
|
|
}
|
|
|
|
Json StorageMedia::upload_simple(const std::string& bucket, const std::string& object,
|
|
const std::string& data, const std::string& content_type) const {
|
|
if (data.size() > DataCaps::kMaxSimpleUploadBytes)
|
|
throw GoogleError::invalid("simple upload over " +
|
|
std::to_string(DataCaps::kMaxSimpleUploadBytes) +
|
|
" bytes; use the resumable upload");
|
|
check_type(content_type);
|
|
clients::ClientRequest req(clients::Method::kPost,
|
|
clients::Url::parse(upload_url(bucket, object, "media")));
|
|
req.bytes(data, content_type);
|
|
clients::ClientResponse r = send(std::move(req));
|
|
if (!r.is_success()) fail(r);
|
|
return parse(r);
|
|
}
|
|
|
|
std::string StorageMedia::resumable_start(const std::string& bucket, const std::string& object,
|
|
const std::string& content_type, long long total,
|
|
const Json& metadata) const {
|
|
if (total < 0 || total > static_cast<long long>(DataCaps::kMaxResumableTotal))
|
|
throw GoogleError::invalid("total must be 0.." + std::to_string(DataCaps::kMaxResumableTotal));
|
|
check_type(content_type);
|
|
if (!metadata.is_null() && !metadata.is_object())
|
|
throw GoogleError::invalid("metadata must be a JSON object");
|
|
clients::ClientRequest req(clients::Method::kPost,
|
|
clients::Url::parse(upload_url(bucket, object, "resumable")));
|
|
req.header("X-Upload-Content-Type", content_type);
|
|
req.header("X-Upload-Content-Length", std::to_string(total));
|
|
req.json(metadata.is_object() ? metadata : Json::object());
|
|
clients::ClientResponse r = send(std::move(req));
|
|
if (!r.is_success()) fail(r);
|
|
std::string loc = r.headers().get("location").value_or("");
|
|
if (loc.empty())
|
|
throw GoogleError(502, "BAD_RESPONSE", "resumable start returned no Location", "storage");
|
|
return check_session(loc);
|
|
}
|
|
|
|
ResumableProgress StorageMedia::resumable_put(const std::string& session_url, long long offset,
|
|
const std::string& chunk, long long total) const {
|
|
check_session(session_url);
|
|
if (total < 0 || total > static_cast<long long>(DataCaps::kMaxResumableTotal))
|
|
throw GoogleError::invalid("total out of range");
|
|
if (offset < 0 || chunk.empty() || chunk.size() > DataCaps::kMaxResumableChunk)
|
|
throw GoogleError::invalid("chunk must be 1.." + std::to_string(DataCaps::kMaxResumableChunk) +
|
|
" bytes at a non-negative offset");
|
|
long long end = offset + static_cast<long long>(chunk.size());
|
|
if (end > total) throw GoogleError::invalid("chunk runs past total");
|
|
if (end < total && chunk.size() % DataCaps::kChunkGranularity != 0)
|
|
throw GoogleError::invalid("a chunk that does not end the object must be a multiple of 256 KiB");
|
|
clients::ClientRequest req(clients::Method::kPut, clients::Url::parse(session_url));
|
|
req.header("Content-Range", "bytes " + std::to_string(offset) + "-" + std::to_string(end - 1) +
|
|
"/" + std::to_string(total));
|
|
req.bytes(chunk, "application/octet-stream");
|
|
clients::ClientResponse r = send(std::move(req));
|
|
ResumableProgress p;
|
|
if (r.status() == 308) {
|
|
std::string range = r.headers().get("range").value_or(""); // "bytes=0-N"
|
|
auto dash = range.find('-');
|
|
p.next_offset = dash == std::string::npos ? 0 : std::stoll(range.substr(dash + 1)) + 1;
|
|
return p;
|
|
}
|
|
if (!r.is_success()) fail(r);
|
|
p.done = true;
|
|
p.next_offset = total;
|
|
p.object = parse(r);
|
|
return p;
|
|
}
|
|
|
|
ResumableProgress StorageMedia::resumable_status(const std::string& session_url,
|
|
long long total) const {
|
|
check_session(session_url);
|
|
if (total < 0 || total > static_cast<long long>(DataCaps::kMaxResumableTotal))
|
|
throw GoogleError::invalid("total out of range");
|
|
clients::ClientRequest req(clients::Method::kPut, clients::Url::parse(session_url));
|
|
req.header("Content-Range", "bytes */" + std::to_string(total));
|
|
req.bytes("", "application/octet-stream");
|
|
clients::ClientResponse r = send(std::move(req));
|
|
ResumableProgress p;
|
|
if (r.status() == 308) {
|
|
std::string range = r.headers().get("range").value_or("");
|
|
auto dash = range.find('-');
|
|
p.next_offset = dash == std::string::npos ? 0 : std::stoll(range.substr(dash + 1)) + 1;
|
|
return p;
|
|
}
|
|
if (!r.is_success()) fail(r);
|
|
p.done = true;
|
|
p.next_offset = total;
|
|
p.object = parse(r);
|
|
return p;
|
|
}
|
|
|
|
void StorageMedia::resumable_cancel(const std::string& session_url) const {
|
|
check_session(session_url);
|
|
clients::ClientRequest req(clients::Method::kDelete, clients::Url::parse(session_url));
|
|
clients::ClientResponse r = send(std::move(req));
|
|
if (r.status() != 499 && !r.is_success()) fail(r);
|
|
}
|
|
|
|
} // namespace molo::google::data
|