95 lines
3.7 KiB
C++
95 lines
3.7 KiB
C++
#include "openai/platform/uploads_api.h"
|
|
|
|
#include <algorithm>
|
|
|
|
#include "llm/error.h"
|
|
#include "openai/platform/caps.h"
|
|
|
|
namespace molo::openai::platform {
|
|
|
|
namespace {
|
|
[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("openai", why); }
|
|
} // namespace
|
|
|
|
UploadPlan UploadPlan::make(std::size_t total, std::size_t part_bytes) {
|
|
if (total == 0) bad("upload is empty");
|
|
if (total > Caps::kMaxFileBytes) bad("upload over the byte cap");
|
|
if (part_bytes == 0 || part_bytes > Caps::kMaxUploadPartBytes)
|
|
bad("part size must be 1 byte to 64 MiB");
|
|
UploadPlan plan;
|
|
std::size_t count = (total + part_bytes - 1) / part_bytes;
|
|
if (count > Caps::kMaxUploadParts) bad("upload needs more parts than the cap");
|
|
for (std::size_t off = 0; off < total; off += part_bytes)
|
|
plan.parts.push_back({off, std::min(part_bytes, total - off)});
|
|
return plan;
|
|
}
|
|
|
|
Json UploadsApi::create(const std::string& filename, const std::string& mime_type, const std::string& purpose,
|
|
std::size_t bytes, const Json* expires_after) {
|
|
static const char* kPurposes[] = {"assistants", "batch", "fine-tune", "vision"};
|
|
if (std::none_of(std::begin(kPurposes), std::end(kPurposes), [&](const char* k) { return purpose == k; }))
|
|
bad("purpose must be assistants, batch, fine-tune or vision");
|
|
if (!Form::valid_filename(filename)) bad("bad filename");
|
|
if (mime_type.empty() || mime_type.size() > 128) bad("bad mime type");
|
|
if (bytes == 0 || bytes > Caps::kMaxFileBytes) bad("bytes must be 1 to the byte cap");
|
|
Json body = Json::object();
|
|
body.set("filename", Json(filename));
|
|
body.set("purpose", Json(purpose));
|
|
body.set("bytes", Json(static_cast<double>(bytes)));
|
|
body.set("mime_type", Json(mime_type));
|
|
if (expires_after && !expires_after->is_null()) body.set("expires_after", *expires_after);
|
|
return invoke(Op::kCreateUpload, {}, &body);
|
|
}
|
|
|
|
Json UploadsApi::add_part(const std::string& upload_id, const std::string& data) {
|
|
if (data.empty() || data.size() > Caps::kMaxUploadPartBytes) bad("part must be 1 byte to 64 MiB");
|
|
Form f;
|
|
f.file("data", "part", "application/octet-stream", data);
|
|
return invoke_form(Op::kAddUploadPart, {{"upload_id", upload_id}}, f);
|
|
}
|
|
|
|
Json UploadsApi::complete(const std::string& upload_id, const std::vector<std::string>& part_ids,
|
|
const std::string& md5) {
|
|
if (part_ids.empty() || part_ids.size() > Caps::kMaxUploadParts) bad("part_ids must be 1 to the part cap");
|
|
Json ids = Json::array();
|
|
for (const std::string& id : part_ids) {
|
|
check_id("part", id);
|
|
ids.push(Json(id));
|
|
}
|
|
Json body = Json::object();
|
|
body.set("part_ids", ids);
|
|
if (!md5.empty()) {
|
|
if (md5.size() != 32) bad("md5 must be 32 hex characters");
|
|
body.set("md5", Json(md5));
|
|
}
|
|
return invoke(Op::kCompleteUpload, {{"upload_id", upload_id}}, &body);
|
|
}
|
|
|
|
Json UploadsApi::cancel(const std::string& upload_id) {
|
|
return invoke(Op::kCancelUpload, {{"upload_id", upload_id}});
|
|
}
|
|
|
|
Json UploadsApi::upload_all(const std::string& filename, const std::string& mime_type,
|
|
const std::string& purpose, const std::string& data, std::size_t part_bytes) {
|
|
UploadPlan plan = UploadPlan::make(data.size(), part_bytes);
|
|
Json upload = create(filename, mime_type, purpose, data.size());
|
|
std::string id = string_at(upload, "id");
|
|
check_id("upload", id);
|
|
try {
|
|
std::vector<std::string> part_ids;
|
|
for (const UploadPlan::Part& p : plan.parts) {
|
|
Json part = add_part(id, data.substr(p.offset, p.size));
|
|
part_ids.push_back(string_at(part, "id"));
|
|
}
|
|
return complete(id, part_ids);
|
|
} catch (...) {
|
|
try {
|
|
cancel(id);
|
|
} catch (...) {
|
|
}
|
|
throw;
|
|
}
|
|
}
|
|
|
|
} // namespace molo::openai::platform
|