Files
moloapp/src/xai/core/batches.cpp
T

184 lines
7.2 KiB
C++
Raw Normal View History

#include "xai/core/batches.h"
#include <set>
#include "xai/core/caps.h"
namespace molo::xai::core {
namespace {
void check_batch_id(const std::string& id) {
Check::id("batch", id);
if (id.find(':') != std::string::npos) Check::fail("bad batch id");
}
Query page_query(Call call, std::optional<int> limit, const std::string& token) {
Query q;
if (limit) {
Check::range("limit", *limit, 1, Caps::kMaxBatchPageLimit);
q.add(call, "limit", static_cast<long>(*limit));
}
if (!token.empty()) {
Check::text("pagination_token", token, Caps::kMaxQueryValueLen);
q.add(call, "pagination_token", token);
}
return q;
}
} // namespace
BatchEntry BatchEntry::make(const std::string& id, const char* kind, Json body) {
Check::id("batch request", id);
Json inner = Json::object();
inner.set(kind, std::move(body));
Json o = Json::object();
o.set("batch_request_id", Json(id));
o.set("batch_request", inner);
BatchEntry e(std::move(o));
if (e.bytes() > Caps::kMaxBatchRequestBytes) Check::fail("batch request over 25 MB");
return e;
}
BatchEntry BatchEntry::chat(const std::string& id, const ChatRequest& r) { return make(id, "chat_get_completion", r.to_json()); }
BatchEntry BatchEntry::responses(const std::string& id, const ResponseRequest& r) { return make(id, "responses", r.to_json()); }
BatchEntry BatchEntry::image_generation(const std::string& id, const ImageGenerationRequest& r) { return make(id, "image_generation", r.to_json()); }
BatchEntry BatchEntry::image_edit(const std::string& id, const ImageEditRequest& r) { return make(id, "image_edit", r.to_json()); }
BatchEntry BatchEntry::video_generation(const std::string& id, const VideoGenerationRequest& r) { return make(id, "video_generation", r.to_json()); }
BatchEntry BatchEntry::video_edit(const std::string& id, const VideoEditRequest& r) { return make(id, "video_generation", r.to_json()); }
BatchEntry BatchEntry::video_extension(const std::string& id, const VideoExtendRequest& r) { return make(id, "video_extension", r.to_json()); }
Batch Batch::parse(const Json& d) {
Batch b;
b.raw = d;
b.batch_id = jx::str(d, "batch_id");
b.name = jx::str(d, "name");
b.create_time = jx::str(d, "create_time");
b.expire_time = jx::str(d, "expire_time");
b.cancel_time = jx::str(d, "cancel_time");
b.cancel_by_xai_message = jx::str(d, "cancel_by_xai_message");
b.create_api_key_id = jx::str(d, "create_api_key_id");
Json s = jx::object(d, "state");
b.state = {jx::num(s, "num_requests"), jx::num(s, "num_pending"), jx::num(s, "num_success"),
jx::num(s, "num_error"), jx::num(s, "num_cancelled")};
return b;
}
Batch BatchService::create(const std::string& name) {
Check::text("name", name, Caps::kMaxNameLen);
Json body = Json::object();
body.set("name", Json(name));
return Batch::parse(call_json(Call::kBatchCreate, {}, &body));
}
Batch BatchService::add_requests(const std::string& batch_id, const std::vector<BatchEntry>& entries) {
check_batch_id(batch_id);
Check::count("batch requests", entries.size(), 1, Caps::kMaxBatchRequestsPerAdd);
std::set<std::string> seen;
Json arr = Json::array();
for (const auto& e : entries) {
if (!seen.insert(jx::str(e.json(), "batch_request_id")).second)
Check::fail("batch_request_id must be unique within the call");
arr.push(e.json());
}
Json body = Json::object();
body.set("batch_requests", arr);
call_json(Call::kBatchRequestsAdd, {batch_id}, &body);
return get(batch_id);
}
Batch BatchService::get(const std::string& batch_id) {
check_batch_id(batch_id);
return Batch::parse(call_json(Call::kBatchGet, {batch_id}));
}
BatchPage BatchService::list(std::optional<int> limit, const std::string& token) {
Json doc = call_json(Call::kBatchList, {}, nullptr, page_query(Call::kBatchList, limit, token).str());
BatchPage p;
const Json* arr = jx::get(doc, "batches");
if (arr && arr->is_array())
for (const Json& b : arr->as_array()) {
if (p.batches.size() >= static_cast<std::size_t>(Caps::kMaxBatchPageLimit)) break;
p.batches.push_back(Batch::parse(b));
}
p.pagination_token = jx::str(doc, "pagination_token");
return p;
}
BatchRequestPage BatchService::list_requests(const std::string& batch_id, std::optional<int> limit,
const std::string& token) {
check_batch_id(batch_id);
Json doc = call_json(Call::kBatchRequestsList, {batch_id}, nullptr,
page_query(Call::kBatchRequestsList, limit, token).str());
BatchRequestPage p;
const Json* arr = jx::get(doc, "batch_request_metadata");
if (arr && arr->is_array())
for (const Json& m : arr->as_array()) {
if (p.requests.size() >= static_cast<std::size_t>(Caps::kMaxBatchPageLimit)) break;
p.requests.push_back({jx::str(m, "batch_request_id"), jx::str(m, "endpoint"), jx::str(m, "model"),
jx::str(m, "state"), jx::str(m, "create_time"), jx::str(m, "finish_time")});
}
p.pagination_token = jx::str(doc, "pagination_token");
return p;
}
BatchResultPage BatchService::results(const std::string& batch_id, std::optional<int> limit,
const std::string& token) {
check_batch_id(batch_id);
Json doc = call_json(Call::kBatchResults, {batch_id}, nullptr, page_query(Call::kBatchResults, limit, token).str());
BatchResultPage p;
const Json* arr = jx::get(doc, "results");
if (arr && arr->is_array())
for (const Json& r : arr->as_array()) {
if (p.results.size() >= static_cast<std::size_t>(Caps::kMaxBatchPageLimit)) break;
BatchResult o;
o.batch_request_id = jx::str(r, "batch_request_id");
Json result = jx::object(r, "batch_result");
Json response = jx::object(result, "response");
if (response.is_object() && !response.as_object().empty()) {
// The response object carries exactly one typed member.
const auto& members = response.as_object();
auto it = members.begin();
o.kind = it->first;
o.response = it->second;
o.succeeded = it->second.is_object();
if (o.kind == "chat_get_completion")
o.cost_usd_ticks = jx::num(jx::object(o.response, "usage"), "cost_in_usd_ticks");
}
std::string err = jx::str(result, "error");
if (err.empty()) err = jx::str(r, "error_message");
if (!err.empty()) {
o.succeeded = false;
o.error = err;
}
p.results.push_back(std::move(o));
}
p.pagination_token = jx::str(doc, "pagination_token");
return p;
}
BatchResultSet BatchService::all_results(const std::string& batch_id, std::size_t max_items) {
Check::range("max_items", static_cast<double>(max_items), 1, 100000);
BatchResultSet out;
std::string token;
for (std::size_t page = 0; page < Caps::kMaxBatchPagesWalked; ++page) {
BatchResultPage p = results(batch_id, 1000, token);
for (auto& r : p.results) {
if (out.results.size() >= max_items) {
out.truncated = true;
return out;
}
out.results.push_back(std::move(r));
}
if (p.pagination_token.empty()) return out;
token = p.pagination_token;
}
out.truncated = true;
return out;
}
Batch BatchService::cancel(const std::string& batch_id) {
check_batch_id(batch_id);
return Batch::parse(call_json(Call::kBatchCancel, {batch_id}));
}
} // namespace molo::xai::core