184 lines
7.2 KiB
C++
184 lines
7.2 KiB
C++
#include "xai/core/batches.h"
|
|
|
|
#include <set>
|
|
|
|
#include "xai/core/caps.h"
|
|
|
|
namespace molo::xai::core {
|
|
|
|
namespace {
|
|
void check_batch_id(const std::string& id) {
|
|
Check::id("batch", id);
|
|
if (id.find(':') != std::string::npos) Check::fail("bad batch id");
|
|
}
|
|
|
|
Query page_query(Call call, std::optional<int> limit, const std::string& token) {
|
|
Query q;
|
|
if (limit) {
|
|
Check::range("limit", *limit, 1, Caps::kMaxBatchPageLimit);
|
|
q.add(call, "limit", static_cast<long>(*limit));
|
|
}
|
|
if (!token.empty()) {
|
|
Check::text("pagination_token", token, Caps::kMaxQueryValueLen);
|
|
q.add(call, "pagination_token", token);
|
|
}
|
|
return q;
|
|
}
|
|
} // namespace
|
|
|
|
BatchEntry BatchEntry::make(const std::string& id, const char* kind, Json body) {
|
|
Check::id("batch request", id);
|
|
Json inner = Json::object();
|
|
inner.set(kind, std::move(body));
|
|
Json o = Json::object();
|
|
o.set("batch_request_id", Json(id));
|
|
o.set("batch_request", inner);
|
|
BatchEntry e(std::move(o));
|
|
if (e.bytes() > Caps::kMaxBatchRequestBytes) Check::fail("batch request over 25 MB");
|
|
return e;
|
|
}
|
|
|
|
BatchEntry BatchEntry::chat(const std::string& id, const ChatRequest& r) { return make(id, "chat_get_completion", r.to_json()); }
|
|
BatchEntry BatchEntry::responses(const std::string& id, const ResponseRequest& r) { return make(id, "responses", r.to_json()); }
|
|
BatchEntry BatchEntry::image_generation(const std::string& id, const ImageGenerationRequest& r) { return make(id, "image_generation", r.to_json()); }
|
|
BatchEntry BatchEntry::image_edit(const std::string& id, const ImageEditRequest& r) { return make(id, "image_edit", r.to_json()); }
|
|
BatchEntry BatchEntry::video_generation(const std::string& id, const VideoGenerationRequest& r) { return make(id, "video_generation", r.to_json()); }
|
|
BatchEntry BatchEntry::video_edit(const std::string& id, const VideoEditRequest& r) { return make(id, "video_generation", r.to_json()); }
|
|
BatchEntry BatchEntry::video_extension(const std::string& id, const VideoExtendRequest& r) { return make(id, "video_extension", r.to_json()); }
|
|
|
|
Batch Batch::parse(const Json& d) {
|
|
Batch b;
|
|
b.raw = d;
|
|
b.batch_id = jx::str(d, "batch_id");
|
|
b.name = jx::str(d, "name");
|
|
b.create_time = jx::str(d, "create_time");
|
|
b.expire_time = jx::str(d, "expire_time");
|
|
b.cancel_time = jx::str(d, "cancel_time");
|
|
b.cancel_by_xai_message = jx::str(d, "cancel_by_xai_message");
|
|
b.create_api_key_id = jx::str(d, "create_api_key_id");
|
|
Json s = jx::object(d, "state");
|
|
b.state = {jx::num(s, "num_requests"), jx::num(s, "num_pending"), jx::num(s, "num_success"),
|
|
jx::num(s, "num_error"), jx::num(s, "num_cancelled")};
|
|
return b;
|
|
}
|
|
|
|
Batch BatchService::create(const std::string& name) {
|
|
Check::text("name", name, Caps::kMaxNameLen);
|
|
Json body = Json::object();
|
|
body.set("name", Json(name));
|
|
return Batch::parse(call_json(Call::kBatchCreate, {}, &body));
|
|
}
|
|
|
|
Batch BatchService::add_requests(const std::string& batch_id, const std::vector<BatchEntry>& entries) {
|
|
check_batch_id(batch_id);
|
|
Check::count("batch requests", entries.size(), 1, Caps::kMaxBatchRequestsPerAdd);
|
|
std::set<std::string> seen;
|
|
Json arr = Json::array();
|
|
for (const auto& e : entries) {
|
|
if (!seen.insert(jx::str(e.json(), "batch_request_id")).second)
|
|
Check::fail("batch_request_id must be unique within the call");
|
|
arr.push(e.json());
|
|
}
|
|
Json body = Json::object();
|
|
body.set("batch_requests", arr);
|
|
call_json(Call::kBatchRequestsAdd, {batch_id}, &body);
|
|
return get(batch_id);
|
|
}
|
|
|
|
Batch BatchService::get(const std::string& batch_id) {
|
|
check_batch_id(batch_id);
|
|
return Batch::parse(call_json(Call::kBatchGet, {batch_id}));
|
|
}
|
|
|
|
BatchPage BatchService::list(std::optional<int> limit, const std::string& token) {
|
|
Json doc = call_json(Call::kBatchList, {}, nullptr, page_query(Call::kBatchList, limit, token).str());
|
|
BatchPage p;
|
|
const Json* arr = jx::get(doc, "batches");
|
|
if (arr && arr->is_array())
|
|
for (const Json& b : arr->as_array()) {
|
|
if (p.batches.size() >= static_cast<std::size_t>(Caps::kMaxBatchPageLimit)) break;
|
|
p.batches.push_back(Batch::parse(b));
|
|
}
|
|
p.pagination_token = jx::str(doc, "pagination_token");
|
|
return p;
|
|
}
|
|
|
|
BatchRequestPage BatchService::list_requests(const std::string& batch_id, std::optional<int> limit,
|
|
const std::string& token) {
|
|
check_batch_id(batch_id);
|
|
Json doc = call_json(Call::kBatchRequestsList, {batch_id}, nullptr,
|
|
page_query(Call::kBatchRequestsList, limit, token).str());
|
|
BatchRequestPage p;
|
|
const Json* arr = jx::get(doc, "batch_request_metadata");
|
|
if (arr && arr->is_array())
|
|
for (const Json& m : arr->as_array()) {
|
|
if (p.requests.size() >= static_cast<std::size_t>(Caps::kMaxBatchPageLimit)) break;
|
|
p.requests.push_back({jx::str(m, "batch_request_id"), jx::str(m, "endpoint"), jx::str(m, "model"),
|
|
jx::str(m, "state"), jx::str(m, "create_time"), jx::str(m, "finish_time")});
|
|
}
|
|
p.pagination_token = jx::str(doc, "pagination_token");
|
|
return p;
|
|
}
|
|
|
|
BatchResultPage BatchService::results(const std::string& batch_id, std::optional<int> limit,
|
|
const std::string& token) {
|
|
check_batch_id(batch_id);
|
|
Json doc = call_json(Call::kBatchResults, {batch_id}, nullptr, page_query(Call::kBatchResults, limit, token).str());
|
|
BatchResultPage p;
|
|
const Json* arr = jx::get(doc, "results");
|
|
if (arr && arr->is_array())
|
|
for (const Json& r : arr->as_array()) {
|
|
if (p.results.size() >= static_cast<std::size_t>(Caps::kMaxBatchPageLimit)) break;
|
|
BatchResult o;
|
|
o.batch_request_id = jx::str(r, "batch_request_id");
|
|
Json result = jx::object(r, "batch_result");
|
|
Json response = jx::object(result, "response");
|
|
if (response.is_object() && !response.as_object().empty()) {
|
|
// The response object carries exactly one typed member.
|
|
const auto& members = response.as_object();
|
|
auto it = members.begin();
|
|
o.kind = it->first;
|
|
o.response = it->second;
|
|
o.succeeded = it->second.is_object();
|
|
if (o.kind == "chat_get_completion")
|
|
o.cost_usd_ticks = jx::num(jx::object(o.response, "usage"), "cost_in_usd_ticks");
|
|
}
|
|
std::string err = jx::str(result, "error");
|
|
if (err.empty()) err = jx::str(r, "error_message");
|
|
if (!err.empty()) {
|
|
o.succeeded = false;
|
|
o.error = err;
|
|
}
|
|
p.results.push_back(std::move(o));
|
|
}
|
|
p.pagination_token = jx::str(doc, "pagination_token");
|
|
return p;
|
|
}
|
|
|
|
BatchResultSet BatchService::all_results(const std::string& batch_id, std::size_t max_items) {
|
|
Check::range("max_items", static_cast<double>(max_items), 1, 100000);
|
|
BatchResultSet out;
|
|
std::string token;
|
|
for (std::size_t page = 0; page < Caps::kMaxBatchPagesWalked; ++page) {
|
|
BatchResultPage p = results(batch_id, 1000, token);
|
|
for (auto& r : p.results) {
|
|
if (out.results.size() >= max_items) {
|
|
out.truncated = true;
|
|
return out;
|
|
}
|
|
out.results.push_back(std::move(r));
|
|
}
|
|
if (p.pagination_token.empty()) return out;
|
|
token = p.pagination_token;
|
|
}
|
|
out.truncated = true;
|
|
return out;
|
|
}
|
|
|
|
Batch BatchService::cancel(const std::string& batch_id) {
|
|
check_batch_id(batch_id);
|
|
return Batch::parse(call_json(Call::kBatchCancel, {batch_id}));
|
|
}
|
|
|
|
} // namespace molo::xai::core
|