Files
moloapp/src/meta/responses.cpp
T

105 lines
3.9 KiB
C++

#include "meta/responses.h"
#include "meta/caps.h"
#include "meta/check.h"
namespace molo::meta {
namespace {
ResponseResult from_raw(const llm::RawResponse& raw) {
ResponseResult r;
try {
r.body = raw.body.find_first_not_of(" \t\r\n") == std::string::npos ? Json::object()
: Json::parse(raw.body);
} catch (const std::exception& e) {
throw llm::ApiError("meta", 502, "BAD_RESPONSE", e.what());
}
r.limits = RateLimits::from_headers(raw.headers);
return r;
}
} // namespace
std::string ResponseResult::output_text() const {
std::string out;
if (const Json* o = JsonX::arr(body, "output"))
for (const Json& item : o->as_array()) {
if (JsonX::str(item, "type") != "message") continue;
if (const Json* c = JsonX::arr(item, "content"))
for (const Json& part : c->as_array())
if (JsonX::str(part, "type") == "output_text") out += JsonX::str(part, "text");
}
return out;
}
MetaUsage ResponseResult::usage() const {
const Json* u = JsonX::obj(body, "usage");
return u ? MetaUsage::from_responses(*u) : MetaUsage();
}
ResponseResult ResponsesService::create(const ResponseRequest& request) {
Json body = request.to_json(false);
std::string wire = body.dump();
return from_raw(client_->request_raw(method_of(Call::kResponsesCreate),
path_for(Call::kResponsesCreate), &wire,
"application/json"));
}
ResponseAccumulator ResponsesService::stream(const ResponseRequest& request,
const EventSink& sink) {
Json body = request.to_json(true);
ResponseAccumulator acc;
std::size_t seen = 0;
client_->stream(path_for(Call::kResponsesCreate), body, [&](const llm::SseEvent& e) {
if (++seen > MetaCaps::kMaxStreamEvents) return false; // hard cap on events per stream
ResponseEvent ev = ResponseEvent::from_sse(e);
bool more = acc.feed(ev);
if (sink && !sink(ev)) return false;
return more;
});
return acc;
}
ResponseResult ResponsesService::retrieve(const std::string& response_id) {
Check::segment("response_id", response_id);
return from_raw(client_->request_raw(method_of(Call::kResponsesRetrieve),
path_for(Call::kResponsesRetrieve, response_id)));
}
ResponseAccumulator ResponsesService::retrieve_stream(const std::string& response_id,
long starting_after, const EventSink& sink) {
Check::segment("response_id", response_id);
std::string path = path_for(Call::kResponsesRetrieve, response_id) + "?stream=true";
if (starting_after >= 0) path += "&starting_after=" + std::to_string(starting_after);
llm::RawResponse raw = client_->request_raw(method_of(Call::kResponsesRetrieve), path, nullptr,
"", {{"Accept", "text/event-stream"}});
ResponseAccumulator acc;
llm::SseDecoder decoder("meta", [&](const llm::SseEvent& e) {
ResponseEvent ev = ResponseEvent::from_sse(e);
bool more = acc.feed(ev);
if (sink && !sink(ev)) return false;
return more;
});
decoder.feed(raw.body);
decoder.finish();
return acc;
}
Json ResponsesService::remove(const std::string& response_id) {
Check::segment("response_id", response_id);
return client_->del(path_for(Call::kResponsesDelete, response_id));
}
Json ResponsesService::cancel(const std::string& response_id) {
Check::segment("response_id", response_id);
Json empty = Json::object();
return client_->post(path_for(Call::kResponsesCancel, response_id), empty);
}
long ResponsesService::count_input_tokens(const ResponseRequest& request) {
Json body = request.to_count_json();
Json out = client_->post(path_for(Call::kResponsesInputTokens), body);
return JsonX::num(out, "input_tokens", -1);
}
} // namespace molo::meta