Files
moloapp/src/meta/chat_model.cpp
T

286 lines
12 KiB
C++
Raw Normal View History

#include "meta/chat_model.h"
#include <map>
#include "meta/check.h"
#include "meta/response_input.h"
#include "security/crypto/Base64.h"
namespace molo::meta {
namespace {
std::string image_url_of(const llm::ImageData& im) {
if (!im.url.empty()) return im.url;
if (im.bytes.empty()) Check::fail("image has neither url nor bytes");
return "data:" + im.mime + ";base64," + security::crypto::Base64::encode(im.bytes);
}
void common_checks(const llm::ChatRequest& r) {
if (!r.stop.empty()) Check::fail("stop sequences are not supported by Muse models");
if (!r.tool_choice.empty() && r.tool_choice != "auto")
Check::fail("tool_choice must be auto (the only value Meta accepts)");
}
std::string finish_of(const std::string& v) {
if (v == "stop" || v == "length" || v == "tool_calls" || v == "content_filter") return v;
if (v == "function_call") return "tool_calls";
return v;
}
} // namespace
MetaChatModel::MetaChatModel(std::shared_ptr<llm::ApiClient> client, Endpoint endpoint)
: responses_(client), chat_(client), endpoint_(endpoint) {}
ResponseRequest MetaChatModel::to_response_request(const llm::ChatRequest& r) const {
common_checks(r);
ResponseRequest q;
q.model = r.model;
q.instructions = r.system;
Json input = Json::array();
for (const llm::Message& m : r.messages) {
if (m.role == llm::Role::kSystem) {
input.push(ResponseInput::developer(m.text()));
continue;
}
Json parts = Json::array();
bool user = m.role == llm::Role::kUser;
auto flush = [&](const char* role) {
if (parts.as_array().empty()) return;
input.push(ResponseInput::message(role, parts));
parts = Json::array();
};
for (const llm::ContentPart& p : m.parts) {
if (!p.reasoning_signature.empty()) {
flush("assistant");
input.push(ResponseInput::reasoning(p.reasoning_signature));
}
if (!p.text.empty()) {
if (user) parts.push(ResponseInput::text(p.text));
else {
Json t = Json::object();
t.set("type", Json("output_text"));
t.set("text", Json(p.text));
t.set("annotations", Json::array());
parts.push(t);
}
}
if (p.image) parts.push(ResponseInput::image_url(image_url_of(*p.image)));
if (p.tool_call) {
flush("assistant");
input.push(ResponseInput::function_call(p.tool_call->id, p.tool_call->name,
p.tool_call->arguments.dump()));
}
if (p.tool_result) {
flush(user ? "user" : "assistant");
input.push(ResponseInput::function_call_output(p.tool_result->call_id, p.tool_result->content));
}
}
flush(m.role == llm::Role::kAssistant ? "assistant" : "user");
}
q.input = input;
for (const llm::ToolSpec& t : r.tools)
q.tools.push_back(ToolBuilder::function(t.name, t.description, t.parameters));
if (r.max_tokens) q.max_output_tokens = *r.max_tokens;
q.temperature = r.temperature;
q.top_p = r.top_p;
q.reasoning_effort = r.reasoning_effort;
if (request_reasoning_summary) q.reasoning_summary = "auto";
if (r.json_mode) q.text_format = ResponseRequest::format_json_object();
if (r.response_schema.is_object())
q.text_format = ResponseRequest::format_json_schema("output", r.response_schema);
q.store = false; // stateless: replay reasoning ourselves
q.include = {"reasoning.encrypted_content"};
q.extra = r.extra;
return q;
}
ChatCompletionRequest MetaChatModel::to_chat_request(const llm::ChatRequest& r) const {
common_checks(r);
ChatCompletionRequest q;
q.model = r.model;
if (!r.system.empty()) q.messages.push_back(ChatMessage::developer(r.system));
for (const llm::Message& m : r.messages) {
if (m.role == llm::Role::kSystem) { q.messages.push_back(ChatMessage::developer(m.text())); continue; }
if (m.role == llm::Role::kAssistant) {
std::vector<Json> calls;
for (const llm::ContentPart& p : m.parts)
if (p.tool_call)
calls.push_back(ChatMessage::tool_call(p.tool_call->id, p.tool_call->name, p.tool_call->arguments.dump()));
q.messages.push_back(calls.empty() ? ChatMessage::assistant(m.text())
: ChatMessage::assistant_tool_calls(m.text(), calls));
continue;
}
std::vector<Json> parts;
bool images = false;
for (const llm::ContentPart& p : m.parts) {
if (p.tool_result) {
q.messages.push_back(ChatMessage::tool_result(p.tool_result->call_id, p.tool_result->content));
continue;
}
if (!p.text.empty()) parts.push_back(ChatMessage::text_part(p.text));
if (p.image) { parts.push_back(ChatMessage::image_part(image_url_of(*p.image))); images = true; }
}
if (parts.empty()) continue;
if (images) q.messages.push_back(ChatMessage::user_parts(parts));
else q.messages.push_back(ChatMessage::user(m.text()));
}
for (const llm::ToolSpec& t : r.tools)
q.tools.push_back(ChatMessage::function_tool(t.name, t.description, t.parameters));
if (r.max_tokens) q.max_completion_tokens = *r.max_tokens;
q.temperature = r.temperature;
q.top_p = r.top_p;
q.reasoning_effort = r.reasoning_effort;
if (r.json_mode) q.response_format = ChatCompletionRequest::format_json_object();
if (r.response_schema.is_object())
q.response_format = ChatCompletionRequest::format_json_schema("output", r.response_schema);
q.extra = r.extra;
return q;
}
llm::ChatResponse MetaChatModel::from_response(const Json& doc) {
llm::ChatResponse out;
out.raw = doc;
out.id = JsonX::str(doc, "id");
out.model = JsonX::str(doc, "model");
out.message.role = llm::Role::kAssistant;
bool calls = false, refusal = false;
if (const Json* items = JsonX::arr(doc, "output"))
for (const Json& item : items->as_array()) {
std::string type = JsonX::str(item, "type");
if (type == "message") {
if (const Json* c = JsonX::arr(item, "content"))
for (const Json& p : c->as_array()) {
if (JsonX::str(p, "type") == "output_text")
out.message.parts.push_back(llm::ContentPart::of_text(JsonX::str(p, "text")));
else if (JsonX::str(p, "type") == "refusal")
refusal = true;
}
} else if (type == "reasoning") {
llm::ContentPart part;
if (const Json* s = JsonX::arr(item, "summary"))
for (const Json& t : s->as_array()) part.reasoning += JsonX::str(t, "text");
part.reasoning_signature = JsonX::str(item, "encrypted_content");
if (!part.reasoning.empty() || !part.reasoning_signature.empty())
out.message.parts.push_back(std::move(part));
} else if (type == "function_call") {
llm::ToolCall call;
call.id = JsonX::str(item, "call_id");
call.name = JsonX::str(item, "name");
try {
call.arguments = Json::parse(JsonX::str(item, "arguments", "{}"));
} catch (const std::exception&) {
call.arguments = Json::object();
}
out.message.parts.push_back(llm::ContentPart::of_call(std::move(call)));
calls = true;
}
}
std::string status = JsonX::str(doc, "status");
out.finish_reason = calls ? "tool_calls" : refusal ? "content_filter"
: status == "incomplete" ? "length" : "stop";
if (const Json* u = JsonX::obj(doc, "usage")) out.usage = MetaUsage::from_responses(*u).to_neutral();
return out;
}
llm::ChatResponse MetaChatModel::from_chat(const Json& doc) {
llm::ChatResponse out;
out.raw = doc;
out.id = JsonX::str(doc, "id");
out.model = JsonX::str(doc, "model");
out.message.role = llm::Role::kAssistant;
if (const Json* ch = JsonX::arr(doc, "choices"); ch && !ch->as_array().empty()) {
const Json& c = ch->as_array()[0];
out.finish_reason = finish_of(JsonX::str(c, "finish_reason"));
if (const Json* m = JsonX::obj(c, "message")) {
std::string text = JsonX::str(*m, "content");
if (!text.empty()) out.message.parts.push_back(llm::ContentPart::of_text(text));
if (const Json* tcs = JsonX::arr(*m, "tool_calls"))
for (const Json& tc : tcs->as_array()) {
llm::ToolCall call;
call.id = JsonX::str(tc, "id");
if (const Json* f = JsonX::obj(tc, "function")) {
call.name = JsonX::str(*f, "name");
try {
call.arguments = Json::parse(JsonX::str(*f, "arguments", "{}"));
} catch (const std::exception&) {
call.arguments = Json::object();
}
}
out.message.parts.push_back(llm::ContentPart::of_call(std::move(call)));
}
}
}
if (const Json* u = JsonX::obj(doc, "usage")) out.usage = MetaUsage::from_chat(*u).to_neutral();
return out;
}
llm::ChatResponse MetaChatModel::generate(const llm::ChatRequest& request) {
if (endpoint_ == Endpoint::kResponses)
return from_response(responses_.create(to_response_request(request)).body);
return from_chat(chat_.create(to_chat_request(request)));
}
void MetaChatModel::stream(const llm::ChatRequest& request,
const std::function<bool(const llm::ChatChunk&)>& on_chunk) {
if (endpoint_ == Endpoint::kChatCompletions) {
chat_.stream(to_chat_request(request), [&](const Json& chunk) {
llm::ChatChunk c;
std::string finish;
if (const Json* ch = JsonX::arr(chunk, "choices"); ch && !ch->as_array().empty()) {
const Json& first = ch->as_array()[0];
finish = finish_of(JsonX::str(first, "finish_reason"));
if (const Json* d = JsonX::obj(first, "delta")) {
c.text_delta = JsonX::str(*d, "content");
c.reasoning_delta = JsonX::str(*d, "reasoning_content");
if (const Json* tcs = JsonX::arr(*d, "tool_calls"))
for (const Json& tc : tcs->as_array()) {
c.tool_call_id = JsonX::str(tc, "id");
if (const Json* f = JsonX::obj(tc, "function")) {
c.tool_call_name = JsonX::str(*f, "name");
c.tool_args_delta = JsonX::str(*f, "arguments");
}
}
}
}
c.finish_reason = finish;
if (const Json* u = JsonX::obj(chunk, "usage")) c.usage = MetaUsage::from_chat(*u).to_neutral();
return on_chunk(c);
});
return;
}
std::map<std::string, std::pair<std::string, std::string>> by_item; // item id -> call id, name
bool saw_call = false;
responses_.stream(to_response_request(request), [&](const ResponseEvent& ev) {
llm::ChatChunk c;
const Json& d = ev.data;
if (ev.type == "response.output_text.delta") c.text_delta = JsonX::str(d, "delta");
else if (ev.type == "response.reasoning_summary_text.delta") c.reasoning_delta = JsonX::str(d, "delta");
else if (ev.type == "response.output_item.added") {
const Json* item = JsonX::obj(d, "item");
if (!item || JsonX::str(*item, "type") != "function_call") return true;
saw_call = true;
by_item[JsonX::str(*item, "id")] = {JsonX::str(*item, "call_id"), JsonX::str(*item, "name")};
c.tool_call_id = JsonX::str(*item, "call_id");
c.tool_call_name = JsonX::str(*item, "name");
} else if (ev.type == "response.function_call_arguments.delta") {
auto it = by_item.find(JsonX::str(d, "item_id"));
if (it != by_item.end()) c.tool_call_id = it->second.first;
c.tool_args_delta = JsonX::str(d, "delta");
} else if (ev.type == "response.completed" || ev.type == "response.incomplete") {
const Json* r = JsonX::obj(d, "response");
c.finish_reason = saw_call ? "tool_calls" : ev.type == "response.incomplete" ? "length" : "stop";
if (r)
if (const Json* u = JsonX::obj(*r, "usage")) c.usage = MetaUsage::from_responses(*u).to_neutral();
} else if (ev.type == "response.failed" || ev.type == "error") {
throw llm::ApiError("meta", 502, "stream_failed", JsonX::str(d, "message", "response failed"));
} else {
return true;
}
return on_chunk(c);
});
}
} // namespace molo::meta