286 lines
12 KiB
C++
286 lines
12 KiB
C++
#include "meta/chat_model.h"
|
|||
|
|
|
||
|
|
#include <map>
|
||
|
|
|
||
|
|
#include "meta/check.h"
|
||
|
|
#include "meta/response_input.h"
|
||
|
|
#include "security/crypto/Base64.h"
|
||
|
|
|
||
|
|
namespace molo::meta {
|
||
|
|
|
||
|
|
namespace {
|
||
|
|
|
||
|
|
std::string image_url_of(const llm::ImageData& im) {
|
||
|
|
if (!im.url.empty()) return im.url;
|
||
|
|
if (im.bytes.empty()) Check::fail("image has neither url nor bytes");
|
||
|
|
return "data:" + im.mime + ";base64," + security::crypto::Base64::encode(im.bytes);
|
||
|
|
}
|
||
|
|
|
||
|
|
void common_checks(const llm::ChatRequest& r) {
|
||
|
|
if (!r.stop.empty()) Check::fail("stop sequences are not supported by Muse models");
|
||
|
|
if (!r.tool_choice.empty() && r.tool_choice != "auto")
|
||
|
|
Check::fail("tool_choice must be auto (the only value Meta accepts)");
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string finish_of(const std::string& v) {
|
||
|
|
if (v == "stop" || v == "length" || v == "tool_calls" || v == "content_filter") return v;
|
||
|
|
if (v == "function_call") return "tool_calls";
|
||
|
|
return v;
|
||
|
|
}
|
||
|
|
|
||
|
|
} // namespace
|
||
|
|
|
||
|
|
MetaChatModel::MetaChatModel(std::shared_ptr<llm::ApiClient> client, Endpoint endpoint)
|
||
|
|
: responses_(client), chat_(client), endpoint_(endpoint) {}
|
||
|
|
|
||
|
|
ResponseRequest MetaChatModel::to_response_request(const llm::ChatRequest& r) const {
|
||
|
|
common_checks(r);
|
||
|
|
ResponseRequest q;
|
||
|
|
q.model = r.model;
|
||
|
|
q.instructions = r.system;
|
||
|
|
Json input = Json::array();
|
||
|
|
for (const llm::Message& m : r.messages) {
|
||
|
|
if (m.role == llm::Role::kSystem) {
|
||
|
|
input.push(ResponseInput::developer(m.text()));
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
Json parts = Json::array();
|
||
|
|
bool user = m.role == llm::Role::kUser;
|
||
|
|
auto flush = [&](const char* role) {
|
||
|
|
if (parts.as_array().empty()) return;
|
||
|
|
input.push(ResponseInput::message(role, parts));
|
||
|
|
parts = Json::array();
|
||
|
|
};
|
||
|
|
for (const llm::ContentPart& p : m.parts) {
|
||
|
|
if (!p.reasoning_signature.empty()) {
|
||
|
|
flush("assistant");
|
||
|
|
input.push(ResponseInput::reasoning(p.reasoning_signature));
|
||
|
|
}
|
||
|
|
if (!p.text.empty()) {
|
||
|
|
if (user) parts.push(ResponseInput::text(p.text));
|
||
|
|
else {
|
||
|
|
Json t = Json::object();
|
||
|
|
t.set("type", Json("output_text"));
|
||
|
|
t.set("text", Json(p.text));
|
||
|
|
t.set("annotations", Json::array());
|
||
|
|
parts.push(t);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (p.image) parts.push(ResponseInput::image_url(image_url_of(*p.image)));
|
||
|
|
if (p.tool_call) {
|
||
|
|
flush("assistant");
|
||
|
|
input.push(ResponseInput::function_call(p.tool_call->id, p.tool_call->name,
|
||
|
|
p.tool_call->arguments.dump()));
|
||
|
|
}
|
||
|
|
if (p.tool_result) {
|
||
|
|
flush(user ? "user" : "assistant");
|
||
|
|
input.push(ResponseInput::function_call_output(p.tool_result->call_id, p.tool_result->content));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
flush(m.role == llm::Role::kAssistant ? "assistant" : "user");
|
||
|
|
}
|
||
|
|
q.input = input;
|
||
|
|
for (const llm::ToolSpec& t : r.tools)
|
||
|
|
q.tools.push_back(ToolBuilder::function(t.name, t.description, t.parameters));
|
||
|
|
if (r.max_tokens) q.max_output_tokens = *r.max_tokens;
|
||
|
|
q.temperature = r.temperature;
|
||
|
|
q.top_p = r.top_p;
|
||
|
|
q.reasoning_effort = r.reasoning_effort;
|
||
|
|
if (request_reasoning_summary) q.reasoning_summary = "auto";
|
||
|
|
if (r.json_mode) q.text_format = ResponseRequest::format_json_object();
|
||
|
|
if (r.response_schema.is_object())
|
||
|
|
q.text_format = ResponseRequest::format_json_schema("output", r.response_schema);
|
||
|
|
q.store = false; // stateless: replay reasoning ourselves
|
||
|
|
q.include = {"reasoning.encrypted_content"};
|
||
|
|
q.extra = r.extra;
|
||
|
|
return q;
|
||
|
|
}
|
||
|
|
|
||
|
|
ChatCompletionRequest MetaChatModel::to_chat_request(const llm::ChatRequest& r) const {
|
||
|
|
common_checks(r);
|
||
|
|
ChatCompletionRequest q;
|
||
|
|
q.model = r.model;
|
||
|
|
if (!r.system.empty()) q.messages.push_back(ChatMessage::developer(r.system));
|
||
|
|
for (const llm::Message& m : r.messages) {
|
||
|
|
if (m.role == llm::Role::kSystem) { q.messages.push_back(ChatMessage::developer(m.text())); continue; }
|
||
|
|
if (m.role == llm::Role::kAssistant) {
|
||
|
|
std::vector<Json> calls;
|
||
|
|
for (const llm::ContentPart& p : m.parts)
|
||
|
|
if (p.tool_call)
|
||
|
|
calls.push_back(ChatMessage::tool_call(p.tool_call->id, p.tool_call->name, p.tool_call->arguments.dump()));
|
||
|
|
q.messages.push_back(calls.empty() ? ChatMessage::assistant(m.text())
|
||
|
|
: ChatMessage::assistant_tool_calls(m.text(), calls));
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
std::vector<Json> parts;
|
||
|
|
bool images = false;
|
||
|
|
for (const llm::ContentPart& p : m.parts) {
|
||
|
|
if (p.tool_result) {
|
||
|
|
q.messages.push_back(ChatMessage::tool_result(p.tool_result->call_id, p.tool_result->content));
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if (!p.text.empty()) parts.push_back(ChatMessage::text_part(p.text));
|
||
|
|
if (p.image) { parts.push_back(ChatMessage::image_part(image_url_of(*p.image))); images = true; }
|
||
|
|
}
|
||
|
|
if (parts.empty()) continue;
|
||
|
|
if (images) q.messages.push_back(ChatMessage::user_parts(parts));
|
||
|
|
else q.messages.push_back(ChatMessage::user(m.text()));
|
||
|
|
}
|
||
|
|
for (const llm::ToolSpec& t : r.tools)
|
||
|
|
q.tools.push_back(ChatMessage::function_tool(t.name, t.description, t.parameters));
|
||
|
|
if (r.max_tokens) q.max_completion_tokens = *r.max_tokens;
|
||
|
|
q.temperature = r.temperature;
|
||
|
|
q.top_p = r.top_p;
|
||
|
|
q.reasoning_effort = r.reasoning_effort;
|
||
|
|
if (r.json_mode) q.response_format = ChatCompletionRequest::format_json_object();
|
||
|
|
if (r.response_schema.is_object())
|
||
|
|
q.response_format = ChatCompletionRequest::format_json_schema("output", r.response_schema);
|
||
|
|
q.extra = r.extra;
|
||
|
|
return q;
|
||
|
|
}
|
||
|
|
|
||
|
|
llm::ChatResponse MetaChatModel::from_response(const Json& doc) {
|
||
|
|
llm::ChatResponse out;
|
||
|
|
out.raw = doc;
|
||
|
|
out.id = JsonX::str(doc, "id");
|
||
|
|
out.model = JsonX::str(doc, "model");
|
||
|
|
out.message.role = llm::Role::kAssistant;
|
||
|
|
bool calls = false, refusal = false;
|
||
|
|
if (const Json* items = JsonX::arr(doc, "output"))
|
||
|
|
for (const Json& item : items->as_array()) {
|
||
|
|
std::string type = JsonX::str(item, "type");
|
||
|
|
if (type == "message") {
|
||
|
|
if (const Json* c = JsonX::arr(item, "content"))
|
||
|
|
for (const Json& p : c->as_array()) {
|
||
|
|
if (JsonX::str(p, "type") == "output_text")
|
||
|
|
out.message.parts.push_back(llm::ContentPart::of_text(JsonX::str(p, "text")));
|
||
|
|
else if (JsonX::str(p, "type") == "refusal")
|
||
|
|
refusal = true;
|
||
|
|
}
|
||
|
|
} else if (type == "reasoning") {
|
||
|
|
llm::ContentPart part;
|
||
|
|
if (const Json* s = JsonX::arr(item, "summary"))
|
||
|
|
for (const Json& t : s->as_array()) part.reasoning += JsonX::str(t, "text");
|
||
|
|
part.reasoning_signature = JsonX::str(item, "encrypted_content");
|
||
|
|
if (!part.reasoning.empty() || !part.reasoning_signature.empty())
|
||
|
|
out.message.parts.push_back(std::move(part));
|
||
|
|
} else if (type == "function_call") {
|
||
|
|
llm::ToolCall call;
|
||
|
|
call.id = JsonX::str(item, "call_id");
|
||
|
|
call.name = JsonX::str(item, "name");
|
||
|
|
try {
|
||
|
|
call.arguments = Json::parse(JsonX::str(item, "arguments", "{}"));
|
||
|
|
} catch (const std::exception&) {
|
||
|
|
call.arguments = Json::object();
|
||
|
|
}
|
||
|
|
out.message.parts.push_back(llm::ContentPart::of_call(std::move(call)));
|
||
|
|
calls = true;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
std::string status = JsonX::str(doc, "status");
|
||
|
|
out.finish_reason = calls ? "tool_calls" : refusal ? "content_filter"
|
||
|
|
: status == "incomplete" ? "length" : "stop";
|
||
|
|
if (const Json* u = JsonX::obj(doc, "usage")) out.usage = MetaUsage::from_responses(*u).to_neutral();
|
||
|
|
return out;
|
||
|
|
}
|
||
|
|
|
||
|
|
llm::ChatResponse MetaChatModel::from_chat(const Json& doc) {
|
||
|
|
llm::ChatResponse out;
|
||
|
|
out.raw = doc;
|
||
|
|
out.id = JsonX::str(doc, "id");
|
||
|
|
out.model = JsonX::str(doc, "model");
|
||
|
|
out.message.role = llm::Role::kAssistant;
|
||
|
|
if (const Json* ch = JsonX::arr(doc, "choices"); ch && !ch->as_array().empty()) {
|
||
|
|
const Json& c = ch->as_array()[0];
|
||
|
|
out.finish_reason = finish_of(JsonX::str(c, "finish_reason"));
|
||
|
|
if (const Json* m = JsonX::obj(c, "message")) {
|
||
|
|
std::string text = JsonX::str(*m, "content");
|
||
|
|
if (!text.empty()) out.message.parts.push_back(llm::ContentPart::of_text(text));
|
||
|
|
if (const Json* tcs = JsonX::arr(*m, "tool_calls"))
|
||
|
|
for (const Json& tc : tcs->as_array()) {
|
||
|
|
llm::ToolCall call;
|
||
|
|
call.id = JsonX::str(tc, "id");
|
||
|
|
if (const Json* f = JsonX::obj(tc, "function")) {
|
||
|
|
call.name = JsonX::str(*f, "name");
|
||
|
|
try {
|
||
|
|
call.arguments = Json::parse(JsonX::str(*f, "arguments", "{}"));
|
||
|
|
} catch (const std::exception&) {
|
||
|
|
call.arguments = Json::object();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
out.message.parts.push_back(llm::ContentPart::of_call(std::move(call)));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (const Json* u = JsonX::obj(doc, "usage")) out.usage = MetaUsage::from_chat(*u).to_neutral();
|
||
|
|
return out;
|
||
|
|
}
|
||
|
|
|
||
|
|
llm::ChatResponse MetaChatModel::generate(const llm::ChatRequest& request) {
|
||
|
|
if (endpoint_ == Endpoint::kResponses)
|
||
|
|
return from_response(responses_.create(to_response_request(request)).body);
|
||
|
|
return from_chat(chat_.create(to_chat_request(request)));
|
||
|
|
}
|
||
|
|
|
||
|
|
void MetaChatModel::stream(const llm::ChatRequest& request,
|
||
|
|
const std::function<bool(const llm::ChatChunk&)>& on_chunk) {
|
||
|
|
if (endpoint_ == Endpoint::kChatCompletions) {
|
||
|
|
chat_.stream(to_chat_request(request), [&](const Json& chunk) {
|
||
|
|
llm::ChatChunk c;
|
||
|
|
std::string finish;
|
||
|
|
if (const Json* ch = JsonX::arr(chunk, "choices"); ch && !ch->as_array().empty()) {
|
||
|
|
const Json& first = ch->as_array()[0];
|
||
|
|
finish = finish_of(JsonX::str(first, "finish_reason"));
|
||
|
|
if (const Json* d = JsonX::obj(first, "delta")) {
|
||
|
|
c.text_delta = JsonX::str(*d, "content");
|
||
|
|
c.reasoning_delta = JsonX::str(*d, "reasoning_content");
|
||
|
|
if (const Json* tcs = JsonX::arr(*d, "tool_calls"))
|
||
|
|
for (const Json& tc : tcs->as_array()) {
|
||
|
|
c.tool_call_id = JsonX::str(tc, "id");
|
||
|
|
if (const Json* f = JsonX::obj(tc, "function")) {
|
||
|
|
c.tool_call_name = JsonX::str(*f, "name");
|
||
|
|
c.tool_args_delta = JsonX::str(*f, "arguments");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
c.finish_reason = finish;
|
||
|
|
if (const Json* u = JsonX::obj(chunk, "usage")) c.usage = MetaUsage::from_chat(*u).to_neutral();
|
||
|
|
return on_chunk(c);
|
||
|
|
});
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
std::map<std::string, std::pair<std::string, std::string>> by_item; // item id -> call id, name
|
||
|
|
bool saw_call = false;
|
||
|
|
responses_.stream(to_response_request(request), [&](const ResponseEvent& ev) {
|
||
|
|
llm::ChatChunk c;
|
||
|
|
const Json& d = ev.data;
|
||
|
|
if (ev.type == "response.output_text.delta") c.text_delta = JsonX::str(d, "delta");
|
||
|
|
else if (ev.type == "response.reasoning_summary_text.delta") c.reasoning_delta = JsonX::str(d, "delta");
|
||
|
|
else if (ev.type == "response.output_item.added") {
|
||
|
|
const Json* item = JsonX::obj(d, "item");
|
||
|
|
if (!item || JsonX::str(*item, "type") != "function_call") return true;
|
||
|
|
saw_call = true;
|
||
|
|
by_item[JsonX::str(*item, "id")] = {JsonX::str(*item, "call_id"), JsonX::str(*item, "name")};
|
||
|
|
c.tool_call_id = JsonX::str(*item, "call_id");
|
||
|
|
c.tool_call_name = JsonX::str(*item, "name");
|
||
|
|
} else if (ev.type == "response.function_call_arguments.delta") {
|
||
|
|
auto it = by_item.find(JsonX::str(d, "item_id"));
|
||
|
|
if (it != by_item.end()) c.tool_call_id = it->second.first;
|
||
|
|
c.tool_args_delta = JsonX::str(d, "delta");
|
||
|
|
} else if (ev.type == "response.completed" || ev.type == "response.incomplete") {
|
||
|
|
const Json* r = JsonX::obj(d, "response");
|
||
|
|
c.finish_reason = saw_call ? "tool_calls" : ev.type == "response.incomplete" ? "length" : "stop";
|
||
|
|
if (r)
|
||
|
|
if (const Json* u = JsonX::obj(*r, "usage")) c.usage = MetaUsage::from_responses(*u).to_neutral();
|
||
|
|
} else if (ev.type == "response.failed" || ev.type == "error") {
|
||
|
|
throw llm::ApiError("meta", 502, "stream_failed", JsonX::str(d, "message", "response failed"));
|
||
|
|
} else {
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
return on_chunk(c);
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
} // namespace molo::meta
|