feat(clients): compat: OpenAI-compatible provider registry (39 rows), chat model, services, demo
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
+291
@@ -0,0 +1,291 @@
|
||||
// CompatChatModel: request mapping, row quirks, parsing and streaming. Replies
|
||||
// are documented OpenAI examples (FIXTURE) or transcripts built from the
|
||||
// documented chunk shape (LABELLED, not recorded provider responses).
|
||||
#include "support.h"
|
||||
#include "vendors/compat/chat_model.h"
|
||||
|
||||
using namespace molo::compat;
|
||||
using namespace compat_test;
|
||||
using molo::llm::ApiError;
|
||||
using molo::llm::ChatChunk;
|
||||
using molo::llm::ChatRequest;
|
||||
using molo::llm::Message;
|
||||
|
||||
namespace {
|
||||
|
||||
ChatRequest base() {
|
||||
ChatRequest r;
|
||||
r.model = "m1";
|
||||
r.messages.push_back(Message::user("hi"));
|
||||
return r;
|
||||
}
|
||||
|
||||
// FIXTURE: OpenAI API reference, createChatCompletion streaming example chunks.
|
||||
const char* kStream =
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}]}\n\n"
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[],\"usage\":{\"prompt_tokens\":9,\"completion_tokens\":2,\"total_tokens\":11}}\n\n"
|
||||
"data: [DONE]\n\n";
|
||||
|
||||
// LABELLED: tool call fragments in the documented delta shape, with a reasoning field.
|
||||
const char* kToolStream =
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"thinking\"},\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\":\"}}]},\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"Rome\\\"}\"}}]},\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\n"
|
||||
"data: [DONE]\n\n";
|
||||
} // namespace
|
||||
|
||||
TEST(compat_chat, body_maps_the_neutral_request) {
|
||||
Rig rig("together");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
r.system = "be brief";
|
||||
r.max_tokens = 16;
|
||||
r.temperature = 0.5;
|
||||
r.top_p = 0.9;
|
||||
r.stop = {"END"};
|
||||
r.reasoning_effort = "low";
|
||||
r.json_mode = true;
|
||||
Json b = m.to_body(r, false);
|
||||
ASSERT_EQ(b.find("model")->as_string(), std::string("m1"));
|
||||
ASSERT_EQ(b.find("messages")->as_array().size(), 2u);
|
||||
ASSERT_EQ(b.find("messages")->as_array()[0].find("role")->as_string(), std::string("system"));
|
||||
ASSERT_EQ(b.find("max_tokens")->as_int(), 16L);
|
||||
ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_object"));
|
||||
ASSERT_EQ(b.find("reasoning_effort")->as_string(), std::string("low"));
|
||||
ASSERT_TRUE(b.find("stream") == nullptr);
|
||||
ASSERT_TRUE(m.to_body(r, true).find("stream")->as_bool());
|
||||
}
|
||||
|
||||
TEST(compat_chat, schema_tools_and_tool_choice) {
|
||||
Rig rig("together");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
r.response_schema = Json::parse("{\"type\":\"object\"}");
|
||||
r.tools.push_back({"get_weather", "weather", Json::parse("{\"type\":\"object\"}")});
|
||||
r.tool_choice = "get_weather";
|
||||
Json b = m.to_body(r, false);
|
||||
ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_schema"));
|
||||
ASSERT_TRUE(b.find("response_format")->find("json_schema")->find("strict")->as_bool());
|
||||
ASSERT_EQ(b.find("tool_choice")->find("function")->find("name")->as_string(), std::string("get_weather"));
|
||||
r.tool_choice = "required";
|
||||
ASSERT_EQ(m.to_body(r, false).find("tool_choice")->as_string(), std::string("required"));
|
||||
ChatRequest none = base();
|
||||
none.tool_choice = "auto";
|
||||
ASSERT_THROW(m.to_body(none, false), ApiError); // tool_choice needs tools
|
||||
ChatRequest both = base();
|
||||
both.json_mode = true;
|
||||
both.response_schema = Json::parse("{}");
|
||||
ASSERT_THROW(m.to_body(both, false), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, tool_round_trip_messages_and_images) {
|
||||
Rig rig("together");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
Message a;
|
||||
a.role = molo::llm::Role::kAssistant;
|
||||
molo::llm::ToolCall tc;
|
||||
tc.id = "call_1";
|
||||
tc.name = "f";
|
||||
tc.arguments = Json::parse("{\"a\":1}");
|
||||
a.parts.push_back(molo::llm::ContentPart::of_call(tc));
|
||||
r.messages.push_back(a);
|
||||
Message t;
|
||||
t.role = molo::llm::Role::kTool;
|
||||
t.parts.push_back(molo::llm::ContentPart::of_result({"call_1", "42", false}));
|
||||
r.messages.push_back(t);
|
||||
Message u;
|
||||
u.role = molo::llm::Role::kUser;
|
||||
u.parts.push_back(molo::llm::ContentPart::of_text("look"));
|
||||
u.parts.push_back(molo::llm::ContentPart::of_image({"image/png", "abc", ""}));
|
||||
r.messages.push_back(u);
|
||||
Json b = m.to_body(r, false);
|
||||
const auto& msgs = b.find("messages")->as_array();
|
||||
ASSERT_EQ(msgs[1].find("tool_calls")->as_array()[0].find("function")->find("arguments")->as_string(), std::string("{\"a\":1}"));
|
||||
ASSERT_TRUE(msgs[1].find("content")->is_null());
|
||||
ASSERT_EQ(msgs[2].find("tool_call_id")->as_string(), std::string("call_1"));
|
||||
ASSERT_TRUE(msgs[3].find("content")->is_array());
|
||||
ASSERT_TRUE(has(msgs[3].find("content")->as_array()[1].find("image_url")->find("url")->as_string(), "data:image/png;base64,"));
|
||||
Message empty_tool;
|
||||
empty_tool.role = molo::llm::Role::kTool;
|
||||
ChatRequest bad = base();
|
||||
bad.messages.push_back(empty_tool);
|
||||
ASSERT_THROW(m.to_body(bad, false), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, input_rules_run_before_any_traffic) {
|
||||
Rig rig("together");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
r.model = "";
|
||||
ASSERT_THROW(m.generate(r), ApiError); // no default model
|
||||
r = base();
|
||||
r.messages.clear();
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.max_tokens = 0;
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.temperature = 2.5;
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.top_p = 0.0;
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.stop = std::vector<std::string>(17, "x");
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.extra = Json::parse("{\"messages\":[]}");
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.extra = Json::parse("[1]");
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.model = std::string(300, 'x');
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
ASSERT_EQ(rig.transport->send_count(), 0u);
|
||||
CompatChatModel with_default(rig.provider, "dflt");
|
||||
ASSERT_EQ(with_default.to_body(base(), false).find("model")->as_string(), std::string("m1"));
|
||||
ChatRequest nomodel = base();
|
||||
nomodel.model = "";
|
||||
ASSERT_EQ(with_default.to_body(nomodel, false).find("model")->as_string(), std::string("dflt"));
|
||||
}
|
||||
|
||||
TEST(compat_chat, groq_quirks_are_rejected_loudly) {
|
||||
Rig rig("groq");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest named = base();
|
||||
named.messages[0].name = "alice";
|
||||
ASSERT_THROW(m.to_body(named, false), ApiError);
|
||||
for (const char* key : {"logprobs", "top_logprobs", "logit_bias"}) {
|
||||
ChatRequest r = base();
|
||||
r.extra = Json::object();
|
||||
r.extra.set(key, Json(1));
|
||||
ASSERT_THROW(m.to_body(r, false), ApiError);
|
||||
}
|
||||
ChatRequest n2 = base();
|
||||
n2.extra = Json::parse("{\"n\":2}");
|
||||
ASSERT_THROW(m.to_body(n2, false), ApiError);
|
||||
ChatRequest n1 = base();
|
||||
n1.extra = Json::parse("{\"n\":1}");
|
||||
ASSERT_TRUE(m.to_body(n1, false).find("n") != nullptr);
|
||||
ASSERT_EQ(rig.transport->send_count(), 0u);
|
||||
// The same request is fine on a row without the quirk.
|
||||
Rig other("together");
|
||||
CompatChatModel o(other.provider);
|
||||
ChatRequest ok = base();
|
||||
ok.messages[0].name = "alice";
|
||||
ok.extra = Json::parse("{\"logprobs\":true,\"n\":2}");
|
||||
ASSERT_TRUE(o.to_body(ok, false).find("logprobs") != nullptr);
|
||||
ChatRequest big = base();
|
||||
big.extra = Json::parse("{\"n\":9}");
|
||||
ASSERT_THROW(o.to_body(big, false), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, cerebras_rejects_tools_with_response_format) {
|
||||
Rig rig("cerebras");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
r.json_mode = true;
|
||||
ASSERT_TRUE(m.to_body(r, false).find("response_format") != nullptr);
|
||||
r.tools.push_back({"f", "", Json::object()});
|
||||
ASSERT_THROW(m.to_body(r, false), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, parses_reasoning_fields_and_tool_calls) {
|
||||
Json doc = Json::parse(
|
||||
"{\"id\":\"x\",\"model\":\"m\",\"choices\":[{\"index\":0,\"finish_reason\":\"tool_calls\",\"message\":"
|
||||
"{\"role\":\"assistant\",\"content\":null,\"reasoning\":\"why\",\"tool_calls\":[{\"id\":\"c1\",\"type\":\"function\","
|
||||
"\"function\":{\"name\":\"f\",\"arguments\":\"{\\\"a\\\":1}\"}},{\"id\":\"c2\",\"type\":\"function\","
|
||||
"\"function\":{\"name\":\"g\",\"arguments\":\"not json\"}}]}}],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":4,"
|
||||
"\"total_tokens\":7,\"prompt_tokens_details\":{\"cached_tokens\":2},\"completion_tokens_details\":{\"reasoning_tokens\":1}}}");
|
||||
molo::llm::ChatResponse r = CompatChatModel::parse_response(doc);
|
||||
ASSERT_EQ(r.finish_reason, std::string("tool_calls"));
|
||||
ASSERT_EQ(r.message.parts[0].reasoning, std::string("why"));
|
||||
auto calls = r.tool_calls();
|
||||
ASSERT_EQ(calls.size(), 2u);
|
||||
ASSERT_EQ(calls[0].arguments.find("a")->as_int(), 1L);
|
||||
ASSERT_TRUE(calls[1].arguments.find("_invalid_json") != nullptr);
|
||||
ASSERT_EQ(r.usage.cached_tokens, 2);
|
||||
ASSERT_EQ(r.usage.reasoning_tokens, 1);
|
||||
Json rc = Json::parse("{\"choices\":[{\"message\":{\"content\":\"a\",\"reasoning_content\":\"b\"}}]}");
|
||||
ASSERT_EQ(CompatChatModel::parse_response(rc).message.parts[0].reasoning, std::string("b"));
|
||||
ASSERT_THROW(CompatChatModel::parse_response(Json::parse("{\"choices\":[]}")), ApiError);
|
||||
ASSERT_THROW(CompatChatModel::parse_response(Json::parse("{\"choices\":[{}]}")), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, stream_documented_transcript_with_usage_chunk) {
|
||||
Rig rig("fireworks");
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse(kStream));
|
||||
CompatChatModel m(rig.provider);
|
||||
std::string text, finish;
|
||||
int total = 0;
|
||||
m.stream(base(), [&](const ChatChunk& c) {
|
||||
text += c.text_delta;
|
||||
if (!c.finish_reason.empty()) finish = c.finish_reason;
|
||||
if (c.usage) total = c.usage->total_tokens;
|
||||
return true;
|
||||
});
|
||||
ASSERT_EQ(text, std::string("Hello!"));
|
||||
ASSERT_EQ(finish, std::string("stop"));
|
||||
ASSERT_EQ(total, 11);
|
||||
ASSERT_TRUE(has(rig.transport->seen()[0].body, "\"stream\":true"));
|
||||
ASSERT_FALSE(has(rig.transport->seen()[0].body, "stream_options"));
|
||||
}
|
||||
|
||||
TEST(compat_chat, stream_tool_fragments_carry_id_and_name) {
|
||||
Rig rig("fireworks");
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse(kToolStream));
|
||||
CompatChatModel m(rig.provider);
|
||||
std::string args, id, name, reasoning;
|
||||
m.stream(base(), [&](const ChatChunk& c) {
|
||||
args += c.tool_args_delta;
|
||||
if (!c.tool_call_id.empty()) id = c.tool_call_id;
|
||||
if (!c.tool_call_name.empty()) name = c.tool_call_name;
|
||||
reasoning += c.reasoning_delta;
|
||||
return true;
|
||||
});
|
||||
ASSERT_EQ(args, std::string("{\"city\":\"Rome\"}"));
|
||||
ASSERT_EQ(id, std::string("call_1"));
|
||||
ASSERT_EQ(name, std::string("get_weather"));
|
||||
ASSERT_EQ(reasoning, std::string("thinking"));
|
||||
}
|
||||
|
||||
TEST(compat_chat, stream_without_finish_reason_is_truncated_and_sink_can_stop) {
|
||||
Rig rig("fireworks");
|
||||
rig.transport->expect_response(
|
||||
Method::kPost, rig.path("/chat/completions"),
|
||||
sse("data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"a\"},\"finish_reason\":null}]}\n\n"));
|
||||
CompatChatModel m(rig.provider);
|
||||
try {
|
||||
m.stream(base(), [](const ChatChunk&) { return true; });
|
||||
FAIL("expected truncation");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.http_status(), 502);
|
||||
ASSERT_EQ(e.type(), std::string("STREAM_TRUNCATED"));
|
||||
}
|
||||
Rig rig2("fireworks");
|
||||
rig2.transport->expect_response(Method::kPost, rig2.path("/chat/completions"), sse(kStream));
|
||||
CompatChatModel m2(rig2.provider);
|
||||
int seen = 0;
|
||||
m2.stream(base(), [&](const ChatChunk&) { return ++seen < 2; }); // stop early: no throw
|
||||
ASSERT_EQ(seen, 2);
|
||||
}
|
||||
|
||||
TEST(compat_chat, stream_error_event_throws) {
|
||||
Rig rig("fireworks");
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"),
|
||||
sse("data: {\"error\":{\"type\":\"overloaded\",\"message\":\"busy\"}}\n\n"));
|
||||
CompatChatModel m(rig.provider);
|
||||
try {
|
||||
m.stream(base(), [](const ChatChunk&) { return true; });
|
||||
FAIL("expected");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.type(), std::string("overloaded"));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user