feat(clients): compat: OpenAI-compatible provider registry (39 rows), chat model, services, demo

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-25 15:01:38 +01:00
co-authored by Claude Sonnet 5.5
parent 3db67fcf1f
commit aaddb29d42
23 changed files with 3348 additions and 3 deletions
+291
View File
@@ -0,0 +1,291 @@
// CompatChatModel: request mapping, row quirks, parsing and streaming. Replies
// are documented OpenAI examples (FIXTURE) or transcripts built from the
// documented chunk shape (LABELLED, not recorded provider responses).
#include "support.h"
#include "vendors/compat/chat_model.h"
using namespace molo::compat;
using namespace compat_test;
using molo::llm::ApiError;
using molo::llm::ChatChunk;
using molo::llm::ChatRequest;
using molo::llm::Message;
namespace {
ChatRequest base() {
ChatRequest r;
r.model = "m1";
r.messages.push_back(Message::user("hi"));
return r;
}
// FIXTURE: OpenAI API reference, createChatCompletion streaming example chunks.
const char* kStream =
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}]}\n\n"
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[],\"usage\":{\"prompt_tokens\":9,\"completion_tokens\":2,\"total_tokens\":11}}\n\n"
"data: [DONE]\n\n";
// LABELLED: tool call fragments in the documented delta shape, with a reasoning field.
const char* kToolStream =
"data: {\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"thinking\"},\"finish_reason\":null}]}\n\n"
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null}]}\n\n"
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\":\"}}]},\"finish_reason\":null}]}\n\n"
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"Rome\\\"}\"}}]},\"finish_reason\":null}]}\n\n"
"data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\n"
"data: [DONE]\n\n";
} // namespace
TEST(compat_chat, body_maps_the_neutral_request) {
Rig rig("together");
CompatChatModel m(rig.provider);
ChatRequest r = base();
r.system = "be brief";
r.max_tokens = 16;
r.temperature = 0.5;
r.top_p = 0.9;
r.stop = {"END"};
r.reasoning_effort = "low";
r.json_mode = true;
Json b = m.to_body(r, false);
ASSERT_EQ(b.find("model")->as_string(), std::string("m1"));
ASSERT_EQ(b.find("messages")->as_array().size(), 2u);
ASSERT_EQ(b.find("messages")->as_array()[0].find("role")->as_string(), std::string("system"));
ASSERT_EQ(b.find("max_tokens")->as_int(), 16L);
ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_object"));
ASSERT_EQ(b.find("reasoning_effort")->as_string(), std::string("low"));
ASSERT_TRUE(b.find("stream") == nullptr);
ASSERT_TRUE(m.to_body(r, true).find("stream")->as_bool());
}
TEST(compat_chat, schema_tools_and_tool_choice) {
Rig rig("together");
CompatChatModel m(rig.provider);
ChatRequest r = base();
r.response_schema = Json::parse("{\"type\":\"object\"}");
r.tools.push_back({"get_weather", "weather", Json::parse("{\"type\":\"object\"}")});
r.tool_choice = "get_weather";
Json b = m.to_body(r, false);
ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_schema"));
ASSERT_TRUE(b.find("response_format")->find("json_schema")->find("strict")->as_bool());
ASSERT_EQ(b.find("tool_choice")->find("function")->find("name")->as_string(), std::string("get_weather"));
r.tool_choice = "required";
ASSERT_EQ(m.to_body(r, false).find("tool_choice")->as_string(), std::string("required"));
ChatRequest none = base();
none.tool_choice = "auto";
ASSERT_THROW(m.to_body(none, false), ApiError); // tool_choice needs tools
ChatRequest both = base();
both.json_mode = true;
both.response_schema = Json::parse("{}");
ASSERT_THROW(m.to_body(both, false), ApiError);
}
TEST(compat_chat, tool_round_trip_messages_and_images) {
Rig rig("together");
CompatChatModel m(rig.provider);
ChatRequest r = base();
Message a;
a.role = molo::llm::Role::kAssistant;
molo::llm::ToolCall tc;
tc.id = "call_1";
tc.name = "f";
tc.arguments = Json::parse("{\"a\":1}");
a.parts.push_back(molo::llm::ContentPart::of_call(tc));
r.messages.push_back(a);
Message t;
t.role = molo::llm::Role::kTool;
t.parts.push_back(molo::llm::ContentPart::of_result({"call_1", "42", false}));
r.messages.push_back(t);
Message u;
u.role = molo::llm::Role::kUser;
u.parts.push_back(molo::llm::ContentPart::of_text("look"));
u.parts.push_back(molo::llm::ContentPart::of_image({"image/png", "abc", ""}));
r.messages.push_back(u);
Json b = m.to_body(r, false);
const auto& msgs = b.find("messages")->as_array();
ASSERT_EQ(msgs[1].find("tool_calls")->as_array()[0].find("function")->find("arguments")->as_string(), std::string("{\"a\":1}"));
ASSERT_TRUE(msgs[1].find("content")->is_null());
ASSERT_EQ(msgs[2].find("tool_call_id")->as_string(), std::string("call_1"));
ASSERT_TRUE(msgs[3].find("content")->is_array());
ASSERT_TRUE(has(msgs[3].find("content")->as_array()[1].find("image_url")->find("url")->as_string(), "data:image/png;base64,"));
Message empty_tool;
empty_tool.role = molo::llm::Role::kTool;
ChatRequest bad = base();
bad.messages.push_back(empty_tool);
ASSERT_THROW(m.to_body(bad, false), ApiError);
}
TEST(compat_chat, input_rules_run_before_any_traffic) {
Rig rig("together");
CompatChatModel m(rig.provider);
ChatRequest r = base();
r.model = "";
ASSERT_THROW(m.generate(r), ApiError); // no default model
r = base();
r.messages.clear();
ASSERT_THROW(m.generate(r), ApiError);
r = base();
r.max_tokens = 0;
ASSERT_THROW(m.generate(r), ApiError);
r = base();
r.temperature = 2.5;
ASSERT_THROW(m.generate(r), ApiError);
r = base();
r.top_p = 0.0;
ASSERT_THROW(m.generate(r), ApiError);
r = base();
r.stop = std::vector<std::string>(17, "x");
ASSERT_THROW(m.generate(r), ApiError);
r = base();
r.extra = Json::parse("{\"messages\":[]}");
ASSERT_THROW(m.generate(r), ApiError);
r = base();
r.extra = Json::parse("[1]");
ASSERT_THROW(m.generate(r), ApiError);
r = base();
r.model = std::string(300, 'x');
ASSERT_THROW(m.generate(r), ApiError);
ASSERT_EQ(rig.transport->send_count(), 0u);
CompatChatModel with_default(rig.provider, "dflt");
ASSERT_EQ(with_default.to_body(base(), false).find("model")->as_string(), std::string("m1"));
ChatRequest nomodel = base();
nomodel.model = "";
ASSERT_EQ(with_default.to_body(nomodel, false).find("model")->as_string(), std::string("dflt"));
}
TEST(compat_chat, groq_quirks_are_rejected_loudly) {
Rig rig("groq");
CompatChatModel m(rig.provider);
ChatRequest named = base();
named.messages[0].name = "alice";
ASSERT_THROW(m.to_body(named, false), ApiError);
for (const char* key : {"logprobs", "top_logprobs", "logit_bias"}) {
ChatRequest r = base();
r.extra = Json::object();
r.extra.set(key, Json(1));
ASSERT_THROW(m.to_body(r, false), ApiError);
}
ChatRequest n2 = base();
n2.extra = Json::parse("{\"n\":2}");
ASSERT_THROW(m.to_body(n2, false), ApiError);
ChatRequest n1 = base();
n1.extra = Json::parse("{\"n\":1}");
ASSERT_TRUE(m.to_body(n1, false).find("n") != nullptr);
ASSERT_EQ(rig.transport->send_count(), 0u);
// The same request is fine on a row without the quirk.
Rig other("together");
CompatChatModel o(other.provider);
ChatRequest ok = base();
ok.messages[0].name = "alice";
ok.extra = Json::parse("{\"logprobs\":true,\"n\":2}");
ASSERT_TRUE(o.to_body(ok, false).find("logprobs") != nullptr);
ChatRequest big = base();
big.extra = Json::parse("{\"n\":9}");
ASSERT_THROW(o.to_body(big, false), ApiError);
}
TEST(compat_chat, cerebras_rejects_tools_with_response_format) {
Rig rig("cerebras");
CompatChatModel m(rig.provider);
ChatRequest r = base();
r.json_mode = true;
ASSERT_TRUE(m.to_body(r, false).find("response_format") != nullptr);
r.tools.push_back({"f", "", Json::object()});
ASSERT_THROW(m.to_body(r, false), ApiError);
}
TEST(compat_chat, parses_reasoning_fields_and_tool_calls) {
Json doc = Json::parse(
"{\"id\":\"x\",\"model\":\"m\",\"choices\":[{\"index\":0,\"finish_reason\":\"tool_calls\",\"message\":"
"{\"role\":\"assistant\",\"content\":null,\"reasoning\":\"why\",\"tool_calls\":[{\"id\":\"c1\",\"type\":\"function\","
"\"function\":{\"name\":\"f\",\"arguments\":\"{\\\"a\\\":1}\"}},{\"id\":\"c2\",\"type\":\"function\","
"\"function\":{\"name\":\"g\",\"arguments\":\"not json\"}}]}}],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":4,"
"\"total_tokens\":7,\"prompt_tokens_details\":{\"cached_tokens\":2},\"completion_tokens_details\":{\"reasoning_tokens\":1}}}");
molo::llm::ChatResponse r = CompatChatModel::parse_response(doc);
ASSERT_EQ(r.finish_reason, std::string("tool_calls"));
ASSERT_EQ(r.message.parts[0].reasoning, std::string("why"));
auto calls = r.tool_calls();
ASSERT_EQ(calls.size(), 2u);
ASSERT_EQ(calls[0].arguments.find("a")->as_int(), 1L);
ASSERT_TRUE(calls[1].arguments.find("_invalid_json") != nullptr);
ASSERT_EQ(r.usage.cached_tokens, 2);
ASSERT_EQ(r.usage.reasoning_tokens, 1);
Json rc = Json::parse("{\"choices\":[{\"message\":{\"content\":\"a\",\"reasoning_content\":\"b\"}}]}");
ASSERT_EQ(CompatChatModel::parse_response(rc).message.parts[0].reasoning, std::string("b"));
ASSERT_THROW(CompatChatModel::parse_response(Json::parse("{\"choices\":[]}")), ApiError);
ASSERT_THROW(CompatChatModel::parse_response(Json::parse("{\"choices\":[{}]}")), ApiError);
}
TEST(compat_chat, stream_documented_transcript_with_usage_chunk) {
Rig rig("fireworks");
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse(kStream));
CompatChatModel m(rig.provider);
std::string text, finish;
int total = 0;
m.stream(base(), [&](const ChatChunk& c) {
text += c.text_delta;
if (!c.finish_reason.empty()) finish = c.finish_reason;
if (c.usage) total = c.usage->total_tokens;
return true;
});
ASSERT_EQ(text, std::string("Hello!"));
ASSERT_EQ(finish, std::string("stop"));
ASSERT_EQ(total, 11);
ASSERT_TRUE(has(rig.transport->seen()[0].body, "\"stream\":true"));
ASSERT_FALSE(has(rig.transport->seen()[0].body, "stream_options"));
}
TEST(compat_chat, stream_tool_fragments_carry_id_and_name) {
Rig rig("fireworks");
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse(kToolStream));
CompatChatModel m(rig.provider);
std::string args, id, name, reasoning;
m.stream(base(), [&](const ChatChunk& c) {
args += c.tool_args_delta;
if (!c.tool_call_id.empty()) id = c.tool_call_id;
if (!c.tool_call_name.empty()) name = c.tool_call_name;
reasoning += c.reasoning_delta;
return true;
});
ASSERT_EQ(args, std::string("{\"city\":\"Rome\"}"));
ASSERT_EQ(id, std::string("call_1"));
ASSERT_EQ(name, std::string("get_weather"));
ASSERT_EQ(reasoning, std::string("thinking"));
}
TEST(compat_chat, stream_without_finish_reason_is_truncated_and_sink_can_stop) {
Rig rig("fireworks");
rig.transport->expect_response(
Method::kPost, rig.path("/chat/completions"),
sse("data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"a\"},\"finish_reason\":null}]}\n\n"));
CompatChatModel m(rig.provider);
try {
m.stream(base(), [](const ChatChunk&) { return true; });
FAIL("expected truncation");
} catch (const ApiError& e) {
ASSERT_EQ(e.http_status(), 502);
ASSERT_EQ(e.type(), std::string("STREAM_TRUNCATED"));
}
Rig rig2("fireworks");
rig2.transport->expect_response(Method::kPost, rig2.path("/chat/completions"), sse(kStream));
CompatChatModel m2(rig2.provider);
int seen = 0;
m2.stream(base(), [&](const ChatChunk&) { return ++seen < 2; }); // stop early: no throw
ASSERT_EQ(seen, 2);
}
TEST(compat_chat, stream_error_event_throws) {
Rig rig("fireworks");
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"),
sse("data: {\"error\":{\"type\":\"overloaded\",\"message\":\"busy\"}}\n\n"));
CompatChatModel m(rig.provider);
try {
m.stream(base(), [](const ChatChunk&) { return true; });
FAIL("expected");
} catch (const ApiError& e) {
ASSERT_EQ(e.type(), std::string("overloaded"));
}
}