// CompatChatModel: request mapping, row quirks, parsing and streaming. Replies // are documented OpenAI examples (FIXTURE) or transcripts built from the // documented chunk shape (LABELLED, not recorded provider responses). #include "support.h" #include "vendors/compat/chat_model.h" using namespace molo::compat; using namespace compat_test; using molo::llm::ApiError; using molo::llm::ChatChunk; using molo::llm::ChatRequest; using molo::llm::Message; namespace { ChatRequest base() { ChatRequest r; r.model = "m1"; r.messages.push_back(Message::user("hi")); return r; } // FIXTURE: OpenAI API reference, createChatCompletion streaming example chunks. const char* kStream = "data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n" "data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n" "data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n" "data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}]}\n\n" "data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[],\"usage\":{\"prompt_tokens\":9,\"completion_tokens\":2,\"total_tokens\":11}}\n\n" "data: [DONE]\n\n"; // LABELLED: tool call fragments in the documented delta shape, with a reasoning field. const char* kToolStream = "data: {\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"thinking\"},\"finish_reason\":null}]}\n\n" "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null}]}\n\n" "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\":\"}}]},\"finish_reason\":null}]}\n\n" "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"Rome\\\"}\"}}]},\"finish_reason\":null}]}\n\n" "data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\n" "data: [DONE]\n\n"; } // namespace TEST(compat_chat, body_maps_the_neutral_request) { Rig rig("together"); CompatChatModel m(rig.provider); ChatRequest r = base(); r.system = "be brief"; r.max_tokens = 16; r.temperature = 0.5; r.top_p = 0.9; r.stop = {"END"}; r.reasoning_effort = "low"; r.json_mode = true; Json b = m.to_body(r, false); ASSERT_EQ(b.find("model")->as_string(), std::string("m1")); ASSERT_EQ(b.find("messages")->as_array().size(), 2u); ASSERT_EQ(b.find("messages")->as_array()[0].find("role")->as_string(), std::string("system")); ASSERT_EQ(b.find("max_tokens")->as_int(), 16L); ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_object")); ASSERT_EQ(b.find("reasoning_effort")->as_string(), std::string("low")); ASSERT_TRUE(b.find("stream") == nullptr); ASSERT_TRUE(m.to_body(r, true).find("stream")->as_bool()); } TEST(compat_chat, schema_tools_and_tool_choice) { Rig rig("together"); CompatChatModel m(rig.provider); ChatRequest r = base(); r.response_schema = Json::parse("{\"type\":\"object\"}"); r.tools.push_back({"get_weather", "weather", Json::parse("{\"type\":\"object\"}")}); r.tool_choice = "get_weather"; Json b = m.to_body(r, false); ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_schema")); ASSERT_TRUE(b.find("response_format")->find("json_schema")->find("strict")->as_bool()); ASSERT_EQ(b.find("tool_choice")->find("function")->find("name")->as_string(), std::string("get_weather")); r.tool_choice = "required"; ASSERT_EQ(m.to_body(r, false).find("tool_choice")->as_string(), std::string("required")); ChatRequest none = base(); none.tool_choice = "auto"; ASSERT_THROW(m.to_body(none, false), ApiError); // tool_choice needs tools ChatRequest both = base(); both.json_mode = true; both.response_schema = Json::parse("{}"); ASSERT_THROW(m.to_body(both, false), ApiError); } TEST(compat_chat, tool_round_trip_messages_and_images) { Rig rig("together"); CompatChatModel m(rig.provider); ChatRequest r = base(); Message a; a.role = molo::llm::Role::kAssistant; molo::llm::ToolCall tc; tc.id = "call_1"; tc.name = "f"; tc.arguments = Json::parse("{\"a\":1}"); a.parts.push_back(molo::llm::ContentPart::of_call(tc)); r.messages.push_back(a); Message t; t.role = molo::llm::Role::kTool; t.parts.push_back(molo::llm::ContentPart::of_result({"call_1", "42", false})); r.messages.push_back(t); Message u; u.role = molo::llm::Role::kUser; u.parts.push_back(molo::llm::ContentPart::of_text("look")); u.parts.push_back(molo::llm::ContentPart::of_image({"image/png", "abc", ""})); r.messages.push_back(u); Json b = m.to_body(r, false); const auto& msgs = b.find("messages")->as_array(); ASSERT_EQ(msgs[1].find("tool_calls")->as_array()[0].find("function")->find("arguments")->as_string(), std::string("{\"a\":1}")); ASSERT_TRUE(msgs[1].find("content")->is_null()); ASSERT_EQ(msgs[2].find("tool_call_id")->as_string(), std::string("call_1")); ASSERT_TRUE(msgs[3].find("content")->is_array()); ASSERT_TRUE(has(msgs[3].find("content")->as_array()[1].find("image_url")->find("url")->as_string(), "data:image/png;base64,")); Message empty_tool; empty_tool.role = molo::llm::Role::kTool; ChatRequest bad = base(); bad.messages.push_back(empty_tool); ASSERT_THROW(m.to_body(bad, false), ApiError); } TEST(compat_chat, input_rules_run_before_any_traffic) { Rig rig("together"); CompatChatModel m(rig.provider); ChatRequest r = base(); r.model = ""; ASSERT_THROW(m.generate(r), ApiError); // no default model r = base(); r.messages.clear(); ASSERT_THROW(m.generate(r), ApiError); r = base(); r.max_tokens = 0; ASSERT_THROW(m.generate(r), ApiError); r = base(); r.temperature = 2.5; ASSERT_THROW(m.generate(r), ApiError); r = base(); r.top_p = 0.0; ASSERT_THROW(m.generate(r), ApiError); r = base(); r.stop = std::vector(17, "x"); ASSERT_THROW(m.generate(r), ApiError); r = base(); r.extra = Json::parse("{\"messages\":[]}"); ASSERT_THROW(m.generate(r), ApiError); r = base(); r.extra = Json::parse("[1]"); ASSERT_THROW(m.generate(r), ApiError); r = base(); r.model = std::string(300, 'x'); ASSERT_THROW(m.generate(r), ApiError); ASSERT_EQ(rig.transport->send_count(), 0u); CompatChatModel with_default(rig.provider, "dflt"); ASSERT_EQ(with_default.to_body(base(), false).find("model")->as_string(), std::string("m1")); ChatRequest nomodel = base(); nomodel.model = ""; ASSERT_EQ(with_default.to_body(nomodel, false).find("model")->as_string(), std::string("dflt")); } TEST(compat_chat, groq_quirks_are_rejected_loudly) { Rig rig("groq"); CompatChatModel m(rig.provider); ChatRequest named = base(); named.messages[0].name = "alice"; ASSERT_THROW(m.to_body(named, false), ApiError); for (const char* key : {"logprobs", "top_logprobs", "logit_bias"}) { ChatRequest r = base(); r.extra = Json::object(); r.extra.set(key, Json(1)); ASSERT_THROW(m.to_body(r, false), ApiError); } ChatRequest n2 = base(); n2.extra = Json::parse("{\"n\":2}"); ASSERT_THROW(m.to_body(n2, false), ApiError); ChatRequest n1 = base(); n1.extra = Json::parse("{\"n\":1}"); ASSERT_TRUE(m.to_body(n1, false).find("n") != nullptr); ASSERT_EQ(rig.transport->send_count(), 0u); // The same request is fine on a row without the quirk. Rig other("together"); CompatChatModel o(other.provider); ChatRequest ok = base(); ok.messages[0].name = "alice"; ok.extra = Json::parse("{\"logprobs\":true,\"n\":2}"); ASSERT_TRUE(o.to_body(ok, false).find("logprobs") != nullptr); ChatRequest big = base(); big.extra = Json::parse("{\"n\":9}"); ASSERT_THROW(o.to_body(big, false), ApiError); } TEST(compat_chat, cerebras_rejects_tools_with_response_format) { Rig rig("cerebras"); CompatChatModel m(rig.provider); ChatRequest r = base(); r.json_mode = true; ASSERT_TRUE(m.to_body(r, false).find("response_format") != nullptr); r.tools.push_back({"f", "", Json::object()}); ASSERT_THROW(m.to_body(r, false), ApiError); } TEST(compat_chat, parses_reasoning_fields_and_tool_calls) { Json doc = Json::parse( "{\"id\":\"x\",\"model\":\"m\",\"choices\":[{\"index\":0,\"finish_reason\":\"tool_calls\",\"message\":" "{\"role\":\"assistant\",\"content\":null,\"reasoning\":\"why\",\"tool_calls\":[{\"id\":\"c1\",\"type\":\"function\"," "\"function\":{\"name\":\"f\",\"arguments\":\"{\\\"a\\\":1}\"}},{\"id\":\"c2\",\"type\":\"function\"," "\"function\":{\"name\":\"g\",\"arguments\":\"not json\"}}]}}],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":4," "\"total_tokens\":7,\"prompt_tokens_details\":{\"cached_tokens\":2},\"completion_tokens_details\":{\"reasoning_tokens\":1}}}"); molo::llm::ChatResponse r = CompatChatModel::parse_response(doc); ASSERT_EQ(r.finish_reason, std::string("tool_calls")); ASSERT_EQ(r.message.parts[0].reasoning, std::string("why")); auto calls = r.tool_calls(); ASSERT_EQ(calls.size(), 2u); ASSERT_EQ(calls[0].arguments.find("a")->as_int(), 1L); ASSERT_TRUE(calls[1].arguments.find("_invalid_json") != nullptr); ASSERT_EQ(r.usage.cached_tokens, 2); ASSERT_EQ(r.usage.reasoning_tokens, 1); Json rc = Json::parse("{\"choices\":[{\"message\":{\"content\":\"a\",\"reasoning_content\":\"b\"}}]}"); ASSERT_EQ(CompatChatModel::parse_response(rc).message.parts[0].reasoning, std::string("b")); ASSERT_THROW(CompatChatModel::parse_response(Json::parse("{\"choices\":[]}")), ApiError); ASSERT_THROW(CompatChatModel::parse_response(Json::parse("{\"choices\":[{}]}")), ApiError); } TEST(compat_chat, stream_documented_transcript_with_usage_chunk) { Rig rig("fireworks"); rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse(kStream)); CompatChatModel m(rig.provider); std::string text, finish; int total = 0; m.stream(base(), [&](const ChatChunk& c) { text += c.text_delta; if (!c.finish_reason.empty()) finish = c.finish_reason; if (c.usage) total = c.usage->total_tokens; return true; }); ASSERT_EQ(text, std::string("Hello!")); ASSERT_EQ(finish, std::string("stop")); ASSERT_EQ(total, 11); ASSERT_TRUE(has(rig.transport->seen()[0].body, "\"stream\":true")); ASSERT_FALSE(has(rig.transport->seen()[0].body, "stream_options")); } TEST(compat_chat, stream_tool_fragments_carry_id_and_name) { Rig rig("fireworks"); rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse(kToolStream)); CompatChatModel m(rig.provider); std::string args, id, name, reasoning; m.stream(base(), [&](const ChatChunk& c) { args += c.tool_args_delta; if (!c.tool_call_id.empty()) id = c.tool_call_id; if (!c.tool_call_name.empty()) name = c.tool_call_name; reasoning += c.reasoning_delta; return true; }); ASSERT_EQ(args, std::string("{\"city\":\"Rome\"}")); ASSERT_EQ(id, std::string("call_1")); ASSERT_EQ(name, std::string("get_weather")); ASSERT_EQ(reasoning, std::string("thinking")); } TEST(compat_chat, stream_without_finish_reason_is_truncated_and_sink_can_stop) { Rig rig("fireworks"); rig.transport->expect_response( Method::kPost, rig.path("/chat/completions"), sse("data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"a\"},\"finish_reason\":null}]}\n\n")); CompatChatModel m(rig.provider); try { m.stream(base(), [](const ChatChunk&) { return true; }); FAIL("expected truncation"); } catch (const ApiError& e) { ASSERT_EQ(e.http_status(), 502); ASSERT_EQ(e.type(), std::string("STREAM_TRUNCATED")); } Rig rig2("fireworks"); rig2.transport->expect_response(Method::kPost, rig2.path("/chat/completions"), sse(kStream)); CompatChatModel m2(rig2.provider); int seen = 0; m2.stream(base(), [&](const ChatChunk&) { return ++seen < 2; }); // stop early: no throw ASSERT_EQ(seen, 2); } TEST(compat_chat, stream_error_event_throws) { Rig rig("fireworks"); rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse("data: {\"error\":{\"type\":\"overloaded\",\"message\":\"busy\"}}\n\n")); CompatChatModel m(rig.provider); try { m.stream(base(), [](const ChatChunk&) { return true; }); FAIL("expected"); } catch (const ApiError& e) { ASSERT_EQ(e.type(), std::string("overloaded")); } }