feat(clients): compat: OpenAI-compatible provider registry (39 rows), chat model, services, demo

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-25 15:01:38 +01:00
co-authored by Claude Sonnet 5.5
parent 3db67fcf1f
commit aaddb29d42
23 changed files with 3348 additions and 3 deletions
+63
View File
@@ -0,0 +1,63 @@
#pragma once
// Shared rig for the compat tests: a Provider over a MockTransport with a
// fake key. The mock only records the requests the real client builds;
// replies are documented examples taken verbatim from OpenAI's API reference
// (the shape every row shares) and labelled FIXTURE where they appear.
// Nothing here verifies a provider's behavior (see SPEC.md ## Unverified).
#include <memory>
#include <string>
#include "clients/transports/mock_transport.h"
#include "framework.h"
#include "vendors/compat/provider.h"
namespace compat_test {
using molo::clients::ClientResponse;
using molo::clients::HeaderMap;
using molo::clients::Method;
using molo::clients::transports::MockTransport;
using Json = molo::clients::body::Json;
inline ClientResponse reply(const std::string& body, int status = 200,
const std::string& type = "application/json") {
HeaderMap h;
h.set("Content-Type", type);
return ClientResponse(status, h, body);
}
inline ClientResponse sse(const std::string& body) { return reply(body, 200, "text/event-stream"); }
inline const char* kKey = "sk-TESTKEY-DO-NOT-LEAK";
inline bool has(const std::string& hay, const std::string& needle) { return hay.find(needle) != std::string::npos; }
// "https://host/a/b" -> "/a/b"
inline std::string path_of(const std::string& url) {
std::size_t scheme = url.find("://");
std::size_t slash = url.find('/', scheme == std::string::npos ? 0 : scheme + 3);
return slash == std::string::npos ? "" : url.substr(slash);
}
struct Rig {
MockTransport* transport = nullptr;
molo::compat::Provider provider;
explicit Rig(const std::string& id, const std::string& key = kKey, const std::string& base_override = "",
const std::string& account = "acct123")
: provider(make(id, key, base_override, account, &transport)) {}
std::string path(const char* suffix) const { return path_of(provider.base_url()) + suffix; }
private:
static molo::compat::Provider make(const std::string& id, const std::string& key, const std::string& base,
const std::string& account, MockTransport** out) {
auto t = std::make_unique<MockTransport>();
*out = t.get();
return molo::compat::Provider::with(id, std::move(t), key, base, account);
}
};
} // namespace compat_test
+291
View File
@@ -0,0 +1,291 @@
// CompatChatModel: request mapping, row quirks, parsing and streaming. Replies
// are documented OpenAI examples (FIXTURE) or transcripts built from the
// documented chunk shape (LABELLED, not recorded provider responses).
#include "support.h"
#include "vendors/compat/chat_model.h"
using namespace molo::compat;
using namespace compat_test;
using molo::llm::ApiError;
using molo::llm::ChatChunk;
using molo::llm::ChatRequest;
using molo::llm::Message;
namespace {
ChatRequest base() {
ChatRequest r;
r.model = "m1";
r.messages.push_back(Message::user("hi"));
return r;
}
// FIXTURE: OpenAI API reference, createChatCompletion streaming example chunks.
const char* kStream =
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}]}\n\n"
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[],\"usage\":{\"prompt_tokens\":9,\"completion_tokens\":2,\"total_tokens\":11}}\n\n"
"data: [DONE]\n\n";
// LABELLED: tool call fragments in the documented delta shape, with a reasoning field.
const char* kToolStream =
"data: {\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"thinking\"},\"finish_reason\":null}]}\n\n"
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null}]}\n\n"
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\":\"}}]},\"finish_reason\":null}]}\n\n"
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"Rome\\\"}\"}}]},\"finish_reason\":null}]}\n\n"
"data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\n"
"data: [DONE]\n\n";
} // namespace
TEST(compat_chat, body_maps_the_neutral_request) {
Rig rig("together");
CompatChatModel m(rig.provider);
ChatRequest r = base();
r.system = "be brief";
r.max_tokens = 16;
r.temperature = 0.5;
r.top_p = 0.9;
r.stop = {"END"};
r.reasoning_effort = "low";
r.json_mode = true;
Json b = m.to_body(r, false);
ASSERT_EQ(b.find("model")->as_string(), std::string("m1"));
ASSERT_EQ(b.find("messages")->as_array().size(), 2u);
ASSERT_EQ(b.find("messages")->as_array()[0].find("role")->as_string(), std::string("system"));
ASSERT_EQ(b.find("max_tokens")->as_int(), 16L);
ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_object"));
ASSERT_EQ(b.find("reasoning_effort")->as_string(), std::string("low"));
ASSERT_TRUE(b.find("stream") == nullptr);
ASSERT_TRUE(m.to_body(r, true).find("stream")->as_bool());
}
TEST(compat_chat, schema_tools_and_tool_choice) {
Rig rig("together");
CompatChatModel m(rig.provider);
ChatRequest r = base();
r.response_schema = Json::parse("{\"type\":\"object\"}");
r.tools.push_back({"get_weather", "weather", Json::parse("{\"type\":\"object\"}")});
r.tool_choice = "get_weather";
Json b = m.to_body(r, false);
ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_schema"));
ASSERT_TRUE(b.find("response_format")->find("json_schema")->find("strict")->as_bool());
ASSERT_EQ(b.find("tool_choice")->find("function")->find("name")->as_string(), std::string("get_weather"));
r.tool_choice = "required";
ASSERT_EQ(m.to_body(r, false).find("tool_choice")->as_string(), std::string("required"));
ChatRequest none = base();
none.tool_choice = "auto";
ASSERT_THROW(m.to_body(none, false), ApiError); // tool_choice needs tools
ChatRequest both = base();
both.json_mode = true;
both.response_schema = Json::parse("{}");
ASSERT_THROW(m.to_body(both, false), ApiError);
}
TEST(compat_chat, tool_round_trip_messages_and_images) {
Rig rig("together");
CompatChatModel m(rig.provider);
ChatRequest r = base();
Message a;
a.role = molo::llm::Role::kAssistant;
molo::llm::ToolCall tc;
tc.id = "call_1";
tc.name = "f";
tc.arguments = Json::parse("{\"a\":1}");
a.parts.push_back(molo::llm::ContentPart::of_call(tc));
r.messages.push_back(a);
Message t;
t.role = molo::llm::Role::kTool;
t.parts.push_back(molo::llm::ContentPart::of_result({"call_1", "42", false}));
r.messages.push_back(t);
Message u;
u.role = molo::llm::Role::kUser;
u.parts.push_back(molo::llm::ContentPart::of_text("look"));
u.parts.push_back(molo::llm::ContentPart::of_image({"image/png", "abc", ""}));
r.messages.push_back(u);
Json b = m.to_body(r, false);
const auto& msgs = b.find("messages")->as_array();
ASSERT_EQ(msgs[1].find("tool_calls")->as_array()[0].find("function")->find("arguments")->as_string(), std::string("{\"a\":1}"));
ASSERT_TRUE(msgs[1].find("content")->is_null());
ASSERT_EQ(msgs[2].find("tool_call_id")->as_string(), std::string("call_1"));
ASSERT_TRUE(msgs[3].find("content")->is_array());
ASSERT_TRUE(has(msgs[3].find("content")->as_array()[1].find("image_url")->find("url")->as_string(), "data:image/png;base64,"));
Message empty_tool;
empty_tool.role = molo::llm::Role::kTool;
ChatRequest bad = base();
bad.messages.push_back(empty_tool);
ASSERT_THROW(m.to_body(bad, false), ApiError);
}
TEST(compat_chat, input_rules_run_before_any_traffic) {
Rig rig("together");
CompatChatModel m(rig.provider);
ChatRequest r = base();
r.model = "";
ASSERT_THROW(m.generate(r), ApiError); // no default model
r = base();
r.messages.clear();
ASSERT_THROW(m.generate(r), ApiError);
r = base();
r.max_tokens = 0;
ASSERT_THROW(m.generate(r), ApiError);
r = base();
r.temperature = 2.5;
ASSERT_THROW(m.generate(r), ApiError);
r = base();
r.top_p = 0.0;
ASSERT_THROW(m.generate(r), ApiError);
r = base();
r.stop = std::vector<std::string>(17, "x");
ASSERT_THROW(m.generate(r), ApiError);
r = base();
r.extra = Json::parse("{\"messages\":[]}");
ASSERT_THROW(m.generate(r), ApiError);
r = base();
r.extra = Json::parse("[1]");
ASSERT_THROW(m.generate(r), ApiError);
r = base();
r.model = std::string(300, 'x');
ASSERT_THROW(m.generate(r), ApiError);
ASSERT_EQ(rig.transport->send_count(), 0u);
CompatChatModel with_default(rig.provider, "dflt");
ASSERT_EQ(with_default.to_body(base(), false).find("model")->as_string(), std::string("m1"));
ChatRequest nomodel = base();
nomodel.model = "";
ASSERT_EQ(with_default.to_body(nomodel, false).find("model")->as_string(), std::string("dflt"));
}
TEST(compat_chat, groq_quirks_are_rejected_loudly) {
Rig rig("groq");
CompatChatModel m(rig.provider);
ChatRequest named = base();
named.messages[0].name = "alice";
ASSERT_THROW(m.to_body(named, false), ApiError);
for (const char* key : {"logprobs", "top_logprobs", "logit_bias"}) {
ChatRequest r = base();
r.extra = Json::object();
r.extra.set(key, Json(1));
ASSERT_THROW(m.to_body(r, false), ApiError);
}
ChatRequest n2 = base();
n2.extra = Json::parse("{\"n\":2}");
ASSERT_THROW(m.to_body(n2, false), ApiError);
ChatRequest n1 = base();
n1.extra = Json::parse("{\"n\":1}");
ASSERT_TRUE(m.to_body(n1, false).find("n") != nullptr);
ASSERT_EQ(rig.transport->send_count(), 0u);
// The same request is fine on a row without the quirk.
Rig other("together");
CompatChatModel o(other.provider);
ChatRequest ok = base();
ok.messages[0].name = "alice";
ok.extra = Json::parse("{\"logprobs\":true,\"n\":2}");
ASSERT_TRUE(o.to_body(ok, false).find("logprobs") != nullptr);
ChatRequest big = base();
big.extra = Json::parse("{\"n\":9}");
ASSERT_THROW(o.to_body(big, false), ApiError);
}
TEST(compat_chat, cerebras_rejects_tools_with_response_format) {
Rig rig("cerebras");
CompatChatModel m(rig.provider);
ChatRequest r = base();
r.json_mode = true;
ASSERT_TRUE(m.to_body(r, false).find("response_format") != nullptr);
r.tools.push_back({"f", "", Json::object()});
ASSERT_THROW(m.to_body(r, false), ApiError);
}
TEST(compat_chat, parses_reasoning_fields_and_tool_calls) {
Json doc = Json::parse(
"{\"id\":\"x\",\"model\":\"m\",\"choices\":[{\"index\":0,\"finish_reason\":\"tool_calls\",\"message\":"
"{\"role\":\"assistant\",\"content\":null,\"reasoning\":\"why\",\"tool_calls\":[{\"id\":\"c1\",\"type\":\"function\","
"\"function\":{\"name\":\"f\",\"arguments\":\"{\\\"a\\\":1}\"}},{\"id\":\"c2\",\"type\":\"function\","
"\"function\":{\"name\":\"g\",\"arguments\":\"not json\"}}]}}],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":4,"
"\"total_tokens\":7,\"prompt_tokens_details\":{\"cached_tokens\":2},\"completion_tokens_details\":{\"reasoning_tokens\":1}}}");
molo::llm::ChatResponse r = CompatChatModel::parse_response(doc);
ASSERT_EQ(r.finish_reason, std::string("tool_calls"));
ASSERT_EQ(r.message.parts[0].reasoning, std::string("why"));
auto calls = r.tool_calls();
ASSERT_EQ(calls.size(), 2u);
ASSERT_EQ(calls[0].arguments.find("a")->as_int(), 1L);
ASSERT_TRUE(calls[1].arguments.find("_invalid_json") != nullptr);
ASSERT_EQ(r.usage.cached_tokens, 2);
ASSERT_EQ(r.usage.reasoning_tokens, 1);
Json rc = Json::parse("{\"choices\":[{\"message\":{\"content\":\"a\",\"reasoning_content\":\"b\"}}]}");
ASSERT_EQ(CompatChatModel::parse_response(rc).message.parts[0].reasoning, std::string("b"));
ASSERT_THROW(CompatChatModel::parse_response(Json::parse("{\"choices\":[]}")), ApiError);
ASSERT_THROW(CompatChatModel::parse_response(Json::parse("{\"choices\":[{}]}")), ApiError);
}
TEST(compat_chat, stream_documented_transcript_with_usage_chunk) {
Rig rig("fireworks");
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse(kStream));
CompatChatModel m(rig.provider);
std::string text, finish;
int total = 0;
m.stream(base(), [&](const ChatChunk& c) {
text += c.text_delta;
if (!c.finish_reason.empty()) finish = c.finish_reason;
if (c.usage) total = c.usage->total_tokens;
return true;
});
ASSERT_EQ(text, std::string("Hello!"));
ASSERT_EQ(finish, std::string("stop"));
ASSERT_EQ(total, 11);
ASSERT_TRUE(has(rig.transport->seen()[0].body, "\"stream\":true"));
ASSERT_FALSE(has(rig.transport->seen()[0].body, "stream_options"));
}
TEST(compat_chat, stream_tool_fragments_carry_id_and_name) {
Rig rig("fireworks");
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse(kToolStream));
CompatChatModel m(rig.provider);
std::string args, id, name, reasoning;
m.stream(base(), [&](const ChatChunk& c) {
args += c.tool_args_delta;
if (!c.tool_call_id.empty()) id = c.tool_call_id;
if (!c.tool_call_name.empty()) name = c.tool_call_name;
reasoning += c.reasoning_delta;
return true;
});
ASSERT_EQ(args, std::string("{\"city\":\"Rome\"}"));
ASSERT_EQ(id, std::string("call_1"));
ASSERT_EQ(name, std::string("get_weather"));
ASSERT_EQ(reasoning, std::string("thinking"));
}
TEST(compat_chat, stream_without_finish_reason_is_truncated_and_sink_can_stop) {
Rig rig("fireworks");
rig.transport->expect_response(
Method::kPost, rig.path("/chat/completions"),
sse("data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"a\"},\"finish_reason\":null}]}\n\n"));
CompatChatModel m(rig.provider);
try {
m.stream(base(), [](const ChatChunk&) { return true; });
FAIL("expected truncation");
} catch (const ApiError& e) {
ASSERT_EQ(e.http_status(), 502);
ASSERT_EQ(e.type(), std::string("STREAM_TRUNCATED"));
}
Rig rig2("fireworks");
rig2.transport->expect_response(Method::kPost, rig2.path("/chat/completions"), sse(kStream));
CompatChatModel m2(rig2.provider);
int seen = 0;
m2.stream(base(), [&](const ChatChunk&) { return ++seen < 2; }); // stop early: no throw
ASSERT_EQ(seen, 2);
}
TEST(compat_chat, stream_error_event_throws) {
Rig rig("fireworks");
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"),
sse("data: {\"error\":{\"type\":\"overloaded\",\"message\":\"busy\"}}\n\n"));
CompatChatModel m(rig.provider);
try {
m.stream(base(), [](const ChatChunk&) { return true; });
FAIL("expected");
} catch (const ApiError& e) {
ASSERT_EQ(e.type(), std::string("overloaded"));
}
}
+115
View File
@@ -0,0 +1,115 @@
// Opt-in live checks, driven by the registry table. MOLO_LIVE_NET=1 enables
// the network. Per row:
// - probe (no key needed): an invalid key must be answered like an API, never
// 404 and never a transport failure; the status and error type are printed;
// - with the row's key env set: list models (when the row claims it) and ONE
// completion of max_tokens 8, on the model named by COMPAT_<ID>_MODEL
// (upper case id). A missing key or model prints a loud SKIP line.
// Nothing is created or deleted. Local rows run only when their base env is set.
#include <algorithm>
#include <cstdlib>
#include <iostream>
#include "clients/transports/http_transport.h"
#include "llm/config.h"
#include "support.h"
#include "vendors/compat/chat_model.h"
#include "vendors/compat/services.h"
using namespace molo::compat;
using namespace compat_test;
using molo::llm::ApiError;
namespace {
// Read at static-init time: another suite in the shared binary unsets
// MOLO_LIVE_NET before the tests run.
const bool kNet = std::getenv("MOLO_LIVE_NET") != nullptr;
bool net() { return kNet; }
std::string upper(std::string s) {
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return static_cast<char>(std::toupper(c)); });
return s;
}
Provider probe_provider(const ProviderRow& row) {
return Provider::with(row.id, std::make_unique<molo::clients::transports::HttpTransport>(), "sk-invalid-molo-probe",
"", "probe-account");
}
} // namespace
TEST(compat_live, invalid_key_probe_answers_like_an_api_for_every_hosted_row) {
if (!net()) {
std::cout << "SKIP compat_live.invalid_key_probe (set MOLO_LIVE_NET=1)\n";
return;
}
int unreachable = 0;
for (std::size_t i = 0; i < Registry::count(); ++i) {
const ProviderRow& row = Registry::at(i);
if (row.local || std::string(row.extra_env).size() > 0) continue; // no hosted URL to probe
Provider p = probe_provider(row);
int status = 200;
std::string type = "ok", detail;
try {
if (Registry::has(row, Feature::kModels)) {
CompatModels(p).list();
} else {
CompatChatModel m(p);
molo::llm::ChatRequest r;
r.model = "molo-probe";
r.messages.push_back(molo::llm::Message::user("hi"));
r.max_tokens = 1;
m.generate(r);
}
} catch (const ApiError& e) {
status = e.http_status();
type = e.type();
detail = e.message().substr(0, 160);
}
std::cout << " " << row.id << ": " << status << " " << type << "\n";
if (status == 502) {
++unreachable; // transport failure: reported, not asserted (network flake)
std::cout << " WARN " << row.id << " unreachable from here: " << detail << "\n";
continue;
}
ASSERT_TRUE(status != 404);
}
std::cout << " unreachable rows: " << unreachable << "\n";
}
TEST(compat_live, models_and_one_tiny_completion_for_each_row_with_a_key) {
if (!net()) {
std::cout << "SKIP compat_live.models_and_one_tiny_completion (set MOLO_LIVE_NET=1 and a provider key env)\n";
return;
}
int ran = 0;
for (std::size_t i = 0; i < Registry::count(); ++i) {
const ProviderRow& row = Registry::at(i);
Provider p = Provider::from_env(row.id);
if (!p.configured()) {
std::cout << "SKIP compat_live." << row.id << " (set " << p.missing_env() << ")\n";
continue;
}
++ran;
if (Registry::has(row, Feature::kModels) && !std::string(row.models_path).empty()) {
ModelList list = CompatModels(p).list();
std::cout << " " << row.id << " models: " << list.models.size() << "\n";
ASSERT_TRUE(!list.models.empty());
}
const std::string model_env = "COMPAT_" + upper(row.id) + "_MODEL";
std::string model = molo::llm::Config::get(model_env.c_str());
if (model.empty()) {
std::cout << "SKIP compat_live." << row.id << " completion (set " << model_env << ")\n";
continue;
}
CompatChatModel chat(p, model);
molo::llm::ChatRequest r;
r.messages.push_back(molo::llm::Message::user("Reply with the single word: ok"));
r.max_tokens = 8;
molo::llm::ChatResponse out = chat.generate(r);
std::cout << " " << row.id << " completion finish=" << out.finish_reason << " tokens=" << out.usage.total_tokens << "\n";
ASSERT_TRUE(!out.id.empty() || !out.finish_reason.empty());
}
if (ran == 0) std::cout << "SKIP compat_live: no provider key is configured\n";
}
+176
View File
@@ -0,0 +1,176 @@
// Provider wiring over the real ApiClient with a recording transport, driven
// by the registry table: every row is exercised, so a new row is tested
// without a new test.
#include <cstdlib>
#include "support.h"
#include "vendors/compat/chat_model.h"
#include "vendors/compat/services.h"
using namespace molo::compat;
using namespace compat_test;
using molo::llm::ApiError;
namespace {
// FIXTURE: OpenAI API reference, createChatCompletion "Default" response example.
const char* kChatExample =
"{\"id\":\"chatcmpl-B9MBs8CjcvOU2jLn4n570S5qMJKcT\",\"object\":\"chat.completion\",\"created\":1741569952,"
"\"model\":\"gpt-4.1-2025-04-14\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\","
"\"content\":\"Hello! How can I assist you today?\",\"refusal\":null,\"annotations\":[]},\"logprobs\":null,"
"\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":19,\"completion_tokens\":10,\"total_tokens\":29,"
"\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":"
"{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},"
"\"service_tier\":\"default\"}";
// FIXTURE: OpenAI API reference, listModels response example (two entries).
const char* kModelsExample =
"{\"object\":\"list\",\"data\":[{\"id\":\"model-id-0\",\"object\":\"model\",\"created\":1686935002,"
"\"owned_by\":\"organization-owner\"},{\"id\":\"model-id-1\",\"object\":\"model\",\"created\":1686935002,"
"\"owned_by\":\"organization-owner\"}]}";
molo::llm::ChatRequest hello() {
molo::llm::ChatRequest r;
r.model = "some-model";
r.messages.push_back(molo::llm::Message::user("Hello!"));
return r;
}
} // namespace
TEST(compat_provider, every_row_posts_chat_to_its_own_base_with_a_bearer_key) {
for (std::size_t i = 0; i < Registry::count(); ++i) {
const ProviderRow& row = Registry::at(i);
Rig rig(row.id);
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), reply(kChatExample));
CompatChatModel model(rig.provider);
molo::llm::ChatResponse out = model.generate(hello());
ASSERT_EQ(out.text(), std::string("Hello! How can I assist you today?"));
ASSERT_EQ(out.usage.total_tokens, 29);
auto s = rig.transport->seen()[0];
ASSERT_EQ(s.method, std::string("POST"));
ASSERT_EQ(s.headers.get_or("Authorization", ""), std::string("Bearer ") + kKey);
ASSERT_EQ(s.target.find(kKey), std::string::npos);
ASSERT_EQ(model.vendor(), std::string("compat:") + row.id);
}
}
TEST(compat_provider, every_models_row_lists_and_every_other_row_refuses_without_traffic) {
for (std::size_t i = 0; i < Registry::count(); ++i) {
const ProviderRow& row = Registry::at(i);
Rig rig(row.id);
CompatModels models(rig.provider);
if (Registry::has(row, Feature::kModels)) {
rig.transport->expect_response(Method::kGet, rig.path("/models"), reply(kModelsExample));
ModelList list = models.list();
ASSERT_EQ(list.models.size(), 2u);
ASSERT_TRUE(list.contains("model-id-1"));
ASSERT_EQ(rig.transport->seen()[0].method, std::string("GET"));
} else {
ASSERT_THROW(models.list(), ApiError);
ASSERT_EQ(rig.transport->send_count(), 0u);
}
}
}
TEST(compat_provider, features_not_in_the_row_never_reach_the_network) {
Rig rig("cerebras");
CompatEmbeddings emb(rig.provider);
ASSERT_THROW(emb.embed("m", {"x"}), ApiError);
CompatImages img(rig.provider);
ImageRequest ir;
ir.model = "m";
ir.prompt = "p";
ASSERT_THROW(img.generate(ir), ApiError);
CompatAudio audio(rig.provider);
SpeechRequest sr;
sr.model = "m";
sr.input = "x";
sr.voice = "v";
std::string ct;
ASSERT_THROW(audio.speak(sr, ct), ApiError);
ASSERT_EQ(rig.transport->send_count(), 0u);
}
TEST(compat_provider, unconfigured_provider_has_no_traffic_and_names_the_env) {
Rig rig("groq", "");
ASSERT_FALSE(rig.provider.configured());
ASSERT_EQ(rig.provider.missing_env(), std::string("GROQ_API_KEY"));
CompatChatModel model(rig.provider);
try {
model.generate(hello());
FAIL("expected");
} catch (const molo::llm::NotConfiguredError& e) {
ASSERT_EQ(e.http_status(), 503);
ASSERT_TRUE(has(e.message(), "GROQ_API_KEY"));
}
ASSERT_EQ(rig.transport->send_count(), 0u);
}
TEST(compat_provider, cloudflare_substitutes_the_account_id_and_validates_it) {
Rig rig("cloudflare", kKey, "", "a1b2c3");
ASSERT_EQ(rig.provider.base_url(), std::string("https://api.cloudflare.com/client/v4/accounts/a1b2c3/ai/v1"));
ASSERT_THROW(Rig("cloudflare", kKey, "", ""), ApiError);
ASSERT_THROW(Rig("cloudflare", kKey, "", "a/../b"), ApiError);
ASSERT_THROW(Rig("cloudflare", kKey, "", "a b"), ApiError);
}
TEST(compat_provider, base_override_is_validated_and_trailing_slash_removed) {
Rig ok("vllm", "", "http://10.0.0.5:9000/v1/");
ASSERT_EQ(ok.provider.base_url(), std::string("http://10.0.0.5:9000/v1"));
ASSERT_THROW(Rig("vllm", "", "ftp://x"), ApiError);
ASSERT_THROW(Rig("vllm", "", "http://x/v1?a=b"), ApiError);
ASSERT_THROW(Rig("vllm", "", "http://x/../v1"), ApiError);
ASSERT_THROW(Rig("vllm", "", "http://x y/v1"), ApiError);
ASSERT_THROW(Rig("vllm", "", std::string("http://x/") + std::string(600, 'a')), ApiError);
}
TEST(compat_provider, local_rows_are_configured_only_when_their_base_env_is_set) {
setenv("MOLO_ENV_FILES", "off", 1);
unsetenv("OLLAMA_BASE_URL");
unsetenv("OLLAMA_API_KEY");
Provider off = Provider::from_env("ollama");
ASSERT_FALSE(off.configured());
ASSERT_EQ(off.missing_env(), std::string("OLLAMA_BASE_URL"));
setenv("OLLAMA_BASE_URL", "http://127.0.0.1:11434/v1", 1);
Provider on = Provider::from_env("ollama");
ASSERT_TRUE(on.configured());
ASSERT_EQ(on.base_url(), std::string("http://127.0.0.1:11434/v1"));
unsetenv("OLLAMA_BASE_URL");
}
TEST(compat_provider, hosted_from_env_needs_its_key_and_cloudflare_needs_the_account) {
setenv("MOLO_ENV_FILES", "off", 1);
unsetenv("CLOUDFLARE_API_TOKEN");
unsetenv("CLOUDFLARE_ACCOUNT_ID");
unsetenv("GROQ_API_KEY");
ASSERT_FALSE(Provider::from_env("groq").configured());
setenv("CLOUDFLARE_API_TOKEN", "t", 1);
Provider cf = Provider::from_env("cloudflare");
ASSERT_FALSE(cf.configured());
ASSERT_EQ(cf.missing_env(), std::string("CLOUDFLARE_ACCOUNT_ID"));
setenv("CLOUDFLARE_ACCOUNT_ID", "acct9", 1);
ASSERT_TRUE(Provider::from_env("cloudflare").configured());
unsetenv("CLOUDFLARE_API_TOKEN");
unsetenv("CLOUDFLARE_ACCOUNT_ID");
}
TEST(compat_provider, error_envelopes_of_different_providers_map_to_api_error) {
// Shapes seen in the unauthenticated probes of SPEC section 5 (labelled: probe output, 2026-10-25).
struct Case { const char* id; int status; const char* body; const char* type; };
const Case cases[] = {
{"groq", 401, "{\"error\":{\"message\":\"Invalid API Key\",\"type\":\"invalid_request_error\",\"code\":\"invalid_api_key\"}}", "invalid_request_error"},
{"cerebras", 401, "{\"message\":\"Wrong API Key\",\"type\":\"invalid_request_error\",\"param\":\"api_key\",\"code\":\"wrong_api_key\"}", "invalid_request_error"},
{"moonshot", 401, "{\"error\":{\"message\":\"Incorrect API key provided\",\"type\":\"incorrect_api_key_error\"}}", "incorrect_api_key_error"},
};
for (const Case& c : cases) {
Rig rig(c.id);
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), reply(c.body, c.status));
CompatChatModel model(rig.provider);
try {
model.generate(hello());
FAIL("expected");
} catch (const ApiError& e) {
ASSERT_EQ(e.http_status(), c.status);
ASSERT_EQ(e.type(), std::string(c.type));
}
}
}
+105
View File
@@ -0,0 +1,105 @@
// The registry table is the single source: these tests pin its invariants so
// that adding a row cannot break consistency between components.
#include <set>
#include "support.h"
#include "vendors/compat/calls.h"
#include "vendors/compat/registry.h"
using namespace molo::compat;
using namespace compat_test;
TEST(compat_registry, rows_are_unique_slugs_with_complete_fields) {
std::set<std::string> ids;
ASSERT_TRUE(Registry::count() >= 35);
for (std::size_t i = 0; i < Registry::count(); ++i) {
const ProviderRow& r = Registry::at(i);
std::string id = r.id;
ASSERT_TRUE(!id.empty() && id.size() <= 32);
for (char c : id) ASSERT_TRUE((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'));
ASSERT_TRUE(ids.insert(id).second);
ASSERT_TRUE(std::string(r.name).size() > 2);
ASSERT_TRUE(std::string(r.key_env).size() > 3);
ASSERT_TRUE(std::string(r.doc_url).rfind("https://", 0) == 0);
ASSERT_TRUE(std::string(r.note).size() > 10);
ASSERT_TRUE((r.features & Feature::kChat) != 0); // every row is a chat provider
}
}
TEST(compat_registry, base_urls_are_wellformed) {
std::set<std::string> envs;
for (std::size_t i = 0; i < Registry::count(); ++i) {
const ProviderRow& r = Registry::at(i);
std::string base = r.base_url;
ASSERT_TRUE(base.back() != '/');
ASSERT_TRUE(base.find(' ') == std::string::npos);
if (r.local) {
ASSERT_TRUE(base.rfind("http://localhost:", 0) == 0);
ASSERT_TRUE(std::string(r.base_env).size() > 3); // the "configured" switch
} else {
ASSERT_TRUE(base.rfind("https://", 0) == 0);
ASSERT_TRUE(std::string(r.base_env).empty());
}
ASSERT_EQ(base.find("{account_id}") != std::string::npos, std::string(r.extra_env).size() > 0);
// Credential names are unique except where two rows deliberately share none.
ASSERT_TRUE(envs.insert(r.key_env).second);
}
}
TEST(compat_registry, models_feature_and_path_agree) {
for (std::size_t i = 0; i < Registry::count(); ++i) {
const ProviderRow& r = Registry::at(i);
bool claims = Registry::has(r, Feature::kModels);
bool path = std::string(r.models_path).size() > 0;
ASSERT_EQ(claims, path);
if (path) ASSERT_EQ(std::string(r.models_path), std::string("/models"));
}
}
TEST(compat_registry, quirk_rows_carry_their_documented_flag) {
const ProviderRow* groq = Registry::find("groq");
ASSERT_TRUE(groq != nullptr);
ASSERT_TRUE((groq->quirks & Quirk::kRejectMessageName) != 0);
ASSERT_TRUE((groq->quirks & Quirk::kRejectLogprobs) != 0);
ASSERT_TRUE((groq->quirks & Quirk::kSingleChoice) != 0);
ASSERT_TRUE((Registry::get("cerebras").quirks & Quirk::kNoToolsWithFormat) != 0);
ASSERT_EQ(Registry::get("cerebras").features & Feature::kEmbeddings, 0u);
}
TEST(compat_registry, evidence_matches_row_kind) {
for (std::size_t i = 0; i < Registry::count(); ++i) {
const ProviderRow& r = Registry::at(i);
if (r.local) ASSERT_TRUE(r.evidence == Evidence::kDocsOnly);
ASSERT_TRUE(std::string(Registry::evidence_name(r.evidence)).size() > 4);
}
ASSERT_TRUE(Registry::get("cloudflare").evidence == Evidence::kDocsOnly);
}
TEST(compat_registry, no_em_dash_in_any_string) {
const std::string em = "\xE2\x80\x94";
for (std::size_t i = 0; i < Registry::count(); ++i) {
const ProviderRow& r = Registry::at(i);
for (const char* s : {r.id, r.name, r.base_url, r.key_env, r.doc_url, r.note})
ASSERT_TRUE(std::string(s).find(em) == std::string::npos);
}
}
TEST(compat_registry, lookup_and_unknown_id) {
ASSERT_TRUE(Registry::find("groq") != nullptr);
ASSERT_TRUE(Registry::find("nope") == nullptr);
ASSERT_THROW(Registry::get("nope"), molo::llm::ApiError);
ASSERT_THROW(Registry::at(Registry::count()), molo::llm::ApiError);
ASSERT_EQ(Registry::feature_names(Registry::get("cerebras")), std::string("chat,models"));
}
TEST(compat_registry, call_table_rows_map_to_openai_operations) {
ASSERT_EQ(Calls::count(), 6u);
ASSERT_EQ(std::string(Calls::spec_of(Call::kChat).path), std::string("/chat/completions"));
ASSERT_EQ(std::string(Calls::spec_of(Call::kModels).operation_id), std::string("listModels"));
ASSERT_TRUE(Calls::method_of(Call::kModels) == Method::kGet);
ASSERT_TRUE(Calls::method_of(Call::kSpeech) == Method::kPost);
for (std::size_t i = 0; i < Calls::count(); ++i) {
ASSERT_TRUE(std::string(Calls::at(i).path).front() == '/');
ASSERT_TRUE(std::string(Calls::at(i).operation_id).size() > 5);
}
}
+142
View File
@@ -0,0 +1,142 @@
// Models, embeddings, images and audio over the recording transport. Replies
// are the documented OpenAI examples (FIXTURE); assertions are on what WE send
// and how WE parse.
#include "support.h"
#include "vendors/compat/services.h"
using namespace molo::compat;
using namespace compat_test;
using molo::llm::ApiError;
TEST(compat_services, models_accepts_a_bare_array_and_skips_entries_without_id) {
// Together returns a bare array; shape recorded from its docs page (LABELLED).
ModelList l = CompatModels::parse(Json::parse("[{\"id\":\"a\"},{\"name\":\"b\"},{\"x\":1}]"));
ASSERT_EQ(l.models.size(), 2u);
ASSERT_TRUE(l.contains("b"));
ASSERT_THROW(CompatModels::parse(Json::parse("{\"data\":5}")), ApiError);
ASSERT_THROW(CompatModels::parse(Json::parse("{}")), ApiError);
}
TEST(compat_services, models_list_truncates_loudly) {
Json arr = Json::array();
for (std::size_t i = 0; i < 5010; ++i) {
Json o = Json::object();
o.set("id", Json("m" + std::to_string(i)));
arr.push(o);
}
ModelList l = CompatModels::parse(arr);
ASSERT_EQ(l.models.size(), 5000u);
ASSERT_TRUE(l.truncated);
}
TEST(compat_services, row_without_list_endpoint_refuses) {
Rig rig("dashscope");
CompatModels m(rig.provider);
ASSERT_THROW(m.list(), ApiError);
ASSERT_EQ(rig.transport->send_count(), 0u);
}
TEST(compat_services, embeddings_body_rules_and_parse) {
Rig rig("nvidia");
CompatEmbeddings e(rig.provider);
Json b = e.to_body("m", {"a", "b"}, 256);
ASSERT_EQ(b.find("input")->as_array().size(), 2u);
ASSERT_EQ(b.find("dimensions")->as_int(), 256L);
ASSERT_TRUE(e.to_body("m", {"a"}, 0).find("dimensions") == nullptr);
ASSERT_THROW(e.to_body("", {"a"}, 0), ApiError);
ASSERT_THROW(e.to_body("m", {}, 0), ApiError);
ASSERT_THROW(e.to_body("m", {""}, 0), ApiError);
ASSERT_THROW(e.to_body("m", {"a"}, -1), ApiError);
ASSERT_THROW(e.to_body("m", std::vector<std::string>(2049, "x"), 0), ApiError);
// FIXTURE: OpenAI API reference, createEmbedding response example.
rig.transport->expect_response(
Method::kPost, rig.path("/embeddings"),
reply("{\"object\":\"list\",\"data\":[{\"object\":\"embedding\",\"embedding\":[0.0023064255,-0.009327292,-0.0028842222],"
"\"index\":0}],\"model\":\"text-embedding-ada-002\",\"usage\":{\"prompt_tokens\":8,\"total_tokens\":8}}"));
EmbeddingResult r = e.embed("text-embedding-ada-002", {"The food was delicious"});
ASSERT_EQ(r.items.size(), 1u);
ASSERT_EQ(r.items[0].vector.size(), 3u);
ASSERT_EQ(r.total_tokens, 8);
ASSERT_THROW(CompatEmbeddings::parse(Json::parse("{\"data\":[{\"embedding\":5}]}")), ApiError);
}
TEST(compat_services, images_body_rules_and_parse) {
Rig rig("together");
CompatImages img(rig.provider);
ImageRequest r;
r.model = "m";
r.prompt = "a cat";
r.size = "512x512";
r.response_format = "b64_json";
Json b = img.to_body(r);
ASSERT_EQ(b.find("size")->as_string(), std::string("512x512"));
r.size = "big";
ASSERT_THROW(img.to_body(r), ApiError);
r.size = "x512";
ASSERT_THROW(img.to_body(r), ApiError);
r.size = "";
r.n = 11;
ASSERT_THROW(img.to_body(r), ApiError);
r.n = 1;
r.response_format = "png";
ASSERT_THROW(img.to_body(r), ApiError);
r.response_format = "";
r.prompt = "";
ASSERT_THROW(img.to_body(r), ApiError);
// FIXTURE: OpenAI API reference, createImage response example.
auto items = CompatImages::parse(Json::parse("{\"created\":1713833628,\"data\":[{\"b64_json\":\"...\",\"revised_prompt\":\"x\"}]}"));
ASSERT_EQ(items.size(), 1u);
ASSERT_EQ(items[0].revised_prompt, std::string("x"));
ASSERT_THROW(CompatImages::parse(Json::parse("{}")), ApiError);
}
TEST(compat_services, transcription_sends_multipart_and_validates) {
Rig rig("groq");
CompatAudio audio(rig.provider);
TranscribeRequest t;
t.model = "whisper-large-v3";
t.filename = "a.wav";
t.audio = "RIFFxxxx";
t.language = "en";
// FIXTURE: OpenAI API reference, createTranscription json response example.
rig.transport->expect_response(Method::kPost, rig.path("/audio/transcriptions"),
reply("{\"text\":\"Imagine the wildest idea that you've ever had.\"}"));
ASSERT_TRUE(has(audio.transcribe(t), "wildest idea"));
auto s = rig.transport->seen()[0];
ASSERT_TRUE(has(s.headers.get_or("Content-Type", ""), "multipart/form-data"));
ASSERT_TRUE(has(s.body, "name=\"file\"; filename=\"a.wav\""));
ASSERT_TRUE(has(s.body, "whisper-large-v3"));
TranscribeRequest bad = t;
bad.audio.clear();
ASSERT_THROW(audio.transcribe(bad), ApiError);
bad = t;
bad.filename = "a\".wav";
ASSERT_THROW(audio.transcribe(bad), ApiError);
bad = t;
bad.filename = "../a.wav";
ASSERT_THROW(audio.transcribe(bad), ApiError);
Rig no("cerebras");
ASSERT_THROW(CompatAudio(no.provider).transcribe(t), ApiError);
}
TEST(compat_services, speech_returns_bytes_and_content_type) {
Rig rig("together");
CompatAudio audio(rig.provider);
SpeechRequest s;
s.model = "m";
s.input = "hello";
s.voice = "alloy";
s.format = "mp3";
rig.transport->expect_response(Method::kPost, rig.path("/audio/speech"), reply("ID3bytes", 200, "audio/mpeg"));
std::string ct;
ASSERT_EQ(audio.speak(s, ct), std::string("ID3bytes"));
ASSERT_EQ(ct, std::string("audio/mpeg"));
s.format = "ogg";
ASSERT_THROW(audio.speech_body(s), ApiError);
s.format = "";
s.voice = "";
ASSERT_THROW(audio.speech_body(s), ApiError);
s.voice = "v";
s.input = std::string(4097, 'x');
ASSERT_THROW(audio.speech_body(s), ApiError);
}