feat(clients): compat: OpenAI-compatible provider registry (39 rows), chat model, services, demo
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Vendored
+63
@@ -0,0 +1,63 @@
|
||||
#pragma once
|
||||
|
||||
// Shared rig for the compat tests: a Provider over a MockTransport with a
|
||||
// fake key. The mock only records the requests the real client builds;
|
||||
// replies are documented examples taken verbatim from OpenAI's API reference
|
||||
// (the shape every row shares) and labelled FIXTURE where they appear.
|
||||
// Nothing here verifies a provider's behavior (see SPEC.md ## Unverified).
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "clients/transports/mock_transport.h"
|
||||
#include "framework.h"
|
||||
#include "vendors/compat/provider.h"
|
||||
|
||||
namespace compat_test {
|
||||
|
||||
using molo::clients::ClientResponse;
|
||||
using molo::clients::HeaderMap;
|
||||
using molo::clients::Method;
|
||||
using molo::clients::transports::MockTransport;
|
||||
using Json = molo::clients::body::Json;
|
||||
|
||||
inline ClientResponse reply(const std::string& body, int status = 200,
|
||||
const std::string& type = "application/json") {
|
||||
HeaderMap h;
|
||||
h.set("Content-Type", type);
|
||||
return ClientResponse(status, h, body);
|
||||
}
|
||||
|
||||
inline ClientResponse sse(const std::string& body) { return reply(body, 200, "text/event-stream"); }
|
||||
|
||||
inline const char* kKey = "sk-TESTKEY-DO-NOT-LEAK";
|
||||
|
||||
inline bool has(const std::string& hay, const std::string& needle) { return hay.find(needle) != std::string::npos; }
|
||||
|
||||
// "https://host/a/b" -> "/a/b"
|
||||
inline std::string path_of(const std::string& url) {
|
||||
std::size_t scheme = url.find("://");
|
||||
std::size_t slash = url.find('/', scheme == std::string::npos ? 0 : scheme + 3);
|
||||
return slash == std::string::npos ? "" : url.substr(slash);
|
||||
}
|
||||
|
||||
struct Rig {
|
||||
MockTransport* transport = nullptr;
|
||||
molo::compat::Provider provider;
|
||||
|
||||
explicit Rig(const std::string& id, const std::string& key = kKey, const std::string& base_override = "",
|
||||
const std::string& account = "acct123")
|
||||
: provider(make(id, key, base_override, account, &transport)) {}
|
||||
|
||||
std::string path(const char* suffix) const { return path_of(provider.base_url()) + suffix; }
|
||||
|
||||
private:
|
||||
static molo::compat::Provider make(const std::string& id, const std::string& key, const std::string& base,
|
||||
const std::string& account, MockTransport** out) {
|
||||
auto t = std::make_unique<MockTransport>();
|
||||
*out = t.get();
|
||||
return molo::compat::Provider::with(id, std::move(t), key, base, account);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace compat_test
|
||||
+291
@@ -0,0 +1,291 @@
|
||||
// CompatChatModel: request mapping, row quirks, parsing and streaming. Replies
|
||||
// are documented OpenAI examples (FIXTURE) or transcripts built from the
|
||||
// documented chunk shape (LABELLED, not recorded provider responses).
|
||||
#include "support.h"
|
||||
#include "vendors/compat/chat_model.h"
|
||||
|
||||
using namespace molo::compat;
|
||||
using namespace compat_test;
|
||||
using molo::llm::ApiError;
|
||||
using molo::llm::ChatChunk;
|
||||
using molo::llm::ChatRequest;
|
||||
using molo::llm::Message;
|
||||
|
||||
namespace {
|
||||
|
||||
ChatRequest base() {
|
||||
ChatRequest r;
|
||||
r.model = "m1";
|
||||
r.messages.push_back(Message::user("hi"));
|
||||
return r;
|
||||
}
|
||||
|
||||
// FIXTURE: OpenAI API reference, createChatCompletion streaming example chunks.
|
||||
const char* kStream =
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}]}\n\n"
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[],\"usage\":{\"prompt_tokens\":9,\"completion_tokens\":2,\"total_tokens\":11}}\n\n"
|
||||
"data: [DONE]\n\n";
|
||||
|
||||
// LABELLED: tool call fragments in the documented delta shape, with a reasoning field.
|
||||
const char* kToolStream =
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"thinking\"},\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\":\"}}]},\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"Rome\\\"}\"}}]},\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\n"
|
||||
"data: [DONE]\n\n";
|
||||
} // namespace
|
||||
|
||||
TEST(compat_chat, body_maps_the_neutral_request) {
|
||||
Rig rig("together");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
r.system = "be brief";
|
||||
r.max_tokens = 16;
|
||||
r.temperature = 0.5;
|
||||
r.top_p = 0.9;
|
||||
r.stop = {"END"};
|
||||
r.reasoning_effort = "low";
|
||||
r.json_mode = true;
|
||||
Json b = m.to_body(r, false);
|
||||
ASSERT_EQ(b.find("model")->as_string(), std::string("m1"));
|
||||
ASSERT_EQ(b.find("messages")->as_array().size(), 2u);
|
||||
ASSERT_EQ(b.find("messages")->as_array()[0].find("role")->as_string(), std::string("system"));
|
||||
ASSERT_EQ(b.find("max_tokens")->as_int(), 16L);
|
||||
ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_object"));
|
||||
ASSERT_EQ(b.find("reasoning_effort")->as_string(), std::string("low"));
|
||||
ASSERT_TRUE(b.find("stream") == nullptr);
|
||||
ASSERT_TRUE(m.to_body(r, true).find("stream")->as_bool());
|
||||
}
|
||||
|
||||
TEST(compat_chat, schema_tools_and_tool_choice) {
|
||||
Rig rig("together");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
r.response_schema = Json::parse("{\"type\":\"object\"}");
|
||||
r.tools.push_back({"get_weather", "weather", Json::parse("{\"type\":\"object\"}")});
|
||||
r.tool_choice = "get_weather";
|
||||
Json b = m.to_body(r, false);
|
||||
ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_schema"));
|
||||
ASSERT_TRUE(b.find("response_format")->find("json_schema")->find("strict")->as_bool());
|
||||
ASSERT_EQ(b.find("tool_choice")->find("function")->find("name")->as_string(), std::string("get_weather"));
|
||||
r.tool_choice = "required";
|
||||
ASSERT_EQ(m.to_body(r, false).find("tool_choice")->as_string(), std::string("required"));
|
||||
ChatRequest none = base();
|
||||
none.tool_choice = "auto";
|
||||
ASSERT_THROW(m.to_body(none, false), ApiError); // tool_choice needs tools
|
||||
ChatRequest both = base();
|
||||
both.json_mode = true;
|
||||
both.response_schema = Json::parse("{}");
|
||||
ASSERT_THROW(m.to_body(both, false), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, tool_round_trip_messages_and_images) {
|
||||
Rig rig("together");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
Message a;
|
||||
a.role = molo::llm::Role::kAssistant;
|
||||
molo::llm::ToolCall tc;
|
||||
tc.id = "call_1";
|
||||
tc.name = "f";
|
||||
tc.arguments = Json::parse("{\"a\":1}");
|
||||
a.parts.push_back(molo::llm::ContentPart::of_call(tc));
|
||||
r.messages.push_back(a);
|
||||
Message t;
|
||||
t.role = molo::llm::Role::kTool;
|
||||
t.parts.push_back(molo::llm::ContentPart::of_result({"call_1", "42", false}));
|
||||
r.messages.push_back(t);
|
||||
Message u;
|
||||
u.role = molo::llm::Role::kUser;
|
||||
u.parts.push_back(molo::llm::ContentPart::of_text("look"));
|
||||
u.parts.push_back(molo::llm::ContentPart::of_image({"image/png", "abc", ""}));
|
||||
r.messages.push_back(u);
|
||||
Json b = m.to_body(r, false);
|
||||
const auto& msgs = b.find("messages")->as_array();
|
||||
ASSERT_EQ(msgs[1].find("tool_calls")->as_array()[0].find("function")->find("arguments")->as_string(), std::string("{\"a\":1}"));
|
||||
ASSERT_TRUE(msgs[1].find("content")->is_null());
|
||||
ASSERT_EQ(msgs[2].find("tool_call_id")->as_string(), std::string("call_1"));
|
||||
ASSERT_TRUE(msgs[3].find("content")->is_array());
|
||||
ASSERT_TRUE(has(msgs[3].find("content")->as_array()[1].find("image_url")->find("url")->as_string(), "data:image/png;base64,"));
|
||||
Message empty_tool;
|
||||
empty_tool.role = molo::llm::Role::kTool;
|
||||
ChatRequest bad = base();
|
||||
bad.messages.push_back(empty_tool);
|
||||
ASSERT_THROW(m.to_body(bad, false), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, input_rules_run_before_any_traffic) {
|
||||
Rig rig("together");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
r.model = "";
|
||||
ASSERT_THROW(m.generate(r), ApiError); // no default model
|
||||
r = base();
|
||||
r.messages.clear();
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.max_tokens = 0;
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.temperature = 2.5;
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.top_p = 0.0;
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.stop = std::vector<std::string>(17, "x");
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.extra = Json::parse("{\"messages\":[]}");
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.extra = Json::parse("[1]");
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.model = std::string(300, 'x');
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
ASSERT_EQ(rig.transport->send_count(), 0u);
|
||||
CompatChatModel with_default(rig.provider, "dflt");
|
||||
ASSERT_EQ(with_default.to_body(base(), false).find("model")->as_string(), std::string("m1"));
|
||||
ChatRequest nomodel = base();
|
||||
nomodel.model = "";
|
||||
ASSERT_EQ(with_default.to_body(nomodel, false).find("model")->as_string(), std::string("dflt"));
|
||||
}
|
||||
|
||||
TEST(compat_chat, groq_quirks_are_rejected_loudly) {
|
||||
Rig rig("groq");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest named = base();
|
||||
named.messages[0].name = "alice";
|
||||
ASSERT_THROW(m.to_body(named, false), ApiError);
|
||||
for (const char* key : {"logprobs", "top_logprobs", "logit_bias"}) {
|
||||
ChatRequest r = base();
|
||||
r.extra = Json::object();
|
||||
r.extra.set(key, Json(1));
|
||||
ASSERT_THROW(m.to_body(r, false), ApiError);
|
||||
}
|
||||
ChatRequest n2 = base();
|
||||
n2.extra = Json::parse("{\"n\":2}");
|
||||
ASSERT_THROW(m.to_body(n2, false), ApiError);
|
||||
ChatRequest n1 = base();
|
||||
n1.extra = Json::parse("{\"n\":1}");
|
||||
ASSERT_TRUE(m.to_body(n1, false).find("n") != nullptr);
|
||||
ASSERT_EQ(rig.transport->send_count(), 0u);
|
||||
// The same request is fine on a row without the quirk.
|
||||
Rig other("together");
|
||||
CompatChatModel o(other.provider);
|
||||
ChatRequest ok = base();
|
||||
ok.messages[0].name = "alice";
|
||||
ok.extra = Json::parse("{\"logprobs\":true,\"n\":2}");
|
||||
ASSERT_TRUE(o.to_body(ok, false).find("logprobs") != nullptr);
|
||||
ChatRequest big = base();
|
||||
big.extra = Json::parse("{\"n\":9}");
|
||||
ASSERT_THROW(o.to_body(big, false), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, cerebras_rejects_tools_with_response_format) {
|
||||
Rig rig("cerebras");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
r.json_mode = true;
|
||||
ASSERT_TRUE(m.to_body(r, false).find("response_format") != nullptr);
|
||||
r.tools.push_back({"f", "", Json::object()});
|
||||
ASSERT_THROW(m.to_body(r, false), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, parses_reasoning_fields_and_tool_calls) {
|
||||
Json doc = Json::parse(
|
||||
"{\"id\":\"x\",\"model\":\"m\",\"choices\":[{\"index\":0,\"finish_reason\":\"tool_calls\",\"message\":"
|
||||
"{\"role\":\"assistant\",\"content\":null,\"reasoning\":\"why\",\"tool_calls\":[{\"id\":\"c1\",\"type\":\"function\","
|
||||
"\"function\":{\"name\":\"f\",\"arguments\":\"{\\\"a\\\":1}\"}},{\"id\":\"c2\",\"type\":\"function\","
|
||||
"\"function\":{\"name\":\"g\",\"arguments\":\"not json\"}}]}}],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":4,"
|
||||
"\"total_tokens\":7,\"prompt_tokens_details\":{\"cached_tokens\":2},\"completion_tokens_details\":{\"reasoning_tokens\":1}}}");
|
||||
molo::llm::ChatResponse r = CompatChatModel::parse_response(doc);
|
||||
ASSERT_EQ(r.finish_reason, std::string("tool_calls"));
|
||||
ASSERT_EQ(r.message.parts[0].reasoning, std::string("why"));
|
||||
auto calls = r.tool_calls();
|
||||
ASSERT_EQ(calls.size(), 2u);
|
||||
ASSERT_EQ(calls[0].arguments.find("a")->as_int(), 1L);
|
||||
ASSERT_TRUE(calls[1].arguments.find("_invalid_json") != nullptr);
|
||||
ASSERT_EQ(r.usage.cached_tokens, 2);
|
||||
ASSERT_EQ(r.usage.reasoning_tokens, 1);
|
||||
Json rc = Json::parse("{\"choices\":[{\"message\":{\"content\":\"a\",\"reasoning_content\":\"b\"}}]}");
|
||||
ASSERT_EQ(CompatChatModel::parse_response(rc).message.parts[0].reasoning, std::string("b"));
|
||||
ASSERT_THROW(CompatChatModel::parse_response(Json::parse("{\"choices\":[]}")), ApiError);
|
||||
ASSERT_THROW(CompatChatModel::parse_response(Json::parse("{\"choices\":[{}]}")), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, stream_documented_transcript_with_usage_chunk) {
|
||||
Rig rig("fireworks");
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse(kStream));
|
||||
CompatChatModel m(rig.provider);
|
||||
std::string text, finish;
|
||||
int total = 0;
|
||||
m.stream(base(), [&](const ChatChunk& c) {
|
||||
text += c.text_delta;
|
||||
if (!c.finish_reason.empty()) finish = c.finish_reason;
|
||||
if (c.usage) total = c.usage->total_tokens;
|
||||
return true;
|
||||
});
|
||||
ASSERT_EQ(text, std::string("Hello!"));
|
||||
ASSERT_EQ(finish, std::string("stop"));
|
||||
ASSERT_EQ(total, 11);
|
||||
ASSERT_TRUE(has(rig.transport->seen()[0].body, "\"stream\":true"));
|
||||
ASSERT_FALSE(has(rig.transport->seen()[0].body, "stream_options"));
|
||||
}
|
||||
|
||||
TEST(compat_chat, stream_tool_fragments_carry_id_and_name) {
|
||||
Rig rig("fireworks");
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse(kToolStream));
|
||||
CompatChatModel m(rig.provider);
|
||||
std::string args, id, name, reasoning;
|
||||
m.stream(base(), [&](const ChatChunk& c) {
|
||||
args += c.tool_args_delta;
|
||||
if (!c.tool_call_id.empty()) id = c.tool_call_id;
|
||||
if (!c.tool_call_name.empty()) name = c.tool_call_name;
|
||||
reasoning += c.reasoning_delta;
|
||||
return true;
|
||||
});
|
||||
ASSERT_EQ(args, std::string("{\"city\":\"Rome\"}"));
|
||||
ASSERT_EQ(id, std::string("call_1"));
|
||||
ASSERT_EQ(name, std::string("get_weather"));
|
||||
ASSERT_EQ(reasoning, std::string("thinking"));
|
||||
}
|
||||
|
||||
TEST(compat_chat, stream_without_finish_reason_is_truncated_and_sink_can_stop) {
|
||||
Rig rig("fireworks");
|
||||
rig.transport->expect_response(
|
||||
Method::kPost, rig.path("/chat/completions"),
|
||||
sse("data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"a\"},\"finish_reason\":null}]}\n\n"));
|
||||
CompatChatModel m(rig.provider);
|
||||
try {
|
||||
m.stream(base(), [](const ChatChunk&) { return true; });
|
||||
FAIL("expected truncation");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.http_status(), 502);
|
||||
ASSERT_EQ(e.type(), std::string("STREAM_TRUNCATED"));
|
||||
}
|
||||
Rig rig2("fireworks");
|
||||
rig2.transport->expect_response(Method::kPost, rig2.path("/chat/completions"), sse(kStream));
|
||||
CompatChatModel m2(rig2.provider);
|
||||
int seen = 0;
|
||||
m2.stream(base(), [&](const ChatChunk&) { return ++seen < 2; }); // stop early: no throw
|
||||
ASSERT_EQ(seen, 2);
|
||||
}
|
||||
|
||||
TEST(compat_chat, stream_error_event_throws) {
|
||||
Rig rig("fireworks");
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"),
|
||||
sse("data: {\"error\":{\"type\":\"overloaded\",\"message\":\"busy\"}}\n\n"));
|
||||
CompatChatModel m(rig.provider);
|
||||
try {
|
||||
m.stream(base(), [](const ChatChunk&) { return true; });
|
||||
FAIL("expected");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.type(), std::string("overloaded"));
|
||||
}
|
||||
}
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
// Opt-in live checks, driven by the registry table. MOLO_LIVE_NET=1 enables
|
||||
// the network. Per row:
|
||||
// - probe (no key needed): an invalid key must be answered like an API, never
|
||||
// 404 and never a transport failure; the status and error type are printed;
|
||||
// - with the row's key env set: list models (when the row claims it) and ONE
|
||||
// completion of max_tokens 8, on the model named by COMPAT_<ID>_MODEL
|
||||
// (upper case id). A missing key or model prints a loud SKIP line.
|
||||
// Nothing is created or deleted. Local rows run only when their base env is set.
|
||||
#include <algorithm>
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
|
||||
#include "clients/transports/http_transport.h"
|
||||
#include "llm/config.h"
|
||||
#include "support.h"
|
||||
#include "vendors/compat/chat_model.h"
|
||||
#include "vendors/compat/services.h"
|
||||
|
||||
using namespace molo::compat;
|
||||
using namespace compat_test;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
namespace {
|
||||
|
||||
// Read at static-init time: another suite in the shared binary unsets
|
||||
// MOLO_LIVE_NET before the tests run.
|
||||
const bool kNet = std::getenv("MOLO_LIVE_NET") != nullptr;
|
||||
bool net() { return kNet; }
|
||||
|
||||
std::string upper(std::string s) {
|
||||
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return static_cast<char>(std::toupper(c)); });
|
||||
return s;
|
||||
}
|
||||
|
||||
Provider probe_provider(const ProviderRow& row) {
|
||||
return Provider::with(row.id, std::make_unique<molo::clients::transports::HttpTransport>(), "sk-invalid-molo-probe",
|
||||
"", "probe-account");
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST(compat_live, invalid_key_probe_answers_like_an_api_for_every_hosted_row) {
|
||||
if (!net()) {
|
||||
std::cout << "SKIP compat_live.invalid_key_probe (set MOLO_LIVE_NET=1)\n";
|
||||
return;
|
||||
}
|
||||
int unreachable = 0;
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& row = Registry::at(i);
|
||||
if (row.local || std::string(row.extra_env).size() > 0) continue; // no hosted URL to probe
|
||||
Provider p = probe_provider(row);
|
||||
int status = 200;
|
||||
std::string type = "ok", detail;
|
||||
try {
|
||||
if (Registry::has(row, Feature::kModels)) {
|
||||
CompatModels(p).list();
|
||||
} else {
|
||||
CompatChatModel m(p);
|
||||
molo::llm::ChatRequest r;
|
||||
r.model = "molo-probe";
|
||||
r.messages.push_back(molo::llm::Message::user("hi"));
|
||||
r.max_tokens = 1;
|
||||
m.generate(r);
|
||||
}
|
||||
} catch (const ApiError& e) {
|
||||
status = e.http_status();
|
||||
type = e.type();
|
||||
detail = e.message().substr(0, 160);
|
||||
}
|
||||
std::cout << " " << row.id << ": " << status << " " << type << "\n";
|
||||
if (status == 502) {
|
||||
++unreachable; // transport failure: reported, not asserted (network flake)
|
||||
std::cout << " WARN " << row.id << " unreachable from here: " << detail << "\n";
|
||||
continue;
|
||||
}
|
||||
ASSERT_TRUE(status != 404);
|
||||
}
|
||||
std::cout << " unreachable rows: " << unreachable << "\n";
|
||||
}
|
||||
|
||||
TEST(compat_live, models_and_one_tiny_completion_for_each_row_with_a_key) {
|
||||
if (!net()) {
|
||||
std::cout << "SKIP compat_live.models_and_one_tiny_completion (set MOLO_LIVE_NET=1 and a provider key env)\n";
|
||||
return;
|
||||
}
|
||||
int ran = 0;
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& row = Registry::at(i);
|
||||
Provider p = Provider::from_env(row.id);
|
||||
if (!p.configured()) {
|
||||
std::cout << "SKIP compat_live." << row.id << " (set " << p.missing_env() << ")\n";
|
||||
continue;
|
||||
}
|
||||
++ran;
|
||||
if (Registry::has(row, Feature::kModels) && !std::string(row.models_path).empty()) {
|
||||
ModelList list = CompatModels(p).list();
|
||||
std::cout << " " << row.id << " models: " << list.models.size() << "\n";
|
||||
ASSERT_TRUE(!list.models.empty());
|
||||
}
|
||||
const std::string model_env = "COMPAT_" + upper(row.id) + "_MODEL";
|
||||
std::string model = molo::llm::Config::get(model_env.c_str());
|
||||
if (model.empty()) {
|
||||
std::cout << "SKIP compat_live." << row.id << " completion (set " << model_env << ")\n";
|
||||
continue;
|
||||
}
|
||||
CompatChatModel chat(p, model);
|
||||
molo::llm::ChatRequest r;
|
||||
r.messages.push_back(molo::llm::Message::user("Reply with the single word: ok"));
|
||||
r.max_tokens = 8;
|
||||
molo::llm::ChatResponse out = chat.generate(r);
|
||||
std::cout << " " << row.id << " completion finish=" << out.finish_reason << " tokens=" << out.usage.total_tokens << "\n";
|
||||
ASSERT_TRUE(!out.id.empty() || !out.finish_reason.empty());
|
||||
}
|
||||
if (ran == 0) std::cout << "SKIP compat_live: no provider key is configured\n";
|
||||
}
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
// Provider wiring over the real ApiClient with a recording transport, driven
|
||||
// by the registry table: every row is exercised, so a new row is tested
|
||||
// without a new test.
|
||||
#include <cstdlib>
|
||||
|
||||
#include "support.h"
|
||||
#include "vendors/compat/chat_model.h"
|
||||
#include "vendors/compat/services.h"
|
||||
|
||||
using namespace molo::compat;
|
||||
using namespace compat_test;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
namespace {
|
||||
// FIXTURE: OpenAI API reference, createChatCompletion "Default" response example.
|
||||
const char* kChatExample =
|
||||
"{\"id\":\"chatcmpl-B9MBs8CjcvOU2jLn4n570S5qMJKcT\",\"object\":\"chat.completion\",\"created\":1741569952,"
|
||||
"\"model\":\"gpt-4.1-2025-04-14\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\","
|
||||
"\"content\":\"Hello! How can I assist you today?\",\"refusal\":null,\"annotations\":[]},\"logprobs\":null,"
|
||||
"\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":19,\"completion_tokens\":10,\"total_tokens\":29,"
|
||||
"\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":"
|
||||
"{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},"
|
||||
"\"service_tier\":\"default\"}";
|
||||
|
||||
// FIXTURE: OpenAI API reference, listModels response example (two entries).
|
||||
const char* kModelsExample =
|
||||
"{\"object\":\"list\",\"data\":[{\"id\":\"model-id-0\",\"object\":\"model\",\"created\":1686935002,"
|
||||
"\"owned_by\":\"organization-owner\"},{\"id\":\"model-id-1\",\"object\":\"model\",\"created\":1686935002,"
|
||||
"\"owned_by\":\"organization-owner\"}]}";
|
||||
|
||||
molo::llm::ChatRequest hello() {
|
||||
molo::llm::ChatRequest r;
|
||||
r.model = "some-model";
|
||||
r.messages.push_back(molo::llm::Message::user("Hello!"));
|
||||
return r;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(compat_provider, every_row_posts_chat_to_its_own_base_with_a_bearer_key) {
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& row = Registry::at(i);
|
||||
Rig rig(row.id);
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), reply(kChatExample));
|
||||
CompatChatModel model(rig.provider);
|
||||
molo::llm::ChatResponse out = model.generate(hello());
|
||||
ASSERT_EQ(out.text(), std::string("Hello! How can I assist you today?"));
|
||||
ASSERT_EQ(out.usage.total_tokens, 29);
|
||||
auto s = rig.transport->seen()[0];
|
||||
ASSERT_EQ(s.method, std::string("POST"));
|
||||
ASSERT_EQ(s.headers.get_or("Authorization", ""), std::string("Bearer ") + kKey);
|
||||
ASSERT_EQ(s.target.find(kKey), std::string::npos);
|
||||
ASSERT_EQ(model.vendor(), std::string("compat:") + row.id);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(compat_provider, every_models_row_lists_and_every_other_row_refuses_without_traffic) {
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& row = Registry::at(i);
|
||||
Rig rig(row.id);
|
||||
CompatModels models(rig.provider);
|
||||
if (Registry::has(row, Feature::kModels)) {
|
||||
rig.transport->expect_response(Method::kGet, rig.path("/models"), reply(kModelsExample));
|
||||
ModelList list = models.list();
|
||||
ASSERT_EQ(list.models.size(), 2u);
|
||||
ASSERT_TRUE(list.contains("model-id-1"));
|
||||
ASSERT_EQ(rig.transport->seen()[0].method, std::string("GET"));
|
||||
} else {
|
||||
ASSERT_THROW(models.list(), ApiError);
|
||||
ASSERT_EQ(rig.transport->send_count(), 0u);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(compat_provider, features_not_in_the_row_never_reach_the_network) {
|
||||
Rig rig("cerebras");
|
||||
CompatEmbeddings emb(rig.provider);
|
||||
ASSERT_THROW(emb.embed("m", {"x"}), ApiError);
|
||||
CompatImages img(rig.provider);
|
||||
ImageRequest ir;
|
||||
ir.model = "m";
|
||||
ir.prompt = "p";
|
||||
ASSERT_THROW(img.generate(ir), ApiError);
|
||||
CompatAudio audio(rig.provider);
|
||||
SpeechRequest sr;
|
||||
sr.model = "m";
|
||||
sr.input = "x";
|
||||
sr.voice = "v";
|
||||
std::string ct;
|
||||
ASSERT_THROW(audio.speak(sr, ct), ApiError);
|
||||
ASSERT_EQ(rig.transport->send_count(), 0u);
|
||||
}
|
||||
|
||||
TEST(compat_provider, unconfigured_provider_has_no_traffic_and_names_the_env) {
|
||||
Rig rig("groq", "");
|
||||
ASSERT_FALSE(rig.provider.configured());
|
||||
ASSERT_EQ(rig.provider.missing_env(), std::string("GROQ_API_KEY"));
|
||||
CompatChatModel model(rig.provider);
|
||||
try {
|
||||
model.generate(hello());
|
||||
FAIL("expected");
|
||||
} catch (const molo::llm::NotConfiguredError& e) {
|
||||
ASSERT_EQ(e.http_status(), 503);
|
||||
ASSERT_TRUE(has(e.message(), "GROQ_API_KEY"));
|
||||
}
|
||||
ASSERT_EQ(rig.transport->send_count(), 0u);
|
||||
}
|
||||
|
||||
TEST(compat_provider, cloudflare_substitutes_the_account_id_and_validates_it) {
|
||||
Rig rig("cloudflare", kKey, "", "a1b2c3");
|
||||
ASSERT_EQ(rig.provider.base_url(), std::string("https://api.cloudflare.com/client/v4/accounts/a1b2c3/ai/v1"));
|
||||
ASSERT_THROW(Rig("cloudflare", kKey, "", ""), ApiError);
|
||||
ASSERT_THROW(Rig("cloudflare", kKey, "", "a/../b"), ApiError);
|
||||
ASSERT_THROW(Rig("cloudflare", kKey, "", "a b"), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_provider, base_override_is_validated_and_trailing_slash_removed) {
|
||||
Rig ok("vllm", "", "http://10.0.0.5:9000/v1/");
|
||||
ASSERT_EQ(ok.provider.base_url(), std::string("http://10.0.0.5:9000/v1"));
|
||||
ASSERT_THROW(Rig("vllm", "", "ftp://x"), ApiError);
|
||||
ASSERT_THROW(Rig("vllm", "", "http://x/v1?a=b"), ApiError);
|
||||
ASSERT_THROW(Rig("vllm", "", "http://x/../v1"), ApiError);
|
||||
ASSERT_THROW(Rig("vllm", "", "http://x y/v1"), ApiError);
|
||||
ASSERT_THROW(Rig("vllm", "", std::string("http://x/") + std::string(600, 'a')), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_provider, local_rows_are_configured_only_when_their_base_env_is_set) {
|
||||
setenv("MOLO_ENV_FILES", "off", 1);
|
||||
unsetenv("OLLAMA_BASE_URL");
|
||||
unsetenv("OLLAMA_API_KEY");
|
||||
Provider off = Provider::from_env("ollama");
|
||||
ASSERT_FALSE(off.configured());
|
||||
ASSERT_EQ(off.missing_env(), std::string("OLLAMA_BASE_URL"));
|
||||
setenv("OLLAMA_BASE_URL", "http://127.0.0.1:11434/v1", 1);
|
||||
Provider on = Provider::from_env("ollama");
|
||||
ASSERT_TRUE(on.configured());
|
||||
ASSERT_EQ(on.base_url(), std::string("http://127.0.0.1:11434/v1"));
|
||||
unsetenv("OLLAMA_BASE_URL");
|
||||
}
|
||||
|
||||
TEST(compat_provider, hosted_from_env_needs_its_key_and_cloudflare_needs_the_account) {
|
||||
setenv("MOLO_ENV_FILES", "off", 1);
|
||||
unsetenv("CLOUDFLARE_API_TOKEN");
|
||||
unsetenv("CLOUDFLARE_ACCOUNT_ID");
|
||||
unsetenv("GROQ_API_KEY");
|
||||
ASSERT_FALSE(Provider::from_env("groq").configured());
|
||||
setenv("CLOUDFLARE_API_TOKEN", "t", 1);
|
||||
Provider cf = Provider::from_env("cloudflare");
|
||||
ASSERT_FALSE(cf.configured());
|
||||
ASSERT_EQ(cf.missing_env(), std::string("CLOUDFLARE_ACCOUNT_ID"));
|
||||
setenv("CLOUDFLARE_ACCOUNT_ID", "acct9", 1);
|
||||
ASSERT_TRUE(Provider::from_env("cloudflare").configured());
|
||||
unsetenv("CLOUDFLARE_API_TOKEN");
|
||||
unsetenv("CLOUDFLARE_ACCOUNT_ID");
|
||||
}
|
||||
|
||||
TEST(compat_provider, error_envelopes_of_different_providers_map_to_api_error) {
|
||||
// Shapes seen in the unauthenticated probes of SPEC section 5 (labelled: probe output, 2026-10-25).
|
||||
struct Case { const char* id; int status; const char* body; const char* type; };
|
||||
const Case cases[] = {
|
||||
{"groq", 401, "{\"error\":{\"message\":\"Invalid API Key\",\"type\":\"invalid_request_error\",\"code\":\"invalid_api_key\"}}", "invalid_request_error"},
|
||||
{"cerebras", 401, "{\"message\":\"Wrong API Key\",\"type\":\"invalid_request_error\",\"param\":\"api_key\",\"code\":\"wrong_api_key\"}", "invalid_request_error"},
|
||||
{"moonshot", 401, "{\"error\":{\"message\":\"Incorrect API key provided\",\"type\":\"incorrect_api_key_error\"}}", "incorrect_api_key_error"},
|
||||
};
|
||||
for (const Case& c : cases) {
|
||||
Rig rig(c.id);
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), reply(c.body, c.status));
|
||||
CompatChatModel model(rig.provider);
|
||||
try {
|
||||
model.generate(hello());
|
||||
FAIL("expected");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.http_status(), c.status);
|
||||
ASSERT_EQ(e.type(), std::string(c.type));
|
||||
}
|
||||
}
|
||||
}
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
// The registry table is the single source: these tests pin its invariants so
|
||||
// that adding a row cannot break consistency between components.
|
||||
#include <set>
|
||||
|
||||
#include "support.h"
|
||||
#include "vendors/compat/calls.h"
|
||||
#include "vendors/compat/registry.h"
|
||||
|
||||
using namespace molo::compat;
|
||||
using namespace compat_test;
|
||||
|
||||
TEST(compat_registry, rows_are_unique_slugs_with_complete_fields) {
|
||||
std::set<std::string> ids;
|
||||
ASSERT_TRUE(Registry::count() >= 35);
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& r = Registry::at(i);
|
||||
std::string id = r.id;
|
||||
ASSERT_TRUE(!id.empty() && id.size() <= 32);
|
||||
for (char c : id) ASSERT_TRUE((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'));
|
||||
ASSERT_TRUE(ids.insert(id).second);
|
||||
ASSERT_TRUE(std::string(r.name).size() > 2);
|
||||
ASSERT_TRUE(std::string(r.key_env).size() > 3);
|
||||
ASSERT_TRUE(std::string(r.doc_url).rfind("https://", 0) == 0);
|
||||
ASSERT_TRUE(std::string(r.note).size() > 10);
|
||||
ASSERT_TRUE((r.features & Feature::kChat) != 0); // every row is a chat provider
|
||||
}
|
||||
}
|
||||
|
||||
TEST(compat_registry, base_urls_are_wellformed) {
|
||||
std::set<std::string> envs;
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& r = Registry::at(i);
|
||||
std::string base = r.base_url;
|
||||
ASSERT_TRUE(base.back() != '/');
|
||||
ASSERT_TRUE(base.find(' ') == std::string::npos);
|
||||
if (r.local) {
|
||||
ASSERT_TRUE(base.rfind("http://localhost:", 0) == 0);
|
||||
ASSERT_TRUE(std::string(r.base_env).size() > 3); // the "configured" switch
|
||||
} else {
|
||||
ASSERT_TRUE(base.rfind("https://", 0) == 0);
|
||||
ASSERT_TRUE(std::string(r.base_env).empty());
|
||||
}
|
||||
ASSERT_EQ(base.find("{account_id}") != std::string::npos, std::string(r.extra_env).size() > 0);
|
||||
// Credential names are unique except where two rows deliberately share none.
|
||||
ASSERT_TRUE(envs.insert(r.key_env).second);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(compat_registry, models_feature_and_path_agree) {
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& r = Registry::at(i);
|
||||
bool claims = Registry::has(r, Feature::kModels);
|
||||
bool path = std::string(r.models_path).size() > 0;
|
||||
ASSERT_EQ(claims, path);
|
||||
if (path) ASSERT_EQ(std::string(r.models_path), std::string("/models"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(compat_registry, quirk_rows_carry_their_documented_flag) {
|
||||
const ProviderRow* groq = Registry::find("groq");
|
||||
ASSERT_TRUE(groq != nullptr);
|
||||
ASSERT_TRUE((groq->quirks & Quirk::kRejectMessageName) != 0);
|
||||
ASSERT_TRUE((groq->quirks & Quirk::kRejectLogprobs) != 0);
|
||||
ASSERT_TRUE((groq->quirks & Quirk::kSingleChoice) != 0);
|
||||
ASSERT_TRUE((Registry::get("cerebras").quirks & Quirk::kNoToolsWithFormat) != 0);
|
||||
ASSERT_EQ(Registry::get("cerebras").features & Feature::kEmbeddings, 0u);
|
||||
}
|
||||
|
||||
TEST(compat_registry, evidence_matches_row_kind) {
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& r = Registry::at(i);
|
||||
if (r.local) ASSERT_TRUE(r.evidence == Evidence::kDocsOnly);
|
||||
ASSERT_TRUE(std::string(Registry::evidence_name(r.evidence)).size() > 4);
|
||||
}
|
||||
ASSERT_TRUE(Registry::get("cloudflare").evidence == Evidence::kDocsOnly);
|
||||
}
|
||||
|
||||
TEST(compat_registry, no_em_dash_in_any_string) {
|
||||
const std::string em = "\xE2\x80\x94";
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& r = Registry::at(i);
|
||||
for (const char* s : {r.id, r.name, r.base_url, r.key_env, r.doc_url, r.note})
|
||||
ASSERT_TRUE(std::string(s).find(em) == std::string::npos);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(compat_registry, lookup_and_unknown_id) {
|
||||
ASSERT_TRUE(Registry::find("groq") != nullptr);
|
||||
ASSERT_TRUE(Registry::find("nope") == nullptr);
|
||||
ASSERT_THROW(Registry::get("nope"), molo::llm::ApiError);
|
||||
ASSERT_THROW(Registry::at(Registry::count()), molo::llm::ApiError);
|
||||
ASSERT_EQ(Registry::feature_names(Registry::get("cerebras")), std::string("chat,models"));
|
||||
}
|
||||
|
||||
TEST(compat_registry, call_table_rows_map_to_openai_operations) {
|
||||
ASSERT_EQ(Calls::count(), 6u);
|
||||
ASSERT_EQ(std::string(Calls::spec_of(Call::kChat).path), std::string("/chat/completions"));
|
||||
ASSERT_EQ(std::string(Calls::spec_of(Call::kModels).operation_id), std::string("listModels"));
|
||||
ASSERT_TRUE(Calls::method_of(Call::kModels) == Method::kGet);
|
||||
ASSERT_TRUE(Calls::method_of(Call::kSpeech) == Method::kPost);
|
||||
for (std::size_t i = 0; i < Calls::count(); ++i) {
|
||||
ASSERT_TRUE(std::string(Calls::at(i).path).front() == '/');
|
||||
ASSERT_TRUE(std::string(Calls::at(i).operation_id).size() > 5);
|
||||
}
|
||||
}
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
// Models, embeddings, images and audio over the recording transport. Replies
|
||||
// are the documented OpenAI examples (FIXTURE); assertions are on what WE send
|
||||
// and how WE parse.
|
||||
#include "support.h"
|
||||
#include "vendors/compat/services.h"
|
||||
|
||||
using namespace molo::compat;
|
||||
using namespace compat_test;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
TEST(compat_services, models_accepts_a_bare_array_and_skips_entries_without_id) {
|
||||
// Together returns a bare array; shape recorded from its docs page (LABELLED).
|
||||
ModelList l = CompatModels::parse(Json::parse("[{\"id\":\"a\"},{\"name\":\"b\"},{\"x\":1}]"));
|
||||
ASSERT_EQ(l.models.size(), 2u);
|
||||
ASSERT_TRUE(l.contains("b"));
|
||||
ASSERT_THROW(CompatModels::parse(Json::parse("{\"data\":5}")), ApiError);
|
||||
ASSERT_THROW(CompatModels::parse(Json::parse("{}")), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_services, models_list_truncates_loudly) {
|
||||
Json arr = Json::array();
|
||||
for (std::size_t i = 0; i < 5010; ++i) {
|
||||
Json o = Json::object();
|
||||
o.set("id", Json("m" + std::to_string(i)));
|
||||
arr.push(o);
|
||||
}
|
||||
ModelList l = CompatModels::parse(arr);
|
||||
ASSERT_EQ(l.models.size(), 5000u);
|
||||
ASSERT_TRUE(l.truncated);
|
||||
}
|
||||
|
||||
TEST(compat_services, row_without_list_endpoint_refuses) {
|
||||
Rig rig("dashscope");
|
||||
CompatModels m(rig.provider);
|
||||
ASSERT_THROW(m.list(), ApiError);
|
||||
ASSERT_EQ(rig.transport->send_count(), 0u);
|
||||
}
|
||||
|
||||
TEST(compat_services, embeddings_body_rules_and_parse) {
|
||||
Rig rig("nvidia");
|
||||
CompatEmbeddings e(rig.provider);
|
||||
Json b = e.to_body("m", {"a", "b"}, 256);
|
||||
ASSERT_EQ(b.find("input")->as_array().size(), 2u);
|
||||
ASSERT_EQ(b.find("dimensions")->as_int(), 256L);
|
||||
ASSERT_TRUE(e.to_body("m", {"a"}, 0).find("dimensions") == nullptr);
|
||||
ASSERT_THROW(e.to_body("", {"a"}, 0), ApiError);
|
||||
ASSERT_THROW(e.to_body("m", {}, 0), ApiError);
|
||||
ASSERT_THROW(e.to_body("m", {""}, 0), ApiError);
|
||||
ASSERT_THROW(e.to_body("m", {"a"}, -1), ApiError);
|
||||
ASSERT_THROW(e.to_body("m", std::vector<std::string>(2049, "x"), 0), ApiError);
|
||||
// FIXTURE: OpenAI API reference, createEmbedding response example.
|
||||
rig.transport->expect_response(
|
||||
Method::kPost, rig.path("/embeddings"),
|
||||
reply("{\"object\":\"list\",\"data\":[{\"object\":\"embedding\",\"embedding\":[0.0023064255,-0.009327292,-0.0028842222],"
|
||||
"\"index\":0}],\"model\":\"text-embedding-ada-002\",\"usage\":{\"prompt_tokens\":8,\"total_tokens\":8}}"));
|
||||
EmbeddingResult r = e.embed("text-embedding-ada-002", {"The food was delicious"});
|
||||
ASSERT_EQ(r.items.size(), 1u);
|
||||
ASSERT_EQ(r.items[0].vector.size(), 3u);
|
||||
ASSERT_EQ(r.total_tokens, 8);
|
||||
ASSERT_THROW(CompatEmbeddings::parse(Json::parse("{\"data\":[{\"embedding\":5}]}")), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_services, images_body_rules_and_parse) {
|
||||
Rig rig("together");
|
||||
CompatImages img(rig.provider);
|
||||
ImageRequest r;
|
||||
r.model = "m";
|
||||
r.prompt = "a cat";
|
||||
r.size = "512x512";
|
||||
r.response_format = "b64_json";
|
||||
Json b = img.to_body(r);
|
||||
ASSERT_EQ(b.find("size")->as_string(), std::string("512x512"));
|
||||
r.size = "big";
|
||||
ASSERT_THROW(img.to_body(r), ApiError);
|
||||
r.size = "x512";
|
||||
ASSERT_THROW(img.to_body(r), ApiError);
|
||||
r.size = "";
|
||||
r.n = 11;
|
||||
ASSERT_THROW(img.to_body(r), ApiError);
|
||||
r.n = 1;
|
||||
r.response_format = "png";
|
||||
ASSERT_THROW(img.to_body(r), ApiError);
|
||||
r.response_format = "";
|
||||
r.prompt = "";
|
||||
ASSERT_THROW(img.to_body(r), ApiError);
|
||||
// FIXTURE: OpenAI API reference, createImage response example.
|
||||
auto items = CompatImages::parse(Json::parse("{\"created\":1713833628,\"data\":[{\"b64_json\":\"...\",\"revised_prompt\":\"x\"}]}"));
|
||||
ASSERT_EQ(items.size(), 1u);
|
||||
ASSERT_EQ(items[0].revised_prompt, std::string("x"));
|
||||
ASSERT_THROW(CompatImages::parse(Json::parse("{}")), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_services, transcription_sends_multipart_and_validates) {
|
||||
Rig rig("groq");
|
||||
CompatAudio audio(rig.provider);
|
||||
TranscribeRequest t;
|
||||
t.model = "whisper-large-v3";
|
||||
t.filename = "a.wav";
|
||||
t.audio = "RIFFxxxx";
|
||||
t.language = "en";
|
||||
// FIXTURE: OpenAI API reference, createTranscription json response example.
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/audio/transcriptions"),
|
||||
reply("{\"text\":\"Imagine the wildest idea that you've ever had.\"}"));
|
||||
ASSERT_TRUE(has(audio.transcribe(t), "wildest idea"));
|
||||
auto s = rig.transport->seen()[0];
|
||||
ASSERT_TRUE(has(s.headers.get_or("Content-Type", ""), "multipart/form-data"));
|
||||
ASSERT_TRUE(has(s.body, "name=\"file\"; filename=\"a.wav\""));
|
||||
ASSERT_TRUE(has(s.body, "whisper-large-v3"));
|
||||
TranscribeRequest bad = t;
|
||||
bad.audio.clear();
|
||||
ASSERT_THROW(audio.transcribe(bad), ApiError);
|
||||
bad = t;
|
||||
bad.filename = "a\".wav";
|
||||
ASSERT_THROW(audio.transcribe(bad), ApiError);
|
||||
bad = t;
|
||||
bad.filename = "../a.wav";
|
||||
ASSERT_THROW(audio.transcribe(bad), ApiError);
|
||||
Rig no("cerebras");
|
||||
ASSERT_THROW(CompatAudio(no.provider).transcribe(t), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_services, speech_returns_bytes_and_content_type) {
|
||||
Rig rig("together");
|
||||
CompatAudio audio(rig.provider);
|
||||
SpeechRequest s;
|
||||
s.model = "m";
|
||||
s.input = "hello";
|
||||
s.voice = "alloy";
|
||||
s.format = "mp3";
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/audio/speech"), reply("ID3bytes", 200, "audio/mpeg"));
|
||||
std::string ct;
|
||||
ASSERT_EQ(audio.speak(s, ct), std::string("ID3bytes"));
|
||||
ASSERT_EQ(ct, std::string("audio/mpeg"));
|
||||
s.format = "ogg";
|
||||
ASSERT_THROW(audio.speech_body(s), ApiError);
|
||||
s.format = "";
|
||||
s.voice = "";
|
||||
ASSERT_THROW(audio.speech_body(s), ApiError);
|
||||
s.voice = "v";
|
||||
s.input = std::string(4097, 'x');
|
||||
ASSERT_THROW(audio.speech_body(s), ApiError);
|
||||
}
|
||||
Reference in New Issue
Block a user