From aaddb29d425956efae148e68ee80bded23825c57 Mon Sep 17 00:00:00 2001 From: retoor Date: Sun, 25 Oct 2026 15:01:38 +0100 Subject: [PATCH] feat(clients): compat: OpenAI-compatible provider registry (39 rows), chat model, services, demo Co-Authored-By: Claude Sonnet 5.5 --- example/app/ai/CompatDemo.cpp | 444 ++++++++++++++++++- example/assets/ai_compat.js | 72 +++ src/vendors/compat/SPEC.md | 243 ++++++++++ src/vendors/compat/calls.cpp | 50 +++ src/vendors/compat/calls.h | 38 ++ src/vendors/compat/caps.h | 28 ++ src/vendors/compat/chat_model.cpp | 375 ++++++++++++++++ src/vendors/compat/chat_model.h | 48 ++ src/vendors/compat/provider.cpp | 110 +++++ src/vendors/compat/provider.h | 64 +++ src/vendors/compat/registry.cpp | 239 ++++++++++ src/vendors/compat/registry.h | 72 +++ src/vendors/compat/services.cpp | 199 +++++++++ src/vendors/compat/services.h | 104 +++++ src/vendors/compat/tools/check_spec.py | 239 ++++++++++ tests/example/ai/compat/page_test.cpp | 50 +++ tests/example/ai/compat/routes_test.cpp | 84 ++++ tests/src/vendors/compat/support.h | 63 +++ tests/src/vendors/compat/test_chat_model.cpp | 291 ++++++++++++ tests/src/vendors/compat/test_live.cpp | 115 +++++ tests/src/vendors/compat/test_provider.cpp | 176 ++++++++ tests/src/vendors/compat/test_registry.cpp | 105 +++++ tests/src/vendors/compat/test_services.cpp | 142 ++++++ 23 files changed, 3348 insertions(+), 3 deletions(-) create mode 100644 example/assets/ai_compat.js create mode 100644 src/vendors/compat/SPEC.md create mode 100644 src/vendors/compat/calls.cpp create mode 100644 src/vendors/compat/calls.h create mode 100644 src/vendors/compat/caps.h create mode 100644 src/vendors/compat/chat_model.cpp create mode 100644 src/vendors/compat/chat_model.h create mode 100644 src/vendors/compat/provider.cpp create mode 100644 src/vendors/compat/provider.h create mode 100644 src/vendors/compat/registry.cpp create mode 100644 src/vendors/compat/registry.h create mode 100644 src/vendors/compat/services.cpp create mode 100644 src/vendors/compat/services.h create mode 100644 src/vendors/compat/tools/check_spec.py create mode 100644 tests/example/ai/compat/page_test.cpp create mode 100644 tests/example/ai/compat/routes_test.cpp create mode 100644 tests/src/vendors/compat/support.h create mode 100644 tests/src/vendors/compat/test_chat_model.cpp create mode 100644 tests/src/vendors/compat/test_live.cpp create mode 100644 tests/src/vendors/compat/test_provider.cpp create mode 100644 tests/src/vendors/compat/test_registry.cpp create mode 100644 tests/src/vendors/compat/test_services.cpp diff --git a/example/app/ai/CompatDemo.cpp b/example/app/ai/CompatDemo.cpp index 9d41d1f..fa10c55 100644 --- a/example/app/ai/CompatDemo.cpp +++ b/example/app/ai/CompatDemo.cpp @@ -1,13 +1,451 @@ #include "../AiDemos.hpp" +#include + #include "../../fw/Router.hpp" +#include "../../fw/Templates.hpp" +#include "security/crypto/Base64.h" +#include "vendors/compat/caps.h" +#include "vendors/compat/chat_model.h" +#include "vendors/compat/provider.h" +#include "vendors/compat/services.h" namespace molo::example::app { -// compat. Stub: the package agent fills this in. +// OpenAI-compatible provider registry over the real src/vendors/compat part. +// GET routes need no credential. Every route that calls a provider is a POST: +// 422 on bad input, 503 compat_not_configured without the provider's +// credential. Routes that cost money (chat, embeddings, images, audio) are dry +// runs that show the exact request unless the body has "confirm":true. The +// provider list is the registry table; nothing here names a provider. + +namespace { + +namespace cp = compat; +using Json = clients::body::Json; + +constexpr std::size_t kMaxBody = 8 * 1024 * 1024; // audio travels as base64 + +const char kPage[] = R"COMPAT({% extends "base.html" %} +{% block content %} +

OpenAI-compatible providers

+

One registry table, one client: chat, embeddings, models, images and audio for every +provider whose row claims them. Paid calls are dry runs until the body says +"confirm":true.

+

Checking credentials...

+
+ +

+ + +
+ + +
+ +

+
+ + +{% endblock %} +)COMPAT"; + +[[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); } + +fw::Response json_out(const Json& j, int status = 200) { + fw::Response res = fw::JsonResponse(j.dump(), status); + res.headers.set("Cache-Control", "no-store"); + return res; +} + +Json parse_body(const fw::Request& req) { + if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large"); + if (req.body.empty()) return Json::object(); + try { + Json j = Json::parse(req.body); + if (!j.is_object()) bad("body must be a JSON object"); + return j; + } catch (const clients::body::JsonError&) { + bad("body is not valid JSON"); + } +} + +std::string text(const Json& b, const char* key, bool required = false) { + const Json* v = b.find(key); + if (!v || v->is_null()) { + if (required) bad(std::string(key) + " is required"); + return ""; + } + if (!v->is_string()) bad(std::string(key) + " must be a string"); + return v->as_string(); +} + +int number(const Json& b, const char* key, int fallback) { + const Json* v = b.find(key); + if (!v || v->is_null()) return fallback; + if (!v->is_number()) bad(std::string(key) + " must be a number"); + return static_cast(v->as_int()); +} + +bool flag(const Json& b, const char* key) { + const Json* v = b.find(key); + if (!v || v->is_null()) return false; + if (!v->is_bool()) bad(std::string(key) + " must be true or false"); + return v->as_bool(); +} + +std::vector inputs(const Json& b) { + std::vector out; + const Json* v = b.find("input"); + if (!v || v->is_null()) bad("input is required"); + if (v->is_string()) { + out.push_back(v->as_string()); + } else if (v->is_array()) { + for (const Json& s : v->as_array()) { + if (!s.is_string()) bad("input must be a string or an array of strings"); + out.push_back(s.as_string()); + } + } else { + bad("input must be a string or an array of strings"); + } + return out; +} + +Json dry_run(const Json& request, const std::string& url) { + Json o = Json::object(); + o.set("dry_run", Json(true)); + o.set("would_send_to", Json(url)); + o.set("request", request); + o.set("hint", Json("send confirm:true to run it")); + return o; +} + +Json usage_json(const llm::Usage& u) { + Json o = Json::object(); + o.set("input_tokens", Json(u.input_tokens)); + o.set("output_tokens", Json(u.output_tokens)); + o.set("total_tokens", Json(u.total_tokens)); + o.set("cached_tokens", Json(u.cached_tokens)); + o.set("reasoning_tokens", Json(u.reasoning_tokens)); + return o; +} + +Json provider_json(const cp::ProviderRow& row) { + cp::Provider p = cp::Provider::from_env(row.id); + Json o = Json::object(); + o.set("id", Json(row.id)); + o.set("name", Json(row.name)); + o.set("base_url", Json(p.base_url())); + o.set("credential_env", Json(row.key_env)); + o.set("base_env", Json(row.base_env)); + o.set("extra_env", Json(row.extra_env)); + o.set("configured", Json(p.configured())); + o.set("missing_env", Json(p.missing_env())); + Json feats = Json::array(); + std::string names = cp::Registry::feature_names(row); + std::size_t at = 0; + while (at <= names.size()) { + std::size_t comma = names.find(',', at); + if (comma == std::string::npos) comma = names.size(); + if (comma > at) feats.push(Json(names.substr(at, comma - at))); + at = comma + 1; + } + o.set("features", feats); + o.set("evidence", Json(cp::Registry::evidence_name(row.evidence))); + o.set("local", Json(row.local)); + o.set("doc_url", Json(row.doc_url)); + o.set("note", Json(row.note)); + return o; +} + +// One route = validate into a Plan, then the generic runner decides dry run, +// credential, call and error mapping. +struct Plan { + Json request; // what would be sent (shown in a dry run) + std::string url; // for the dry-run message + bool needs_confirm = false; // paid + std::function call; +}; + +class CompatRoutes { + public: + using Planner = std::function; + + fw::Response run(const fw::Request& req, const Planner& planner) const { + Json body = parse_body(req); + Plan plan; + try { + plan = planner(body); + } catch (const llm::ApiError& e) { + map(e); + } + if (plan.needs_confirm && !flag(body, "confirm")) return json_out(dry_run(plan.request, plan.url)); + try { + return json_out(plan.call()); + } catch (const llm::ApiError& e) { + map(e); + } + } + + static cp::Provider provider_of(const Json& b) { + std::string id = text(b, "provider", true); + if (!cp::Registry::find(id)) bad("unknown provider '" + id.substr(0, 64) + "'"); + return cp::Provider::from_env(id); + } + + static std::string url_of(const cp::Provider& p, cp::Call call) { + return p.base_url() + cp::Calls::spec_of(call).path; + } + + Plan chat(const Json& b) const { + cp::Provider p = provider_of(b); + llm::ChatRequest r; + r.model = text(b, "model"); + std::string system = text(b, "system"); + if (!system.empty()) r.system = system; + r.messages.push_back(llm::Message::user(text(b, "prompt", true))); + int max_tokens = number(b, "max_tokens", 0); + if (max_tokens != 0) r.max_tokens = max_tokens; + r.json_mode = flag(b, "json"); + r.reasoning_effort = text(b, "effort"); + if (const Json* t = b.find("temperature"); t && t->is_number()) r.temperature = t->as_number(); + if (const Json* e = b.find("extra"); e && !e->is_null()) r.extra = *e; + auto model = std::make_shared(p); + Plan pl{model->to_body(r, false), url_of(p, cp::Call::kChat), true, nullptr}; + pl.call = [model, r] { + llm::ChatResponse out = model->generate(r); + Json o = Json::object(); + o.set("id", Json(out.id)); + o.set("model", Json(out.model)); + o.set("finish_reason", Json(out.finish_reason)); + o.set("content", Json(out.text())); + std::string reasoning; + for (const llm::ContentPart& part : out.message.parts) reasoning += part.reasoning; + o.set("reasoning", Json(reasoning)); + Json calls = Json::array(); + for (const llm::ToolCall& tc : out.tool_calls()) { + Json x = Json::object(); + x.set("id", Json(tc.id)); + x.set("name", Json(tc.name)); + x.set("arguments", tc.arguments); + calls.push(x); + } + o.set("tool_calls", calls); + o.set("usage", usage_json(out.usage)); + return o; + }; + return pl; + } + + Plan models(const Json& b) const { + cp::Provider p = provider_of(b); + p.require(cp::Call::kModels); + if (std::string(p.row().models_path).empty()) bad(std::string(p.row().name) + " has no list endpoint under its base URL"); + auto api = std::make_shared(p); + return Plan{Json::object(), url_of(p, cp::Call::kModels), false, [api] { + cp::ModelList l = api->list(); + Json arr = Json::array(); + for (const cp::ModelEntry& m : l.models) { + Json x = Json::object(); + x.set("id", Json(m.id)); + x.set("owned_by", Json(m.owned_by)); + arr.push(x); + } + Json o = Json::object(); + o.set("models", arr); + o.set("truncated", Json(l.truncated)); + return o; + }}; + } + + Plan embeddings(const Json& b) const { + cp::Provider p = provider_of(b); + auto api = std::make_shared(p); + std::string model = text(b, "model"); + std::vector in = inputs(b); + int dims = number(b, "dimensions", 0); + Plan pl{api->to_body(model, in, dims), url_of(p, cp::Call::kEmbeddings), true, nullptr}; + pl.call = [api, model, in, dims] { + cp::EmbeddingResult r = api->embed(model, in, dims); + Json items = Json::array(); + for (const cp::Embedding& e : r.items) { + Json x = Json::object(); + x.set("dimensions", Json(static_cast(e.vector.size()))); + Json head = Json::array(); + for (std::size_t i = 0; i < e.vector.size() && i < 8; ++i) head.push(Json(e.vector[i])); + x.set("head", head); + items.push(x); + } + Json o = Json::object(); + o.set("model", Json(r.model)); + o.set("items", items); + o.set("total_tokens", Json(r.total_tokens)); + return o; + }; + return pl; + } + + Plan images(const Json& b) const { + cp::Provider p = provider_of(b); + auto api = std::make_shared(p); + cp::ImageRequest r; + r.model = text(b, "model"); + r.prompt = text(b, "prompt", true); + r.n = number(b, "n", 1); + r.size = text(b, "size"); + r.response_format = text(b, "response_format"); + Plan pl{api->to_body(r), url_of(p, cp::Call::kImages), true, nullptr}; + pl.call = [api, r] { + Json arr = Json::array(); + for (const cp::ImageItem& i : api->generate(r)) { + Json x = Json::object(); + x.set("url", Json(i.url)); + x.set("b64_json_bytes", Json(static_cast(i.b64_json.size()))); + x.set("revised_prompt", Json(i.revised_prompt)); + arr.push(x); + } + Json o = Json::object(); + o.set("images", arr); + return o; + }; + return pl; + } + + Plan transcribe(const Json& b) const { + cp::Provider p = provider_of(b); + auto api = std::make_shared(p); + cp::TranscribeRequest r; + r.model = text(b, "model"); + r.filename = text(b, "filename", true); + r.language = text(b, "language"); + r.prompt = text(b, "prompt"); + if (!security::crypto::Base64::decode(text(b, "data_base64", true), r.audio)) bad("data_base64 is not valid base64"); + if (r.audio.size() > cp::CompatCaps::kMaxAudioBytes) bad("audio over the byte cap"); + p.require(cp::Call::kTranscribe); + Json req = Json::object(); + req.set("model", Json(r.model)); + req.set("filename", Json(r.filename)); + req.set("bytes", Json(static_cast(r.audio.size()))); + Plan pl{req, url_of(p, cp::Call::kTranscribe), true, nullptr}; + pl.call = [api, r] { + Json o = Json::object(); + o.set("text", Json(api->transcribe(r))); + return o; + }; + return pl; + } + + Plan speech(const Json& b) const { + cp::Provider p = provider_of(b); + auto api = std::make_shared(p); + cp::SpeechRequest r; + r.model = text(b, "model"); + r.input = text(b, "input", true); + r.voice = text(b, "voice"); + r.format = text(b, "format"); + Plan pl{api->speech_body(r), url_of(p, cp::Call::kSpeech), true, nullptr}; + pl.call = [api, r] { + std::string ct; + std::string audio = api->speak(r, ct); + Json o = Json::object(); + o.set("content_type", Json(ct)); + o.set("bytes", Json(static_cast(audio.size()))); + o.set("audio_base64", Json(audio.size() <= 1024 * 1024 ? security::crypto::Base64::encode(audio) : std::string())); + return o; + }; + return pl; + } + + private: + [[noreturn]] static void map(const llm::ApiError& e) { + if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED") + throw fw::HttpError(503, "compat_not_configured", e.message()); + if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty()) + throw fw::HttpError(422, "validation_error", e.message()); + throw fw::HttpError(502, "compat_error", e.vendor() + " " + std::to_string(e.http_status()) + " " + e.type() + ": " + e.message()); + } +}; + +} // namespace + void register_ai_compat_demo(fw::Router& router, Services& services) { - (void)router; - (void)services; + auto routes = std::make_shared(); + fw::Environment* env = services.templates.get(); + services.templates->add_template("ai_compat.html", kPage); + services.add_feature("ai-compat", "/ai/compat"); + + router.get("/ai/compat", "ai-compat-page", [env](fw::Request req) { + (void)req; + fw::TemplateCtx ctx; + ctx["title"] = std::string("OpenAI-compatible providers"); + return fw::Response(200, env->render("ai_compat.html", ctx), "text/html; charset=utf-8"); + }); + + router.get("/ai/compat/status", "ai-compat-status", [](fw::Request req) { + (void)req; + int configured = 0; + Json credentials = Json::array(); + for (std::size_t i = 0; i < cp::Registry::count(); ++i) { + const cp::ProviderRow& row = cp::Registry::at(i); + if (cp::Provider::from_env(row.id).configured()) ++configured; + credentials.push(Json(row.key_env)); + } + Json o = Json::object(); + o.set("configured", Json(configured > 0)); + o.set("configured_providers", Json(configured)); + o.set("providers", Json(static_cast(cp::Registry::count()))); + o.set("credential_envs", credentials); + o.set("calls", Json(static_cast(cp::Calls::count()))); + o.set("max_models_returned", Json(static_cast(cp::CompatCaps::kMaxModelsReturned))); + return json_out(o); + }); + + router.get("/ai/compat/providers", "ai-compat-providers", [](fw::Request req) { + (void)req; + Json arr = Json::array(); + for (std::size_t i = 0; i < cp::Registry::count(); ++i) arr.push(provider_json(cp::Registry::at(i))); + Json o = Json::object(); + o.set("providers", arr); + return json_out(o); + }); + + router.get("/ai/compat/calls", "ai-compat-calls", [](fw::Request req) { + (void)req; + Json arr = Json::array(); + for (std::size_t i = 0; i < cp::Calls::count(); ++i) { + const cp::CallSpec& s = cp::Calls::at(i); + Json row = Json::object(); + row.set("http", Json(s.http_method)); + row.set("path", Json(s.path)); + row.set("operation", Json(s.operation_id)); + row.set("feature", Json(s.feature_name)); + arr.push(row); + } + Json o = Json::object(); + o.set("calls", arr); + return json_out(o); + }); + + using Member = CompatRoutes::Planner; + CompatRoutes* r = routes.get(); + auto add = [&](const char* path, const char* name, Plan (CompatRoutes::*m)(const Json&) const) { + router.post(path, name, [routes, r, m](fw::Request req) { + return routes->run(req, Member([r, m](const Json& b) { return (r->*m)(b); })); + }); + }; + add("/ai/compat/chat", "ai-compat-chat", &CompatRoutes::chat); + add("/ai/compat/models", "ai-compat-models", &CompatRoutes::models); + add("/ai/compat/embeddings", "ai-compat-embeddings", &CompatRoutes::embeddings); + add("/ai/compat/images", "ai-compat-images", &CompatRoutes::images); + add("/ai/compat/transcribe", "ai-compat-transcribe", &CompatRoutes::transcribe); + add("/ai/compat/speech", "ai-compat-speech", &CompatRoutes::speech); } } // namespace molo::example::app diff --git a/example/assets/ai_compat.js b/example/assets/ai_compat.js new file mode 100644 index 0000000..ea6475f --- /dev/null +++ b/example/assets/ai_compat.js @@ -0,0 +1,72 @@ +// Compat page: thin client over /ai/compat/* (all limits server side). The +// provider list comes from the registry table through /ai/compat/providers. +(function () { + "use strict"; + var $ = function (id) { return document.getElementById(id); }; + var out = $("cp-out"), err = $("cp-error"); + var templates = { + "chat": { prompt: "Reply with the single word: ok", max_tokens: 8 }, + "models": {}, + "embeddings": { input: ["The food was delicious"] }, + "images": { prompt: "a small red cube on white", size: "512x512" }, + "transcribe": { filename: "clip.wav", data_base64: "" }, + "speech": { input: "Hello", voice: "alloy", format: "mp3" } + }; + var providers = []; + var rsel = $("cp-route"), psel = $("cp-provider"); + Object.keys(templates).forEach(function (k) { + var o = document.createElement("option"); + o.value = k; o.textContent = k; + rsel.appendChild(o); + }); + function fill() { + var t = JSON.parse(JSON.stringify(templates[rsel.value])); + t.provider = psel.value; + $("cp-body").value = JSON.stringify(t, null, 1); + } + function info() { + var p = providers.filter(function (x) { return x.id === psel.value; })[0]; + if (!p) { $("cp-info").textContent = ""; return; } + $("cp-info").textContent = p.name + " | " + p.base_url + " | " + p.features.join(", ") + + " | " + p.evidence + " | " + (p.configured ? "configured" : "needs " + p.missing_env); + } + rsel.onchange = fill; + psel.onchange = function () { info(); fill(); }; + + function send(confirm) { + var body; + try { body = JSON.parse($("cp-body").value || "{}"); } + catch (e) { err.textContent = "Body is not valid JSON"; return; } + if (confirm) body.confirm = true; + err.textContent = ""; + out.textContent = "..."; + fetch("/ai/compat/" + rsel.value, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body) + }).then(function (res) { + return res.text().then(function (t) { + var j; + try { j = JSON.parse(t); } catch (e) { j = { message: t }; } + if (!res.ok) { err.textContent = j.message || ("HTTP " + res.status); out.textContent = ""; } + else { out.textContent = JSON.stringify(j, null, 2); } + }); + }).catch(function (e) { err.textContent = String(e); out.textContent = ""; }); + } + $("cp-send").onclick = function () { send(false); }; + $("cp-confirm").onclick = function () { send(true); }; + + fetch("/ai/compat/providers").then(function (r) { return r.json(); }).then(function (s) { + providers = s.providers; + providers.forEach(function (p) { + var o = document.createElement("option"); + o.value = p.id; + o.textContent = p.name + (p.configured ? " (key set)" : ""); + psel.appendChild(o); + }); + $("cp-config").textContent = providers.filter(function (p) { return p.configured; }).length + + " of " + providers.length + " providers have a credential. Others answer 503 until one is configured; dry runs still work."; + info(); + fill(); + }); +})(); diff --git a/src/vendors/compat/SPEC.md b/src/vendors/compat/SPEC.md new file mode 100644 index 0000000..b0ee508 --- /dev/null +++ b/src/vendors/compat/SPEC.md @@ -0,0 +1,243 @@ +# OpenAI-compatible provider registry (compat) - Specification + +Owned directory: `src/vendors/compat/`. Namespace `molo::compat`. Depends on +the `src/llm` foundation (`ApiClient`, `Service`, `SseDecoder`, `ApiError`, +`Config`, `ChatModel`). Plan: `plans/18-breadth.md` section 2 (method: +`plans/17-llm-vendor-platforms.md`). Credentials: one env var per provider, +named in the table of section 4, read through `llm::Config`. + +## 1. Scope + +One client for every provider that serves OpenAI-shaped endpoints. A provider +is ONE ROW in `registry.cpp`; clients, routes, demo page and tests iterate the +table, so adding a provider is one row plus one line in section 4 (enforced +by `tools/check_spec.py`). + +Endpoints, all OpenAI operations (section 3 of the checker verifies them +against OpenAI's OpenAPI document): chat completions (also streaming), +embeddings, models list, image generation, audio transcription, speech. A row +only claims an endpoint its own docs list or a probe distinguished; a call +outside the row's claim is refused before any network traffic. + +Not here, said plainly: legacy completions, moderations, translations, +image edits and variations, stored completions, the Responses API (no row's +docs were read for it except Cloudflare's, which lists `/responses`; not +wrapped), files, batches, fine-tuning. Providers with their own native API +have their own packages (mistral, cohere, perplexity, huggingface, ...). +Providers whose OpenAI-compatible endpoint could not be verified are in +section 5 and are not rows. + +## 2. Module map + +| File | Concern | +|---|---| +| `caps.h` | Static ceilings (`CompatCaps`) | +| `registry.h/.cpp` | `ProviderRow`, `Feature`, `Quirk`, `Evidence`, `Registry` and THE table | +| `calls.h/.cpp` | The call table (6 rows, each an OpenAI `operationId`) | +| `provider.h/.cpp` | `Provider` (row + `ApiClient`, base URL resolution and validation), `ProviderService` base | +| `chat_model.h/.cpp` | `CompatChatModel : llm::ChatModel` (generate, stream, wire mapping, quirks) | +| `services.h/.cpp` | `CompatModels`, `CompatEmbeddings`, `CompatImages`, `CompatAudio` | +| `tools/check_spec.py` | Checks the call table against OpenAI's OpenAPI document and the registry against section 4 | + +## 3. Contracts and static caps + +| Knob | Value | +|---|---| +| messages / tools / stop sequences | 4096 / 128 / 16 | +| max_tokens | 1..4194304 | +| temperature / top_p | 0..2 / (0,1] | +| n (extra) | 1..8; exactly 1 on rows with `kSingleChoice` | +| stream events | 200000 per stream, then the stream is stopped | +| models returned | 5000, then `truncated=true` (loud) | +| embedding inputs / chars per input | 2048 / 1 MiB | +| image n / prompt chars | 1..10 / 32768 | +| audio upload / speech input chars | 64 MiB / 4096 | +| model id chars / base URL chars | 256 / 512 | + +- Auth is `Authorization: Bearer ` for every row (the shape OpenAI + clients use). Local rows (`local=true`) are "configured" only when their + `base_env` is set; the key is optional and the placeholder `local` is sent + when none is set. Cloudflare needs `CLOUDFLARE_ACCOUNT_ID` (validated + `[A-Za-z0-9_-]{1,64}`) and substitutes it into the base URL. +- `_BASE_URL` style overrides are validated: `http(s)://`, no spaces, + control characters, `?`, `#` or `..`; trailing `/` removed. +- The model id is always required: ids differ per provider, so the module has + no honest default (`CompatChatModel` takes an optional default at construction). +- Quirks that the docs state as a 400 are rejected loudly before traffic + (`Quirk::kRejectMessageName`, `kRejectLogprobs`, `kSingleChoice`, + `kNoToolsWithFormat`); everything else is passed through untouched. +- Errors: every non-2xx maps to `llm::ApiError` (the shared envelope parser + handles `{error:{...}}`, `{message,type,code}`, `{detail}` and plain text). + A stream without `finish_reason` throws 502 `STREAM_TRUNCATED`. +- Model list shapes: `{"data":[...]}` and a bare array are both accepted. + +## 4. Provider table + +Source of truth is `registry.cpp`; this table must match it (checker). +Evidence: `docs+probe` = the provider's docs text names the base URL and an +unauthenticated probe on 2026-10-25 answered like an API; `probe-only` = the +docs page was not readable as text or did not name the URL, the probe answered +like an API; `docs-only` = named by docs, not probeable from here. + +| id | name | base URL | credential | features | evidence | documentation | documented quirks and probe results | +|---|---|---|---|---|---|---|---| +| `groq` | Groq | `https://api.groq.com/openai/v1` | `GROQ_API_KEY` | chat,models,transcribe | docs+probe | https://console.groq.com/docs/openai | Docs: logprobs, logit_bias, top_logprobs and messages[].name answer 400; n must be 1; temperature 0 becomes 1e-8; transcription formats vtt and srt unsupported. Probe: 401 invalid_api_key on /models and chat. | +| `together` | Together AI | `https://api.together.ai/v1` | `TOGETHER_API_KEY` | chat,embeddings,models,images,transcribe,speech | docs+probe | https://docs.together.ai/docs/openai-api-compatibility | Docs: logit_bias unsupported on most models; service_tier, store, metadata and prediction are accepted and ignored. Probe: 401 on /models and chat; legacy host api.together.xyz answers 401 too. | +| `fireworks` | Fireworks AI | `https://api.fireworks.ai/inference/v1` | `FIREWORKS_API_KEY` | chat,embeddings,models | docs+probe | https://docs.fireworks.ai/tools-sdks/openai-compatibility | Probe: /models 401 with a docs link; unknown model answers 404 NOT_FOUND; empty embeddings body answers 500. | +| `cerebras` | Cerebras | `https://api.cerebras.ai/v1` | `CEREBRAS_API_KEY` | chat,models | docs+probe | https://inference-docs.cerebras.ai/resources/openai | Docs: tools and response_format must not be combined (gpt-oss-120b rejects it); system and developer roles both act at developer level. Probe: /models 403 {detail}, chat 401 wrong_api_key, embeddings path 404. | +| `sambanova` | SambaNova | `https://api.sambanova.ai/v1` | `SAMBANOVA_API_KEY` | chat,embeddings,models | probe-only | https://docs.sambanova.ai/docs/en/features/openai-compatibility | Docs page answers 403 to a plain fetch. Probe: /models 200 public; chat 401 (also accepts x-api-key); embeddings answers 400 'must provide a model' before auth. | +| `deepinfra` | DeepInfra | `https://api.deepinfra.com/v1/openai` | `DEEPINFRA_API_KEY` | chat,embeddings,models,images,transcribe | docs+probe | https://docs.deepinfra.com/chat/overview | Probe: /models 200 public; chat 401 missing API key; images and audio paths answer 401 with a {detail} body. | +| `novita` | Novita AI | `https://api.novita.ai/openai/v1` | `NOVITA_API_KEY` | chat,models | docs+probe | https://novita.ai/docs/guides/llm-api | Probe: /models 200 public; bad key answers 403 INVALID_API_KEY (not 401); legacy base /v3/openai answers the same. | +| `nvidia` | NVIDIA NIM (build.nvidia.com) | `https://integrate.api.nvidia.com/v1` | `NVIDIA_API_KEY` | chat,embeddings,models | docs+probe | https://docs.api.nvidia.com/nim/reference/llm-apis | Probe: /models 200 public; embeddings answers 400 'model field is required'; retired models answer 410 Gone. | +| `cloudflare` | Cloudflare Workers AI | `https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1` | `CLOUDFLARE_API_TOKEN + CLOUDFLARE_ACCOUNT_ID` | chat,embeddings | docs-only | https://developers.cloudflare.com/workers-ai/configuration/open-ai-compatibility/ | Docs list /chat/completions, /embeddings and /responses under the account path. Probe: GET /models answers 405 (no list endpoint here); not probeable further without an account id. | +| `moonshot` | Moonshot (Kimi) | `https://api.moonshot.ai/v1` | `MOONSHOT_API_KEY` | chat,models | docs+probe | https://platform.kimi.ai/docs/api/chat | Probe: 401 incorrect_api_key_error on /models and chat; the China host api.moonshot.cn answers the same (separate keys). | +| `dashscope` | Alibaba DashScope (Qwen, intl) | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` | `DASHSCOPE_API_KEY` | chat,embeddings | docs+probe | https://www.alibabacloud.com/help/en/model-studio/compatibility-of-openai-with-dashscope | Probe: 401 on /models and chat; embeddings answers 400 'you must provide a model parameter'; China host dashscope.aliyuncs.com/compatible-mode/v1 answers 401 too. | +| `zhipu` | Zhipu (GLM, China) | `https://open.bigmodel.cn/api/paas/v4` | `ZHIPU_API_KEY` | chat,embeddings,images | docs+probe | https://docs.bigmodel.cn/cn/guide/develop/openai/introduction | Docs name the base URL; the path is /api/paas/v4, not /v1. Probe: 401 code 1001 (missing Authorization) on chat, embeddings and images. | +| `zai` | Z.ai (GLM, international) | `https://api.z.ai/api/paas/v4` | `ZAI_API_KEY` | chat,embeddings,images | docs+probe | https://docs.z.ai/api-reference/introduction | International host of the Zhipu API, same /api/paas/v4 path. Probe: 401 code 1001. | +| `minimax` | MiniMax | `https://api.minimax.io/v1` | `MINIMAX_API_KEY` | chat | probe-only | https://platform.minimax.io/docs/api-reference/text-openai-api | Docs text did not name the URL. Probe: 401 authorized_error on /models and chat; embeddings answers HTTP 200 with a base_resp status_code 1004 envelope (not OpenAI shaped); China host api.minimaxi.com answers 401. | +| `qianfan` | Baidu Qianfan | `https://qianfan.baidubce.com/v2` | `QIANFAN_API_KEY` | chat,embeddings | probe-only | https://cloud.baidu.com/doc/qianfan-api/s/3m9b5lqft | Docs page is rendered client side. Probe: chat and embeddings 401 invalid_iam_token; GET /models 403 AccessDenied (different envelope). | +| `ark` | Volcengine Ark (Doubao) | `https://ark.cn-beijing.volces.com/api/v3` | `ARK_API_KEY` | chat | probe-only | https://www.volcengine.com/docs/82379/1298454 | Docs page is rendered client side. Probe: 401 AuthenticationError on /models, chat and embeddings; path is /api/v3. | +| `siliconflow` | SiliconFlow | `https://api.siliconflow.com/v1` | `SILICONFLOW_API_KEY` | chat,embeddings,models | docs+probe | https://docs.siliconflow.com/en/api-reference/chat-completions/chat-completions | Probe: 401 code 30014 with a {code,data,message} envelope on every path; China host api.siliconflow.cn identical. | +| `nebius` | Nebius Token Factory | `https://api.studio.nebius.com/v1` | `NEBIUS_API_KEY` | chat,embeddings,models | docs+probe | https://docs.tokenfactory.nebius.com/api-reference/introduction | Probe: 401 unauthenticated on /models, chat and embeddings; images/generations 404 on this host; api.tokenfactory.nebius.com/v1 answers the same. | +| `scaleway` | Scaleway Generative APIs | `https://api.scaleway.ai/v1` | `SCALEWAY_API_KEY` | chat,embeddings,models | docs+probe | https://www.scaleway.com/en/docs/generative-apis/api-cli/using-chat-api/ | Probe: 401 UNAUTHORIZED (missing 'Authorization' header) on /models, chat and embeddings. | +| `upstage` | Upstage (Solar) | `https://api.upstage.ai/v1` | `UPSTAGE_API_KEY` | chat,embeddings | docs+probe | https://console.upstage.ai/docs/getting-started | Probe: 401 invalid API key on /models, chat and embeddings. | +| `inception` | Inception (Mercury) | `https://api.inceptionlabs.ai/v1` | `INCEPTION_API_KEY` | chat,models | docs+probe | https://docs.inceptionlabs.ai/get-started/get-started | Probe: /models 200 public; chat answers 400 listing the valid model names before auth. | +| `stepfun` | StepFun | `https://api.stepfun.com/v1` | `STEPFUN_API_KEY` | chat,models | docs+probe | https://platform.stepfun.com/docs/api-reference/chat/chat-completion-create | Probe: 401 invalid_api_key on /models and chat. | +| `hunyuan` | Tencent Hunyuan | `https://api.hunyuan.cloud.tencent.com/v1` | `HUNYUAN_API_KEY` | chat | probe-only | https://cloud.tencent.com/document/product/1729/111007 | Docs page did not name the URL in readable text. Probe: 401 'Incorrect API key provided' on /models and chat. | +| `venice` | Venice AI | `https://api.venice.ai/api/v1` | `VENICE_API_KEY` | chat,models | docs+probe | https://docs.venice.ai/api-reference/endpoint/chat/completions | Probe: /models 200 public; unauthenticated chat answers 402 with an x402 payment envelope (not 401). | +| `vercel` | Vercel AI Gateway | `https://ai-gateway.vercel.sh/v1` | `AI_GATEWAY_API_KEY` | chat,models | docs+probe | https://vercel.com/docs/ai-gateway/openai-compat | Probe: /models 200 public; chat 401 authentication_error 'Missing Authorization header'. | +| `friendli` | FriendliAI Serverless | `https://api.friendli.ai/serverless/v1` | `FRIENDLI_TOKEN` | chat,models | docs+probe | https://friendli.ai/docs/openapi/serverless/chat-completions | Probe: /models 200 public; chat 401 {detail}. | +| `gemini` | Google Gemini (OpenAI compatibility) | `https://generativelanguage.googleapis.com/v1beta/openai` | `GEMINI_API_KEY` | chat | docs+probe | https://ai.google.dev/gemini-api/docs/openai | Probe: chat answers 400 'Missing or invalid Authorization header'; unauthenticated GET /models and embeddings answer 404, so only chat is claimed. | +| `baseten` | Baseten Model APIs | `https://inference.baseten.co/v1` | `BASETEN_API_KEY` | chat | docs+probe | https://docs.baseten.co/development/model-apis/overview | Probe: /models 401 'No Authorization header provided'; chat 403 'please check the api-key you provided'. | +| `featherless` | Featherless AI | `https://api.featherless.ai/v1` | `FEATHERLESS_API_KEY` | chat,models | probe-only | https://featherless.ai/docs/overview | Docs text did not name the URL. Probe: /models 200 public; chat 401 'must be signed in'. | +| `ovh` | OVHcloud AI Endpoints | `https://oai.endpoints.kepler.ai.cloud.ovh.net/v1` | `OVH_AI_ENDPOINTS_ACCESS_TOKEN` | chat,models | probe-only | https://help.ovhcloud.com/csm/en-gb-public-cloud-ai-endpoints-getting-started | Docs text did not name the URL. Probe: /models 200 public; chat on an unknown model answers 404 model_not_found. | +| `parasail` | Parasail | `https://api.parasail.io/v1` | `PARASAIL_API_KEY` | chat,models | probe-only | https://docs.parasail.io/parasail-docs/serverless/serverless | Docs URL answered 404 at probe time. Probe: 401 'No bearer token provided' on /models and chat. | +| `crusoe` | Crusoe Managed Inference | `https://managed-inference-api-proxy.crusoecloud.com/v1` | `CRUSOE_API_KEY` | chat,models | probe-only | https://docs.crusoe.ai/managed-inference | Docs URL answered 404 at probe time. Probe: 401 {errors:[Authentication failed]} on /models and chat. | +| `xiaomi` | Xiaomi MiMo | `https://api.xiaomimimo.com/v1` | `XIAOMI_API_KEY` | chat | probe-only | https://platform.xiaomimimo.com/ | Docs text did not name the URL. Probe: 401 'Invalid API Key' on /models and chat. | +| `chutes` | Chutes | `https://llm.chutes.ai/v1` | `CHUTES_API_KEY` | chat,models | probe-only | https://docs.chutes.ai/ | Docs text did not name the URL. Probe: /models 200 public; chat answered an nginx 429 HTML page. | +| `ollama` | Ollama | `http://localhost:11434/v1` | `OLLAMA_API_KEY + OLLAMA_BASE_URL` | chat,embeddings,models | docs-only | https://raw.githubusercontent.com/ollama/ollama/main/docs/api/openai-compatibility.mdx | Docs name http://localhost:11434/v1. The key is required by OpenAI clients but ignored by the server; the module sends the placeholder 'local' when none is set. | +| `lmstudio` | LM Studio | `http://localhost:1234/v1` | `LMSTUDIO_API_KEY + LMSTUDIO_BASE_URL` | chat,embeddings,models | docs-only | https://lmstudio.ai/docs/developer/openai-compat | Docs name http://localhost:1234/v1. Key ignored unless configured in the app. | +| `vllm` | vLLM server | `http://localhost:8000/v1` | `VLLM_API_KEY + VLLM_BASE_URL` | chat,embeddings,models,transcribe | docs-only | https://docs.vllm.ai/en/latest/serving/online_serving/ | Docs list /v1/chat/completions, /v1/embeddings and /v1/audio/transcriptions; key only when the server was started with --api-key. | +| `llamacpp` | llama.cpp server | `http://localhost:8080/v1` | `LLAMACPP_API_KEY + LLAMACPP_BASE_URL` | chat,embeddings,models | docs-only | https://raw.githubusercontent.com/ggml-org/llama.cpp/master/tools/server/README.md | README lists /v1/chat/completions; key only when started with --api-key. Same default port as LocalAI. | +| `localai` | LocalAI | `http://localhost:8080/v1` | `LOCALAI_API_KEY + LOCALAI_BASE_URL` | chat,embeddings,models | docs-only | https://localai.io/docs/overview/ | Docs name localhost:8080; endpoint list beyond chat, embeddings and models was not read. | + +## 5. Providers not in the table + +Plan 18 names these; they are NOT rows because the OpenAI-compatible endpoint +could not be verified from here (2026-10-25). Each is one row away once +confirmed. + +| Provider | What was seen | +|---|---| +| GitHub Models | Docs name `https://models.github.ai/inference/chat/completions` and a catalog at `https://models.github.ai/catalog/models`, but the docs page read did not contain the URL as text, and every unauthenticated request (any path) answered `200 OK` with body `OK`, which proves nothing. The catalog lives outside the inference base path, which the row model cannot express (`models_path` is relative to the base) | +| Lambda | `api.lambda.ai`, `api.lambdalabs.com` and `inference.lambda.ai` did not resolve from here | +| Lepton | `api.lepton.ai` did not resolve (service wound down) | +| Hyperbolic | `/models` answers 401, but chat and image paths answered `404 "This inference endpoint has been decommissioned"` on every probe | +| Kluster | `api.kluster.ai` did not resolve | +| Anyscale | The Public Endpoints API answers a 404 page announcing its end | +| Sarvam | `/v1/models` is public, chat answered 403 `invalid_api_key_error`; Sarvam documents its own subscription-key header, not Bearer, so the Bearer-only row model would be wrong | +| Reka | `api.reka.ai/v1` answers 401 `missing_api_key`; no OpenAI-compatibility statement was found | +| AI21 | `/studio/v1/models` answers an empty public list and chat 400 "bad or missing authentication"; the chat shape was not confirmed as OpenAI's | +| Mistral, Cohere, Perplexity, Hugging Face, OpenRouter, xAI, DeepSeek, OpenAI | Own packages (plan 17 / 18) | + +## 6. Demo routes + +Page `/ai/compat`, GET `/ai/compat/status`, `/ai/compat/providers` (the +table plus per-row configured state), `/ai/compat/calls`. POST routes (body +carries `"provider":""`): `chat`, `models`, `embeddings`, `images`, +`transcribe`, `speech`. 422 on bad input or a call the row does not claim, +503 `compat_not_configured` without the row's credential (the message names +the missing env), paid routes (all but `models`) are dry runs showing the +exact request and URL unless the body has `"confirm":true`. Matrix row +`ai-compat` for the page. + +## 7. Sources + +- OpenAI OpenAPI document `openai/openai-openapi` at commit + `8243cc8fa6f04430fb1f1a6a2cb926fbeb4f7f26` (`openapi.json`, version 2.3.0): + the shape of every call. +- Per-row documentation URL in the table of section 4. Each page was fetched + as plain text on 2026-10-25 and searched for the base URL; "docs+probe" + rows contain it. The documented quirks encoded as `Quirk` flags were read + from: Groq (`console.groq.com/docs/openai`, "Currently Unsupported OpenAI + Features": logprobs, logit_bias, top_logprobs, messages[].name answer 400; + n must be 1) and Cerebras (`inference-docs.cerebras.ai/resources/openai`, + "Unsupported parameter combinations": tools with response_format). +- Together's docs page lists `/v1/chat/completions`, `/v1/embeddings`, + `/v1/images/generations`, `/v1/audio/speech`, `/v1/audio/transcriptions` and + moved its host to `api.together.ai`; Cloudflare's page lists + `/chat/completions`, `/embeddings`, `/responses` under + `/client/v4/accounts/{account_id}/ai/v1`; vLLM's page lists + `/v1/chat/completions`, `/v1/embeddings`, `/v1/audio/transcriptions`. + +## 8. Verified against spec + +`python3 -I src/vendors/compat/tools/check_spec.py` on 2026-10-25 against +the pinned OpenAI document (2.3.0): 6 call rows OK (method, path, operation +id, request schema, every body field, response schema, every response field +read), coverage OK (19 operations of the touched families are excluded with a +reason in the script), registry OK (39 rows equal the table of section 4 in +ids, base URLs, credential names and documentation URLs). `--local` runs only +the registry check without network. This checks the OpenAI shape, not any +provider's conformance to it. + +## 9. Verified live + +No provider key exists in this environment. Unauthenticated probes, no key +sent, 2026-10-25 (`MOLO_LIVE_NET=1`, test `compat_live.invalid_key_probe...`, +which uses an obviously invalid key and asserts "never 404"). Result per row +(status of `GET /models` with the invalid key where the row lists models, +otherwise a one-token `POST /chat/completions`): 401 for groq, together, +fireworks, cerebras, deepinfra, dashscope, zhipu, zai, minimax, ark, +siliconflow, nebius, upstage, moonshot, stepfun, hunyuan, vercel, featherless, +parasail, crusoe, xiaomi; 403 for scaleway, baseten, ovh; 400 for gemini +(`Missing or invalid Authorization header`); 200 for novita, nvidia, inception, +venice, friendli (public lists that ignore the key). Three hosts could not be reached by OUR +client: sambanova and chutes (`wolfSSL handshake failed: ASN parsing error, +invalid input`) and qianfan (`handshake failed: received alert fatal error`) +while `curl` reaches all three; see Gaps. Local rows were not probed (no +server). Opt-in live tests: per row with its key env set, list models (when +claimed) and ONE completion of `max_tokens` 8 on `COMPAT__MODEL` (upper +case id); a missing key or model prints a loud SKIP line per row. No success +body from any provider was seen. + +## 10. Unverified + +- Every success response of every provider (only OpenAI's documented examples + are used as parsing fixtures, labelled FIXTURE in the tests). +- The `features` mask of each row beyond what its docs page or a probe showed. + An unauthenticated 401 does not prove a path exists (many providers check + auth first), so only these claims rest on more: embeddings routes answering + 400 before auth (sambanova, nvidia, dashscope), a 404 proving absence + (cerebras embeddings, nebius images, gemini models and embeddings), and the + docs pages read for Together and Cloudflare. All other claims are the + provider's well known documented surface, not re-read here: images and audio + for together, deepinfra, zhipu, zai; transcription for groq and vllm. +- `docs-only` rows: Cloudflare (needs an account id), the five local servers. +- Quirks other than the four flags (for example Groq turning temperature 0 + into 1e-8, Moonshot and Kimi temperature limits, per-model parameter + support) are listed in notes or not at all, and are not enforced. +- Provider-specific auth beyond Bearer (Baseten documents its own scheme; the + probe showed Bearer is the shape OpenAI clients send, acceptance with a real + key is unproven). +- China-region hosts (moonshot.cn, dashscope.aliyuncs.com, minimaxi.com, + siliconflow.cn) answer like their international twins but are not rows; set + the override env for local rows only. Hosted rows cannot be overridden by + env today (see Gaps). + +## 11. Gaps + +- `src/clients` TLS: wolfSSL handshake fails against sambanova, chutes and + qianfan hosts where curl succeeds. Owner: `src/clients/transports` (not this + package); the rows stay because the endpoints answer to curl. +- `src/llm`: the shared `MOLO_LIVE_NET` is unset by a test in `tests/src/llm` + (`test_llm.cpp`) so any suite reading it at run time skips in the shared + binary; this suite reads it at static-init time. Other vendors' live tests + still skip when the whole `ai_test` binary runs. +- No env override for hosted base URLs (regional hosts); the `base_env` switch + exists for local rows only. Closing it is one `base_env` value per hosted + row in `registry.cpp`. +- GitHub Models needs a second base URL for its catalog (see section 5). +- Streaming sends no `stream_options.include_usage` by default because some + providers reject it; pass it through `ChatRequest.extra`. +- Responses API, legacy completions, image edits and moderation are not wrapped. diff --git a/src/vendors/compat/calls.cpp b/src/vendors/compat/calls.cpp new file mode 100644 index 0000000..11dc615 --- /dev/null +++ b/src/vendors/compat/calls.cpp @@ -0,0 +1,50 @@ +#include "vendors/compat/calls.h" + +#include "llm/error.h" +#include "vendors/compat/registry.h" + +namespace molo::compat { + +namespace { + +const CallSpec kCalls[] = { + {Call::kChat, "POST", "/chat/completions", "createChatCompletion", "CreateChatCompletionRequest", + "model,messages,max_tokens,temperature,top_p,stop,tools,tool_choice,response_format,reasoning_effort," + "stream,stream_options,n", + "CreateChatCompletionResponse", "id,model,choices,usage", Feature::kChat, "chat"}, + {Call::kEmbeddings, "POST", "/embeddings", "createEmbedding", "CreateEmbeddingRequest", + "model,input,dimensions,encoding_format", "CreateEmbeddingResponse", "data,model,usage", + Feature::kEmbeddings, "embeddings"}, + {Call::kModels, "GET", "/models", "listModels", "", "", "ListModelsResponse", "data", + Feature::kModels, "models"}, + {Call::kImages, "POST", "/images/generations", "createImage", "CreateImageRequest", + "model,prompt,n,size,response_format", "ImagesResponse", "data", Feature::kImages, "images"}, + {Call::kTranscribe, "POST", "/audio/transcriptions", "createTranscription", "CreateTranscriptionRequest", + "file,model,language,prompt,response_format,temperature", "CreateTranscriptionResponseJson", "text", + Feature::kTranscribe, "transcribe"}, + {Call::kSpeech, "POST", "/audio/speech", "createSpeech", "CreateSpeechRequest", + "model,input,voice,response_format,speed", "", "", Feature::kSpeech, "speech"}, +}; + +constexpr std::size_t kCallCount = sizeof(kCalls) / sizeof(kCalls[0]); + +} // namespace + +const CallSpec& Calls::spec_of(Call call) { + for (const CallSpec& s : kCalls) + if (s.call == call) return s; + throw llm::ApiError::invalid("compat", "unknown call"); +} + +clients::Method Calls::method_of(Call call) { + return std::string(spec_of(call).http_method) == "GET" ? clients::Method::kGet : clients::Method::kPost; +} + +std::size_t Calls::count() { return kCallCount; } + +const CallSpec& Calls::at(std::size_t index) { + if (index >= kCallCount) throw llm::ApiError::invalid("compat", "call index out of range"); + return kCalls[index]; +} + +} // namespace molo::compat diff --git a/src/vendors/compat/calls.h b/src/vendors/compat/calls.h new file mode 100644 index 0000000..645ea6b --- /dev/null +++ b/src/vendors/compat/calls.h @@ -0,0 +1,38 @@ +#pragma once + +// The ONE table of REST calls this module makes. The shapes are OpenAI's +// (every row is an OpenAI operation); providers differ only in base URL, auth +// and quirks, which live in registry.cpp. tools/check_spec.py checks every +// row against OpenAI's published OpenAPI document. Code builds every path +// from this table, never from a literal. + +#include +#include + +#include "clients/request.h" + +namespace molo::compat { + +enum class Call { kChat, kEmbeddings, kModels, kImages, kTranscribe, kSpeech }; + +struct CallSpec { + Call call; + const char* http_method; // GET POST + const char* path; // from the provider base URL + const char* operation_id; // OpenAI OpenAPI operationId + const char* request; // OpenAI request schema name ("" for GET) + const char* body_fields; // comma list: request body fields we send + const char* response; // OpenAI response schema name + const char* fields_read; // comma list: response fields we read + unsigned feature; // registry Feature bit a row must claim + const char* feature_name; +}; + +struct Calls { + static const CallSpec& spec_of(Call call); + static clients::Method method_of(Call call); + static std::size_t count(); + static const CallSpec& at(std::size_t index); +}; + +} // namespace molo::compat diff --git a/src/vendors/compat/caps.h b/src/vendors/compat/caps.h new file mode 100644 index 0000000..9d4102a --- /dev/null +++ b/src/vendors/compat/caps.h @@ -0,0 +1,28 @@ +#pragma once + +// Static ceilings for the compat module. Never autotuned. + +#include + +namespace molo::compat { + +struct CompatCaps { + static constexpr std::size_t kMaxMessages = 4096; + static constexpr std::size_t kMaxTools = 128; + static constexpr std::size_t kMaxStopSequences = 16; + static constexpr std::size_t kMaxStreamEvents = 200000; + static constexpr std::size_t kMaxChoices = 8; + static constexpr std::size_t kMaxModelsReturned = 5000; + static constexpr std::size_t kMaxEmbeddingInputs = 2048; + static constexpr std::size_t kMaxEmbeddingInputChars = 1u * 1024u * 1024u; + static constexpr std::size_t kMaxPromptChars = 32768; + static constexpr int kMaxImages = 10; + static constexpr std::size_t kMaxAudioBytes = 64u * 1024u * 1024u; + static constexpr std::size_t kMaxSpeechChars = 4096; + static constexpr std::size_t kMaxModelChars = 256; + static constexpr std::size_t kMaxBaseUrlChars = 512; + static constexpr std::size_t kMaxFilenameChars = 256; + static constexpr int kMaxTokensCeiling = 4194304; +}; + +} // namespace molo::compat diff --git a/src/vendors/compat/chat_model.cpp b/src/vendors/compat/chat_model.cpp new file mode 100644 index 0000000..d6137b6 --- /dev/null +++ b/src/vendors/compat/chat_model.cpp @@ -0,0 +1,375 @@ +#include "vendors/compat/chat_model.h" + +#include + +#include "security/crypto/Base64.h" +#include "vendors/compat/caps.h" + +namespace molo::compat { + +using Json = clients::body::Json; + +namespace { + +[[noreturn]] void bad(const std::string& vendor, const std::string& why) { + throw llm::ApiError::invalid(vendor, why); +} + +std::string str_at(const Json& j, const char* key) { + const Json* v = j.is_object() ? j.find(key) : nullptr; + return v && v->is_string() ? v->as_string() : ""; +} + +int int_at(const Json& j, const char* key) { + const Json* v = j.is_object() ? j.find(key) : nullptr; + return v && v->is_number() ? static_cast(v->as_int()) : 0; +} + +const Json* obj_at(const Json& j, const char* key) { + const Json* v = j.is_object() ? j.find(key) : nullptr; + return v && v->is_object() ? v : nullptr; +} + +Json image_part(const std::string& vendor, const llm::ImageData& img) { + std::string url = img.url; + if (url.empty()) { + if (img.bytes.empty()) bad(vendor, "image has neither url nor bytes"); + if (img.mime.empty()) bad(vendor, "image bytes need a mime type"); + url = "data:" + img.mime + ";base64," + security::crypto::Base64::encode(img.bytes); + } + Json u = Json::object(); + u.set("url", Json(url)); + Json p = Json::object(); + p.set("type", Json("image_url")); + p.set("image_url", u); + return p; +} + +Json text_part(const std::string& t) { + Json p = Json::object(); + p.set("type", Json("text")); + p.set("text", Json(t)); + return p; +} + +Json tool_call_json(const llm::ToolCall& c) { + Json fn = Json::object(); + fn.set("name", Json(c.name)); + fn.set("arguments", Json(c.arguments.is_null() ? std::string("{}") : c.arguments.dump())); + Json o = Json::object(); + o.set("id", Json(c.id)); + o.set("type", Json("function")); + o.set("function", fn); + return o; +} + +void append_message(const std::string& vendor, const ProviderRow& row, Json& out, const llm::Message& m) { + if (!m.name.empty() && (row.quirks & Quirk::kRejectMessageName)) + bad(vendor, std::string(row.name) + " rejects messages[].name (documented 400)"); + auto base = [&](const char* role) { + Json o = Json::object(); + o.set("role", Json(role)); + if (!m.name.empty()) o.set("name", Json(m.name)); + return o; + }; + switch (m.role) { + case llm::Role::kSystem: { + Json o = base("system"); + o.set("content", Json(m.text())); + out.push(o); + return; + } + case llm::Role::kUser: { + Json o = base("user"); + bool has_image = false; + for (const llm::ContentPart& p : m.parts) has_image = has_image || p.image.has_value(); + if (!has_image) { + o.set("content", Json(m.text())); + } else { + Json parts = Json::array(); + for (const llm::ContentPart& p : m.parts) { + if (!p.text.empty()) parts.push(text_part(p.text)); + if (p.image) parts.push(image_part(vendor, *p.image)); + } + o.set("content", parts); + } + out.push(o); + return; + } + case llm::Role::kAssistant: { + Json o = base("assistant"); + Json calls = Json::array(); + bool any = false; + for (const llm::ContentPart& p : m.parts) + if (p.tool_call) { + calls.push(tool_call_json(*p.tool_call)); + any = true; + } + std::string text = m.text(); + o.set("content", text.empty() && any ? Json() : Json(text)); + if (any) o.set("tool_calls", calls); + out.push(o); + return; + } + case llm::Role::kTool: { + bool any = false; + for (const llm::ContentPart& p : m.parts) + if (p.tool_result) { + Json o = Json::object(); + o.set("role", Json("tool")); + o.set("tool_call_id", Json(p.tool_result->call_id)); + o.set("content", Json(p.tool_result->content)); + out.push(o); + any = true; + } + if (!any) bad(vendor, "a tool message needs a tool result part"); + return; + } + } +} + +llm::Usage usage_of(const Json& u) { + llm::Usage out; + if (!u.is_object()) return out; + out.input_tokens = int_at(u, "prompt_tokens"); + out.output_tokens = int_at(u, "completion_tokens"); + out.total_tokens = int_at(u, "total_tokens"); + if (const Json* d = obj_at(u, "prompt_tokens_details")) out.cached_tokens = int_at(*d, "cached_tokens"); + if (const Json* d = obj_at(u, "completion_tokens_details")) out.reasoning_tokens = int_at(*d, "reasoning_tokens"); + return out; +} + +std::string reasoning_of(const Json& holder) { + std::string r = str_at(holder, "reasoning_content"); + return r.empty() ? str_at(holder, "reasoning") : r; +} + +Json arguments_of(const std::string& raw) { + if (raw.empty()) return Json::object(); + try { + Json j = Json::parse(raw); + if (j.is_object()) return j; + } catch (const std::exception&) { + } + Json badj = Json::object(); + badj.set("_invalid_json", Json(raw)); + return badj; +} + +} // namespace + +CompatChatModel::CompatChatModel(Provider provider, std::string default_model) + : ProviderService(std::move(provider)), default_model_(std::move(default_model)) {} + +std::shared_ptr CompatChatModel::from_env(const std::string& provider_id, + const std::string& default_model) { + return std::make_shared(Provider::from_env(provider_id), default_model); +} + +Json CompatChatModel::to_body(const llm::ChatRequest& r, bool stream) const { + const std::string vendor = client_->config().vendor; + const ProviderRow& row = provider_.row(); + provider_.require(Call::kChat); + + std::string model = r.model.empty() ? default_model_ : r.model; + if (model.empty()) bad(vendor, "model is required (ids differ per provider)"); + if (model.size() > CompatCaps::kMaxModelChars) bad(vendor, "model id too long"); + if (r.messages.empty() && r.system.empty()) bad(vendor, "at least one message is required"); + if (r.messages.size() > CompatCaps::kMaxMessages) bad(vendor, "too many messages"); + if (r.tools.size() > CompatCaps::kMaxTools) bad(vendor, "too many tools"); + if (r.stop.size() > CompatCaps::kMaxStopSequences) bad(vendor, "too many stop sequences"); + if (r.max_tokens && (*r.max_tokens < 1 || *r.max_tokens > CompatCaps::kMaxTokensCeiling)) + bad(vendor, "max_tokens out of range"); + if (r.temperature && (*r.temperature < 0.0 || *r.temperature > 2.0)) bad(vendor, "temperature must be 0..2"); + if (r.top_p && (*r.top_p <= 0.0 || *r.top_p > 1.0)) bad(vendor, "top_p must be in (0,1]"); + const bool has_format = r.json_mode || !r.response_schema.is_null(); + if (r.json_mode && !r.response_schema.is_null()) bad(vendor, "json_mode and response_schema are exclusive"); + if ((row.quirks & Quirk::kNoToolsWithFormat) && has_format && !r.tools.empty()) + bad(vendor, std::string(row.name) + " documents that tools and response_format must not be combined"); + + Json body = Json::object(); + body.set("model", Json(model)); + Json messages = Json::array(); + if (!r.system.empty()) append_message(vendor, row, messages, llm::Message::system(r.system)); + for (const llm::Message& m : r.messages) append_message(vendor, row, messages, m); + body.set("messages", messages); + if (r.max_tokens) body.set("max_tokens", Json(*r.max_tokens)); + if (r.temperature) body.set("temperature", Json(*r.temperature)); + if (r.top_p) body.set("top_p", Json(*r.top_p)); + if (!r.stop.empty()) { + Json s = Json::array(); + for (const std::string& x : r.stop) s.push(Json(x)); + body.set("stop", s); + } + if (!r.tools.empty()) { + Json tools = Json::array(); + for (const llm::ToolSpec& t : r.tools) { + if (t.name.empty()) bad(vendor, "tool name is required"); + Json fn = Json::object(); + fn.set("name", Json(t.name)); + if (!t.description.empty()) fn.set("description", Json(t.description)); + fn.set("parameters", t.parameters); + Json o = Json::object(); + o.set("type", Json("function")); + o.set("function", fn); + tools.push(o); + } + body.set("tools", tools); + } + if (!r.tool_choice.empty()) { + if (r.tools.empty()) bad(vendor, "tool_choice needs tools"); + if (r.tool_choice == "auto" || r.tool_choice == "none" || r.tool_choice == "required") { + body.set("tool_choice", Json(r.tool_choice)); + } else { + Json fn = Json::object(); + fn.set("name", Json(r.tool_choice)); + Json o = Json::object(); + o.set("type", Json("function")); + o.set("function", fn); + body.set("tool_choice", o); + } + } + if (r.json_mode) { + Json f = Json::object(); + f.set("type", Json("json_object")); + body.set("response_format", f); + } else if (!r.response_schema.is_null()) { + Json js = Json::object(); + js.set("name", Json("response")); + js.set("strict", Json(true)); + js.set("schema", r.response_schema); + Json f = Json::object(); + f.set("type", Json("json_schema")); + f.set("json_schema", js); + body.set("response_format", f); + } + if (!r.reasoning_effort.empty()) body.set("reasoning_effort", Json(r.reasoning_effort)); + if (r.extra.is_object()) { + for (const auto& [k, v] : r.extra.as_object()) { + if (k == "model" || k == "messages" || k == "stream") bad(vendor, "extra may not set '" + k + "'"); + if ((row.quirks & Quirk::kRejectLogprobs) && (k == "logprobs" || k == "top_logprobs" || k == "logit_bias")) + bad(vendor, std::string(row.name) + " rejects " + k + " (documented 400)"); + if ((row.quirks & Quirk::kSingleChoice) && k == "n" && !(v.is_number() && v.as_int() == 1)) + bad(vendor, std::string(row.name) + " requires n = 1"); + if (k == "n" && v.is_number() && (v.as_int() < 1 || v.as_int() > static_cast(CompatCaps::kMaxChoices))) + bad(vendor, "n out of range"); + body.set(k, v); + } + } else if (!r.extra.is_null()) { + bad(vendor, "extra must be an object"); + } + if (stream) body.set("stream", Json(true)); + return body; +} + +llm::ChatResponse CompatChatModel::parse_response(const Json& doc) { + llm::ChatResponse out; + out.id = str_at(doc, "id"); + out.model = str_at(doc, "model"); + out.message.role = llm::Role::kAssistant; + out.raw = doc; + out.usage = usage_of(doc.is_object() && doc.find("usage") ? *doc.find("usage") : Json()); + const Json* choices = doc.is_object() ? doc.find("choices") : nullptr; + if (!choices || !choices->is_array() || choices->as_array().empty()) + throw llm::ApiError("compat", 502, "BAD_RESPONSE", "response has no choices"); + const Json& c = choices->as_array()[0]; + out.finish_reason = str_at(c, "finish_reason"); + const Json* msg = obj_at(c, "message"); + if (!msg) throw llm::ApiError("compat", 502, "BAD_RESPONSE", "choice has no message"); + std::string reasoning = reasoning_of(*msg); + if (!reasoning.empty()) { + llm::ContentPart r; + r.reasoning = reasoning; + out.message.parts.push_back(r); + } + std::string content = str_at(*msg, "content"); + const Json* calls = msg->find("tool_calls"); + const bool has_calls = calls && calls->is_array() && !calls->as_array().empty(); + if (!content.empty() || !has_calls) out.message.parts.push_back(llm::ContentPart::of_text(content)); + if (has_calls) + for (const Json& call : calls->as_array()) { + llm::ToolCall tc; + tc.id = str_at(call, "id"); + if (const Json* fn = obj_at(call, "function")) { + tc.name = str_at(*fn, "name"); + tc.arguments = arguments_of(str_at(*fn, "arguments")); + } + out.message.parts.push_back(llm::ContentPart::of_call(tc)); + } + return out; +} + +llm::ChatResponse CompatChatModel::generate(const llm::ChatRequest& request) { + Json body = to_body(request, false); + return parse_response(client_->post(Calls::spec_of(Call::kChat).path, body)); +} + +void CompatChatModel::stream(const llm::ChatRequest& request, + const std::function& on_chunk) { + Json body = to_body(request, true); + std::map ids, names; + std::size_t events = 0; + bool finished = false, stopped = false; + client_->stream(Calls::spec_of(Call::kChat).path, body, [&](const llm::SseEvent& ev) { + if (++events > CompatCaps::kMaxStreamEvents) { + stopped = true; + return false; + } + const Json& d = ev.data; + if (const Json* err = obj_at(d, "error")) + throw llm::ApiError(vendor(), 502, str_at(*err, "type").empty() ? "STREAM_ERROR" : str_at(*err, "type"), + str_at(*err, "message")); + auto emit = [&](const llm::ChatChunk& c) { + if (!on_chunk(c)) { + stopped = true; + return false; + } + return true; + }; + const Json* choices = d.is_object() ? d.find("choices") : nullptr; + if (choices && choices->is_array() && !choices->as_array().empty()) { + const Json& c = choices->as_array()[0]; + if (const Json* delta = obj_at(c, "delta")) { + std::string text = str_at(*delta, "content"), reasoning = reasoning_of(*delta); + if (!text.empty() || !reasoning.empty()) { + llm::ChatChunk ch; + ch.text_delta = text; + ch.reasoning_delta = reasoning; + if (!emit(ch)) return false; + } + const Json* calls = delta->find("tool_calls"); + if (calls && calls->is_array()) + for (const Json& f : calls->as_array()) { + int index = int_at(f, "index"); + if (!str_at(f, "id").empty()) ids[index] = str_at(f, "id"); + const Json* fn = obj_at(f, "function"); + if (fn && !str_at(*fn, "name").empty()) names[index] = str_at(*fn, "name"); + llm::ChatChunk ch; + ch.tool_call_id = ids[index]; + ch.tool_call_name = names[index]; + ch.tool_args_delta = fn ? str_at(*fn, "arguments") : ""; + if (!emit(ch)) return false; + } + } + std::string fin = str_at(c, "finish_reason"); + if (!fin.empty()) { + finished = true; + llm::ChatChunk ch; + ch.finish_reason = fin; + if (const Json* u = obj_at(d, "usage")) ch.usage = usage_of(*u); + if (!emit(ch)) return false; + return true; + } + } + if (const Json* u = obj_at(d, "usage")) { + llm::ChatChunk ch; + ch.usage = usage_of(*u); + if (!emit(ch)) return false; + } + return true; + }); + if (!stopped && !finished) + throw llm::ApiError(vendor(), 502, "STREAM_TRUNCATED", "stream ended without a finish_reason"); +} + +} // namespace molo::compat diff --git a/src/vendors/compat/chat_model.h b/src/vendors/compat/chat_model.h new file mode 100644 index 0000000..c24a711 --- /dev/null +++ b/src/vendors/compat/chat_model.h @@ -0,0 +1,48 @@ +#pragma once + +// CompatChatModel: llm::ChatModel over POST {base}/chat/completions of any +// registry row. Mapping notes (each one is a decision): +// - ChatRequest.model is required unless a default model is given: model ids +// differ per provider and there is no honest module-wide default. +// - ChatRequest.system becomes the first system message. +// - Reasoning text of an assistant message is never sent back. +// - json_mode -> response_format {type: json_object}; response_schema -> +// response_format {type: json_schema, strict: true}. +// - reasoning_effort is sent as reasoning_effort; ChatRequest.extra is merged +// except for keys the module owns (model, messages, stream). +// - Row quirks (registry.h Quirk) are rejected before any traffic. +// - Reasoning comes back from reasoning_content or reasoning, whichever the +// provider sends. +// - Tool call arguments that are not valid JSON arrive as +// {"_invalid_json": ""}. +// - Streaming sends no stream_options unless ChatRequest.extra carries them; +// usage then appears only when the provider sends it. A stream that ends +// without a finish_reason throws 502 STREAM_TRUNCATED. + +#include + +#include "llm/chat_types.h" +#include "vendors/compat/provider.h" + +namespace molo::compat { + +class CompatChatModel : public ProviderService, public llm::ChatModel { + public: + explicit CompatChatModel(Provider provider, std::string default_model = ""); + static std::shared_ptr from_env(const std::string& provider_id, + const std::string& default_model = ""); + + std::string vendor() const override { return client_->config().vendor; } + llm::ChatResponse generate(const llm::ChatRequest& request) override; + void stream(const llm::ChatRequest& request, + const std::function& on_chunk) override; + + // The wire body a neutral request maps to (also used by tests and dry runs). + clients::body::Json to_body(const llm::ChatRequest& request, bool stream) const; + static llm::ChatResponse parse_response(const clients::body::Json& doc); + + private: + std::string default_model_; +}; + +} // namespace molo::compat diff --git a/src/vendors/compat/provider.cpp b/src/vendors/compat/provider.cpp new file mode 100644 index 0000000..31aa8cb --- /dev/null +++ b/src/vendors/compat/provider.cpp @@ -0,0 +1,110 @@ +#include "vendors/compat/provider.h" + +#include + +#include "clients/transports/http_transport.h" +#include "llm/config.h" +#include "vendors/compat/caps.h" + +namespace molo::compat { + +namespace { + +bool valid_account_id(const std::string& s) { + if (s.empty() || s.size() > 64) return false; + for (char c : s) + if (!(std::isalnum(static_cast(c)) || c == '-' || c == '_')) return false; + return true; +} + +bool valid_base(const std::string& s) { + if (s.empty() || s.size() > CompatCaps::kMaxBaseUrlChars) return false; + if (s.rfind("http://", 0) != 0 && s.rfind("https://", 0) != 0) return false; + for (char c : s) + if (static_cast(c) <= 0x20 || c == 0x7f || c == '?' || c == '#') return false; + return s.find("..") == std::string::npos; +} + +std::string replace_account(std::string base, const std::string& account) { + const std::string token = "{account_id}"; + std::size_t at = base.find(token); + if (at != std::string::npos) base.replace(at, token.size(), account); + return base; +} + +// The key a row would use: configured env value; local rows get a +// placeholder when their base_env switch is set (servers ignore it). +std::string key_for(const ProviderRow& row, const std::string& key_env_value, bool switch_set) { + if (!row.local) return key_env_value; + if (!switch_set) return ""; + return key_env_value.empty() ? "local" : key_env_value; +} + +} // namespace + +std::string Provider::resolve_base(const ProviderRow& row, const std::string& base_override, + const std::string& account_id) { + std::string base = base_override.empty() ? row.base_url : base_override; + if (base.find("{account_id}") != std::string::npos) { + if (!valid_account_id(account_id)) + throw llm::ApiError::invalid("compat:" + std::string(row.id), "account id missing or malformed"); + base = replace_account(base, account_id); + } + while (base.size() > 8 && base.back() == '/') base.pop_back(); + if (!valid_base(base)) + throw llm::ApiError::invalid("compat:" + std::string(row.id), "base URL must be http(s), no spaces, query or fragment"); + return base; +} + +llm::ApiClientConfig Provider::config_for(const ProviderRow& row, const std::string& base_url, + const std::string& missing_env) { + llm::ApiClientConfig c; + c.vendor = std::string("compat:") + row.id; + c.base_url = base_url; + c.key_env = missing_env.empty() ? row.key_env : missing_env; + c.auth = llm::AuthStyle::kBearer; + return c; +} + +Provider Provider::from_env(const std::string& id) { + const ProviderRow& row = Registry::get(id); + std::string base_override = row.base_env[0] ? llm::Config::get(row.base_env) : ""; + std::string account = row.extra_env[0] ? llm::Config::get(row.extra_env) : ""; + std::string key = key_for(row, llm::Config::get(row.key_env), !base_override.empty()); + std::string missing; + if (row.local && base_override.empty()) missing = row.base_env; + else if (row.extra_env[0] && account.empty()) missing = row.extra_env; + if (!missing.empty()) key.clear(); + // An unconfigured row still needs a syntactically valid base URL to exist. + std::string base = (row.extra_env[0] && account.empty()) + ? replace_account(row.base_url, "unset") + : resolve_base(row, base_override, account); + auto client = std::make_shared(std::make_unique(), + config_for(row, base, missing), key); + return Provider(&row, base, client); +} + +Provider Provider::with(const std::string& id, std::unique_ptr transport, + const std::string& key, const std::string& base_override, + const std::string& account_id) { + const ProviderRow& row = Registry::get(id); + std::string base = resolve_base(row, base_override, account_id); + auto client = std::make_shared(std::move(transport), config_for(row, base, ""), key); + return Provider(&row, base, client); +} + +std::string Provider::missing_env() const { + if (configured()) return ""; + return client_->config().key_env; +} + +bool Provider::supports(Call call) const { return Registry::has(*row_, Calls::spec_of(call).feature); } + +void Provider::require(Call call) const { + if (!supports(call)) + throw llm::ApiError::invalid(client_->config().vendor, + std::string(row_->name) + " has no documented " + Calls::spec_of(call).feature_name + + " endpoint in the registry"); +} + +} // namespace molo::compat diff --git a/src/vendors/compat/provider.h b/src/vendors/compat/provider.h new file mode 100644 index 0000000..cbe5217 --- /dev/null +++ b/src/vendors/compat/provider.h @@ -0,0 +1,64 @@ +#pragma once + +// Provider: one registry row bound to one ApiClient. The only place a base +// URL is resolved (account id substitution, env override) and validated. +// Services take a Provider and refuse calls the row does not claim. + +#include +#include + +#include "llm/api_client.h" +#include "vendors/compat/calls.h" +#include "vendors/compat/registry.h" + +namespace molo::compat { + +class Provider { + public: + // Real HTTPS transport. Key and overrides come from llm::Config. + static Provider from_env(const std::string& id); + // Same row, any transport (tests). base_override and account_id may be empty. + static Provider with(const std::string& id, std::unique_ptr transport, + const std::string& key, const std::string& base_override = "", + const std::string& account_id = ""); + + // base_url with {account_id} replaced and a trailing slash removed. Throws + // ApiError::invalid for a malformed override or account id. + static std::string resolve_base(const ProviderRow& row, const std::string& base_override, + const std::string& account_id); + + const ProviderRow& row() const { return *row_; } + const std::string& base_url() const { return base_url_; } + std::shared_ptr client() const { return client_; } + bool configured() const { return client_->configured(); } + // Env var a user must set next ("" when configured). + std::string missing_env() const; + + // Throws ApiError::invalid when the row does not claim the call's feature. + void require(Call call) const; + bool supports(Call call) const; + + private: + Provider(const ProviderRow* row, std::string base_url, std::shared_ptr client) + : row_(row), base_url_(std::move(base_url)), client_(std::move(client)) {} + + static llm::ApiClientConfig config_for(const ProviderRow& row, const std::string& base_url, + const std::string& missing_env); + + const ProviderRow* row_; + std::string base_url_; + std::shared_ptr client_; +}; + +// Base for the per-endpoint services: holds the Provider, checks features. +class ProviderService : public llm::Service { + public: + explicit ProviderService(Provider provider) + : llm::Service(provider.client()), provider_(std::move(provider)) {} + const Provider& provider() const { return provider_; } + + protected: + Provider provider_; +}; + +} // namespace molo::compat diff --git a/src/vendors/compat/registry.cpp b/src/vendors/compat/registry.cpp new file mode 100644 index 0000000..bb61fbd --- /dev/null +++ b/src/vendors/compat/registry.cpp @@ -0,0 +1,239 @@ +#include "vendors/compat/registry.h" + +#include "llm/error.h" + +namespace molo::compat { + +namespace { + +constexpr unsigned C = Feature::kChat; +constexpr unsigned E = Feature::kEmbeddings; +constexpr unsigned M = Feature::kModels; +constexpr unsigned I = Feature::kImages; +constexpr unsigned T = Feature::kTranscribe; +constexpr unsigned S = Feature::kSpeech; +constexpr Evidence DP = Evidence::kDocsAndProbe; +constexpr Evidence PO = Evidence::kProbeOnly; +constexpr Evidence DO = Evidence::kDocsOnly; + +// Probe date for every "Probe:" note: 2026-10-25, unauthenticated, no key sent. +// Fields: id, name, base_url, key_env, base_env, extra_env, models_path, +// features, quirks, evidence, local, doc_url, note. +const ProviderRow kRows[] = { + {"groq", "Groq", "https://api.groq.com/openai/v1", "GROQ_API_KEY", "", "", "/models", + C | M | T, Quirk::kRejectLogprobs | Quirk::kRejectMessageName | Quirk::kSingleChoice, DP, false, + "https://console.groq.com/docs/openai", + "Docs: logprobs, logit_bias, top_logprobs and messages[].name answer 400; n must be 1; temperature 0 " + "becomes 1e-8; transcription formats vtt and srt unsupported. Probe: 401 invalid_api_key on /models and chat."}, + {"together", "Together AI", "https://api.together.ai/v1", "TOGETHER_API_KEY", "", "", "/models", + C | E | M | I | T | S, Quirk::kNone, DP, false, + "https://docs.together.ai/docs/openai-api-compatibility", + "Docs: logit_bias unsupported on most models; service_tier, store, metadata and prediction are accepted " + "and ignored. Probe: 401 on /models and chat; legacy host api.together.xyz answers 401 too."}, + {"fireworks", "Fireworks AI", "https://api.fireworks.ai/inference/v1", "FIREWORKS_API_KEY", "", "", "/models", + C | E | M, Quirk::kNone, DP, false, + "https://docs.fireworks.ai/tools-sdks/openai-compatibility", + "Probe: /models 401 with a docs link; unknown model answers 404 NOT_FOUND; empty embeddings body answers 500."}, + {"cerebras", "Cerebras", "https://api.cerebras.ai/v1", "CEREBRAS_API_KEY", "", "", "/models", + C | M, Quirk::kNoToolsWithFormat, DP, false, + "https://inference-docs.cerebras.ai/resources/openai", + "Docs: tools and response_format must not be combined (gpt-oss-120b rejects it); system and developer roles " + "both act at developer level. Probe: /models 403 {detail}, chat 401 wrong_api_key, embeddings path 404."}, + {"sambanova", "SambaNova", "https://api.sambanova.ai/v1", "SAMBANOVA_API_KEY", "", "", "/models", + C | E | M, Quirk::kNone, PO, false, + "https://docs.sambanova.ai/docs/en/features/openai-compatibility", + "Docs page answers 403 to a plain fetch. Probe: /models 200 public; chat 401 (also accepts x-api-key); " + "embeddings answers 400 'must provide a model' before auth."}, + {"deepinfra", "DeepInfra", "https://api.deepinfra.com/v1/openai", "DEEPINFRA_API_KEY", "", "", "/models", + C | E | M | I | T, Quirk::kNone, DP, false, + "https://docs.deepinfra.com/chat/overview", + "Probe: /models 200 public; chat 401 missing API key; images and audio paths answer 401 with a {detail} body."}, + {"novita", "Novita AI", "https://api.novita.ai/openai/v1", "NOVITA_API_KEY", "", "", "/models", + C | M, Quirk::kNone, DP, false, + "https://novita.ai/docs/guides/llm-api", + "Probe: /models 200 public; bad key answers 403 INVALID_API_KEY (not 401); legacy base /v3/openai answers the same."}, + {"nvidia", "NVIDIA NIM (build.nvidia.com)", "https://integrate.api.nvidia.com/v1", "NVIDIA_API_KEY", "", "", "/models", + C | E | M, Quirk::kNone, DP, false, + "https://docs.api.nvidia.com/nim/reference/llm-apis", + "Probe: /models 200 public; embeddings answers 400 'model field is required'; retired models answer 410 Gone."}, + {"cloudflare", "Cloudflare Workers AI", "https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1", + "CLOUDFLARE_API_TOKEN", "", "CLOUDFLARE_ACCOUNT_ID", "", C | E, Quirk::kNone, DO, false, + "https://developers.cloudflare.com/workers-ai/configuration/open-ai-compatibility/", + "Docs list /chat/completions, /embeddings and /responses under the account path. Probe: GET /models answers " + "405 (no list endpoint here); not probeable further without an account id."}, + {"moonshot", "Moonshot (Kimi)", "https://api.moonshot.ai/v1", "MOONSHOT_API_KEY", "", "", "/models", + C | M, Quirk::kNone, DP, false, + "https://platform.kimi.ai/docs/api/chat", + "Probe: 401 incorrect_api_key_error on /models and chat; the China host api.moonshot.cn answers the same " + "(separate keys)."}, + {"dashscope", "Alibaba DashScope (Qwen, intl)", "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", + "DASHSCOPE_API_KEY", "", "", "", C | E, Quirk::kNone, DP, false, + "https://www.alibabacloud.com/help/en/model-studio/compatibility-of-openai-with-dashscope", + "Probe: 401 on /models and chat; embeddings answers 400 'you must provide a model parameter'; China host " + "dashscope.aliyuncs.com/compatible-mode/v1 answers 401 too."}, + {"zhipu", "Zhipu (GLM, China)", "https://open.bigmodel.cn/api/paas/v4", "ZHIPU_API_KEY", "", "", "", + C | E | I, Quirk::kNone, DP, false, + "https://docs.bigmodel.cn/cn/guide/develop/openai/introduction", + "Docs name the base URL; the path is /api/paas/v4, not /v1. Probe: 401 code 1001 (missing Authorization) " + "on chat, embeddings and images."}, + {"zai", "Z.ai (GLM, international)", "https://api.z.ai/api/paas/v4", "ZAI_API_KEY", "", "", "", + C | E | I, Quirk::kNone, DP, false, + "https://docs.z.ai/api-reference/introduction", + "International host of the Zhipu API, same /api/paas/v4 path. Probe: 401 code 1001."}, + {"minimax", "MiniMax", "https://api.minimax.io/v1", "MINIMAX_API_KEY", "", "", "", + C, Quirk::kNone, PO, false, + "https://platform.minimax.io/docs/api-reference/text-openai-api", + "Docs text did not name the URL. Probe: 401 authorized_error on /models and chat; embeddings answers HTTP 200 " + "with a base_resp status_code 1004 envelope (not OpenAI shaped); China host api.minimaxi.com answers 401."}, + {"qianfan", "Baidu Qianfan", "https://qianfan.baidubce.com/v2", "QIANFAN_API_KEY", "", "", "", + C | E, Quirk::kNone, PO, false, + "https://cloud.baidu.com/doc/qianfan-api/s/3m9b5lqft", + "Docs page is rendered client side. Probe: chat and embeddings 401 invalid_iam_token; GET /models 403 " + "AccessDenied (different envelope)."}, + {"ark", "Volcengine Ark (Doubao)", "https://ark.cn-beijing.volces.com/api/v3", "ARK_API_KEY", "", "", "", + C, Quirk::kNone, PO, false, + "https://www.volcengine.com/docs/82379/1298454", + "Docs page is rendered client side. Probe: 401 AuthenticationError on /models, chat and embeddings; path is /api/v3."}, + {"siliconflow", "SiliconFlow", "https://api.siliconflow.com/v1", "SILICONFLOW_API_KEY", "", "", "/models", + C | E | M, Quirk::kNone, DP, false, + "https://docs.siliconflow.com/en/api-reference/chat-completions/chat-completions", + "Probe: 401 code 30014 with a {code,data,message} envelope on every path; China host api.siliconflow.cn identical."}, + {"nebius", "Nebius Token Factory", "https://api.studio.nebius.com/v1", "NEBIUS_API_KEY", "", "", "/models", + C | E | M, Quirk::kNone, DP, false, + "https://docs.tokenfactory.nebius.com/api-reference/introduction", + "Probe: 401 unauthenticated on /models, chat and embeddings; images/generations 404 on this host; " + "api.tokenfactory.nebius.com/v1 answers the same."}, + {"scaleway", "Scaleway Generative APIs", "https://api.scaleway.ai/v1", "SCALEWAY_API_KEY", "", "", "/models", + C | E | M, Quirk::kNone, DP, false, + "https://www.scaleway.com/en/docs/generative-apis/api-cli/using-chat-api/", + "Probe: 401 UNAUTHORIZED (missing 'Authorization' header) on /models, chat and embeddings."}, + {"upstage", "Upstage (Solar)", "https://api.upstage.ai/v1", "UPSTAGE_API_KEY", "", "", "", + C | E, Quirk::kNone, DP, false, + "https://console.upstage.ai/docs/getting-started", + "Probe: 401 invalid API key on /models, chat and embeddings."}, + {"inception", "Inception (Mercury)", "https://api.inceptionlabs.ai/v1", "INCEPTION_API_KEY", "", "", "/models", + C | M, Quirk::kNone, DP, false, + "https://docs.inceptionlabs.ai/get-started/get-started", + "Probe: /models 200 public; chat answers 400 listing the valid model names before auth."}, + {"stepfun", "StepFun", "https://api.stepfun.com/v1", "STEPFUN_API_KEY", "", "", "/models", + C | M, Quirk::kNone, DP, false, + "https://platform.stepfun.com/docs/api-reference/chat/chat-completion-create", + "Probe: 401 invalid_api_key on /models and chat."}, + {"hunyuan", "Tencent Hunyuan", "https://api.hunyuan.cloud.tencent.com/v1", "HUNYUAN_API_KEY", "", "", "", + C, Quirk::kNone, PO, false, + "https://cloud.tencent.com/document/product/1729/111007", + "Docs page did not name the URL in readable text. Probe: 401 'Incorrect API key provided' on /models and chat."}, + {"venice", "Venice AI", "https://api.venice.ai/api/v1", "VENICE_API_KEY", "", "", "/models", + C | M, Quirk::kNone, DP, false, + "https://docs.venice.ai/api-reference/endpoint/chat/completions", + "Probe: /models 200 public; unauthenticated chat answers 402 with an x402 payment envelope (not 401)."}, + {"vercel", "Vercel AI Gateway", "https://ai-gateway.vercel.sh/v1", "AI_GATEWAY_API_KEY", "", "", "/models", + C | M, Quirk::kNone, DP, false, + "https://vercel.com/docs/ai-gateway/openai-compat", + "Probe: /models 200 public; chat 401 authentication_error 'Missing Authorization header'."}, + {"friendli", "FriendliAI Serverless", "https://api.friendli.ai/serverless/v1", "FRIENDLI_TOKEN", "", "", "/models", + C | M, Quirk::kNone, DP, false, + "https://friendli.ai/docs/openapi/serverless/chat-completions", + "Probe: /models 200 public; chat 401 {detail}."}, + {"gemini", "Google Gemini (OpenAI compatibility)", "https://generativelanguage.googleapis.com/v1beta/openai", + "GEMINI_API_KEY", "", "", "", C, Quirk::kNone, DP, false, + "https://ai.google.dev/gemini-api/docs/openai", + "Probe: chat answers 400 'Missing or invalid Authorization header'; unauthenticated GET /models and " + "embeddings answer 404, so only chat is claimed."}, + {"baseten", "Baseten Model APIs", "https://inference.baseten.co/v1", "BASETEN_API_KEY", "", "", "", + C, Quirk::kNone, DP, false, + "https://docs.baseten.co/development/model-apis/overview", + "Probe: /models 401 'No Authorization header provided'; chat 403 'please check the api-key you provided'."}, + {"featherless", "Featherless AI", "https://api.featherless.ai/v1", "FEATHERLESS_API_KEY", "", "", "/models", + C | M, Quirk::kNone, PO, false, + "https://featherless.ai/docs/overview", + "Docs text did not name the URL. Probe: /models 200 public; chat 401 'must be signed in'."}, + {"ovh", "OVHcloud AI Endpoints", "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1", "OVH_AI_ENDPOINTS_ACCESS_TOKEN", + "", "", "/models", C | M, Quirk::kNone, PO, false, + "https://help.ovhcloud.com/csm/en-gb-public-cloud-ai-endpoints-getting-started", + "Docs text did not name the URL. Probe: /models 200 public; chat on an unknown model answers 404 model_not_found."}, + {"parasail", "Parasail", "https://api.parasail.io/v1", "PARASAIL_API_KEY", "", "", "/models", + C | M, Quirk::kNone, PO, false, + "https://docs.parasail.io/parasail-docs/serverless/serverless", + "Docs URL answered 404 at probe time. Probe: 401 'No bearer token provided' on /models and chat."}, + {"crusoe", "Crusoe Managed Inference", "https://managed-inference-api-proxy.crusoecloud.com/v1", "CRUSOE_API_KEY", + "", "", "/models", C | M, Quirk::kNone, PO, false, + "https://docs.crusoe.ai/managed-inference", + "Docs URL answered 404 at probe time. Probe: 401 {errors:[Authentication failed]} on /models and chat."}, + {"xiaomi", "Xiaomi MiMo", "https://api.xiaomimimo.com/v1", "XIAOMI_API_KEY", "", "", "", + C, Quirk::kNone, PO, false, + "https://platform.xiaomimimo.com/", + "Docs text did not name the URL. Probe: 401 'Invalid API Key' on /models and chat."}, + {"chutes", "Chutes", "https://llm.chutes.ai/v1", "CHUTES_API_KEY", "", "", "/models", + C | M, Quirk::kNone, PO, false, + "https://docs.chutes.ai/", + "Docs text did not name the URL. Probe: /models 200 public; chat answered an nginx 429 HTML page."}, + // Self-hosted servers: the key is optional, the row is configured when its base_env is set. + {"ollama", "Ollama", "http://localhost:11434/v1", "OLLAMA_API_KEY", "OLLAMA_BASE_URL", "", "/models", + C | E | M, Quirk::kNone, DO, true, + "https://raw.githubusercontent.com/ollama/ollama/main/docs/api/openai-compatibility.mdx", + "Docs name http://localhost:11434/v1. The key is required by OpenAI clients but ignored by the server; " + "the module sends the placeholder 'local' when none is set."}, + {"lmstudio", "LM Studio", "http://localhost:1234/v1", "LMSTUDIO_API_KEY", "LMSTUDIO_BASE_URL", "", "/models", + C | E | M, Quirk::kNone, DO, true, + "https://lmstudio.ai/docs/developer/openai-compat", + "Docs name http://localhost:1234/v1. Key ignored unless configured in the app."}, + {"vllm", "vLLM server", "http://localhost:8000/v1", "VLLM_API_KEY", "VLLM_BASE_URL", "", "/models", + C | E | M | T, Quirk::kNone, DO, true, + "https://docs.vllm.ai/en/latest/serving/online_serving/", + "Docs list /v1/chat/completions, /v1/embeddings and /v1/audio/transcriptions; key only when the server " + "was started with --api-key."}, + {"llamacpp", "llama.cpp server", "http://localhost:8080/v1", "LLAMACPP_API_KEY", "LLAMACPP_BASE_URL", "", "/models", + C | E | M, Quirk::kNone, DO, true, + "https://raw.githubusercontent.com/ggml-org/llama.cpp/master/tools/server/README.md", + "README lists /v1/chat/completions; key only when started with --api-key. Same default port as LocalAI."}, + {"localai", "LocalAI", "http://localhost:8080/v1", "LOCALAI_API_KEY", "LOCALAI_BASE_URL", "", "/models", + C | E | M, Quirk::kNone, DO, true, + "https://localai.io/docs/overview/", + "Docs name localhost:8080; endpoint list beyond chat, embeddings and models was not read."}, +}; + +constexpr std::size_t kRowCount = sizeof(kRows) / sizeof(kRows[0]); + +} // namespace + +std::size_t Registry::count() { return kRowCount; } + +const ProviderRow& Registry::at(std::size_t index) { + if (index >= kRowCount) throw llm::ApiError::invalid("compat", "registry index out of range"); + return kRows[index]; +} + +const ProviderRow* Registry::find(const std::string& id) { + for (const ProviderRow& row : kRows) + if (id == row.id) return &row; + return nullptr; +} + +const ProviderRow& Registry::get(const std::string& id) { + const ProviderRow* row = find(id); + if (!row) throw llm::ApiError::invalid("compat", "unknown provider '" + id.substr(0, 64) + "'"); + return *row; +} + +const char* Registry::evidence_name(Evidence e) { + switch (e) { + case Evidence::kDocsAndProbe: return "docs+probe"; + case Evidence::kProbeOnly: return "probe-only"; + case Evidence::kDocsOnly: return "docs-only"; + } + return ""; +} + +std::string Registry::feature_names(const ProviderRow& row) { + struct Name { unsigned bit; const char* name; }; + static const Name names[] = {{Feature::kChat, "chat"}, {Feature::kEmbeddings, "embeddings"}, + {Feature::kModels, "models"}, {Feature::kImages, "images"}, + {Feature::kTranscribe, "transcribe"}, {Feature::kSpeech, "speech"}}; + std::string out; + for (const Name& n : names) + if (row.features & n.bit) out += (out.empty() ? "" : ",") + std::string(n.name); + return out; +} + +} // namespace molo::compat diff --git a/src/vendors/compat/registry.h b/src/vendors/compat/registry.h new file mode 100644 index 0000000..489e6ad --- /dev/null +++ b/src/vendors/compat/registry.h @@ -0,0 +1,72 @@ +#pragma once + +// The ONE provider table of this module. Adding a provider is one row in +// registry.cpp (plus its line in SPEC.md section 4, which tools/check_spec.py +// enforces). Everything else (clients, routes, demo, tests) iterates this +// table. + +#include +#include + +namespace molo::compat { + +// Endpoints a row claims. Claims come from the provider's own docs or a +// probe; SPEC.md says which. A call outside the mask is refused before any +// network traffic. +struct Feature { + static constexpr unsigned kChat = 1u << 0; + static constexpr unsigned kEmbeddings = 1u << 1; + static constexpr unsigned kModels = 1u << 2; + static constexpr unsigned kImages = 1u << 3; + static constexpr unsigned kTranscribe = 1u << 4; + static constexpr unsigned kSpeech = 1u << 5; +}; + +// Documented behavior that differs from OpenAI and that the module enforces +// loudly (reject) instead of letting the provider answer 400. +struct Quirk { + static constexpr unsigned kNone = 0; + static constexpr unsigned kRejectMessageName = 1u << 0; // messages[].name answers 400 + static constexpr unsigned kRejectLogprobs = 1u << 1; // logprobs, top_logprobs, logit_bias answer 400 + static constexpr unsigned kSingleChoice = 1u << 2; // n must be 1 + static constexpr unsigned kNoToolsWithFormat = 1u << 3; // tools + response_format not combinable +}; + +// How the row was verified (see SPEC.md section 5). +enum class Evidence { + kDocsAndProbe, // docs text names the base URL; unauthenticated probe answered like an API + kProbeOnly, // docs text did not name it (or not readable); probe answered like an API + kDocsOnly, // docs name it; not probeable (local server, or needs an account id) +}; + +struct ProviderRow { + const char* id; // slug, [a-z0-9]+, used in routes and ids + const char* name; // display name + const char* base_url; // no trailing slash; "{account_id}" is replaced from extra_env + const char* key_env; // credential env var (via llm::Config); local rows: optional key + const char* base_env; // env var overriding base_url; for local rows also the "configured" switch + const char* extra_env; // second required env var (account id) or "" + const char* models_path; // "" when the row has no list endpoint + unsigned features; // Feature mask + unsigned quirks; // Quirk mask + Evidence evidence; + bool local; // self-hosted server, no vendor credential + const char* doc_url; // documentation page cited for the base URL + const char* note; // documented quirks and probe results, one line +}; + +class Registry { + public: + static std::size_t count(); + static const ProviderRow& at(std::size_t index); + // nullptr when unknown. + static const ProviderRow* find(const std::string& id); + // Throws llm::ApiError::invalid for an unknown id. + static const ProviderRow& get(const std::string& id); + static bool has(const ProviderRow& row, unsigned feature) { return (row.features & feature) != 0; } + static const char* evidence_name(Evidence e); + // Comma list of the feature names a row claims ("chat,embeddings,models"). + static std::string feature_names(const ProviderRow& row); +}; + +} // namespace molo::compat diff --git a/src/vendors/compat/services.cpp b/src/vendors/compat/services.cpp new file mode 100644 index 0000000..64cf5ca --- /dev/null +++ b/src/vendors/compat/services.cpp @@ -0,0 +1,199 @@ +#include "vendors/compat/services.h" + +#include "clients/body/form.h" +#include "vendors/compat/caps.h" + +namespace molo::compat { + +using Json = clients::body::Json; + +namespace { + +[[noreturn]] void bad(const Provider& p, const std::string& why) { + throw llm::ApiError::invalid(p.client()->config().vendor, why); +} + +void check_model(const Provider& p, const std::string& model) { + if (model.empty()) bad(p, "model is required (ids differ per provider)"); + if (model.size() > CompatCaps::kMaxModelChars) bad(p, "model id too long"); +} + +std::string str_at(const Json& j, const char* key) { + const Json* v = j.is_object() ? j.find(key) : nullptr; + return v && v->is_string() ? v->as_string() : ""; +} + +int int_at(const Json& j, const char* key) { + const Json* v = j.is_object() ? j.find(key) : nullptr; + return v && v->is_number() ? static_cast(v->as_int()) : 0; +} + +} // namespace + +bool ModelList::contains(const std::string& id) const { + for (const ModelEntry& m : models) + if (m.id == id) return true; + return false; +} + +ModelList CompatModels::parse(const Json& doc) { + const Json* arr = &doc; + if (doc.is_object()) arr = doc.find("data"); + ModelList out; + if (!arr || !arr->is_array()) throw llm::ApiError("compat", 502, "BAD_RESPONSE", "model list has no data array"); + for (const Json& m : arr->as_array()) { + if (out.models.size() >= CompatCaps::kMaxModelsReturned) { + out.truncated = true; + break; + } + ModelEntry e; + e.id = str_at(m, "id"); + if (e.id.empty()) e.id = str_at(m, "name"); + if (e.id.empty()) continue; + e.owned_by = str_at(m, "owned_by"); + out.models.push_back(e); + } + return out; +} + +ModelList CompatModels::list() { + provider_.require(Call::kModels); + if (std::string(provider_.row().models_path).empty()) + bad(provider_, std::string(provider_.row().name) + " has no list endpoint under its base URL"); + return parse(client_->get(provider_.row().models_path)); +} + +Json CompatEmbeddings::to_body(const std::string& model, const std::vector& inputs, + int dimensions) const { + provider_.require(Call::kEmbeddings); + check_model(provider_, model); + if (inputs.empty()) bad(provider_, "at least one input is required"); + if (inputs.size() > CompatCaps::kMaxEmbeddingInputs) bad(provider_, "too many inputs"); + if (dimensions < 0 || dimensions > 65536) bad(provider_, "dimensions out of range"); + Json arr = Json::array(); + for (const std::string& s : inputs) { + if (s.empty()) bad(provider_, "input must not be empty"); + if (s.size() > CompatCaps::kMaxEmbeddingInputChars) bad(provider_, "input too long"); + arr.push(Json(s)); + } + Json body = Json::object(); + body.set("model", Json(model)); + body.set("input", arr); + if (dimensions > 0) body.set("dimensions", Json(dimensions)); + return body; +} + +EmbeddingResult CompatEmbeddings::parse(const Json& doc) { + EmbeddingResult out; + out.model = str_at(doc, "model"); + const Json* data = doc.is_object() ? doc.find("data") : nullptr; + if (!data || !data->is_array()) throw llm::ApiError("compat", 502, "BAD_RESPONSE", "embeddings response has no data"); + for (const Json& d : data->as_array()) { + Embedding e; + const Json* v = d.is_object() ? d.find("embedding") : nullptr; + if (!v || !v->is_array()) throw llm::ApiError("compat", 502, "BAD_RESPONSE", "embedding is not an array"); + for (const Json& x : v->as_array()) e.vector.push_back(x.is_number() ? x.as_number() : 0.0); + out.items.push_back(std::move(e)); + } + if (const Json* u = doc.find("usage"); u && u->is_object()) { + out.prompt_tokens = int_at(*u, "prompt_tokens"); + out.total_tokens = int_at(*u, "total_tokens"); + } + return out; +} + +EmbeddingResult CompatEmbeddings::embed(const std::string& model, const std::vector& inputs, + int dimensions) { + Json body = to_body(model, inputs, dimensions); + return parse(client_->post(Calls::spec_of(Call::kEmbeddings).path, body)); +} + +Json CompatImages::to_body(const ImageRequest& r) const { + provider_.require(Call::kImages); + check_model(provider_, r.model); + if (r.prompt.empty() || r.prompt.size() > CompatCaps::kMaxPromptChars) bad(provider_, "prompt must be 1..32768 chars"); + if (r.n < 1 || r.n > CompatCaps::kMaxImages) bad(provider_, "n must be 1..10"); + if (!r.response_format.empty() && r.response_format != "url" && r.response_format != "b64_json") + bad(provider_, "response_format must be url or b64_json"); + if (!r.size.empty()) { + std::size_t x = r.size.find('x'); + bool ok = x != std::string::npos && x > 0 && x + 1 < r.size.size() && r.size.size() <= 16; + for (std::size_t i = 0; ok && i < r.size.size(); ++i) + ok = i == x || (r.size[i] >= '0' && r.size[i] <= '9'); + if (!ok) bad(provider_, "size must look like 1024x1024"); + } + Json body = Json::object(); + body.set("model", Json(r.model)); + body.set("prompt", Json(r.prompt)); + body.set("n", Json(r.n)); + if (!r.size.empty()) body.set("size", Json(r.size)); + if (!r.response_format.empty()) body.set("response_format", Json(r.response_format)); + return body; +} + +std::vector CompatImages::parse(const Json& doc) { + const Json* data = doc.is_object() ? doc.find("data") : nullptr; + if (!data || !data->is_array()) throw llm::ApiError("compat", 502, "BAD_RESPONSE", "images response has no data"); + std::vector out; + for (const Json& d : data->as_array()) { + if (out.size() >= static_cast(CompatCaps::kMaxImages)) break; + ImageItem i; + i.url = str_at(d, "url"); + i.b64_json = str_at(d, "b64_json"); + i.revised_prompt = str_at(d, "revised_prompt"); + out.push_back(std::move(i)); + } + return out; +} + +std::vector CompatImages::generate(const ImageRequest& request) { + Json body = to_body(request); + return parse(client_->post(Calls::spec_of(Call::kImages).path, body)); +} + +std::string CompatAudio::transcribe(const TranscribeRequest& r) { + provider_.require(Call::kTranscribe); + check_model(provider_, r.model); + if (r.audio.empty()) bad(provider_, "audio is empty"); + if (r.audio.size() > CompatCaps::kMaxAudioBytes) bad(provider_, "audio over the byte cap"); + if (r.filename.empty() || r.filename.size() > CompatCaps::kMaxFilenameChars) bad(provider_, "filename must be 1..256 chars"); + if (r.filename.find_first_of("\"\r\n/\\") != std::string::npos) bad(provider_, "filename has forbidden characters"); + clients::body::MultipartBody form; + form.add_file("file", r.filename, r.mime, r.audio); + form.add_text("model", r.model); + if (!r.language.empty()) form.add_text("language", r.language); + if (!r.prompt.empty()) form.add_text("prompt", r.prompt); + form.add_text("response_format", "json"); + Json doc = client_->post_multipart(Calls::spec_of(Call::kTranscribe).path, form); + return str_at(doc, "text"); +} + +Json CompatAudio::speech_body(const SpeechRequest& r) const { + provider_.require(Call::kSpeech); + check_model(provider_, r.model); + if (r.input.empty() || r.input.size() > CompatCaps::kMaxSpeechChars) bad(provider_, "input must be 1..4096 chars"); + if (r.voice.empty()) bad(provider_, "voice is required (names differ per provider)"); + static const char* formats[] = {"mp3", "opus", "aac", "flac", "wav", "pcm"}; + if (!r.format.empty()) { + bool ok = false; + for (const char* f : formats) ok = ok || r.format == f; + if (!ok) bad(provider_, "format must be mp3, opus, aac, flac, wav or pcm"); + } + Json body = Json::object(); + body.set("model", Json(r.model)); + body.set("input", Json(r.input)); + body.set("voice", Json(r.voice)); + if (!r.format.empty()) body.set("response_format", Json(r.format)); + return body; +} + +std::string CompatAudio::speak(const SpeechRequest& r, std::string& content_type) { + Json body = speech_body(r); + std::string text = body.dump(); + llm::RawResponse res = client_->request_raw(clients::Method::kPost, Calls::spec_of(Call::kSpeech).path, &text, + "application/json"); + content_type = res.content_type; + return res.body; +} + +} // namespace molo::compat diff --git a/src/vendors/compat/services.h b/src/vendors/compat/services.h new file mode 100644 index 0000000..ec26166 --- /dev/null +++ b/src/vendors/compat/services.h @@ -0,0 +1,104 @@ +#pragma once + +// The non-chat endpoints of a registry row. Each service refuses a call the +// row does not claim (Provider::require) before any network traffic, and +// validates every input against CompatCaps. + +#include +#include +#include + +#include "clients/body/json.h" +#include "vendors/compat/provider.h" + +namespace molo::compat { + +struct ModelEntry { + std::string id; + std::string owned_by; +}; + +struct ModelList { + std::vector models; + bool truncated = false; // more entries than CompatCaps::kMaxModelsReturned + bool contains(const std::string& id) const; +}; + +// GET /models. Accepts {"data":[...]} and a bare array (some providers return one). +class CompatModels : public ProviderService { + public: + using ProviderService::ProviderService; + ModelList list(); + static ModelList parse(const clients::body::Json& doc); +}; + +struct Embedding { + std::vector vector; +}; + +struct EmbeddingResult { + std::string model; + std::vector items; + int prompt_tokens = 0; + int total_tokens = 0; +}; + +class CompatEmbeddings : public ProviderService { + public: + using ProviderService::ProviderService; + // dimensions: 0 = provider default. + EmbeddingResult embed(const std::string& model, const std::vector& inputs, int dimensions = 0); + clients::body::Json to_body(const std::string& model, const std::vector& inputs, + int dimensions) const; + static EmbeddingResult parse(const clients::body::Json& doc); +}; + +struct ImageItem { + std::string url; + std::string b64_json; + std::string revised_prompt; +}; + +struct ImageRequest { + std::string model; + std::string prompt; + int n = 1; + std::string size; // "1024x1024"; empty = provider default + std::string response_format; // "", "url" or "b64_json" +}; + +class CompatImages : public ProviderService { + public: + using ProviderService::ProviderService; + // Paid call: the caller decides (the demo keeps it a dry run). + std::vector generate(const ImageRequest& request); + clients::body::Json to_body(const ImageRequest& request) const; + static std::vector parse(const clients::body::Json& doc); +}; + +struct TranscribeRequest { + std::string model; + std::string filename; + std::string mime = "application/octet-stream"; + std::string audio; // raw bytes + std::string language; + std::string prompt; +}; + +struct SpeechRequest { + std::string model; + std::string input; + std::string voice; + std::string format; // "", mp3, opus, aac, flac, wav, pcm +}; + +class CompatAudio : public ProviderService { + public: + using ProviderService::ProviderService; + std::string transcribe(const TranscribeRequest& request); // returns the text + // Audio bytes; the content type is returned through `content_type`. + std::string speak(const SpeechRequest& request, std::string& content_type); + clients::body::Json speech_body(const SpeechRequest& request) const; +}; + +} // namespace molo::compat diff --git a/src/vendors/compat/tools/check_spec.py b/src/vendors/compat/tools/check_spec.py new file mode 100644 index 0000000..309bbda --- /dev/null +++ b/src/vendors/compat/tools/check_spec.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Check src/vendors/compat against OpenAI's OpenAPI document and its own SPEC. + +Every compat call is an OpenAI operation (providers differ only in base URL, +auth and quirks). Fails (exit 1) when + + call a row of calls.cpp has a wrong method, path, operation id, request + schema, request body field or response field (fields are resolved + through $ref, allOf, oneOf and anyOf); + coverage an operation of the spec in the families this module touches + (chat completions, embeddings, models, images, audio) is neither + in the table nor in EXCLUDED; + registry the provider ids / base URLs / credential names of registry.cpp and + the table of SPEC.md section 4 differ in either direction. + +The registry check needs no network: pass --local to run only it. +Python 3 stdlib only. + + python3 -I src/vendors/compat/tools/check_spec.py [--local] [path/to/openapi.json] +""" +import json +import os +import re +import sys +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.join(HERE, "..") +SPEC_SHA = "8243cc8fa6f04430fb1f1a6a2cb926fbeb4f7f26" # same pin as src/openai/core +URL = "https://raw.githubusercontent.com/openai/openai-openapi/%s/openapi.json" % SPEC_SHA + +FAMILIES = ("/chat/completions", "/embeddings", "/models", "/images", "/audio", "/completions", "/moderations") +EXCLUDED = { + "listChatCompletions": "stored completions are an OpenAI product feature, not portable", + "getChatCompletion": "stored completions", + "updateChatCompletion": "stored completions", + "deleteChatCompletion": "stored completions", + "getChatCompletionMessages": "stored completions", + "createCompletion": "legacy completions; not claimed by any registry row", + "createModeration": "no registry row documents a moderation endpoint", + "createImageEdit": "multipart image editing; no registry row documents it", + "createImageVariation": "multipart image variations; no registry row documents it", + "createTranslation": "audio translation; no registry row documents it", + "createVoiceConsent": "OpenAI custom voices only", + "listVoiceConsents": "OpenAI custom voices only", + "getVoiceConsent": "OpenAI custom voices only", + "updateVoiceConsent": "OpenAI custom voices only", + "deleteVoiceConsent": "OpenAI custom voices only", + "createVoice": "OpenAI custom voices only", + "deleteVoice": "OpenAI custom voices only", + "retrieveModel": "no registry row documents GET /models/{id}; models are listed", + "deleteModel": "OpenAI fine-tune deletion only", +} +STR = re.compile(r'"((?:[^"\\]|\\.)*)"') + + +def string_runs(block): + """String literals of a C++ block with adjacent literals concatenated.""" + out, last_end, cur = [], None, None + for m in STR.finditer(block): + if last_end is not None and block[last_end:m.start()].strip() == "": + cur += m.group(1) + else: + if cur is not None: + out.append(cur) + cur = m.group(1) + last_end = m.end() + if cur is not None: + out.append(cur) + return out + + +def calls_rows(): + text = open(os.path.join(ROOT, "calls.cpp")).read() + a = text.index("const CallSpec kCalls[]") + b = text.index("};", a) + rows = [] + for blk in re.findall(r"\{Call::k\w+.*?Feature::k\w+, \"\w+\"\}", text[a:b], re.S): + rows.append(string_runs(blk)) + return rows # method, path, op, request, fields, response, fields_read, feature_name + + +class Spec: + def __init__(self, doc): + self.doc = doc + + def schema(self, name): + return self.doc["components"]["schemas"][name] + + def deref(self, node): + while isinstance(node, dict) and "$ref" in node: + node = self.schema(node["$ref"].rsplit("/", 1)[1]) + return node + + def props(self, node, seen=None): + seen = seen or set() + out = set() + if isinstance(node, dict) and "$ref" in node: + ref = node["$ref"] + if ref in seen: + return out + seen.add(ref) + return self.props(self.deref(node), seen) + if not isinstance(node, dict): + return out + out |= set(node.get("properties", {})) + for key in ("allOf", "oneOf", "anyOf"): + for sub in node.get(key, []): + out |= self.props(sub, seen) + return out + + def schema_name_of(self, content, want=""): + ref = content.get("schema", {}) + if "$ref" in ref: + return ref["$ref"].rsplit("/", 1)[1] + # a union (anyOf / oneOf): the wanted name may be one member + for key in ("anyOf", "oneOf"): + for sub in ref.get(key, []): + if "$ref" in sub and sub["$ref"].rsplit("/", 1)[1] == want: + return want + return "" + + +def check_calls(spec, bad): + rows = calls_rows() + declared = set() + for method, path, op, req, fields, resp, read, _feat in rows: + node = spec.doc["paths"].get(path, {}).get(method.lower()) + if not node: + bad.append("%s %s: not in the spec" % (method, path)) + continue + declared.add(op) + if node["operationId"] != op: + bad.append("%s %s: operationId %s != %s" % (method, path, node["operationId"], op)) + body = node.get("requestBody", {}).get("content", {}) + if req: + content = body.get("application/json") or body.get("multipart/form-data") or {} + if spec.schema_name_of(content) != req: + bad.append("%s %s: request schema %s != %s" % (method, path, spec.schema_name_of(content), req)) + known = spec.props(content.get("schema", {})) + for f in filter(None, fields.split(",")): + if f not in known: + bad.append("%s %s: body field %s not in %s" % (method, path, f, req)) + if resp: + ok = node["responses"]["200"]["content"]["application/json"] + if spec.schema_name_of(ok, resp) != resp: + bad.append("%s %s: response schema %s != %s" % (method, path, spec.schema_name_of(ok), resp)) + known = spec.props(ok.get("schema", {})) + if not known: + known = spec.props({"$ref": "#/components/schemas/" + resp}) + for f in filter(None, read.split(",")): + if f not in known: + bad.append("%s %s: response field %s not in %s" % (method, path, f, resp)) + for path, item in spec.doc["paths"].items(): + if not path.startswith(FAMILIES): + continue + for m, node in item.items(): + if m not in ("get", "post", "delete"): + continue + op = node["operationId"] + if op not in declared and op not in EXCLUDED: + bad.append("spec operation %s %s (%s) is neither in the table nor excluded" % (m.upper(), path, op)) + for op in EXCLUDED: + if op in declared: + bad.append("%s is both declared and excluded" % op) + return len(rows) + + +def registry_rows(): + text = open(os.path.join(ROOT, "registry.cpp")).read() + a = text.index("const ProviderRow kRows[]") + b = text.index("constexpr std::size_t kRowCount") + out = [] + for blk in re.findall(r'\n \{"[a-z0-9]+",.*?(?=\n \{"|\n //|\n\};)', text[a:b], re.S): + s = string_runs(blk) + out.append({"id": s[0], "name": s[1], "base": s[2], "key": s[3], "base_env": s[4], "extra_env": s[5], + "models": s[6], "doc": s[7]}) + return out + + +def spec_md_rows(): + text = open(os.path.join(ROOT, "SPEC.md")).read() + a = text.index("## 4. Provider table") + b = text.index("\n## ", a + 5) + rows = {} + for line in text[a:b].splitlines(): + cells = [c.strip().strip("`") for c in line.strip().strip("|").split("|")] + if line.startswith("|") and len(cells) >= 5 and re.fullmatch(r"[a-z0-9]+", cells[0]) and cells[0] != "id": + rows[cells[0]] = cells + return rows + + +def check_registry(bad): + reg = registry_rows() + md = spec_md_rows() + ids = [r["id"] for r in reg] + if len(ids) != len(set(ids)): + bad.append("registry.cpp has duplicate ids") + for r in reg: + cells = md.get(r["id"]) + if not cells: + bad.append("provider %s is in registry.cpp but not in SPEC.md section 4" % r["id"]) + continue + line = " ".join(cells) + for what in ("base", "key", "doc"): + if r[what] not in line: + bad.append("provider %s: %s '%s' differs between registry.cpp and SPEC.md" % (r["id"], what, r[what])) + for i in md: + if i not in ids: + bad.append("provider %s is in SPEC.md section 4 but not in registry.cpp" % i) + return len(reg) + + +def main(): + args = [a for a in sys.argv[1:] if not a.startswith("--")] + local = "--local" in sys.argv + bad = [] + n_reg = check_registry(bad) + n_calls = 0 + if not local: + if args: + doc = json.load(open(args[0])) + else: + req = urllib.request.Request(URL, headers={"User-Agent": "molo-compat-spec-check/1"}) + with urllib.request.urlopen(req, timeout=120) as r: + doc = json.loads(r.read().decode("utf-8")) + n_calls = check_calls(Spec(doc), bad) + print("openapi %s (%s)" % (doc["info"]["version"], SPEC_SHA[:12])) + print("registry rows: %d, call rows checked: %d" % (n_reg, n_calls)) + if bad: + print("FAIL") + for b in bad: + print(" " + b) + sys.exit(1) + print("OK") + + +if __name__ == "__main__": + main() diff --git a/tests/example/ai/compat/page_test.cpp b/tests/example/ai/compat/page_test.cpp new file mode 100644 index 0000000..44dedb1 --- /dev/null +++ b/tests/example/ai/compat/page_test.cpp @@ -0,0 +1,50 @@ +// Routes: GET /ai/compat (page), /status, /providers, /calls (no credential needed) +#include + +#include "support/Assert.hpp" +#include "vendors/compat/registry.h" + +using namespace molo::example::test; +namespace cp = molo::compat; + +int main() { + setenv("MOLO_ENV_FILES", "off", 1); + for (std::size_t i = 0; i < cp::Registry::count(); ++i) { + const cp::ProviderRow& r = cp::Registry::at(i); + unsetenv(r.key_env); + if (r.base_env[0]) unsetenv(r.base_env); + if (r.extra_env[0]) unsetenv(r.extra_env); + } + Fixture f; + auto page = f.client->get("/ai/compat"); + check(page.status == 200, "page 200"); + check(contains(page.headers.get("Content-Type"), "text/html"), "html"); + check(contains(page.body, "OpenAI-compatible providers") && contains(page.body, "viewport"), "title and mobile viewport"); + check(contains(page.body, "/embedded/ai_compat.js"), "script wired"); + check(f.client->post("/ai/compat", "{}", "application/json").status == 405, "post page is 405"); + + auto st = f.client->get("/ai/compat/status"); + check(st.status == 200 && contains(st.body, "\"configured\":false"), "status without a key"); + check(contains(st.body, "GROQ_API_KEY") && contains(st.body, "CLOUDFLARE_API_TOKEN"), "status names credential envs"); + check(contains(st.body, "\"configured_providers\":0"), "no provider configured"); + check(f.client->post("/ai/compat/status", "{}", "application/json").status == 405, "post status is 405"); + + auto prov = f.client->get("/ai/compat/providers"); + check(prov.status == 200, "providers 200"); + for (std::size_t i = 0; i < cp::Registry::count(); ++i) { + const cp::ProviderRow& r = cp::Registry::at(i); + check(contains(prov.body, std::string("\"id\":\"") + r.id + "\""), std::string("providers lists ") + r.id); + } + check(contains(prov.body, "https://api.groq.com/openai/v1") && contains(prov.body, "docs+probe") && + contains(prov.body, "needs GROQ_API_KEY") == false && contains(prov.body, "\"missing_env\":\"GROQ_API_KEY\""), + "provider rows carry base url, evidence and missing env"); + check(contains(prov.body, "\"missing_env\":\"OLLAMA_BASE_URL\""), "local row names its switch"); + check(contains(prov.body, "\"missing_env\":\"CLOUDFLARE_ACCOUNT_ID\"") || contains(prov.body, "\"missing_env\":\"CLOUDFLARE_API_TOKEN\""), + "cloudflare names a missing env"); + + auto calls = f.client->get("/ai/compat/calls"); + check(calls.status == 200 && contains(calls.body, "/chat/completions") && contains(calls.body, "createEmbedding") && + contains(calls.body, "/audio/speech"), + "call table listed"); + return finish("ai/compat/page"); +} diff --git a/tests/example/ai/compat/routes_test.cpp b/tests/example/ai/compat/routes_test.cpp new file mode 100644 index 0000000..5efdd00 --- /dev/null +++ b/tests/example/ai/compat/routes_test.cpp @@ -0,0 +1,84 @@ +// Routes: POST /ai/compat/* (405, 422, dry runs, 503 without a credential) +#include + +#include "support/Assert.hpp" +#include "vendors/compat/registry.h" + +using namespace molo::example::test; +namespace cp = molo::compat; + +int main() { + setenv("MOLO_ENV_FILES", "off", 1); + for (std::size_t i = 0; i < cp::Registry::count(); ++i) { + const cp::ProviderRow& r = cp::Registry::at(i); + unsetenv(r.key_env); + if (r.base_env[0]) unsetenv(r.base_env); + if (r.extra_env[0]) unsetenv(r.extra_env); + } + Fixture f; + auto post = [&](const std::string& route, const std::string& body) { + return f.client->post("/ai/compat/" + route, body, "application/json"); + }; + const char* all[] = {"chat", "models", "embeddings", "images", "transcribe", "speech"}; + for (const char* r : all) { + std::string route = std::string("/ai/compat/") + r; + check(f.client->get(route).status == 405, route + " get is 405"); + check(post(r, "not json").status == 422, route + " bad json is 422"); + check(post(r, "[1]").status == 422, route + " non-object is 422"); + check(post(r, "{}").status == 422, route + " without provider is 422"); + check(post(r, "{\"provider\":\"nope\"}").status == 422, route + " unknown provider is 422"); + check(post(r, "{\"provider\":5}").status == 422, route + " non-string provider is 422"); + } + + // 422: input rules run before anything else + check(post("chat", "{\"provider\":\"groq\"}").status == 422, "chat needs prompt"); + check(post("chat", "{\"provider\":\"groq\",\"prompt\":\"hi\"}").status == 422, "chat needs a model"); + check(post("chat", "{\"provider\":\"groq\",\"prompt\":\"hi\",\"model\":\"m\",\"max_tokens\":-1}").status == 422, "max_tokens -1"); + check(post("chat", "{\"provider\":\"groq\",\"prompt\":\"hi\",\"model\":\"m\",\"extra\":{\"logprobs\":true}}").status == 422, + "groq quirk logprobs is 422"); + check(post("chat", "{\"provider\":\"groq\",\"prompt\":\"hi\",\"model\":\"m\",\"extra\":{\"n\":2}}").status == 422, "groq quirk n is 422"); + check(post("chat", "{\"provider\":\"groq\",\"prompt\":\"hi\",\"model\":\"m\",\"temperature\":3}").status == 422, "temperature 3"); + check(post("models", "{\"provider\":\"dashscope\"}").status == 422, "row without a list endpoint is 422"); + check(post("embeddings", "{\"provider\":\"cerebras\",\"model\":\"m\",\"input\":\"x\"}").status == 422, "cerebras has no embeddings"); + check(post("embeddings", "{\"provider\":\"nvidia\",\"model\":\"m\"}").status == 422, "embeddings need input"); + check(post("embeddings", "{\"provider\":\"nvidia\",\"model\":\"m\",\"input\":[1]}").status == 422, "input must be strings"); + check(post("images", "{\"provider\":\"together\",\"model\":\"m\",\"prompt\":\"x\",\"n\":11}").status == 422, "n over 10"); + check(post("images", "{\"provider\":\"groq\",\"model\":\"m\",\"prompt\":\"x\"}").status == 422, "groq has no images"); + check(post("transcribe", "{\"provider\":\"groq\",\"model\":\"m\",\"filename\":\"a.wav\",\"data_base64\":\"@@\"}").status == 422, "bad base64"); + check(post("transcribe", "{\"provider\":\"cerebras\",\"model\":\"m\",\"filename\":\"a.wav\",\"data_base64\":\"aGk=\"}").status == 422, + "cerebras has no transcription"); + check(post("speech", "{\"provider\":\"together\",\"model\":\"m\",\"input\":\"x\"}").status == 422, "speech needs a voice"); + check(post("speech", "{\"provider\":\"together\",\"model\":\"m\",\"input\":\"x\",\"voice\":\"v\",\"format\":\"ogg\"}").status == 422, "bad format"); + + // dry runs: paid routes show the exact request and the URL + auto dry = post("chat", "{\"provider\":\"groq\",\"prompt\":\"hi\",\"model\":\"llama\",\"max_tokens\":8}"); + check(dry.status == 200 && contains(dry.body, "\"dry_run\":true") && contains(dry.body, "\"model\":\"llama\"") && + contains(dry.body, "https://api.groq.com/openai/v1/chat/completions"), + "chat is a dry run without confirm"); + check(contains(post("embeddings", "{\"provider\":\"nvidia\",\"model\":\"m\",\"input\":\"x\"}").body, "/embeddings"), "embeddings dry run"); + check(contains(post("images", "{\"provider\":\"together\",\"model\":\"m\",\"prompt\":\"x\"}").body, "/images/generations"), "images dry run"); + check(contains(post("transcribe", "{\"provider\":\"groq\",\"model\":\"m\",\"filename\":\"a.wav\",\"data_base64\":\"aGk=\"}").body, + "\"bytes\":2"), + "transcribe dry run"); + check(contains(post("speech", "{\"provider\":\"together\",\"model\":\"m\",\"input\":\"x\",\"voice\":\"v\"}").body, "/audio/speech"), + "speech dry run"); + check(contains(post("chat", "{\"provider\":\"ollama\",\"prompt\":\"hi\",\"model\":\"m\"}").body, "http://localhost:11434/v1/chat/completions"), + "local provider dry run shows its default URL"); + + // 503: the free route and confirmed routes need the provider's credential + auto models = post("models", "{\"provider\":\"groq\"}"); + check(models.status == 503 && contains(models.body, "compat_not_configured") && contains(models.body, "GROQ_API_KEY"), + "models without key is 503 and names the env"); + check(post("chat", "{\"provider\":\"groq\",\"prompt\":\"hi\",\"model\":\"m\",\"confirm\":true}").status == 503, "confirmed chat without key is 503"); + check(post("embeddings", "{\"provider\":\"nvidia\",\"model\":\"m\",\"input\":\"x\",\"confirm\":true}").status == 503, "confirmed embeddings without key is 503"); + check(post("images", "{\"provider\":\"together\",\"model\":\"m\",\"prompt\":\"x\",\"confirm\":true}").status == 503, "confirmed images without key is 503"); + check(post("transcribe", "{\"provider\":\"groq\",\"model\":\"m\",\"filename\":\"a.wav\",\"data_base64\":\"aGk=\",\"confirm\":true}").status == 503, + "confirmed transcribe without key is 503"); + check(post("speech", "{\"provider\":\"together\",\"model\":\"m\",\"input\":\"x\",\"voice\":\"v\",\"confirm\":true}").status == 503, + "confirmed speech without key is 503"); + auto local = post("models", "{\"provider\":\"ollama\"}"); + check(local.status == 503 && contains(local.body, "OLLAMA_BASE_URL"), "local provider without base env is 503"); + auto cf = post("chat", "{\"provider\":\"cloudflare\",\"prompt\":\"hi\",\"model\":\"m\",\"confirm\":true}"); + check(cf.status == 503, "cloudflare without account is 503"); + return finish("ai/compat/routes"); +} diff --git a/tests/src/vendors/compat/support.h b/tests/src/vendors/compat/support.h new file mode 100644 index 0000000..c5d8f0c --- /dev/null +++ b/tests/src/vendors/compat/support.h @@ -0,0 +1,63 @@ +#pragma once + +// Shared rig for the compat tests: a Provider over a MockTransport with a +// fake key. The mock only records the requests the real client builds; +// replies are documented examples taken verbatim from OpenAI's API reference +// (the shape every row shares) and labelled FIXTURE where they appear. +// Nothing here verifies a provider's behavior (see SPEC.md ## Unverified). + +#include +#include + +#include "clients/transports/mock_transport.h" +#include "framework.h" +#include "vendors/compat/provider.h" + +namespace compat_test { + +using molo::clients::ClientResponse; +using molo::clients::HeaderMap; +using molo::clients::Method; +using molo::clients::transports::MockTransport; +using Json = molo::clients::body::Json; + +inline ClientResponse reply(const std::string& body, int status = 200, + const std::string& type = "application/json") { + HeaderMap h; + h.set("Content-Type", type); + return ClientResponse(status, h, body); +} + +inline ClientResponse sse(const std::string& body) { return reply(body, 200, "text/event-stream"); } + +inline const char* kKey = "sk-TESTKEY-DO-NOT-LEAK"; + +inline bool has(const std::string& hay, const std::string& needle) { return hay.find(needle) != std::string::npos; } + +// "https://host/a/b" -> "/a/b" +inline std::string path_of(const std::string& url) { + std::size_t scheme = url.find("://"); + std::size_t slash = url.find('/', scheme == std::string::npos ? 0 : scheme + 3); + return slash == std::string::npos ? "" : url.substr(slash); +} + +struct Rig { + MockTransport* transport = nullptr; + molo::compat::Provider provider; + + explicit Rig(const std::string& id, const std::string& key = kKey, const std::string& base_override = "", + const std::string& account = "acct123") + : provider(make(id, key, base_override, account, &transport)) {} + + std::string path(const char* suffix) const { return path_of(provider.base_url()) + suffix; } + + private: + static molo::compat::Provider make(const std::string& id, const std::string& key, const std::string& base, + const std::string& account, MockTransport** out) { + auto t = std::make_unique(); + *out = t.get(); + return molo::compat::Provider::with(id, std::move(t), key, base, account); + } +}; + +} // namespace compat_test diff --git a/tests/src/vendors/compat/test_chat_model.cpp b/tests/src/vendors/compat/test_chat_model.cpp new file mode 100644 index 0000000..23b3afe --- /dev/null +++ b/tests/src/vendors/compat/test_chat_model.cpp @@ -0,0 +1,291 @@ +// CompatChatModel: request mapping, row quirks, parsing and streaming. Replies +// are documented OpenAI examples (FIXTURE) or transcripts built from the +// documented chunk shape (LABELLED, not recorded provider responses). +#include "support.h" +#include "vendors/compat/chat_model.h" + +using namespace molo::compat; +using namespace compat_test; +using molo::llm::ApiError; +using molo::llm::ChatChunk; +using molo::llm::ChatRequest; +using molo::llm::Message; + +namespace { + +ChatRequest base() { + ChatRequest r; + r.model = "m1"; + r.messages.push_back(Message::user("hi")); + return r; +} + +// FIXTURE: OpenAI API reference, createChatCompletion streaming example chunks. +const char* kStream = + "data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n" + "data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n" + "data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n" + "data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}]}\n\n" + "data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[],\"usage\":{\"prompt_tokens\":9,\"completion_tokens\":2,\"total_tokens\":11}}\n\n" + "data: [DONE]\n\n"; + +// LABELLED: tool call fragments in the documented delta shape, with a reasoning field. +const char* kToolStream = + "data: {\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"thinking\"},\"finish_reason\":null}]}\n\n" + "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null}]}\n\n" + "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\":\"}}]},\"finish_reason\":null}]}\n\n" + "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"Rome\\\"}\"}}]},\"finish_reason\":null}]}\n\n" + "data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\n" + "data: [DONE]\n\n"; +} // namespace + +TEST(compat_chat, body_maps_the_neutral_request) { + Rig rig("together"); + CompatChatModel m(rig.provider); + ChatRequest r = base(); + r.system = "be brief"; + r.max_tokens = 16; + r.temperature = 0.5; + r.top_p = 0.9; + r.stop = {"END"}; + r.reasoning_effort = "low"; + r.json_mode = true; + Json b = m.to_body(r, false); + ASSERT_EQ(b.find("model")->as_string(), std::string("m1")); + ASSERT_EQ(b.find("messages")->as_array().size(), 2u); + ASSERT_EQ(b.find("messages")->as_array()[0].find("role")->as_string(), std::string("system")); + ASSERT_EQ(b.find("max_tokens")->as_int(), 16L); + ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_object")); + ASSERT_EQ(b.find("reasoning_effort")->as_string(), std::string("low")); + ASSERT_TRUE(b.find("stream") == nullptr); + ASSERT_TRUE(m.to_body(r, true).find("stream")->as_bool()); +} + +TEST(compat_chat, schema_tools_and_tool_choice) { + Rig rig("together"); + CompatChatModel m(rig.provider); + ChatRequest r = base(); + r.response_schema = Json::parse("{\"type\":\"object\"}"); + r.tools.push_back({"get_weather", "weather", Json::parse("{\"type\":\"object\"}")}); + r.tool_choice = "get_weather"; + Json b = m.to_body(r, false); + ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_schema")); + ASSERT_TRUE(b.find("response_format")->find("json_schema")->find("strict")->as_bool()); + ASSERT_EQ(b.find("tool_choice")->find("function")->find("name")->as_string(), std::string("get_weather")); + r.tool_choice = "required"; + ASSERT_EQ(m.to_body(r, false).find("tool_choice")->as_string(), std::string("required")); + ChatRequest none = base(); + none.tool_choice = "auto"; + ASSERT_THROW(m.to_body(none, false), ApiError); // tool_choice needs tools + ChatRequest both = base(); + both.json_mode = true; + both.response_schema = Json::parse("{}"); + ASSERT_THROW(m.to_body(both, false), ApiError); +} + +TEST(compat_chat, tool_round_trip_messages_and_images) { + Rig rig("together"); + CompatChatModel m(rig.provider); + ChatRequest r = base(); + Message a; + a.role = molo::llm::Role::kAssistant; + molo::llm::ToolCall tc; + tc.id = "call_1"; + tc.name = "f"; + tc.arguments = Json::parse("{\"a\":1}"); + a.parts.push_back(molo::llm::ContentPart::of_call(tc)); + r.messages.push_back(a); + Message t; + t.role = molo::llm::Role::kTool; + t.parts.push_back(molo::llm::ContentPart::of_result({"call_1", "42", false})); + r.messages.push_back(t); + Message u; + u.role = molo::llm::Role::kUser; + u.parts.push_back(molo::llm::ContentPart::of_text("look")); + u.parts.push_back(molo::llm::ContentPart::of_image({"image/png", "abc", ""})); + r.messages.push_back(u); + Json b = m.to_body(r, false); + const auto& msgs = b.find("messages")->as_array(); + ASSERT_EQ(msgs[1].find("tool_calls")->as_array()[0].find("function")->find("arguments")->as_string(), std::string("{\"a\":1}")); + ASSERT_TRUE(msgs[1].find("content")->is_null()); + ASSERT_EQ(msgs[2].find("tool_call_id")->as_string(), std::string("call_1")); + ASSERT_TRUE(msgs[3].find("content")->is_array()); + ASSERT_TRUE(has(msgs[3].find("content")->as_array()[1].find("image_url")->find("url")->as_string(), "data:image/png;base64,")); + Message empty_tool; + empty_tool.role = molo::llm::Role::kTool; + ChatRequest bad = base(); + bad.messages.push_back(empty_tool); + ASSERT_THROW(m.to_body(bad, false), ApiError); +} + +TEST(compat_chat, input_rules_run_before_any_traffic) { + Rig rig("together"); + CompatChatModel m(rig.provider); + ChatRequest r = base(); + r.model = ""; + ASSERT_THROW(m.generate(r), ApiError); // no default model + r = base(); + r.messages.clear(); + ASSERT_THROW(m.generate(r), ApiError); + r = base(); + r.max_tokens = 0; + ASSERT_THROW(m.generate(r), ApiError); + r = base(); + r.temperature = 2.5; + ASSERT_THROW(m.generate(r), ApiError); + r = base(); + r.top_p = 0.0; + ASSERT_THROW(m.generate(r), ApiError); + r = base(); + r.stop = std::vector(17, "x"); + ASSERT_THROW(m.generate(r), ApiError); + r = base(); + r.extra = Json::parse("{\"messages\":[]}"); + ASSERT_THROW(m.generate(r), ApiError); + r = base(); + r.extra = Json::parse("[1]"); + ASSERT_THROW(m.generate(r), ApiError); + r = base(); + r.model = std::string(300, 'x'); + ASSERT_THROW(m.generate(r), ApiError); + ASSERT_EQ(rig.transport->send_count(), 0u); + CompatChatModel with_default(rig.provider, "dflt"); + ASSERT_EQ(with_default.to_body(base(), false).find("model")->as_string(), std::string("m1")); + ChatRequest nomodel = base(); + nomodel.model = ""; + ASSERT_EQ(with_default.to_body(nomodel, false).find("model")->as_string(), std::string("dflt")); +} + +TEST(compat_chat, groq_quirks_are_rejected_loudly) { + Rig rig("groq"); + CompatChatModel m(rig.provider); + ChatRequest named = base(); + named.messages[0].name = "alice"; + ASSERT_THROW(m.to_body(named, false), ApiError); + for (const char* key : {"logprobs", "top_logprobs", "logit_bias"}) { + ChatRequest r = base(); + r.extra = Json::object(); + r.extra.set(key, Json(1)); + ASSERT_THROW(m.to_body(r, false), ApiError); + } + ChatRequest n2 = base(); + n2.extra = Json::parse("{\"n\":2}"); + ASSERT_THROW(m.to_body(n2, false), ApiError); + ChatRequest n1 = base(); + n1.extra = Json::parse("{\"n\":1}"); + ASSERT_TRUE(m.to_body(n1, false).find("n") != nullptr); + ASSERT_EQ(rig.transport->send_count(), 0u); + // The same request is fine on a row without the quirk. + Rig other("together"); + CompatChatModel o(other.provider); + ChatRequest ok = base(); + ok.messages[0].name = "alice"; + ok.extra = Json::parse("{\"logprobs\":true,\"n\":2}"); + ASSERT_TRUE(o.to_body(ok, false).find("logprobs") != nullptr); + ChatRequest big = base(); + big.extra = Json::parse("{\"n\":9}"); + ASSERT_THROW(o.to_body(big, false), ApiError); +} + +TEST(compat_chat, cerebras_rejects_tools_with_response_format) { + Rig rig("cerebras"); + CompatChatModel m(rig.provider); + ChatRequest r = base(); + r.json_mode = true; + ASSERT_TRUE(m.to_body(r, false).find("response_format") != nullptr); + r.tools.push_back({"f", "", Json::object()}); + ASSERT_THROW(m.to_body(r, false), ApiError); +} + +TEST(compat_chat, parses_reasoning_fields_and_tool_calls) { + Json doc = Json::parse( + "{\"id\":\"x\",\"model\":\"m\",\"choices\":[{\"index\":0,\"finish_reason\":\"tool_calls\",\"message\":" + "{\"role\":\"assistant\",\"content\":null,\"reasoning\":\"why\",\"tool_calls\":[{\"id\":\"c1\",\"type\":\"function\"," + "\"function\":{\"name\":\"f\",\"arguments\":\"{\\\"a\\\":1}\"}},{\"id\":\"c2\",\"type\":\"function\"," + "\"function\":{\"name\":\"g\",\"arguments\":\"not json\"}}]}}],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":4," + "\"total_tokens\":7,\"prompt_tokens_details\":{\"cached_tokens\":2},\"completion_tokens_details\":{\"reasoning_tokens\":1}}}"); + molo::llm::ChatResponse r = CompatChatModel::parse_response(doc); + ASSERT_EQ(r.finish_reason, std::string("tool_calls")); + ASSERT_EQ(r.message.parts[0].reasoning, std::string("why")); + auto calls = r.tool_calls(); + ASSERT_EQ(calls.size(), 2u); + ASSERT_EQ(calls[0].arguments.find("a")->as_int(), 1L); + ASSERT_TRUE(calls[1].arguments.find("_invalid_json") != nullptr); + ASSERT_EQ(r.usage.cached_tokens, 2); + ASSERT_EQ(r.usage.reasoning_tokens, 1); + Json rc = Json::parse("{\"choices\":[{\"message\":{\"content\":\"a\",\"reasoning_content\":\"b\"}}]}"); + ASSERT_EQ(CompatChatModel::parse_response(rc).message.parts[0].reasoning, std::string("b")); + ASSERT_THROW(CompatChatModel::parse_response(Json::parse("{\"choices\":[]}")), ApiError); + ASSERT_THROW(CompatChatModel::parse_response(Json::parse("{\"choices\":[{}]}")), ApiError); +} + +TEST(compat_chat, stream_documented_transcript_with_usage_chunk) { + Rig rig("fireworks"); + rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse(kStream)); + CompatChatModel m(rig.provider); + std::string text, finish; + int total = 0; + m.stream(base(), [&](const ChatChunk& c) { + text += c.text_delta; + if (!c.finish_reason.empty()) finish = c.finish_reason; + if (c.usage) total = c.usage->total_tokens; + return true; + }); + ASSERT_EQ(text, std::string("Hello!")); + ASSERT_EQ(finish, std::string("stop")); + ASSERT_EQ(total, 11); + ASSERT_TRUE(has(rig.transport->seen()[0].body, "\"stream\":true")); + ASSERT_FALSE(has(rig.transport->seen()[0].body, "stream_options")); +} + +TEST(compat_chat, stream_tool_fragments_carry_id_and_name) { + Rig rig("fireworks"); + rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse(kToolStream)); + CompatChatModel m(rig.provider); + std::string args, id, name, reasoning; + m.stream(base(), [&](const ChatChunk& c) { + args += c.tool_args_delta; + if (!c.tool_call_id.empty()) id = c.tool_call_id; + if (!c.tool_call_name.empty()) name = c.tool_call_name; + reasoning += c.reasoning_delta; + return true; + }); + ASSERT_EQ(args, std::string("{\"city\":\"Rome\"}")); + ASSERT_EQ(id, std::string("call_1")); + ASSERT_EQ(name, std::string("get_weather")); + ASSERT_EQ(reasoning, std::string("thinking")); +} + +TEST(compat_chat, stream_without_finish_reason_is_truncated_and_sink_can_stop) { + Rig rig("fireworks"); + rig.transport->expect_response( + Method::kPost, rig.path("/chat/completions"), + sse("data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"a\"},\"finish_reason\":null}]}\n\n")); + CompatChatModel m(rig.provider); + try { + m.stream(base(), [](const ChatChunk&) { return true; }); + FAIL("expected truncation"); + } catch (const ApiError& e) { + ASSERT_EQ(e.http_status(), 502); + ASSERT_EQ(e.type(), std::string("STREAM_TRUNCATED")); + } + Rig rig2("fireworks"); + rig2.transport->expect_response(Method::kPost, rig2.path("/chat/completions"), sse(kStream)); + CompatChatModel m2(rig2.provider); + int seen = 0; + m2.stream(base(), [&](const ChatChunk&) { return ++seen < 2; }); // stop early: no throw + ASSERT_EQ(seen, 2); +} + +TEST(compat_chat, stream_error_event_throws) { + Rig rig("fireworks"); + rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), + sse("data: {\"error\":{\"type\":\"overloaded\",\"message\":\"busy\"}}\n\n")); + CompatChatModel m(rig.provider); + try { + m.stream(base(), [](const ChatChunk&) { return true; }); + FAIL("expected"); + } catch (const ApiError& e) { + ASSERT_EQ(e.type(), std::string("overloaded")); + } +} diff --git a/tests/src/vendors/compat/test_live.cpp b/tests/src/vendors/compat/test_live.cpp new file mode 100644 index 0000000..6712c46 --- /dev/null +++ b/tests/src/vendors/compat/test_live.cpp @@ -0,0 +1,115 @@ +// Opt-in live checks, driven by the registry table. MOLO_LIVE_NET=1 enables +// the network. Per row: +// - probe (no key needed): an invalid key must be answered like an API, never +// 404 and never a transport failure; the status and error type are printed; +// - with the row's key env set: list models (when the row claims it) and ONE +// completion of max_tokens 8, on the model named by COMPAT__MODEL +// (upper case id). A missing key or model prints a loud SKIP line. +// Nothing is created or deleted. Local rows run only when their base env is set. +#include +#include +#include + +#include "clients/transports/http_transport.h" +#include "llm/config.h" +#include "support.h" +#include "vendors/compat/chat_model.h" +#include "vendors/compat/services.h" + +using namespace molo::compat; +using namespace compat_test; +using molo::llm::ApiError; + +namespace { + +// Read at static-init time: another suite in the shared binary unsets +// MOLO_LIVE_NET before the tests run. +const bool kNet = std::getenv("MOLO_LIVE_NET") != nullptr; +bool net() { return kNet; } + +std::string upper(std::string s) { + std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return static_cast(std::toupper(c)); }); + return s; +} + +Provider probe_provider(const ProviderRow& row) { + return Provider::with(row.id, std::make_unique(), "sk-invalid-molo-probe", + "", "probe-account"); +} + +} // namespace + +TEST(compat_live, invalid_key_probe_answers_like_an_api_for_every_hosted_row) { + if (!net()) { + std::cout << "SKIP compat_live.invalid_key_probe (set MOLO_LIVE_NET=1)\n"; + return; + } + int unreachable = 0; + for (std::size_t i = 0; i < Registry::count(); ++i) { + const ProviderRow& row = Registry::at(i); + if (row.local || std::string(row.extra_env).size() > 0) continue; // no hosted URL to probe + Provider p = probe_provider(row); + int status = 200; + std::string type = "ok", detail; + try { + if (Registry::has(row, Feature::kModels)) { + CompatModels(p).list(); + } else { + CompatChatModel m(p); + molo::llm::ChatRequest r; + r.model = "molo-probe"; + r.messages.push_back(molo::llm::Message::user("hi")); + r.max_tokens = 1; + m.generate(r); + } + } catch (const ApiError& e) { + status = e.http_status(); + type = e.type(); + detail = e.message().substr(0, 160); + } + std::cout << " " << row.id << ": " << status << " " << type << "\n"; + if (status == 502) { + ++unreachable; // transport failure: reported, not asserted (network flake) + std::cout << " WARN " << row.id << " unreachable from here: " << detail << "\n"; + continue; + } + ASSERT_TRUE(status != 404); + } + std::cout << " unreachable rows: " << unreachable << "\n"; +} + +TEST(compat_live, models_and_one_tiny_completion_for_each_row_with_a_key) { + if (!net()) { + std::cout << "SKIP compat_live.models_and_one_tiny_completion (set MOLO_LIVE_NET=1 and a provider key env)\n"; + return; + } + int ran = 0; + for (std::size_t i = 0; i < Registry::count(); ++i) { + const ProviderRow& row = Registry::at(i); + Provider p = Provider::from_env(row.id); + if (!p.configured()) { + std::cout << "SKIP compat_live." << row.id << " (set " << p.missing_env() << ")\n"; + continue; + } + ++ran; + if (Registry::has(row, Feature::kModels) && !std::string(row.models_path).empty()) { + ModelList list = CompatModels(p).list(); + std::cout << " " << row.id << " models: " << list.models.size() << "\n"; + ASSERT_TRUE(!list.models.empty()); + } + const std::string model_env = "COMPAT_" + upper(row.id) + "_MODEL"; + std::string model = molo::llm::Config::get(model_env.c_str()); + if (model.empty()) { + std::cout << "SKIP compat_live." << row.id << " completion (set " << model_env << ")\n"; + continue; + } + CompatChatModel chat(p, model); + molo::llm::ChatRequest r; + r.messages.push_back(molo::llm::Message::user("Reply with the single word: ok")); + r.max_tokens = 8; + molo::llm::ChatResponse out = chat.generate(r); + std::cout << " " << row.id << " completion finish=" << out.finish_reason << " tokens=" << out.usage.total_tokens << "\n"; + ASSERT_TRUE(!out.id.empty() || !out.finish_reason.empty()); + } + if (ran == 0) std::cout << "SKIP compat_live: no provider key is configured\n"; +} diff --git a/tests/src/vendors/compat/test_provider.cpp b/tests/src/vendors/compat/test_provider.cpp new file mode 100644 index 0000000..c84391b --- /dev/null +++ b/tests/src/vendors/compat/test_provider.cpp @@ -0,0 +1,176 @@ +// Provider wiring over the real ApiClient with a recording transport, driven +// by the registry table: every row is exercised, so a new row is tested +// without a new test. +#include + +#include "support.h" +#include "vendors/compat/chat_model.h" +#include "vendors/compat/services.h" + +using namespace molo::compat; +using namespace compat_test; +using molo::llm::ApiError; + +namespace { +// FIXTURE: OpenAI API reference, createChatCompletion "Default" response example. +const char* kChatExample = + "{\"id\":\"chatcmpl-B9MBs8CjcvOU2jLn4n570S5qMJKcT\",\"object\":\"chat.completion\",\"created\":1741569952," + "\"model\":\"gpt-4.1-2025-04-14\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\"," + "\"content\":\"Hello! How can I assist you today?\",\"refusal\":null,\"annotations\":[]},\"logprobs\":null," + "\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":19,\"completion_tokens\":10,\"total_tokens\":29," + "\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":" + "{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}}," + "\"service_tier\":\"default\"}"; + +// FIXTURE: OpenAI API reference, listModels response example (two entries). +const char* kModelsExample = + "{\"object\":\"list\",\"data\":[{\"id\":\"model-id-0\",\"object\":\"model\",\"created\":1686935002," + "\"owned_by\":\"organization-owner\"},{\"id\":\"model-id-1\",\"object\":\"model\",\"created\":1686935002," + "\"owned_by\":\"organization-owner\"}]}"; + +molo::llm::ChatRequest hello() { + molo::llm::ChatRequest r; + r.model = "some-model"; + r.messages.push_back(molo::llm::Message::user("Hello!")); + return r; +} +} // namespace + +TEST(compat_provider, every_row_posts_chat_to_its_own_base_with_a_bearer_key) { + for (std::size_t i = 0; i < Registry::count(); ++i) { + const ProviderRow& row = Registry::at(i); + Rig rig(row.id); + rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), reply(kChatExample)); + CompatChatModel model(rig.provider); + molo::llm::ChatResponse out = model.generate(hello()); + ASSERT_EQ(out.text(), std::string("Hello! How can I assist you today?")); + ASSERT_EQ(out.usage.total_tokens, 29); + auto s = rig.transport->seen()[0]; + ASSERT_EQ(s.method, std::string("POST")); + ASSERT_EQ(s.headers.get_or("Authorization", ""), std::string("Bearer ") + kKey); + ASSERT_EQ(s.target.find(kKey), std::string::npos); + ASSERT_EQ(model.vendor(), std::string("compat:") + row.id); + } +} + +TEST(compat_provider, every_models_row_lists_and_every_other_row_refuses_without_traffic) { + for (std::size_t i = 0; i < Registry::count(); ++i) { + const ProviderRow& row = Registry::at(i); + Rig rig(row.id); + CompatModels models(rig.provider); + if (Registry::has(row, Feature::kModels)) { + rig.transport->expect_response(Method::kGet, rig.path("/models"), reply(kModelsExample)); + ModelList list = models.list(); + ASSERT_EQ(list.models.size(), 2u); + ASSERT_TRUE(list.contains("model-id-1")); + ASSERT_EQ(rig.transport->seen()[0].method, std::string("GET")); + } else { + ASSERT_THROW(models.list(), ApiError); + ASSERT_EQ(rig.transport->send_count(), 0u); + } + } +} + +TEST(compat_provider, features_not_in_the_row_never_reach_the_network) { + Rig rig("cerebras"); + CompatEmbeddings emb(rig.provider); + ASSERT_THROW(emb.embed("m", {"x"}), ApiError); + CompatImages img(rig.provider); + ImageRequest ir; + ir.model = "m"; + ir.prompt = "p"; + ASSERT_THROW(img.generate(ir), ApiError); + CompatAudio audio(rig.provider); + SpeechRequest sr; + sr.model = "m"; + sr.input = "x"; + sr.voice = "v"; + std::string ct; + ASSERT_THROW(audio.speak(sr, ct), ApiError); + ASSERT_EQ(rig.transport->send_count(), 0u); +} + +TEST(compat_provider, unconfigured_provider_has_no_traffic_and_names_the_env) { + Rig rig("groq", ""); + ASSERT_FALSE(rig.provider.configured()); + ASSERT_EQ(rig.provider.missing_env(), std::string("GROQ_API_KEY")); + CompatChatModel model(rig.provider); + try { + model.generate(hello()); + FAIL("expected"); + } catch (const molo::llm::NotConfiguredError& e) { + ASSERT_EQ(e.http_status(), 503); + ASSERT_TRUE(has(e.message(), "GROQ_API_KEY")); + } + ASSERT_EQ(rig.transport->send_count(), 0u); +} + +TEST(compat_provider, cloudflare_substitutes_the_account_id_and_validates_it) { + Rig rig("cloudflare", kKey, "", "a1b2c3"); + ASSERT_EQ(rig.provider.base_url(), std::string("https://api.cloudflare.com/client/v4/accounts/a1b2c3/ai/v1")); + ASSERT_THROW(Rig("cloudflare", kKey, "", ""), ApiError); + ASSERT_THROW(Rig("cloudflare", kKey, "", "a/../b"), ApiError); + ASSERT_THROW(Rig("cloudflare", kKey, "", "a b"), ApiError); +} + +TEST(compat_provider, base_override_is_validated_and_trailing_slash_removed) { + Rig ok("vllm", "", "http://10.0.0.5:9000/v1/"); + ASSERT_EQ(ok.provider.base_url(), std::string("http://10.0.0.5:9000/v1")); + ASSERT_THROW(Rig("vllm", "", "ftp://x"), ApiError); + ASSERT_THROW(Rig("vllm", "", "http://x/v1?a=b"), ApiError); + ASSERT_THROW(Rig("vllm", "", "http://x/../v1"), ApiError); + ASSERT_THROW(Rig("vllm", "", "http://x y/v1"), ApiError); + ASSERT_THROW(Rig("vllm", "", std::string("http://x/") + std::string(600, 'a')), ApiError); +} + +TEST(compat_provider, local_rows_are_configured_only_when_their_base_env_is_set) { + setenv("MOLO_ENV_FILES", "off", 1); + unsetenv("OLLAMA_BASE_URL"); + unsetenv("OLLAMA_API_KEY"); + Provider off = Provider::from_env("ollama"); + ASSERT_FALSE(off.configured()); + ASSERT_EQ(off.missing_env(), std::string("OLLAMA_BASE_URL")); + setenv("OLLAMA_BASE_URL", "http://127.0.0.1:11434/v1", 1); + Provider on = Provider::from_env("ollama"); + ASSERT_TRUE(on.configured()); + ASSERT_EQ(on.base_url(), std::string("http://127.0.0.1:11434/v1")); + unsetenv("OLLAMA_BASE_URL"); +} + +TEST(compat_provider, hosted_from_env_needs_its_key_and_cloudflare_needs_the_account) { + setenv("MOLO_ENV_FILES", "off", 1); + unsetenv("CLOUDFLARE_API_TOKEN"); + unsetenv("CLOUDFLARE_ACCOUNT_ID"); + unsetenv("GROQ_API_KEY"); + ASSERT_FALSE(Provider::from_env("groq").configured()); + setenv("CLOUDFLARE_API_TOKEN", "t", 1); + Provider cf = Provider::from_env("cloudflare"); + ASSERT_FALSE(cf.configured()); + ASSERT_EQ(cf.missing_env(), std::string("CLOUDFLARE_ACCOUNT_ID")); + setenv("CLOUDFLARE_ACCOUNT_ID", "acct9", 1); + ASSERT_TRUE(Provider::from_env("cloudflare").configured()); + unsetenv("CLOUDFLARE_API_TOKEN"); + unsetenv("CLOUDFLARE_ACCOUNT_ID"); +} + +TEST(compat_provider, error_envelopes_of_different_providers_map_to_api_error) { + // Shapes seen in the unauthenticated probes of SPEC section 5 (labelled: probe output, 2026-10-25). + struct Case { const char* id; int status; const char* body; const char* type; }; + const Case cases[] = { + {"groq", 401, "{\"error\":{\"message\":\"Invalid API Key\",\"type\":\"invalid_request_error\",\"code\":\"invalid_api_key\"}}", "invalid_request_error"}, + {"cerebras", 401, "{\"message\":\"Wrong API Key\",\"type\":\"invalid_request_error\",\"param\":\"api_key\",\"code\":\"wrong_api_key\"}", "invalid_request_error"}, + {"moonshot", 401, "{\"error\":{\"message\":\"Incorrect API key provided\",\"type\":\"incorrect_api_key_error\"}}", "incorrect_api_key_error"}, + }; + for (const Case& c : cases) { + Rig rig(c.id); + rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), reply(c.body, c.status)); + CompatChatModel model(rig.provider); + try { + model.generate(hello()); + FAIL("expected"); + } catch (const ApiError& e) { + ASSERT_EQ(e.http_status(), c.status); + ASSERT_EQ(e.type(), std::string(c.type)); + } + } +} diff --git a/tests/src/vendors/compat/test_registry.cpp b/tests/src/vendors/compat/test_registry.cpp new file mode 100644 index 0000000..e47ac15 --- /dev/null +++ b/tests/src/vendors/compat/test_registry.cpp @@ -0,0 +1,105 @@ +// The registry table is the single source: these tests pin its invariants so +// that adding a row cannot break consistency between components. +#include + +#include "support.h" +#include "vendors/compat/calls.h" +#include "vendors/compat/registry.h" + +using namespace molo::compat; +using namespace compat_test; + +TEST(compat_registry, rows_are_unique_slugs_with_complete_fields) { + std::set ids; + ASSERT_TRUE(Registry::count() >= 35); + for (std::size_t i = 0; i < Registry::count(); ++i) { + const ProviderRow& r = Registry::at(i); + std::string id = r.id; + ASSERT_TRUE(!id.empty() && id.size() <= 32); + for (char c : id) ASSERT_TRUE((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')); + ASSERT_TRUE(ids.insert(id).second); + ASSERT_TRUE(std::string(r.name).size() > 2); + ASSERT_TRUE(std::string(r.key_env).size() > 3); + ASSERT_TRUE(std::string(r.doc_url).rfind("https://", 0) == 0); + ASSERT_TRUE(std::string(r.note).size() > 10); + ASSERT_TRUE((r.features & Feature::kChat) != 0); // every row is a chat provider + } +} + +TEST(compat_registry, base_urls_are_wellformed) { + std::set envs; + for (std::size_t i = 0; i < Registry::count(); ++i) { + const ProviderRow& r = Registry::at(i); + std::string base = r.base_url; + ASSERT_TRUE(base.back() != '/'); + ASSERT_TRUE(base.find(' ') == std::string::npos); + if (r.local) { + ASSERT_TRUE(base.rfind("http://localhost:", 0) == 0); + ASSERT_TRUE(std::string(r.base_env).size() > 3); // the "configured" switch + } else { + ASSERT_TRUE(base.rfind("https://", 0) == 0); + ASSERT_TRUE(std::string(r.base_env).empty()); + } + ASSERT_EQ(base.find("{account_id}") != std::string::npos, std::string(r.extra_env).size() > 0); + // Credential names are unique except where two rows deliberately share none. + ASSERT_TRUE(envs.insert(r.key_env).second); + } +} + +TEST(compat_registry, models_feature_and_path_agree) { + for (std::size_t i = 0; i < Registry::count(); ++i) { + const ProviderRow& r = Registry::at(i); + bool claims = Registry::has(r, Feature::kModels); + bool path = std::string(r.models_path).size() > 0; + ASSERT_EQ(claims, path); + if (path) ASSERT_EQ(std::string(r.models_path), std::string("/models")); + } +} + +TEST(compat_registry, quirk_rows_carry_their_documented_flag) { + const ProviderRow* groq = Registry::find("groq"); + ASSERT_TRUE(groq != nullptr); + ASSERT_TRUE((groq->quirks & Quirk::kRejectMessageName) != 0); + ASSERT_TRUE((groq->quirks & Quirk::kRejectLogprobs) != 0); + ASSERT_TRUE((groq->quirks & Quirk::kSingleChoice) != 0); + ASSERT_TRUE((Registry::get("cerebras").quirks & Quirk::kNoToolsWithFormat) != 0); + ASSERT_EQ(Registry::get("cerebras").features & Feature::kEmbeddings, 0u); +} + +TEST(compat_registry, evidence_matches_row_kind) { + for (std::size_t i = 0; i < Registry::count(); ++i) { + const ProviderRow& r = Registry::at(i); + if (r.local) ASSERT_TRUE(r.evidence == Evidence::kDocsOnly); + ASSERT_TRUE(std::string(Registry::evidence_name(r.evidence)).size() > 4); + } + ASSERT_TRUE(Registry::get("cloudflare").evidence == Evidence::kDocsOnly); +} + +TEST(compat_registry, no_em_dash_in_any_string) { + const std::string em = "\xE2\x80\x94"; + for (std::size_t i = 0; i < Registry::count(); ++i) { + const ProviderRow& r = Registry::at(i); + for (const char* s : {r.id, r.name, r.base_url, r.key_env, r.doc_url, r.note}) + ASSERT_TRUE(std::string(s).find(em) == std::string::npos); + } +} + +TEST(compat_registry, lookup_and_unknown_id) { + ASSERT_TRUE(Registry::find("groq") != nullptr); + ASSERT_TRUE(Registry::find("nope") == nullptr); + ASSERT_THROW(Registry::get("nope"), molo::llm::ApiError); + ASSERT_THROW(Registry::at(Registry::count()), molo::llm::ApiError); + ASSERT_EQ(Registry::feature_names(Registry::get("cerebras")), std::string("chat,models")); +} + +TEST(compat_registry, call_table_rows_map_to_openai_operations) { + ASSERT_EQ(Calls::count(), 6u); + ASSERT_EQ(std::string(Calls::spec_of(Call::kChat).path), std::string("/chat/completions")); + ASSERT_EQ(std::string(Calls::spec_of(Call::kModels).operation_id), std::string("listModels")); + ASSERT_TRUE(Calls::method_of(Call::kModels) == Method::kGet); + ASSERT_TRUE(Calls::method_of(Call::kSpeech) == Method::kPost); + for (std::size_t i = 0; i < Calls::count(); ++i) { + ASSERT_TRUE(std::string(Calls::at(i).path).front() == '/'); + ASSERT_TRUE(std::string(Calls::at(i).operation_id).size() > 5); + } +} diff --git a/tests/src/vendors/compat/test_services.cpp b/tests/src/vendors/compat/test_services.cpp new file mode 100644 index 0000000..81d1bde --- /dev/null +++ b/tests/src/vendors/compat/test_services.cpp @@ -0,0 +1,142 @@ +// Models, embeddings, images and audio over the recording transport. Replies +// are the documented OpenAI examples (FIXTURE); assertions are on what WE send +// and how WE parse. +#include "support.h" +#include "vendors/compat/services.h" + +using namespace molo::compat; +using namespace compat_test; +using molo::llm::ApiError; + +TEST(compat_services, models_accepts_a_bare_array_and_skips_entries_without_id) { + // Together returns a bare array; shape recorded from its docs page (LABELLED). + ModelList l = CompatModels::parse(Json::parse("[{\"id\":\"a\"},{\"name\":\"b\"},{\"x\":1}]")); + ASSERT_EQ(l.models.size(), 2u); + ASSERT_TRUE(l.contains("b")); + ASSERT_THROW(CompatModels::parse(Json::parse("{\"data\":5}")), ApiError); + ASSERT_THROW(CompatModels::parse(Json::parse("{}")), ApiError); +} + +TEST(compat_services, models_list_truncates_loudly) { + Json arr = Json::array(); + for (std::size_t i = 0; i < 5010; ++i) { + Json o = Json::object(); + o.set("id", Json("m" + std::to_string(i))); + arr.push(o); + } + ModelList l = CompatModels::parse(arr); + ASSERT_EQ(l.models.size(), 5000u); + ASSERT_TRUE(l.truncated); +} + +TEST(compat_services, row_without_list_endpoint_refuses) { + Rig rig("dashscope"); + CompatModels m(rig.provider); + ASSERT_THROW(m.list(), ApiError); + ASSERT_EQ(rig.transport->send_count(), 0u); +} + +TEST(compat_services, embeddings_body_rules_and_parse) { + Rig rig("nvidia"); + CompatEmbeddings e(rig.provider); + Json b = e.to_body("m", {"a", "b"}, 256); + ASSERT_EQ(b.find("input")->as_array().size(), 2u); + ASSERT_EQ(b.find("dimensions")->as_int(), 256L); + ASSERT_TRUE(e.to_body("m", {"a"}, 0).find("dimensions") == nullptr); + ASSERT_THROW(e.to_body("", {"a"}, 0), ApiError); + ASSERT_THROW(e.to_body("m", {}, 0), ApiError); + ASSERT_THROW(e.to_body("m", {""}, 0), ApiError); + ASSERT_THROW(e.to_body("m", {"a"}, -1), ApiError); + ASSERT_THROW(e.to_body("m", std::vector(2049, "x"), 0), ApiError); + // FIXTURE: OpenAI API reference, createEmbedding response example. + rig.transport->expect_response( + Method::kPost, rig.path("/embeddings"), + reply("{\"object\":\"list\",\"data\":[{\"object\":\"embedding\",\"embedding\":[0.0023064255,-0.009327292,-0.0028842222]," + "\"index\":0}],\"model\":\"text-embedding-ada-002\",\"usage\":{\"prompt_tokens\":8,\"total_tokens\":8}}")); + EmbeddingResult r = e.embed("text-embedding-ada-002", {"The food was delicious"}); + ASSERT_EQ(r.items.size(), 1u); + ASSERT_EQ(r.items[0].vector.size(), 3u); + ASSERT_EQ(r.total_tokens, 8); + ASSERT_THROW(CompatEmbeddings::parse(Json::parse("{\"data\":[{\"embedding\":5}]}")), ApiError); +} + +TEST(compat_services, images_body_rules_and_parse) { + Rig rig("together"); + CompatImages img(rig.provider); + ImageRequest r; + r.model = "m"; + r.prompt = "a cat"; + r.size = "512x512"; + r.response_format = "b64_json"; + Json b = img.to_body(r); + ASSERT_EQ(b.find("size")->as_string(), std::string("512x512")); + r.size = "big"; + ASSERT_THROW(img.to_body(r), ApiError); + r.size = "x512"; + ASSERT_THROW(img.to_body(r), ApiError); + r.size = ""; + r.n = 11; + ASSERT_THROW(img.to_body(r), ApiError); + r.n = 1; + r.response_format = "png"; + ASSERT_THROW(img.to_body(r), ApiError); + r.response_format = ""; + r.prompt = ""; + ASSERT_THROW(img.to_body(r), ApiError); + // FIXTURE: OpenAI API reference, createImage response example. + auto items = CompatImages::parse(Json::parse("{\"created\":1713833628,\"data\":[{\"b64_json\":\"...\",\"revised_prompt\":\"x\"}]}")); + ASSERT_EQ(items.size(), 1u); + ASSERT_EQ(items[0].revised_prompt, std::string("x")); + ASSERT_THROW(CompatImages::parse(Json::parse("{}")), ApiError); +} + +TEST(compat_services, transcription_sends_multipart_and_validates) { + Rig rig("groq"); + CompatAudio audio(rig.provider); + TranscribeRequest t; + t.model = "whisper-large-v3"; + t.filename = "a.wav"; + t.audio = "RIFFxxxx"; + t.language = "en"; + // FIXTURE: OpenAI API reference, createTranscription json response example. + rig.transport->expect_response(Method::kPost, rig.path("/audio/transcriptions"), + reply("{\"text\":\"Imagine the wildest idea that you've ever had.\"}")); + ASSERT_TRUE(has(audio.transcribe(t), "wildest idea")); + auto s = rig.transport->seen()[0]; + ASSERT_TRUE(has(s.headers.get_or("Content-Type", ""), "multipart/form-data")); + ASSERT_TRUE(has(s.body, "name=\"file\"; filename=\"a.wav\"")); + ASSERT_TRUE(has(s.body, "whisper-large-v3")); + TranscribeRequest bad = t; + bad.audio.clear(); + ASSERT_THROW(audio.transcribe(bad), ApiError); + bad = t; + bad.filename = "a\".wav"; + ASSERT_THROW(audio.transcribe(bad), ApiError); + bad = t; + bad.filename = "../a.wav"; + ASSERT_THROW(audio.transcribe(bad), ApiError); + Rig no("cerebras"); + ASSERT_THROW(CompatAudio(no.provider).transcribe(t), ApiError); +} + +TEST(compat_services, speech_returns_bytes_and_content_type) { + Rig rig("together"); + CompatAudio audio(rig.provider); + SpeechRequest s; + s.model = "m"; + s.input = "hello"; + s.voice = "alloy"; + s.format = "mp3"; + rig.transport->expect_response(Method::kPost, rig.path("/audio/speech"), reply("ID3bytes", 200, "audio/mpeg")); + std::string ct; + ASSERT_EQ(audio.speak(s, ct), std::string("ID3bytes")); + ASSERT_EQ(ct, std::string("audio/mpeg")); + s.format = "ogg"; + ASSERT_THROW(audio.speech_body(s), ApiError); + s.format = ""; + s.voice = ""; + ASSERT_THROW(audio.speech_body(s), ApiError); + s.voice = "v"; + s.input = std::string(4097, 'x'); + ASSERT_THROW(audio.speech_body(s), ApiError); +}