Merge branch 'worktree-agent-a89776b81b60a5fca'
This commit is contained in:
@@ -1,13 +1,451 @@
|
||||
#include "../AiDemos.hpp"
|
||||
|
||||
#include <functional>
|
||||
|
||||
#include "../../fw/Router.hpp"
|
||||
#include "../../fw/Templates.hpp"
|
||||
#include "security/crypto/Base64.h"
|
||||
#include "vendors/compat/caps.h"
|
||||
#include "vendors/compat/chat_model.h"
|
||||
#include "vendors/compat/provider.h"
|
||||
#include "vendors/compat/services.h"
|
||||
|
||||
namespace molo::example::app {
|
||||
|
||||
// compat. Stub: the package agent fills this in.
|
||||
// OpenAI-compatible provider registry over the real src/vendors/compat part.
|
||||
// GET routes need no credential. Every route that calls a provider is a POST:
|
||||
// 422 on bad input, 503 compat_not_configured without the provider's
|
||||
// credential. Routes that cost money (chat, embeddings, images, audio) are dry
|
||||
// runs that show the exact request unless the body has "confirm":true. The
|
||||
// provider list is the registry table; nothing here names a provider.
|
||||
|
||||
namespace {
|
||||
|
||||
namespace cp = compat;
|
||||
using Json = clients::body::Json;
|
||||
|
||||
constexpr std::size_t kMaxBody = 8 * 1024 * 1024; // audio travels as base64
|
||||
|
||||
const char kPage[] = R"COMPAT({% extends "base.html" %}
|
||||
{% block content %}
|
||||
<h1>OpenAI-compatible providers</h1>
|
||||
<p class="hint">One registry table, one client: chat, embeddings, models, images and audio for every
|
||||
provider whose row claims them. Paid calls are dry runs until the body says
|
||||
<code>"confirm":true</code>.</p>
|
||||
<p id="cp-config" class="hint">Checking credentials...</p>
|
||||
<section class="cp">
|
||||
<label>Provider
|
||||
<select id="cp-provider"></select></label>
|
||||
<p id="cp-info" class="hint"></p>
|
||||
<label>Route
|
||||
<select id="cp-route"></select></label>
|
||||
<label>Body (JSON object) <textarea id="cp-body" rows="8"></textarea></label>
|
||||
<div class="cp-actions">
|
||||
<button id="cp-send" type="button" class="primary">Send</button>
|
||||
<button id="cp-confirm" type="button">Send with confirm</button>
|
||||
</div>
|
||||
<p id="cp-error" class="error" role="alert"></p>
|
||||
<pre id="cp-out" class="cp-out"></pre>
|
||||
</section>
|
||||
<style>
|
||||
.cp label{display:block;margin:.5rem 0}
|
||||
.cp select,.cp textarea{width:100%;box-sizing:border-box}
|
||||
.cp-actions{display:flex;flex-wrap:wrap;gap:.5rem;margin:.5rem 0}
|
||||
.cp-out{overflow:auto;max-height:24rem;white-space:pre-wrap;word-break:break-word}
|
||||
</style>
|
||||
<script src="/embedded/ai_compat.js"></script>
|
||||
{% endblock %}
|
||||
)COMPAT";
|
||||
|
||||
[[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); }
|
||||
|
||||
fw::Response json_out(const Json& j, int status = 200) {
|
||||
fw::Response res = fw::JsonResponse(j.dump(), status);
|
||||
res.headers.set("Cache-Control", "no-store");
|
||||
return res;
|
||||
}
|
||||
|
||||
Json parse_body(const fw::Request& req) {
|
||||
if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large");
|
||||
if (req.body.empty()) return Json::object();
|
||||
try {
|
||||
Json j = Json::parse(req.body);
|
||||
if (!j.is_object()) bad("body must be a JSON object");
|
||||
return j;
|
||||
} catch (const clients::body::JsonError&) {
|
||||
bad("body is not valid JSON");
|
||||
}
|
||||
}
|
||||
|
||||
std::string text(const Json& b, const char* key, bool required = false) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) {
|
||||
if (required) bad(std::string(key) + " is required");
|
||||
return "";
|
||||
}
|
||||
if (!v->is_string()) bad(std::string(key) + " must be a string");
|
||||
return v->as_string();
|
||||
}
|
||||
|
||||
int number(const Json& b, const char* key, int fallback) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return fallback;
|
||||
if (!v->is_number()) bad(std::string(key) + " must be a number");
|
||||
return static_cast<int>(v->as_int());
|
||||
}
|
||||
|
||||
bool flag(const Json& b, const char* key) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return false;
|
||||
if (!v->is_bool()) bad(std::string(key) + " must be true or false");
|
||||
return v->as_bool();
|
||||
}
|
||||
|
||||
std::vector<std::string> inputs(const Json& b) {
|
||||
std::vector<std::string> out;
|
||||
const Json* v = b.find("input");
|
||||
if (!v || v->is_null()) bad("input is required");
|
||||
if (v->is_string()) {
|
||||
out.push_back(v->as_string());
|
||||
} else if (v->is_array()) {
|
||||
for (const Json& s : v->as_array()) {
|
||||
if (!s.is_string()) bad("input must be a string or an array of strings");
|
||||
out.push_back(s.as_string());
|
||||
}
|
||||
} else {
|
||||
bad("input must be a string or an array of strings");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
Json dry_run(const Json& request, const std::string& url) {
|
||||
Json o = Json::object();
|
||||
o.set("dry_run", Json(true));
|
||||
o.set("would_send_to", Json(url));
|
||||
o.set("request", request);
|
||||
o.set("hint", Json("send confirm:true to run it"));
|
||||
return o;
|
||||
}
|
||||
|
||||
Json usage_json(const llm::Usage& u) {
|
||||
Json o = Json::object();
|
||||
o.set("input_tokens", Json(u.input_tokens));
|
||||
o.set("output_tokens", Json(u.output_tokens));
|
||||
o.set("total_tokens", Json(u.total_tokens));
|
||||
o.set("cached_tokens", Json(u.cached_tokens));
|
||||
o.set("reasoning_tokens", Json(u.reasoning_tokens));
|
||||
return o;
|
||||
}
|
||||
|
||||
Json provider_json(const cp::ProviderRow& row) {
|
||||
cp::Provider p = cp::Provider::from_env(row.id);
|
||||
Json o = Json::object();
|
||||
o.set("id", Json(row.id));
|
||||
o.set("name", Json(row.name));
|
||||
o.set("base_url", Json(p.base_url()));
|
||||
o.set("credential_env", Json(row.key_env));
|
||||
o.set("base_env", Json(row.base_env));
|
||||
o.set("extra_env", Json(row.extra_env));
|
||||
o.set("configured", Json(p.configured()));
|
||||
o.set("missing_env", Json(p.missing_env()));
|
||||
Json feats = Json::array();
|
||||
std::string names = cp::Registry::feature_names(row);
|
||||
std::size_t at = 0;
|
||||
while (at <= names.size()) {
|
||||
std::size_t comma = names.find(',', at);
|
||||
if (comma == std::string::npos) comma = names.size();
|
||||
if (comma > at) feats.push(Json(names.substr(at, comma - at)));
|
||||
at = comma + 1;
|
||||
}
|
||||
o.set("features", feats);
|
||||
o.set("evidence", Json(cp::Registry::evidence_name(row.evidence)));
|
||||
o.set("local", Json(row.local));
|
||||
o.set("doc_url", Json(row.doc_url));
|
||||
o.set("note", Json(row.note));
|
||||
return o;
|
||||
}
|
||||
|
||||
// One route = validate into a Plan, then the generic runner decides dry run,
|
||||
// credential, call and error mapping.
|
||||
struct Plan {
|
||||
Json request; // what would be sent (shown in a dry run)
|
||||
std::string url; // for the dry-run message
|
||||
bool needs_confirm = false; // paid
|
||||
std::function<Json()> call;
|
||||
};
|
||||
|
||||
class CompatRoutes {
|
||||
public:
|
||||
using Planner = std::function<Plan(const Json&)>;
|
||||
|
||||
fw::Response run(const fw::Request& req, const Planner& planner) const {
|
||||
Json body = parse_body(req);
|
||||
Plan plan;
|
||||
try {
|
||||
plan = planner(body);
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
if (plan.needs_confirm && !flag(body, "confirm")) return json_out(dry_run(plan.request, plan.url));
|
||||
try {
|
||||
return json_out(plan.call());
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
}
|
||||
|
||||
static cp::Provider provider_of(const Json& b) {
|
||||
std::string id = text(b, "provider", true);
|
||||
if (!cp::Registry::find(id)) bad("unknown provider '" + id.substr(0, 64) + "'");
|
||||
return cp::Provider::from_env(id);
|
||||
}
|
||||
|
||||
static std::string url_of(const cp::Provider& p, cp::Call call) {
|
||||
return p.base_url() + cp::Calls::spec_of(call).path;
|
||||
}
|
||||
|
||||
Plan chat(const Json& b) const {
|
||||
cp::Provider p = provider_of(b);
|
||||
llm::ChatRequest r;
|
||||
r.model = text(b, "model");
|
||||
std::string system = text(b, "system");
|
||||
if (!system.empty()) r.system = system;
|
||||
r.messages.push_back(llm::Message::user(text(b, "prompt", true)));
|
||||
int max_tokens = number(b, "max_tokens", 0);
|
||||
if (max_tokens != 0) r.max_tokens = max_tokens;
|
||||
r.json_mode = flag(b, "json");
|
||||
r.reasoning_effort = text(b, "effort");
|
||||
if (const Json* t = b.find("temperature"); t && t->is_number()) r.temperature = t->as_number();
|
||||
if (const Json* e = b.find("extra"); e && !e->is_null()) r.extra = *e;
|
||||
auto model = std::make_shared<cp::CompatChatModel>(p);
|
||||
Plan pl{model->to_body(r, false), url_of(p, cp::Call::kChat), true, nullptr};
|
||||
pl.call = [model, r] {
|
||||
llm::ChatResponse out = model->generate(r);
|
||||
Json o = Json::object();
|
||||
o.set("id", Json(out.id));
|
||||
o.set("model", Json(out.model));
|
||||
o.set("finish_reason", Json(out.finish_reason));
|
||||
o.set("content", Json(out.text()));
|
||||
std::string reasoning;
|
||||
for (const llm::ContentPart& part : out.message.parts) reasoning += part.reasoning;
|
||||
o.set("reasoning", Json(reasoning));
|
||||
Json calls = Json::array();
|
||||
for (const llm::ToolCall& tc : out.tool_calls()) {
|
||||
Json x = Json::object();
|
||||
x.set("id", Json(tc.id));
|
||||
x.set("name", Json(tc.name));
|
||||
x.set("arguments", tc.arguments);
|
||||
calls.push(x);
|
||||
}
|
||||
o.set("tool_calls", calls);
|
||||
o.set("usage", usage_json(out.usage));
|
||||
return o;
|
||||
};
|
||||
return pl;
|
||||
}
|
||||
|
||||
Plan models(const Json& b) const {
|
||||
cp::Provider p = provider_of(b);
|
||||
p.require(cp::Call::kModels);
|
||||
if (std::string(p.row().models_path).empty()) bad(std::string(p.row().name) + " has no list endpoint under its base URL");
|
||||
auto api = std::make_shared<cp::CompatModels>(p);
|
||||
return Plan{Json::object(), url_of(p, cp::Call::kModels), false, [api] {
|
||||
cp::ModelList l = api->list();
|
||||
Json arr = Json::array();
|
||||
for (const cp::ModelEntry& m : l.models) {
|
||||
Json x = Json::object();
|
||||
x.set("id", Json(m.id));
|
||||
x.set("owned_by", Json(m.owned_by));
|
||||
arr.push(x);
|
||||
}
|
||||
Json o = Json::object();
|
||||
o.set("models", arr);
|
||||
o.set("truncated", Json(l.truncated));
|
||||
return o;
|
||||
}};
|
||||
}
|
||||
|
||||
Plan embeddings(const Json& b) const {
|
||||
cp::Provider p = provider_of(b);
|
||||
auto api = std::make_shared<cp::CompatEmbeddings>(p);
|
||||
std::string model = text(b, "model");
|
||||
std::vector<std::string> in = inputs(b);
|
||||
int dims = number(b, "dimensions", 0);
|
||||
Plan pl{api->to_body(model, in, dims), url_of(p, cp::Call::kEmbeddings), true, nullptr};
|
||||
pl.call = [api, model, in, dims] {
|
||||
cp::EmbeddingResult r = api->embed(model, in, dims);
|
||||
Json items = Json::array();
|
||||
for (const cp::Embedding& e : r.items) {
|
||||
Json x = Json::object();
|
||||
x.set("dimensions", Json(static_cast<int>(e.vector.size())));
|
||||
Json head = Json::array();
|
||||
for (std::size_t i = 0; i < e.vector.size() && i < 8; ++i) head.push(Json(e.vector[i]));
|
||||
x.set("head", head);
|
||||
items.push(x);
|
||||
}
|
||||
Json o = Json::object();
|
||||
o.set("model", Json(r.model));
|
||||
o.set("items", items);
|
||||
o.set("total_tokens", Json(r.total_tokens));
|
||||
return o;
|
||||
};
|
||||
return pl;
|
||||
}
|
||||
|
||||
Plan images(const Json& b) const {
|
||||
cp::Provider p = provider_of(b);
|
||||
auto api = std::make_shared<cp::CompatImages>(p);
|
||||
cp::ImageRequest r;
|
||||
r.model = text(b, "model");
|
||||
r.prompt = text(b, "prompt", true);
|
||||
r.n = number(b, "n", 1);
|
||||
r.size = text(b, "size");
|
||||
r.response_format = text(b, "response_format");
|
||||
Plan pl{api->to_body(r), url_of(p, cp::Call::kImages), true, nullptr};
|
||||
pl.call = [api, r] {
|
||||
Json arr = Json::array();
|
||||
for (const cp::ImageItem& i : api->generate(r)) {
|
||||
Json x = Json::object();
|
||||
x.set("url", Json(i.url));
|
||||
x.set("b64_json_bytes", Json(static_cast<int>(i.b64_json.size())));
|
||||
x.set("revised_prompt", Json(i.revised_prompt));
|
||||
arr.push(x);
|
||||
}
|
||||
Json o = Json::object();
|
||||
o.set("images", arr);
|
||||
return o;
|
||||
};
|
||||
return pl;
|
||||
}
|
||||
|
||||
Plan transcribe(const Json& b) const {
|
||||
cp::Provider p = provider_of(b);
|
||||
auto api = std::make_shared<cp::CompatAudio>(p);
|
||||
cp::TranscribeRequest r;
|
||||
r.model = text(b, "model");
|
||||
r.filename = text(b, "filename", true);
|
||||
r.language = text(b, "language");
|
||||
r.prompt = text(b, "prompt");
|
||||
if (!security::crypto::Base64::decode(text(b, "data_base64", true), r.audio)) bad("data_base64 is not valid base64");
|
||||
if (r.audio.size() > cp::CompatCaps::kMaxAudioBytes) bad("audio over the byte cap");
|
||||
p.require(cp::Call::kTranscribe);
|
||||
Json req = Json::object();
|
||||
req.set("model", Json(r.model));
|
||||
req.set("filename", Json(r.filename));
|
||||
req.set("bytes", Json(static_cast<int>(r.audio.size())));
|
||||
Plan pl{req, url_of(p, cp::Call::kTranscribe), true, nullptr};
|
||||
pl.call = [api, r] {
|
||||
Json o = Json::object();
|
||||
o.set("text", Json(api->transcribe(r)));
|
||||
return o;
|
||||
};
|
||||
return pl;
|
||||
}
|
||||
|
||||
Plan speech(const Json& b) const {
|
||||
cp::Provider p = provider_of(b);
|
||||
auto api = std::make_shared<cp::CompatAudio>(p);
|
||||
cp::SpeechRequest r;
|
||||
r.model = text(b, "model");
|
||||
r.input = text(b, "input", true);
|
||||
r.voice = text(b, "voice");
|
||||
r.format = text(b, "format");
|
||||
Plan pl{api->speech_body(r), url_of(p, cp::Call::kSpeech), true, nullptr};
|
||||
pl.call = [api, r] {
|
||||
std::string ct;
|
||||
std::string audio = api->speak(r, ct);
|
||||
Json o = Json::object();
|
||||
o.set("content_type", Json(ct));
|
||||
o.set("bytes", Json(static_cast<int>(audio.size())));
|
||||
o.set("audio_base64", Json(audio.size() <= 1024 * 1024 ? security::crypto::Base64::encode(audio) : std::string()));
|
||||
return o;
|
||||
};
|
||||
return pl;
|
||||
}
|
||||
|
||||
private:
|
||||
[[noreturn]] static void map(const llm::ApiError& e) {
|
||||
if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED")
|
||||
throw fw::HttpError(503, "compat_not_configured", e.message());
|
||||
if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty())
|
||||
throw fw::HttpError(422, "validation_error", e.message());
|
||||
throw fw::HttpError(502, "compat_error", e.vendor() + " " + std::to_string(e.http_status()) + " " + e.type() + ": " + e.message());
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
void register_ai_compat_demo(fw::Router& router, Services& services) {
|
||||
(void)router;
|
||||
(void)services;
|
||||
auto routes = std::make_shared<CompatRoutes>();
|
||||
fw::Environment* env = services.templates.get();
|
||||
services.templates->add_template("ai_compat.html", kPage);
|
||||
services.add_feature("ai-compat", "/ai/compat");
|
||||
|
||||
router.get("/ai/compat", "ai-compat-page", [env](fw::Request req) {
|
||||
(void)req;
|
||||
fw::TemplateCtx ctx;
|
||||
ctx["title"] = std::string("OpenAI-compatible providers");
|
||||
return fw::Response(200, env->render("ai_compat.html", ctx), "text/html; charset=utf-8");
|
||||
});
|
||||
|
||||
router.get("/ai/compat/status", "ai-compat-status", [](fw::Request req) {
|
||||
(void)req;
|
||||
int configured = 0;
|
||||
Json credentials = Json::array();
|
||||
for (std::size_t i = 0; i < cp::Registry::count(); ++i) {
|
||||
const cp::ProviderRow& row = cp::Registry::at(i);
|
||||
if (cp::Provider::from_env(row.id).configured()) ++configured;
|
||||
credentials.push(Json(row.key_env));
|
||||
}
|
||||
Json o = Json::object();
|
||||
o.set("configured", Json(configured > 0));
|
||||
o.set("configured_providers", Json(configured));
|
||||
o.set("providers", Json(static_cast<int>(cp::Registry::count())));
|
||||
o.set("credential_envs", credentials);
|
||||
o.set("calls", Json(static_cast<int>(cp::Calls::count())));
|
||||
o.set("max_models_returned", Json(static_cast<int>(cp::CompatCaps::kMaxModelsReturned)));
|
||||
return json_out(o);
|
||||
});
|
||||
|
||||
router.get("/ai/compat/providers", "ai-compat-providers", [](fw::Request req) {
|
||||
(void)req;
|
||||
Json arr = Json::array();
|
||||
for (std::size_t i = 0; i < cp::Registry::count(); ++i) arr.push(provider_json(cp::Registry::at(i)));
|
||||
Json o = Json::object();
|
||||
o.set("providers", arr);
|
||||
return json_out(o);
|
||||
});
|
||||
|
||||
router.get("/ai/compat/calls", "ai-compat-calls", [](fw::Request req) {
|
||||
(void)req;
|
||||
Json arr = Json::array();
|
||||
for (std::size_t i = 0; i < cp::Calls::count(); ++i) {
|
||||
const cp::CallSpec& s = cp::Calls::at(i);
|
||||
Json row = Json::object();
|
||||
row.set("http", Json(s.http_method));
|
||||
row.set("path", Json(s.path));
|
||||
row.set("operation", Json(s.operation_id));
|
||||
row.set("feature", Json(s.feature_name));
|
||||
arr.push(row);
|
||||
}
|
||||
Json o = Json::object();
|
||||
o.set("calls", arr);
|
||||
return json_out(o);
|
||||
});
|
||||
|
||||
using Member = CompatRoutes::Planner;
|
||||
CompatRoutes* r = routes.get();
|
||||
auto add = [&](const char* path, const char* name, Plan (CompatRoutes::*m)(const Json&) const) {
|
||||
router.post(path, name, [routes, r, m](fw::Request req) {
|
||||
return routes->run(req, Member([r, m](const Json& b) { return (r->*m)(b); }));
|
||||
});
|
||||
};
|
||||
add("/ai/compat/chat", "ai-compat-chat", &CompatRoutes::chat);
|
||||
add("/ai/compat/models", "ai-compat-models", &CompatRoutes::models);
|
||||
add("/ai/compat/embeddings", "ai-compat-embeddings", &CompatRoutes::embeddings);
|
||||
add("/ai/compat/images", "ai-compat-images", &CompatRoutes::images);
|
||||
add("/ai/compat/transcribe", "ai-compat-transcribe", &CompatRoutes::transcribe);
|
||||
add("/ai/compat/speech", "ai-compat-speech", &CompatRoutes::speech);
|
||||
}
|
||||
|
||||
} // namespace molo::example::app
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
// Compat page: thin client over /ai/compat/* (all limits server side). The
|
||||
// provider list comes from the registry table through /ai/compat/providers.
|
||||
(function () {
|
||||
"use strict";
|
||||
var $ = function (id) { return document.getElementById(id); };
|
||||
var out = $("cp-out"), err = $("cp-error");
|
||||
var templates = {
|
||||
"chat": { prompt: "Reply with the single word: ok", max_tokens: 8 },
|
||||
"models": {},
|
||||
"embeddings": { input: ["The food was delicious"] },
|
||||
"images": { prompt: "a small red cube on white", size: "512x512" },
|
||||
"transcribe": { filename: "clip.wav", data_base64: "" },
|
||||
"speech": { input: "Hello", voice: "alloy", format: "mp3" }
|
||||
};
|
||||
var providers = [];
|
||||
var rsel = $("cp-route"), psel = $("cp-provider");
|
||||
Object.keys(templates).forEach(function (k) {
|
||||
var o = document.createElement("option");
|
||||
o.value = k; o.textContent = k;
|
||||
rsel.appendChild(o);
|
||||
});
|
||||
function fill() {
|
||||
var t = JSON.parse(JSON.stringify(templates[rsel.value]));
|
||||
t.provider = psel.value;
|
||||
$("cp-body").value = JSON.stringify(t, null, 1);
|
||||
}
|
||||
function info() {
|
||||
var p = providers.filter(function (x) { return x.id === psel.value; })[0];
|
||||
if (!p) { $("cp-info").textContent = ""; return; }
|
||||
$("cp-info").textContent = p.name + " | " + p.base_url + " | " + p.features.join(", ") +
|
||||
" | " + p.evidence + " | " + (p.configured ? "configured" : "needs " + p.missing_env);
|
||||
}
|
||||
rsel.onchange = fill;
|
||||
psel.onchange = function () { info(); fill(); };
|
||||
|
||||
function send(confirm) {
|
||||
var body;
|
||||
try { body = JSON.parse($("cp-body").value || "{}"); }
|
||||
catch (e) { err.textContent = "Body is not valid JSON"; return; }
|
||||
if (confirm) body.confirm = true;
|
||||
err.textContent = "";
|
||||
out.textContent = "...";
|
||||
fetch("/ai/compat/" + rsel.value, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body)
|
||||
}).then(function (res) {
|
||||
return res.text().then(function (t) {
|
||||
var j;
|
||||
try { j = JSON.parse(t); } catch (e) { j = { message: t }; }
|
||||
if (!res.ok) { err.textContent = j.message || ("HTTP " + res.status); out.textContent = ""; }
|
||||
else { out.textContent = JSON.stringify(j, null, 2); }
|
||||
});
|
||||
}).catch(function (e) { err.textContent = String(e); out.textContent = ""; });
|
||||
}
|
||||
$("cp-send").onclick = function () { send(false); };
|
||||
$("cp-confirm").onclick = function () { send(true); };
|
||||
|
||||
fetch("/ai/compat/providers").then(function (r) { return r.json(); }).then(function (s) {
|
||||
providers = s.providers;
|
||||
providers.forEach(function (p) {
|
||||
var o = document.createElement("option");
|
||||
o.value = p.id;
|
||||
o.textContent = p.name + (p.configured ? " (key set)" : "");
|
||||
psel.appendChild(o);
|
||||
});
|
||||
$("cp-config").textContent = providers.filter(function (p) { return p.configured; }).length +
|
||||
" of " + providers.length + " providers have a credential. Others answer 503 until one is configured; dry runs still work.";
|
||||
info();
|
||||
fill();
|
||||
});
|
||||
})();
|
||||
Vendored
+243
@@ -0,0 +1,243 @@
|
||||
# OpenAI-compatible provider registry (compat) - Specification
|
||||
|
||||
Owned directory: `src/vendors/compat/`. Namespace `molo::compat`. Depends on
|
||||
the `src/llm` foundation (`ApiClient`, `Service`, `SseDecoder`, `ApiError`,
|
||||
`Config`, `ChatModel`). Plan: `plans/18-breadth.md` section 2 (method:
|
||||
`plans/17-llm-vendor-platforms.md`). Credentials: one env var per provider,
|
||||
named in the table of section 4, read through `llm::Config`.
|
||||
|
||||
## 1. Scope
|
||||
|
||||
One client for every provider that serves OpenAI-shaped endpoints. A provider
|
||||
is ONE ROW in `registry.cpp`; clients, routes, demo page and tests iterate the
|
||||
table, so adding a provider is one row plus one line in section 4 (enforced
|
||||
by `tools/check_spec.py`).
|
||||
|
||||
Endpoints, all OpenAI operations (section 3 of the checker verifies them
|
||||
against OpenAI's OpenAPI document): chat completions (also streaming),
|
||||
embeddings, models list, image generation, audio transcription, speech. A row
|
||||
only claims an endpoint its own docs list or a probe distinguished; a call
|
||||
outside the row's claim is refused before any network traffic.
|
||||
|
||||
Not here, said plainly: legacy completions, moderations, translations,
|
||||
image edits and variations, stored completions, the Responses API (no row's
|
||||
docs were read for it except Cloudflare's, which lists `/responses`; not
|
||||
wrapped), files, batches, fine-tuning. Providers with their own native API
|
||||
have their own packages (mistral, cohere, perplexity, huggingface, ...).
|
||||
Providers whose OpenAI-compatible endpoint could not be verified are in
|
||||
section 5 and are not rows.
|
||||
|
||||
## 2. Module map
|
||||
|
||||
| File | Concern |
|
||||
|---|---|
|
||||
| `caps.h` | Static ceilings (`CompatCaps`) |
|
||||
| `registry.h/.cpp` | `ProviderRow`, `Feature`, `Quirk`, `Evidence`, `Registry` and THE table |
|
||||
| `calls.h/.cpp` | The call table (6 rows, each an OpenAI `operationId`) |
|
||||
| `provider.h/.cpp` | `Provider` (row + `ApiClient`, base URL resolution and validation), `ProviderService` base |
|
||||
| `chat_model.h/.cpp` | `CompatChatModel : llm::ChatModel` (generate, stream, wire mapping, quirks) |
|
||||
| `services.h/.cpp` | `CompatModels`, `CompatEmbeddings`, `CompatImages`, `CompatAudio` |
|
||||
| `tools/check_spec.py` | Checks the call table against OpenAI's OpenAPI document and the registry against section 4 |
|
||||
|
||||
## 3. Contracts and static caps
|
||||
|
||||
| Knob | Value |
|
||||
|---|---|
|
||||
| messages / tools / stop sequences | 4096 / 128 / 16 |
|
||||
| max_tokens | 1..4194304 |
|
||||
| temperature / top_p | 0..2 / (0,1] |
|
||||
| n (extra) | 1..8; exactly 1 on rows with `kSingleChoice` |
|
||||
| stream events | 200000 per stream, then the stream is stopped |
|
||||
| models returned | 5000, then `truncated=true` (loud) |
|
||||
| embedding inputs / chars per input | 2048 / 1 MiB |
|
||||
| image n / prompt chars | 1..10 / 32768 |
|
||||
| audio upload / speech input chars | 64 MiB / 4096 |
|
||||
| model id chars / base URL chars | 256 / 512 |
|
||||
|
||||
- Auth is `Authorization: Bearer <key>` for every row (the shape OpenAI
|
||||
clients use). Local rows (`local=true`) are "configured" only when their
|
||||
`base_env` is set; the key is optional and the placeholder `local` is sent
|
||||
when none is set. Cloudflare needs `CLOUDFLARE_ACCOUNT_ID` (validated
|
||||
`[A-Za-z0-9_-]{1,64}`) and substitutes it into the base URL.
|
||||
- `<ID>_BASE_URL` style overrides are validated: `http(s)://`, no spaces,
|
||||
control characters, `?`, `#` or `..`; trailing `/` removed.
|
||||
- The model id is always required: ids differ per provider, so the module has
|
||||
no honest default (`CompatChatModel` takes an optional default at construction).
|
||||
- Quirks that the docs state as a 400 are rejected loudly before traffic
|
||||
(`Quirk::kRejectMessageName`, `kRejectLogprobs`, `kSingleChoice`,
|
||||
`kNoToolsWithFormat`); everything else is passed through untouched.
|
||||
- Errors: every non-2xx maps to `llm::ApiError` (the shared envelope parser
|
||||
handles `{error:{...}}`, `{message,type,code}`, `{detail}` and plain text).
|
||||
A stream without `finish_reason` throws 502 `STREAM_TRUNCATED`.
|
||||
- Model list shapes: `{"data":[...]}` and a bare array are both accepted.
|
||||
|
||||
## 4. Provider table
|
||||
|
||||
Source of truth is `registry.cpp`; this table must match it (checker).
|
||||
Evidence: `docs+probe` = the provider's docs text names the base URL and an
|
||||
unauthenticated probe on 2026-10-25 answered like an API; `probe-only` = the
|
||||
docs page was not readable as text or did not name the URL, the probe answered
|
||||
like an API; `docs-only` = named by docs, not probeable from here.
|
||||
|
||||
| id | name | base URL | credential | features | evidence | documentation | documented quirks and probe results |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| `groq` | Groq | `https://api.groq.com/openai/v1` | `GROQ_API_KEY` | chat,models,transcribe | docs+probe | https://console.groq.com/docs/openai | Docs: logprobs, logit_bias, top_logprobs and messages[].name answer 400; n must be 1; temperature 0 becomes 1e-8; transcription formats vtt and srt unsupported. Probe: 401 invalid_api_key on /models and chat. |
|
||||
| `together` | Together AI | `https://api.together.ai/v1` | `TOGETHER_API_KEY` | chat,embeddings,models,images,transcribe,speech | docs+probe | https://docs.together.ai/docs/openai-api-compatibility | Docs: logit_bias unsupported on most models; service_tier, store, metadata and prediction are accepted and ignored. Probe: 401 on /models and chat; legacy host api.together.xyz answers 401 too. |
|
||||
| `fireworks` | Fireworks AI | `https://api.fireworks.ai/inference/v1` | `FIREWORKS_API_KEY` | chat,embeddings,models | docs+probe | https://docs.fireworks.ai/tools-sdks/openai-compatibility | Probe: /models 401 with a docs link; unknown model answers 404 NOT_FOUND; empty embeddings body answers 500. |
|
||||
| `cerebras` | Cerebras | `https://api.cerebras.ai/v1` | `CEREBRAS_API_KEY` | chat,models | docs+probe | https://inference-docs.cerebras.ai/resources/openai | Docs: tools and response_format must not be combined (gpt-oss-120b rejects it); system and developer roles both act at developer level. Probe: /models 403 {detail}, chat 401 wrong_api_key, embeddings path 404. |
|
||||
| `sambanova` | SambaNova | `https://api.sambanova.ai/v1` | `SAMBANOVA_API_KEY` | chat,embeddings,models | probe-only | https://docs.sambanova.ai/docs/en/features/openai-compatibility | Docs page answers 403 to a plain fetch. Probe: /models 200 public; chat 401 (also accepts x-api-key); embeddings answers 400 'must provide a model' before auth. |
|
||||
| `deepinfra` | DeepInfra | `https://api.deepinfra.com/v1/openai` | `DEEPINFRA_API_KEY` | chat,embeddings,models,images,transcribe | docs+probe | https://docs.deepinfra.com/chat/overview | Probe: /models 200 public; chat 401 missing API key; images and audio paths answer 401 with a {detail} body. |
|
||||
| `novita` | Novita AI | `https://api.novita.ai/openai/v1` | `NOVITA_API_KEY` | chat,models | docs+probe | https://novita.ai/docs/guides/llm-api | Probe: /models 200 public; bad key answers 403 INVALID_API_KEY (not 401); legacy base /v3/openai answers the same. |
|
||||
| `nvidia` | NVIDIA NIM (build.nvidia.com) | `https://integrate.api.nvidia.com/v1` | `NVIDIA_API_KEY` | chat,embeddings,models | docs+probe | https://docs.api.nvidia.com/nim/reference/llm-apis | Probe: /models 200 public; embeddings answers 400 'model field is required'; retired models answer 410 Gone. |
|
||||
| `cloudflare` | Cloudflare Workers AI | `https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1` | `CLOUDFLARE_API_TOKEN + CLOUDFLARE_ACCOUNT_ID` | chat,embeddings | docs-only | https://developers.cloudflare.com/workers-ai/configuration/open-ai-compatibility/ | Docs list /chat/completions, /embeddings and /responses under the account path. Probe: GET /models answers 405 (no list endpoint here); not probeable further without an account id. |
|
||||
| `moonshot` | Moonshot (Kimi) | `https://api.moonshot.ai/v1` | `MOONSHOT_API_KEY` | chat,models | docs+probe | https://platform.kimi.ai/docs/api/chat | Probe: 401 incorrect_api_key_error on /models and chat; the China host api.moonshot.cn answers the same (separate keys). |
|
||||
| `dashscope` | Alibaba DashScope (Qwen, intl) | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` | `DASHSCOPE_API_KEY` | chat,embeddings | docs+probe | https://www.alibabacloud.com/help/en/model-studio/compatibility-of-openai-with-dashscope | Probe: 401 on /models and chat; embeddings answers 400 'you must provide a model parameter'; China host dashscope.aliyuncs.com/compatible-mode/v1 answers 401 too. |
|
||||
| `zhipu` | Zhipu (GLM, China) | `https://open.bigmodel.cn/api/paas/v4` | `ZHIPU_API_KEY` | chat,embeddings,images | docs+probe | https://docs.bigmodel.cn/cn/guide/develop/openai/introduction | Docs name the base URL; the path is /api/paas/v4, not /v1. Probe: 401 code 1001 (missing Authorization) on chat, embeddings and images. |
|
||||
| `zai` | Z.ai (GLM, international) | `https://api.z.ai/api/paas/v4` | `ZAI_API_KEY` | chat,embeddings,images | docs+probe | https://docs.z.ai/api-reference/introduction | International host of the Zhipu API, same /api/paas/v4 path. Probe: 401 code 1001. |
|
||||
| `minimax` | MiniMax | `https://api.minimax.io/v1` | `MINIMAX_API_KEY` | chat | probe-only | https://platform.minimax.io/docs/api-reference/text-openai-api | Docs text did not name the URL. Probe: 401 authorized_error on /models and chat; embeddings answers HTTP 200 with a base_resp status_code 1004 envelope (not OpenAI shaped); China host api.minimaxi.com answers 401. |
|
||||
| `qianfan` | Baidu Qianfan | `https://qianfan.baidubce.com/v2` | `QIANFAN_API_KEY` | chat,embeddings | probe-only | https://cloud.baidu.com/doc/qianfan-api/s/3m9b5lqft | Docs page is rendered client side. Probe: chat and embeddings 401 invalid_iam_token; GET /models 403 AccessDenied (different envelope). |
|
||||
| `ark` | Volcengine Ark (Doubao) | `https://ark.cn-beijing.volces.com/api/v3` | `ARK_API_KEY` | chat | probe-only | https://www.volcengine.com/docs/82379/1298454 | Docs page is rendered client side. Probe: 401 AuthenticationError on /models, chat and embeddings; path is /api/v3. |
|
||||
| `siliconflow` | SiliconFlow | `https://api.siliconflow.com/v1` | `SILICONFLOW_API_KEY` | chat,embeddings,models | docs+probe | https://docs.siliconflow.com/en/api-reference/chat-completions/chat-completions | Probe: 401 code 30014 with a {code,data,message} envelope on every path; China host api.siliconflow.cn identical. |
|
||||
| `nebius` | Nebius Token Factory | `https://api.studio.nebius.com/v1` | `NEBIUS_API_KEY` | chat,embeddings,models | docs+probe | https://docs.tokenfactory.nebius.com/api-reference/introduction | Probe: 401 unauthenticated on /models, chat and embeddings; images/generations 404 on this host; api.tokenfactory.nebius.com/v1 answers the same. |
|
||||
| `scaleway` | Scaleway Generative APIs | `https://api.scaleway.ai/v1` | `SCALEWAY_API_KEY` | chat,embeddings,models | docs+probe | https://www.scaleway.com/en/docs/generative-apis/api-cli/using-chat-api/ | Probe: 401 UNAUTHORIZED (missing 'Authorization' header) on /models, chat and embeddings. |
|
||||
| `upstage` | Upstage (Solar) | `https://api.upstage.ai/v1` | `UPSTAGE_API_KEY` | chat,embeddings | docs+probe | https://console.upstage.ai/docs/getting-started | Probe: 401 invalid API key on /models, chat and embeddings. |
|
||||
| `inception` | Inception (Mercury) | `https://api.inceptionlabs.ai/v1` | `INCEPTION_API_KEY` | chat,models | docs+probe | https://docs.inceptionlabs.ai/get-started/get-started | Probe: /models 200 public; chat answers 400 listing the valid model names before auth. |
|
||||
| `stepfun` | StepFun | `https://api.stepfun.com/v1` | `STEPFUN_API_KEY` | chat,models | docs+probe | https://platform.stepfun.com/docs/api-reference/chat/chat-completion-create | Probe: 401 invalid_api_key on /models and chat. |
|
||||
| `hunyuan` | Tencent Hunyuan | `https://api.hunyuan.cloud.tencent.com/v1` | `HUNYUAN_API_KEY` | chat | probe-only | https://cloud.tencent.com/document/product/1729/111007 | Docs page did not name the URL in readable text. Probe: 401 'Incorrect API key provided' on /models and chat. |
|
||||
| `venice` | Venice AI | `https://api.venice.ai/api/v1` | `VENICE_API_KEY` | chat,models | docs+probe | https://docs.venice.ai/api-reference/endpoint/chat/completions | Probe: /models 200 public; unauthenticated chat answers 402 with an x402 payment envelope (not 401). |
|
||||
| `vercel` | Vercel AI Gateway | `https://ai-gateway.vercel.sh/v1` | `AI_GATEWAY_API_KEY` | chat,models | docs+probe | https://vercel.com/docs/ai-gateway/openai-compat | Probe: /models 200 public; chat 401 authentication_error 'Missing Authorization header'. |
|
||||
| `friendli` | FriendliAI Serverless | `https://api.friendli.ai/serverless/v1` | `FRIENDLI_TOKEN` | chat,models | docs+probe | https://friendli.ai/docs/openapi/serverless/chat-completions | Probe: /models 200 public; chat 401 {detail}. |
|
||||
| `gemini` | Google Gemini (OpenAI compatibility) | `https://generativelanguage.googleapis.com/v1beta/openai` | `GEMINI_API_KEY` | chat | docs+probe | https://ai.google.dev/gemini-api/docs/openai | Probe: chat answers 400 'Missing or invalid Authorization header'; unauthenticated GET /models and embeddings answer 404, so only chat is claimed. |
|
||||
| `baseten` | Baseten Model APIs | `https://inference.baseten.co/v1` | `BASETEN_API_KEY` | chat | docs+probe | https://docs.baseten.co/development/model-apis/overview | Probe: /models 401 'No Authorization header provided'; chat 403 'please check the api-key you provided'. |
|
||||
| `featherless` | Featherless AI | `https://api.featherless.ai/v1` | `FEATHERLESS_API_KEY` | chat,models | probe-only | https://featherless.ai/docs/overview | Docs text did not name the URL. Probe: /models 200 public; chat 401 'must be signed in'. |
|
||||
| `ovh` | OVHcloud AI Endpoints | `https://oai.endpoints.kepler.ai.cloud.ovh.net/v1` | `OVH_AI_ENDPOINTS_ACCESS_TOKEN` | chat,models | probe-only | https://help.ovhcloud.com/csm/en-gb-public-cloud-ai-endpoints-getting-started | Docs text did not name the URL. Probe: /models 200 public; chat on an unknown model answers 404 model_not_found. |
|
||||
| `parasail` | Parasail | `https://api.parasail.io/v1` | `PARASAIL_API_KEY` | chat,models | probe-only | https://docs.parasail.io/parasail-docs/serverless/serverless | Docs URL answered 404 at probe time. Probe: 401 'No bearer token provided' on /models and chat. |
|
||||
| `crusoe` | Crusoe Managed Inference | `https://managed-inference-api-proxy.crusoecloud.com/v1` | `CRUSOE_API_KEY` | chat,models | probe-only | https://docs.crusoe.ai/managed-inference | Docs URL answered 404 at probe time. Probe: 401 {errors:[Authentication failed]} on /models and chat. |
|
||||
| `xiaomi` | Xiaomi MiMo | `https://api.xiaomimimo.com/v1` | `XIAOMI_API_KEY` | chat | probe-only | https://platform.xiaomimimo.com/ | Docs text did not name the URL. Probe: 401 'Invalid API Key' on /models and chat. |
|
||||
| `chutes` | Chutes | `https://llm.chutes.ai/v1` | `CHUTES_API_KEY` | chat,models | probe-only | https://docs.chutes.ai/ | Docs text did not name the URL. Probe: /models 200 public; chat answered an nginx 429 HTML page. |
|
||||
| `ollama` | Ollama | `http://localhost:11434/v1` | `OLLAMA_API_KEY + OLLAMA_BASE_URL` | chat,embeddings,models | docs-only | https://raw.githubusercontent.com/ollama/ollama/main/docs/api/openai-compatibility.mdx | Docs name http://localhost:11434/v1. The key is required by OpenAI clients but ignored by the server; the module sends the placeholder 'local' when none is set. |
|
||||
| `lmstudio` | LM Studio | `http://localhost:1234/v1` | `LMSTUDIO_API_KEY + LMSTUDIO_BASE_URL` | chat,embeddings,models | docs-only | https://lmstudio.ai/docs/developer/openai-compat | Docs name http://localhost:1234/v1. Key ignored unless configured in the app. |
|
||||
| `vllm` | vLLM server | `http://localhost:8000/v1` | `VLLM_API_KEY + VLLM_BASE_URL` | chat,embeddings,models,transcribe | docs-only | https://docs.vllm.ai/en/latest/serving/online_serving/ | Docs list /v1/chat/completions, /v1/embeddings and /v1/audio/transcriptions; key only when the server was started with --api-key. |
|
||||
| `llamacpp` | llama.cpp server | `http://localhost:8080/v1` | `LLAMACPP_API_KEY + LLAMACPP_BASE_URL` | chat,embeddings,models | docs-only | https://raw.githubusercontent.com/ggml-org/llama.cpp/master/tools/server/README.md | README lists /v1/chat/completions; key only when started with --api-key. Same default port as LocalAI. |
|
||||
| `localai` | LocalAI | `http://localhost:8080/v1` | `LOCALAI_API_KEY + LOCALAI_BASE_URL` | chat,embeddings,models | docs-only | https://localai.io/docs/overview/ | Docs name localhost:8080; endpoint list beyond chat, embeddings and models was not read. |
|
||||
|
||||
## 5. Providers not in the table
|
||||
|
||||
Plan 18 names these; they are NOT rows because the OpenAI-compatible endpoint
|
||||
could not be verified from here (2026-10-25). Each is one row away once
|
||||
confirmed.
|
||||
|
||||
| Provider | What was seen |
|
||||
|---|---|
|
||||
| GitHub Models | Docs name `https://models.github.ai/inference/chat/completions` and a catalog at `https://models.github.ai/catalog/models`, but the docs page read did not contain the URL as text, and every unauthenticated request (any path) answered `200 OK` with body `OK`, which proves nothing. The catalog lives outside the inference base path, which the row model cannot express (`models_path` is relative to the base) |
|
||||
| Lambda | `api.lambda.ai`, `api.lambdalabs.com` and `inference.lambda.ai` did not resolve from here |
|
||||
| Lepton | `api.lepton.ai` did not resolve (service wound down) |
|
||||
| Hyperbolic | `/models` answers 401, but chat and image paths answered `404 "This inference endpoint has been decommissioned"` on every probe |
|
||||
| Kluster | `api.kluster.ai` did not resolve |
|
||||
| Anyscale | The Public Endpoints API answers a 404 page announcing its end |
|
||||
| Sarvam | `/v1/models` is public, chat answered 403 `invalid_api_key_error`; Sarvam documents its own subscription-key header, not Bearer, so the Bearer-only row model would be wrong |
|
||||
| Reka | `api.reka.ai/v1` answers 401 `missing_api_key`; no OpenAI-compatibility statement was found |
|
||||
| AI21 | `/studio/v1/models` answers an empty public list and chat 400 "bad or missing authentication"; the chat shape was not confirmed as OpenAI's |
|
||||
| Mistral, Cohere, Perplexity, Hugging Face, OpenRouter, xAI, DeepSeek, OpenAI | Own packages (plan 17 / 18) |
|
||||
|
||||
## 6. Demo routes
|
||||
|
||||
Page `/ai/compat`, GET `/ai/compat/status`, `/ai/compat/providers` (the
|
||||
table plus per-row configured state), `/ai/compat/calls`. POST routes (body
|
||||
carries `"provider":"<id>"`): `chat`, `models`, `embeddings`, `images`,
|
||||
`transcribe`, `speech`. 422 on bad input or a call the row does not claim,
|
||||
503 `compat_not_configured` without the row's credential (the message names
|
||||
the missing env), paid routes (all but `models`) are dry runs showing the
|
||||
exact request and URL unless the body has `"confirm":true`. Matrix row
|
||||
`ai-compat` for the page.
|
||||
|
||||
## 7. Sources
|
||||
|
||||
- OpenAI OpenAPI document `openai/openai-openapi` at commit
|
||||
`8243cc8fa6f04430fb1f1a6a2cb926fbeb4f7f26` (`openapi.json`, version 2.3.0):
|
||||
the shape of every call.
|
||||
- Per-row documentation URL in the table of section 4. Each page was fetched
|
||||
as plain text on 2026-10-25 and searched for the base URL; "docs+probe"
|
||||
rows contain it. The documented quirks encoded as `Quirk` flags were read
|
||||
from: Groq (`console.groq.com/docs/openai`, "Currently Unsupported OpenAI
|
||||
Features": logprobs, logit_bias, top_logprobs, messages[].name answer 400;
|
||||
n must be 1) and Cerebras (`inference-docs.cerebras.ai/resources/openai`,
|
||||
"Unsupported parameter combinations": tools with response_format).
|
||||
- Together's docs page lists `/v1/chat/completions`, `/v1/embeddings`,
|
||||
`/v1/images/generations`, `/v1/audio/speech`, `/v1/audio/transcriptions` and
|
||||
moved its host to `api.together.ai`; Cloudflare's page lists
|
||||
`/chat/completions`, `/embeddings`, `/responses` under
|
||||
`/client/v4/accounts/{account_id}/ai/v1`; vLLM's page lists
|
||||
`/v1/chat/completions`, `/v1/embeddings`, `/v1/audio/transcriptions`.
|
||||
|
||||
## 8. Verified against spec
|
||||
|
||||
`python3 -I src/vendors/compat/tools/check_spec.py` on 2026-10-25 against
|
||||
the pinned OpenAI document (2.3.0): 6 call rows OK (method, path, operation
|
||||
id, request schema, every body field, response schema, every response field
|
||||
read), coverage OK (19 operations of the touched families are excluded with a
|
||||
reason in the script), registry OK (39 rows equal the table of section 4 in
|
||||
ids, base URLs, credential names and documentation URLs). `--local` runs only
|
||||
the registry check without network. This checks the OpenAI shape, not any
|
||||
provider's conformance to it.
|
||||
|
||||
## 9. Verified live
|
||||
|
||||
No provider key exists in this environment. Unauthenticated probes, no key
|
||||
sent, 2026-10-25 (`MOLO_LIVE_NET=1`, test `compat_live.invalid_key_probe...`,
|
||||
which uses an obviously invalid key and asserts "never 404"). Result per row
|
||||
(status of `GET /models` with the invalid key where the row lists models,
|
||||
otherwise a one-token `POST /chat/completions`): 401 for groq, together,
|
||||
fireworks, cerebras, deepinfra, dashscope, zhipu, zai, minimax, ark,
|
||||
siliconflow, nebius, upstage, moonshot, stepfun, hunyuan, vercel, featherless,
|
||||
parasail, crusoe, xiaomi; 403 for scaleway, baseten, ovh; 400 for gemini
|
||||
(`Missing or invalid Authorization header`); 200 for novita, nvidia, inception,
|
||||
venice, friendli (public lists that ignore the key). Three hosts could not be reached by OUR
|
||||
client: sambanova and chutes (`wolfSSL handshake failed: ASN parsing error,
|
||||
invalid input`) and qianfan (`handshake failed: received alert fatal error`)
|
||||
while `curl` reaches all three; see Gaps. Local rows were not probed (no
|
||||
server). Opt-in live tests: per row with its key env set, list models (when
|
||||
claimed) and ONE completion of `max_tokens` 8 on `COMPAT_<ID>_MODEL` (upper
|
||||
case id); a missing key or model prints a loud SKIP line per row. No success
|
||||
body from any provider was seen.
|
||||
|
||||
## 10. Unverified
|
||||
|
||||
- Every success response of every provider (only OpenAI's documented examples
|
||||
are used as parsing fixtures, labelled FIXTURE in the tests).
|
||||
- The `features` mask of each row beyond what its docs page or a probe showed.
|
||||
An unauthenticated 401 does not prove a path exists (many providers check
|
||||
auth first), so only these claims rest on more: embeddings routes answering
|
||||
400 before auth (sambanova, nvidia, dashscope), a 404 proving absence
|
||||
(cerebras embeddings, nebius images, gemini models and embeddings), and the
|
||||
docs pages read for Together and Cloudflare. All other claims are the
|
||||
provider's well known documented surface, not re-read here: images and audio
|
||||
for together, deepinfra, zhipu, zai; transcription for groq and vllm.
|
||||
- `docs-only` rows: Cloudflare (needs an account id), the five local servers.
|
||||
- Quirks other than the four flags (for example Groq turning temperature 0
|
||||
into 1e-8, Moonshot and Kimi temperature limits, per-model parameter
|
||||
support) are listed in notes or not at all, and are not enforced.
|
||||
- Provider-specific auth beyond Bearer (Baseten documents its own scheme; the
|
||||
probe showed Bearer is the shape OpenAI clients send, acceptance with a real
|
||||
key is unproven).
|
||||
- China-region hosts (moonshot.cn, dashscope.aliyuncs.com, minimaxi.com,
|
||||
siliconflow.cn) answer like their international twins but are not rows; set
|
||||
the override env for local rows only. Hosted rows cannot be overridden by
|
||||
env today (see Gaps).
|
||||
|
||||
## 11. Gaps
|
||||
|
||||
- `src/clients` TLS: wolfSSL handshake fails against sambanova, chutes and
|
||||
qianfan hosts where curl succeeds. Owner: `src/clients/transports` (not this
|
||||
package); the rows stay because the endpoints answer to curl.
|
||||
- `src/llm`: the shared `MOLO_LIVE_NET` is unset by a test in `tests/src/llm`
|
||||
(`test_llm.cpp`) so any suite reading it at run time skips in the shared
|
||||
binary; this suite reads it at static-init time. Other vendors' live tests
|
||||
still skip when the whole `ai_test` binary runs.
|
||||
- No env override for hosted base URLs (regional hosts); the `base_env` switch
|
||||
exists for local rows only. Closing it is one `base_env` value per hosted
|
||||
row in `registry.cpp`.
|
||||
- GitHub Models needs a second base URL for its catalog (see section 5).
|
||||
- Streaming sends no `stream_options.include_usage` by default because some
|
||||
providers reject it; pass it through `ChatRequest.extra`.
|
||||
- Responses API, legacy completions, image edits and moderation are not wrapped.
|
||||
Vendored
+50
@@ -0,0 +1,50 @@
|
||||
#include "vendors/compat/calls.h"
|
||||
|
||||
#include "llm/error.h"
|
||||
#include "vendors/compat/registry.h"
|
||||
|
||||
namespace molo::compat {
|
||||
|
||||
namespace {
|
||||
|
||||
const CallSpec kCalls[] = {
|
||||
{Call::kChat, "POST", "/chat/completions", "createChatCompletion", "CreateChatCompletionRequest",
|
||||
"model,messages,max_tokens,temperature,top_p,stop,tools,tool_choice,response_format,reasoning_effort,"
|
||||
"stream,stream_options,n",
|
||||
"CreateChatCompletionResponse", "id,model,choices,usage", Feature::kChat, "chat"},
|
||||
{Call::kEmbeddings, "POST", "/embeddings", "createEmbedding", "CreateEmbeddingRequest",
|
||||
"model,input,dimensions,encoding_format", "CreateEmbeddingResponse", "data,model,usage",
|
||||
Feature::kEmbeddings, "embeddings"},
|
||||
{Call::kModels, "GET", "/models", "listModels", "", "", "ListModelsResponse", "data",
|
||||
Feature::kModels, "models"},
|
||||
{Call::kImages, "POST", "/images/generations", "createImage", "CreateImageRequest",
|
||||
"model,prompt,n,size,response_format", "ImagesResponse", "data", Feature::kImages, "images"},
|
||||
{Call::kTranscribe, "POST", "/audio/transcriptions", "createTranscription", "CreateTranscriptionRequest",
|
||||
"file,model,language,prompt,response_format,temperature", "CreateTranscriptionResponseJson", "text",
|
||||
Feature::kTranscribe, "transcribe"},
|
||||
{Call::kSpeech, "POST", "/audio/speech", "createSpeech", "CreateSpeechRequest",
|
||||
"model,input,voice,response_format,speed", "", "", Feature::kSpeech, "speech"},
|
||||
};
|
||||
|
||||
constexpr std::size_t kCallCount = sizeof(kCalls) / sizeof(kCalls[0]);
|
||||
|
||||
} // namespace
|
||||
|
||||
const CallSpec& Calls::spec_of(Call call) {
|
||||
for (const CallSpec& s : kCalls)
|
||||
if (s.call == call) return s;
|
||||
throw llm::ApiError::invalid("compat", "unknown call");
|
||||
}
|
||||
|
||||
clients::Method Calls::method_of(Call call) {
|
||||
return std::string(spec_of(call).http_method) == "GET" ? clients::Method::kGet : clients::Method::kPost;
|
||||
}
|
||||
|
||||
std::size_t Calls::count() { return kCallCount; }
|
||||
|
||||
const CallSpec& Calls::at(std::size_t index) {
|
||||
if (index >= kCallCount) throw llm::ApiError::invalid("compat", "call index out of range");
|
||||
return kCalls[index];
|
||||
}
|
||||
|
||||
} // namespace molo::compat
|
||||
Vendored
+38
@@ -0,0 +1,38 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE table of REST calls this module makes. The shapes are OpenAI's
|
||||
// (every row is an OpenAI operation); providers differ only in base URL, auth
|
||||
// and quirks, which live in registry.cpp. tools/check_spec.py checks every
|
||||
// row against OpenAI's published OpenAPI document. Code builds every path
|
||||
// from this table, never from a literal.
|
||||
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
|
||||
#include "clients/request.h"
|
||||
|
||||
namespace molo::compat {
|
||||
|
||||
enum class Call { kChat, kEmbeddings, kModels, kImages, kTranscribe, kSpeech };
|
||||
|
||||
struct CallSpec {
|
||||
Call call;
|
||||
const char* http_method; // GET POST
|
||||
const char* path; // from the provider base URL
|
||||
const char* operation_id; // OpenAI OpenAPI operationId
|
||||
const char* request; // OpenAI request schema name ("" for GET)
|
||||
const char* body_fields; // comma list: request body fields we send
|
||||
const char* response; // OpenAI response schema name
|
||||
const char* fields_read; // comma list: response fields we read
|
||||
unsigned feature; // registry Feature bit a row must claim
|
||||
const char* feature_name;
|
||||
};
|
||||
|
||||
struct Calls {
|
||||
static const CallSpec& spec_of(Call call);
|
||||
static clients::Method method_of(Call call);
|
||||
static std::size_t count();
|
||||
static const CallSpec& at(std::size_t index);
|
||||
};
|
||||
|
||||
} // namespace molo::compat
|
||||
Vendored
+28
@@ -0,0 +1,28 @@
|
||||
#pragma once
|
||||
|
||||
// Static ceilings for the compat module. Never autotuned.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
namespace molo::compat {
|
||||
|
||||
struct CompatCaps {
|
||||
static constexpr std::size_t kMaxMessages = 4096;
|
||||
static constexpr std::size_t kMaxTools = 128;
|
||||
static constexpr std::size_t kMaxStopSequences = 16;
|
||||
static constexpr std::size_t kMaxStreamEvents = 200000;
|
||||
static constexpr std::size_t kMaxChoices = 8;
|
||||
static constexpr std::size_t kMaxModelsReturned = 5000;
|
||||
static constexpr std::size_t kMaxEmbeddingInputs = 2048;
|
||||
static constexpr std::size_t kMaxEmbeddingInputChars = 1u * 1024u * 1024u;
|
||||
static constexpr std::size_t kMaxPromptChars = 32768;
|
||||
static constexpr int kMaxImages = 10;
|
||||
static constexpr std::size_t kMaxAudioBytes = 64u * 1024u * 1024u;
|
||||
static constexpr std::size_t kMaxSpeechChars = 4096;
|
||||
static constexpr std::size_t kMaxModelChars = 256;
|
||||
static constexpr std::size_t kMaxBaseUrlChars = 512;
|
||||
static constexpr std::size_t kMaxFilenameChars = 256;
|
||||
static constexpr int kMaxTokensCeiling = 4194304;
|
||||
};
|
||||
|
||||
} // namespace molo::compat
|
||||
Vendored
+375
@@ -0,0 +1,375 @@
|
||||
#include "vendors/compat/chat_model.h"
|
||||
|
||||
#include <map>
|
||||
|
||||
#include "security/crypto/Base64.h"
|
||||
#include "vendors/compat/caps.h"
|
||||
|
||||
namespace molo::compat {
|
||||
|
||||
using Json = clients::body::Json;
|
||||
|
||||
namespace {
|
||||
|
||||
[[noreturn]] void bad(const std::string& vendor, const std::string& why) {
|
||||
throw llm::ApiError::invalid(vendor, why);
|
||||
}
|
||||
|
||||
std::string str_at(const Json& j, const char* key) {
|
||||
const Json* v = j.is_object() ? j.find(key) : nullptr;
|
||||
return v && v->is_string() ? v->as_string() : "";
|
||||
}
|
||||
|
||||
int int_at(const Json& j, const char* key) {
|
||||
const Json* v = j.is_object() ? j.find(key) : nullptr;
|
||||
return v && v->is_number() ? static_cast<int>(v->as_int()) : 0;
|
||||
}
|
||||
|
||||
const Json* obj_at(const Json& j, const char* key) {
|
||||
const Json* v = j.is_object() ? j.find(key) : nullptr;
|
||||
return v && v->is_object() ? v : nullptr;
|
||||
}
|
||||
|
||||
Json image_part(const std::string& vendor, const llm::ImageData& img) {
|
||||
std::string url = img.url;
|
||||
if (url.empty()) {
|
||||
if (img.bytes.empty()) bad(vendor, "image has neither url nor bytes");
|
||||
if (img.mime.empty()) bad(vendor, "image bytes need a mime type");
|
||||
url = "data:" + img.mime + ";base64," + security::crypto::Base64::encode(img.bytes);
|
||||
}
|
||||
Json u = Json::object();
|
||||
u.set("url", Json(url));
|
||||
Json p = Json::object();
|
||||
p.set("type", Json("image_url"));
|
||||
p.set("image_url", u);
|
||||
return p;
|
||||
}
|
||||
|
||||
Json text_part(const std::string& t) {
|
||||
Json p = Json::object();
|
||||
p.set("type", Json("text"));
|
||||
p.set("text", Json(t));
|
||||
return p;
|
||||
}
|
||||
|
||||
Json tool_call_json(const llm::ToolCall& c) {
|
||||
Json fn = Json::object();
|
||||
fn.set("name", Json(c.name));
|
||||
fn.set("arguments", Json(c.arguments.is_null() ? std::string("{}") : c.arguments.dump()));
|
||||
Json o = Json::object();
|
||||
o.set("id", Json(c.id));
|
||||
o.set("type", Json("function"));
|
||||
o.set("function", fn);
|
||||
return o;
|
||||
}
|
||||
|
||||
void append_message(const std::string& vendor, const ProviderRow& row, Json& out, const llm::Message& m) {
|
||||
if (!m.name.empty() && (row.quirks & Quirk::kRejectMessageName))
|
||||
bad(vendor, std::string(row.name) + " rejects messages[].name (documented 400)");
|
||||
auto base = [&](const char* role) {
|
||||
Json o = Json::object();
|
||||
o.set("role", Json(role));
|
||||
if (!m.name.empty()) o.set("name", Json(m.name));
|
||||
return o;
|
||||
};
|
||||
switch (m.role) {
|
||||
case llm::Role::kSystem: {
|
||||
Json o = base("system");
|
||||
o.set("content", Json(m.text()));
|
||||
out.push(o);
|
||||
return;
|
||||
}
|
||||
case llm::Role::kUser: {
|
||||
Json o = base("user");
|
||||
bool has_image = false;
|
||||
for (const llm::ContentPart& p : m.parts) has_image = has_image || p.image.has_value();
|
||||
if (!has_image) {
|
||||
o.set("content", Json(m.text()));
|
||||
} else {
|
||||
Json parts = Json::array();
|
||||
for (const llm::ContentPart& p : m.parts) {
|
||||
if (!p.text.empty()) parts.push(text_part(p.text));
|
||||
if (p.image) parts.push(image_part(vendor, *p.image));
|
||||
}
|
||||
o.set("content", parts);
|
||||
}
|
||||
out.push(o);
|
||||
return;
|
||||
}
|
||||
case llm::Role::kAssistant: {
|
||||
Json o = base("assistant");
|
||||
Json calls = Json::array();
|
||||
bool any = false;
|
||||
for (const llm::ContentPart& p : m.parts)
|
||||
if (p.tool_call) {
|
||||
calls.push(tool_call_json(*p.tool_call));
|
||||
any = true;
|
||||
}
|
||||
std::string text = m.text();
|
||||
o.set("content", text.empty() && any ? Json() : Json(text));
|
||||
if (any) o.set("tool_calls", calls);
|
||||
out.push(o);
|
||||
return;
|
||||
}
|
||||
case llm::Role::kTool: {
|
||||
bool any = false;
|
||||
for (const llm::ContentPart& p : m.parts)
|
||||
if (p.tool_result) {
|
||||
Json o = Json::object();
|
||||
o.set("role", Json("tool"));
|
||||
o.set("tool_call_id", Json(p.tool_result->call_id));
|
||||
o.set("content", Json(p.tool_result->content));
|
||||
out.push(o);
|
||||
any = true;
|
||||
}
|
||||
if (!any) bad(vendor, "a tool message needs a tool result part");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
llm::Usage usage_of(const Json& u) {
|
||||
llm::Usage out;
|
||||
if (!u.is_object()) return out;
|
||||
out.input_tokens = int_at(u, "prompt_tokens");
|
||||
out.output_tokens = int_at(u, "completion_tokens");
|
||||
out.total_tokens = int_at(u, "total_tokens");
|
||||
if (const Json* d = obj_at(u, "prompt_tokens_details")) out.cached_tokens = int_at(*d, "cached_tokens");
|
||||
if (const Json* d = obj_at(u, "completion_tokens_details")) out.reasoning_tokens = int_at(*d, "reasoning_tokens");
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string reasoning_of(const Json& holder) {
|
||||
std::string r = str_at(holder, "reasoning_content");
|
||||
return r.empty() ? str_at(holder, "reasoning") : r;
|
||||
}
|
||||
|
||||
Json arguments_of(const std::string& raw) {
|
||||
if (raw.empty()) return Json::object();
|
||||
try {
|
||||
Json j = Json::parse(raw);
|
||||
if (j.is_object()) return j;
|
||||
} catch (const std::exception&) {
|
||||
}
|
||||
Json badj = Json::object();
|
||||
badj.set("_invalid_json", Json(raw));
|
||||
return badj;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
CompatChatModel::CompatChatModel(Provider provider, std::string default_model)
|
||||
: ProviderService(std::move(provider)), default_model_(std::move(default_model)) {}
|
||||
|
||||
std::shared_ptr<CompatChatModel> CompatChatModel::from_env(const std::string& provider_id,
|
||||
const std::string& default_model) {
|
||||
return std::make_shared<CompatChatModel>(Provider::from_env(provider_id), default_model);
|
||||
}
|
||||
|
||||
Json CompatChatModel::to_body(const llm::ChatRequest& r, bool stream) const {
|
||||
const std::string vendor = client_->config().vendor;
|
||||
const ProviderRow& row = provider_.row();
|
||||
provider_.require(Call::kChat);
|
||||
|
||||
std::string model = r.model.empty() ? default_model_ : r.model;
|
||||
if (model.empty()) bad(vendor, "model is required (ids differ per provider)");
|
||||
if (model.size() > CompatCaps::kMaxModelChars) bad(vendor, "model id too long");
|
||||
if (r.messages.empty() && r.system.empty()) bad(vendor, "at least one message is required");
|
||||
if (r.messages.size() > CompatCaps::kMaxMessages) bad(vendor, "too many messages");
|
||||
if (r.tools.size() > CompatCaps::kMaxTools) bad(vendor, "too many tools");
|
||||
if (r.stop.size() > CompatCaps::kMaxStopSequences) bad(vendor, "too many stop sequences");
|
||||
if (r.max_tokens && (*r.max_tokens < 1 || *r.max_tokens > CompatCaps::kMaxTokensCeiling))
|
||||
bad(vendor, "max_tokens out of range");
|
||||
if (r.temperature && (*r.temperature < 0.0 || *r.temperature > 2.0)) bad(vendor, "temperature must be 0..2");
|
||||
if (r.top_p && (*r.top_p <= 0.0 || *r.top_p > 1.0)) bad(vendor, "top_p must be in (0,1]");
|
||||
const bool has_format = r.json_mode || !r.response_schema.is_null();
|
||||
if (r.json_mode && !r.response_schema.is_null()) bad(vendor, "json_mode and response_schema are exclusive");
|
||||
if ((row.quirks & Quirk::kNoToolsWithFormat) && has_format && !r.tools.empty())
|
||||
bad(vendor, std::string(row.name) + " documents that tools and response_format must not be combined");
|
||||
|
||||
Json body = Json::object();
|
||||
body.set("model", Json(model));
|
||||
Json messages = Json::array();
|
||||
if (!r.system.empty()) append_message(vendor, row, messages, llm::Message::system(r.system));
|
||||
for (const llm::Message& m : r.messages) append_message(vendor, row, messages, m);
|
||||
body.set("messages", messages);
|
||||
if (r.max_tokens) body.set("max_tokens", Json(*r.max_tokens));
|
||||
if (r.temperature) body.set("temperature", Json(*r.temperature));
|
||||
if (r.top_p) body.set("top_p", Json(*r.top_p));
|
||||
if (!r.stop.empty()) {
|
||||
Json s = Json::array();
|
||||
for (const std::string& x : r.stop) s.push(Json(x));
|
||||
body.set("stop", s);
|
||||
}
|
||||
if (!r.tools.empty()) {
|
||||
Json tools = Json::array();
|
||||
for (const llm::ToolSpec& t : r.tools) {
|
||||
if (t.name.empty()) bad(vendor, "tool name is required");
|
||||
Json fn = Json::object();
|
||||
fn.set("name", Json(t.name));
|
||||
if (!t.description.empty()) fn.set("description", Json(t.description));
|
||||
fn.set("parameters", t.parameters);
|
||||
Json o = Json::object();
|
||||
o.set("type", Json("function"));
|
||||
o.set("function", fn);
|
||||
tools.push(o);
|
||||
}
|
||||
body.set("tools", tools);
|
||||
}
|
||||
if (!r.tool_choice.empty()) {
|
||||
if (r.tools.empty()) bad(vendor, "tool_choice needs tools");
|
||||
if (r.tool_choice == "auto" || r.tool_choice == "none" || r.tool_choice == "required") {
|
||||
body.set("tool_choice", Json(r.tool_choice));
|
||||
} else {
|
||||
Json fn = Json::object();
|
||||
fn.set("name", Json(r.tool_choice));
|
||||
Json o = Json::object();
|
||||
o.set("type", Json("function"));
|
||||
o.set("function", fn);
|
||||
body.set("tool_choice", o);
|
||||
}
|
||||
}
|
||||
if (r.json_mode) {
|
||||
Json f = Json::object();
|
||||
f.set("type", Json("json_object"));
|
||||
body.set("response_format", f);
|
||||
} else if (!r.response_schema.is_null()) {
|
||||
Json js = Json::object();
|
||||
js.set("name", Json("response"));
|
||||
js.set("strict", Json(true));
|
||||
js.set("schema", r.response_schema);
|
||||
Json f = Json::object();
|
||||
f.set("type", Json("json_schema"));
|
||||
f.set("json_schema", js);
|
||||
body.set("response_format", f);
|
||||
}
|
||||
if (!r.reasoning_effort.empty()) body.set("reasoning_effort", Json(r.reasoning_effort));
|
||||
if (r.extra.is_object()) {
|
||||
for (const auto& [k, v] : r.extra.as_object()) {
|
||||
if (k == "model" || k == "messages" || k == "stream") bad(vendor, "extra may not set '" + k + "'");
|
||||
if ((row.quirks & Quirk::kRejectLogprobs) && (k == "logprobs" || k == "top_logprobs" || k == "logit_bias"))
|
||||
bad(vendor, std::string(row.name) + " rejects " + k + " (documented 400)");
|
||||
if ((row.quirks & Quirk::kSingleChoice) && k == "n" && !(v.is_number() && v.as_int() == 1))
|
||||
bad(vendor, std::string(row.name) + " requires n = 1");
|
||||
if (k == "n" && v.is_number() && (v.as_int() < 1 || v.as_int() > static_cast<long>(CompatCaps::kMaxChoices)))
|
||||
bad(vendor, "n out of range");
|
||||
body.set(k, v);
|
||||
}
|
||||
} else if (!r.extra.is_null()) {
|
||||
bad(vendor, "extra must be an object");
|
||||
}
|
||||
if (stream) body.set("stream", Json(true));
|
||||
return body;
|
||||
}
|
||||
|
||||
llm::ChatResponse CompatChatModel::parse_response(const Json& doc) {
|
||||
llm::ChatResponse out;
|
||||
out.id = str_at(doc, "id");
|
||||
out.model = str_at(doc, "model");
|
||||
out.message.role = llm::Role::kAssistant;
|
||||
out.raw = doc;
|
||||
out.usage = usage_of(doc.is_object() && doc.find("usage") ? *doc.find("usage") : Json());
|
||||
const Json* choices = doc.is_object() ? doc.find("choices") : nullptr;
|
||||
if (!choices || !choices->is_array() || choices->as_array().empty())
|
||||
throw llm::ApiError("compat", 502, "BAD_RESPONSE", "response has no choices");
|
||||
const Json& c = choices->as_array()[0];
|
||||
out.finish_reason = str_at(c, "finish_reason");
|
||||
const Json* msg = obj_at(c, "message");
|
||||
if (!msg) throw llm::ApiError("compat", 502, "BAD_RESPONSE", "choice has no message");
|
||||
std::string reasoning = reasoning_of(*msg);
|
||||
if (!reasoning.empty()) {
|
||||
llm::ContentPart r;
|
||||
r.reasoning = reasoning;
|
||||
out.message.parts.push_back(r);
|
||||
}
|
||||
std::string content = str_at(*msg, "content");
|
||||
const Json* calls = msg->find("tool_calls");
|
||||
const bool has_calls = calls && calls->is_array() && !calls->as_array().empty();
|
||||
if (!content.empty() || !has_calls) out.message.parts.push_back(llm::ContentPart::of_text(content));
|
||||
if (has_calls)
|
||||
for (const Json& call : calls->as_array()) {
|
||||
llm::ToolCall tc;
|
||||
tc.id = str_at(call, "id");
|
||||
if (const Json* fn = obj_at(call, "function")) {
|
||||
tc.name = str_at(*fn, "name");
|
||||
tc.arguments = arguments_of(str_at(*fn, "arguments"));
|
||||
}
|
||||
out.message.parts.push_back(llm::ContentPart::of_call(tc));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
llm::ChatResponse CompatChatModel::generate(const llm::ChatRequest& request) {
|
||||
Json body = to_body(request, false);
|
||||
return parse_response(client_->post(Calls::spec_of(Call::kChat).path, body));
|
||||
}
|
||||
|
||||
void CompatChatModel::stream(const llm::ChatRequest& request,
|
||||
const std::function<bool(const llm::ChatChunk&)>& on_chunk) {
|
||||
Json body = to_body(request, true);
|
||||
std::map<int, std::string> ids, names;
|
||||
std::size_t events = 0;
|
||||
bool finished = false, stopped = false;
|
||||
client_->stream(Calls::spec_of(Call::kChat).path, body, [&](const llm::SseEvent& ev) {
|
||||
if (++events > CompatCaps::kMaxStreamEvents) {
|
||||
stopped = true;
|
||||
return false;
|
||||
}
|
||||
const Json& d = ev.data;
|
||||
if (const Json* err = obj_at(d, "error"))
|
||||
throw llm::ApiError(vendor(), 502, str_at(*err, "type").empty() ? "STREAM_ERROR" : str_at(*err, "type"),
|
||||
str_at(*err, "message"));
|
||||
auto emit = [&](const llm::ChatChunk& c) {
|
||||
if (!on_chunk(c)) {
|
||||
stopped = true;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
const Json* choices = d.is_object() ? d.find("choices") : nullptr;
|
||||
if (choices && choices->is_array() && !choices->as_array().empty()) {
|
||||
const Json& c = choices->as_array()[0];
|
||||
if (const Json* delta = obj_at(c, "delta")) {
|
||||
std::string text = str_at(*delta, "content"), reasoning = reasoning_of(*delta);
|
||||
if (!text.empty() || !reasoning.empty()) {
|
||||
llm::ChatChunk ch;
|
||||
ch.text_delta = text;
|
||||
ch.reasoning_delta = reasoning;
|
||||
if (!emit(ch)) return false;
|
||||
}
|
||||
const Json* calls = delta->find("tool_calls");
|
||||
if (calls && calls->is_array())
|
||||
for (const Json& f : calls->as_array()) {
|
||||
int index = int_at(f, "index");
|
||||
if (!str_at(f, "id").empty()) ids[index] = str_at(f, "id");
|
||||
const Json* fn = obj_at(f, "function");
|
||||
if (fn && !str_at(*fn, "name").empty()) names[index] = str_at(*fn, "name");
|
||||
llm::ChatChunk ch;
|
||||
ch.tool_call_id = ids[index];
|
||||
ch.tool_call_name = names[index];
|
||||
ch.tool_args_delta = fn ? str_at(*fn, "arguments") : "";
|
||||
if (!emit(ch)) return false;
|
||||
}
|
||||
}
|
||||
std::string fin = str_at(c, "finish_reason");
|
||||
if (!fin.empty()) {
|
||||
finished = true;
|
||||
llm::ChatChunk ch;
|
||||
ch.finish_reason = fin;
|
||||
if (const Json* u = obj_at(d, "usage")) ch.usage = usage_of(*u);
|
||||
if (!emit(ch)) return false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (const Json* u = obj_at(d, "usage")) {
|
||||
llm::ChatChunk ch;
|
||||
ch.usage = usage_of(*u);
|
||||
if (!emit(ch)) return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
if (!stopped && !finished)
|
||||
throw llm::ApiError(vendor(), 502, "STREAM_TRUNCATED", "stream ended without a finish_reason");
|
||||
}
|
||||
|
||||
} // namespace molo::compat
|
||||
Vendored
+48
@@ -0,0 +1,48 @@
|
||||
#pragma once
|
||||
|
||||
// CompatChatModel: llm::ChatModel over POST {base}/chat/completions of any
|
||||
// registry row. Mapping notes (each one is a decision):
|
||||
// - ChatRequest.model is required unless a default model is given: model ids
|
||||
// differ per provider and there is no honest module-wide default.
|
||||
// - ChatRequest.system becomes the first system message.
|
||||
// - Reasoning text of an assistant message is never sent back.
|
||||
// - json_mode -> response_format {type: json_object}; response_schema ->
|
||||
// response_format {type: json_schema, strict: true}.
|
||||
// - reasoning_effort is sent as reasoning_effort; ChatRequest.extra is merged
|
||||
// except for keys the module owns (model, messages, stream).
|
||||
// - Row quirks (registry.h Quirk) are rejected before any traffic.
|
||||
// - Reasoning comes back from reasoning_content or reasoning, whichever the
|
||||
// provider sends.
|
||||
// - Tool call arguments that are not valid JSON arrive as
|
||||
// {"_invalid_json": "<raw text>"}.
|
||||
// - Streaming sends no stream_options unless ChatRequest.extra carries them;
|
||||
// usage then appears only when the provider sends it. A stream that ends
|
||||
// without a finish_reason throws 502 STREAM_TRUNCATED.
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "llm/chat_types.h"
|
||||
#include "vendors/compat/provider.h"
|
||||
|
||||
namespace molo::compat {
|
||||
|
||||
class CompatChatModel : public ProviderService, public llm::ChatModel {
|
||||
public:
|
||||
explicit CompatChatModel(Provider provider, std::string default_model = "");
|
||||
static std::shared_ptr<CompatChatModel> from_env(const std::string& provider_id,
|
||||
const std::string& default_model = "");
|
||||
|
||||
std::string vendor() const override { return client_->config().vendor; }
|
||||
llm::ChatResponse generate(const llm::ChatRequest& request) override;
|
||||
void stream(const llm::ChatRequest& request,
|
||||
const std::function<bool(const llm::ChatChunk&)>& on_chunk) override;
|
||||
|
||||
// The wire body a neutral request maps to (also used by tests and dry runs).
|
||||
clients::body::Json to_body(const llm::ChatRequest& request, bool stream) const;
|
||||
static llm::ChatResponse parse_response(const clients::body::Json& doc);
|
||||
|
||||
private:
|
||||
std::string default_model_;
|
||||
};
|
||||
|
||||
} // namespace molo::compat
|
||||
Vendored
+110
@@ -0,0 +1,110 @@
|
||||
#include "vendors/compat/provider.h"
|
||||
|
||||
#include <cctype>
|
||||
|
||||
#include "clients/transports/http_transport.h"
|
||||
#include "llm/config.h"
|
||||
#include "vendors/compat/caps.h"
|
||||
|
||||
namespace molo::compat {
|
||||
|
||||
namespace {
|
||||
|
||||
bool valid_account_id(const std::string& s) {
|
||||
if (s.empty() || s.size() > 64) return false;
|
||||
for (char c : s)
|
||||
if (!(std::isalnum(static_cast<unsigned char>(c)) || c == '-' || c == '_')) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool valid_base(const std::string& s) {
|
||||
if (s.empty() || s.size() > CompatCaps::kMaxBaseUrlChars) return false;
|
||||
if (s.rfind("http://", 0) != 0 && s.rfind("https://", 0) != 0) return false;
|
||||
for (char c : s)
|
||||
if (static_cast<unsigned char>(c) <= 0x20 || c == 0x7f || c == '?' || c == '#') return false;
|
||||
return s.find("..") == std::string::npos;
|
||||
}
|
||||
|
||||
std::string replace_account(std::string base, const std::string& account) {
|
||||
const std::string token = "{account_id}";
|
||||
std::size_t at = base.find(token);
|
||||
if (at != std::string::npos) base.replace(at, token.size(), account);
|
||||
return base;
|
||||
}
|
||||
|
||||
// The key a row would use: configured env value; local rows get a
|
||||
// placeholder when their base_env switch is set (servers ignore it).
|
||||
std::string key_for(const ProviderRow& row, const std::string& key_env_value, bool switch_set) {
|
||||
if (!row.local) return key_env_value;
|
||||
if (!switch_set) return "";
|
||||
return key_env_value.empty() ? "local" : key_env_value;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string Provider::resolve_base(const ProviderRow& row, const std::string& base_override,
|
||||
const std::string& account_id) {
|
||||
std::string base = base_override.empty() ? row.base_url : base_override;
|
||||
if (base.find("{account_id}") != std::string::npos) {
|
||||
if (!valid_account_id(account_id))
|
||||
throw llm::ApiError::invalid("compat:" + std::string(row.id), "account id missing or malformed");
|
||||
base = replace_account(base, account_id);
|
||||
}
|
||||
while (base.size() > 8 && base.back() == '/') base.pop_back();
|
||||
if (!valid_base(base))
|
||||
throw llm::ApiError::invalid("compat:" + std::string(row.id), "base URL must be http(s), no spaces, query or fragment");
|
||||
return base;
|
||||
}
|
||||
|
||||
llm::ApiClientConfig Provider::config_for(const ProviderRow& row, const std::string& base_url,
|
||||
const std::string& missing_env) {
|
||||
llm::ApiClientConfig c;
|
||||
c.vendor = std::string("compat:") + row.id;
|
||||
c.base_url = base_url;
|
||||
c.key_env = missing_env.empty() ? row.key_env : missing_env;
|
||||
c.auth = llm::AuthStyle::kBearer;
|
||||
return c;
|
||||
}
|
||||
|
||||
Provider Provider::from_env(const std::string& id) {
|
||||
const ProviderRow& row = Registry::get(id);
|
||||
std::string base_override = row.base_env[0] ? llm::Config::get(row.base_env) : "";
|
||||
std::string account = row.extra_env[0] ? llm::Config::get(row.extra_env) : "";
|
||||
std::string key = key_for(row, llm::Config::get(row.key_env), !base_override.empty());
|
||||
std::string missing;
|
||||
if (row.local && base_override.empty()) missing = row.base_env;
|
||||
else if (row.extra_env[0] && account.empty()) missing = row.extra_env;
|
||||
if (!missing.empty()) key.clear();
|
||||
// An unconfigured row still needs a syntactically valid base URL to exist.
|
||||
std::string base = (row.extra_env[0] && account.empty())
|
||||
? replace_account(row.base_url, "unset")
|
||||
: resolve_base(row, base_override, account);
|
||||
auto client = std::make_shared<llm::ApiClient>(std::make_unique<clients::transports::HttpTransport>(),
|
||||
config_for(row, base, missing), key);
|
||||
return Provider(&row, base, client);
|
||||
}
|
||||
|
||||
Provider Provider::with(const std::string& id, std::unique_ptr<clients::Transport> transport,
|
||||
const std::string& key, const std::string& base_override,
|
||||
const std::string& account_id) {
|
||||
const ProviderRow& row = Registry::get(id);
|
||||
std::string base = resolve_base(row, base_override, account_id);
|
||||
auto client = std::make_shared<llm::ApiClient>(std::move(transport), config_for(row, base, ""), key);
|
||||
return Provider(&row, base, client);
|
||||
}
|
||||
|
||||
std::string Provider::missing_env() const {
|
||||
if (configured()) return "";
|
||||
return client_->config().key_env;
|
||||
}
|
||||
|
||||
bool Provider::supports(Call call) const { return Registry::has(*row_, Calls::spec_of(call).feature); }
|
||||
|
||||
void Provider::require(Call call) const {
|
||||
if (!supports(call))
|
||||
throw llm::ApiError::invalid(client_->config().vendor,
|
||||
std::string(row_->name) + " has no documented " + Calls::spec_of(call).feature_name +
|
||||
" endpoint in the registry");
|
||||
}
|
||||
|
||||
} // namespace molo::compat
|
||||
Vendored
+64
@@ -0,0 +1,64 @@
|
||||
#pragma once
|
||||
|
||||
// Provider: one registry row bound to one ApiClient. The only place a base
|
||||
// URL is resolved (account id substitution, env override) and validated.
|
||||
// Services take a Provider and refuse calls the row does not claim.
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "llm/api_client.h"
|
||||
#include "vendors/compat/calls.h"
|
||||
#include "vendors/compat/registry.h"
|
||||
|
||||
namespace molo::compat {
|
||||
|
||||
class Provider {
|
||||
public:
|
||||
// Real HTTPS transport. Key and overrides come from llm::Config.
|
||||
static Provider from_env(const std::string& id);
|
||||
// Same row, any transport (tests). base_override and account_id may be empty.
|
||||
static Provider with(const std::string& id, std::unique_ptr<clients::Transport> transport,
|
||||
const std::string& key, const std::string& base_override = "",
|
||||
const std::string& account_id = "");
|
||||
|
||||
// base_url with {account_id} replaced and a trailing slash removed. Throws
|
||||
// ApiError::invalid for a malformed override or account id.
|
||||
static std::string resolve_base(const ProviderRow& row, const std::string& base_override,
|
||||
const std::string& account_id);
|
||||
|
||||
const ProviderRow& row() const { return *row_; }
|
||||
const std::string& base_url() const { return base_url_; }
|
||||
std::shared_ptr<llm::ApiClient> client() const { return client_; }
|
||||
bool configured() const { return client_->configured(); }
|
||||
// Env var a user must set next ("" when configured).
|
||||
std::string missing_env() const;
|
||||
|
||||
// Throws ApiError::invalid when the row does not claim the call's feature.
|
||||
void require(Call call) const;
|
||||
bool supports(Call call) const;
|
||||
|
||||
private:
|
||||
Provider(const ProviderRow* row, std::string base_url, std::shared_ptr<llm::ApiClient> client)
|
||||
: row_(row), base_url_(std::move(base_url)), client_(std::move(client)) {}
|
||||
|
||||
static llm::ApiClientConfig config_for(const ProviderRow& row, const std::string& base_url,
|
||||
const std::string& missing_env);
|
||||
|
||||
const ProviderRow* row_;
|
||||
std::string base_url_;
|
||||
std::shared_ptr<llm::ApiClient> client_;
|
||||
};
|
||||
|
||||
// Base for the per-endpoint services: holds the Provider, checks features.
|
||||
class ProviderService : public llm::Service {
|
||||
public:
|
||||
explicit ProviderService(Provider provider)
|
||||
: llm::Service(provider.client()), provider_(std::move(provider)) {}
|
||||
const Provider& provider() const { return provider_; }
|
||||
|
||||
protected:
|
||||
Provider provider_;
|
||||
};
|
||||
|
||||
} // namespace molo::compat
|
||||
Vendored
+239
@@ -0,0 +1,239 @@
|
||||
#include "vendors/compat/registry.h"
|
||||
|
||||
#include "llm/error.h"
|
||||
|
||||
namespace molo::compat {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr unsigned C = Feature::kChat;
|
||||
constexpr unsigned E = Feature::kEmbeddings;
|
||||
constexpr unsigned M = Feature::kModels;
|
||||
constexpr unsigned I = Feature::kImages;
|
||||
constexpr unsigned T = Feature::kTranscribe;
|
||||
constexpr unsigned S = Feature::kSpeech;
|
||||
constexpr Evidence DP = Evidence::kDocsAndProbe;
|
||||
constexpr Evidence PO = Evidence::kProbeOnly;
|
||||
constexpr Evidence DO = Evidence::kDocsOnly;
|
||||
|
||||
// Probe date for every "Probe:" note: 2026-10-25, unauthenticated, no key sent.
|
||||
// Fields: id, name, base_url, key_env, base_env, extra_env, models_path,
|
||||
// features, quirks, evidence, local, doc_url, note.
|
||||
const ProviderRow kRows[] = {
|
||||
{"groq", "Groq", "https://api.groq.com/openai/v1", "GROQ_API_KEY", "", "", "/models",
|
||||
C | M | T, Quirk::kRejectLogprobs | Quirk::kRejectMessageName | Quirk::kSingleChoice, DP, false,
|
||||
"https://console.groq.com/docs/openai",
|
||||
"Docs: logprobs, logit_bias, top_logprobs and messages[].name answer 400; n must be 1; temperature 0 "
|
||||
"becomes 1e-8; transcription formats vtt and srt unsupported. Probe: 401 invalid_api_key on /models and chat."},
|
||||
{"together", "Together AI", "https://api.together.ai/v1", "TOGETHER_API_KEY", "", "", "/models",
|
||||
C | E | M | I | T | S, Quirk::kNone, DP, false,
|
||||
"https://docs.together.ai/docs/openai-api-compatibility",
|
||||
"Docs: logit_bias unsupported on most models; service_tier, store, metadata and prediction are accepted "
|
||||
"and ignored. Probe: 401 on /models and chat; legacy host api.together.xyz answers 401 too."},
|
||||
{"fireworks", "Fireworks AI", "https://api.fireworks.ai/inference/v1", "FIREWORKS_API_KEY", "", "", "/models",
|
||||
C | E | M, Quirk::kNone, DP, false,
|
||||
"https://docs.fireworks.ai/tools-sdks/openai-compatibility",
|
||||
"Probe: /models 401 with a docs link; unknown model answers 404 NOT_FOUND; empty embeddings body answers 500."},
|
||||
{"cerebras", "Cerebras", "https://api.cerebras.ai/v1", "CEREBRAS_API_KEY", "", "", "/models",
|
||||
C | M, Quirk::kNoToolsWithFormat, DP, false,
|
||||
"https://inference-docs.cerebras.ai/resources/openai",
|
||||
"Docs: tools and response_format must not be combined (gpt-oss-120b rejects it); system and developer roles "
|
||||
"both act at developer level. Probe: /models 403 {detail}, chat 401 wrong_api_key, embeddings path 404."},
|
||||
{"sambanova", "SambaNova", "https://api.sambanova.ai/v1", "SAMBANOVA_API_KEY", "", "", "/models",
|
||||
C | E | M, Quirk::kNone, PO, false,
|
||||
"https://docs.sambanova.ai/docs/en/features/openai-compatibility",
|
||||
"Docs page answers 403 to a plain fetch. Probe: /models 200 public; chat 401 (also accepts x-api-key); "
|
||||
"embeddings answers 400 'must provide a model' before auth."},
|
||||
{"deepinfra", "DeepInfra", "https://api.deepinfra.com/v1/openai", "DEEPINFRA_API_KEY", "", "", "/models",
|
||||
C | E | M | I | T, Quirk::kNone, DP, false,
|
||||
"https://docs.deepinfra.com/chat/overview",
|
||||
"Probe: /models 200 public; chat 401 missing API key; images and audio paths answer 401 with a {detail} body."},
|
||||
{"novita", "Novita AI", "https://api.novita.ai/openai/v1", "NOVITA_API_KEY", "", "", "/models",
|
||||
C | M, Quirk::kNone, DP, false,
|
||||
"https://novita.ai/docs/guides/llm-api",
|
||||
"Probe: /models 200 public; bad key answers 403 INVALID_API_KEY (not 401); legacy base /v3/openai answers the same."},
|
||||
{"nvidia", "NVIDIA NIM (build.nvidia.com)", "https://integrate.api.nvidia.com/v1", "NVIDIA_API_KEY", "", "", "/models",
|
||||
C | E | M, Quirk::kNone, DP, false,
|
||||
"https://docs.api.nvidia.com/nim/reference/llm-apis",
|
||||
"Probe: /models 200 public; embeddings answers 400 'model field is required'; retired models answer 410 Gone."},
|
||||
{"cloudflare", "Cloudflare Workers AI", "https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/v1",
|
||||
"CLOUDFLARE_API_TOKEN", "", "CLOUDFLARE_ACCOUNT_ID", "", C | E, Quirk::kNone, DO, false,
|
||||
"https://developers.cloudflare.com/workers-ai/configuration/open-ai-compatibility/",
|
||||
"Docs list /chat/completions, /embeddings and /responses under the account path. Probe: GET /models answers "
|
||||
"405 (no list endpoint here); not probeable further without an account id."},
|
||||
{"moonshot", "Moonshot (Kimi)", "https://api.moonshot.ai/v1", "MOONSHOT_API_KEY", "", "", "/models",
|
||||
C | M, Quirk::kNone, DP, false,
|
||||
"https://platform.kimi.ai/docs/api/chat",
|
||||
"Probe: 401 incorrect_api_key_error on /models and chat; the China host api.moonshot.cn answers the same "
|
||||
"(separate keys)."},
|
||||
{"dashscope", "Alibaba DashScope (Qwen, intl)", "https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
|
||||
"DASHSCOPE_API_KEY", "", "", "", C | E, Quirk::kNone, DP, false,
|
||||
"https://www.alibabacloud.com/help/en/model-studio/compatibility-of-openai-with-dashscope",
|
||||
"Probe: 401 on /models and chat; embeddings answers 400 'you must provide a model parameter'; China host "
|
||||
"dashscope.aliyuncs.com/compatible-mode/v1 answers 401 too."},
|
||||
{"zhipu", "Zhipu (GLM, China)", "https://open.bigmodel.cn/api/paas/v4", "ZHIPU_API_KEY", "", "", "",
|
||||
C | E | I, Quirk::kNone, DP, false,
|
||||
"https://docs.bigmodel.cn/cn/guide/develop/openai/introduction",
|
||||
"Docs name the base URL; the path is /api/paas/v4, not /v1. Probe: 401 code 1001 (missing Authorization) "
|
||||
"on chat, embeddings and images."},
|
||||
{"zai", "Z.ai (GLM, international)", "https://api.z.ai/api/paas/v4", "ZAI_API_KEY", "", "", "",
|
||||
C | E | I, Quirk::kNone, DP, false,
|
||||
"https://docs.z.ai/api-reference/introduction",
|
||||
"International host of the Zhipu API, same /api/paas/v4 path. Probe: 401 code 1001."},
|
||||
{"minimax", "MiniMax", "https://api.minimax.io/v1", "MINIMAX_API_KEY", "", "", "",
|
||||
C, Quirk::kNone, PO, false,
|
||||
"https://platform.minimax.io/docs/api-reference/text-openai-api",
|
||||
"Docs text did not name the URL. Probe: 401 authorized_error on /models and chat; embeddings answers HTTP 200 "
|
||||
"with a base_resp status_code 1004 envelope (not OpenAI shaped); China host api.minimaxi.com answers 401."},
|
||||
{"qianfan", "Baidu Qianfan", "https://qianfan.baidubce.com/v2", "QIANFAN_API_KEY", "", "", "",
|
||||
C | E, Quirk::kNone, PO, false,
|
||||
"https://cloud.baidu.com/doc/qianfan-api/s/3m9b5lqft",
|
||||
"Docs page is rendered client side. Probe: chat and embeddings 401 invalid_iam_token; GET /models 403 "
|
||||
"AccessDenied (different envelope)."},
|
||||
{"ark", "Volcengine Ark (Doubao)", "https://ark.cn-beijing.volces.com/api/v3", "ARK_API_KEY", "", "", "",
|
||||
C, Quirk::kNone, PO, false,
|
||||
"https://www.volcengine.com/docs/82379/1298454",
|
||||
"Docs page is rendered client side. Probe: 401 AuthenticationError on /models, chat and embeddings; path is /api/v3."},
|
||||
{"siliconflow", "SiliconFlow", "https://api.siliconflow.com/v1", "SILICONFLOW_API_KEY", "", "", "/models",
|
||||
C | E | M, Quirk::kNone, DP, false,
|
||||
"https://docs.siliconflow.com/en/api-reference/chat-completions/chat-completions",
|
||||
"Probe: 401 code 30014 with a {code,data,message} envelope on every path; China host api.siliconflow.cn identical."},
|
||||
{"nebius", "Nebius Token Factory", "https://api.studio.nebius.com/v1", "NEBIUS_API_KEY", "", "", "/models",
|
||||
C | E | M, Quirk::kNone, DP, false,
|
||||
"https://docs.tokenfactory.nebius.com/api-reference/introduction",
|
||||
"Probe: 401 unauthenticated on /models, chat and embeddings; images/generations 404 on this host; "
|
||||
"api.tokenfactory.nebius.com/v1 answers the same."},
|
||||
{"scaleway", "Scaleway Generative APIs", "https://api.scaleway.ai/v1", "SCALEWAY_API_KEY", "", "", "/models",
|
||||
C | E | M, Quirk::kNone, DP, false,
|
||||
"https://www.scaleway.com/en/docs/generative-apis/api-cli/using-chat-api/",
|
||||
"Probe: 401 UNAUTHORIZED (missing 'Authorization' header) on /models, chat and embeddings."},
|
||||
{"upstage", "Upstage (Solar)", "https://api.upstage.ai/v1", "UPSTAGE_API_KEY", "", "", "",
|
||||
C | E, Quirk::kNone, DP, false,
|
||||
"https://console.upstage.ai/docs/getting-started",
|
||||
"Probe: 401 invalid API key on /models, chat and embeddings."},
|
||||
{"inception", "Inception (Mercury)", "https://api.inceptionlabs.ai/v1", "INCEPTION_API_KEY", "", "", "/models",
|
||||
C | M, Quirk::kNone, DP, false,
|
||||
"https://docs.inceptionlabs.ai/get-started/get-started",
|
||||
"Probe: /models 200 public; chat answers 400 listing the valid model names before auth."},
|
||||
{"stepfun", "StepFun", "https://api.stepfun.com/v1", "STEPFUN_API_KEY", "", "", "/models",
|
||||
C | M, Quirk::kNone, DP, false,
|
||||
"https://platform.stepfun.com/docs/api-reference/chat/chat-completion-create",
|
||||
"Probe: 401 invalid_api_key on /models and chat."},
|
||||
{"hunyuan", "Tencent Hunyuan", "https://api.hunyuan.cloud.tencent.com/v1", "HUNYUAN_API_KEY", "", "", "",
|
||||
C, Quirk::kNone, PO, false,
|
||||
"https://cloud.tencent.com/document/product/1729/111007",
|
||||
"Docs page did not name the URL in readable text. Probe: 401 'Incorrect API key provided' on /models and chat."},
|
||||
{"venice", "Venice AI", "https://api.venice.ai/api/v1", "VENICE_API_KEY", "", "", "/models",
|
||||
C | M, Quirk::kNone, DP, false,
|
||||
"https://docs.venice.ai/api-reference/endpoint/chat/completions",
|
||||
"Probe: /models 200 public; unauthenticated chat answers 402 with an x402 payment envelope (not 401)."},
|
||||
{"vercel", "Vercel AI Gateway", "https://ai-gateway.vercel.sh/v1", "AI_GATEWAY_API_KEY", "", "", "/models",
|
||||
C | M, Quirk::kNone, DP, false,
|
||||
"https://vercel.com/docs/ai-gateway/openai-compat",
|
||||
"Probe: /models 200 public; chat 401 authentication_error 'Missing Authorization header'."},
|
||||
{"friendli", "FriendliAI Serverless", "https://api.friendli.ai/serverless/v1", "FRIENDLI_TOKEN", "", "", "/models",
|
||||
C | M, Quirk::kNone, DP, false,
|
||||
"https://friendli.ai/docs/openapi/serverless/chat-completions",
|
||||
"Probe: /models 200 public; chat 401 {detail}."},
|
||||
{"gemini", "Google Gemini (OpenAI compatibility)", "https://generativelanguage.googleapis.com/v1beta/openai",
|
||||
"GEMINI_API_KEY", "", "", "", C, Quirk::kNone, DP, false,
|
||||
"https://ai.google.dev/gemini-api/docs/openai",
|
||||
"Probe: chat answers 400 'Missing or invalid Authorization header'; unauthenticated GET /models and "
|
||||
"embeddings answer 404, so only chat is claimed."},
|
||||
{"baseten", "Baseten Model APIs", "https://inference.baseten.co/v1", "BASETEN_API_KEY", "", "", "",
|
||||
C, Quirk::kNone, DP, false,
|
||||
"https://docs.baseten.co/development/model-apis/overview",
|
||||
"Probe: /models 401 'No Authorization header provided'; chat 403 'please check the api-key you provided'."},
|
||||
{"featherless", "Featherless AI", "https://api.featherless.ai/v1", "FEATHERLESS_API_KEY", "", "", "/models",
|
||||
C | M, Quirk::kNone, PO, false,
|
||||
"https://featherless.ai/docs/overview",
|
||||
"Docs text did not name the URL. Probe: /models 200 public; chat 401 'must be signed in'."},
|
||||
{"ovh", "OVHcloud AI Endpoints", "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1", "OVH_AI_ENDPOINTS_ACCESS_TOKEN",
|
||||
"", "", "/models", C | M, Quirk::kNone, PO, false,
|
||||
"https://help.ovhcloud.com/csm/en-gb-public-cloud-ai-endpoints-getting-started",
|
||||
"Docs text did not name the URL. Probe: /models 200 public; chat on an unknown model answers 404 model_not_found."},
|
||||
{"parasail", "Parasail", "https://api.parasail.io/v1", "PARASAIL_API_KEY", "", "", "/models",
|
||||
C | M, Quirk::kNone, PO, false,
|
||||
"https://docs.parasail.io/parasail-docs/serverless/serverless",
|
||||
"Docs URL answered 404 at probe time. Probe: 401 'No bearer token provided' on /models and chat."},
|
||||
{"crusoe", "Crusoe Managed Inference", "https://managed-inference-api-proxy.crusoecloud.com/v1", "CRUSOE_API_KEY",
|
||||
"", "", "/models", C | M, Quirk::kNone, PO, false,
|
||||
"https://docs.crusoe.ai/managed-inference",
|
||||
"Docs URL answered 404 at probe time. Probe: 401 {errors:[Authentication failed]} on /models and chat."},
|
||||
{"xiaomi", "Xiaomi MiMo", "https://api.xiaomimimo.com/v1", "XIAOMI_API_KEY", "", "", "",
|
||||
C, Quirk::kNone, PO, false,
|
||||
"https://platform.xiaomimimo.com/",
|
||||
"Docs text did not name the URL. Probe: 401 'Invalid API Key' on /models and chat."},
|
||||
{"chutes", "Chutes", "https://llm.chutes.ai/v1", "CHUTES_API_KEY", "", "", "/models",
|
||||
C | M, Quirk::kNone, PO, false,
|
||||
"https://docs.chutes.ai/",
|
||||
"Docs text did not name the URL. Probe: /models 200 public; chat answered an nginx 429 HTML page."},
|
||||
// Self-hosted servers: the key is optional, the row is configured when its base_env is set.
|
||||
{"ollama", "Ollama", "http://localhost:11434/v1", "OLLAMA_API_KEY", "OLLAMA_BASE_URL", "", "/models",
|
||||
C | E | M, Quirk::kNone, DO, true,
|
||||
"https://raw.githubusercontent.com/ollama/ollama/main/docs/api/openai-compatibility.mdx",
|
||||
"Docs name http://localhost:11434/v1. The key is required by OpenAI clients but ignored by the server; "
|
||||
"the module sends the placeholder 'local' when none is set."},
|
||||
{"lmstudio", "LM Studio", "http://localhost:1234/v1", "LMSTUDIO_API_KEY", "LMSTUDIO_BASE_URL", "", "/models",
|
||||
C | E | M, Quirk::kNone, DO, true,
|
||||
"https://lmstudio.ai/docs/developer/openai-compat",
|
||||
"Docs name http://localhost:1234/v1. Key ignored unless configured in the app."},
|
||||
{"vllm", "vLLM server", "http://localhost:8000/v1", "VLLM_API_KEY", "VLLM_BASE_URL", "", "/models",
|
||||
C | E | M | T, Quirk::kNone, DO, true,
|
||||
"https://docs.vllm.ai/en/latest/serving/online_serving/",
|
||||
"Docs list /v1/chat/completions, /v1/embeddings and /v1/audio/transcriptions; key only when the server "
|
||||
"was started with --api-key."},
|
||||
{"llamacpp", "llama.cpp server", "http://localhost:8080/v1", "LLAMACPP_API_KEY", "LLAMACPP_BASE_URL", "", "/models",
|
||||
C | E | M, Quirk::kNone, DO, true,
|
||||
"https://raw.githubusercontent.com/ggml-org/llama.cpp/master/tools/server/README.md",
|
||||
"README lists /v1/chat/completions; key only when started with --api-key. Same default port as LocalAI."},
|
||||
{"localai", "LocalAI", "http://localhost:8080/v1", "LOCALAI_API_KEY", "LOCALAI_BASE_URL", "", "/models",
|
||||
C | E | M, Quirk::kNone, DO, true,
|
||||
"https://localai.io/docs/overview/",
|
||||
"Docs name localhost:8080; endpoint list beyond chat, embeddings and models was not read."},
|
||||
};
|
||||
|
||||
constexpr std::size_t kRowCount = sizeof(kRows) / sizeof(kRows[0]);
|
||||
|
||||
} // namespace
|
||||
|
||||
std::size_t Registry::count() { return kRowCount; }
|
||||
|
||||
const ProviderRow& Registry::at(std::size_t index) {
|
||||
if (index >= kRowCount) throw llm::ApiError::invalid("compat", "registry index out of range");
|
||||
return kRows[index];
|
||||
}
|
||||
|
||||
const ProviderRow* Registry::find(const std::string& id) {
|
||||
for (const ProviderRow& row : kRows)
|
||||
if (id == row.id) return &row;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const ProviderRow& Registry::get(const std::string& id) {
|
||||
const ProviderRow* row = find(id);
|
||||
if (!row) throw llm::ApiError::invalid("compat", "unknown provider '" + id.substr(0, 64) + "'");
|
||||
return *row;
|
||||
}
|
||||
|
||||
const char* Registry::evidence_name(Evidence e) {
|
||||
switch (e) {
|
||||
case Evidence::kDocsAndProbe: return "docs+probe";
|
||||
case Evidence::kProbeOnly: return "probe-only";
|
||||
case Evidence::kDocsOnly: return "docs-only";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
std::string Registry::feature_names(const ProviderRow& row) {
|
||||
struct Name { unsigned bit; const char* name; };
|
||||
static const Name names[] = {{Feature::kChat, "chat"}, {Feature::kEmbeddings, "embeddings"},
|
||||
{Feature::kModels, "models"}, {Feature::kImages, "images"},
|
||||
{Feature::kTranscribe, "transcribe"}, {Feature::kSpeech, "speech"}};
|
||||
std::string out;
|
||||
for (const Name& n : names)
|
||||
if (row.features & n.bit) out += (out.empty() ? "" : ",") + std::string(n.name);
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace molo::compat
|
||||
Vendored
+72
@@ -0,0 +1,72 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE provider table of this module. Adding a provider is one row in
|
||||
// registry.cpp (plus its line in SPEC.md section 4, which tools/check_spec.py
|
||||
// enforces). Everything else (clients, routes, demo, tests) iterates this
|
||||
// table.
|
||||
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
|
||||
namespace molo::compat {
|
||||
|
||||
// Endpoints a row claims. Claims come from the provider's own docs or a
|
||||
// probe; SPEC.md says which. A call outside the mask is refused before any
|
||||
// network traffic.
|
||||
struct Feature {
|
||||
static constexpr unsigned kChat = 1u << 0;
|
||||
static constexpr unsigned kEmbeddings = 1u << 1;
|
||||
static constexpr unsigned kModels = 1u << 2;
|
||||
static constexpr unsigned kImages = 1u << 3;
|
||||
static constexpr unsigned kTranscribe = 1u << 4;
|
||||
static constexpr unsigned kSpeech = 1u << 5;
|
||||
};
|
||||
|
||||
// Documented behavior that differs from OpenAI and that the module enforces
|
||||
// loudly (reject) instead of letting the provider answer 400.
|
||||
struct Quirk {
|
||||
static constexpr unsigned kNone = 0;
|
||||
static constexpr unsigned kRejectMessageName = 1u << 0; // messages[].name answers 400
|
||||
static constexpr unsigned kRejectLogprobs = 1u << 1; // logprobs, top_logprobs, logit_bias answer 400
|
||||
static constexpr unsigned kSingleChoice = 1u << 2; // n must be 1
|
||||
static constexpr unsigned kNoToolsWithFormat = 1u << 3; // tools + response_format not combinable
|
||||
};
|
||||
|
||||
// How the row was verified (see SPEC.md section 5).
|
||||
enum class Evidence {
|
||||
kDocsAndProbe, // docs text names the base URL; unauthenticated probe answered like an API
|
||||
kProbeOnly, // docs text did not name it (or not readable); probe answered like an API
|
||||
kDocsOnly, // docs name it; not probeable (local server, or needs an account id)
|
||||
};
|
||||
|
||||
struct ProviderRow {
|
||||
const char* id; // slug, [a-z0-9]+, used in routes and ids
|
||||
const char* name; // display name
|
||||
const char* base_url; // no trailing slash; "{account_id}" is replaced from extra_env
|
||||
const char* key_env; // credential env var (via llm::Config); local rows: optional key
|
||||
const char* base_env; // env var overriding base_url; for local rows also the "configured" switch
|
||||
const char* extra_env; // second required env var (account id) or ""
|
||||
const char* models_path; // "" when the row has no list endpoint
|
||||
unsigned features; // Feature mask
|
||||
unsigned quirks; // Quirk mask
|
||||
Evidence evidence;
|
||||
bool local; // self-hosted server, no vendor credential
|
||||
const char* doc_url; // documentation page cited for the base URL
|
||||
const char* note; // documented quirks and probe results, one line
|
||||
};
|
||||
|
||||
class Registry {
|
||||
public:
|
||||
static std::size_t count();
|
||||
static const ProviderRow& at(std::size_t index);
|
||||
// nullptr when unknown.
|
||||
static const ProviderRow* find(const std::string& id);
|
||||
// Throws llm::ApiError::invalid for an unknown id.
|
||||
static const ProviderRow& get(const std::string& id);
|
||||
static bool has(const ProviderRow& row, unsigned feature) { return (row.features & feature) != 0; }
|
||||
static const char* evidence_name(Evidence e);
|
||||
// Comma list of the feature names a row claims ("chat,embeddings,models").
|
||||
static std::string feature_names(const ProviderRow& row);
|
||||
};
|
||||
|
||||
} // namespace molo::compat
|
||||
Vendored
+199
@@ -0,0 +1,199 @@
|
||||
#include "vendors/compat/services.h"
|
||||
|
||||
#include "clients/body/form.h"
|
||||
#include "vendors/compat/caps.h"
|
||||
|
||||
namespace molo::compat {
|
||||
|
||||
using Json = clients::body::Json;
|
||||
|
||||
namespace {
|
||||
|
||||
[[noreturn]] void bad(const Provider& p, const std::string& why) {
|
||||
throw llm::ApiError::invalid(p.client()->config().vendor, why);
|
||||
}
|
||||
|
||||
void check_model(const Provider& p, const std::string& model) {
|
||||
if (model.empty()) bad(p, "model is required (ids differ per provider)");
|
||||
if (model.size() > CompatCaps::kMaxModelChars) bad(p, "model id too long");
|
||||
}
|
||||
|
||||
std::string str_at(const Json& j, const char* key) {
|
||||
const Json* v = j.is_object() ? j.find(key) : nullptr;
|
||||
return v && v->is_string() ? v->as_string() : "";
|
||||
}
|
||||
|
||||
int int_at(const Json& j, const char* key) {
|
||||
const Json* v = j.is_object() ? j.find(key) : nullptr;
|
||||
return v && v->is_number() ? static_cast<int>(v->as_int()) : 0;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool ModelList::contains(const std::string& id) const {
|
||||
for (const ModelEntry& m : models)
|
||||
if (m.id == id) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
ModelList CompatModels::parse(const Json& doc) {
|
||||
const Json* arr = &doc;
|
||||
if (doc.is_object()) arr = doc.find("data");
|
||||
ModelList out;
|
||||
if (!arr || !arr->is_array()) throw llm::ApiError("compat", 502, "BAD_RESPONSE", "model list has no data array");
|
||||
for (const Json& m : arr->as_array()) {
|
||||
if (out.models.size() >= CompatCaps::kMaxModelsReturned) {
|
||||
out.truncated = true;
|
||||
break;
|
||||
}
|
||||
ModelEntry e;
|
||||
e.id = str_at(m, "id");
|
||||
if (e.id.empty()) e.id = str_at(m, "name");
|
||||
if (e.id.empty()) continue;
|
||||
e.owned_by = str_at(m, "owned_by");
|
||||
out.models.push_back(e);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
ModelList CompatModels::list() {
|
||||
provider_.require(Call::kModels);
|
||||
if (std::string(provider_.row().models_path).empty())
|
||||
bad(provider_, std::string(provider_.row().name) + " has no list endpoint under its base URL");
|
||||
return parse(client_->get(provider_.row().models_path));
|
||||
}
|
||||
|
||||
Json CompatEmbeddings::to_body(const std::string& model, const std::vector<std::string>& inputs,
|
||||
int dimensions) const {
|
||||
provider_.require(Call::kEmbeddings);
|
||||
check_model(provider_, model);
|
||||
if (inputs.empty()) bad(provider_, "at least one input is required");
|
||||
if (inputs.size() > CompatCaps::kMaxEmbeddingInputs) bad(provider_, "too many inputs");
|
||||
if (dimensions < 0 || dimensions > 65536) bad(provider_, "dimensions out of range");
|
||||
Json arr = Json::array();
|
||||
for (const std::string& s : inputs) {
|
||||
if (s.empty()) bad(provider_, "input must not be empty");
|
||||
if (s.size() > CompatCaps::kMaxEmbeddingInputChars) bad(provider_, "input too long");
|
||||
arr.push(Json(s));
|
||||
}
|
||||
Json body = Json::object();
|
||||
body.set("model", Json(model));
|
||||
body.set("input", arr);
|
||||
if (dimensions > 0) body.set("dimensions", Json(dimensions));
|
||||
return body;
|
||||
}
|
||||
|
||||
EmbeddingResult CompatEmbeddings::parse(const Json& doc) {
|
||||
EmbeddingResult out;
|
||||
out.model = str_at(doc, "model");
|
||||
const Json* data = doc.is_object() ? doc.find("data") : nullptr;
|
||||
if (!data || !data->is_array()) throw llm::ApiError("compat", 502, "BAD_RESPONSE", "embeddings response has no data");
|
||||
for (const Json& d : data->as_array()) {
|
||||
Embedding e;
|
||||
const Json* v = d.is_object() ? d.find("embedding") : nullptr;
|
||||
if (!v || !v->is_array()) throw llm::ApiError("compat", 502, "BAD_RESPONSE", "embedding is not an array");
|
||||
for (const Json& x : v->as_array()) e.vector.push_back(x.is_number() ? x.as_number() : 0.0);
|
||||
out.items.push_back(std::move(e));
|
||||
}
|
||||
if (const Json* u = doc.find("usage"); u && u->is_object()) {
|
||||
out.prompt_tokens = int_at(*u, "prompt_tokens");
|
||||
out.total_tokens = int_at(*u, "total_tokens");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
EmbeddingResult CompatEmbeddings::embed(const std::string& model, const std::vector<std::string>& inputs,
|
||||
int dimensions) {
|
||||
Json body = to_body(model, inputs, dimensions);
|
||||
return parse(client_->post(Calls::spec_of(Call::kEmbeddings).path, body));
|
||||
}
|
||||
|
||||
Json CompatImages::to_body(const ImageRequest& r) const {
|
||||
provider_.require(Call::kImages);
|
||||
check_model(provider_, r.model);
|
||||
if (r.prompt.empty() || r.prompt.size() > CompatCaps::kMaxPromptChars) bad(provider_, "prompt must be 1..32768 chars");
|
||||
if (r.n < 1 || r.n > CompatCaps::kMaxImages) bad(provider_, "n must be 1..10");
|
||||
if (!r.response_format.empty() && r.response_format != "url" && r.response_format != "b64_json")
|
||||
bad(provider_, "response_format must be url or b64_json");
|
||||
if (!r.size.empty()) {
|
||||
std::size_t x = r.size.find('x');
|
||||
bool ok = x != std::string::npos && x > 0 && x + 1 < r.size.size() && r.size.size() <= 16;
|
||||
for (std::size_t i = 0; ok && i < r.size.size(); ++i)
|
||||
ok = i == x || (r.size[i] >= '0' && r.size[i] <= '9');
|
||||
if (!ok) bad(provider_, "size must look like 1024x1024");
|
||||
}
|
||||
Json body = Json::object();
|
||||
body.set("model", Json(r.model));
|
||||
body.set("prompt", Json(r.prompt));
|
||||
body.set("n", Json(r.n));
|
||||
if (!r.size.empty()) body.set("size", Json(r.size));
|
||||
if (!r.response_format.empty()) body.set("response_format", Json(r.response_format));
|
||||
return body;
|
||||
}
|
||||
|
||||
std::vector<ImageItem> CompatImages::parse(const Json& doc) {
|
||||
const Json* data = doc.is_object() ? doc.find("data") : nullptr;
|
||||
if (!data || !data->is_array()) throw llm::ApiError("compat", 502, "BAD_RESPONSE", "images response has no data");
|
||||
std::vector<ImageItem> out;
|
||||
for (const Json& d : data->as_array()) {
|
||||
if (out.size() >= static_cast<std::size_t>(CompatCaps::kMaxImages)) break;
|
||||
ImageItem i;
|
||||
i.url = str_at(d, "url");
|
||||
i.b64_json = str_at(d, "b64_json");
|
||||
i.revised_prompt = str_at(d, "revised_prompt");
|
||||
out.push_back(std::move(i));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<ImageItem> CompatImages::generate(const ImageRequest& request) {
|
||||
Json body = to_body(request);
|
||||
return parse(client_->post(Calls::spec_of(Call::kImages).path, body));
|
||||
}
|
||||
|
||||
std::string CompatAudio::transcribe(const TranscribeRequest& r) {
|
||||
provider_.require(Call::kTranscribe);
|
||||
check_model(provider_, r.model);
|
||||
if (r.audio.empty()) bad(provider_, "audio is empty");
|
||||
if (r.audio.size() > CompatCaps::kMaxAudioBytes) bad(provider_, "audio over the byte cap");
|
||||
if (r.filename.empty() || r.filename.size() > CompatCaps::kMaxFilenameChars) bad(provider_, "filename must be 1..256 chars");
|
||||
if (r.filename.find_first_of("\"\r\n/\\") != std::string::npos) bad(provider_, "filename has forbidden characters");
|
||||
clients::body::MultipartBody form;
|
||||
form.add_file("file", r.filename, r.mime, r.audio);
|
||||
form.add_text("model", r.model);
|
||||
if (!r.language.empty()) form.add_text("language", r.language);
|
||||
if (!r.prompt.empty()) form.add_text("prompt", r.prompt);
|
||||
form.add_text("response_format", "json");
|
||||
Json doc = client_->post_multipart(Calls::spec_of(Call::kTranscribe).path, form);
|
||||
return str_at(doc, "text");
|
||||
}
|
||||
|
||||
Json CompatAudio::speech_body(const SpeechRequest& r) const {
|
||||
provider_.require(Call::kSpeech);
|
||||
check_model(provider_, r.model);
|
||||
if (r.input.empty() || r.input.size() > CompatCaps::kMaxSpeechChars) bad(provider_, "input must be 1..4096 chars");
|
||||
if (r.voice.empty()) bad(provider_, "voice is required (names differ per provider)");
|
||||
static const char* formats[] = {"mp3", "opus", "aac", "flac", "wav", "pcm"};
|
||||
if (!r.format.empty()) {
|
||||
bool ok = false;
|
||||
for (const char* f : formats) ok = ok || r.format == f;
|
||||
if (!ok) bad(provider_, "format must be mp3, opus, aac, flac, wav or pcm");
|
||||
}
|
||||
Json body = Json::object();
|
||||
body.set("model", Json(r.model));
|
||||
body.set("input", Json(r.input));
|
||||
body.set("voice", Json(r.voice));
|
||||
if (!r.format.empty()) body.set("response_format", Json(r.format));
|
||||
return body;
|
||||
}
|
||||
|
||||
std::string CompatAudio::speak(const SpeechRequest& r, std::string& content_type) {
|
||||
Json body = speech_body(r);
|
||||
std::string text = body.dump();
|
||||
llm::RawResponse res = client_->request_raw(clients::Method::kPost, Calls::spec_of(Call::kSpeech).path, &text,
|
||||
"application/json");
|
||||
content_type = res.content_type;
|
||||
return res.body;
|
||||
}
|
||||
|
||||
} // namespace molo::compat
|
||||
Vendored
+104
@@ -0,0 +1,104 @@
|
||||
#pragma once
|
||||
|
||||
// The non-chat endpoints of a registry row. Each service refuses a call the
|
||||
// row does not claim (Provider::require) before any network traffic, and
|
||||
// validates every input against CompatCaps.
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "clients/body/json.h"
|
||||
#include "vendors/compat/provider.h"
|
||||
|
||||
namespace molo::compat {
|
||||
|
||||
struct ModelEntry {
|
||||
std::string id;
|
||||
std::string owned_by;
|
||||
};
|
||||
|
||||
struct ModelList {
|
||||
std::vector<ModelEntry> models;
|
||||
bool truncated = false; // more entries than CompatCaps::kMaxModelsReturned
|
||||
bool contains(const std::string& id) const;
|
||||
};
|
||||
|
||||
// GET /models. Accepts {"data":[...]} and a bare array (some providers return one).
|
||||
class CompatModels : public ProviderService {
|
||||
public:
|
||||
using ProviderService::ProviderService;
|
||||
ModelList list();
|
||||
static ModelList parse(const clients::body::Json& doc);
|
||||
};
|
||||
|
||||
struct Embedding {
|
||||
std::vector<double> vector;
|
||||
};
|
||||
|
||||
struct EmbeddingResult {
|
||||
std::string model;
|
||||
std::vector<Embedding> items;
|
||||
int prompt_tokens = 0;
|
||||
int total_tokens = 0;
|
||||
};
|
||||
|
||||
class CompatEmbeddings : public ProviderService {
|
||||
public:
|
||||
using ProviderService::ProviderService;
|
||||
// dimensions: 0 = provider default.
|
||||
EmbeddingResult embed(const std::string& model, const std::vector<std::string>& inputs, int dimensions = 0);
|
||||
clients::body::Json to_body(const std::string& model, const std::vector<std::string>& inputs,
|
||||
int dimensions) const;
|
||||
static EmbeddingResult parse(const clients::body::Json& doc);
|
||||
};
|
||||
|
||||
struct ImageItem {
|
||||
std::string url;
|
||||
std::string b64_json;
|
||||
std::string revised_prompt;
|
||||
};
|
||||
|
||||
struct ImageRequest {
|
||||
std::string model;
|
||||
std::string prompt;
|
||||
int n = 1;
|
||||
std::string size; // "1024x1024"; empty = provider default
|
||||
std::string response_format; // "", "url" or "b64_json"
|
||||
};
|
||||
|
||||
class CompatImages : public ProviderService {
|
||||
public:
|
||||
using ProviderService::ProviderService;
|
||||
// Paid call: the caller decides (the demo keeps it a dry run).
|
||||
std::vector<ImageItem> generate(const ImageRequest& request);
|
||||
clients::body::Json to_body(const ImageRequest& request) const;
|
||||
static std::vector<ImageItem> parse(const clients::body::Json& doc);
|
||||
};
|
||||
|
||||
struct TranscribeRequest {
|
||||
std::string model;
|
||||
std::string filename;
|
||||
std::string mime = "application/octet-stream";
|
||||
std::string audio; // raw bytes
|
||||
std::string language;
|
||||
std::string prompt;
|
||||
};
|
||||
|
||||
struct SpeechRequest {
|
||||
std::string model;
|
||||
std::string input;
|
||||
std::string voice;
|
||||
std::string format; // "", mp3, opus, aac, flac, wav, pcm
|
||||
};
|
||||
|
||||
class CompatAudio : public ProviderService {
|
||||
public:
|
||||
using ProviderService::ProviderService;
|
||||
std::string transcribe(const TranscribeRequest& request); // returns the text
|
||||
// Audio bytes; the content type is returned through `content_type`.
|
||||
std::string speak(const SpeechRequest& request, std::string& content_type);
|
||||
clients::body::Json speech_body(const SpeechRequest& request) const;
|
||||
};
|
||||
|
||||
} // namespace molo::compat
|
||||
+239
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check src/vendors/compat against OpenAI's OpenAPI document and its own SPEC.
|
||||
|
||||
Every compat call is an OpenAI operation (providers differ only in base URL,
|
||||
auth and quirks). Fails (exit 1) when
|
||||
|
||||
call a row of calls.cpp has a wrong method, path, operation id, request
|
||||
schema, request body field or response field (fields are resolved
|
||||
through $ref, allOf, oneOf and anyOf);
|
||||
coverage an operation of the spec in the families this module touches
|
||||
(chat completions, embeddings, models, images, audio) is neither
|
||||
in the table nor in EXCLUDED;
|
||||
registry the provider ids / base URLs / credential names of registry.cpp and
|
||||
the table of SPEC.md section 4 differ in either direction.
|
||||
|
||||
The registry check needs no network: pass --local to run only it.
|
||||
Python 3 stdlib only.
|
||||
|
||||
python3 -I src/vendors/compat/tools/check_spec.py [--local] [path/to/openapi.json]
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.join(HERE, "..")
|
||||
SPEC_SHA = "8243cc8fa6f04430fb1f1a6a2cb926fbeb4f7f26" # same pin as src/openai/core
|
||||
URL = "https://raw.githubusercontent.com/openai/openai-openapi/%s/openapi.json" % SPEC_SHA
|
||||
|
||||
FAMILIES = ("/chat/completions", "/embeddings", "/models", "/images", "/audio", "/completions", "/moderations")
|
||||
EXCLUDED = {
|
||||
"listChatCompletions": "stored completions are an OpenAI product feature, not portable",
|
||||
"getChatCompletion": "stored completions",
|
||||
"updateChatCompletion": "stored completions",
|
||||
"deleteChatCompletion": "stored completions",
|
||||
"getChatCompletionMessages": "stored completions",
|
||||
"createCompletion": "legacy completions; not claimed by any registry row",
|
||||
"createModeration": "no registry row documents a moderation endpoint",
|
||||
"createImageEdit": "multipart image editing; no registry row documents it",
|
||||
"createImageVariation": "multipart image variations; no registry row documents it",
|
||||
"createTranslation": "audio translation; no registry row documents it",
|
||||
"createVoiceConsent": "OpenAI custom voices only",
|
||||
"listVoiceConsents": "OpenAI custom voices only",
|
||||
"getVoiceConsent": "OpenAI custom voices only",
|
||||
"updateVoiceConsent": "OpenAI custom voices only",
|
||||
"deleteVoiceConsent": "OpenAI custom voices only",
|
||||
"createVoice": "OpenAI custom voices only",
|
||||
"deleteVoice": "OpenAI custom voices only",
|
||||
"retrieveModel": "no registry row documents GET /models/{id}; models are listed",
|
||||
"deleteModel": "OpenAI fine-tune deletion only",
|
||||
}
|
||||
STR = re.compile(r'"((?:[^"\\]|\\.)*)"')
|
||||
|
||||
|
||||
def string_runs(block):
|
||||
"""String literals of a C++ block with adjacent literals concatenated."""
|
||||
out, last_end, cur = [], None, None
|
||||
for m in STR.finditer(block):
|
||||
if last_end is not None and block[last_end:m.start()].strip() == "":
|
||||
cur += m.group(1)
|
||||
else:
|
||||
if cur is not None:
|
||||
out.append(cur)
|
||||
cur = m.group(1)
|
||||
last_end = m.end()
|
||||
if cur is not None:
|
||||
out.append(cur)
|
||||
return out
|
||||
|
||||
|
||||
def calls_rows():
|
||||
text = open(os.path.join(ROOT, "calls.cpp")).read()
|
||||
a = text.index("const CallSpec kCalls[]")
|
||||
b = text.index("};", a)
|
||||
rows = []
|
||||
for blk in re.findall(r"\{Call::k\w+.*?Feature::k\w+, \"\w+\"\}", text[a:b], re.S):
|
||||
rows.append(string_runs(blk))
|
||||
return rows # method, path, op, request, fields, response, fields_read, feature_name
|
||||
|
||||
|
||||
class Spec:
|
||||
def __init__(self, doc):
|
||||
self.doc = doc
|
||||
|
||||
def schema(self, name):
|
||||
return self.doc["components"]["schemas"][name]
|
||||
|
||||
def deref(self, node):
|
||||
while isinstance(node, dict) and "$ref" in node:
|
||||
node = self.schema(node["$ref"].rsplit("/", 1)[1])
|
||||
return node
|
||||
|
||||
def props(self, node, seen=None):
|
||||
seen = seen or set()
|
||||
out = set()
|
||||
if isinstance(node, dict) and "$ref" in node:
|
||||
ref = node["$ref"]
|
||||
if ref in seen:
|
||||
return out
|
||||
seen.add(ref)
|
||||
return self.props(self.deref(node), seen)
|
||||
if not isinstance(node, dict):
|
||||
return out
|
||||
out |= set(node.get("properties", {}))
|
||||
for key in ("allOf", "oneOf", "anyOf"):
|
||||
for sub in node.get(key, []):
|
||||
out |= self.props(sub, seen)
|
||||
return out
|
||||
|
||||
def schema_name_of(self, content, want=""):
|
||||
ref = content.get("schema", {})
|
||||
if "$ref" in ref:
|
||||
return ref["$ref"].rsplit("/", 1)[1]
|
||||
# a union (anyOf / oneOf): the wanted name may be one member
|
||||
for key in ("anyOf", "oneOf"):
|
||||
for sub in ref.get(key, []):
|
||||
if "$ref" in sub and sub["$ref"].rsplit("/", 1)[1] == want:
|
||||
return want
|
||||
return ""
|
||||
|
||||
|
||||
def check_calls(spec, bad):
|
||||
rows = calls_rows()
|
||||
declared = set()
|
||||
for method, path, op, req, fields, resp, read, _feat in rows:
|
||||
node = spec.doc["paths"].get(path, {}).get(method.lower())
|
||||
if not node:
|
||||
bad.append("%s %s: not in the spec" % (method, path))
|
||||
continue
|
||||
declared.add(op)
|
||||
if node["operationId"] != op:
|
||||
bad.append("%s %s: operationId %s != %s" % (method, path, node["operationId"], op))
|
||||
body = node.get("requestBody", {}).get("content", {})
|
||||
if req:
|
||||
content = body.get("application/json") or body.get("multipart/form-data") or {}
|
||||
if spec.schema_name_of(content) != req:
|
||||
bad.append("%s %s: request schema %s != %s" % (method, path, spec.schema_name_of(content), req))
|
||||
known = spec.props(content.get("schema", {}))
|
||||
for f in filter(None, fields.split(",")):
|
||||
if f not in known:
|
||||
bad.append("%s %s: body field %s not in %s" % (method, path, f, req))
|
||||
if resp:
|
||||
ok = node["responses"]["200"]["content"]["application/json"]
|
||||
if spec.schema_name_of(ok, resp) != resp:
|
||||
bad.append("%s %s: response schema %s != %s" % (method, path, spec.schema_name_of(ok), resp))
|
||||
known = spec.props(ok.get("schema", {}))
|
||||
if not known:
|
||||
known = spec.props({"$ref": "#/components/schemas/" + resp})
|
||||
for f in filter(None, read.split(",")):
|
||||
if f not in known:
|
||||
bad.append("%s %s: response field %s not in %s" % (method, path, f, resp))
|
||||
for path, item in spec.doc["paths"].items():
|
||||
if not path.startswith(FAMILIES):
|
||||
continue
|
||||
for m, node in item.items():
|
||||
if m not in ("get", "post", "delete"):
|
||||
continue
|
||||
op = node["operationId"]
|
||||
if op not in declared and op not in EXCLUDED:
|
||||
bad.append("spec operation %s %s (%s) is neither in the table nor excluded" % (m.upper(), path, op))
|
||||
for op in EXCLUDED:
|
||||
if op in declared:
|
||||
bad.append("%s is both declared and excluded" % op)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def registry_rows():
|
||||
text = open(os.path.join(ROOT, "registry.cpp")).read()
|
||||
a = text.index("const ProviderRow kRows[]")
|
||||
b = text.index("constexpr std::size_t kRowCount")
|
||||
out = []
|
||||
for blk in re.findall(r'\n \{"[a-z0-9]+",.*?(?=\n \{"|\n //|\n\};)', text[a:b], re.S):
|
||||
s = string_runs(blk)
|
||||
out.append({"id": s[0], "name": s[1], "base": s[2], "key": s[3], "base_env": s[4], "extra_env": s[5],
|
||||
"models": s[6], "doc": s[7]})
|
||||
return out
|
||||
|
||||
|
||||
def spec_md_rows():
|
||||
text = open(os.path.join(ROOT, "SPEC.md")).read()
|
||||
a = text.index("## 4. Provider table")
|
||||
b = text.index("\n## ", a + 5)
|
||||
rows = {}
|
||||
for line in text[a:b].splitlines():
|
||||
cells = [c.strip().strip("`") for c in line.strip().strip("|").split("|")]
|
||||
if line.startswith("|") and len(cells) >= 5 and re.fullmatch(r"[a-z0-9]+", cells[0]) and cells[0] != "id":
|
||||
rows[cells[0]] = cells
|
||||
return rows
|
||||
|
||||
|
||||
def check_registry(bad):
|
||||
reg = registry_rows()
|
||||
md = spec_md_rows()
|
||||
ids = [r["id"] for r in reg]
|
||||
if len(ids) != len(set(ids)):
|
||||
bad.append("registry.cpp has duplicate ids")
|
||||
for r in reg:
|
||||
cells = md.get(r["id"])
|
||||
if not cells:
|
||||
bad.append("provider %s is in registry.cpp but not in SPEC.md section 4" % r["id"])
|
||||
continue
|
||||
line = " ".join(cells)
|
||||
for what in ("base", "key", "doc"):
|
||||
if r[what] not in line:
|
||||
bad.append("provider %s: %s '%s' differs between registry.cpp and SPEC.md" % (r["id"], what, r[what]))
|
||||
for i in md:
|
||||
if i not in ids:
|
||||
bad.append("provider %s is in SPEC.md section 4 but not in registry.cpp" % i)
|
||||
return len(reg)
|
||||
|
||||
|
||||
def main():
|
||||
args = [a for a in sys.argv[1:] if not a.startswith("--")]
|
||||
local = "--local" in sys.argv
|
||||
bad = []
|
||||
n_reg = check_registry(bad)
|
||||
n_calls = 0
|
||||
if not local:
|
||||
if args:
|
||||
doc = json.load(open(args[0]))
|
||||
else:
|
||||
req = urllib.request.Request(URL, headers={"User-Agent": "molo-compat-spec-check/1"})
|
||||
with urllib.request.urlopen(req, timeout=120) as r:
|
||||
doc = json.loads(r.read().decode("utf-8"))
|
||||
n_calls = check_calls(Spec(doc), bad)
|
||||
print("openapi %s (%s)" % (doc["info"]["version"], SPEC_SHA[:12]))
|
||||
print("registry rows: %d, call rows checked: %d" % (n_reg, n_calls))
|
||||
if bad:
|
||||
print("FAIL")
|
||||
for b in bad:
|
||||
print(" " + b)
|
||||
sys.exit(1)
|
||||
print("OK")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,50 @@
|
||||
// Routes: GET /ai/compat (page), /status, /providers, /calls (no credential needed)
|
||||
#include <cstdlib>
|
||||
|
||||
#include "support/Assert.hpp"
|
||||
#include "vendors/compat/registry.h"
|
||||
|
||||
using namespace molo::example::test;
|
||||
namespace cp = molo::compat;
|
||||
|
||||
int main() {
|
||||
setenv("MOLO_ENV_FILES", "off", 1);
|
||||
for (std::size_t i = 0; i < cp::Registry::count(); ++i) {
|
||||
const cp::ProviderRow& r = cp::Registry::at(i);
|
||||
unsetenv(r.key_env);
|
||||
if (r.base_env[0]) unsetenv(r.base_env);
|
||||
if (r.extra_env[0]) unsetenv(r.extra_env);
|
||||
}
|
||||
Fixture f;
|
||||
auto page = f.client->get("/ai/compat");
|
||||
check(page.status == 200, "page 200");
|
||||
check(contains(page.headers.get("Content-Type"), "text/html"), "html");
|
||||
check(contains(page.body, "OpenAI-compatible providers") && contains(page.body, "viewport"), "title and mobile viewport");
|
||||
check(contains(page.body, "/embedded/ai_compat.js"), "script wired");
|
||||
check(f.client->post("/ai/compat", "{}", "application/json").status == 405, "post page is 405");
|
||||
|
||||
auto st = f.client->get("/ai/compat/status");
|
||||
check(st.status == 200 && contains(st.body, "\"configured\":false"), "status without a key");
|
||||
check(contains(st.body, "GROQ_API_KEY") && contains(st.body, "CLOUDFLARE_API_TOKEN"), "status names credential envs");
|
||||
check(contains(st.body, "\"configured_providers\":0"), "no provider configured");
|
||||
check(f.client->post("/ai/compat/status", "{}", "application/json").status == 405, "post status is 405");
|
||||
|
||||
auto prov = f.client->get("/ai/compat/providers");
|
||||
check(prov.status == 200, "providers 200");
|
||||
for (std::size_t i = 0; i < cp::Registry::count(); ++i) {
|
||||
const cp::ProviderRow& r = cp::Registry::at(i);
|
||||
check(contains(prov.body, std::string("\"id\":\"") + r.id + "\""), std::string("providers lists ") + r.id);
|
||||
}
|
||||
check(contains(prov.body, "https://api.groq.com/openai/v1") && contains(prov.body, "docs+probe") &&
|
||||
contains(prov.body, "needs GROQ_API_KEY") == false && contains(prov.body, "\"missing_env\":\"GROQ_API_KEY\""),
|
||||
"provider rows carry base url, evidence and missing env");
|
||||
check(contains(prov.body, "\"missing_env\":\"OLLAMA_BASE_URL\""), "local row names its switch");
|
||||
check(contains(prov.body, "\"missing_env\":\"CLOUDFLARE_ACCOUNT_ID\"") || contains(prov.body, "\"missing_env\":\"CLOUDFLARE_API_TOKEN\""),
|
||||
"cloudflare names a missing env");
|
||||
|
||||
auto calls = f.client->get("/ai/compat/calls");
|
||||
check(calls.status == 200 && contains(calls.body, "/chat/completions") && contains(calls.body, "createEmbedding") &&
|
||||
contains(calls.body, "/audio/speech"),
|
||||
"call table listed");
|
||||
return finish("ai/compat/page");
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
// Routes: POST /ai/compat/* (405, 422, dry runs, 503 without a credential)
|
||||
#include <cstdlib>
|
||||
|
||||
#include "support/Assert.hpp"
|
||||
#include "vendors/compat/registry.h"
|
||||
|
||||
using namespace molo::example::test;
|
||||
namespace cp = molo::compat;
|
||||
|
||||
int main() {
|
||||
setenv("MOLO_ENV_FILES", "off", 1);
|
||||
for (std::size_t i = 0; i < cp::Registry::count(); ++i) {
|
||||
const cp::ProviderRow& r = cp::Registry::at(i);
|
||||
unsetenv(r.key_env);
|
||||
if (r.base_env[0]) unsetenv(r.base_env);
|
||||
if (r.extra_env[0]) unsetenv(r.extra_env);
|
||||
}
|
||||
Fixture f;
|
||||
auto post = [&](const std::string& route, const std::string& body) {
|
||||
return f.client->post("/ai/compat/" + route, body, "application/json");
|
||||
};
|
||||
const char* all[] = {"chat", "models", "embeddings", "images", "transcribe", "speech"};
|
||||
for (const char* r : all) {
|
||||
std::string route = std::string("/ai/compat/") + r;
|
||||
check(f.client->get(route).status == 405, route + " get is 405");
|
||||
check(post(r, "not json").status == 422, route + " bad json is 422");
|
||||
check(post(r, "[1]").status == 422, route + " non-object is 422");
|
||||
check(post(r, "{}").status == 422, route + " without provider is 422");
|
||||
check(post(r, "{\"provider\":\"nope\"}").status == 422, route + " unknown provider is 422");
|
||||
check(post(r, "{\"provider\":5}").status == 422, route + " non-string provider is 422");
|
||||
}
|
||||
|
||||
// 422: input rules run before anything else
|
||||
check(post("chat", "{\"provider\":\"groq\"}").status == 422, "chat needs prompt");
|
||||
check(post("chat", "{\"provider\":\"groq\",\"prompt\":\"hi\"}").status == 422, "chat needs a model");
|
||||
check(post("chat", "{\"provider\":\"groq\",\"prompt\":\"hi\",\"model\":\"m\",\"max_tokens\":-1}").status == 422, "max_tokens -1");
|
||||
check(post("chat", "{\"provider\":\"groq\",\"prompt\":\"hi\",\"model\":\"m\",\"extra\":{\"logprobs\":true}}").status == 422,
|
||||
"groq quirk logprobs is 422");
|
||||
check(post("chat", "{\"provider\":\"groq\",\"prompt\":\"hi\",\"model\":\"m\",\"extra\":{\"n\":2}}").status == 422, "groq quirk n is 422");
|
||||
check(post("chat", "{\"provider\":\"groq\",\"prompt\":\"hi\",\"model\":\"m\",\"temperature\":3}").status == 422, "temperature 3");
|
||||
check(post("models", "{\"provider\":\"dashscope\"}").status == 422, "row without a list endpoint is 422");
|
||||
check(post("embeddings", "{\"provider\":\"cerebras\",\"model\":\"m\",\"input\":\"x\"}").status == 422, "cerebras has no embeddings");
|
||||
check(post("embeddings", "{\"provider\":\"nvidia\",\"model\":\"m\"}").status == 422, "embeddings need input");
|
||||
check(post("embeddings", "{\"provider\":\"nvidia\",\"model\":\"m\",\"input\":[1]}").status == 422, "input must be strings");
|
||||
check(post("images", "{\"provider\":\"together\",\"model\":\"m\",\"prompt\":\"x\",\"n\":11}").status == 422, "n over 10");
|
||||
check(post("images", "{\"provider\":\"groq\",\"model\":\"m\",\"prompt\":\"x\"}").status == 422, "groq has no images");
|
||||
check(post("transcribe", "{\"provider\":\"groq\",\"model\":\"m\",\"filename\":\"a.wav\",\"data_base64\":\"@@\"}").status == 422, "bad base64");
|
||||
check(post("transcribe", "{\"provider\":\"cerebras\",\"model\":\"m\",\"filename\":\"a.wav\",\"data_base64\":\"aGk=\"}").status == 422,
|
||||
"cerebras has no transcription");
|
||||
check(post("speech", "{\"provider\":\"together\",\"model\":\"m\",\"input\":\"x\"}").status == 422, "speech needs a voice");
|
||||
check(post("speech", "{\"provider\":\"together\",\"model\":\"m\",\"input\":\"x\",\"voice\":\"v\",\"format\":\"ogg\"}").status == 422, "bad format");
|
||||
|
||||
// dry runs: paid routes show the exact request and the URL
|
||||
auto dry = post("chat", "{\"provider\":\"groq\",\"prompt\":\"hi\",\"model\":\"llama\",\"max_tokens\":8}");
|
||||
check(dry.status == 200 && contains(dry.body, "\"dry_run\":true") && contains(dry.body, "\"model\":\"llama\"") &&
|
||||
contains(dry.body, "https://api.groq.com/openai/v1/chat/completions"),
|
||||
"chat is a dry run without confirm");
|
||||
check(contains(post("embeddings", "{\"provider\":\"nvidia\",\"model\":\"m\",\"input\":\"x\"}").body, "/embeddings"), "embeddings dry run");
|
||||
check(contains(post("images", "{\"provider\":\"together\",\"model\":\"m\",\"prompt\":\"x\"}").body, "/images/generations"), "images dry run");
|
||||
check(contains(post("transcribe", "{\"provider\":\"groq\",\"model\":\"m\",\"filename\":\"a.wav\",\"data_base64\":\"aGk=\"}").body,
|
||||
"\"bytes\":2"),
|
||||
"transcribe dry run");
|
||||
check(contains(post("speech", "{\"provider\":\"together\",\"model\":\"m\",\"input\":\"x\",\"voice\":\"v\"}").body, "/audio/speech"),
|
||||
"speech dry run");
|
||||
check(contains(post("chat", "{\"provider\":\"ollama\",\"prompt\":\"hi\",\"model\":\"m\"}").body, "http://localhost:11434/v1/chat/completions"),
|
||||
"local provider dry run shows its default URL");
|
||||
|
||||
// 503: the free route and confirmed routes need the provider's credential
|
||||
auto models = post("models", "{\"provider\":\"groq\"}");
|
||||
check(models.status == 503 && contains(models.body, "compat_not_configured") && contains(models.body, "GROQ_API_KEY"),
|
||||
"models without key is 503 and names the env");
|
||||
check(post("chat", "{\"provider\":\"groq\",\"prompt\":\"hi\",\"model\":\"m\",\"confirm\":true}").status == 503, "confirmed chat without key is 503");
|
||||
check(post("embeddings", "{\"provider\":\"nvidia\",\"model\":\"m\",\"input\":\"x\",\"confirm\":true}").status == 503, "confirmed embeddings without key is 503");
|
||||
check(post("images", "{\"provider\":\"together\",\"model\":\"m\",\"prompt\":\"x\",\"confirm\":true}").status == 503, "confirmed images without key is 503");
|
||||
check(post("transcribe", "{\"provider\":\"groq\",\"model\":\"m\",\"filename\":\"a.wav\",\"data_base64\":\"aGk=\",\"confirm\":true}").status == 503,
|
||||
"confirmed transcribe without key is 503");
|
||||
check(post("speech", "{\"provider\":\"together\",\"model\":\"m\",\"input\":\"x\",\"voice\":\"v\",\"confirm\":true}").status == 503,
|
||||
"confirmed speech without key is 503");
|
||||
auto local = post("models", "{\"provider\":\"ollama\"}");
|
||||
check(local.status == 503 && contains(local.body, "OLLAMA_BASE_URL"), "local provider without base env is 503");
|
||||
auto cf = post("chat", "{\"provider\":\"cloudflare\",\"prompt\":\"hi\",\"model\":\"m\",\"confirm\":true}");
|
||||
check(cf.status == 503, "cloudflare without account is 503");
|
||||
return finish("ai/compat/routes");
|
||||
}
|
||||
Vendored
+63
@@ -0,0 +1,63 @@
|
||||
#pragma once
|
||||
|
||||
// Shared rig for the compat tests: a Provider over a MockTransport with a
|
||||
// fake key. The mock only records the requests the real client builds;
|
||||
// replies are documented examples taken verbatim from OpenAI's API reference
|
||||
// (the shape every row shares) and labelled FIXTURE where they appear.
|
||||
// Nothing here verifies a provider's behavior (see SPEC.md ## Unverified).
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "clients/transports/mock_transport.h"
|
||||
#include "framework.h"
|
||||
#include "vendors/compat/provider.h"
|
||||
|
||||
namespace compat_test {
|
||||
|
||||
using molo::clients::ClientResponse;
|
||||
using molo::clients::HeaderMap;
|
||||
using molo::clients::Method;
|
||||
using molo::clients::transports::MockTransport;
|
||||
using Json = molo::clients::body::Json;
|
||||
|
||||
inline ClientResponse reply(const std::string& body, int status = 200,
|
||||
const std::string& type = "application/json") {
|
||||
HeaderMap h;
|
||||
h.set("Content-Type", type);
|
||||
return ClientResponse(status, h, body);
|
||||
}
|
||||
|
||||
inline ClientResponse sse(const std::string& body) { return reply(body, 200, "text/event-stream"); }
|
||||
|
||||
inline const char* kKey = "sk-TESTKEY-DO-NOT-LEAK";
|
||||
|
||||
inline bool has(const std::string& hay, const std::string& needle) { return hay.find(needle) != std::string::npos; }
|
||||
|
||||
// "https://host/a/b" -> "/a/b"
|
||||
inline std::string path_of(const std::string& url) {
|
||||
std::size_t scheme = url.find("://");
|
||||
std::size_t slash = url.find('/', scheme == std::string::npos ? 0 : scheme + 3);
|
||||
return slash == std::string::npos ? "" : url.substr(slash);
|
||||
}
|
||||
|
||||
struct Rig {
|
||||
MockTransport* transport = nullptr;
|
||||
molo::compat::Provider provider;
|
||||
|
||||
explicit Rig(const std::string& id, const std::string& key = kKey, const std::string& base_override = "",
|
||||
const std::string& account = "acct123")
|
||||
: provider(make(id, key, base_override, account, &transport)) {}
|
||||
|
||||
std::string path(const char* suffix) const { return path_of(provider.base_url()) + suffix; }
|
||||
|
||||
private:
|
||||
static molo::compat::Provider make(const std::string& id, const std::string& key, const std::string& base,
|
||||
const std::string& account, MockTransport** out) {
|
||||
auto t = std::make_unique<MockTransport>();
|
||||
*out = t.get();
|
||||
return molo::compat::Provider::with(id, std::move(t), key, base, account);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace compat_test
|
||||
+291
@@ -0,0 +1,291 @@
|
||||
// CompatChatModel: request mapping, row quirks, parsing and streaming. Replies
|
||||
// are documented OpenAI examples (FIXTURE) or transcripts built from the
|
||||
// documented chunk shape (LABELLED, not recorded provider responses).
|
||||
#include "support.h"
|
||||
#include "vendors/compat/chat_model.h"
|
||||
|
||||
using namespace molo::compat;
|
||||
using namespace compat_test;
|
||||
using molo::llm::ApiError;
|
||||
using molo::llm::ChatChunk;
|
||||
using molo::llm::ChatRequest;
|
||||
using molo::llm::Message;
|
||||
|
||||
namespace {
|
||||
|
||||
ChatRequest base() {
|
||||
ChatRequest r;
|
||||
r.model = "m1";
|
||||
r.messages.push_back(Message::user("hi"));
|
||||
return r;
|
||||
}
|
||||
|
||||
// FIXTURE: OpenAI API reference, createChatCompletion streaming example chunks.
|
||||
const char* kStream =
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"Hello\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"!\"},\"logprobs\":null,\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{},\"logprobs\":null,\"finish_reason\":\"stop\"}]}\n\n"
|
||||
"data: {\"id\":\"chatcmpl-123\",\"object\":\"chat.completion.chunk\",\"created\":1694268190,\"model\":\"gpt-4o-mini\",\"choices\":[],\"usage\":{\"prompt_tokens\":9,\"completion_tokens\":2,\"total_tokens\":11}}\n\n"
|
||||
"data: [DONE]\n\n";
|
||||
|
||||
// LABELLED: tool call fragments in the documented delta shape, with a reasoning field.
|
||||
const char* kToolStream =
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{\"reasoning\":\"thinking\"},\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]},\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"{\\\"city\\\":\"}}]},\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"Rome\\\"}\"}}]},\"finish_reason\":null}]}\n\n"
|
||||
"data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\n"
|
||||
"data: [DONE]\n\n";
|
||||
} // namespace
|
||||
|
||||
TEST(compat_chat, body_maps_the_neutral_request) {
|
||||
Rig rig("together");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
r.system = "be brief";
|
||||
r.max_tokens = 16;
|
||||
r.temperature = 0.5;
|
||||
r.top_p = 0.9;
|
||||
r.stop = {"END"};
|
||||
r.reasoning_effort = "low";
|
||||
r.json_mode = true;
|
||||
Json b = m.to_body(r, false);
|
||||
ASSERT_EQ(b.find("model")->as_string(), std::string("m1"));
|
||||
ASSERT_EQ(b.find("messages")->as_array().size(), 2u);
|
||||
ASSERT_EQ(b.find("messages")->as_array()[0].find("role")->as_string(), std::string("system"));
|
||||
ASSERT_EQ(b.find("max_tokens")->as_int(), 16L);
|
||||
ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_object"));
|
||||
ASSERT_EQ(b.find("reasoning_effort")->as_string(), std::string("low"));
|
||||
ASSERT_TRUE(b.find("stream") == nullptr);
|
||||
ASSERT_TRUE(m.to_body(r, true).find("stream")->as_bool());
|
||||
}
|
||||
|
||||
TEST(compat_chat, schema_tools_and_tool_choice) {
|
||||
Rig rig("together");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
r.response_schema = Json::parse("{\"type\":\"object\"}");
|
||||
r.tools.push_back({"get_weather", "weather", Json::parse("{\"type\":\"object\"}")});
|
||||
r.tool_choice = "get_weather";
|
||||
Json b = m.to_body(r, false);
|
||||
ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_schema"));
|
||||
ASSERT_TRUE(b.find("response_format")->find("json_schema")->find("strict")->as_bool());
|
||||
ASSERT_EQ(b.find("tool_choice")->find("function")->find("name")->as_string(), std::string("get_weather"));
|
||||
r.tool_choice = "required";
|
||||
ASSERT_EQ(m.to_body(r, false).find("tool_choice")->as_string(), std::string("required"));
|
||||
ChatRequest none = base();
|
||||
none.tool_choice = "auto";
|
||||
ASSERT_THROW(m.to_body(none, false), ApiError); // tool_choice needs tools
|
||||
ChatRequest both = base();
|
||||
both.json_mode = true;
|
||||
both.response_schema = Json::parse("{}");
|
||||
ASSERT_THROW(m.to_body(both, false), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, tool_round_trip_messages_and_images) {
|
||||
Rig rig("together");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
Message a;
|
||||
a.role = molo::llm::Role::kAssistant;
|
||||
molo::llm::ToolCall tc;
|
||||
tc.id = "call_1";
|
||||
tc.name = "f";
|
||||
tc.arguments = Json::parse("{\"a\":1}");
|
||||
a.parts.push_back(molo::llm::ContentPart::of_call(tc));
|
||||
r.messages.push_back(a);
|
||||
Message t;
|
||||
t.role = molo::llm::Role::kTool;
|
||||
t.parts.push_back(molo::llm::ContentPart::of_result({"call_1", "42", false}));
|
||||
r.messages.push_back(t);
|
||||
Message u;
|
||||
u.role = molo::llm::Role::kUser;
|
||||
u.parts.push_back(molo::llm::ContentPart::of_text("look"));
|
||||
u.parts.push_back(molo::llm::ContentPart::of_image({"image/png", "abc", ""}));
|
||||
r.messages.push_back(u);
|
||||
Json b = m.to_body(r, false);
|
||||
const auto& msgs = b.find("messages")->as_array();
|
||||
ASSERT_EQ(msgs[1].find("tool_calls")->as_array()[0].find("function")->find("arguments")->as_string(), std::string("{\"a\":1}"));
|
||||
ASSERT_TRUE(msgs[1].find("content")->is_null());
|
||||
ASSERT_EQ(msgs[2].find("tool_call_id")->as_string(), std::string("call_1"));
|
||||
ASSERT_TRUE(msgs[3].find("content")->is_array());
|
||||
ASSERT_TRUE(has(msgs[3].find("content")->as_array()[1].find("image_url")->find("url")->as_string(), "data:image/png;base64,"));
|
||||
Message empty_tool;
|
||||
empty_tool.role = molo::llm::Role::kTool;
|
||||
ChatRequest bad = base();
|
||||
bad.messages.push_back(empty_tool);
|
||||
ASSERT_THROW(m.to_body(bad, false), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, input_rules_run_before_any_traffic) {
|
||||
Rig rig("together");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
r.model = "";
|
||||
ASSERT_THROW(m.generate(r), ApiError); // no default model
|
||||
r = base();
|
||||
r.messages.clear();
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.max_tokens = 0;
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.temperature = 2.5;
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.top_p = 0.0;
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.stop = std::vector<std::string>(17, "x");
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.extra = Json::parse("{\"messages\":[]}");
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.extra = Json::parse("[1]");
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
r = base();
|
||||
r.model = std::string(300, 'x');
|
||||
ASSERT_THROW(m.generate(r), ApiError);
|
||||
ASSERT_EQ(rig.transport->send_count(), 0u);
|
||||
CompatChatModel with_default(rig.provider, "dflt");
|
||||
ASSERT_EQ(with_default.to_body(base(), false).find("model")->as_string(), std::string("m1"));
|
||||
ChatRequest nomodel = base();
|
||||
nomodel.model = "";
|
||||
ASSERT_EQ(with_default.to_body(nomodel, false).find("model")->as_string(), std::string("dflt"));
|
||||
}
|
||||
|
||||
TEST(compat_chat, groq_quirks_are_rejected_loudly) {
|
||||
Rig rig("groq");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest named = base();
|
||||
named.messages[0].name = "alice";
|
||||
ASSERT_THROW(m.to_body(named, false), ApiError);
|
||||
for (const char* key : {"logprobs", "top_logprobs", "logit_bias"}) {
|
||||
ChatRequest r = base();
|
||||
r.extra = Json::object();
|
||||
r.extra.set(key, Json(1));
|
||||
ASSERT_THROW(m.to_body(r, false), ApiError);
|
||||
}
|
||||
ChatRequest n2 = base();
|
||||
n2.extra = Json::parse("{\"n\":2}");
|
||||
ASSERT_THROW(m.to_body(n2, false), ApiError);
|
||||
ChatRequest n1 = base();
|
||||
n1.extra = Json::parse("{\"n\":1}");
|
||||
ASSERT_TRUE(m.to_body(n1, false).find("n") != nullptr);
|
||||
ASSERT_EQ(rig.transport->send_count(), 0u);
|
||||
// The same request is fine on a row without the quirk.
|
||||
Rig other("together");
|
||||
CompatChatModel o(other.provider);
|
||||
ChatRequest ok = base();
|
||||
ok.messages[0].name = "alice";
|
||||
ok.extra = Json::parse("{\"logprobs\":true,\"n\":2}");
|
||||
ASSERT_TRUE(o.to_body(ok, false).find("logprobs") != nullptr);
|
||||
ChatRequest big = base();
|
||||
big.extra = Json::parse("{\"n\":9}");
|
||||
ASSERT_THROW(o.to_body(big, false), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, cerebras_rejects_tools_with_response_format) {
|
||||
Rig rig("cerebras");
|
||||
CompatChatModel m(rig.provider);
|
||||
ChatRequest r = base();
|
||||
r.json_mode = true;
|
||||
ASSERT_TRUE(m.to_body(r, false).find("response_format") != nullptr);
|
||||
r.tools.push_back({"f", "", Json::object()});
|
||||
ASSERT_THROW(m.to_body(r, false), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, parses_reasoning_fields_and_tool_calls) {
|
||||
Json doc = Json::parse(
|
||||
"{\"id\":\"x\",\"model\":\"m\",\"choices\":[{\"index\":0,\"finish_reason\":\"tool_calls\",\"message\":"
|
||||
"{\"role\":\"assistant\",\"content\":null,\"reasoning\":\"why\",\"tool_calls\":[{\"id\":\"c1\",\"type\":\"function\","
|
||||
"\"function\":{\"name\":\"f\",\"arguments\":\"{\\\"a\\\":1}\"}},{\"id\":\"c2\",\"type\":\"function\","
|
||||
"\"function\":{\"name\":\"g\",\"arguments\":\"not json\"}}]}}],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":4,"
|
||||
"\"total_tokens\":7,\"prompt_tokens_details\":{\"cached_tokens\":2},\"completion_tokens_details\":{\"reasoning_tokens\":1}}}");
|
||||
molo::llm::ChatResponse r = CompatChatModel::parse_response(doc);
|
||||
ASSERT_EQ(r.finish_reason, std::string("tool_calls"));
|
||||
ASSERT_EQ(r.message.parts[0].reasoning, std::string("why"));
|
||||
auto calls = r.tool_calls();
|
||||
ASSERT_EQ(calls.size(), 2u);
|
||||
ASSERT_EQ(calls[0].arguments.find("a")->as_int(), 1L);
|
||||
ASSERT_TRUE(calls[1].arguments.find("_invalid_json") != nullptr);
|
||||
ASSERT_EQ(r.usage.cached_tokens, 2);
|
||||
ASSERT_EQ(r.usage.reasoning_tokens, 1);
|
||||
Json rc = Json::parse("{\"choices\":[{\"message\":{\"content\":\"a\",\"reasoning_content\":\"b\"}}]}");
|
||||
ASSERT_EQ(CompatChatModel::parse_response(rc).message.parts[0].reasoning, std::string("b"));
|
||||
ASSERT_THROW(CompatChatModel::parse_response(Json::parse("{\"choices\":[]}")), ApiError);
|
||||
ASSERT_THROW(CompatChatModel::parse_response(Json::parse("{\"choices\":[{}]}")), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_chat, stream_documented_transcript_with_usage_chunk) {
|
||||
Rig rig("fireworks");
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse(kStream));
|
||||
CompatChatModel m(rig.provider);
|
||||
std::string text, finish;
|
||||
int total = 0;
|
||||
m.stream(base(), [&](const ChatChunk& c) {
|
||||
text += c.text_delta;
|
||||
if (!c.finish_reason.empty()) finish = c.finish_reason;
|
||||
if (c.usage) total = c.usage->total_tokens;
|
||||
return true;
|
||||
});
|
||||
ASSERT_EQ(text, std::string("Hello!"));
|
||||
ASSERT_EQ(finish, std::string("stop"));
|
||||
ASSERT_EQ(total, 11);
|
||||
ASSERT_TRUE(has(rig.transport->seen()[0].body, "\"stream\":true"));
|
||||
ASSERT_FALSE(has(rig.transport->seen()[0].body, "stream_options"));
|
||||
}
|
||||
|
||||
TEST(compat_chat, stream_tool_fragments_carry_id_and_name) {
|
||||
Rig rig("fireworks");
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), sse(kToolStream));
|
||||
CompatChatModel m(rig.provider);
|
||||
std::string args, id, name, reasoning;
|
||||
m.stream(base(), [&](const ChatChunk& c) {
|
||||
args += c.tool_args_delta;
|
||||
if (!c.tool_call_id.empty()) id = c.tool_call_id;
|
||||
if (!c.tool_call_name.empty()) name = c.tool_call_name;
|
||||
reasoning += c.reasoning_delta;
|
||||
return true;
|
||||
});
|
||||
ASSERT_EQ(args, std::string("{\"city\":\"Rome\"}"));
|
||||
ASSERT_EQ(id, std::string("call_1"));
|
||||
ASSERT_EQ(name, std::string("get_weather"));
|
||||
ASSERT_EQ(reasoning, std::string("thinking"));
|
||||
}
|
||||
|
||||
TEST(compat_chat, stream_without_finish_reason_is_truncated_and_sink_can_stop) {
|
||||
Rig rig("fireworks");
|
||||
rig.transport->expect_response(
|
||||
Method::kPost, rig.path("/chat/completions"),
|
||||
sse("data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"a\"},\"finish_reason\":null}]}\n\n"));
|
||||
CompatChatModel m(rig.provider);
|
||||
try {
|
||||
m.stream(base(), [](const ChatChunk&) { return true; });
|
||||
FAIL("expected truncation");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.http_status(), 502);
|
||||
ASSERT_EQ(e.type(), std::string("STREAM_TRUNCATED"));
|
||||
}
|
||||
Rig rig2("fireworks");
|
||||
rig2.transport->expect_response(Method::kPost, rig2.path("/chat/completions"), sse(kStream));
|
||||
CompatChatModel m2(rig2.provider);
|
||||
int seen = 0;
|
||||
m2.stream(base(), [&](const ChatChunk&) { return ++seen < 2; }); // stop early: no throw
|
||||
ASSERT_EQ(seen, 2);
|
||||
}
|
||||
|
||||
TEST(compat_chat, stream_error_event_throws) {
|
||||
Rig rig("fireworks");
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"),
|
||||
sse("data: {\"error\":{\"type\":\"overloaded\",\"message\":\"busy\"}}\n\n"));
|
||||
CompatChatModel m(rig.provider);
|
||||
try {
|
||||
m.stream(base(), [](const ChatChunk&) { return true; });
|
||||
FAIL("expected");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.type(), std::string("overloaded"));
|
||||
}
|
||||
}
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
// Opt-in live checks, driven by the registry table. MOLO_LIVE_NET=1 enables
|
||||
// the network. Per row:
|
||||
// - probe (no key needed): an invalid key must be answered like an API, never
|
||||
// 404 and never a transport failure; the status and error type are printed;
|
||||
// - with the row's key env set: list models (when the row claims it) and ONE
|
||||
// completion of max_tokens 8, on the model named by COMPAT_<ID>_MODEL
|
||||
// (upper case id). A missing key or model prints a loud SKIP line.
|
||||
// Nothing is created or deleted. Local rows run only when their base env is set.
|
||||
#include <algorithm>
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
|
||||
#include "clients/transports/http_transport.h"
|
||||
#include "llm/config.h"
|
||||
#include "support.h"
|
||||
#include "vendors/compat/chat_model.h"
|
||||
#include "vendors/compat/services.h"
|
||||
|
||||
using namespace molo::compat;
|
||||
using namespace compat_test;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
namespace {
|
||||
|
||||
// Read at static-init time: another suite in the shared binary unsets
|
||||
// MOLO_LIVE_NET before the tests run.
|
||||
const bool kNet = std::getenv("MOLO_LIVE_NET") != nullptr;
|
||||
bool net() { return kNet; }
|
||||
|
||||
std::string upper(std::string s) {
|
||||
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return static_cast<char>(std::toupper(c)); });
|
||||
return s;
|
||||
}
|
||||
|
||||
Provider probe_provider(const ProviderRow& row) {
|
||||
return Provider::with(row.id, std::make_unique<molo::clients::transports::HttpTransport>(), "sk-invalid-molo-probe",
|
||||
"", "probe-account");
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST(compat_live, invalid_key_probe_answers_like_an_api_for_every_hosted_row) {
|
||||
if (!net()) {
|
||||
std::cout << "SKIP compat_live.invalid_key_probe (set MOLO_LIVE_NET=1)\n";
|
||||
return;
|
||||
}
|
||||
int unreachable = 0;
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& row = Registry::at(i);
|
||||
if (row.local || std::string(row.extra_env).size() > 0) continue; // no hosted URL to probe
|
||||
Provider p = probe_provider(row);
|
||||
int status = 200;
|
||||
std::string type = "ok", detail;
|
||||
try {
|
||||
if (Registry::has(row, Feature::kModels)) {
|
||||
CompatModels(p).list();
|
||||
} else {
|
||||
CompatChatModel m(p);
|
||||
molo::llm::ChatRequest r;
|
||||
r.model = "molo-probe";
|
||||
r.messages.push_back(molo::llm::Message::user("hi"));
|
||||
r.max_tokens = 1;
|
||||
m.generate(r);
|
||||
}
|
||||
} catch (const ApiError& e) {
|
||||
status = e.http_status();
|
||||
type = e.type();
|
||||
detail = e.message().substr(0, 160);
|
||||
}
|
||||
std::cout << " " << row.id << ": " << status << " " << type << "\n";
|
||||
if (status == 502) {
|
||||
++unreachable; // transport failure: reported, not asserted (network flake)
|
||||
std::cout << " WARN " << row.id << " unreachable from here: " << detail << "\n";
|
||||
continue;
|
||||
}
|
||||
ASSERT_TRUE(status != 404);
|
||||
}
|
||||
std::cout << " unreachable rows: " << unreachable << "\n";
|
||||
}
|
||||
|
||||
TEST(compat_live, models_and_one_tiny_completion_for_each_row_with_a_key) {
|
||||
if (!net()) {
|
||||
std::cout << "SKIP compat_live.models_and_one_tiny_completion (set MOLO_LIVE_NET=1 and a provider key env)\n";
|
||||
return;
|
||||
}
|
||||
int ran = 0;
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& row = Registry::at(i);
|
||||
Provider p = Provider::from_env(row.id);
|
||||
if (!p.configured()) {
|
||||
std::cout << "SKIP compat_live." << row.id << " (set " << p.missing_env() << ")\n";
|
||||
continue;
|
||||
}
|
||||
++ran;
|
||||
if (Registry::has(row, Feature::kModels) && !std::string(row.models_path).empty()) {
|
||||
ModelList list = CompatModels(p).list();
|
||||
std::cout << " " << row.id << " models: " << list.models.size() << "\n";
|
||||
ASSERT_TRUE(!list.models.empty());
|
||||
}
|
||||
const std::string model_env = "COMPAT_" + upper(row.id) + "_MODEL";
|
||||
std::string model = molo::llm::Config::get(model_env.c_str());
|
||||
if (model.empty()) {
|
||||
std::cout << "SKIP compat_live." << row.id << " completion (set " << model_env << ")\n";
|
||||
continue;
|
||||
}
|
||||
CompatChatModel chat(p, model);
|
||||
molo::llm::ChatRequest r;
|
||||
r.messages.push_back(molo::llm::Message::user("Reply with the single word: ok"));
|
||||
r.max_tokens = 8;
|
||||
molo::llm::ChatResponse out = chat.generate(r);
|
||||
std::cout << " " << row.id << " completion finish=" << out.finish_reason << " tokens=" << out.usage.total_tokens << "\n";
|
||||
ASSERT_TRUE(!out.id.empty() || !out.finish_reason.empty());
|
||||
}
|
||||
if (ran == 0) std::cout << "SKIP compat_live: no provider key is configured\n";
|
||||
}
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
// Provider wiring over the real ApiClient with a recording transport, driven
|
||||
// by the registry table: every row is exercised, so a new row is tested
|
||||
// without a new test.
|
||||
#include <cstdlib>
|
||||
|
||||
#include "support.h"
|
||||
#include "vendors/compat/chat_model.h"
|
||||
#include "vendors/compat/services.h"
|
||||
|
||||
using namespace molo::compat;
|
||||
using namespace compat_test;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
namespace {
|
||||
// FIXTURE: OpenAI API reference, createChatCompletion "Default" response example.
|
||||
const char* kChatExample =
|
||||
"{\"id\":\"chatcmpl-B9MBs8CjcvOU2jLn4n570S5qMJKcT\",\"object\":\"chat.completion\",\"created\":1741569952,"
|
||||
"\"model\":\"gpt-4.1-2025-04-14\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\","
|
||||
"\"content\":\"Hello! How can I assist you today?\",\"refusal\":null,\"annotations\":[]},\"logprobs\":null,"
|
||||
"\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":19,\"completion_tokens\":10,\"total_tokens\":29,"
|
||||
"\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":"
|
||||
"{\"reasoning_tokens\":0,\"audio_tokens\":0,\"accepted_prediction_tokens\":0,\"rejected_prediction_tokens\":0}},"
|
||||
"\"service_tier\":\"default\"}";
|
||||
|
||||
// FIXTURE: OpenAI API reference, listModels response example (two entries).
|
||||
const char* kModelsExample =
|
||||
"{\"object\":\"list\",\"data\":[{\"id\":\"model-id-0\",\"object\":\"model\",\"created\":1686935002,"
|
||||
"\"owned_by\":\"organization-owner\"},{\"id\":\"model-id-1\",\"object\":\"model\",\"created\":1686935002,"
|
||||
"\"owned_by\":\"organization-owner\"}]}";
|
||||
|
||||
molo::llm::ChatRequest hello() {
|
||||
molo::llm::ChatRequest r;
|
||||
r.model = "some-model";
|
||||
r.messages.push_back(molo::llm::Message::user("Hello!"));
|
||||
return r;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(compat_provider, every_row_posts_chat_to_its_own_base_with_a_bearer_key) {
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& row = Registry::at(i);
|
||||
Rig rig(row.id);
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), reply(kChatExample));
|
||||
CompatChatModel model(rig.provider);
|
||||
molo::llm::ChatResponse out = model.generate(hello());
|
||||
ASSERT_EQ(out.text(), std::string("Hello! How can I assist you today?"));
|
||||
ASSERT_EQ(out.usage.total_tokens, 29);
|
||||
auto s = rig.transport->seen()[0];
|
||||
ASSERT_EQ(s.method, std::string("POST"));
|
||||
ASSERT_EQ(s.headers.get_or("Authorization", ""), std::string("Bearer ") + kKey);
|
||||
ASSERT_EQ(s.target.find(kKey), std::string::npos);
|
||||
ASSERT_EQ(model.vendor(), std::string("compat:") + row.id);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(compat_provider, every_models_row_lists_and_every_other_row_refuses_without_traffic) {
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& row = Registry::at(i);
|
||||
Rig rig(row.id);
|
||||
CompatModels models(rig.provider);
|
||||
if (Registry::has(row, Feature::kModels)) {
|
||||
rig.transport->expect_response(Method::kGet, rig.path("/models"), reply(kModelsExample));
|
||||
ModelList list = models.list();
|
||||
ASSERT_EQ(list.models.size(), 2u);
|
||||
ASSERT_TRUE(list.contains("model-id-1"));
|
||||
ASSERT_EQ(rig.transport->seen()[0].method, std::string("GET"));
|
||||
} else {
|
||||
ASSERT_THROW(models.list(), ApiError);
|
||||
ASSERT_EQ(rig.transport->send_count(), 0u);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(compat_provider, features_not_in_the_row_never_reach_the_network) {
|
||||
Rig rig("cerebras");
|
||||
CompatEmbeddings emb(rig.provider);
|
||||
ASSERT_THROW(emb.embed("m", {"x"}), ApiError);
|
||||
CompatImages img(rig.provider);
|
||||
ImageRequest ir;
|
||||
ir.model = "m";
|
||||
ir.prompt = "p";
|
||||
ASSERT_THROW(img.generate(ir), ApiError);
|
||||
CompatAudio audio(rig.provider);
|
||||
SpeechRequest sr;
|
||||
sr.model = "m";
|
||||
sr.input = "x";
|
||||
sr.voice = "v";
|
||||
std::string ct;
|
||||
ASSERT_THROW(audio.speak(sr, ct), ApiError);
|
||||
ASSERT_EQ(rig.transport->send_count(), 0u);
|
||||
}
|
||||
|
||||
TEST(compat_provider, unconfigured_provider_has_no_traffic_and_names_the_env) {
|
||||
Rig rig("groq", "");
|
||||
ASSERT_FALSE(rig.provider.configured());
|
||||
ASSERT_EQ(rig.provider.missing_env(), std::string("GROQ_API_KEY"));
|
||||
CompatChatModel model(rig.provider);
|
||||
try {
|
||||
model.generate(hello());
|
||||
FAIL("expected");
|
||||
} catch (const molo::llm::NotConfiguredError& e) {
|
||||
ASSERT_EQ(e.http_status(), 503);
|
||||
ASSERT_TRUE(has(e.message(), "GROQ_API_KEY"));
|
||||
}
|
||||
ASSERT_EQ(rig.transport->send_count(), 0u);
|
||||
}
|
||||
|
||||
TEST(compat_provider, cloudflare_substitutes_the_account_id_and_validates_it) {
|
||||
Rig rig("cloudflare", kKey, "", "a1b2c3");
|
||||
ASSERT_EQ(rig.provider.base_url(), std::string("https://api.cloudflare.com/client/v4/accounts/a1b2c3/ai/v1"));
|
||||
ASSERT_THROW(Rig("cloudflare", kKey, "", ""), ApiError);
|
||||
ASSERT_THROW(Rig("cloudflare", kKey, "", "a/../b"), ApiError);
|
||||
ASSERT_THROW(Rig("cloudflare", kKey, "", "a b"), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_provider, base_override_is_validated_and_trailing_slash_removed) {
|
||||
Rig ok("vllm", "", "http://10.0.0.5:9000/v1/");
|
||||
ASSERT_EQ(ok.provider.base_url(), std::string("http://10.0.0.5:9000/v1"));
|
||||
ASSERT_THROW(Rig("vllm", "", "ftp://x"), ApiError);
|
||||
ASSERT_THROW(Rig("vllm", "", "http://x/v1?a=b"), ApiError);
|
||||
ASSERT_THROW(Rig("vllm", "", "http://x/../v1"), ApiError);
|
||||
ASSERT_THROW(Rig("vllm", "", "http://x y/v1"), ApiError);
|
||||
ASSERT_THROW(Rig("vllm", "", std::string("http://x/") + std::string(600, 'a')), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_provider, local_rows_are_configured_only_when_their_base_env_is_set) {
|
||||
setenv("MOLO_ENV_FILES", "off", 1);
|
||||
unsetenv("OLLAMA_BASE_URL");
|
||||
unsetenv("OLLAMA_API_KEY");
|
||||
Provider off = Provider::from_env("ollama");
|
||||
ASSERT_FALSE(off.configured());
|
||||
ASSERT_EQ(off.missing_env(), std::string("OLLAMA_BASE_URL"));
|
||||
setenv("OLLAMA_BASE_URL", "http://127.0.0.1:11434/v1", 1);
|
||||
Provider on = Provider::from_env("ollama");
|
||||
ASSERT_TRUE(on.configured());
|
||||
ASSERT_EQ(on.base_url(), std::string("http://127.0.0.1:11434/v1"));
|
||||
unsetenv("OLLAMA_BASE_URL");
|
||||
}
|
||||
|
||||
TEST(compat_provider, hosted_from_env_needs_its_key_and_cloudflare_needs_the_account) {
|
||||
setenv("MOLO_ENV_FILES", "off", 1);
|
||||
unsetenv("CLOUDFLARE_API_TOKEN");
|
||||
unsetenv("CLOUDFLARE_ACCOUNT_ID");
|
||||
unsetenv("GROQ_API_KEY");
|
||||
ASSERT_FALSE(Provider::from_env("groq").configured());
|
||||
setenv("CLOUDFLARE_API_TOKEN", "t", 1);
|
||||
Provider cf = Provider::from_env("cloudflare");
|
||||
ASSERT_FALSE(cf.configured());
|
||||
ASSERT_EQ(cf.missing_env(), std::string("CLOUDFLARE_ACCOUNT_ID"));
|
||||
setenv("CLOUDFLARE_ACCOUNT_ID", "acct9", 1);
|
||||
ASSERT_TRUE(Provider::from_env("cloudflare").configured());
|
||||
unsetenv("CLOUDFLARE_API_TOKEN");
|
||||
unsetenv("CLOUDFLARE_ACCOUNT_ID");
|
||||
}
|
||||
|
||||
TEST(compat_provider, error_envelopes_of_different_providers_map_to_api_error) {
|
||||
// Shapes seen in the unauthenticated probes of SPEC section 5 (labelled: probe output, 2026-10-25).
|
||||
struct Case { const char* id; int status; const char* body; const char* type; };
|
||||
const Case cases[] = {
|
||||
{"groq", 401, "{\"error\":{\"message\":\"Invalid API Key\",\"type\":\"invalid_request_error\",\"code\":\"invalid_api_key\"}}", "invalid_request_error"},
|
||||
{"cerebras", 401, "{\"message\":\"Wrong API Key\",\"type\":\"invalid_request_error\",\"param\":\"api_key\",\"code\":\"wrong_api_key\"}", "invalid_request_error"},
|
||||
{"moonshot", 401, "{\"error\":{\"message\":\"Incorrect API key provided\",\"type\":\"incorrect_api_key_error\"}}", "incorrect_api_key_error"},
|
||||
};
|
||||
for (const Case& c : cases) {
|
||||
Rig rig(c.id);
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/chat/completions"), reply(c.body, c.status));
|
||||
CompatChatModel model(rig.provider);
|
||||
try {
|
||||
model.generate(hello());
|
||||
FAIL("expected");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.http_status(), c.status);
|
||||
ASSERT_EQ(e.type(), std::string(c.type));
|
||||
}
|
||||
}
|
||||
}
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
// The registry table is the single source: these tests pin its invariants so
|
||||
// that adding a row cannot break consistency between components.
|
||||
#include <set>
|
||||
|
||||
#include "support.h"
|
||||
#include "vendors/compat/calls.h"
|
||||
#include "vendors/compat/registry.h"
|
||||
|
||||
using namespace molo::compat;
|
||||
using namespace compat_test;
|
||||
|
||||
TEST(compat_registry, rows_are_unique_slugs_with_complete_fields) {
|
||||
std::set<std::string> ids;
|
||||
ASSERT_TRUE(Registry::count() >= 35);
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& r = Registry::at(i);
|
||||
std::string id = r.id;
|
||||
ASSERT_TRUE(!id.empty() && id.size() <= 32);
|
||||
for (char c : id) ASSERT_TRUE((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'));
|
||||
ASSERT_TRUE(ids.insert(id).second);
|
||||
ASSERT_TRUE(std::string(r.name).size() > 2);
|
||||
ASSERT_TRUE(std::string(r.key_env).size() > 3);
|
||||
ASSERT_TRUE(std::string(r.doc_url).rfind("https://", 0) == 0);
|
||||
ASSERT_TRUE(std::string(r.note).size() > 10);
|
||||
ASSERT_TRUE((r.features & Feature::kChat) != 0); // every row is a chat provider
|
||||
}
|
||||
}
|
||||
|
||||
TEST(compat_registry, base_urls_are_wellformed) {
|
||||
std::set<std::string> envs;
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& r = Registry::at(i);
|
||||
std::string base = r.base_url;
|
||||
ASSERT_TRUE(base.back() != '/');
|
||||
ASSERT_TRUE(base.find(' ') == std::string::npos);
|
||||
if (r.local) {
|
||||
ASSERT_TRUE(base.rfind("http://localhost:", 0) == 0);
|
||||
ASSERT_TRUE(std::string(r.base_env).size() > 3); // the "configured" switch
|
||||
} else {
|
||||
ASSERT_TRUE(base.rfind("https://", 0) == 0);
|
||||
ASSERT_TRUE(std::string(r.base_env).empty());
|
||||
}
|
||||
ASSERT_EQ(base.find("{account_id}") != std::string::npos, std::string(r.extra_env).size() > 0);
|
||||
// Credential names are unique except where two rows deliberately share none.
|
||||
ASSERT_TRUE(envs.insert(r.key_env).second);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(compat_registry, models_feature_and_path_agree) {
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& r = Registry::at(i);
|
||||
bool claims = Registry::has(r, Feature::kModels);
|
||||
bool path = std::string(r.models_path).size() > 0;
|
||||
ASSERT_EQ(claims, path);
|
||||
if (path) ASSERT_EQ(std::string(r.models_path), std::string("/models"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(compat_registry, quirk_rows_carry_their_documented_flag) {
|
||||
const ProviderRow* groq = Registry::find("groq");
|
||||
ASSERT_TRUE(groq != nullptr);
|
||||
ASSERT_TRUE((groq->quirks & Quirk::kRejectMessageName) != 0);
|
||||
ASSERT_TRUE((groq->quirks & Quirk::kRejectLogprobs) != 0);
|
||||
ASSERT_TRUE((groq->quirks & Quirk::kSingleChoice) != 0);
|
||||
ASSERT_TRUE((Registry::get("cerebras").quirks & Quirk::kNoToolsWithFormat) != 0);
|
||||
ASSERT_EQ(Registry::get("cerebras").features & Feature::kEmbeddings, 0u);
|
||||
}
|
||||
|
||||
TEST(compat_registry, evidence_matches_row_kind) {
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& r = Registry::at(i);
|
||||
if (r.local) ASSERT_TRUE(r.evidence == Evidence::kDocsOnly);
|
||||
ASSERT_TRUE(std::string(Registry::evidence_name(r.evidence)).size() > 4);
|
||||
}
|
||||
ASSERT_TRUE(Registry::get("cloudflare").evidence == Evidence::kDocsOnly);
|
||||
}
|
||||
|
||||
TEST(compat_registry, no_em_dash_in_any_string) {
|
||||
const std::string em = "\xE2\x80\x94";
|
||||
for (std::size_t i = 0; i < Registry::count(); ++i) {
|
||||
const ProviderRow& r = Registry::at(i);
|
||||
for (const char* s : {r.id, r.name, r.base_url, r.key_env, r.doc_url, r.note})
|
||||
ASSERT_TRUE(std::string(s).find(em) == std::string::npos);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(compat_registry, lookup_and_unknown_id) {
|
||||
ASSERT_TRUE(Registry::find("groq") != nullptr);
|
||||
ASSERT_TRUE(Registry::find("nope") == nullptr);
|
||||
ASSERT_THROW(Registry::get("nope"), molo::llm::ApiError);
|
||||
ASSERT_THROW(Registry::at(Registry::count()), molo::llm::ApiError);
|
||||
ASSERT_EQ(Registry::feature_names(Registry::get("cerebras")), std::string("chat,models"));
|
||||
}
|
||||
|
||||
TEST(compat_registry, call_table_rows_map_to_openai_operations) {
|
||||
ASSERT_EQ(Calls::count(), 6u);
|
||||
ASSERT_EQ(std::string(Calls::spec_of(Call::kChat).path), std::string("/chat/completions"));
|
||||
ASSERT_EQ(std::string(Calls::spec_of(Call::kModels).operation_id), std::string("listModels"));
|
||||
ASSERT_TRUE(Calls::method_of(Call::kModels) == Method::kGet);
|
||||
ASSERT_TRUE(Calls::method_of(Call::kSpeech) == Method::kPost);
|
||||
for (std::size_t i = 0; i < Calls::count(); ++i) {
|
||||
ASSERT_TRUE(std::string(Calls::at(i).path).front() == '/');
|
||||
ASSERT_TRUE(std::string(Calls::at(i).operation_id).size() > 5);
|
||||
}
|
||||
}
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
// Models, embeddings, images and audio over the recording transport. Replies
|
||||
// are the documented OpenAI examples (FIXTURE); assertions are on what WE send
|
||||
// and how WE parse.
|
||||
#include "support.h"
|
||||
#include "vendors/compat/services.h"
|
||||
|
||||
using namespace molo::compat;
|
||||
using namespace compat_test;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
TEST(compat_services, models_accepts_a_bare_array_and_skips_entries_without_id) {
|
||||
// Together returns a bare array; shape recorded from its docs page (LABELLED).
|
||||
ModelList l = CompatModels::parse(Json::parse("[{\"id\":\"a\"},{\"name\":\"b\"},{\"x\":1}]"));
|
||||
ASSERT_EQ(l.models.size(), 2u);
|
||||
ASSERT_TRUE(l.contains("b"));
|
||||
ASSERT_THROW(CompatModels::parse(Json::parse("{\"data\":5}")), ApiError);
|
||||
ASSERT_THROW(CompatModels::parse(Json::parse("{}")), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_services, models_list_truncates_loudly) {
|
||||
Json arr = Json::array();
|
||||
for (std::size_t i = 0; i < 5010; ++i) {
|
||||
Json o = Json::object();
|
||||
o.set("id", Json("m" + std::to_string(i)));
|
||||
arr.push(o);
|
||||
}
|
||||
ModelList l = CompatModels::parse(arr);
|
||||
ASSERT_EQ(l.models.size(), 5000u);
|
||||
ASSERT_TRUE(l.truncated);
|
||||
}
|
||||
|
||||
TEST(compat_services, row_without_list_endpoint_refuses) {
|
||||
Rig rig("dashscope");
|
||||
CompatModels m(rig.provider);
|
||||
ASSERT_THROW(m.list(), ApiError);
|
||||
ASSERT_EQ(rig.transport->send_count(), 0u);
|
||||
}
|
||||
|
||||
TEST(compat_services, embeddings_body_rules_and_parse) {
|
||||
Rig rig("nvidia");
|
||||
CompatEmbeddings e(rig.provider);
|
||||
Json b = e.to_body("m", {"a", "b"}, 256);
|
||||
ASSERT_EQ(b.find("input")->as_array().size(), 2u);
|
||||
ASSERT_EQ(b.find("dimensions")->as_int(), 256L);
|
||||
ASSERT_TRUE(e.to_body("m", {"a"}, 0).find("dimensions") == nullptr);
|
||||
ASSERT_THROW(e.to_body("", {"a"}, 0), ApiError);
|
||||
ASSERT_THROW(e.to_body("m", {}, 0), ApiError);
|
||||
ASSERT_THROW(e.to_body("m", {""}, 0), ApiError);
|
||||
ASSERT_THROW(e.to_body("m", {"a"}, -1), ApiError);
|
||||
ASSERT_THROW(e.to_body("m", std::vector<std::string>(2049, "x"), 0), ApiError);
|
||||
// FIXTURE: OpenAI API reference, createEmbedding response example.
|
||||
rig.transport->expect_response(
|
||||
Method::kPost, rig.path("/embeddings"),
|
||||
reply("{\"object\":\"list\",\"data\":[{\"object\":\"embedding\",\"embedding\":[0.0023064255,-0.009327292,-0.0028842222],"
|
||||
"\"index\":0}],\"model\":\"text-embedding-ada-002\",\"usage\":{\"prompt_tokens\":8,\"total_tokens\":8}}"));
|
||||
EmbeddingResult r = e.embed("text-embedding-ada-002", {"The food was delicious"});
|
||||
ASSERT_EQ(r.items.size(), 1u);
|
||||
ASSERT_EQ(r.items[0].vector.size(), 3u);
|
||||
ASSERT_EQ(r.total_tokens, 8);
|
||||
ASSERT_THROW(CompatEmbeddings::parse(Json::parse("{\"data\":[{\"embedding\":5}]}")), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_services, images_body_rules_and_parse) {
|
||||
Rig rig("together");
|
||||
CompatImages img(rig.provider);
|
||||
ImageRequest r;
|
||||
r.model = "m";
|
||||
r.prompt = "a cat";
|
||||
r.size = "512x512";
|
||||
r.response_format = "b64_json";
|
||||
Json b = img.to_body(r);
|
||||
ASSERT_EQ(b.find("size")->as_string(), std::string("512x512"));
|
||||
r.size = "big";
|
||||
ASSERT_THROW(img.to_body(r), ApiError);
|
||||
r.size = "x512";
|
||||
ASSERT_THROW(img.to_body(r), ApiError);
|
||||
r.size = "";
|
||||
r.n = 11;
|
||||
ASSERT_THROW(img.to_body(r), ApiError);
|
||||
r.n = 1;
|
||||
r.response_format = "png";
|
||||
ASSERT_THROW(img.to_body(r), ApiError);
|
||||
r.response_format = "";
|
||||
r.prompt = "";
|
||||
ASSERT_THROW(img.to_body(r), ApiError);
|
||||
// FIXTURE: OpenAI API reference, createImage response example.
|
||||
auto items = CompatImages::parse(Json::parse("{\"created\":1713833628,\"data\":[{\"b64_json\":\"...\",\"revised_prompt\":\"x\"}]}"));
|
||||
ASSERT_EQ(items.size(), 1u);
|
||||
ASSERT_EQ(items[0].revised_prompt, std::string("x"));
|
||||
ASSERT_THROW(CompatImages::parse(Json::parse("{}")), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_services, transcription_sends_multipart_and_validates) {
|
||||
Rig rig("groq");
|
||||
CompatAudio audio(rig.provider);
|
||||
TranscribeRequest t;
|
||||
t.model = "whisper-large-v3";
|
||||
t.filename = "a.wav";
|
||||
t.audio = "RIFFxxxx";
|
||||
t.language = "en";
|
||||
// FIXTURE: OpenAI API reference, createTranscription json response example.
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/audio/transcriptions"),
|
||||
reply("{\"text\":\"Imagine the wildest idea that you've ever had.\"}"));
|
||||
ASSERT_TRUE(has(audio.transcribe(t), "wildest idea"));
|
||||
auto s = rig.transport->seen()[0];
|
||||
ASSERT_TRUE(has(s.headers.get_or("Content-Type", ""), "multipart/form-data"));
|
||||
ASSERT_TRUE(has(s.body, "name=\"file\"; filename=\"a.wav\""));
|
||||
ASSERT_TRUE(has(s.body, "whisper-large-v3"));
|
||||
TranscribeRequest bad = t;
|
||||
bad.audio.clear();
|
||||
ASSERT_THROW(audio.transcribe(bad), ApiError);
|
||||
bad = t;
|
||||
bad.filename = "a\".wav";
|
||||
ASSERT_THROW(audio.transcribe(bad), ApiError);
|
||||
bad = t;
|
||||
bad.filename = "../a.wav";
|
||||
ASSERT_THROW(audio.transcribe(bad), ApiError);
|
||||
Rig no("cerebras");
|
||||
ASSERT_THROW(CompatAudio(no.provider).transcribe(t), ApiError);
|
||||
}
|
||||
|
||||
TEST(compat_services, speech_returns_bytes_and_content_type) {
|
||||
Rig rig("together");
|
||||
CompatAudio audio(rig.provider);
|
||||
SpeechRequest s;
|
||||
s.model = "m";
|
||||
s.input = "hello";
|
||||
s.voice = "alloy";
|
||||
s.format = "mp3";
|
||||
rig.transport->expect_response(Method::kPost, rig.path("/audio/speech"), reply("ID3bytes", 200, "audio/mpeg"));
|
||||
std::string ct;
|
||||
ASSERT_EQ(audio.speak(s, ct), std::string("ID3bytes"));
|
||||
ASSERT_EQ(ct, std::string("audio/mpeg"));
|
||||
s.format = "ogg";
|
||||
ASSERT_THROW(audio.speech_body(s), ApiError);
|
||||
s.format = "";
|
||||
s.voice = "";
|
||||
ASSERT_THROW(audio.speech_body(s), ApiError);
|
||||
s.voice = "v";
|
||||
s.input = std::string(4097, 'x');
|
||||
ASSERT_THROW(audio.speech_body(s), ApiError);
|
||||
}
|
||||
Reference in New Issue
Block a user