#include "../AiDemos.hpp" #include #include "../../fw/Router.hpp" #include "../../fw/Templates.hpp" #include "security/crypto/Base64.h" #include "vendors/compat/caps.h" #include "vendors/compat/chat_model.h" #include "vendors/compat/provider.h" #include "vendors/compat/services.h" namespace molo::example::app { // OpenAI-compatible provider registry over the real src/vendors/compat part. // GET routes need no credential. Every route that calls a provider is a POST: // 422 on bad input, 503 compat_not_configured without the provider's // credential. Routes that cost money (chat, embeddings, images, audio) are dry // runs that show the exact request unless the body has "confirm":true. The // provider list is the registry table; nothing here names a provider. namespace { namespace cp = compat; using Json = clients::body::Json; constexpr std::size_t kMaxBody = 8 * 1024 * 1024; // audio travels as base64 const char kPage[] = R"COMPAT({% extends "base.html" %} {% block content %}

OpenAI-compatible providers

One registry table, one client: chat, embeddings, models, images and audio for every provider whose row claims them. Paid calls are dry runs until the body says "confirm":true.

Checking credentials...


{% endblock %} )COMPAT"; [[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); } fw::Response json_out(const Json& j, int status = 200) { fw::Response res = fw::JsonResponse(j.dump(), status); res.headers.set("Cache-Control", "no-store"); return res; } Json parse_body(const fw::Request& req) { if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large"); if (req.body.empty()) return Json::object(); try { Json j = Json::parse(req.body); if (!j.is_object()) bad("body must be a JSON object"); return j; } catch (const clients::body::JsonError&) { bad("body is not valid JSON"); } } std::string text(const Json& b, const char* key, bool required = false) { const Json* v = b.find(key); if (!v || v->is_null()) { if (required) bad(std::string(key) + " is required"); return ""; } if (!v->is_string()) bad(std::string(key) + " must be a string"); return v->as_string(); } int number(const Json& b, const char* key, int fallback) { const Json* v = b.find(key); if (!v || v->is_null()) return fallback; if (!v->is_number()) bad(std::string(key) + " must be a number"); return static_cast(v->as_int()); } bool flag(const Json& b, const char* key) { const Json* v = b.find(key); if (!v || v->is_null()) return false; if (!v->is_bool()) bad(std::string(key) + " must be true or false"); return v->as_bool(); } std::vector inputs(const Json& b) { std::vector out; const Json* v = b.find("input"); if (!v || v->is_null()) bad("input is required"); if (v->is_string()) { out.push_back(v->as_string()); } else if (v->is_array()) { for (const Json& s : v->as_array()) { if (!s.is_string()) bad("input must be a string or an array of strings"); out.push_back(s.as_string()); } } else { bad("input must be a string or an array of strings"); } return out; } Json dry_run(const Json& request, const std::string& url) { Json o = Json::object(); o.set("dry_run", Json(true)); o.set("would_send_to", Json(url)); o.set("request", request); o.set("hint", Json("send confirm:true to run it")); return o; } Json usage_json(const llm::Usage& u) { Json o = Json::object(); o.set("input_tokens", Json(u.input_tokens)); o.set("output_tokens", Json(u.output_tokens)); o.set("total_tokens", Json(u.total_tokens)); o.set("cached_tokens", Json(u.cached_tokens)); o.set("reasoning_tokens", Json(u.reasoning_tokens)); return o; } Json provider_json(const cp::ProviderRow& row) { cp::Provider p = cp::Provider::from_env(row.id); Json o = Json::object(); o.set("id", Json(row.id)); o.set("name", Json(row.name)); o.set("base_url", Json(p.base_url())); o.set("credential_env", Json(row.key_env)); o.set("base_env", Json(row.base_env)); o.set("extra_env", Json(row.extra_env)); o.set("configured", Json(p.configured())); o.set("missing_env", Json(p.missing_env())); Json feats = Json::array(); std::string names = cp::Registry::feature_names(row); std::size_t at = 0; while (at <= names.size()) { std::size_t comma = names.find(',', at); if (comma == std::string::npos) comma = names.size(); if (comma > at) feats.push(Json(names.substr(at, comma - at))); at = comma + 1; } o.set("features", feats); o.set("evidence", Json(cp::Registry::evidence_name(row.evidence))); o.set("local", Json(row.local)); o.set("doc_url", Json(row.doc_url)); o.set("note", Json(row.note)); return o; } // One route = validate into a Plan, then the generic runner decides dry run, // credential, call and error mapping. struct Plan { Json request; // what would be sent (shown in a dry run) std::string url; // for the dry-run message bool needs_confirm = false; // paid std::function call; }; class CompatRoutes { public: using Planner = std::function; fw::Response run(const fw::Request& req, const Planner& planner) const { Json body = parse_body(req); Plan plan; try { plan = planner(body); } catch (const llm::ApiError& e) { map(e); } if (plan.needs_confirm && !flag(body, "confirm")) return json_out(dry_run(plan.request, plan.url)); try { return json_out(plan.call()); } catch (const llm::ApiError& e) { map(e); } } static cp::Provider provider_of(const Json& b) { std::string id = text(b, "provider", true); if (!cp::Registry::find(id)) bad("unknown provider '" + id.substr(0, 64) + "'"); return cp::Provider::from_env(id); } static std::string url_of(const cp::Provider& p, cp::Call call) { return p.base_url() + cp::Calls::spec_of(call).path; } Plan chat(const Json& b) const { cp::Provider p = provider_of(b); llm::ChatRequest r; r.model = text(b, "model"); std::string system = text(b, "system"); if (!system.empty()) r.system = system; r.messages.push_back(llm::Message::user(text(b, "prompt", true))); int max_tokens = number(b, "max_tokens", 0); if (max_tokens != 0) r.max_tokens = max_tokens; r.json_mode = flag(b, "json"); r.reasoning_effort = text(b, "effort"); if (const Json* t = b.find("temperature"); t && t->is_number()) r.temperature = t->as_number(); if (const Json* e = b.find("extra"); e && !e->is_null()) r.extra = *e; auto model = std::make_shared(p); Plan pl{model->to_body(r, false), url_of(p, cp::Call::kChat), true, nullptr}; pl.call = [model, r] { llm::ChatResponse out = model->generate(r); Json o = Json::object(); o.set("id", Json(out.id)); o.set("model", Json(out.model)); o.set("finish_reason", Json(out.finish_reason)); o.set("content", Json(out.text())); std::string reasoning; for (const llm::ContentPart& part : out.message.parts) reasoning += part.reasoning; o.set("reasoning", Json(reasoning)); Json calls = Json::array(); for (const llm::ToolCall& tc : out.tool_calls()) { Json x = Json::object(); x.set("id", Json(tc.id)); x.set("name", Json(tc.name)); x.set("arguments", tc.arguments); calls.push(x); } o.set("tool_calls", calls); o.set("usage", usage_json(out.usage)); return o; }; return pl; } Plan models(const Json& b) const { cp::Provider p = provider_of(b); p.require(cp::Call::kModels); if (std::string(p.row().models_path).empty()) bad(std::string(p.row().name) + " has no list endpoint under its base URL"); auto api = std::make_shared(p); return Plan{Json::object(), url_of(p, cp::Call::kModels), false, [api] { cp::ModelList l = api->list(); Json arr = Json::array(); for (const cp::ModelEntry& m : l.models) { Json x = Json::object(); x.set("id", Json(m.id)); x.set("owned_by", Json(m.owned_by)); arr.push(x); } Json o = Json::object(); o.set("models", arr); o.set("truncated", Json(l.truncated)); return o; }}; } Plan embeddings(const Json& b) const { cp::Provider p = provider_of(b); auto api = std::make_shared(p); std::string model = text(b, "model"); std::vector in = inputs(b); int dims = number(b, "dimensions", 0); Plan pl{api->to_body(model, in, dims), url_of(p, cp::Call::kEmbeddings), true, nullptr}; pl.call = [api, model, in, dims] { cp::EmbeddingResult r = api->embed(model, in, dims); Json items = Json::array(); for (const cp::Embedding& e : r.items) { Json x = Json::object(); x.set("dimensions", Json(static_cast(e.vector.size()))); Json head = Json::array(); for (std::size_t i = 0; i < e.vector.size() && i < 8; ++i) head.push(Json(e.vector[i])); x.set("head", head); items.push(x); } Json o = Json::object(); o.set("model", Json(r.model)); o.set("items", items); o.set("total_tokens", Json(r.total_tokens)); return o; }; return pl; } Plan images(const Json& b) const { cp::Provider p = provider_of(b); auto api = std::make_shared(p); cp::ImageRequest r; r.model = text(b, "model"); r.prompt = text(b, "prompt", true); r.n = number(b, "n", 1); r.size = text(b, "size"); r.response_format = text(b, "response_format"); Plan pl{api->to_body(r), url_of(p, cp::Call::kImages), true, nullptr}; pl.call = [api, r] { Json arr = Json::array(); for (const cp::ImageItem& i : api->generate(r)) { Json x = Json::object(); x.set("url", Json(i.url)); x.set("b64_json_bytes", Json(static_cast(i.b64_json.size()))); x.set("revised_prompt", Json(i.revised_prompt)); arr.push(x); } Json o = Json::object(); o.set("images", arr); return o; }; return pl; } Plan transcribe(const Json& b) const { cp::Provider p = provider_of(b); auto api = std::make_shared(p); cp::TranscribeRequest r; r.model = text(b, "model"); r.filename = text(b, "filename", true); r.language = text(b, "language"); r.prompt = text(b, "prompt"); if (!security::crypto::Base64::decode(text(b, "data_base64", true), r.audio)) bad("data_base64 is not valid base64"); if (r.audio.size() > cp::CompatCaps::kMaxAudioBytes) bad("audio over the byte cap"); p.require(cp::Call::kTranscribe); Json req = Json::object(); req.set("model", Json(r.model)); req.set("filename", Json(r.filename)); req.set("bytes", Json(static_cast(r.audio.size()))); Plan pl{req, url_of(p, cp::Call::kTranscribe), true, nullptr}; pl.call = [api, r] { Json o = Json::object(); o.set("text", Json(api->transcribe(r))); return o; }; return pl; } Plan speech(const Json& b) const { cp::Provider p = provider_of(b); auto api = std::make_shared(p); cp::SpeechRequest r; r.model = text(b, "model"); r.input = text(b, "input", true); r.voice = text(b, "voice"); r.format = text(b, "format"); Plan pl{api->speech_body(r), url_of(p, cp::Call::kSpeech), true, nullptr}; pl.call = [api, r] { std::string ct; std::string audio = api->speak(r, ct); Json o = Json::object(); o.set("content_type", Json(ct)); o.set("bytes", Json(static_cast(audio.size()))); o.set("audio_base64", Json(audio.size() <= 1024 * 1024 ? security::crypto::Base64::encode(audio) : std::string())); return o; }; return pl; } private: [[noreturn]] static void map(const llm::ApiError& e) { if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED") throw fw::HttpError(503, "compat_not_configured", e.message()); if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty()) throw fw::HttpError(422, "validation_error", e.message()); throw fw::HttpError(502, "compat_error", e.vendor() + " " + std::to_string(e.http_status()) + " " + e.type() + ": " + e.message()); } }; } // namespace void register_ai_compat_demo(fw::Router& router, Services& services) { auto routes = std::make_shared(); fw::Environment* env = services.templates.get(); services.templates->add_template("ai_compat.html", kPage); services.add_feature("ai-compat", "/ai/compat"); router.get("/ai/compat", "ai-compat-page", [env](fw::Request req) { (void)req; fw::TemplateCtx ctx; ctx["title"] = std::string("OpenAI-compatible providers"); return fw::Response(200, env->render("ai_compat.html", ctx), "text/html; charset=utf-8"); }); router.get("/ai/compat/status", "ai-compat-status", [](fw::Request req) { (void)req; int configured = 0; Json credentials = Json::array(); for (std::size_t i = 0; i < cp::Registry::count(); ++i) { const cp::ProviderRow& row = cp::Registry::at(i); if (cp::Provider::from_env(row.id).configured()) ++configured; credentials.push(Json(row.key_env)); } Json o = Json::object(); o.set("configured", Json(configured > 0)); o.set("configured_providers", Json(configured)); o.set("providers", Json(static_cast(cp::Registry::count()))); o.set("credential_envs", credentials); o.set("calls", Json(static_cast(cp::Calls::count()))); o.set("max_models_returned", Json(static_cast(cp::CompatCaps::kMaxModelsReturned))); return json_out(o); }); router.get("/ai/compat/providers", "ai-compat-providers", [](fw::Request req) { (void)req; Json arr = Json::array(); for (std::size_t i = 0; i < cp::Registry::count(); ++i) arr.push(provider_json(cp::Registry::at(i))); Json o = Json::object(); o.set("providers", arr); return json_out(o); }); router.get("/ai/compat/calls", "ai-compat-calls", [](fw::Request req) { (void)req; Json arr = Json::array(); for (std::size_t i = 0; i < cp::Calls::count(); ++i) { const cp::CallSpec& s = cp::Calls::at(i); Json row = Json::object(); row.set("http", Json(s.http_method)); row.set("path", Json(s.path)); row.set("operation", Json(s.operation_id)); row.set("feature", Json(s.feature_name)); arr.push(row); } Json o = Json::object(); o.set("calls", arr); return json_out(o); }); using Member = CompatRoutes::Planner; CompatRoutes* r = routes.get(); auto add = [&](const char* path, const char* name, Plan (CompatRoutes::*m)(const Json&) const) { router.post(path, name, [routes, r, m](fw::Request req) { return routes->run(req, Member([r, m](const Json& b) { return (r->*m)(b); })); }); }; add("/ai/compat/chat", "ai-compat-chat", &CompatRoutes::chat); add("/ai/compat/models", "ai-compat-models", &CompatRoutes::models); add("/ai/compat/embeddings", "ai-compat-embeddings", &CompatRoutes::embeddings); add("/ai/compat/images", "ai-compat-images", &CompatRoutes::images); add("/ai/compat/transcribe", "ai-compat-transcribe", &CompatRoutes::transcribe); add("/ai/compat/speech", "ai-compat-speech", &CompatRoutes::speech); } } // namespace molo::example::app