#include "../AiDemos.hpp" #include #include #include "../../fw/Router.hpp" #include "../../fw/Templates.hpp" #include "vendors/retrieval/brave/brave.h" #include "vendors/retrieval/caps.h" #include "vendors/retrieval/exa/exa.h" #include "vendors/retrieval/jina/jina.h" #include "vendors/retrieval/mixedbread/mixedbread.h" #include "vendors/retrieval/pinecone/pinecone.h" #include "vendors/retrieval/qdrant/qdrant.h" #include "vendors/retrieval/tavily/tavily.h" #include "vendors/retrieval/voyage/voyage.h" #include "vendors/retrieval/weaviate/weaviate.h" namespace molo::example::app { // Retrieval vendors (Voyage, Jina, Pinecone, Qdrant, Weaviate, Mixedbread, // Tavily, Exa, Brave) over the real src/vendors/retrieval package. GET routes // need no credential. Every route that calls a provider is a POST: 422 on bad // input, 503 retrieval_not_configured without the credential. Calls that cost // money, write, or destroy are dry runs that show the exact request unless the // body has "confirm":true. namespace { namespace rt = retrieval; using Json = clients::body::Json; const char kPage[] = R"RETRIEVAL({% extends "base.html" %} {% block content %}

Retrieval vendors

Embeddings, rerank, web search and vector databases from Voyage, Jina, Pinecone, Qdrant, Weaviate, Mixedbread, Tavily, Exa and Brave. Paid, writing and destructive calls are dry runs until the body says "confirm":true.

Checking credentials...


{% endblock %} )RETRIEVAL"; [[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); } fw::Response json_out(const Json& j, int status = 200) { fw::Response res = fw::JsonResponse(j.dump(), status); res.headers.set("Cache-Control", "no-store"); return res; } Json parse_body(const fw::Request& req) { if (req.body.size() > rt::RetrievalCaps::kMaxBodyBytes) throw fw::HttpError(413, "payload_too_large", "body too large"); if (req.body.empty()) return Json::object(); try { Json j = Json::parse(req.body); if (!j.is_object()) bad("body must be a JSON object"); return j; } catch (const clients::body::JsonError&) { bad("body is not valid JSON"); } } std::string text(const Json& b, const char* key, bool required = false) { const Json* v = b.find(key); if (!v || v->is_null()) { if (required) bad(std::string(key) + " is required"); return ""; } if (!v->is_string()) bad(std::string(key) + " must be a string"); return v->as_string(); } int number(const Json& b, const char* key, int fallback) { const Json* v = b.find(key); if (!v || v->is_null()) return fallback; if (!v->is_number()) bad(std::string(key) + " must be a number"); return static_cast(v->as_int()); } bool flag(const Json& b, const char* key) { const Json* v = b.find(key); if (!v || v->is_null()) return false; if (!v->is_bool()) bad(std::string(key) + " must be true or false"); return v->as_bool(); } std::vector strings(const Json& b, const char* key, bool required) { std::vector out; const Json* v = b.find(key); if (!v || v->is_null()) { if (required) bad(std::string(key) + " is required"); return out; } if (!v->is_array()) bad(std::string(key) + " must be an array of strings"); for (const Json& s : v->as_array()) { if (!s.is_string()) bad(std::string(key) + " must be an array of strings"); out.push_back(s.as_string()); } return out; } std::string scalar_text(const Json& v, const std::string& what) { if (v.is_string()) return v.as_string(); if (v.is_bool()) return v.as_bool() ? "true" : "false"; if (v.is_number()) return std::to_string(v.as_int()); bad(what + " values must be strings, numbers or booleans"); } std::map string_map(const Json& b, const char* key) { std::map out; const Json* v = b.find(key); if (!v || v->is_null()) return out; if (!v->is_object()) bad(std::string(key) + " must be an object"); for (const auto& [k, x] : v->as_object()) out[k] = scalar_text(x, key); return out; } Json dry_run(const Json& request) { Json o = Json::object(); o.set("dry_run", Json(true)); o.set("request", request); o.set("hint", Json("send confirm:true to run it")); return o; } // One provider's wiring: how to reach its Gateway and which seams it fills. struct Entry { std::string id; std::string title; std::string key_env; std::string url_env; // empty = fixed vendor host rt::CallTable (*table)(); std::function(const std::string& index_host)> gateway; std::function()> embedder; std::function()> reranker; std::function()> web; }; template std::shared_ptr gateway_of(P p) { return p.shared_gateway(); } class RetrievalRoutes { public: RetrievalRoutes() { add({"voyage", "Voyage AI", rt::Voyage::kKeyEnv, "", [] { return rt::voyage_calls(); }, [](const std::string&) { return gateway_of(rt::Voyage::from_env()); }, [] { return std::make_shared(rt::Voyage::from_env()); }, [] { return std::make_shared(rt::Voyage::from_env()); }, nullptr}); add({"jina", "Jina AI", rt::Jina::kKeyEnv, "", [] { return rt::jina_calls(); }, [](const std::string&) { return gateway_of(rt::Jina::from_env()); }, [] { return std::make_shared(rt::Jina::from_env()); }, [] { return std::make_shared(rt::Jina::from_env()); }, [] { return std::make_shared(rt::Jina::from_env()); }}); add({"pinecone", "Pinecone", rt::Pinecone::kKeyEnv, "", [] { return rt::pinecone_calls(); }, [](const std::string& host) { return gateway_of(rt::Pinecone::from_env(host)); }, [] { return std::make_shared(rt::Pinecone::from_env()); }, [] { return std::make_shared(rt::Pinecone::from_env()); }, nullptr}); add({"qdrant", "Qdrant", rt::Qdrant::kKeyEnv, rt::Qdrant::kUrlEnv, [] { return rt::qdrant_calls(); }, [](const std::string&) { return gateway_of(rt::Qdrant::from_env()); }, nullptr, nullptr, nullptr}); add({"weaviate", "Weaviate", rt::Weaviate::kKeyEnv, rt::Weaviate::kUrlEnv, [] { return rt::weaviate_calls(); }, [](const std::string&) { return gateway_of(rt::Weaviate::from_env()); }, nullptr, nullptr, nullptr}); add({"mixedbread", "Mixedbread", rt::Mixedbread::kKeyEnv, "", [] { return rt::mixedbread_calls(); }, [](const std::string&) { return gateway_of(rt::Mixedbread::from_env()); }, [] { return std::make_shared(rt::Mixedbread::from_env()); }, [] { return std::make_shared(rt::Mixedbread::from_env()); }, nullptr}); add({"tavily", "Tavily", rt::Tavily::kKeyEnv, "", [] { return rt::tavily_calls(); }, [](const std::string&) { return gateway_of(rt::Tavily::from_env()); }, nullptr, nullptr, [] { return std::make_shared(rt::Tavily::from_env()); }}); add({"exa", "Exa", rt::Exa::kKeyEnv, "", [] { return rt::exa_calls(); }, [](const std::string&) { return gateway_of(rt::Exa::from_env()); }, nullptr, nullptr, [] { return std::make_shared(rt::Exa::from_env()); }}); add({"brave", "Brave Search", rt::Brave::kKeyEnv, "", [] { return rt::brave_calls(); }, [](const std::string&) { return gateway_of(rt::Brave::from_env()); }, nullptr, nullptr, [] { return std::make_shared(rt::Brave::from_env()); }}); } const std::vector& entries() const { return entries_; } const Entry& entry(const std::string& id) const { for (const Entry& e : entries_) if (e.id == id) return e; bad("provider must be one of: " + ids()); } std::string ids() const { std::string out; for (const Entry& e : entries_) out += (out.empty() ? "" : ", ") + e.id; return out; } Json status() const { Json arr = Json::array(); for (const Entry& e : entries_) { Json p = Json::object(); p.set("id", Json(e.id)); p.set("title", Json(e.title)); p.set("credential_env", Json(e.key_env)); p.set("url_env", Json(e.url_env)); p.set("configured", Json(e.gateway("")->configured())); p.set("calls", Json(static_cast(e.table().size()))); Json seams = Json::array(); if (e.embedder) seams.push(Json("embed")); if (e.reranker) seams.push(Json("rerank")); if (e.web) seams.push(Json("search")); p.set("seams", seams); arr.push(p); } Json o = Json::object(); o.set("providers", arr); Json caps = Json::object(); caps.set("max_embed_texts", Json(static_cast(rt::RetrievalCaps::kMaxEmbedTexts))); caps.set("max_rerank_docs", Json(static_cast(rt::RetrievalCaps::kMaxRerankDocs))); caps.set("max_search_results", Json(static_cast(rt::RetrievalCaps::kMaxSearchResults))); caps.set("max_body_bytes", Json(static_cast(rt::RetrievalCaps::kMaxBodyBytes))); o.set("caps", caps); return o; } Json calls(const std::string& provider) const { Json arr = Json::array(); for (const Entry& e : entries_) { if (!provider.empty() && e.id != provider) continue; rt::CallTable t = e.table(); for (std::size_t i = 0; i < t.size(); ++i) { const rt::CallRow& r = t.at(i); Json row = Json::object(); row.set("provider", Json(e.id)); row.set("name", Json(r.name)); row.set("http", Json(r.method)); row.set("path", Json(r.path)); row.set("host", Json(r.host)); row.set("flags", Json(r.flags)); row.set("unverified", Json(rt::RowInfo::unverified(r))); arr.push(row); } } if (arr.as_array().empty() && !provider.empty()) bad("provider must be one of: " + ids()); Json o = Json::object(); o.set("count", Json(static_cast(arr.as_array().size()))); o.set("calls", arr); return o; } // POST /ai/retrieval/call fw::Response call(const fw::Request& req) const { Json b = parse_body(req); const Entry& e = entry(text(b, "provider", true)); std::string name = text(b, "call", true); rt::CallArgs args; args.path = string_map(b, "path"); args.query = string_map(b, "query"); if (const Json* body = b.find("body"); body && !body->is_null()) args.body = *body; std::string index_host = text(b, "index_host"); std::shared_ptr gw; rt::PlannedCall plan; try { if (!index_host.empty()) rt::check_host(e.id, index_host); gw = e.gateway(index_host); plan = gw->plan(name, args); } catch (const llm::ApiError& err) { map(err); } bool guarded = plan.destructive || plan.writes || plan.paid; if (guarded && !flag(b, "confirm")) { Json r = Json::object(); r.set("provider", Json(e.id)); r.set("call", Json(plan.name)); r.set("method", Json(plan.method)); r.set("host", Json(plan.host)); r.set("target", Json(plan.target)); r.set("flags", Json(plan.flags)); r.set("body", plan.body); return json_out(dry_run(r)); } try { Json out = plan.destructive ? gw->destroy(name, args) : gw->call(name, args); return json_out(out); } catch (const llm::ApiError& err) { map(err); } } // POST /ai/retrieval/embed fw::Response embed(const fw::Request& req) const { Json b = parse_body(req); const Entry& e = entry(text(b, "provider", true)); if (!e.embedder) bad(e.id + " has no embeddings"); rt::EmbedRequest r; r.texts = strings(b, "texts", true); r.model = text(b, "model"); r.input_type = text(b, "input_type"); r.dimensions = number(b, "dimensions", 0); try { rt::Embedder::validate(e.id, r); } catch (const llm::ApiError& err) { map(err); } if (!flag(b, "confirm")) { Json q = Json::object(); q.set("provider", Json(e.id)); q.set("texts", Json(static_cast(r.texts.size()))); q.set("model", Json(r.model.empty() ? "(provider default)" : r.model)); q.set("input_type", Json(r.input_type)); return json_out(dry_run(q)); } try { rt::EmbedResult out = e.embedder()->embed(r); Json o = Json::object(); o.set("model", Json(out.model)); o.set("total_tokens", Json(static_cast(out.total_tokens))); o.set("count", Json(static_cast(out.vectors.size()))); o.set("dimensions", Json(out.vectors.empty() ? 0 : static_cast(out.vectors[0].size()))); Json first = Json::array(); if (!out.vectors.empty()) for (std::size_t i = 0; i < out.vectors[0].size() && i < 8; ++i) first.push(Json(static_cast(out.vectors[0][i]))); o.set("first_vector_head", first); return json_out(o); } catch (const llm::ApiError& err) { map(err); } } // POST /ai/retrieval/rerank fw::Response rerank(const fw::Request& req) const { Json b = parse_body(req); const Entry& e = entry(text(b, "provider", true)); if (!e.reranker) bad(e.id + " has no rerank"); rt::RerankRequest r; r.query = text(b, "query", true); r.documents = strings(b, "documents", true); r.model = text(b, "model"); r.top_n = number(b, "top_n", 0); try { rt::Reranker::validate(e.id, r); } catch (const llm::ApiError& err) { map(err); } if (!flag(b, "confirm")) { Json q = Json::object(); q.set("provider", Json(e.id)); q.set("query", Json(r.query)); q.set("documents", Json(static_cast(r.documents.size()))); q.set("top_n", Json(r.top_n)); return json_out(dry_run(q)); } try { rt::RerankResult out = e.reranker()->rerank(r); Json hits = Json::array(); for (const rt::RerankHit& h : out.hits) { Json x = Json::object(); x.set("index", Json(h.index)); x.set("score", Json(h.score)); hits.push(x); } Json o = Json::object(); o.set("model", Json(out.model)); o.set("total_tokens", Json(static_cast(out.total_tokens))); o.set("hits", hits); return json_out(o); } catch (const llm::ApiError& err) { map(err); } } // POST /ai/retrieval/search fw::Response search(const fw::Request& req) const { Json b = parse_body(req); const Entry& e = entry(text(b, "provider", true)); if (!e.web) bad(e.id + " has no web search"); rt::SearchRequest r; r.query = text(b, "query", true); r.max_results = number(b, "max_results", 5); r.recency = text(b, "recency"); try { rt::WebSearch::validate(e.id, r); } catch (const llm::ApiError& err) { map(err); } if (!flag(b, "confirm")) { Json q = Json::object(); q.set("provider", Json(e.id)); q.set("query", Json(r.query)); q.set("max_results", Json(r.max_results)); q.set("recency", Json(r.recency)); return json_out(dry_run(q)); } try { rt::SearchResult out = e.web()->search(r); Json hits = Json::array(); for (const rt::SearchHit& h : out.hits) { Json x = Json::object(); x.set("title", Json(h.title)); x.set("url", Json(h.url)); x.set("snippet", Json(h.snippet)); x.set("published", Json(h.published)); hits.push(x); } Json o = Json::object(); o.set("answer", Json(out.answer)); o.set("hits", hits); return json_out(o); } catch (const llm::ApiError& err) { map(err); } } private: void add(Entry e) { entries_.push_back(std::move(e)); } [[noreturn]] static void map(const llm::ApiError& e) { if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED") throw fw::HttpError(503, "retrieval_not_configured", e.message()); if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty()) throw fw::HttpError(422, "validation_error", e.message()); throw fw::HttpError(502, "retrieval_error", e.vendor() + " " + std::to_string(e.http_status()) + " " + e.type() + ": " + e.message()); } std::vector entries_; }; } // namespace void register_ai_retrieval_demo(fw::Router& router, Services& services) { auto routes = std::make_shared(); fw::Environment* env = services.templates.get(); services.templates->add_template("ai_retrieval.html", kPage); services.add_feature("ai-retrieval", "/ai/retrieval"); router.get("/ai/retrieval", "ai-retrieval-page", [env](fw::Request req) { (void)req; fw::TemplateCtx ctx; ctx["title"] = std::string("Retrieval vendors"); return fw::Response(200, env->render("ai_retrieval.html", ctx), "text/html; charset=utf-8"); }); router.get("/ai/retrieval/status", "ai-retrieval-status", [routes](fw::Request req) { (void)req; return json_out(routes->status()); }); router.get("/ai/retrieval/calls", "ai-retrieval-calls", [routes](fw::Request req) { return json_out(routes->calls(req.query_or("provider", ""))); }); router.post("/ai/retrieval/call", "ai-retrieval-call", [routes](fw::Request req) { return routes->call(req); }); router.post("/ai/retrieval/embed", "ai-retrieval-embed", [routes](fw::Request req) { return routes->embed(req); }); router.post("/ai/retrieval/rerank", "ai-retrieval-rerank", [routes](fw::Request req) { return routes->rerank(req); }); router.post("/ai/retrieval/search", "ai-retrieval-search", [routes](fw::Request req) { return routes->search(req); }); } } // namespace molo::example::app