Files
moloapp/example/app/ai/RetrievalDemo.cpp
T

476 lines
18 KiB
C++
Raw Normal View History

#include "../AiDemos.hpp"
#include <functional>
#include <map>
#include "../../fw/Router.hpp"
#include "../../fw/Templates.hpp"
#include "vendors/retrieval/brave/brave.h"
#include "vendors/retrieval/caps.h"
#include "vendors/retrieval/exa/exa.h"
#include "vendors/retrieval/jina/jina.h"
#include "vendors/retrieval/mixedbread/mixedbread.h"
#include "vendors/retrieval/pinecone/pinecone.h"
#include "vendors/retrieval/qdrant/qdrant.h"
#include "vendors/retrieval/tavily/tavily.h"
#include "vendors/retrieval/voyage/voyage.h"
#include "vendors/retrieval/weaviate/weaviate.h"
namespace molo::example::app {
// Retrieval vendors (Voyage, Jina, Pinecone, Qdrant, Weaviate, Mixedbread,
// Tavily, Exa, Brave) over the real src/vendors/retrieval package. GET routes
// need no credential. Every route that calls a provider is a POST: 422 on bad
// input, 503 retrieval_not_configured without the credential. Calls that cost
// money, write, or destroy are dry runs that show the exact request unless the
// body has "confirm":true.
namespace {
namespace rt = retrieval;
using Json = clients::body::Json;
const char kPage[] = R"RETRIEVAL({% extends "base.html" %}
{% block content %}
<h1>Retrieval vendors</h1>
<p class="hint">Embeddings, rerank, web search and vector databases from Voyage, Jina, Pinecone, Qdrant,
Weaviate, Mixedbread, Tavily, Exa and Brave. Paid, writing and destructive calls are dry runs until the
body says <code>"confirm":true</code>.</p>
<p id="rt-config" class="hint">Checking credentials...</p>
<section class="rt">
<label>Operation
<select id="rt-op"></select></label>
<label>Provider
<select id="rt-provider"></select></label>
<label>Call (for the generic call route)
<select id="rt-call"></select></label>
<label>Body (JSON object) <textarea id="rt-body" rows="9"></textarea></label>
<div class="rt-actions">
<button id="rt-send" type="button" class="primary">Send</button>
<button id="rt-confirm" type="button">Send with confirm</button>
</div>
<p id="rt-error" class="error" role="alert"></p>
<pre id="rt-out" class="rt-out"></pre>
</section>
<style>
.rt label{display:block;margin:.5rem 0}
.rt select,.rt textarea{width:100%;box-sizing:border-box}
.rt-actions{display:flex;flex-wrap:wrap;gap:.5rem;margin:.5rem 0}
.rt-out{overflow:auto;max-height:24rem;white-space:pre-wrap;word-break:break-word}
</style>
<script src="/embedded/ai_retrieval.js"></script>
{% endblock %}
)RETRIEVAL";
[[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); }
fw::Response json_out(const Json& j, int status = 200) {
fw::Response res = fw::JsonResponse(j.dump(), status);
res.headers.set("Cache-Control", "no-store");
return res;
}
Json parse_body(const fw::Request& req) {
if (req.body.size() > rt::RetrievalCaps::kMaxBodyBytes) throw fw::HttpError(413, "payload_too_large", "body too large");
if (req.body.empty()) return Json::object();
try {
Json j = Json::parse(req.body);
if (!j.is_object()) bad("body must be a JSON object");
return j;
} catch (const clients::body::JsonError&) {
bad("body is not valid JSON");
}
}
std::string text(const Json& b, const char* key, bool required = false) {
const Json* v = b.find(key);
if (!v || v->is_null()) {
if (required) bad(std::string(key) + " is required");
return "";
}
if (!v->is_string()) bad(std::string(key) + " must be a string");
return v->as_string();
}
int number(const Json& b, const char* key, int fallback) {
const Json* v = b.find(key);
if (!v || v->is_null()) return fallback;
if (!v->is_number()) bad(std::string(key) + " must be a number");
return static_cast<int>(v->as_int());
}
bool flag(const Json& b, const char* key) {
const Json* v = b.find(key);
if (!v || v->is_null()) return false;
if (!v->is_bool()) bad(std::string(key) + " must be true or false");
return v->as_bool();
}
std::vector<std::string> strings(const Json& b, const char* key, bool required) {
std::vector<std::string> out;
const Json* v = b.find(key);
if (!v || v->is_null()) {
if (required) bad(std::string(key) + " is required");
return out;
}
if (!v->is_array()) bad(std::string(key) + " must be an array of strings");
for (const Json& s : v->as_array()) {
if (!s.is_string()) bad(std::string(key) + " must be an array of strings");
out.push_back(s.as_string());
}
return out;
}
std::string scalar_text(const Json& v, const std::string& what) {
if (v.is_string()) return v.as_string();
if (v.is_bool()) return v.as_bool() ? "true" : "false";
if (v.is_number()) return std::to_string(v.as_int());
bad(what + " values must be strings, numbers or booleans");
}
std::map<std::string, std::string> string_map(const Json& b, const char* key) {
std::map<std::string, std::string> out;
const Json* v = b.find(key);
if (!v || v->is_null()) return out;
if (!v->is_object()) bad(std::string(key) + " must be an object");
for (const auto& [k, x] : v->as_object()) out[k] = scalar_text(x, key);
return out;
}
Json dry_run(const Json& request) {
Json o = Json::object();
o.set("dry_run", Json(true));
o.set("request", request);
o.set("hint", Json("send confirm:true to run it"));
return o;
}
// One provider's wiring: how to reach its Gateway and which seams it fills.
struct Entry {
std::string id;
std::string title;
std::string key_env;
std::string url_env; // empty = fixed vendor host
rt::CallTable (*table)();
std::function<std::shared_ptr<rt::Gateway>(const std::string& index_host)> gateway;
std::function<std::shared_ptr<rt::Embedder>()> embedder;
std::function<std::shared_ptr<rt::Reranker>()> reranker;
std::function<std::shared_ptr<rt::WebSearch>()> web;
};
template <typename P>
std::shared_ptr<rt::Gateway> gateway_of(P p) {
return p.shared_gateway();
}
class RetrievalRoutes {
public:
RetrievalRoutes() {
add({"voyage", "Voyage AI", rt::Voyage::kKeyEnv, "", [] { return rt::voyage_calls(); },
[](const std::string&) { return gateway_of(rt::Voyage::from_env()); },
[] { return std::make_shared<rt::Voyage>(rt::Voyage::from_env()); },
[] { return std::make_shared<rt::Voyage>(rt::Voyage::from_env()); }, nullptr});
add({"jina", "Jina AI", rt::Jina::kKeyEnv, "", [] { return rt::jina_calls(); },
[](const std::string&) { return gateway_of(rt::Jina::from_env()); },
[] { return std::make_shared<rt::Jina>(rt::Jina::from_env()); },
[] { return std::make_shared<rt::Jina>(rt::Jina::from_env()); },
[] { return std::make_shared<rt::Jina>(rt::Jina::from_env()); }});
add({"pinecone", "Pinecone", rt::Pinecone::kKeyEnv, "", [] { return rt::pinecone_calls(); },
[](const std::string& host) { return gateway_of(rt::Pinecone::from_env(host)); },
[] { return std::make_shared<rt::Pinecone>(rt::Pinecone::from_env()); },
[] { return std::make_shared<rt::Pinecone>(rt::Pinecone::from_env()); }, nullptr});
add({"qdrant", "Qdrant", rt::Qdrant::kKeyEnv, rt::Qdrant::kUrlEnv, [] { return rt::qdrant_calls(); },
[](const std::string&) { return gateway_of(rt::Qdrant::from_env()); }, nullptr, nullptr, nullptr});
add({"weaviate", "Weaviate", rt::Weaviate::kKeyEnv, rt::Weaviate::kUrlEnv, [] { return rt::weaviate_calls(); },
[](const std::string&) { return gateway_of(rt::Weaviate::from_env()); }, nullptr, nullptr, nullptr});
add({"mixedbread", "Mixedbread", rt::Mixedbread::kKeyEnv, "", [] { return rt::mixedbread_calls(); },
[](const std::string&) { return gateway_of(rt::Mixedbread::from_env()); },
[] { return std::make_shared<rt::Mixedbread>(rt::Mixedbread::from_env()); },
[] { return std::make_shared<rt::Mixedbread>(rt::Mixedbread::from_env()); }, nullptr});
add({"tavily", "Tavily", rt::Tavily::kKeyEnv, "", [] { return rt::tavily_calls(); },
[](const std::string&) { return gateway_of(rt::Tavily::from_env()); }, nullptr, nullptr,
[] { return std::make_shared<rt::Tavily>(rt::Tavily::from_env()); }});
add({"exa", "Exa", rt::Exa::kKeyEnv, "", [] { return rt::exa_calls(); },
[](const std::string&) { return gateway_of(rt::Exa::from_env()); }, nullptr, nullptr,
[] { return std::make_shared<rt::Exa>(rt::Exa::from_env()); }});
add({"brave", "Brave Search", rt::Brave::kKeyEnv, "", [] { return rt::brave_calls(); },
[](const std::string&) { return gateway_of(rt::Brave::from_env()); }, nullptr, nullptr,
[] { return std::make_shared<rt::Brave>(rt::Brave::from_env()); }});
}
const std::vector<Entry>& entries() const { return entries_; }
const Entry& entry(const std::string& id) const {
for (const Entry& e : entries_)
if (e.id == id) return e;
bad("provider must be one of: " + ids());
}
std::string ids() const {
std::string out;
for (const Entry& e : entries_) out += (out.empty() ? "" : ", ") + e.id;
return out;
}
Json status() const {
Json arr = Json::array();
for (const Entry& e : entries_) {
Json p = Json::object();
p.set("id", Json(e.id));
p.set("title", Json(e.title));
p.set("credential_env", Json(e.key_env));
p.set("url_env", Json(e.url_env));
p.set("configured", Json(e.gateway("")->configured()));
p.set("calls", Json(static_cast<int>(e.table().size())));
Json seams = Json::array();
if (e.embedder) seams.push(Json("embed"));
if (e.reranker) seams.push(Json("rerank"));
if (e.web) seams.push(Json("search"));
p.set("seams", seams);
arr.push(p);
}
Json o = Json::object();
o.set("providers", arr);
Json caps = Json::object();
caps.set("max_embed_texts", Json(static_cast<int>(rt::RetrievalCaps::kMaxEmbedTexts)));
caps.set("max_rerank_docs", Json(static_cast<int>(rt::RetrievalCaps::kMaxRerankDocs)));
caps.set("max_search_results", Json(static_cast<int>(rt::RetrievalCaps::kMaxSearchResults)));
caps.set("max_body_bytes", Json(static_cast<int>(rt::RetrievalCaps::kMaxBodyBytes)));
o.set("caps", caps);
return o;
}
Json calls(const std::string& provider) const {
Json arr = Json::array();
for (const Entry& e : entries_) {
if (!provider.empty() && e.id != provider) continue;
rt::CallTable t = e.table();
for (std::size_t i = 0; i < t.size(); ++i) {
const rt::CallRow& r = t.at(i);
Json row = Json::object();
row.set("provider", Json(e.id));
row.set("name", Json(r.name));
row.set("http", Json(r.method));
row.set("path", Json(r.path));
row.set("host", Json(r.host));
row.set("flags", Json(r.flags));
row.set("unverified", Json(rt::RowInfo::unverified(r)));
arr.push(row);
}
}
if (arr.as_array().empty() && !provider.empty()) bad("provider must be one of: " + ids());
Json o = Json::object();
o.set("count", Json(static_cast<int>(arr.as_array().size())));
o.set("calls", arr);
return o;
}
// POST /ai/retrieval/call
fw::Response call(const fw::Request& req) const {
Json b = parse_body(req);
const Entry& e = entry(text(b, "provider", true));
std::string name = text(b, "call", true);
rt::CallArgs args;
args.path = string_map(b, "path");
args.query = string_map(b, "query");
if (const Json* body = b.find("body"); body && !body->is_null()) args.body = *body;
std::string index_host = text(b, "index_host");
std::shared_ptr<rt::Gateway> gw;
rt::PlannedCall plan;
try {
if (!index_host.empty()) rt::check_host(e.id, index_host);
gw = e.gateway(index_host);
plan = gw->plan(name, args);
} catch (const llm::ApiError& err) {
map(err);
}
bool guarded = plan.destructive || plan.writes || plan.paid;
if (guarded && !flag(b, "confirm")) {
Json r = Json::object();
r.set("provider", Json(e.id));
r.set("call", Json(plan.name));
r.set("method", Json(plan.method));
r.set("host", Json(plan.host));
r.set("target", Json(plan.target));
r.set("flags", Json(plan.flags));
r.set("body", plan.body);
return json_out(dry_run(r));
}
try {
Json out = plan.destructive ? gw->destroy(name, args) : gw->call(name, args);
return json_out(out);
} catch (const llm::ApiError& err) {
map(err);
}
}
// POST /ai/retrieval/embed
fw::Response embed(const fw::Request& req) const {
Json b = parse_body(req);
const Entry& e = entry(text(b, "provider", true));
if (!e.embedder) bad(e.id + " has no embeddings");
rt::EmbedRequest r;
r.texts = strings(b, "texts", true);
r.model = text(b, "model");
r.input_type = text(b, "input_type");
r.dimensions = number(b, "dimensions", 0);
try {
rt::Embedder::validate(e.id, r);
} catch (const llm::ApiError& err) {
map(err);
}
if (!flag(b, "confirm")) {
Json q = Json::object();
q.set("provider", Json(e.id));
q.set("texts", Json(static_cast<int>(r.texts.size())));
q.set("model", Json(r.model.empty() ? "(provider default)" : r.model));
q.set("input_type", Json(r.input_type));
return json_out(dry_run(q));
}
try {
rt::EmbedResult out = e.embedder()->embed(r);
Json o = Json::object();
o.set("model", Json(out.model));
o.set("total_tokens", Json(static_cast<int>(out.total_tokens)));
o.set("count", Json(static_cast<int>(out.vectors.size())));
o.set("dimensions", Json(out.vectors.empty() ? 0 : static_cast<int>(out.vectors[0].size())));
Json first = Json::array();
if (!out.vectors.empty())
for (std::size_t i = 0; i < out.vectors[0].size() && i < 8; ++i) first.push(Json(static_cast<double>(out.vectors[0][i])));
o.set("first_vector_head", first);
return json_out(o);
} catch (const llm::ApiError& err) {
map(err);
}
}
// POST /ai/retrieval/rerank
fw::Response rerank(const fw::Request& req) const {
Json b = parse_body(req);
const Entry& e = entry(text(b, "provider", true));
if (!e.reranker) bad(e.id + " has no rerank");
rt::RerankRequest r;
r.query = text(b, "query", true);
r.documents = strings(b, "documents", true);
r.model = text(b, "model");
r.top_n = number(b, "top_n", 0);
try {
rt::Reranker::validate(e.id, r);
} catch (const llm::ApiError& err) {
map(err);
}
if (!flag(b, "confirm")) {
Json q = Json::object();
q.set("provider", Json(e.id));
q.set("query", Json(r.query));
q.set("documents", Json(static_cast<int>(r.documents.size())));
q.set("top_n", Json(r.top_n));
return json_out(dry_run(q));
}
try {
rt::RerankResult out = e.reranker()->rerank(r);
Json hits = Json::array();
for (const rt::RerankHit& h : out.hits) {
Json x = Json::object();
x.set("index", Json(h.index));
x.set("score", Json(h.score));
hits.push(x);
}
Json o = Json::object();
o.set("model", Json(out.model));
o.set("total_tokens", Json(static_cast<int>(out.total_tokens)));
o.set("hits", hits);
return json_out(o);
} catch (const llm::ApiError& err) {
map(err);
}
}
// POST /ai/retrieval/search
fw::Response search(const fw::Request& req) const {
Json b = parse_body(req);
const Entry& e = entry(text(b, "provider", true));
if (!e.web) bad(e.id + " has no web search");
rt::SearchRequest r;
r.query = text(b, "query", true);
r.max_results = number(b, "max_results", 5);
r.recency = text(b, "recency");
try {
rt::WebSearch::validate(e.id, r);
} catch (const llm::ApiError& err) {
map(err);
}
if (!flag(b, "confirm")) {
Json q = Json::object();
q.set("provider", Json(e.id));
q.set("query", Json(r.query));
q.set("max_results", Json(r.max_results));
q.set("recency", Json(r.recency));
return json_out(dry_run(q));
}
try {
rt::SearchResult out = e.web()->search(r);
Json hits = Json::array();
for (const rt::SearchHit& h : out.hits) {
Json x = Json::object();
x.set("title", Json(h.title));
x.set("url", Json(h.url));
x.set("snippet", Json(h.snippet));
x.set("published", Json(h.published));
hits.push(x);
}
Json o = Json::object();
o.set("answer", Json(out.answer));
o.set("hits", hits);
return json_out(o);
} catch (const llm::ApiError& err) {
map(err);
}
}
private:
void add(Entry e) { entries_.push_back(std::move(e)); }
[[noreturn]] static void map(const llm::ApiError& e) {
if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED")
throw fw::HttpError(503, "retrieval_not_configured", e.message());
if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty())
throw fw::HttpError(422, "validation_error", e.message());
throw fw::HttpError(502, "retrieval_error", e.vendor() + " " + std::to_string(e.http_status()) + " " + e.type() + ": " + e.message());
}
std::vector<Entry> entries_;
};
} // namespace
void register_ai_retrieval_demo(fw::Router& router, Services& services) {
auto routes = std::make_shared<RetrievalRoutes>();
fw::Environment* env = services.templates.get();
services.templates->add_template("ai_retrieval.html", kPage);
services.add_feature("ai-retrieval", "/ai/retrieval");
router.get("/ai/retrieval", "ai-retrieval-page", [env](fw::Request req) {
(void)req;
fw::TemplateCtx ctx;
ctx["title"] = std::string("Retrieval vendors");
return fw::Response(200, env->render("ai_retrieval.html", ctx), "text/html; charset=utf-8");
});
router.get("/ai/retrieval/status", "ai-retrieval-status", [routes](fw::Request req) {
(void)req;
return json_out(routes->status());
});
router.get("/ai/retrieval/calls", "ai-retrieval-calls", [routes](fw::Request req) {
return json_out(routes->calls(req.query_or("provider", "")));
});
router.post("/ai/retrieval/call", "ai-retrieval-call", [routes](fw::Request req) { return routes->call(req); });
router.post("/ai/retrieval/embed", "ai-retrieval-embed", [routes](fw::Request req) { return routes->embed(req); });
router.post("/ai/retrieval/rerank", "ai-retrieval-rerank", [routes](fw::Request req) { return routes->rerank(req); });
router.post("/ai/retrieval/search", "ai-retrieval-search", [routes](fw::Request req) { return routes->search(req); });
}
} // namespace molo::example::app