From 2ac5859cdb1bf25d7dee20571b8ec7d97ea21d8f Mon Sep 17 00:00:00 2001 From: retoor Date: Sun, 25 Oct 2026 15:15:34 +0100 Subject: [PATCH] feat(clients): retrieval: Voyage, Jina, Pinecone, Qdrant, Weaviate, Mixedbread, Tavily, Exa, Brave Co-Authored-By: Claude Sonnet 5.5 --- example/app/ai/RetrievalDemo.cpp | 468 ++++++++++++- example/assets/ai_retrieval.js | 78 +++ src/vendors/retrieval/SPEC.md | 224 ++++++ src/vendors/retrieval/brave/brave.cpp | 172 +++++ src/vendors/retrieval/brave/brave.h | 56 ++ src/vendors/retrieval/brave/calls.cpp | 34 + src/vendors/retrieval/call_table.cpp | 41 ++ src/vendors/retrieval/call_table.h | 50 ++ src/vendors/retrieval/caps.h | 47 ++ src/vendors/retrieval/exa/calls.cpp | 25 + src/vendors/retrieval/exa/exa.cpp | 173 +++++ src/vendors/retrieval/exa/exa.h | 49 ++ src/vendors/retrieval/gateway.cpp | 120 ++++ src/vendors/retrieval/gateway.h | 81 +++ src/vendors/retrieval/jina/calls.cpp | 33 + src/vendors/retrieval/jina/jina.cpp | 217 ++++++ src/vendors/retrieval/jina/jina.h | 80 +++ src/vendors/retrieval/mixedbread/calls.cpp | 98 +++ .../retrieval/mixedbread/mixedbread.cpp | 160 +++++ src/vendors/retrieval/mixedbread/mixedbread.h | 53 ++ src/vendors/retrieval/pinecone/calls.cpp | 75 ++ src/vendors/retrieval/pinecone/pinecone.cpp | 312 +++++++++ src/vendors/retrieval/pinecone/pinecone.h | 107 +++ src/vendors/retrieval/provider.cpp | 75 ++ src/vendors/retrieval/provider.h | 60 ++ src/vendors/retrieval/qdrant/calls.cpp | 67 ++ src/vendors/retrieval/qdrant/qdrant.cpp | 195 ++++++ src/vendors/retrieval/qdrant/qdrant.h | 66 ++ src/vendors/retrieval/seams.cpp | 48 ++ src/vendors/retrieval/seams.h | 92 +++ src/vendors/retrieval/tavily/calls.cpp | 25 + src/vendors/retrieval/tavily/tavily.cpp | 142 ++++ src/vendors/retrieval/tavily/tavily.h | 68 ++ .../__pycache__/check_spec.cpython-314.pyc | Bin 0 -> 40199 bytes src/vendors/retrieval/tools/check_spec.py | 649 ++++++++++++++++++ src/vendors/retrieval/voyage/calls.cpp | 22 + src/vendors/retrieval/voyage/voyage.cpp | 169 +++++ src/vendors/retrieval/voyage/voyage.h | 65 ++ src/vendors/retrieval/weaviate/calls.cpp | 84 +++ src/vendors/retrieval/weaviate/weaviate.cpp | 162 +++++ src/vendors/retrieval/weaviate/weaviate.h | 54 ++ tests/example/ai/retrieval/call_test.cpp | 58 ++ tests/example/ai/retrieval/page_test.cpp | 41 ++ tests/example/ai/retrieval/routes_test.cpp | 69 ++ tests/src/vendors/retrieval/support.h | 71 ++ tests/src/vendors/retrieval/test_core.cpp | 265 +++++++ tests/src/vendors/retrieval/test_live.cpp | 166 +++++ .../src/vendors/retrieval/test_providers.cpp | 577 ++++++++++++++++ 48 files changed, 6040 insertions(+), 3 deletions(-) create mode 100644 example/assets/ai_retrieval.js create mode 100644 src/vendors/retrieval/SPEC.md create mode 100644 src/vendors/retrieval/brave/brave.cpp create mode 100644 src/vendors/retrieval/brave/brave.h create mode 100644 src/vendors/retrieval/brave/calls.cpp create mode 100644 src/vendors/retrieval/call_table.cpp create mode 100644 src/vendors/retrieval/call_table.h create mode 100644 src/vendors/retrieval/caps.h create mode 100644 src/vendors/retrieval/exa/calls.cpp create mode 100644 src/vendors/retrieval/exa/exa.cpp create mode 100644 src/vendors/retrieval/exa/exa.h create mode 100644 src/vendors/retrieval/gateway.cpp create mode 100644 src/vendors/retrieval/gateway.h create mode 100644 src/vendors/retrieval/jina/calls.cpp create mode 100644 src/vendors/retrieval/jina/jina.cpp create mode 100644 src/vendors/retrieval/jina/jina.h create mode 100644 src/vendors/retrieval/mixedbread/calls.cpp create mode 100644 src/vendors/retrieval/mixedbread/mixedbread.cpp create mode 100644 src/vendors/retrieval/mixedbread/mixedbread.h create mode 100644 src/vendors/retrieval/pinecone/calls.cpp create mode 100644 src/vendors/retrieval/pinecone/pinecone.cpp create mode 100644 src/vendors/retrieval/pinecone/pinecone.h create mode 100644 src/vendors/retrieval/provider.cpp create mode 100644 src/vendors/retrieval/provider.h create mode 100644 src/vendors/retrieval/qdrant/calls.cpp create mode 100644 src/vendors/retrieval/qdrant/qdrant.cpp create mode 100644 src/vendors/retrieval/qdrant/qdrant.h create mode 100644 src/vendors/retrieval/seams.cpp create mode 100644 src/vendors/retrieval/seams.h create mode 100644 src/vendors/retrieval/tavily/calls.cpp create mode 100644 src/vendors/retrieval/tavily/tavily.cpp create mode 100644 src/vendors/retrieval/tavily/tavily.h create mode 100644 src/vendors/retrieval/tools/__pycache__/check_spec.cpython-314.pyc create mode 100644 src/vendors/retrieval/tools/check_spec.py create mode 100644 src/vendors/retrieval/voyage/calls.cpp create mode 100644 src/vendors/retrieval/voyage/voyage.cpp create mode 100644 src/vendors/retrieval/voyage/voyage.h create mode 100644 src/vendors/retrieval/weaviate/calls.cpp create mode 100644 src/vendors/retrieval/weaviate/weaviate.cpp create mode 100644 src/vendors/retrieval/weaviate/weaviate.h create mode 100644 tests/example/ai/retrieval/call_test.cpp create mode 100644 tests/example/ai/retrieval/page_test.cpp create mode 100644 tests/example/ai/retrieval/routes_test.cpp create mode 100644 tests/src/vendors/retrieval/support.h create mode 100644 tests/src/vendors/retrieval/test_core.cpp create mode 100644 tests/src/vendors/retrieval/test_live.cpp create mode 100644 tests/src/vendors/retrieval/test_providers.cpp diff --git a/example/app/ai/RetrievalDemo.cpp b/example/app/ai/RetrievalDemo.cpp index fe9ca0c..5d08243 100644 --- a/example/app/ai/RetrievalDemo.cpp +++ b/example/app/ai/RetrievalDemo.cpp @@ -1,13 +1,475 @@ #include "../AiDemos.hpp" +#include +#include + #include "../../fw/Router.hpp" +#include "../../fw/Templates.hpp" +#include "vendors/retrieval/brave/brave.h" +#include "vendors/retrieval/caps.h" +#include "vendors/retrieval/exa/exa.h" +#include "vendors/retrieval/jina/jina.h" +#include "vendors/retrieval/mixedbread/mixedbread.h" +#include "vendors/retrieval/pinecone/pinecone.h" +#include "vendors/retrieval/qdrant/qdrant.h" +#include "vendors/retrieval/tavily/tavily.h" +#include "vendors/retrieval/voyage/voyage.h" +#include "vendors/retrieval/weaviate/weaviate.h" namespace molo::example::app { -// retrieval. Stub: the package agent fills this in. +// Retrieval vendors (Voyage, Jina, Pinecone, Qdrant, Weaviate, Mixedbread, +// Tavily, Exa, Brave) over the real src/vendors/retrieval package. GET routes +// need no credential. Every route that calls a provider is a POST: 422 on bad +// input, 503 retrieval_not_configured without the credential. Calls that cost +// money, write, or destroy are dry runs that show the exact request unless the +// body has "confirm":true. + +namespace { + +namespace rt = retrieval; +using Json = clients::body::Json; + +const char kPage[] = R"RETRIEVAL({% extends "base.html" %} +{% block content %} +

Retrieval vendors

+

Embeddings, rerank, web search and vector databases from Voyage, Jina, Pinecone, Qdrant, +Weaviate, Mixedbread, Tavily, Exa and Brave. Paid, writing and destructive calls are dry runs until the +body says "confirm":true.

+

Checking credentials...

+
+ + + + +
+ + +
+ +

+
+ + +{% endblock %} +)RETRIEVAL"; + +[[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); } + +fw::Response json_out(const Json& j, int status = 200) { + fw::Response res = fw::JsonResponse(j.dump(), status); + res.headers.set("Cache-Control", "no-store"); + return res; +} + +Json parse_body(const fw::Request& req) { + if (req.body.size() > rt::RetrievalCaps::kMaxBodyBytes) throw fw::HttpError(413, "payload_too_large", "body too large"); + if (req.body.empty()) return Json::object(); + try { + Json j = Json::parse(req.body); + if (!j.is_object()) bad("body must be a JSON object"); + return j; + } catch (const clients::body::JsonError&) { + bad("body is not valid JSON"); + } +} + +std::string text(const Json& b, const char* key, bool required = false) { + const Json* v = b.find(key); + if (!v || v->is_null()) { + if (required) bad(std::string(key) + " is required"); + return ""; + } + if (!v->is_string()) bad(std::string(key) + " must be a string"); + return v->as_string(); +} + +int number(const Json& b, const char* key, int fallback) { + const Json* v = b.find(key); + if (!v || v->is_null()) return fallback; + if (!v->is_number()) bad(std::string(key) + " must be a number"); + return static_cast(v->as_int()); +} + +bool flag(const Json& b, const char* key) { + const Json* v = b.find(key); + if (!v || v->is_null()) return false; + if (!v->is_bool()) bad(std::string(key) + " must be true or false"); + return v->as_bool(); +} + +std::vector strings(const Json& b, const char* key, bool required) { + std::vector out; + const Json* v = b.find(key); + if (!v || v->is_null()) { + if (required) bad(std::string(key) + " is required"); + return out; + } + if (!v->is_array()) bad(std::string(key) + " must be an array of strings"); + for (const Json& s : v->as_array()) { + if (!s.is_string()) bad(std::string(key) + " must be an array of strings"); + out.push_back(s.as_string()); + } + return out; +} + +std::string scalar_text(const Json& v, const std::string& what) { + if (v.is_string()) return v.as_string(); + if (v.is_bool()) return v.as_bool() ? "true" : "false"; + if (v.is_number()) return std::to_string(v.as_int()); + bad(what + " values must be strings, numbers or booleans"); +} + +std::map string_map(const Json& b, const char* key) { + std::map out; + const Json* v = b.find(key); + if (!v || v->is_null()) return out; + if (!v->is_object()) bad(std::string(key) + " must be an object"); + for (const auto& [k, x] : v->as_object()) out[k] = scalar_text(x, key); + return out; +} + +Json dry_run(const Json& request) { + Json o = Json::object(); + o.set("dry_run", Json(true)); + o.set("request", request); + o.set("hint", Json("send confirm:true to run it")); + return o; +} + +// One provider's wiring: how to reach its Gateway and which seams it fills. +struct Entry { + std::string id; + std::string title; + std::string key_env; + std::string url_env; // empty = fixed vendor host + rt::CallTable (*table)(); + std::function(const std::string& index_host)> gateway; + std::function()> embedder; + std::function()> reranker; + std::function()> web; +}; + +template +std::shared_ptr gateway_of(P p) { + return p.shared_gateway(); +} + +class RetrievalRoutes { + public: + RetrievalRoutes() { + add({"voyage", "Voyage AI", rt::Voyage::kKeyEnv, "", [] { return rt::voyage_calls(); }, + [](const std::string&) { return gateway_of(rt::Voyage::from_env()); }, + [] { return std::make_shared(rt::Voyage::from_env()); }, + [] { return std::make_shared(rt::Voyage::from_env()); }, nullptr}); + add({"jina", "Jina AI", rt::Jina::kKeyEnv, "", [] { return rt::jina_calls(); }, + [](const std::string&) { return gateway_of(rt::Jina::from_env()); }, + [] { return std::make_shared(rt::Jina::from_env()); }, + [] { return std::make_shared(rt::Jina::from_env()); }, + [] { return std::make_shared(rt::Jina::from_env()); }}); + add({"pinecone", "Pinecone", rt::Pinecone::kKeyEnv, "", [] { return rt::pinecone_calls(); }, + [](const std::string& host) { return gateway_of(rt::Pinecone::from_env(host)); }, + [] { return std::make_shared(rt::Pinecone::from_env()); }, + [] { return std::make_shared(rt::Pinecone::from_env()); }, nullptr}); + add({"qdrant", "Qdrant", rt::Qdrant::kKeyEnv, rt::Qdrant::kUrlEnv, [] { return rt::qdrant_calls(); }, + [](const std::string&) { return gateway_of(rt::Qdrant::from_env()); }, nullptr, nullptr, nullptr}); + add({"weaviate", "Weaviate", rt::Weaviate::kKeyEnv, rt::Weaviate::kUrlEnv, [] { return rt::weaviate_calls(); }, + [](const std::string&) { return gateway_of(rt::Weaviate::from_env()); }, nullptr, nullptr, nullptr}); + add({"mixedbread", "Mixedbread", rt::Mixedbread::kKeyEnv, "", [] { return rt::mixedbread_calls(); }, + [](const std::string&) { return gateway_of(rt::Mixedbread::from_env()); }, + [] { return std::make_shared(rt::Mixedbread::from_env()); }, + [] { return std::make_shared(rt::Mixedbread::from_env()); }, nullptr}); + add({"tavily", "Tavily", rt::Tavily::kKeyEnv, "", [] { return rt::tavily_calls(); }, + [](const std::string&) { return gateway_of(rt::Tavily::from_env()); }, nullptr, nullptr, + [] { return std::make_shared(rt::Tavily::from_env()); }}); + add({"exa", "Exa", rt::Exa::kKeyEnv, "", [] { return rt::exa_calls(); }, + [](const std::string&) { return gateway_of(rt::Exa::from_env()); }, nullptr, nullptr, + [] { return std::make_shared(rt::Exa::from_env()); }}); + add({"brave", "Brave Search", rt::Brave::kKeyEnv, "", [] { return rt::brave_calls(); }, + [](const std::string&) { return gateway_of(rt::Brave::from_env()); }, nullptr, nullptr, + [] { return std::make_shared(rt::Brave::from_env()); }}); + } + + const std::vector& entries() const { return entries_; } + + const Entry& entry(const std::string& id) const { + for (const Entry& e : entries_) + if (e.id == id) return e; + bad("provider must be one of: " + ids()); + } + + std::string ids() const { + std::string out; + for (const Entry& e : entries_) out += (out.empty() ? "" : ", ") + e.id; + return out; + } + + Json status() const { + Json arr = Json::array(); + for (const Entry& e : entries_) { + Json p = Json::object(); + p.set("id", Json(e.id)); + p.set("title", Json(e.title)); + p.set("credential_env", Json(e.key_env)); + p.set("url_env", Json(e.url_env)); + p.set("configured", Json(e.gateway("")->configured())); + p.set("calls", Json(static_cast(e.table().size()))); + Json seams = Json::array(); + if (e.embedder) seams.push(Json("embed")); + if (e.reranker) seams.push(Json("rerank")); + if (e.web) seams.push(Json("search")); + p.set("seams", seams); + arr.push(p); + } + Json o = Json::object(); + o.set("providers", arr); + Json caps = Json::object(); + caps.set("max_embed_texts", Json(static_cast(rt::RetrievalCaps::kMaxEmbedTexts))); + caps.set("max_rerank_docs", Json(static_cast(rt::RetrievalCaps::kMaxRerankDocs))); + caps.set("max_search_results", Json(static_cast(rt::RetrievalCaps::kMaxSearchResults))); + caps.set("max_body_bytes", Json(static_cast(rt::RetrievalCaps::kMaxBodyBytes))); + o.set("caps", caps); + return o; + } + + Json calls(const std::string& provider) const { + Json arr = Json::array(); + for (const Entry& e : entries_) { + if (!provider.empty() && e.id != provider) continue; + rt::CallTable t = e.table(); + for (std::size_t i = 0; i < t.size(); ++i) { + const rt::CallRow& r = t.at(i); + Json row = Json::object(); + row.set("provider", Json(e.id)); + row.set("name", Json(r.name)); + row.set("http", Json(r.method)); + row.set("path", Json(r.path)); + row.set("host", Json(r.host)); + row.set("flags", Json(r.flags)); + row.set("unverified", Json(rt::RowInfo::unverified(r))); + arr.push(row); + } + } + if (arr.as_array().empty() && !provider.empty()) bad("provider must be one of: " + ids()); + Json o = Json::object(); + o.set("count", Json(static_cast(arr.as_array().size()))); + o.set("calls", arr); + return o; + } + + // POST /ai/retrieval/call + fw::Response call(const fw::Request& req) const { + Json b = parse_body(req); + const Entry& e = entry(text(b, "provider", true)); + std::string name = text(b, "call", true); + rt::CallArgs args; + args.path = string_map(b, "path"); + args.query = string_map(b, "query"); + if (const Json* body = b.find("body"); body && !body->is_null()) args.body = *body; + std::string index_host = text(b, "index_host"); + std::shared_ptr gw; + rt::PlannedCall plan; + try { + if (!index_host.empty()) rt::check_host(e.id, index_host); + gw = e.gateway(index_host); + plan = gw->plan(name, args); + } catch (const llm::ApiError& err) { + map(err); + } + bool guarded = plan.destructive || plan.writes || plan.paid; + if (guarded && !flag(b, "confirm")) { + Json r = Json::object(); + r.set("provider", Json(e.id)); + r.set("call", Json(plan.name)); + r.set("method", Json(plan.method)); + r.set("host", Json(plan.host)); + r.set("target", Json(plan.target)); + r.set("flags", Json(plan.flags)); + r.set("body", plan.body); + return json_out(dry_run(r)); + } + try { + Json out = plan.destructive ? gw->destroy(name, args) : gw->call(name, args); + return json_out(out); + } catch (const llm::ApiError& err) { + map(err); + } + } + + // POST /ai/retrieval/embed + fw::Response embed(const fw::Request& req) const { + Json b = parse_body(req); + const Entry& e = entry(text(b, "provider", true)); + if (!e.embedder) bad(e.id + " has no embeddings"); + rt::EmbedRequest r; + r.texts = strings(b, "texts", true); + r.model = text(b, "model"); + r.input_type = text(b, "input_type"); + r.dimensions = number(b, "dimensions", 0); + try { + rt::Embedder::validate(e.id, r); + } catch (const llm::ApiError& err) { + map(err); + } + if (!flag(b, "confirm")) { + Json q = Json::object(); + q.set("provider", Json(e.id)); + q.set("texts", Json(static_cast(r.texts.size()))); + q.set("model", Json(r.model.empty() ? "(provider default)" : r.model)); + q.set("input_type", Json(r.input_type)); + return json_out(dry_run(q)); + } + try { + rt::EmbedResult out = e.embedder()->embed(r); + Json o = Json::object(); + o.set("model", Json(out.model)); + o.set("total_tokens", Json(static_cast(out.total_tokens))); + o.set("count", Json(static_cast(out.vectors.size()))); + o.set("dimensions", Json(out.vectors.empty() ? 0 : static_cast(out.vectors[0].size()))); + Json first = Json::array(); + if (!out.vectors.empty()) + for (std::size_t i = 0; i < out.vectors[0].size() && i < 8; ++i) first.push(Json(static_cast(out.vectors[0][i]))); + o.set("first_vector_head", first); + return json_out(o); + } catch (const llm::ApiError& err) { + map(err); + } + } + + // POST /ai/retrieval/rerank + fw::Response rerank(const fw::Request& req) const { + Json b = parse_body(req); + const Entry& e = entry(text(b, "provider", true)); + if (!e.reranker) bad(e.id + " has no rerank"); + rt::RerankRequest r; + r.query = text(b, "query", true); + r.documents = strings(b, "documents", true); + r.model = text(b, "model"); + r.top_n = number(b, "top_n", 0); + try { + rt::Reranker::validate(e.id, r); + } catch (const llm::ApiError& err) { + map(err); + } + if (!flag(b, "confirm")) { + Json q = Json::object(); + q.set("provider", Json(e.id)); + q.set("query", Json(r.query)); + q.set("documents", Json(static_cast(r.documents.size()))); + q.set("top_n", Json(r.top_n)); + return json_out(dry_run(q)); + } + try { + rt::RerankResult out = e.reranker()->rerank(r); + Json hits = Json::array(); + for (const rt::RerankHit& h : out.hits) { + Json x = Json::object(); + x.set("index", Json(h.index)); + x.set("score", Json(h.score)); + hits.push(x); + } + Json o = Json::object(); + o.set("model", Json(out.model)); + o.set("total_tokens", Json(static_cast(out.total_tokens))); + o.set("hits", hits); + return json_out(o); + } catch (const llm::ApiError& err) { + map(err); + } + } + + // POST /ai/retrieval/search + fw::Response search(const fw::Request& req) const { + Json b = parse_body(req); + const Entry& e = entry(text(b, "provider", true)); + if (!e.web) bad(e.id + " has no web search"); + rt::SearchRequest r; + r.query = text(b, "query", true); + r.max_results = number(b, "max_results", 5); + r.recency = text(b, "recency"); + try { + rt::WebSearch::validate(e.id, r); + } catch (const llm::ApiError& err) { + map(err); + } + if (!flag(b, "confirm")) { + Json q = Json::object(); + q.set("provider", Json(e.id)); + q.set("query", Json(r.query)); + q.set("max_results", Json(r.max_results)); + q.set("recency", Json(r.recency)); + return json_out(dry_run(q)); + } + try { + rt::SearchResult out = e.web()->search(r); + Json hits = Json::array(); + for (const rt::SearchHit& h : out.hits) { + Json x = Json::object(); + x.set("title", Json(h.title)); + x.set("url", Json(h.url)); + x.set("snippet", Json(h.snippet)); + x.set("published", Json(h.published)); + hits.push(x); + } + Json o = Json::object(); + o.set("answer", Json(out.answer)); + o.set("hits", hits); + return json_out(o); + } catch (const llm::ApiError& err) { + map(err); + } + } + + private: + void add(Entry e) { entries_.push_back(std::move(e)); } + + [[noreturn]] static void map(const llm::ApiError& e) { + if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED") + throw fw::HttpError(503, "retrieval_not_configured", e.message()); + if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty()) + throw fw::HttpError(422, "validation_error", e.message()); + throw fw::HttpError(502, "retrieval_error", e.vendor() + " " + std::to_string(e.http_status()) + " " + e.type() + ": " + e.message()); + } + + std::vector entries_; +}; + +} // namespace + void register_ai_retrieval_demo(fw::Router& router, Services& services) { - (void)router; - (void)services; + auto routes = std::make_shared(); + fw::Environment* env = services.templates.get(); + services.templates->add_template("ai_retrieval.html", kPage); + services.add_feature("ai-retrieval", "/ai/retrieval"); + + router.get("/ai/retrieval", "ai-retrieval-page", [env](fw::Request req) { + (void)req; + fw::TemplateCtx ctx; + ctx["title"] = std::string("Retrieval vendors"); + return fw::Response(200, env->render("ai_retrieval.html", ctx), "text/html; charset=utf-8"); + }); + + router.get("/ai/retrieval/status", "ai-retrieval-status", [routes](fw::Request req) { + (void)req; + return json_out(routes->status()); + }); + + router.get("/ai/retrieval/calls", "ai-retrieval-calls", [routes](fw::Request req) { + return json_out(routes->calls(req.query_or("provider", ""))); + }); + + router.post("/ai/retrieval/call", "ai-retrieval-call", [routes](fw::Request req) { return routes->call(req); }); + router.post("/ai/retrieval/embed", "ai-retrieval-embed", [routes](fw::Request req) { return routes->embed(req); }); + router.post("/ai/retrieval/rerank", "ai-retrieval-rerank", [routes](fw::Request req) { return routes->rerank(req); }); + router.post("/ai/retrieval/search", "ai-retrieval-search", [routes](fw::Request req) { return routes->search(req); }); } } // namespace molo::example::app diff --git a/example/assets/ai_retrieval.js b/example/assets/ai_retrieval.js new file mode 100644 index 0000000..57d53dd --- /dev/null +++ b/example/assets/ai_retrieval.js @@ -0,0 +1,78 @@ +// Retrieval page: thin client over /ai/retrieval/* (all limits server side). +(function () { + "use strict"; + var $ = function (id) { return document.getElementById(id); }; + var out = $("rt-out"), err = $("rt-error"); + var templates = { + "embed": { provider: "voyage", texts: ["The quick brown fox", "A lazy dog"], input_type: "document" }, + "rerank": { provider: "jina", query: "What is a vector database?", documents: ["Pinecone stores vectors.", "Bananas are yellow."], top_n: 1 }, + "search": { provider: "tavily", query: "retrieval augmented generation", max_results: 3, recency: "month" }, + "call": { provider: "qdrant", call: "get_collections", path: {}, query: {}, body: null } + }; + var opSel = $("rt-op"), provSel = $("rt-provider"), callSel = $("rt-call"); + Object.keys(templates).forEach(function (k) { + var o = document.createElement("option"); + o.value = k; o.textContent = k; + opSel.appendChild(o); + }); + function fill() { + var t = JSON.parse(JSON.stringify(templates[opSel.value])); + if (provSel.value) t.provider = provSel.value; + if (opSel.value === "call" && callSel.value) t.call = callSel.value; + $("rt-body").value = JSON.stringify(t, null, 1); + } + function loadCalls() { + callSel.innerHTML = ""; + fetch("/ai/retrieval/calls?provider=" + encodeURIComponent(provSel.value)).then(function (r) { return r.json(); }).then(function (j) { + (j.calls || []).forEach(function (c) { + var o = document.createElement("option"); + o.value = c.name; + o.textContent = c.name + " " + c.http + " " + c.path + (c.flags ? " [" + c.flags + "]" : ""); + callSel.appendChild(o); + }); + fill(); + }); + } + opSel.onchange = function () { fill(); }; + provSel.onchange = loadCalls; + callSel.onchange = fill; + + function send(confirm) { + var body; + try { body = JSON.parse($("rt-body").value || "{}"); } + catch (e) { err.textContent = "Body is not valid JSON"; return; } + if (confirm) body.confirm = true; + err.textContent = ""; + out.textContent = "..."; + fetch("/ai/retrieval/" + opSel.value, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body) + }).then(function (res) { + return res.text().then(function (t) { + var j; + try { j = JSON.parse(t); } catch (e) { j = { message: t }; } + if (!res.ok) { err.textContent = j.message || ("HTTP " + res.status); out.textContent = ""; } + else { out.textContent = JSON.stringify(j, null, 2); } + }); + }).catch(function (e) { err.textContent = String(e); out.textContent = ""; }); + } + $("rt-send").onclick = function () { send(false); }; + $("rt-confirm").onclick = function () { send(true); }; + + fetch("/ai/retrieval/status").then(function (r) { return r.json(); }).then(function (s) { + var on = [], off = []; + s.providers.forEach(function (p) { + var o = document.createElement("option"); + o.value = p.id; o.textContent = p.title; + provSel.appendChild(o); + (p.configured ? on : off).push(p.credential_env + (p.url_env ? "+" + p.url_env : "")); + }); + $("rt-config").textContent = on.length + ? "Configured: " + on.join(", ") + ". Calls are real; paid, writing and destructive ones need confirm." + : "No credential configured: calls answer 503 until one is set; dry runs still work (" + off.join(", ") + ")."; + provSel.value = "voyage"; + fill(); + loadCalls(); + }); +})(); diff --git a/src/vendors/retrieval/SPEC.md b/src/vendors/retrieval/SPEC.md new file mode 100644 index 0000000..0b0d2e3 --- /dev/null +++ b/src/vendors/retrieval/SPEC.md @@ -0,0 +1,224 @@ +# Retrieval vendors - Specification + +Owned directory: `src/vendors/retrieval/`. Namespace `molo::retrieval`. One +subdirectory per provider. Depends on `molo::llm` (`ApiClient`, `Config`, +`ApiError`) and `molo::clients`. Plan: `plans/18-breadth.md` (rules of +`plans/17-llm-vendor-platforms.md` apply). `src/llm` is not edited. + +Providers: Voyage AI, Jina AI, Pinecone, Qdrant, Weaviate, Mixedbread, Tavily, +Exa, Brave Search. + +## 1. Module map + +| File | Concern | +|---|---| +| `caps.h` | Static ceilings, never autotuned (documented limits and our own) | +| `call_table.h/.cpp` | `CallRow`, `CallTable`, `RowInfo` (flags, method, unverified) | +| `gateway.h/.cpp` | `Gateway`: validates a call against its row (path ids, query, body fields, sizes), renders it, sends it. `call()` refuses destructive rows, `destroy()` only runs them, `plan()` renders without traffic | +| `provider.h/.cpp` | `make_gateway` (one `ApiClient` per host), `check_host`, `Provider` base, `J` Json helpers | +| `seams.h/.cpp` | Neutral `Embedder`, `Reranker`, `WebSearch` seams and their request validators | +| `voyage/` | embeddings, multimodal, contextualized, rerank; `Embedder` + `Reranker` | +| `jina/` | embeddings, rerank, models, batch (api.jina.ai), Reader, Search, Segmenter, Classifier; `Embedder` + `Reranker` + `WebSearch` | +| `pinecone/` | control plane, data plane (index host), inference; `Embedder` + `Reranker` | +| `qdrant/` | collections, points, query, snapshots, cluster | +| `weaviate/` | schema, objects, batch, tenants, REST search, GraphQL over HTTP | +| `mixedbread/` | embeddings, reranking, stores, files, parsing, extraction; `Embedder` + `Reranker` | +| `tavily/` | search, extract, crawl, map, research, usage; `WebSearch` | +| `exa/` | search, contents, findSimilar, answer, research; `WebSearch` | +| `brave/` | web, news, images, videos, suggest, spellcheck, local, LLM context, answers; `WebSearch` | +| `tools/check_spec.py` | Verifies every call table against the provider spec | + +Each provider directory holds `calls.cpp` (the table, generated from the spec +by `check_spec.py --rows ` and then reviewed) and `.h/.cpp` +(typed class). `calls.cpp` is the only place paths are written. + +## 2. Seams + +`Embedder::embed(EmbedRequest) -> EmbedResult`, `Reranker::rerank(RerankRequest) +-> RerankResult`, `WebSearch::search(SearchRequest) -> SearchResult`. Shared +validators reject blank or oversized input before any traffic. An adapter whose +provider cannot honour a field says so (Jina search has no `recency`, throws). +A reply with a vector count different from the input count is a `BAD_RESPONSE`. + +## 3. Credentials + +| Provider | Env | Auth header | +|---|---|---| +| Voyage | `VOYAGE_API_KEY` | `Authorization: Bearer` | +| Jina | `JINA_API_KEY` | `Authorization: Bearer` | +| Pinecone | `PINECONE_API_KEY` | `Api-Key` plus `X-Pinecone-Api-Version: 2026-07` | +| Qdrant | `QDRANT_API_KEY` and `QDRANT_URL` | `api-key` | +| Weaviate | `WEAVIATE_API_KEY` and `WEAVIATE_URL` (the `/v1` base is added) | `Authorization: Bearer` | +| Mixedbread | `MXBAI_API_KEY` | `Authorization: Bearer` | +| Tavily | `TAVILY_API_KEY` | `Authorization: Bearer` | +| Exa | `EXA_API_KEY` | `x-api-key` | +| Brave | `BRAVE_API_KEY` | `X-Subscription-Token` | + +Key pages (unverified, the vendors' usual consoles): dash.voyageai.com, +jina.ai/api-dashboard, app.pinecone.io (API keys), cloud.qdrant.io, console.weaviate.cloud, +mixedbread.com (platform), app.tavily.com, dashboard.exa.ai, api-dashboard.search.brave.com. + +## 4. Contracts + +- No key (or no `QDRANT_URL` / `WEAVIATE_URL`): `configured()` is false, every + call throws `NotConfiguredError` (503) before any traffic. +- Rows carry flags: `d` destructive, `w` writes, `r` read-only POST, `p` paid. + The demo runs `d`, `w` and `p` rows only with `"confirm":true`. +- Destructive calls are explicit methods (`Qdrant::delete_collection`, + `Pinecone::delete_index`, `Pinecone::delete_ids/delete_all/delete_by_filter`, + `Weaviate::delete_class/batch_delete`, `Mixedbread::delete_store`, ...) that + go through `Gateway::destroy`. `Gateway::call` refuses them. +- Path ids pass `llm::Service::valid_id` (1..256 of `A-Za-z0-9._:-`); values are + percent-encoded again. Query parameters and body fields must be listed in + the row, otherwise `ApiError::invalid` (400 `INVALID_ARGUMENT`). +- A Pinecone data plane host must be a bare DNS name. The data plane is only + reachable on a `Pinecone` built with that host. +- Hard caps: see `caps.h` (embed 1000 texts, rerank 1000 documents, upsert 1000 + records, top_k 10000, request body 8 MiB, query value 8 KiB, list limit 100, + Mixedbread embed 256 inputs, Tavily extract 20 URLs, Brave web count 20). + List walks are never unbounded: one call is one page. + +## 5. Sources + +| Provider | Source | Revision seen 2026-10-25 | +|---|---|---| +| Voyage | official `voyageai` Python SDK wheel (PyPI): resources `embeddings`, `multimodalembeddings`, `contextualizedembeddings`, `rerank`, base URL, parameters, response objects. Voyage publishes no OpenAPI document | voyageai 0.5.0 | +| Jina | OpenAPI of `api.jina.ai`, `r.jina.ai`, `s.jina.ai`, `segment.jina.ai` (`/openapi.json`) | api 2026.10.13, reader/search 0.5.0, segment 0.1.0 | +| Pinecone | OpenAPI YAML `db_control`, `db_data`, `inference` of `pinecone-io/pinecone-api` | 2026-07 | +| Qdrant | `qdrant/qdrant` `docs/redoc/master/openapi.json` | master | +| Weaviate | `weaviate/weaviate` `openapi-specs/schema.json` (Swagger 2, basePath `/v1`) | 1.41.0-dev | +| Mixedbread | https://api.mixedbread.com/openapi.json | 0.1.0 | +| Tavily | https://docs.tavily.com/documentation/api-reference/openapi.json | 1.0.0 | +| Exa | `exa-labs/openapi-spec` `exa-openapi-spec.yaml` | 1.2.0 | +| Brave | the rendered API reference, one markdown page per operation (`api-dashboard.search.brave.com/api-reference//index.html.md`), plus `llms.txt` | 2026-10-25 | + +Each path and field was cross-checked between the spec and the call table by +`tools/check_spec.py`; typed request builders were written from the same +schemas. The Voyage rows rest on one source (the SDK). + +## 6. Verified against spec + +`python3 -I src/vendors/retrieval/tools/check_spec.py` (stdlib only, network), +run 2026-10-25: **0 problems**. + +| Provider | Spec operations | Table rows | Excluded (with reason in the script) | +|---|---|---|---| +| Qdrant | 69 | 49 | 20 (health, telemetry, issues, locks, shard placement, cluster membership) | +| Weaviate | 123 | 66 | 57 (authz, users, groups, replication, nodes, cluster, backups, classifications, tasks, experimental user export) | +| Pinecone | 57 | 57 | 0 (assistant, admin, nexus, oauth and metrics specs are other products and were not read) | +| Mixedbread | 192 | 80 | 112 (integrations and OAuth callbacks, admin, billing, buckets, events, api-keys, data sources, trace destinations) | +| Jina | 34 | 15 (13 spec + 2 prose) | 21 (chat completions, reader helper routes, path and POST duplicates of GET rows) | +| Tavily | 10 | 7 | 3 (`feedback`, `logs`, `org-usage`: no operationId, account logging) | +| Exa | 7 | 7 | 0 | +| Brave | 16 reference pages | 16 | 0 | +| Voyage | 4 SDK resources | 4 | 0 | + +The checker fails on a row absent from the spec, a body or query field absent +from the spec, an operationId mismatch, and a spec operation that is neither in +the table nor excluded. Its YAML reader (Pinecone and Exa are YAML) is a small +stdlib parser verified against PyYAML on those files (`--selftest`: paths equal). + +## 7. Verified live + +Unauthenticated probes, 2026-10-25 (invalid or missing key; no success body +was seen from any provider): + +| Probe | Result | +|---|---| +| `GET api.pinecone.io/indexes` without key | 401 `{"error":{"code":"UNAUTHENTICATED","message":"Missing api-key header"},"status":401}` | +| `POST api.tavily.com/search` without key | 401 `{"detail":{"error":"Unauthorized: missing or invalid API key."}}` | +| `GET api.jina.ai/v1/embeddings` without key | 401 `{"detail":"Authentication required. ..."}` | +| `GET api.voyageai.com/v1/embeddings` | 405 `{"detail":"Method Not Allowed"}` (route exists) | +| `GET api.mixedbread.com/v1/embeddings` | 405 `{"detail":"Method Not Allowed"}` (route exists) | +| `GET api.exa.ai/search` | 404 `{"error":"Not found","tag":"NOT_FOUND"}` (POST-only route) | +| any `api.search.brave.com/res/v1/*` with an invalid token | 422 `SUBSCRIPTION_TOKEN_INVALID` (auth runs before routing: even `/nonexistent` answers 422, so the probe proves host and TLS, not paths) | +| `openapi.json` of mixedbread, jina (4 hosts), tavily | 200 JSON (the specs above) | + +Through our own classes (`MOLO_LIVE_NET=1 make ai-test`, test +`retrieval_live.unauthenticated_probes_answer_with_an_auth_status`, 2026-10-25), +an invalid key gives: + +| Provider (call) | Result | +|---|---| +| Voyage (embed) | 401 `UNKNOWN` (body `{"detail":...}` is not parsed, see Gaps) | +| Jina (embeddings) | 401 `AUTH_INVALID_API_KEY` | +| Pinecone (list indexes) | 401 `UNAUTHENTICATED` "Invalid API key" | +| Tavily (usage) | 401 `UNKNOWN` | +| Exa (search) | 401, message "Invalid API key. Provide a valid key using 'Authorization: Bearer ' or 'x-api-key: '" (so Exa accepts both headers) | +| Brave (web search) | 422 `SUBSCRIPTION_TOKEN_INVALID` | +| Mixedbread (list stores) | not reachable: `api.mixedbread.com` negotiates only TLS 1.2 (checked with `openssl s_client`) and `src/clients` pins TLS 1.3, so the handshake fails ("record layer version error"). `curl` reaches it and gets 401 `auth_credentials_invalid_error`. The test prints a `FINDING` line instead of failing | + +Qdrant and Weaviate are your own servers and were not probed. + +## 8. Unverified + +Shapes inferred rather than read from a spec or seen: + +- Every success response body: parsers follow the spec schemas (Pinecone, + Qdrant, Mixedbread, Jina embeddings and rerank, Exa, Tavily request side) or + the SDK (Voyage), and the tests use hand-built fixtures labelled FIXTURE, + never recorded replies. +- Tavily response schemas are `inline` in its OpenAPI (not named); the parser + reads `answer`, `results[].title/url/content/score/published_date`, + `failed_results[].url` from the docs' usual shape. +- Jina Reader and Search JSON replies (`data.title/url/content/usage.tokens`, + `data[]`): their OpenAPI documents describe only request parameters. Reader + and Search are called as `GET /?url=` and `GET /search?q=` with + `Accept: application/json`, as the spec's query parameters allow. +- Jina Segmenter `POST /` and Classifier `POST /v1/classify` come from prose + docs only (the segment OpenAPI lists only `/countToken`; `api.jina.ai` lists no + classifier). They are the two `doc:` rows. +- Voyage default model names (`voyage-3.5`, `rerank-2.5`, `voyage-multimodal-3`, + `voyage-context-3`) and the Jina classifier default model are not in any + machine-readable source; the SDK's own default is the old `voyage-2`. Voyage + `return_documents` for rerank is not in the SDK and is not sent. +- Pinecone 2026-07 index creation uses `schema.fields` and `deployment`; the + vector field name `vector` and the data plane vector shape + (`vectors[].id/values/metadata`) follow the 2026-07 `db_data` spec but were + never exercised against a real index. +- Mixedbread query prompt for `input_type=query` + (`Represent this sentence for searching relevant passages:`) is the model + card's, not in the OpenAPI. +- Brave Answers non-streaming reply: the reference documents the streamed + `choices[].delta`; we send `stream:false` and parse `choices[0].message.content` + with a fallback to `delta`. +- Weaviate REST search (`/v1/search/{collection}/...`) is new in the spec + (1.41 dev); request fields are from the spec, behaviour on released servers + is unknown. +- Brave `summarizer` endpoints: not in today's API reference (Answers replaced + them); not wrapped, not probed. + +## 9. Gaps + +Foundation (`src/llm`, `src/clients`, owner of those files): + +- `src/clients/transports/tls_wolfssl.cpp` pins TLS 1.3 (`wolfTLSv1_3_client_method`, min version 1.3). `api.mixedbread.com` is TLS 1.2 only, so the whole Mixedbread package is unreachable until the transport accepts 1.2. Every other host in this package negotiates 1.3. +- `tests/src/llm/test_llm.cpp` unsets `MOLO_LIVE_NET` in the middle of the run, so live tests registered after it (this package's) see the variable gone; `test_live.cpp` here reads it once at start-up to work around that. + +- `ApiClient` needs a key to be "configured": a self-hosted Qdrant or Weaviate + without authentication cannot be called. +- `request()` parses every success body as JSON: Jina batch output downloads, + Mixedbread file downloads and Qdrant snapshot downloads (raw bytes) are not + reachable through `Gateway`; `request_raw` exists but the table engine does + not use it. +- No multipart in the table engine: Qdrant snapshot upload and recover from an + uploaded file, Mixedbread file upload and `upload_store_file`, Pinecone bulk + import from a file are table rows but cannot carry their file. +- `error_from_response` drops the `detail` envelope (Voyage, Jina, Tavily + `{"detail":...}`): the status survives, the message becomes `HTTP `. +- No streaming for Tavily/Brave Answers/Exa answer. +- `Url::encode_component` percent-encodes space as `%20`; both forms are fine. + +Package: + +- gRPC (Weaviate, Qdrant, Pinecone `proto`) is out of scope: the gRPC APIs are + not HTTP and Weaviate has no HTTP gateway for them. +- Pinecone assistant, admin, nexus, oauth and metrics products are not wrapped. +- Mixedbread parsing, extraction, chat and responses rows exist in the table + but have no typed methods; use the generic `/ai/retrieval/call` route or + `Gateway::call`. +- The typed classes cover the main flows; every other row is reachable through + `Gateway::call/destroy` with the same validation. +- No retry or rate-limit handling (each provider documents its own). +- Live success paths are unverified until keys exist: `MOLO_LIVE_NET=1 + MOLO_LIVE_PAID=1 make ai-test` with the keys in `.env.json.local`. diff --git a/src/vendors/retrieval/brave/brave.cpp b/src/vendors/retrieval/brave/brave.cpp new file mode 100644 index 0000000..a17dc1b --- /dev/null +++ b/src/vendors/retrieval/brave/brave.cpp @@ -0,0 +1,172 @@ +#include "vendors/retrieval/brave/brave.h" + +#include "vendors/retrieval/caps.h" + +namespace molo::retrieval { + +llm::ApiClientConfig Brave::config() { + llm::ApiClientConfig c; + c.vendor = "brave"; + c.base_url = kBaseUrl; + c.key_env = kKeyEnv; + c.auth = llm::AuthStyle::kHeader; + c.auth_header = "X-Subscription-Token"; + c.default_headers["Accept"] = "application/json"; + return c; +} + +namespace { +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("brave", why); } + +void check_query(const std::string& q, std::size_t max_chars) { + if (q.find_first_not_of(" \t\r\n") == std::string::npos) bad("query is blank"); + if (q.size() > max_chars) bad("query is over the documented length"); +} +void check_count(int count, int max) { + if (count < 1 || count > max) bad("count must be 1.." + std::to_string(max)); +} +CallArgs query_args(const std::string& q, int count, const std::string& freshness) { + CallArgs a; + a.query["q"] = q; + a.query["count"] = Query::number(count); + if (!freshness.empty()) a.query["freshness"] = freshness; + return a; +} +} // namespace + +Brave Brave::from_env() { return Brave(make_gateway(brave_calls(), config(), {{"main", kBaseUrl}}, "", nullptr)); } +Brave Brave::with(const TransportFactory& factory, std::string key) { + return Brave(make_gateway(brave_calls(), config(), {{"main", kBaseUrl}}, std::move(key), factory)); +} + +std::string Brave::freshness_of(const std::string& r) { + if (r == "day") return "pd"; + if (r == "week") return "pw"; + if (r == "month") return "pm"; + if (r == "year") return "py"; + return ""; +} + +SearchResult Brave::parse_web(const Json& doc) { + SearchResult out; + const Json* web = J::at(doc, "web"); + if (!web) return out; + for (const Json& r : J::array(*web, "results")) { + SearchHit h; + h.title = J::str(r, "title"); + h.url = J::str(r, "url"); + h.snippet = J::str(r, "description"); + h.published = J::str(r, "page_age"); + out.hits.push_back(std::move(h)); + } + return out; +} + +SearchResult Brave::parse_results(const Json& doc) { + SearchResult out; + for (const Json& r : J::array(doc, "results")) { + SearchHit h; + h.title = J::str(r, "title"); + h.url = J::str(r, "url"); + h.snippet = J::str(r, "description"); + h.published = J::str(r, "page_age"); + if (h.published.empty()) h.published = J::str(r, "age"); + out.hits.push_back(std::move(h)); + } + return out; +} + +std::vector Brave::parse_images(const Json& doc) { + std::vector out; + for (const Json& r : J::array(doc, "results")) { + BraveImage i; + i.title = J::str(r, "title"); + i.page_url = J::str(r, "url"); + i.source = J::str(r, "source"); + if (const Json* p = J::at(r, "properties")) i.image_url = J::str(*p, "url"); + out.push_back(std::move(i)); + } + return out; +} + +std::vector Brave::parse_suggest(const Json& doc) { + std::vector out; + for (const Json& r : J::array(doc, "results")) out.push_back(J::str(r, "query")); + return out; +} + +std::string Brave::parse_answer(const Json& doc) { + const Json::Array& choices = J::array(doc, "choices"); + if (choices.empty()) return ""; + if (const Json* m = J::at(choices.front(), "message")) return J::str(*m, "content"); + if (const Json* d = J::at(choices.front(), "delta")) return J::str(*d, "content"); + return ""; +} + +SearchResult Brave::search(const SearchRequest& request) { + WebSearch::validate("brave", request); + check_query(request.query, 600); + check_count(request.max_results, 20); + return parse_web(gw_->call("web_search", query_args(request.query, request.max_results, freshness_of(request.recency)))); +} + +SearchResult Brave::news(const SearchRequest& request) { + WebSearch::validate("brave", request); + check_query(request.query, RetrievalCaps::kBraveQueryChars); + check_count(request.max_results, RetrievalCaps::kBraveMaxCount); + return parse_results(gw_->call("news_search", query_args(request.query, request.max_results, freshness_of(request.recency)))); +} + +SearchResult Brave::videos(const SearchRequest& request) { + WebSearch::validate("brave", request); + check_query(request.query, RetrievalCaps::kBraveQueryChars); + check_count(request.max_results, RetrievalCaps::kBraveMaxCount); + return parse_results(gw_->call("video_search", query_args(request.query, request.max_results, freshness_of(request.recency)))); +} + +std::vector Brave::images(const std::string& query, int count) { + check_query(query, RetrievalCaps::kBraveQueryChars); + check_count(count, 200); + return parse_images(gw_->call("image_search", query_args(query, count, ""))); +} + +std::vector Brave::suggest(const std::string& query, int count) { + check_query(query, RetrievalCaps::kBraveQueryChars); + check_count(count, 20); + CallArgs a; + a.query["q"] = query; + a.query["count"] = Query::number(count); + return parse_suggest(gw_->call("suggest", a)); +} + +Json Brave::spellcheck(const std::string& query) { + check_query(query, RetrievalCaps::kBraveQueryChars); + CallArgs a; + a.query["q"] = query; + return gw_->call("spellcheck", a); +} + +Json Brave::llm_context(const std::string& query) { + check_query(query, RetrievalCaps::kBraveQueryChars); + CallArgs a; + a.query["q"] = query; + return gw_->call("llm_context", a); +} + +std::string Brave::answer(const std::string& prompt) { + if (prompt.find_first_not_of(" \t\r\n") == std::string::npos || prompt.size() > RetrievalCaps::kMaxQueryChars) + bad("prompt must be 1..8192 chars"); + Json msg = Json::object(); + msg.set("role", Json("user")); + msg.set("content", Json(prompt)); + Json msgs = Json::array(); + msgs.push(msg); + Json body = Json::object(); + body.set("messages", msgs); + body.set("stream", Json(false)); + CallArgs a; + a.body = body; + return parse_answer(gw_->call("answers", a)); +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/brave/brave.h b/src/vendors/retrieval/brave/brave.h new file mode 100644 index 0000000..71438f0 --- /dev/null +++ b/src/vendors/retrieval/brave/brave.h @@ -0,0 +1,56 @@ +#pragma once + +// Brave Search API: web, news, images, videos, suggest, spellcheck, local POIs +// and descriptions, place search, LLM context and Answers (chat completions). +// Implements the WebSearch seam over web search. Brave's separate Summarizer +// endpoints are no longer in the published API reference (Answers replaces +// them), so they are not wrapped. + +#include +#include + +#include "vendors/retrieval/provider.h" +#include "vendors/retrieval/seams.h" + +namespace molo::retrieval { + +const CallTable& brave_calls(); + +struct BraveImage { + std::string title; + std::string page_url; + std::string image_url; + std::string source; +}; + +class Brave : public Provider, public WebSearch { + public: + static constexpr const char* kKeyEnv = "BRAVE_API_KEY"; + static constexpr const char* kBaseUrl = "https://api.search.brave.com/res"; + + static llm::ApiClientConfig config(); + static Brave from_env(); + static Brave with(const TransportFactory& factory, std::string key); + explicit Brave(std::shared_ptr gateway) : Provider(std::move(gateway)) {} + + std::string provider() const override { return "brave"; } + SearchResult search(const SearchRequest& request) override; // web, count <= 20 + SearchResult news(const SearchRequest& request); // count <= 50 + SearchResult videos(const SearchRequest& request); // count <= 50 + std::vector images(const std::string& query, int count); // count <= 200 + std::vector suggest(const std::string& query, int count); // count <= 20 + Json spellcheck(const std::string& query); + Json llm_context(const std::string& query); + // One cited answer. Sends stream=false; the reply shape of the non-streaming + // form is not documented (see SPEC.md ## Unverified). + std::string answer(const std::string& prompt); + + static SearchResult parse_web(const Json& doc); + static SearchResult parse_results(const Json& doc); // news and videos + static std::vector parse_images(const Json& doc); + static std::vector parse_suggest(const Json& doc); + static std::string parse_answer(const Json& doc); + static std::string freshness_of(const std::string& recency); +}; + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/brave/calls.cpp b/src/vendors/retrieval/brave/calls.cpp new file mode 100644 index 0000000..c42ed02 --- /dev/null +++ b/src/vendors/retrieval/brave/calls.cpp @@ -0,0 +1,34 @@ +#include "vendors/retrieval/brave/brave.h" + +namespace molo::retrieval { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// name, host, http method, path, spec operation id (or reference page / "doc:"), +// body fields, query parameters, flags (d destructive, w writes, r read-only POST, +// p paid), response schema) +static const CallRow kRows[] = { + {"web_search", "main", "GET", "/v1/web/search", "web/search/get", "", "count,country,enable_rich_callback,extra_snippets,freshness,goggles,goggles_id,include_fetch_metadata,offset,operators,q,result_filter,safesearch,search_lang,spellcheck,summary,text_decorations,ui_lang,units", "p", "-"}, + {"web_search_post", "main", "POST", "/v1/web/search", "web/search/post", "count,country,enable_rich_callback,extra_snippets,freshness,goggles,goggles_id,include_fetch_metadata,offset,operators,q,result_filter,safesearch,search_lang,spellcheck,summary,text_decorations,ui_lang,units", "", "rp", "-"}, + {"local_pois", "main", "GET", "/v1/local/pois", "web/local_pois", "", "ids,search_lang,ui_lang,units", "p", "-"}, + {"local_descriptions", "main", "GET", "/v1/local/descriptions", "web/poi_descriptions", "", "ids", "p", "-"}, + {"rich_search", "main", "GET", "/v1/web/rich", "web/rich_search", "", "callback_key", "p", "-"}, + {"place_search", "main", "GET", "/v1/local/place_search", "web/place_search", "", "count,country,geoloc,latitude,location,longitude,q,radius,safesearch,search_lang,spellcheck,ui_lang,units", "p", "-"}, + {"news_search", "main", "GET", "/v1/news/search", "news/news_search/get", "", "count,country,extra_snippets,freshness,goggles,include_fetch_metadata,offset,operators,q,safesearch,search_lang,spellcheck,ui_lang", "p", "-"}, + {"news_search_post", "main", "POST", "/v1/news/search", "news/news_search/post", "count,country,extra_snippets,freshness,goggles,include_fetch_metadata,offset,operators,q,safesearch,search_lang,spellcheck,ui_lang", "", "rp", "-"}, + {"image_search", "main", "GET", "/v1/images/search", "images/image_search", "", "count,country,q,safesearch,search_lang,spellcheck", "p", "-"}, + {"video_search", "main", "GET", "/v1/videos/search", "videos/video_search/get", "", "count,country,freshness,include_fetch_metadata,offset,operators,q,safesearch,search_lang,spellcheck,ui_lang", "p", "-"}, + {"video_search_post", "main", "POST", "/v1/videos/search", "videos/video_search/post", "count,country,freshness,include_fetch_metadata,offset,operators,q,safesearch,search_lang,spellcheck,ui_lang", "", "rp", "-"}, + {"suggest", "main", "GET", "/v1/suggest/search", "other/suggestions", "", "count,country,lang,q,rich", "p", "-"}, + {"spellcheck", "main", "GET", "/v1/spellcheck/search", "other/spell_check", "", "country,lang,q", "p", "-"}, + {"answers", "main", "POST", "/v1/chat/completions", "ai/answers", "country,enable_citations,enable_entities,enable_research,language,max_completion_tokens,messages,metadata,model,research_allow_thinking,research_maximum_number_of_iterations,research_maximum_number_of_queries,research_maximum_number_of_results_per_query,research_maximum_number_of_seconds,research_maximum_number_of_tokens_per_query,safesearch,seed,stream,web_search_options", "", "rp", "-"}, + {"llm_context", "main", "GET", "/v1/llm/context", "ai/llm_context/get", "", "context_threshold_mode,count,country,enable_local,enable_source_metadata,freshness,goggles,maximum_number_of_snippets,maximum_number_of_snippets_per_url,maximum_number_of_tokens,maximum_number_of_tokens_per_url,maximum_number_of_urls,operators,operators_fallback,q,safesearch,search_lang,spellcheck", "p", "-"}, + {"llm_context_post", "main", "POST", "/v1/llm/context", "ai/llm_context/post", "context_threshold_mode,count,country,enable_local,enable_source_metadata,freshness,goggles,maximum_number_of_snippets,maximum_number_of_snippets_per_url,maximum_number_of_tokens,maximum_number_of_tokens_per_url,maximum_number_of_urls,operators,operators_fallback,q,safesearch,search_lang,spellcheck", "", "rp", "-"}, +}; +// CALLS-END + +const CallTable& brave_calls() { + static const CallTable table{kRows, sizeof(kRows) / sizeof(kRows[0])}; + return table; +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/call_table.cpp b/src/vendors/retrieval/call_table.cpp new file mode 100644 index 0000000..4139695 --- /dev/null +++ b/src/vendors/retrieval/call_table.cpp @@ -0,0 +1,41 @@ +#include "vendors/retrieval/call_table.h" + +#include + +#include "llm/error.h" + +namespace molo::retrieval { + +const CallRow* CallTable::find(const std::string& name) const { + for (std::size_t i = 0; i < count; ++i) + if (name == rows[i].name) return &rows[i]; + return nullptr; +} + +bool RowInfo::has(const CallRow& row, char flag) { return std::strchr(row.flags, flag) != nullptr && flag != '\0'; } + +clients::Method RowInfo::method(const CallRow& row) { + const char* m = row.method; + if (!std::strcmp(m, "GET")) return clients::Method::kGet; + if (!std::strcmp(m, "POST")) return clients::Method::kPost; + if (!std::strcmp(m, "PUT")) return clients::Method::kPut; + if (!std::strcmp(m, "PATCH")) return clients::Method::kPatch; + if (!std::strcmp(m, "DELETE")) return clients::Method::kDelete; + throw llm::ApiError::invalid("retrieval", std::string("bad method in call table: ") + m); +} + +bool RowInfo::listed(const char* csv, const std::string& field) { + std::string s = csv; + std::size_t pos = 0; + while (pos <= s.size()) { + std::size_t end = s.find(',', pos); + if (end == std::string::npos) end = s.size(); + if (s.compare(pos, end - pos, field) == 0 && end - pos == field.size()) return !field.empty(); + pos = end + 1; + } + return false; +} + +bool RowInfo::unverified(const CallRow& row) { return std::strncmp(row.op, "doc:", 4) == 0; } + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/call_table.h b/src/vendors/retrieval/call_table.h new file mode 100644 index 0000000..6667e22 --- /dev/null +++ b/src/vendors/retrieval/call_table.h @@ -0,0 +1,50 @@ +#pragma once + +// The one table of REST calls per provider. Every provider owns a calls.cpp +// with a `CALLS-BEGIN ... CALLS-END` block of CallRow lines that +// tools/check_spec.py compares with the provider's published spec. Code builds +// every path from a row, never from a literal. + +#include +#include + +#include "clients/request.h" + +namespace molo::retrieval { + +// flags: d destructive, w writes, r read-only POST, p paid / metered. +// A GET row has no write flag. `op` starting with "doc:" means the row comes +// from prose documentation only (listed as unverified by check_spec.py). +struct CallRow { + const char* name; // snake_case id used by the demo and by tests + const char* host; // which ApiClient serves it ("main", "control", ...) + const char* method; // GET POST PUT PATCH DELETE + const char* path; // from the host root, {placeholders} for path params + const char* op; // spec operation id (or a reference page slug) + const char* body; // comma list of top-level body fields, "*" = free form + const char* query; // comma list of query parameters + const char* flags; + const char* resp; // response schema name in the spec, "-" when none +}; + +struct CallTable { + const CallRow* rows = nullptr; + std::size_t count = 0; + + const CallRow* find(const std::string& name) const; + const CallRow& at(std::size_t index) const { return rows[index]; } + std::size_t size() const { return count; } +}; + +struct RowInfo { + static bool has(const CallRow& row, char flag); + static bool destructive(const CallRow& row) { return has(row, 'd'); } + static bool paid(const CallRow& row) { return has(row, 'p'); } + static bool writes(const CallRow& row) { return has(row, 'w') || has(row, 'd'); } + static clients::Method method(const CallRow& row); + // True when `field` is in a comma list. + static bool listed(const char* csv, const std::string& field); + static bool unverified(const CallRow& row); +}; + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/caps.h b/src/vendors/retrieval/caps.h new file mode 100644 index 0000000..15f57e1 --- /dev/null +++ b/src/vendors/retrieval/caps.h @@ -0,0 +1,47 @@ +#pragma once + +// Static ceilings for the retrieval package. Never autotuned. "doc" numbers +// come from the provider's published limits (see SPEC.md ## Sources), "own" +// numbers are our hard caps. + +#include + +namespace molo::retrieval { + +struct RetrievalCaps { + // Request shape (own) + static constexpr std::size_t kMaxBodyBytes = 8u * 1024u * 1024u; + static constexpr std::size_t kMaxQueryParams = 32; + static constexpr std::size_t kMaxQueryValueBytes = 8192; + static constexpr std::size_t kMaxPathParams = 4; + static constexpr std::size_t kMaxHostChars = 255; + // Embedding, rerank, search seams + static constexpr std::size_t kMaxEmbedTexts = 1000; // Voyage doc: 1000 inputs + static constexpr std::size_t kMaxTextBytes = 1024u * 1024u; // own, per text + static constexpr std::size_t kMaxRerankDocs = 1000; // Voyage doc: 1000 documents + static constexpr std::size_t kMaxQueryChars = 8192; // own (Brave doc: 400) + static constexpr std::size_t kMaxSearchResults = 100; // own + static constexpr std::size_t kMaxUrls = 100; // own (Tavily doc: 20) + static constexpr int kMaxDimensions = 20000; // Pinecone doc: 20000 + static constexpr int kMaxTopN = 1000; + // Vector databases + static constexpr std::size_t kMaxRecordsPerUpsert = 1000; // Pinecone doc: 1000 + static constexpr std::size_t kMaxIdsPerFetch = 1000; // Pinecone doc: 1000 + static constexpr int kMaxTopK = 10000; // Pinecone doc: 10000 + static constexpr int kMaxListLimit = 100; // Pinecone doc: 100 + static constexpr std::size_t kMaxPointsPerUpsert = 1000; // own + static constexpr int kMaxPointLimit = 10000; // own + static constexpr std::size_t kMaxPages = 20; // list walks + static constexpr std::size_t kMaxItemsReturned = 5000; + // Brave + static constexpr int kBraveMaxCount = 50; // doc: web 20, news 50 + static constexpr int kBraveMaxOffset = 9; // doc + static constexpr std::size_t kBraveQueryChars = 400; // doc + // Tavily and Exa + static constexpr int kMaxCrawlLimit = 500; // own + static constexpr int kMaxCrawlDepth = 5; // doc + static constexpr int kTavilyMaxResults = 20; // doc + static constexpr int kExaMaxResults = 100; // own +}; + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/exa/calls.cpp b/src/vendors/retrieval/exa/calls.cpp new file mode 100644 index 0000000..47e2af9 --- /dev/null +++ b/src/vendors/retrieval/exa/calls.cpp @@ -0,0 +1,25 @@ +#include "vendors/retrieval/exa/exa.h" + +namespace molo::retrieval { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// name, host, http method, path, spec operation id (or reference page / "doc:"), +// body fields, query parameters, flags (d destructive, w writes, r read-only POST, +// p paid), response schema) +static const CallRow kRows[] = { + {"search", "main", "POST", "/search", "search", "additionalQueries,category,contents,context,endCrawlDate,endPublishedDate,excludeDomains,excludeText,includeDomains,includeText,moderation,numResults,outputSchema,query,startCrawlDate,startPublishedDate,type,userLocation", "", "rp", "-"}, + {"find_similar", "main", "POST", "/findSimilar", "findSimilar", "contents,context,endCrawlDate,endPublishedDate,excludeDomains,excludeSourceDomain,excludeText,includeDomains,includeText,moderation,numResults,startCrawlDate,startPublishedDate,url", "", "rp", "-"}, + {"contents", "main", "POST", "/contents", "getContents", "context,extras,highlights,ids,livecrawl,livecrawlTimeout,maxAgeHours,subpageTarget,subpages,summary,text,urls", "", "rp", "-"}, + {"answer", "main", "POST", "/answer", "answer", "outputSchema,query,stream,text", "", "rp", "-"}, + {"research_create", "main", "POST", "/research/v0/tasks", "research-tasks-create", "instructions,model,output", "", "rp", "-"}, + {"research_list", "main", "GET", "/research/v0/tasks", "research-tasks-list", "", "cursor,limit", "", "-"}, + {"research_get", "main", "GET", "/research/v0/tasks/{id}", "ResearchControllerV0_getResearchTask", "", "", "", "-"}, +}; +// CALLS-END + +const CallTable& exa_calls() { + static const CallTable table{kRows, sizeof(kRows) / sizeof(kRows[0])}; + return table; +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/exa/exa.cpp b/src/vendors/retrieval/exa/exa.cpp new file mode 100644 index 0000000..08bb737 --- /dev/null +++ b/src/vendors/retrieval/exa/exa.cpp @@ -0,0 +1,173 @@ +#include "vendors/retrieval/exa/exa.h" + +#include +#include + +#include "vendors/retrieval/caps.h" + +namespace molo::retrieval { + +llm::ApiClientConfig Exa::config() { + llm::ApiClientConfig c; + c.vendor = "exa"; + c.base_url = kBaseUrl; + c.key_env = kKeyEnv; + c.auth = llm::AuthStyle::kHeader; + c.auth_header = "x-api-key"; + return c; +} + +namespace { +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("exa", why); } +CallArgs with_body(Json body) { + CallArgs a; + a.body = std::move(body); + return a; +} +void check_url(const std::string& url) { + bool http = url.rfind("http://", 0) == 0 || url.rfind("https://", 0) == 0; + if (!http || url.size() > 8192 || url.find_first_of(" \t\r\n") != std::string::npos) + bad("url must be http(s) without spaces, at most 8192 chars"); +} +Json text_contents(int max_characters) { + Json text = Json::object(); + text.set("maxCharacters", Json(max_characters)); + Json c = Json::object(); + c.set("text", text); + return c; +} +} // namespace + +Exa Exa::from_env() { return Exa(make_gateway(exa_calls(), config(), {{"main", kBaseUrl}}, "", nullptr)); } +Exa Exa::with(const TransportFactory& factory, std::string key) { + return Exa(make_gateway(exa_calls(), config(), {{"main", kBaseUrl}}, std::move(key), factory)); +} + +int Exa::recency_days(const std::string& r) { + if (r == "day") return 1; + if (r == "week") return 7; + if (r == "month") return 31; + if (r == "year") return 365; + return 0; +} + +std::string Exa::days_ago_iso(long now_epoch_seconds, int days) { + std::time_t t = static_cast(now_epoch_seconds - static_cast(days) * 86400L); + std::tm tm{}; + gmtime_r(&t, &tm); + char buf[32]; + std::strftime(buf, sizeof buf, "%Y-%m-%dT%H:%M:%S.000Z", &tm); + return buf; +} + +SearchResult Exa::parse_results(const Json& doc) { + SearchResult out; + for (const Json& r : J::array(doc, "results")) { + SearchHit h; + h.title = J::str(r, "title"); + h.url = J::str(r, "url"); + h.published = J::str(r, "publishedDate"); + const Json::Array& hl = J::array(r, "highlights"); + if (!hl.empty() && hl.front().is_string()) h.snippet = hl.front().as_string(); + if (h.snippet.empty()) h.snippet = J::str(r, "summary"); + if (h.snippet.empty()) h.snippet = J::str(r, "text").substr(0, 500); + if (const Json* s = J::at(r, "score"); s && s->is_number()) { + h.score = s->as_number(); + h.has_score = true; + } + out.hits.push_back(std::move(h)); + } + return out; +} + +ExaAnswer Exa::parse_answer(const Json& doc) { + ExaAnswer out; + out.answer = J::str(doc, "answer"); + for (const Json& c : J::array(doc, "citations")) { + SearchHit h; + h.title = J::str(c, "title"); + h.url = J::str(c, "url"); + h.published = J::str(c, "publishedDate"); + h.snippet = J::str(c, "text").substr(0, 500); + out.citations.push_back(std::move(h)); + } + return out; +} + +SearchResult Exa::search(const SearchRequest& request) { return search_typed(request, ""); } + +SearchResult Exa::search_typed(const SearchRequest& request, const std::string& type) { + WebSearch::validate("exa", request); + if (request.max_results > RetrievalCaps::kExaMaxResults) bad("max_results is at most 100"); + static const char* types[] = {"", "neural", "fast", "auto", "deep", "deep-reasoning", "instant"}; + bool ok = false; + for (const char* t : types) ok = ok || type == t; + if (!ok) bad("type must be neural, fast, auto, deep, deep-reasoning or instant"); + Json body = Json::object(); + body.set("query", Json(request.query)); + body.set("numResults", Json(request.max_results)); + if (!type.empty()) body.set("type", Json(type)); + body.set("contents", text_contents(500)); + if (int days = recency_days(request.recency); days > 0) { + long now = static_cast(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); + body.set("startPublishedDate", Json(days_ago_iso(now, days))); + } + return parse_results(gw_->call("search", with_body(body))); +} + +SearchResult Exa::find_similar(const std::string& url, int num_results) { + check_url(url); + if (num_results < 1 || num_results > RetrievalCaps::kExaMaxResults) bad("num_results must be 1..100"); + Json body = Json::object(); + body.set("url", Json(url)); + body.set("numResults", Json(num_results)); + body.set("contents", text_contents(500)); + return parse_results(gw_->call("find_similar", with_body(body))); +} + +SearchResult Exa::contents(const std::vector& urls, int max_characters) { + if (urls.empty() || urls.size() > RetrievalCaps::kMaxUrls) bad("urls must hold 1..100 entries"); + for (const std::string& u : urls) check_url(u); + if (max_characters < 1 || max_characters > 1000000) bad("max_characters must be 1..1000000"); + Json body = Json::object(); + body.set("urls", J::strings(urls)); + body.set("text", [&] { + Json t = Json::object(); + t.set("maxCharacters", Json(max_characters)); + return t; + }()); + return parse_results(gw_->call("contents", with_body(body))); +} + +ExaAnswer Exa::answer(const std::string& query) { + if (query.empty() || query.size() > RetrievalCaps::kMaxQueryChars) bad("query must be 1..8192 chars"); + Json body = Json::object(); + body.set("query", Json(query)); + body.set("text", Json(false)); + return parse_answer(gw_->call("answer", with_body(body))); +} + +Json Exa::research_create(const std::string& instructions, const std::string& model) { + if (instructions.empty() || instructions.size() > 4096) bad("instructions must be 1..4096 chars"); + if (!model.empty() && model != "exa-research" && model != "exa-research-pro") bad("model must be exa-research or exa-research-pro"); + Json body = Json::object(); + body.set("instructions", Json(instructions)); + if (!model.empty()) body.set("model", Json(model)); + return gw_->call("research_create", with_body(body)); +} + +Json Exa::research_get(const std::string& id) { + CallArgs a; + a.path["id"] = id; + return gw_->call("research_get", a); +} + +Json Exa::research_list(int limit) { + if (limit < 1 || limit > 200) bad("limit must be 1..200"); + CallArgs a; + a.query["limit"] = Query::number(limit); + return gw_->call("research_list", a); +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/exa/exa.h b/src/vendors/retrieval/exa/exa.h new file mode 100644 index 0000000..002fad6 --- /dev/null +++ b/src/vendors/retrieval/exa/exa.h @@ -0,0 +1,49 @@ +#pragma once + +// Exa: search, contents, findSimilar, answer and research tasks. Implements +// the WebSearch seam. + +#include +#include + +#include "vendors/retrieval/provider.h" +#include "vendors/retrieval/seams.h" + +namespace molo::retrieval { + +const CallTable& exa_calls(); + +struct ExaAnswer { + std::string answer; + std::vector citations; +}; + +class Exa : public Provider, public WebSearch { + public: + static constexpr const char* kKeyEnv = "EXA_API_KEY"; + static constexpr const char* kBaseUrl = "https://api.exa.ai"; + + static llm::ApiClientConfig config(); + static Exa from_env(); + static Exa with(const TransportFactory& factory, std::string key); + explicit Exa(std::shared_ptr gateway) : Provider(std::move(gateway)) {} + + std::string provider() const override { return "exa"; } + // `type`: "" (auto), neural, fast, auto, deep, deep-reasoning, instant. + SearchResult search(const SearchRequest& request) override; + SearchResult search_typed(const SearchRequest& request, const std::string& type); + SearchResult find_similar(const std::string& url, int num_results); + SearchResult contents(const std::vector& urls, int max_characters); + ExaAnswer answer(const std::string& query); + Json research_create(const std::string& instructions, const std::string& model); + Json research_get(const std::string& id); + Json research_list(int limit); + + static SearchResult parse_results(const Json& doc); + static ExaAnswer parse_answer(const Json& doc); + // ISO-8601 UTC timestamp `days` before `now_epoch_seconds` (testable). + static std::string days_ago_iso(long now_epoch_seconds, int days); + static int recency_days(const std::string& recency); +}; + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/gateway.cpp b/src/vendors/retrieval/gateway.cpp new file mode 100644 index 0000000..5d9a3d7 --- /dev/null +++ b/src/vendors/retrieval/gateway.cpp @@ -0,0 +1,120 @@ +#include "vendors/retrieval/gateway.h" + +#include "clients/url.h" +#include "vendors/retrieval/caps.h" + +namespace molo::retrieval { + +namespace { +[[noreturn]] void bad(const std::string& vendor, const std::string& why) { + throw llm::ApiError::invalid(vendor, why); +} + +std::vector placeholders(const std::string& path) { + std::vector out; + std::size_t pos = 0; + while ((pos = path.find('{', pos)) != std::string::npos) { + std::size_t end = path.find('}', pos); + if (end == std::string::npos) break; + out.push_back(path.substr(pos + 1, end - pos - 1)); + pos = end + 1; + } + return out; +} + +bool bad_text(const std::string& v) { + for (unsigned char c : v) + if (c < 0x20 || c == 0x7f) return true; + return false; +} +} // namespace + +Gateway::Gateway(std::string vendor, CallTable table, Hosts hosts) + : vendor_(std::move(vendor)), table_(table), hosts_(std::move(hosts)) {} + +bool Gateway::configured() const { + if (hosts_.empty() || !missing_setting_.empty()) return false; + for (const auto& [name, client] : hosts_) + if (!client || !client->configured()) return false; + return true; +} + +PlannedCall Gateway::plan(const std::string& name, const CallArgs& args) const { + const CallRow* row = table_.find(name); + if (!row) bad(vendor_, "unknown call: " + name); + + // path parameters: exactly the placeholders, each a valid id segment + std::string path = row->path; + std::vector wanted = placeholders(path); + if (wanted.size() > RetrievalCaps::kMaxPathParams) bad(vendor_, "too many path parameters in the table"); + for (const auto& [k, v] : args.path) { + bool known = false; + for (const std::string& w : wanted) known = known || w == k; + if (!known) bad(vendor_, "unexpected path parameter: " + k); + } + for (const std::string& w : wanted) { + auto it = args.path.find(w); + if (it == args.path.end()) bad(vendor_, "missing path parameter: " + w); + if (!llm::Service::valid_id(it->second)) bad(vendor_, "bad " + w + " (1..256 of A-Z a-z 0-9 . _ : -)"); + std::string token = "{" + w + "}"; + path.replace(path.find(token), token.size(), clients::Url::encode_component(it->second)); + } + + // query parameters: listed in the row, bounded + if (args.query.size() + args.query_multi.size() > RetrievalCaps::kMaxQueryParams + RetrievalCaps::kMaxIdsPerFetch) + bad(vendor_, "too many query parameters"); + if (args.query.size() > RetrievalCaps::kMaxQueryParams) bad(vendor_, "too many query parameters"); + std::vector> pairs(args.query.begin(), args.query.end()); + pairs.insert(pairs.end(), args.query_multi.begin(), args.query_multi.end()); + std::string qs; + for (const auto& [k, v] : pairs) { + if (!RowInfo::listed(row->query, k)) bad(vendor_, "query parameter not in the call table: " + k); + if (v.size() > RetrievalCaps::kMaxQueryValueBytes) bad(vendor_, "query value too long: " + k); + if (bad_text(v)) bad(vendor_, "query value has control characters: " + k); + qs += qs.empty() ? '?' : '&'; + qs += clients::Url::encode_component(k) + "=" + clients::Url::encode_component(v); + } + + // body: object whose keys are listed, or free form + std::string body_spec = row->body; + if (!args.body.is_null()) { + if (body_spec.empty()) bad(vendor_, "this call takes no body"); + if (!args.body.is_object() && body_spec != "*") bad(vendor_, "body must be a JSON object"); + if (args.body.is_object() && body_spec != "*") + for (const auto& [k, v] : args.body.as_object()) + if (!RowInfo::listed(row->body, k)) bad(vendor_, "body field not in the call table: " + k); + if (args.body.dump().size() > RetrievalCaps::kMaxBodyBytes) bad(vendor_, "body over the byte cap"); + } + + if (hosts_.find(row->host) == hosts_.end()) bad(vendor_, std::string("no client for host ") + row->host); + + PlannedCall p; + p.name = row->name; + p.vendor = vendor_; + p.host = row->host; + p.method = row->method; + p.target = path + qs; + p.body = args.body; + p.flags = row->flags; + p.destructive = RowInfo::destructive(*row); + p.paid = RowInfo::paid(*row); + p.writes = RowInfo::writes(*row); + return p; +} + +Json Gateway::send(const std::string& name, const CallArgs& args, bool destructive) { + PlannedCall p = plan(name, args); + if (!missing_setting_.empty()) throw llm::NotConfiguredError(vendor_, missing_setting_); + if (p.destructive && !destructive) + bad(vendor_, name + " is destructive: use destroy()"); + if (!p.destructive && destructive) bad(vendor_, name + " is not destructive: use call()"); + const CallRow* row = table_.find(name); + std::shared_ptr client = hosts_.at(row->host); + const Json* body = args.body.is_null() ? nullptr : &args.body; + return client->request(RowInfo::method(*row), p.target, body, args.headers); +} + +Json Gateway::call(const std::string& name, const CallArgs& args) { return send(name, args, false); } +Json Gateway::destroy(const std::string& name, const CallArgs& args) { return send(name, args, true); } + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/gateway.h b/src/vendors/retrieval/gateway.h new file mode 100644 index 0000000..f8fe3da --- /dev/null +++ b/src/vendors/retrieval/gateway.h @@ -0,0 +1,81 @@ +#pragma once + +// Gateway: the table-driven engine behind every provider class. It validates a +// call against its CallRow (path parameters, query parameters, body fields, +// sizes), renders the request, and sends it through llm::ApiClient. Rows +// flagged destructive are refused by call() and only reachable through the +// explicit destroy(). plan() renders without sending (dry runs). + +#include +#include +#include + +#include "clients/body/json.h" +#include "llm/api_client.h" +#include "vendors/retrieval/call_table.h" + +namespace molo::retrieval { + +using Json = clients::body::Json; + +struct CallArgs { + std::map path; // {placeholder} values, ids only + std::map query; // already text; see Query helpers + // Repeated parameters (`ids=a&ids=b`); counted with `query` against the cap. + std::vector> query_multi; + Json body; // null = no body + llm::Headers headers; // extra per-call headers +}; + +struct PlannedCall { + std::string name; + std::string vendor; + std::string host; + std::string method; + std::string target; // path and query string + Json body; + std::string flags; + bool destructive = false; + bool paid = false; + bool writes = false; +}; + +class Gateway { + public: + using Hosts = std::map>; + + Gateway(std::string vendor, CallTable table, Hosts hosts); + + const std::string& vendor() const { return vendor_; } + const CallTable& table() const { return table_; } + // True when every host client has a credential and no setting is missing. + bool configured() const; + // Marks the gateway unusable until a setting exists (for example QDRANT_URL): + // send() then throws NotConfiguredError(vendor, env) before any traffic. + void require_setting(const std::string& env_name) { missing_setting_ = env_name; } + const std::string& missing_setting() const { return missing_setting_; } + const Hosts& hosts() const { return hosts_; } + + // Validates and renders; throws ApiError::invalid (400 INVALID_ARGUMENT). + PlannedCall plan(const std::string& name, const CallArgs& args = {}) const; + // Sends a non-destructive row. + Json call(const std::string& name, const CallArgs& args = {}); + // Sends a destructive row. No default arguments, on purpose. + Json destroy(const std::string& name, const CallArgs& args); + + private: + Json send(const std::string& name, const CallArgs& args, bool destructive); + + std::string vendor_; + CallTable table_; + Hosts hosts_; + std::string missing_setting_; +}; + +// Query text helpers: one place that decides how values are written. +struct Query { + static std::string flag(bool v) { return v ? "true" : "false"; } + static std::string number(long v) { return std::to_string(v); } +}; + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/jina/calls.cpp b/src/vendors/retrieval/jina/calls.cpp new file mode 100644 index 0000000..8301ebb --- /dev/null +++ b/src/vendors/retrieval/jina/calls.cpp @@ -0,0 +1,33 @@ +#include "vendors/retrieval/jina/jina.h" + +namespace molo::retrieval { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// name, host, http method, path, spec operation id (or reference page / "doc:"), +// body fields, query parameters, flags (d destructive, w writes, r read-only POST, +// p paid), response schema) +static const CallRow kRows[] = { + {"embeddings", "api", "POST", "/v1/embeddings", "embeddings_v1_embeddings_post", "dimensions,embedding_type,input,input_type,late_chunking,model,normalized,return_multivector,return_tokenized_input,task,truncate", "", "rp", "EmbeddingResponse"}, + {"rerank", "api", "POST", "/v1/rerank", "rerank_v1_rerank_post", "documents,max_doc_length,model,query,return_documents,return_embeddings,top_n", "", "rp", "RerankingResponse"}, + {"list_models", "api", "GET", "/v1/models", "list_models_v1_models_get", "", "", "", "ModelListResponse"}, + {"get_model", "api", "GET", "/v1/models/{model_id}", "get_model_v1_models__model_id__get", "", "", "", "ModelInfo"}, + {"create_batch", "api", "POST", "/v1/batch/embeddings", "create_batch_job_v1_batch_embeddings_post", "dimensions,input,input_url,model,normalized,task,webhook_url", "", "wp", "BatchStatus"}, + {"get_batch", "api", "GET", "/v1/batch/{batch_id}", "get_batch_status_v1_batch__batch_id__get", "", "", "", "BatchStatus"}, + {"cancel_batch", "api", "DELETE", "/v1/batch/{batch_id}", "cancel_batch_job_v1_batch__batch_id__delete", "", "", "d", "BatchStatus"}, + {"list_batches", "api", "GET", "/v1/batches", "list_batch_jobs_v1_batches_get", "", "limit", "", "inline"}, + {"batch_output", "api", "GET", "/v1/batch/{batch_id}/output", "download_batch_output_v1_batch__batch_id__output_get", "", "", "", "-"}, + {"batch_errors", "api", "GET", "/v1/batch/{batch_id}/errors", "download_batch_errors_v1_batch__batch_id__errors_get", "", "", "", "-"}, + {"read", "reader", "GET", "/", "", "", "assertStatusCode,base,cacheTolerance,detachInvisibles,engine,exportStorageState,file,html,injectFrameScript,injectPageScript,instruction,jsonSchema,keepImgDataUrl,locale,markdownChunking,maxTokens,noCache,noGfm,noServiceWorker,page,pdf,preloadUrl,preset,proxy,proxyResources,proxyUrl,referer,removeOverlay,removeSelector,respondTiming,respondWith,retainImages,retainLinks,retainMedia,robotsTxt,setCookies,skipCertVerification,targetSelector,timeout,tokenBudget,url,userAgent,waitForSelector,webBotAuth,withGeneratedAlt,withIframe,withImagesSummary,withLinksSummary,withShadowDom,zeroDataRetention", "p", "-"}, + {"search", "search", "GET", "/search", "", "", "assertStatusCode,base,cacheTolerance,count,detachInvisibles,engine,exportStorageState,ext,fallback,file,filetype,gl,hl,html,injectFrameScript,injectPageScript,instruction,intitle,jsonSchema,keepImgDataUrl,loc,locale,location,markdownChunking,maxTokens,nfpr,noCache,noGfm,noServiceWorker,num,page,pdf,preloadUrl,preset,provider,proxy,proxyResources,proxyUrl,q,referer,removeOverlay,removeSelector,respondTiming,respondWith,retainImages,retainLinks,retainMedia,robotsTxt,setCookies,site,skipCertVerification,targetSelector,timeout,tokenBudget,type,url,userAgent,waitForSelector,webBotAuth,withGeneratedAlt,withIframe,withImagesSummary,withLinksSummary,withShadowDom,zeroDataRetention", "p", "inline"}, + {"count_token", "segment", "POST", "/countToken", "", "content,head,max_chunk_length,return_chunks,return_tokens,tail,tokenizer", "content,head,max_chunk_length,return_chunks,return_tokens,tail,tokenizer", "rp", "-"}, + {"segment", "segment", "POST", "/", "doc:segmenter", "content,tokenizer,return_tokens,return_chunks,max_chunk_length,head,tail", "", "p", "-"}, + {"classify", "api", "POST", "/v1/classify", "doc:classifier", "model,input,labels", "", "p", "-"}, +}; +// CALLS-END + +const CallTable& jina_calls() { + static const CallTable table{kRows, sizeof(kRows) / sizeof(kRows[0])}; + return table; +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/jina/jina.cpp b/src/vendors/retrieval/jina/jina.cpp new file mode 100644 index 0000000..5fc9a06 --- /dev/null +++ b/src/vendors/retrieval/jina/jina.cpp @@ -0,0 +1,217 @@ +#include "vendors/retrieval/jina/jina.h" + +#include + +#include "vendors/retrieval/caps.h" + +namespace molo::retrieval { + +llm::ApiClientConfig Jina::config() { + llm::ApiClientConfig c; + c.vendor = "jina"; + c.base_url = kApiUrl; + c.key_env = kKeyEnv; + c.auth = llm::AuthStyle::kBearer; + return c; +} + +namespace { +std::vector hosts() { + return {{"api", Jina::kApiUrl}, {"reader", Jina::kReaderUrl}, {"search", Jina::kSearchUrl}, {"segment", Jina::kSegmentUrl}}; +} +llm::Headers json_accept() { return {{"Accept", "application/json"}}; } +} // namespace + +Jina Jina::from_env() { return Jina(make_gateway(jina_calls(), config(), hosts(), "", nullptr)); } +Jina Jina::with(const TransportFactory& factory, std::string key) { + return Jina(make_gateway(jina_calls(), config(), hosts(), std::move(key), factory)); +} + +void Jina::check_url(const std::string& url) { + bool http = url.rfind("http://", 0) == 0 || url.rfind("https://", 0) == 0; + if (!http || url.size() > 8192 || url.find_first_of(" \t\r\n") != std::string::npos) + throw llm::ApiError::invalid("jina", "url must be http(s) without spaces, at most 8192 chars"); +} + +EmbedResult Jina::parse_embeddings(const Json& doc) { + EmbedResult out; + out.model = J::str(doc, "model"); + std::vector>> rows; + for (const Json& d : J::array(doc, "data")) { + const Json* e = J::at(d, "embedding"); + rows.emplace_back(static_cast(J::num(d, "index", static_cast(rows.size()))), + e ? J::floats(*e) : std::vector{}); + } + std::stable_sort(rows.begin(), rows.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); + for (auto& r : rows) out.vectors.push_back(std::move(r.second)); + if (const Json* u = J::at(doc, "usage")) out.total_tokens = static_cast(J::num(*u, "total_tokens")); + return out; +} + +RerankResult Jina::parse_rerank(const Json& doc) { + RerankResult out; + out.model = J::str(doc, "model"); + for (const Json& d : J::array(doc, "results")) { + RerankHit h; + h.index = static_cast(J::num(d, "index")); + h.score = J::num(d, "relevance_score"); + out.hits.push_back(h); + } + std::stable_sort(out.hits.begin(), out.hits.end(), [](const RerankHit& a, const RerankHit& b) { return a.score > b.score; }); + if (const Json* u = J::at(doc, "usage")) out.total_tokens = static_cast(J::num(*u, "total_tokens")); + return out; +} + +JinaPage Jina::parse_page(const Json& doc) { + JinaPage p; + const Json* data = J::at(doc, "data"); + const Json& d = data && data->is_object() ? *data : doc; + p.title = J::str(d, "title"); + p.url = J::str(d, "url"); + p.description = J::str(d, "description"); + p.content = J::str(d, "content"); + if (const Json* u = J::at(d, "usage")) p.tokens = static_cast(J::num(*u, "tokens")); + return p; +} + +SearchResult Jina::parse_search(const Json& doc) { + SearchResult out; + for (const Json& d : J::array(doc, "data")) { + SearchHit h; + h.title = J::str(d, "title"); + h.url = J::str(d, "url"); + h.snippet = J::str(d, "description"); + if (h.snippet.empty()) h.snippet = J::str(d, "content").substr(0, 500); + out.hits.push_back(h); + } + return out; +} + +JinaSegments Jina::parse_segments(const Json& doc) { + JinaSegments s; + s.num_tokens = static_cast(J::num(doc, "num_tokens")); + s.num_chunks = static_cast(J::num(doc, "num_chunks")); + for (const Json& c : J::array(doc, "chunks")) + if (c.is_string()) s.chunks.push_back(c.as_string()); + return s; +} + +std::vector Jina::parse_classification(const Json& doc) { + std::vector out; + for (const Json& d : J::array(doc, "data")) { + JinaPrediction p; + p.index = static_cast(J::num(d, "index", static_cast(out.size()))); + p.label = J::str(d, "prediction"); + p.score = J::num(d, "score"); + out.push_back(p); + } + return out; +} + +EmbedResult Jina::embed(const EmbedRequest& request) { + Embedder::validate("jina", request); + Json body = Json::object(); + body.set("model", Json(request.model.empty() ? std::string(kDefaultEmbedModel) : request.model)); + body.set("input", J::strings(request.texts)); + if (request.input_type == "query") body.set("task", Json("retrieval.query")); + if (request.input_type == "document") body.set("task", Json("retrieval.passage")); + if (request.dimensions > 0) body.set("dimensions", Json(request.dimensions)); + CallArgs args; + args.body = body; + EmbedResult r = parse_embeddings(gw_->call("embeddings", args)); + if (r.vectors.size() != request.texts.size()) + throw llm::ApiError("jina", 502, "BAD_RESPONSE", "embedding count differs from input count"); + return r; +} + +RerankResult Jina::rerank(const RerankRequest& request) { + Reranker::validate("jina", request); + Json body = Json::object(); + body.set("model", Json(request.model.empty() ? std::string(kDefaultRerankModel) : request.model)); + body.set("query", Json(request.query)); + body.set("documents", J::strings(request.documents)); + body.set("return_documents", Json(false)); + if (request.top_n > 0) body.set("top_n", Json(request.top_n)); + CallArgs args; + args.body = body; + return parse_rerank(gw_->call("rerank", args)); +} + +SearchResult Jina::search(const SearchRequest& request) { + WebSearch::validate("jina", request); + if (!request.recency.empty()) throw llm::ApiError::invalid("jina", "jina search has no recency filter"); + CallArgs args; + args.query["q"] = request.query; + args.headers = json_accept(); + return parse_search(gw_->call("search", args)); +} + +JinaPage Jina::read(const std::string& url) { + check_url(url); + CallArgs args; + args.query["url"] = url; + args.headers = json_accept(); + return parse_page(gw_->call("read", args)); +} + +JinaSegments Jina::segment(const std::string& content, const std::string& tokenizer, int max_chunk_length) { + if (content.empty() || content.size() > 65536) throw llm::ApiError::invalid("jina", "content must be 1..65536 bytes"); + if (max_chunk_length < 0 || max_chunk_length > 8192) throw llm::ApiError::invalid("jina", "max_chunk_length out of range"); + Json body = Json::object(); + body.set("content", Json(content)); + if (!tokenizer.empty()) body.set("tokenizer", Json(tokenizer)); + body.set("return_chunks", Json(true)); + if (max_chunk_length > 0) body.set("max_chunk_length", Json(max_chunk_length)); + CallArgs args; + args.body = body; + return parse_segments(gw_->call("segment", args)); +} + +long Jina::count_tokens(const std::string& content, const std::string& tokenizer) { + if (content.empty() || content.size() > 65536) throw llm::ApiError::invalid("jina", "content must be 1..65536 bytes"); + CallArgs args; + args.query["content"] = content; + if (!tokenizer.empty()) args.query["tokenizer"] = tokenizer; + Json doc = gw_->call("count_token", args); + return static_cast(J::num(doc, "num_tokens")); +} + +std::vector Jina::classify(const std::vector& texts, + const std::vector& labels, const std::string& model) { + if (texts.empty() || texts.size() > RetrievalCaps::kMaxEmbedTexts) throw llm::ApiError::invalid("jina", "texts must hold 1..1000 entries"); + if (labels.size() < 2 || labels.size() > 256) throw llm::ApiError::invalid("jina", "labels must hold 2..256 entries"); + Json body = Json::object(); + body.set("model", Json(model.empty() ? std::string(kDefaultClassifyModel) : model)); + body.set("input", J::strings(texts)); + body.set("labels", J::strings(labels)); + CallArgs args; + args.body = body; + return parse_classification(gw_->call("classify", args)); +} + +Json Jina::list_models() { return gw_->call("list_models"); } + +Json Jina::create_batch(const std::string& model, const std::string& input_url, const std::string& task) { + check_url(input_url); + Json body = Json::object(); + body.set("model", Json(model)); + body.set("input_url", Json(input_url)); + if (!task.empty()) body.set("task", Json(task)); + CallArgs args; + args.body = body; + return gw_->call("create_batch", args); +} + +Json Jina::get_batch(const std::string& batch_id) { + CallArgs args; + args.path["batch_id"] = batch_id; + return gw_->call("get_batch", args); +} + +Json Jina::cancel_batch(const std::string& batch_id) { + CallArgs args; + args.path["batch_id"] = batch_id; + return gw_->destroy("cancel_batch", args); +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/jina/jina.h b/src/vendors/retrieval/jina/jina.h new file mode 100644 index 0000000..00380e0 --- /dev/null +++ b/src/vendors/retrieval/jina/jina.h @@ -0,0 +1,80 @@ +#pragma once + +// Jina AI: embeddings, rerank, models and batch embeddings (api.jina.ai), the +// Reader (r.jina.ai), Search (s.jina.ai), the Segmenter (segment.jina.ai) and +// the Classifier. Implements Embedder, Reranker and WebSearch. + +#include +#include + +#include "vendors/retrieval/provider.h" +#include "vendors/retrieval/seams.h" + +namespace molo::retrieval { + +const CallTable& jina_calls(); + +struct JinaPage { + std::string title; + std::string url; + std::string description; + std::string content; // markdown + long tokens = 0; +}; + +struct JinaSegments { + long num_tokens = 0; + long num_chunks = 0; + std::vector chunks; +}; + +struct JinaPrediction { + int index = 0; + std::string label; + double score = 0; +}; + +class Jina : public Provider, public Embedder, public Reranker, public WebSearch { + public: + static constexpr const char* kKeyEnv = "JINA_API_KEY"; + static constexpr const char* kApiUrl = "https://api.jina.ai"; + static constexpr const char* kReaderUrl = "https://r.jina.ai"; + static constexpr const char* kSearchUrl = "https://s.jina.ai"; + static constexpr const char* kSegmentUrl = "https://segment.jina.ai"; + static constexpr const char* kDefaultEmbedModel = "jina-embeddings-v3"; + static constexpr const char* kDefaultRerankModel = "jina-reranker-v3"; + static constexpr const char* kDefaultClassifyModel = "jina-embeddings-v3"; + + static llm::ApiClientConfig config(); + static Jina from_env(); + static Jina with(const TransportFactory& factory, std::string key); + explicit Jina(std::shared_ptr gateway) : Provider(std::move(gateway)) {} + + std::string provider() const override { return "jina"; } + EmbedResult embed(const EmbedRequest& request) override; + RerankResult rerank(const RerankRequest& request) override; + SearchResult search(const SearchRequest& request) override; + + JinaPage read(const std::string& url); + JinaSegments segment(const std::string& content, const std::string& tokenizer, int max_chunk_length); + long count_tokens(const std::string& content, const std::string& tokenizer); + std::vector classify(const std::vector& texts, + const std::vector& labels, const std::string& model); + Json list_models(); + Json create_batch(const std::string& model, const std::string& input_url, const std::string& task); + Json get_batch(const std::string& batch_id); + // Destructive: cancels a running batch job. + Json cancel_batch(const std::string& batch_id); + + static EmbedResult parse_embeddings(const Json& doc); + static RerankResult parse_rerank(const Json& doc); + static JinaPage parse_page(const Json& doc); + static SearchResult parse_search(const Json& doc); + static JinaSegments parse_segments(const Json& doc); + static std::vector parse_classification(const Json& doc); + + // http(s) URL, 1..8192 chars; throws ApiError::invalid. + static void check_url(const std::string& url); +}; + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/mixedbread/calls.cpp b/src/vendors/retrieval/mixedbread/calls.cpp new file mode 100644 index 0000000..eff491b --- /dev/null +++ b/src/vendors/retrieval/mixedbread/calls.cpp @@ -0,0 +1,98 @@ +#include "vendors/retrieval/mixedbread/mixedbread.h" + +namespace molo::retrieval { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// name, host, http method, path, spec operation id (or reference page / "doc:"), +// body fields, query parameters, flags (d destructive, w writes, r read-only POST, +// p paid), response schema) +static const CallRow kRows[] = { + {"create_file", "main", "POST", "/v1/files", "create_file", "file", "", "w", "FileObject"}, + {"list_files", "main", "GET", "/v1/files", "list_files", "", "after,before,include_total,limit,q", "", "FileListResponse"}, + {"list_multipart_uploads", "main", "GET", "/v1/files/uploads", "list_multipart_uploads", "", "", "", "MultipartUploadListResponse"}, + {"create_multipart_upload", "main", "POST", "/v1/files/uploads", "create_multipart_upload", "file_size,filename,mime_type,part_count", "", "w", "CreateMultipartUploadResponse"}, + {"complete_multipart_upload", "main", "POST", "/v1/files/uploads/{upload_id}/complete", "complete_multipart_upload", "parts", "", "w", "FileObject"}, + {"abort_multipart_upload", "main", "POST", "/v1/files/uploads/{upload_id}/abort", "abort_multipart_upload", "", "", "d", "FileDeleted"}, + {"get_multipart_upload", "main", "GET", "/v1/files/uploads/{upload_id}", "get_multipart_upload", "", "", "", "MultipartUploadDetailResponse"}, + {"retrieve_file", "main", "GET", "/v1/files/{file_id}", "retrieve_file", "", "", "", "FileObject"}, + {"update_file", "main", "POST", "/v1/files/{file_id}", "update_file", "file", "", "w", "FileObject"}, + {"delete_file", "main", "DELETE", "/v1/files/{file_id}", "delete_file", "", "", "d", "FileDeleted"}, + {"download_file", "main", "GET", "/v1/files/{file_id}/content", "download_file", "", "", "", "inline"}, + {"list_store_events", "main", "GET", "/v1/stores/{store_identifier}/events", "list_store_events", "", "after,before,event_type,filter_after,filter_before,include_total,limit", "", "StoreEventListResponse"}, + {"get_store_event_histogram", "main", "POST", "/v1/stores/{store_identifier}/events/histogram", "get_store_event_histogram", "bucket_seconds,end_time,event_types,start_time", "", "r", "StoreEventHistogramResponse"}, + {"create_store_file", "main", "POST", "/v1/stores/{store_identifier}/files", "create_store_file", "config,experimental,external_id,file_id,metadata,overwrite", "", "w", "StoreFile"}, + {"list_store_files", "main", "GET", "/v1/stores/{store_identifier}/files", "list_store_files", "", "after,before,include_total,limit,sort_by,sort_order,statuses", "", "StoreFileListResponse"}, + {"upload_store_file", "main", "POST", "/v1/stores/{store_identifier}/files/upload", "upload_store_file", "file,params", "", "w", "StoreFile"}, + {"list_store_files_with_metadata_filter", "main", "POST", "/v1/stores/{store_identifier}/files/list", "list_store_files_with_metadata_filter", "after,before,include_total,limit,metadata_filter,q,sort_by,sort_order,statuses", "", "r", "StoreFileListResponse"}, + {"retrieve_store_file", "main", "GET", "/v1/stores/{store_identifier}/files/{file_identifier}", "retrieve_store_file", "", "return_chunks", "", "StoreFile"}, + {"update_store_file", "main", "PATCH", "/v1/stores/{store_identifier}/files/{file_identifier}", "update_store_file", "metadata", "", "w", "StoreFile"}, + {"delete_store_file", "main", "DELETE", "/v1/stores/{store_identifier}/files/{file_identifier}", "delete_store_file", "", "", "d", "StoreFileDeleted"}, + {"create_search_rule", "main", "POST", "/v1/stores/{store_identifier}/rules", "create_search_rule", "query,rules", "", "r", "mxbai_omni__api__routes__v1__stores__rules__models__SearchRuleResponse"}, + {"retrieve_search_rule", "main", "GET", "/v1/stores/{store_identifier}/rules/{rule_id}", "retrieve_search_rule", "", "", "", "mxbai_omni__api__routes__v1__stores__rules__models__SearchRuleResponse"}, + {"update_search_rule", "main", "PUT", "/v1/stores/{store_identifier}/rules/{rule_id}", "update_search_rule", "query,rules", "", "r", "mxbai_omni__api__routes__v1__stores__rules__models__SearchRuleResponse"}, + {"delete_search_rule", "main", "DELETE", "/v1/stores/{store_identifier}/rules/{rule_id}", "delete_search_rule", "", "", "d", "mxbai_omni__api__routes__v1__stores__rules__models__SearchRuleDeleted"}, + {"delete_specific_search_rule", "main", "DELETE", "/v1/stores/{store_identifier}/rules/{rule_id}/specific", "delete_specific_search_rule", "rule", "", "d", "mxbai_omni__api__routes__v1__stores__rules__models__SearchRuleSpecificDeleted"}, + {"create_store", "main", "POST", "/v1/stores", "create_store", "config,description,expires_after,file_ids,is_public,license,metadata,name,tags", "", "w", "Store"}, + {"list_stores", "main", "GET", "/v1/stores", "list_stores", "", "after,before,include_total,limit,q,tags", "", "StoreListResponse"}, + {"retrieve_store", "main", "GET", "/v1/stores/{store_identifier}", "retrieve_store", "", "", "", "Store"}, + {"update_store", "main", "PUT", "/v1/stores/{store_identifier}", "update_store", "description,expires_after,is_public,license,metadata,name,tags", "", "w", "Store"}, + {"delete_store", "main", "DELETE", "/v1/stores/{store_identifier}", "delete_store", "", "", "d", "StoreDeleted"}, + {"copy_store", "main", "POST", "/v1/stores/{store_identifier}/copy", "copy_store", "description,metadata,name,tags", "", "w", "Store"}, + {"train_store_lsf", "main", "POST", "/v1/stores/{store_identifier}/lsf/train", "train_store_lsf", "", "", "w", "StoreLsfTrainingQueued"}, + {"clear_store_lsf", "main", "DELETE", "/v1/stores/{store_identifier}/lsf", "clear_store_lsf", "", "", "d", "StoreLsfCleared"}, + {"search_store_chunks", "main", "POST", "/v1/stores/search", "search_store_chunks", "file_ids,filters,query,search_options,store_identifiers,stream,top_k", "", "rp", "StoreSearchResponse"}, + {"grep_store_chunks", "main", "POST", "/v1/stores/grep", "grep_store_chunks", "case_sensitive,file_ids,filters,pattern,return_metadata,store_identifiers,targets,top_k", "", "r", "StoreGrepResponse"}, + {"list_store_chunks", "main", "POST", "/v1/stores/list-chunks", "list_store_chunks", "file_ids,filters,search_options,sort_by,store_identifiers,top_k", "", "r", "StoreListChunksResponse"}, + {"create_question_answering", "main", "POST", "/v1/stores/question-answering", "create_question_answering", "file_ids,filters,instructions,qa_options,query,search_options,store_identifiers,stream,top_k", "", "rp", "StoreQAResults"}, + {"enhance_query", "main", "POST", "/v1/stores/queries/enhance", "enhance_query", "file_ids,filters,instructions,query,search_options,store_identifiers,top_k", "", "rp", "QueryEnhanceResults"}, + {"get_metadata_facets", "main", "POST", "/v1/stores/metadata-facets", "get_metadata_facets", "facets,file_ids,filters,max_fields,max_files,max_values_per_field,query,search_options,store_identifiers,top_k", "", "r", "MetadataFacets"}, + {"create_vector_store_file", "main", "POST", "/v1/vector_stores/{vector_store_identifier}/files", "create_vector_store_file", "config,experimental,external_id,file_id,metadata,overwrite", "", "w", "VectorStoreFile"}, + {"list_vector_store_files", "main", "GET", "/v1/vector_stores/{vector_store_identifier}/files", "list_vector_store_files", "", "after,before,include_total,limit,sort_by,sort_order,statuses", "", "VectorStoreFileListResponse"}, + {"list_vector_store_files_with_metadata_filter", "main", "POST", "/v1/vector_stores/{vector_store_identifier}/files/list", "list_vector_store_files_with_metadata_filter", "after,before,include_total,limit,metadata_filter,sort_by,sort_order,statuses", "", "r", "VectorStoreFileListResponse"}, + {"retrieve_vector_store_file", "main", "GET", "/v1/vector_stores/{vector_store_identifier}/files/{file_id}", "retrieve_vector_store_file", "", "return_chunks", "", "VectorStoreFile"}, + {"delete_vector_store_file", "main", "DELETE", "/v1/vector_stores/{vector_store_identifier}/files/{file_id}", "delete_vector_store_file", "", "", "d", "VectorStoreFileDeleted"}, + {"deprecated_create_search_rule", "main", "POST", "/v1/vector_stores/{vector_store_identifier}/rules", "deprecated_create_search_rule", "query,rules", "", "r", "mxbai_omni__api__routes__v1__deprecated_vector_stores__models__SearchRuleResponse"}, + {"deprecated_retrieve_search_rule", "main", "GET", "/v1/vector_stores/{vector_store_identifier}/rules/{rule_id}", "deprecated_retrieve_search_rule", "", "", "", "mxbai_omni__api__routes__v1__deprecated_vector_stores__models__SearchRuleResponse"}, + {"deprecated_update_search_rule", "main", "PUT", "/v1/vector_stores/{vector_store_identifier}/rules/{rule_id}", "deprecated_update_search_rule", "query,rules", "", "r", "mxbai_omni__api__routes__v1__deprecated_vector_stores__models__SearchRuleResponse"}, + {"deprecated_delete_search_rule", "main", "DELETE", "/v1/vector_stores/{vector_store_identifier}/rules/{rule_id}", "deprecated_delete_search_rule", "", "", "d", "mxbai_omni__api__routes__v1__deprecated_vector_stores__models__SearchRuleDeleted"}, + {"deprecated_delete_specific_search_rule", "main", "DELETE", "/v1/vector_stores/{vector_store_identifier}/rules/{rule_id}/specific", "deprecated_delete_specific_search_rule", "rule", "", "d", "mxbai_omni__api__routes__v1__deprecated_vector_stores__models__SearchRuleSpecificDeleted"}, + {"create_vector_store", "main", "POST", "/v1/vector_stores", "create_vector_store", "description,expires_after,file_ids,is_public,metadata,name", "", "w", "VectorStore"}, + {"list_vector_stores", "main", "GET", "/v1/vector_stores", "list_vector_stores", "", "after,before,include_total,limit,q", "", "VectorStoreListResponse"}, + {"retrieve_vector_store", "main", "GET", "/v1/vector_stores/{vector_store_identifier}", "retrieve_vector_store", "", "", "", "VectorStore"}, + {"update_vector_store", "main", "PUT", "/v1/vector_stores/{vector_store_identifier}", "update_vector_store", "description,expires_after,is_public,metadata,name", "", "w", "VectorStore"}, + {"delete_vector_store", "main", "DELETE", "/v1/vector_stores/{vector_store_identifier}", "delete_vector_store", "", "", "d", "VectorStoreDeleted"}, + {"search_vector_store_chunks", "main", "POST", "/v1/vector_stores/search", "search_vector_store_chunks", "file_ids,filters,query,search_options,top_k,vector_store_identifiers", "", "rp", "VectorStoreSearchResponse"}, + {"deprecated_create_question_answering", "main", "POST", "/v1/vector_stores/question-answering", "deprecated_create_question_answering", "file_ids,filters,qa_options,query,search_options,stream,top_k,vector_store_identifiers", "", "rp", "VectorStoreQAResults"}, + {"create_chat_completion", "main", "POST", "/v1/chat/completions", "create_chat_completion", "context_management,include,max_completion_tokens,max_tokens,max_tool_calls,messages,metadata,model,parallel_tool_calls,previous_completion_id,previous_messages,response_format,store,stream,temperature,terminal_tool_name,tool_choice,tools,top_p", "", "wp", "ChatCompletion"}, + {"list_chat_completions", "main", "GET", "/v1/chat/completions", "list_chat_completions", "", "after,include,limit,q", "", "ChatCompletionList"}, + {"retrieve_chat_completion", "main", "GET", "/v1/chat/completions/{completion_id}", "retrieve_chat_completion", "", "include", "", "ChatCompletion"}, + {"delete_chat_completion", "main", "DELETE", "/v1/chat/completions/{completion_id}", "delete_chat_completion", "", "", "d", "ChatCompletionDeleted"}, + {"list_chat_completion_turns", "main", "GET", "/v1/chat/completions/{completion_id}/turns", "list_chat_completion_turns", "", "include", "", "ChatCompletionList"}, + {"list_chat_completion_messages", "main", "GET", "/v1/chat/completions/{completion_id}/messages", "list_chat_completion_messages", "", "", "", "CompletionMessageList"}, + {"close_chat_completion_transcript", "main", "POST", "/v1/chat/completions/{completion_id}/transcript/close", "close_chat_completion_transcript", "answer,tool_call_id,tool_content", "", "w", "ChatCompletion"}, + {"create_response", "main", "POST", "/v1/responses", "create_response", "background,client_metadata,context_management,include,input,instructions,max_output_tokens,max_tool_calls,metadata,model,parallel_tool_calls,previous_response_id,prompt_cache_key,reasoning,safety_identifier,service_tier,store,stream,stream_options,temperature,text,tool_choice,tools,top_p,truncation,user", "", "wp", "Response"}, + {"retrieve_response", "main", "GET", "/v1/responses/{response_id}", "retrieve_response", "", "include[],starting_after,stream", "", "Response"}, + {"delete_response", "main", "DELETE", "/v1/responses/{response_id}", "delete_response", "", "", "d", "ResponseDeleted"}, + {"list_response_input_items", "main", "GET", "/v1/responses/{response_id}/input_items", "list_response_input_items", "", "after,limit,order", "", "ResponseItemList"}, + {"create_parsing_job", "main", "POST", "/v1/parsing/jobs", "create_parsing_job", "chunking_strategy,element_types,file_id,mode,return_format", "", "wp", "ParsingJobResponse"}, + {"list_parsing_jobs", "main", "GET", "/v1/parsing/jobs", "list_parsing_jobs", "", "after,before,include_total,limit,q,statuses", "", "ParsingJobListResponse"}, + {"retrieve_parsing_job", "main", "GET", "/v1/parsing/jobs/{job_id}", "retrieve_parsing_job", "", "", "", "ParsingJobResponse"}, + {"cancel_parsing_job", "main", "PATCH", "/v1/parsing/jobs/{job_id}", "cancel_parsing_job", "", "", "d", "ParsingJobResponse"}, + {"delete_parsing_job", "main", "DELETE", "/v1/parsing/jobs/{job_id}", "delete_parsing_job", "", "", "d", "ParsingJobDeleted"}, + {"create_extraction_job", "main", "POST", "/v1/extractions/jobs", "create_extraction_job", "file_id,json_schema", "", "rp", "ExtractionJob"}, + {"retrieve_extraction_job", "main", "GET", "/v1/extractions/jobs/{job_id}", "retrieve_extraction_job", "", "", "", "ExtractionJob"}, + {"create_schema", "main", "POST", "/v1/extractions/schema", "create_schema", "description", "", "w", "CreatedJsonSchema"}, + {"enhance_schema", "main", "POST", "/v1/extractions/schema/enhance", "enhance_schema", "json_schema", "", "r", "EnhancedJsonSchema"}, + {"validate_schema", "main", "POST", "/v1/extractions/schema/validate", "validate_schema", "json_schema", "", "r", "ValidatedJsonSchema"}, + {"extract_content", "main", "POST", "/v1/extractions/content", "extract_content", "content,instructions,json_schema", "", "rp", "ExtractionResult"}, + {"create_embeddings", "main", "POST", "/v1/embeddings", "create_embeddings", "dimensions,encoding_format,input,model,normalized,prompt", "", "rp", "EmbeddingCreateResponse"}, + {"rerank", "main", "POST", "/v1/reranking", "rerank", "input,model,query,rank_fields,return_input,rewrite_query,top_k", "", "rp", "RerankResponse"}, +}; +// CALLS-END + +const CallTable& mixedbread_calls() { + static const CallTable table{kRows, sizeof(kRows) / sizeof(kRows[0])}; + return table; +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/mixedbread/mixedbread.cpp b/src/vendors/retrieval/mixedbread/mixedbread.cpp new file mode 100644 index 0000000..db834de --- /dev/null +++ b/src/vendors/retrieval/mixedbread/mixedbread.cpp @@ -0,0 +1,160 @@ +#include "vendors/retrieval/mixedbread/mixedbread.h" + +#include + +#include "vendors/retrieval/caps.h" + +namespace molo::retrieval { + +llm::ApiClientConfig Mixedbread::config() { + llm::ApiClientConfig c; + c.vendor = "mixedbread"; + c.base_url = kBaseUrl; + c.key_env = kKeyEnv; + c.auth = llm::AuthStyle::kBearer; + return c; +} + +namespace { +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("mixedbread", why); } +CallArgs with_body(Json body) { + CallArgs a; + a.body = std::move(body); + return a; +} +CallArgs store_arg(const std::string& store) { + CallArgs a; + a.path["store_identifier"] = store; + return a; +} +void check_stores(const std::vector& stores) { + if (stores.empty() || stores.size() > 16) bad("stores must hold 1..16 entries"); + for (const std::string& s : stores) + if (!llm::Service::valid_id(s)) bad("bad store identifier"); +} +} // namespace + +Mixedbread Mixedbread::from_env() { + return Mixedbread(make_gateway(mixedbread_calls(), config(), {{"main", kBaseUrl}}, "", nullptr)); +} +Mixedbread Mixedbread::with(const TransportFactory& factory, std::string key) { + return Mixedbread(make_gateway(mixedbread_calls(), config(), {{"main", kBaseUrl}}, std::move(key), factory)); +} + +EmbedResult Mixedbread::parse_embeddings(const Json& doc) { + EmbedResult out; + out.model = J::str(doc, "model"); + std::vector>> rows; + for (const Json& d : J::array(doc, "data")) { + const Json* e = J::at(d, "embedding"); + rows.emplace_back(static_cast(J::num(d, "index", static_cast(rows.size()))), + e ? J::floats(*e) : std::vector{}); + } + std::stable_sort(rows.begin(), rows.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); + for (auto& r : rows) out.vectors.push_back(std::move(r.second)); + if (const Json* u = J::at(doc, "usage")) out.total_tokens = static_cast(J::num(*u, "total_tokens")); + return out; +} + +RerankResult Mixedbread::parse_rerank(const Json& doc) { + RerankResult out; + out.model = J::str(doc, "model"); + for (const Json& d : J::array(doc, "data")) { + RerankHit h; + h.index = static_cast(J::num(d, "index")); + h.score = J::num(d, "score"); + out.hits.push_back(h); + } + std::stable_sort(out.hits.begin(), out.hits.end(), [](const RerankHit& a, const RerankHit& b) { return a.score > b.score; }); + if (const Json* u = J::at(doc, "usage")) out.total_tokens = static_cast(J::num(*u, "total_tokens")); + return out; +} + +EmbedResult Mixedbread::embed(const EmbedRequest& request) { + Embedder::validate("mixedbread", request); + if (request.texts.size() > kMaxEmbedInputs) bad("at most 256 texts per request"); + Json body = Json::object(); + body.set("model", Json(request.model.empty() ? std::string(kDefaultEmbedModel) : request.model)); + body.set("input", J::strings(request.texts)); + if (request.dimensions > 0) body.set("dimensions", Json(request.dimensions)); + if (request.input_type == "query") body.set("prompt", Json("Represent this sentence for searching relevant passages:")); + EmbedResult r = parse_embeddings(gw_->call("create_embeddings", with_body(body))); + if (r.vectors.size() != request.texts.size()) + throw llm::ApiError("mixedbread", 502, "BAD_RESPONSE", "embedding count differs from input count"); + return r; +} + +RerankResult Mixedbread::rerank(const RerankRequest& request) { + Reranker::validate("mixedbread", request); + Json body = Json::object(); + body.set("model", Json(request.model.empty() ? std::string(kDefaultRerankModel) : request.model)); + body.set("query", Json(request.query)); + body.set("input", J::strings(request.documents)); + body.set("return_input", Json(false)); + if (request.top_n > 0) body.set("top_k", Json(request.top_n)); + return parse_rerank(gw_->call("rerank", with_body(body))); +} + +Json Mixedbread::list_stores() { return gw_->call("list_stores"); } + +Json Mixedbread::create_store(const std::string& name, const std::string& description) { + if (name.empty() || name.size() > 128) bad("store name must be 1..128 chars"); + Json body = Json::object(); + body.set("name", Json(name)); + if (!description.empty()) body.set("description", Json(description)); + return gw_->call("create_store", with_body(body)); +} + +Json Mixedbread::retrieve_store(const std::string& store) { return gw_->call("retrieve_store", store_arg(store)); } +Json Mixedbread::delete_store(const std::string& store) { return gw_->destroy("delete_store", store_arg(store)); } + +Json Mixedbread::search_stores(const std::vector& stores, const std::string& query, int top_k) { + check_stores(stores); + if (query.empty() || query.size() > RetrievalCaps::kMaxQueryChars) bad("query must be 1..8192 chars"); + if (top_k < 1 || top_k > 100) bad("top_k must be 1..100"); + Json body = Json::object(); + body.set("query", Json(query)); + body.set("store_identifiers", J::strings(stores)); + body.set("top_k", Json(top_k)); + return gw_->call("search_store_chunks", with_body(body)); +} + +Json Mixedbread::answer(const std::vector& stores, const std::string& query) { + check_stores(stores); + if (query.empty() || query.size() > RetrievalCaps::kMaxQueryChars) bad("query must be 1..8192 chars"); + Json body = Json::object(); + body.set("query", Json(query)); + body.set("store_identifiers", J::strings(stores)); + return gw_->call("create_question_answering", with_body(body)); +} + +Json Mixedbread::add_file_to_store(const std::string& store, const std::string& file_id) { + if (!llm::Service::valid_id(file_id)) bad("bad file id"); + CallArgs a = store_arg(store); + Json body = Json::object(); + body.set("file_id", Json(file_id)); + a.body = body; + return gw_->call("create_store_file", a); +} + +Json Mixedbread::delete_store_file(const std::string& store, const std::string& file) { + CallArgs a = store_arg(store); + a.path["file_identifier"] = file; + return gw_->destroy("delete_store_file", a); +} + +Json Mixedbread::list_files() { return gw_->call("list_files"); } + +Json Mixedbread::retrieve_file(const std::string& file_id) { + CallArgs a; + a.path["file_id"] = file_id; + return gw_->call("retrieve_file", a); +} + +Json Mixedbread::delete_file(const std::string& file_id) { + CallArgs a; + a.path["file_id"] = file_id; + return gw_->destroy("delete_file", a); +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/mixedbread/mixedbread.h b/src/vendors/retrieval/mixedbread/mixedbread.h new file mode 100644 index 0000000..25b7b83 --- /dev/null +++ b/src/vendors/retrieval/mixedbread/mixedbread.h @@ -0,0 +1,53 @@ +#pragma once + +// Mixedbread: embeddings, reranking, stores (hosted vector stores with +// files, chunk search and question answering), files, parsing and extraction. +// Implements Embedder and Reranker. Organisation administration, integrations +// and OAuth callbacks are not wrapped (see EXCLUDED in tools/check_spec.py). + +#include +#include + +#include "vendors/retrieval/provider.h" +#include "vendors/retrieval/seams.h" + +namespace molo::retrieval { + +const CallTable& mixedbread_calls(); + +class Mixedbread : public Provider, public Embedder, public Reranker { + public: + static constexpr const char* kKeyEnv = "MXBAI_API_KEY"; + static constexpr const char* kBaseUrl = "https://api.mixedbread.com"; + static constexpr const char* kDefaultEmbedModel = "mixedbread-ai/mxbai-embed-large-v1"; + static constexpr const char* kDefaultRerankModel = "mixedbread-ai/mxbai-rerank-v3.1-listwise"; + static constexpr std::size_t kMaxEmbedInputs = 256; // spec: input maxItems 256 + + static llm::ApiClientConfig config(); + static Mixedbread from_env(); + static Mixedbread with(const TransportFactory& factory, std::string key); + explicit Mixedbread(std::shared_ptr gateway) : Provider(std::move(gateway)) {} + + std::string provider() const override { return "mixedbread"; } + EmbedResult embed(const EmbedRequest& request) override; + RerankResult rerank(const RerankRequest& request) override; + + // Stores + Json list_stores(); + Json create_store(const std::string& name, const std::string& description); + Json retrieve_store(const std::string& store); + Json delete_store(const std::string& store); // destructive + Json search_stores(const std::vector& stores, const std::string& query, int top_k); + Json answer(const std::vector& stores, const std::string& query); + Json add_file_to_store(const std::string& store, const std::string& file_id); + Json delete_store_file(const std::string& store, const std::string& file); // destructive + // Files + Json list_files(); + Json retrieve_file(const std::string& file_id); + Json delete_file(const std::string& file_id); // destructive + + static EmbedResult parse_embeddings(const Json& doc); + static RerankResult parse_rerank(const Json& doc); +}; + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/pinecone/calls.cpp b/src/vendors/retrieval/pinecone/calls.cpp new file mode 100644 index 0000000..af695c0 --- /dev/null +++ b/src/vendors/retrieval/pinecone/calls.cpp @@ -0,0 +1,75 @@ +#include "vendors/retrieval/pinecone/pinecone.h" + +namespace molo::retrieval { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// name, host, http method, path, spec operation id (or reference page / "doc:"), +// body fields, query parameters, flags (d destructive, w writes, r read-only POST, +// p paid), response schema) +static const CallRow kRows[] = { + {"list_indexes", "control", "GET", "/indexes", "list_indexes", "", "", "", "IndexList"}, + {"create_index", "control", "POST", "/indexes", "create_index", "cmek_id,deletion_protection,deployment,name,read_capacity,schema,tags", "", "w", "IndexModel"}, + {"describe_index", "control", "GET", "/indexes/{index_name}", "describe_index", "", "", "", "IndexModel"}, + {"delete_index", "control", "DELETE", "/indexes/{index_name}", "delete_index", "", "", "d", "-"}, + {"configure_index", "control", "PATCH", "/indexes/{index_name}", "configure_index", "deletion_protection,deployment,read_capacity,schema,tags", "", "w", "IndexModel"}, + {"list_index_backups", "control", "GET", "/indexes/{index_name}/backups", "list_index_backups", "", "include_deleted,limit,paginationToken", "", "BackupList"}, + {"create_backup", "control", "POST", "/indexes/{index_name}/backups", "create_backup", "description,name", "", "w", "BackupModel"}, + {"list_backup_schedules", "control", "GET", "/indexes/{index_name}/backup-schedules", "list_backup_schedules", "", "limit,paginationToken", "", "BackupScheduleList"}, + {"create_backup_schedule", "control", "POST", "/indexes/{index_name}/backup-schedules", "create_backup_schedule", "name,retention,schedule", "", "w", "BackupScheduleResponse"}, + {"create_index_for_model", "control", "POST", "/indexes/create-for-model", "create_index_for_model", "cloud,deletion_protection,embed,name,read_capacity,region,schema,tags", "", "w", "IndexModel"}, + {"list_project_backups", "control", "GET", "/backups", "list_project_backups", "", "limit,paginationToken", "", "BackupList"}, + {"describe_backup", "control", "GET", "/backups/{backup_id}", "describe_backup", "", "", "", "BackupModel"}, + {"delete_backup", "control", "DELETE", "/backups/{backup_id}", "delete_backup", "", "", "d", "-"}, + {"create_index_from_backup", "control", "POST", "/backups/{backup_id}/create-index", "create_index_from_backup_operation", "deletion_protection,name,read_capacity,tags", "", "w", "CreateIndexFromBackupResponse"}, + {"describe_backup_schedule", "control", "GET", "/backup-schedules/{schedule_id}", "describe_backup_schedule", "", "", "", "BackupScheduleResponse"}, + {"delete_backup_schedule", "control", "DELETE", "/backup-schedules/{schedule_id}", "delete_backup_schedule", "", "", "d", "-"}, + {"update_backup_schedule", "control", "PATCH", "/backup-schedules/{schedule_id}", "update_backup_schedule", "enabled,frequency,retention", "", "w", "BackupScheduleResponse"}, + {"list_backup_schedule_history", "control", "GET", "/backup-schedules/{schedule_id}/history", "list_backup_schedule_history", "", "limit,paginationToken", "", "BackupScheduleHistoryList"}, + {"list_restore_jobs", "control", "GET", "/restore-jobs", "list_restore_jobs", "", "limit,paginationToken", "", "RestoreJobList"}, + {"describe_restore_job", "control", "GET", "/restore-jobs/{job_id}", "describe_restore_job", "", "", "", "RestoreJobModel"}, + {"list_collections", "control", "GET", "/collections", "list_collections", "", "", "", "CollectionList"}, + {"create_collection", "control", "POST", "/collections", "create_collection", "name,source", "", "w", "CollectionModel"}, + {"describe_collection", "control", "GET", "/collections/{collection_name}", "describe_collection", "", "", "", "CollectionModel"}, + {"delete_collection", "control", "DELETE", "/collections/{collection_name}", "delete_collection", "", "", "d", "-"}, + {"list_bulk_imports", "data", "GET", "/bulk/imports", "listBulkImports", "", "limit,paginationToken", "", "ListImportsResponse"}, + {"start_bulk_import", "data", "POST", "/bulk/imports", "startBulkImport", "errorMode,integrationId,uri", "", "w", "StartImportResponse"}, + {"describe_bulk_import", "data", "GET", "/bulk/imports/{id}", "describeBulkImport", "", "", "", "ImportModel"}, + {"cancel_bulk_import", "data", "DELETE", "/bulk/imports/{id}", "cancelBulkImport", "", "", "d", "CancelImportResponse"}, + {"describe_index_stats", "data", "POST", "/describe_index_stats", "describeIndexStats", "filter", "", "r", "IndexDescription"}, + {"query_vectors", "data", "POST", "/query", "queryVectors", "filter,id,includeMetadata,includeValues,maxCandidates,namespace,scanFactor,sparseVector,topK,vector", "", "r", "QueryResponse"}, + {"delete_vectors", "data", "POST", "/vectors/delete", "deleteVectors", "deleteAll,filter,ids,namespace", "", "d", "DeleteResponse"}, + {"fetch_vectors", "data", "GET", "/vectors/fetch", "fetchVectors", "", "ids,namespace", "", "FetchResponse"}, + {"fetch_vectors_by_metadata", "data", "POST", "/vectors/fetch_by_metadata", "fetch_vectors_by_metadata", "filter,limit,namespace,paginationToken", "", "r", "FetchByMetadataResponse"}, + {"list_vectors", "data", "GET", "/vectors/list", "listVectors", "", "limit,namespace,paginationToken,prefix", "", "ListResponse"}, + {"update_vector", "data", "POST", "/vectors/update", "updateVector", "dryRun,filter,id,namespace,setMetadata,sparseValues,values", "", "w", "UpdateResponse"}, + {"upsert_vectors", "data", "POST", "/vectors/upsert", "upsertVectors", "namespace,vectors", "", "w", "UpsertResponse"}, + {"list_namespaces", "data", "GET", "/namespaces", "listNamespacesOperation", "", "limit,paginationToken,prefix", "", "ListNamespacesResponse"}, + {"create_namespace", "data", "POST", "/namespaces", "createNamespace", "name,schema", "", "w", "NamespaceDescription"}, + {"describe_namespace", "data", "GET", "/namespaces/{namespace}", "describeNamespace", "", "", "", "NamespaceDescription"}, + {"delete_namespace", "data", "DELETE", "/namespaces/{namespace}", "deleteNamespace", "", "", "d", "DeleteResponse"}, + {"list_namespace_aliases", "data", "GET", "/namespace-aliases", "listNamespaceAliases", "", "limit,paginationToken,prefix,target_namespace", "", "ListNamespaceAliasesResponse"}, + {"create_namespace_alias", "data", "POST", "/namespace-aliases", "createNamespaceAlias", "name,target_namespace", "", "w", "NamespaceAliasDescription"}, + {"describe_namespace_alias", "data", "GET", "/namespace-aliases/{alias_name}", "describeNamespaceAlias", "", "", "", "NamespaceAliasDescription"}, + {"delete_namespace_alias", "data", "DELETE", "/namespace-aliases/{alias_name}", "deleteNamespaceAlias", "", "", "d", "DeleteResponse"}, + {"repoint_namespace_alias", "data", "PATCH", "/namespace-aliases/{alias_name}", "repointNamespaceAlias", "target_namespace", "", "w", "NamespaceAliasDescription"}, + {"upsert_records_namespace", "data", "POST", "/records/namespaces/{namespace}/upsert", "upsertRecordsNamespace", "*", "", "wp", "-"}, + {"search_records_namespace", "data", "POST", "/records/namespaces/{namespace}/search", "searchRecordsNamespace", "fields,query,rerank", "", "rp", "SearchRecordsResponse"}, + {"upsert_documents", "data", "POST", "/namespaces/{namespace}/documents/upsert", "upsertDocuments", "documents", "", "w", "UpsertDocumentsResponse"}, + {"search_documents", "data", "POST", "/namespaces/{namespace}/documents/search", "searchDocuments", "filter,include_fields,score_by,top_k", "", "r", "SearchDocumentsResponse"}, + {"fetch_documents", "data", "POST", "/namespaces/{namespace}/documents/fetch", "fetchDocuments", "filter,ids,include_fields,limit,pagination_token", "", "r", "FetchDocumentsResponse"}, + {"delete_documents", "data", "POST", "/namespaces/{namespace}/documents/delete", "deleteDocuments", "delete_all,filter,ids", "", "d", "DeleteDocumentsResponse"}, + {"update_documents", "data", "POST", "/namespaces/{namespace}/documents/update", "updateDocuments", "documents,filter,remove_fields,set_fields", "", "w", "UpdateDocumentsResponse"}, + {"list_documents", "data", "POST", "/namespaces/{namespace}/documents/list", "listDocuments", "limit,pagination_token,prefix", "", "r", "ListDocumentsResponse"}, + {"embed", "inference", "POST", "/embed", "embed", "inputs,model,parameters", "", "rp", "EmbeddingsList"}, + {"rerank", "inference", "POST", "/rerank", "rerank", "documents,model,parameters,query,rank_fields,return_documents,top_n", "", "rp", "RerankResult"}, + {"list_models", "inference", "GET", "/models", "list_models", "", "type,vector_type", "", "ModelInfoList"}, + {"get_model", "inference", "GET", "/models/{model_name}", "get_model", "", "", "", "ModelInfo"}, +}; +// CALLS-END + +const CallTable& pinecone_calls() { + static const CallTable table{kRows, sizeof(kRows) / sizeof(kRows[0])}; + return table; +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/pinecone/pinecone.cpp b/src/vendors/retrieval/pinecone/pinecone.cpp new file mode 100644 index 0000000..19c7a69 --- /dev/null +++ b/src/vendors/retrieval/pinecone/pinecone.cpp @@ -0,0 +1,312 @@ +#include "vendors/retrieval/pinecone/pinecone.h" + +#include "vendors/retrieval/caps.h" + +namespace molo::retrieval { + +llm::ApiClientConfig Pinecone::config() { + llm::ApiClientConfig c; + c.vendor = "pinecone"; + c.base_url = kControlUrl; + c.key_env = kKeyEnv; + c.auth = llm::AuthStyle::kHeader; + c.auth_header = "Api-Key"; + c.default_headers["X-Pinecone-Api-Version"] = kApiVersion; + return c; +} + +namespace { +std::vector hosts_for(const std::string& index_host) { + std::vector h = {{"control", Pinecone::kControlUrl}, {"inference", Pinecone::kControlUrl}}; + if (!index_host.empty()) { + check_host("pinecone", index_host); + h.push_back({"data", "https://" + index_host}); + } + return h; +} +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("pinecone", why); } +CallArgs with_body(Json body) { + CallArgs a; + a.body = std::move(body); + return a; +} +CallArgs with_path(const char* key, const std::string& value) { + CallArgs a; + a.path[key] = value; + return a; +} +void put_ns(Json& body, const std::string& ns) { + if (!ns.empty()) body.set("namespace", Json(ns)); +} +} // namespace + +Pinecone Pinecone::from_env(const std::string& index_host) { + return Pinecone(make_gateway(pinecone_calls(), config(), hosts_for(index_host), "", nullptr)); +} + +Pinecone Pinecone::with(const TransportFactory& factory, std::string key, const std::string& index_host) { + return Pinecone(make_gateway(pinecone_calls(), config(), hosts_for(index_host), std::move(key), factory)); +} + +// ---- parsers --------------------------------------------------------------- + +EmbedResult Pinecone::parse_embeddings(const Json& doc) { + EmbedResult out; + out.model = J::str(doc, "model"); + for (const Json& d : J::array(doc, "data")) { + const Json* v = J::at(d, "values"); + out.vectors.push_back(v ? J::floats(*v) : std::vector{}); + } + if (const Json* u = J::at(doc, "usage")) out.total_tokens = static_cast(J::num(*u, "total_tokens")); + return out; +} + +RerankResult Pinecone::parse_rerank(const Json& doc) { + RerankResult out; + out.model = J::str(doc, "model"); + for (const Json& d : J::array(doc, "data")) { + RerankHit h; + h.index = static_cast(J::num(d, "index")); + h.score = J::num(d, "score"); + out.hits.push_back(h); + } + return out; +} + +std::vector Pinecone::parse_matches(const Json& doc) { + std::vector out; + for (const Json& m : J::array(doc, "matches")) { + PineconeMatch x; + x.id = J::str(m, "id"); + x.score = J::num(m, "score"); + if (const Json* v = J::at(m, "values")) x.values = J::floats(*v); + if (const Json* md = J::at(m, "metadata")) x.metadata = *md; + out.push_back(std::move(x)); + } + return out; +} + +PineconePage Pinecone::parse_list(const Json& doc) { + PineconePage p; + for (const Json& v : J::array(doc, "vectors")) p.ids.push_back(J::str(v, "id")); + if (const Json* pg = J::at(doc, "pagination")) p.next_token = J::str(*pg, "next"); + return p; +} + +Json Pinecone::vector_json(const PineconeVector& v) { + if (v.id.empty() || v.id.size() > 512) bad("vector id must be 1..512 chars"); + if (v.values.empty() || v.values.size() > static_cast(RetrievalCaps::kMaxDimensions)) + bad("vector values must hold 1..20000 numbers"); + Json o = Json::object(); + o.set("id", Json(v.id)); + Json vals = Json::array(); + for (float f : v.values) vals.push(Json(static_cast(f))); + o.set("values", vals); + if (!v.metadata.is_null()) { + if (!v.metadata.is_object()) bad("metadata must be an object"); + o.set("metadata", v.metadata); + } + return o; +} + +// ---- inference ------------------------------------------------------------- + +EmbedResult Pinecone::embed(const EmbedRequest& request) { + Embedder::validate("pinecone", request); + Json body = Json::object(); + body.set("model", Json(request.model.empty() ? std::string(kDefaultEmbedModel) : request.model)); + Json inputs = Json::array(); + for (const std::string& t : request.texts) { + Json i = Json::object(); + i.set("text", Json(t)); + inputs.push(i); + } + body.set("inputs", inputs); + Json params = Json::object(); + if (!request.input_type.empty()) params.set("input_type", Json(request.input_type == "query" ? "query" : "passage")); + params.set("truncate", Json("END")); + body.set("parameters", params); + EmbedResult r = parse_embeddings(gw_->call("embed", with_body(body))); + if (r.vectors.size() != request.texts.size()) + throw llm::ApiError("pinecone", 502, "BAD_RESPONSE", "embedding count differs from input count"); + return r; +} + +RerankResult Pinecone::rerank(const RerankRequest& request) { + Reranker::validate("pinecone", request); + Json body = Json::object(); + body.set("model", Json(request.model.empty() ? std::string(kDefaultRerankModel) : request.model)); + body.set("query", Json(request.query)); + Json docs = Json::array(); + for (const std::string& d : request.documents) { + Json i = Json::object(); + i.set("text", Json(d)); + docs.push(i); + } + body.set("documents", docs); + body.set("return_documents", Json(false)); + if (request.top_n > 0) body.set("top_n", Json(request.top_n)); + return parse_rerank(gw_->call("rerank", with_body(body))); +} + +// ---- control plane --------------------------------------------------------- + +Json Pinecone::list_indexes() { return gw_->call("list_indexes"); } +Json Pinecone::describe_index(const std::string& name) { return gw_->call("describe_index", with_path("index_name", name)); } + +Json Pinecone::create_index(const std::string& name, int dimension, const std::string& metric, + const std::string& cloud, const std::string& region) { + if (name.empty() || name.size() > 45) bad("index name must be 1..45 chars"); + if (dimension < 1 || dimension > RetrievalCaps::kMaxDimensions) bad("dimension must be 1..20000"); + if (metric != "cosine" && metric != "dotproduct" && metric != "euclidean") bad("metric must be cosine, dotproduct or euclidean"); + if (cloud.empty() || region.empty()) bad("cloud and region are required"); + Json field = Json::object(); + field.set("type", Json("dense_vector")); + field.set("dimension", Json(dimension)); + field.set("metric", Json(metric)); + Json fields = Json::object(); + fields.set("vector", field); + Json schema = Json::object(); + schema.set("fields", fields); + Json deployment = Json::object(); + deployment.set("deployment_type", Json("managed")); + deployment.set("cloud", Json(cloud)); + deployment.set("region", Json(region)); + Json body = Json::object(); + body.set("name", Json(name)); + body.set("schema", schema); + body.set("deployment", deployment); + return gw_->call("create_index", with_body(body)); +} + +Json Pinecone::configure_index(const std::string& name, const Json& body) { + if (!body.is_object()) bad("body must be an object"); + CallArgs a = with_path("index_name", name); + a.body = body; + return gw_->call("configure_index", a); +} + +Json Pinecone::delete_index(const std::string& name) { return gw_->destroy("delete_index", with_path("index_name", name)); } +Json Pinecone::list_collections() { return gw_->call("list_collections"); } +Json Pinecone::describe_collection(const std::string& name) { + return gw_->call("describe_collection", with_path("collection_name", name)); +} +Json Pinecone::create_collection(const std::string& name, const std::string& source_index) { + if (name.empty() || source_index.empty()) bad("name and source are required"); + Json body = Json::object(); + body.set("name", Json(name)); + body.set("source", Json(source_index)); + return gw_->call("create_collection", with_body(body)); +} +Json Pinecone::delete_collection(const std::string& name) { + return gw_->destroy("delete_collection", with_path("collection_name", name)); +} +Json Pinecone::list_backups(const std::string& index_name) { + return gw_->call("list_index_backups", with_path("index_name", index_name)); +} +Json Pinecone::create_backup(const std::string& index_name, const std::string& name, const std::string& description) { + Json body = Json::object(); + if (!name.empty()) body.set("name", Json(name)); + if (!description.empty()) body.set("description", Json(description)); + CallArgs a = with_path("index_name", index_name); + a.body = body; + return gw_->call("create_backup", a); +} +Json Pinecone::describe_backup(const std::string& backup_id) { return gw_->call("describe_backup", with_path("backup_id", backup_id)); } +Json Pinecone::delete_backup(const std::string& backup_id) { return gw_->destroy("delete_backup", with_path("backup_id", backup_id)); } + +// ---- data plane ------------------------------------------------------------ + +Json Pinecone::upsert(const std::vector& vectors, const std::string& ns) { + if (vectors.empty() || vectors.size() > RetrievalCaps::kMaxRecordsPerUpsert) bad("vectors must hold 1..1000 entries"); + Json arr = Json::array(); + for (const PineconeVector& v : vectors) arr.push(vector_json(v)); + Json body = Json::object(); + body.set("vectors", arr); + put_ns(body, ns); + return gw_->call("upsert_vectors", with_body(body)); +} + +std::vector Pinecone::query(const PineconeQuery& q) { + if (q.top_k < 1 || q.top_k > RetrievalCaps::kMaxTopK) bad("top_k must be 1..10000"); + if (q.vector.empty() == q.id.empty()) bad("give exactly one of vector and id"); + Json body = Json::object(); + body.set("topK", Json(q.top_k)); + if (!q.vector.empty()) { + if (q.vector.size() > static_cast(RetrievalCaps::kMaxDimensions)) bad("vector too long"); + Json v = Json::array(); + for (float f : q.vector) v.push(Json(static_cast(f))); + body.set("vector", v); + } else { + body.set("id", Json(q.id)); + } + put_ns(body, q.ns); + if (!q.filter.is_null()) { + if (!q.filter.is_object()) bad("filter must be an object"); + body.set("filter", q.filter); + } + body.set("includeValues", Json(q.include_values)); + body.set("includeMetadata", Json(q.include_metadata)); + return parse_matches(gw_->call("query_vectors", with_body(body))); +} + +Json Pinecone::fetch(const std::vector& ids, const std::string& ns) { + if (ids.empty() || ids.size() > kFetchIdsPerCall) bad("ids must hold 1..100 entries per call"); + CallArgs a; + for (const std::string& id : ids) a.query_multi.emplace_back("ids", id); + if (!ns.empty()) a.query["namespace"] = ns; + return gw_->call("fetch_vectors", a); +} + +PineconePage Pinecone::list(const std::string& prefix, const std::string& ns, int limit, const std::string& token) { + if (limit < 1 || limit > RetrievalCaps::kMaxListLimit) bad("limit must be 1..100"); + CallArgs a; + a.query["limit"] = Query::number(limit); + if (!prefix.empty()) a.query["prefix"] = prefix; + if (!ns.empty()) a.query["namespace"] = ns; + if (!token.empty()) a.query["paginationToken"] = token; + return parse_list(gw_->call("list_vectors", a)); +} + +Json Pinecone::stats(const Json& filter) { + Json body = Json::object(); + if (!filter.is_null()) body.set("filter", filter); + return gw_->call("describe_index_stats", with_body(body)); +} + +Json Pinecone::list_namespaces(int limit, const std::string& token) { + if (limit < 1 || limit > RetrievalCaps::kMaxListLimit) bad("limit must be 1..100"); + CallArgs a; + a.query["limit"] = Query::number(limit); + if (!token.empty()) a.query["paginationToken"] = token; + return gw_->call("list_namespaces", a); +} + +Json Pinecone::describe_namespace(const std::string& ns) { return gw_->call("describe_namespace", with_path("namespace", ns)); } + +Json Pinecone::delete_ids(const std::vector& ids, const std::string& ns) { + if (ids.empty() || ids.size() > RetrievalCaps::kMaxIdsPerFetch) bad("ids must hold 1..1000 entries"); + Json body = Json::object(); + body.set("ids", J::strings(ids)); + put_ns(body, ns); + return gw_->destroy("delete_vectors", with_body(body)); +} + +Json Pinecone::delete_all(const std::string& ns) { + Json body = Json::object(); + body.set("deleteAll", Json(true)); + put_ns(body, ns); + return gw_->destroy("delete_vectors", with_body(body)); +} + +Json Pinecone::delete_by_filter(const Json& filter, const std::string& ns) { + if (!filter.is_object() || filter.as_object().empty()) bad("filter must be a non-empty object"); + Json body = Json::object(); + body.set("filter", filter); + put_ns(body, ns); + return gw_->destroy("delete_vectors", with_body(body)); +} + +Json Pinecone::delete_namespace(const std::string& ns) { return gw_->destroy("delete_namespace", with_path("namespace", ns)); } + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/pinecone/pinecone.h b/src/vendors/retrieval/pinecone/pinecone.h new file mode 100644 index 0000000..e91b715 --- /dev/null +++ b/src/vendors/retrieval/pinecone/pinecone.h @@ -0,0 +1,107 @@ +#pragma once + +// Pinecone: control plane (indexes, collections, backups), data plane +// (upsert, query, fetch, list, delete, namespaces, stats) and inference +// (embed, rerank). API version 2026-07. The data plane lives on the host of an +// index, so a Pinecone object is built with that host (empty = control plane +// and inference only). Implements Embedder and Reranker through the hosted +// inference models. + +#include +#include + +#include "vendors/retrieval/provider.h" +#include "vendors/retrieval/seams.h" + +namespace molo::retrieval { + +const CallTable& pinecone_calls(); + +struct PineconeVector { + std::string id; + std::vector values; + Json metadata; // null = none +}; + +struct PineconeMatch { + std::string id; + double score = 0; + std::vector values; + Json metadata; +}; + +struct PineconeQuery { + std::vector vector; // or `id` + std::string id; + int top_k = 10; + std::string ns; // namespace; empty = the default namespace + Json filter; // null = none + bool include_values = false; + bool include_metadata = true; +}; + +struct PineconePage { + std::vector ids; + std::string next_token; // empty = last page + bool truncated = false; // the page walk hit the cap +}; + +class Pinecone : public Provider, public Embedder, public Reranker { + public: + static constexpr const char* kKeyEnv = "PINECONE_API_KEY"; + static constexpr const char* kControlUrl = "https://api.pinecone.io"; + static constexpr const char* kApiVersion = "2026-07"; + static constexpr const char* kDefaultEmbedModel = "multilingual-e5-large"; + static constexpr const char* kDefaultRerankModel = "bge-reranker-v2-m3"; + static constexpr std::size_t kFetchIdsPerCall = 100; // ids travel in the URL + + static llm::ApiClientConfig config(); + // index_host: the `host` field of describe_index, e.g. "my-index-abc.svc.pinecone.io". + static Pinecone from_env(const std::string& index_host = ""); + static Pinecone with(const TransportFactory& factory, std::string key, const std::string& index_host = ""); + explicit Pinecone(std::shared_ptr gateway) : Provider(std::move(gateway)) {} + + std::string provider() const override { return "pinecone"; } + EmbedResult embed(const EmbedRequest& request) override; + RerankResult rerank(const RerankRequest& request) override; + + // Control plane + Json list_indexes(); + Json describe_index(const std::string& name); + // A dense-vector index on Pinecone managed infrastructure. + Json create_index(const std::string& name, int dimension, const std::string& metric, + const std::string& cloud, const std::string& region); + Json configure_index(const std::string& name, const Json& body); + // Destructive. + Json delete_index(const std::string& name); + Json list_collections(); + Json describe_collection(const std::string& name); + Json create_collection(const std::string& name, const std::string& source_index); + Json delete_collection(const std::string& name); // destructive + Json list_backups(const std::string& index_name); + Json create_backup(const std::string& index_name, const std::string& name, const std::string& description); + Json describe_backup(const std::string& backup_id); + Json delete_backup(const std::string& backup_id); // destructive + + // Data plane (needs the index host) + Json upsert(const std::vector& vectors, const std::string& ns); + std::vector query(const PineconeQuery& query); + Json fetch(const std::vector& ids, const std::string& ns); + PineconePage list(const std::string& prefix, const std::string& ns, int limit, const std::string& token); + Json stats(const Json& filter); + Json list_namespaces(int limit, const std::string& token); + Json describe_namespace(const std::string& ns); + // Destructive. + Json delete_ids(const std::vector& ids, const std::string& ns); + Json delete_all(const std::string& ns); + Json delete_by_filter(const Json& filter, const std::string& ns); + Json delete_namespace(const std::string& ns); + + static EmbedResult parse_embeddings(const Json& doc); + static RerankResult parse_rerank(const Json& doc); + static std::vector parse_matches(const Json& doc); + static PineconePage parse_list(const Json& doc); + static Json vector_json(const PineconeVector& v); +}; + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/provider.cpp b/src/vendors/retrieval/provider.cpp new file mode 100644 index 0000000..921c3a9 --- /dev/null +++ b/src/vendors/retrieval/provider.cpp @@ -0,0 +1,75 @@ +#include "vendors/retrieval/provider.h" + +#include "clients/transports/http_transport.h" +#include "llm/config.h" +#include "vendors/retrieval/caps.h" + +namespace molo::retrieval { + +std::shared_ptr make_gateway(const CallTable& table, const llm::ApiClientConfig& base, + const std::vector& hosts, std::string key, + const TransportFactory& factory) { + if (key.empty()) key = llm::Config::get(base.key_env.c_str()); + Gateway::Hosts clients; + for (const HostSpec& h : hosts) { + llm::ApiClientConfig cfg = base; + cfg.base_url = h.base_url; + std::unique_ptr transport = + factory ? factory() : std::make_unique(); + clients[h.host] = std::make_shared(std::move(transport), cfg, key); + } + return std::make_shared(base.vendor, table, std::move(clients)); +} + +void check_host(const std::string& vendor, const std::string& host) { + if (host.empty() || host.size() > RetrievalCaps::kMaxHostChars) + throw llm::ApiError::invalid(vendor, "host is empty or too long"); + for (unsigned char c : host) { + bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || c == '.'; + if (!ok) throw llm::ApiError::invalid(vendor, "host must be a bare DNS name (no scheme, port or path)"); + } + if (host.front() == '.' || host.front() == '-' || host.find("..") != std::string::npos) + throw llm::ApiError::invalid(vendor, "malformed host"); +} + +Json J::strings(const std::vector& v) { + Json a = Json::array(); + for (const std::string& s : v) a.push(Json(s)); + return a; +} + +std::string J::str(const Json& doc, const std::string& key, const std::string& fallback) { + const Json* v = doc.is_object() ? doc.find(key) : nullptr; + if (!v) return fallback; + if (v->is_string()) return v->as_string(); + return fallback; +} + +double J::num(const Json& doc, const std::string& key, double fallback) { + const Json* v = doc.is_object() ? doc.find(key) : nullptr; + return v && v->is_number() ? v->as_number() : fallback; +} + +bool J::has_array(const Json& doc, const std::string& key) { + const Json* v = doc.is_object() ? doc.find(key) : nullptr; + return v && v->is_array(); +} + +const Json* J::at(const Json& doc, const std::string& key) { return doc.is_object() ? doc.find(key) : nullptr; } + +std::vector J::floats(const Json& arr) { + std::vector out; + if (!arr.is_array()) return out; + out.reserve(arr.as_array().size()); + for (const Json& x : arr.as_array()) + if (x.is_number()) out.push_back(static_cast(x.as_number())); + return out; +} + +const Json::Array& J::array(const Json& doc, const char* key) { + static const Json::Array empty; + const Json* v = doc.is_object() ? doc.find(key) : nullptr; + return v && v->is_array() ? v->as_array() : empty; +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/provider.h b/src/vendors/retrieval/provider.h new file mode 100644 index 0000000..3b5b747 --- /dev/null +++ b/src/vendors/retrieval/provider.h @@ -0,0 +1,60 @@ +#pragma once + +// Shared plumbing of the provider classes: building a Gateway over one +// ApiClient per host, and small Json helpers so each provider file only holds +// what is specific to the provider. + +#include +#include +#include +#include + +#include "clients/transport.h" +#include "llm/api_client.h" +#include "vendors/retrieval/gateway.h" + +namespace molo::retrieval { + +struct HostSpec { + std::string host; // name used in CallRow::host + std::string base_url; // https://..., no trailing slash +}; + +using TransportFactory = std::function()>; + +// One ApiClient per host. `factory` null = real HTTPS. `key` empty = look up +// base.key_env through llm::Config (empty there too = not configured). +std::shared_ptr make_gateway(const CallTable& table, const llm::ApiClientConfig& base, + const std::vector& hosts, std::string key, + const TransportFactory& factory); + +// A host name from the data plane of a database must be a bare https host. +void check_host(const std::string& vendor, const std::string& host); + +class Provider { + public: + explicit Provider(std::shared_ptr gateway) : gw_(std::move(gateway)) {} + virtual ~Provider() = default; + Gateway& gateway() const { return *gw_; } + std::shared_ptr shared_gateway() const { return gw_; } + bool configured() const { return gw_->configured(); } + const CallTable& table() const { return gw_->table(); } + + protected: + std::shared_ptr gw_; +}; + +// Json helpers +struct J { + static Json strings(const std::vector& v); + static Json object() { return Json::object(); } + static std::string str(const Json& doc, const std::string& key, const std::string& fallback = ""); + static double num(const Json& doc, const std::string& key, double fallback = 0); + static bool has_array(const Json& doc, const std::string& key); + static const Json* at(const Json& doc, const std::string& key); + static std::vector floats(const Json& arr); + // The array at `key`, or an empty static array. + static const Json::Array& array(const Json& doc, const char* key); +}; + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/qdrant/calls.cpp b/src/vendors/retrieval/qdrant/calls.cpp new file mode 100644 index 0000000..b04f927 --- /dev/null +++ b/src/vendors/retrieval/qdrant/calls.cpp @@ -0,0 +1,67 @@ +#include "vendors/retrieval/qdrant/qdrant.h" + +namespace molo::retrieval { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// name, host, http method, path, spec operation id (or reference page / "doc:"), +// body fields, query parameters, flags (d destructive, w writes, r read-only POST, +// p paid), response schema) +static const CallRow kRows[] = { + {"create_shard_key", "main", "PUT", "/collections/{collection_name}/shards", "create_shard_key", "initial_state,placement,replication_factor,shard_key,shards_number", "timeout", "w", "inline"}, + {"list_shard_keys", "main", "GET", "/collections/{collection_name}/shards", "list_shard_keys", "", "", "", "inline"}, + {"cluster_status", "main", "GET", "/cluster", "cluster_status", "", "", "", "inline"}, + {"cluster_telemetry", "main", "GET", "/cluster/telemetry", "cluster_telemetry", "", "details_level,timeout", "", "inline"}, + {"get_quotas", "main", "GET", "/quotas", "get_quotas", "", "", "", "inline"}, + {"update_quotas", "main", "PUT", "/quotas", "update_quotas", "enabled,max_disk_usage_percent,max_resident_memory_percent,release_margin_percent", "wait", "w", "inline"}, + {"get_collections", "main", "GET", "/collections", "get_collections", "", "", "", "inline"}, + {"get_collection", "main", "GET", "/collections/{collection_name}", "get_collection", "", "", "", "inline"}, + {"create_collection", "main", "PUT", "/collections/{collection_name}", "create_collection", "hnsw_config,id_tracker,metadata,on_disk_payload,optimizers_config,payload,quantization_config,replication_factor,shard_number,sharding_method,sparse_vectors,strict_mode_config,vectors,wal_config,write_consistency_factor", "timeout", "w", "inline"}, + {"update_collection", "main", "PATCH", "/collections/{collection_name}", "update_collection", "hnsw_config,metadata,optimizers_config,params,quantization_config,sparse_vectors,strict_mode_config,vectors", "timeout", "w", "inline"}, + {"delete_collection", "main", "DELETE", "/collections/{collection_name}", "delete_collection", "", "timeout", "d", "inline"}, + {"update_aliases", "main", "POST", "/collections/aliases", "update_aliases", "actions", "timeout", "w", "inline"}, + {"create_field_index", "main", "PUT", "/collections/{collection_name}/index", "create_field_index", "field_name,field_schema", "ordering,timeout,wait", "w", "inline"}, + {"collection_exists", "main", "GET", "/collections/{collection_name}/exists", "collection_exists", "", "", "", "inline"}, + {"delete_field_index", "main", "DELETE", "/collections/{collection_name}/index/{field_name}", "delete_field_index", "", "ordering,timeout,wait", "d", "inline"}, + {"create_vector_name", "main", "PUT", "/collections/{collection_name}/vectors/{vector_name}", "create_vector_name", "dense,sparse", "ordering,timeout,wait", "w", "inline"}, + {"delete_vector_name", "main", "DELETE", "/collections/{collection_name}/vectors/{vector_name}", "delete_vector_name", "", "ordering,timeout,wait", "d", "inline"}, + {"get_optimizations", "main", "GET", "/collections/{collection_name}/optimizations", "get_optimizations", "", "completed_limit,with", "", "inline"}, + {"get_collection_aliases", "main", "GET", "/collections/{collection_name}/aliases", "get_collection_aliases", "", "", "", "inline"}, + {"get_collections_aliases", "main", "GET", "/aliases", "get_collections_aliases", "", "", "", "inline"}, + {"recover_from_uploaded_snapshot", "main", "POST", "/collections/{collection_name}/snapshots/upload", "recover_from_uploaded_snapshot", "snapshot", "checksum,priority,wait", "d", "inline"}, + {"recover_from_snapshot", "main", "PUT", "/collections/{collection_name}/snapshots/recover", "recover_from_snapshot", "api_key,checksum,location,priority", "wait", "d", "inline"}, + {"list_snapshots", "main", "GET", "/collections/{collection_name}/snapshots", "list_snapshots", "", "", "", "inline"}, + {"create_snapshot", "main", "POST", "/collections/{collection_name}/snapshots", "create_snapshot", "", "wait", "w", "inline"}, + {"delete_snapshot", "main", "DELETE", "/collections/{collection_name}/snapshots/{snapshot_name}", "delete_snapshot", "", "wait", "d", "inline"}, + {"get_snapshot", "main", "GET", "/collections/{collection_name}/snapshots/{snapshot_name}", "get_snapshot", "", "", "", "inline"}, + {"list_full_snapshots", "main", "GET", "/snapshots", "list_full_snapshots", "", "", "", "inline"}, + {"create_full_snapshot", "main", "POST", "/snapshots", "create_full_snapshot", "", "wait", "w", "inline"}, + {"delete_full_snapshot", "main", "DELETE", "/snapshots/{snapshot_name}", "delete_full_snapshot", "", "wait", "d", "inline"}, + {"get_full_snapshot", "main", "GET", "/snapshots/{snapshot_name}", "get_full_snapshot", "", "", "", "inline"}, + {"get_point", "main", "GET", "/collections/{collection_name}/points/{id}", "get_point", "", "consistency,timeout", "", "inline"}, + {"get_points", "main", "POST", "/collections/{collection_name}/points", "get_points", "ids,shard_key,with_payload,with_vector", "consistency,timeout", "r", "inline"}, + {"upsert_points", "main", "PUT", "/collections/{collection_name}/points", "upsert_points", "batch,points,shard_key,update_filter,update_mode", "ordering,timeout,wait", "w", "inline"}, + {"delete_points", "main", "POST", "/collections/{collection_name}/points/delete", "delete_points", "filter,points,shard_key", "ordering,timeout,wait", "d", "inline"}, + {"update_vectors", "main", "PUT", "/collections/{collection_name}/points/vectors", "update_vectors", "points,shard_key,update_filter", "ordering,timeout,wait", "w", "inline"}, + {"delete_vectors", "main", "POST", "/collections/{collection_name}/points/vectors/delete", "delete_vectors", "filter,points,shard_key,vector", "ordering,timeout,wait", "d", "inline"}, + {"set_payload", "main", "POST", "/collections/{collection_name}/points/payload", "set_payload", "filter,key,payload,points,shard_key", "ordering,timeout,wait", "w", "inline"}, + {"overwrite_payload", "main", "PUT", "/collections/{collection_name}/points/payload", "overwrite_payload", "filter,key,payload,points,shard_key", "ordering,timeout,wait", "w", "inline"}, + {"delete_payload", "main", "POST", "/collections/{collection_name}/points/payload/delete", "delete_payload", "filter,keys,points,shard_key", "ordering,timeout,wait", "d", "inline"}, + {"clear_payload", "main", "POST", "/collections/{collection_name}/points/payload/clear", "clear_payload", "filter,points,shard_key", "ordering,timeout,wait", "d", "inline"}, + {"batch_update", "main", "POST", "/collections/{collection_name}/points/batch", "batch_update", "operations", "ordering,timeout,wait", "w", "inline"}, + {"scroll_points", "main", "POST", "/collections/{collection_name}/points/scroll", "scroll_points", "filter,limit,offset,order_by,shard_key,with_payload,with_vector", "consistency,timeout", "r", "inline"}, + {"count_points", "main", "POST", "/collections/{collection_name}/points/count", "count_points", "exact,filter,shard_key", "consistency,timeout", "r", "inline"}, + {"facet", "main", "POST", "/collections/{collection_name}/facet", "facet", "exact,filter,key,limit,shard_key", "consistency,timeout", "r", "inline"}, + {"query_points", "main", "POST", "/collections/{collection_name}/points/query", "query_points", "filter,limit,lookup_from,offset,params,prefetch,query,score_threshold,shard_key,using,with_payload,with_vector", "consistency,timeout", "r", "inline"}, + {"query_batch_points", "main", "POST", "/collections/{collection_name}/points/query/batch", "query_batch_points", "searches", "consistency,timeout", "r", "inline"}, + {"query_points_groups", "main", "POST", "/collections/{collection_name}/points/query/groups", "query_points_groups", "filter,group_by,group_size,limit,lookup_from,params,prefetch,query,score_threshold,shard_key,using,with_lookup,with_payload,with_vector", "consistency,timeout", "r", "inline"}, + {"search_matrix_pairs", "main", "POST", "/collections/{collection_name}/points/search/matrix/pairs", "search_matrix_pairs", "filter,limit,sample,shard_key,using", "consistency,timeout", "r", "inline"}, + {"search_matrix_offsets", "main", "POST", "/collections/{collection_name}/points/search/matrix/offsets", "search_matrix_offsets", "filter,limit,sample,shard_key,using", "consistency,timeout", "r", "inline"}, +}; +// CALLS-END + +const CallTable& qdrant_calls() { + static const CallTable table{kRows, sizeof(kRows) / sizeof(kRows[0])}; + return table; +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/qdrant/qdrant.cpp b/src/vendors/retrieval/qdrant/qdrant.cpp new file mode 100644 index 0000000..4ac3d6c --- /dev/null +++ b/src/vendors/retrieval/qdrant/qdrant.cpp @@ -0,0 +1,195 @@ +#include "vendors/retrieval/qdrant/qdrant.h" + +#include "llm/config.h" +#include "vendors/retrieval/caps.h" + +namespace molo::retrieval { + +llm::ApiClientConfig Qdrant::config() { + llm::ApiClientConfig c; + c.vendor = "qdrant"; + c.base_url = ""; + c.key_env = kKeyEnv; + c.auth = llm::AuthStyle::kHeader; + c.auth_header = "api-key"; + return c; +} + +namespace { +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("qdrant", why); } + +std::string trim_url(std::string url) { + while (!url.empty() && url.back() == '/') url.pop_back(); + return url; +} + +Qdrant build(const std::string& url, std::string key, const TransportFactory& factory) { + std::string base = trim_url(url); + bool ok = base.rfind("https://", 0) == 0 || base.rfind("http://", 0) == 0; + auto gw = make_gateway(qdrant_calls(), Qdrant::config(), {{"main", ok ? base : "https://qdrant.invalid"}}, + std::move(key), factory); + if (!ok) gw->require_setting(Qdrant::kUrlEnv); + return Qdrant(gw); +} + +CallArgs coll(const std::string& name) { + CallArgs a; + a.path["collection_name"] = name; + return a; +} +} // namespace + +Qdrant Qdrant::from_env() { return build(llm::Config::get(kUrlEnv), "", nullptr); } +Qdrant Qdrant::with(const TransportFactory& factory, std::string key, const std::string& base_url) { + return build(base_url, std::move(key), factory); +} + +const Json& Qdrant::result_of(const Json& doc) { + static const Json empty = Json::object(); + const Json* r = doc.is_object() ? doc.find("result") : nullptr; + return r ? *r : empty; +} + +Json Qdrant::point_json(const QdrantPoint& p) { + bool id_ok = (p.id.is_number() && p.id.as_number() >= 0) || (p.id.is_string() && !p.id.as_string().empty() && p.id.as_string().size() <= 64); + if (!id_ok) bad("point id must be an unsigned integer or a UUID string"); + if (p.vector.empty() || p.vector.size() > static_cast(RetrievalCaps::kMaxDimensions)) bad("vector must hold 1..20000 numbers"); + Json o = Json::object(); + o.set("id", p.id); + Json v = Json::array(); + for (float f : p.vector) v.push(Json(static_cast(f))); + o.set("vector", v); + if (!p.payload.is_null()) { + if (!p.payload.is_object()) bad("payload must be an object"); + o.set("payload", p.payload); + } + return o; +} + +std::vector Qdrant::parse_hits(const Json& doc) { + std::vector out; + const Json& r = result_of(doc); + for (const Json& p : J::array(r, "points")) { + QdrantHit h; + if (const Json* id = J::at(p, "id")) h.id = *id; + h.score = J::num(p, "score"); + if (const Json* pl = J::at(p, "payload")) h.payload = *pl; + out.push_back(std::move(h)); + } + return out; +} + +Json Qdrant::list_collections() { return gw_->call("get_collections"); } + +bool Qdrant::collection_exists(const std::string& name) { + Json doc = gw_->call("collection_exists", coll(name)); + const Json* e = J::at(result_of(doc), "exists"); + return e && e->is_bool() && e->as_bool(); +} + +Json Qdrant::get_collection(const std::string& name) { return gw_->call("get_collection", coll(name)); } + +Json Qdrant::create_collection(const std::string& name, int size, const std::string& distance, bool on_disk_payload) { + if (size < 1 || size > RetrievalCaps::kMaxDimensions) bad("size must be 1..20000"); + if (distance != "Cosine" && distance != "Euclid" && distance != "Dot" && distance != "Manhattan") + bad("distance must be Cosine, Euclid, Dot or Manhattan"); + Json vectors = Json::object(); + vectors.set("size", Json(size)); + vectors.set("distance", Json(distance)); + Json body = Json::object(); + body.set("vectors", vectors); + body.set("on_disk_payload", Json(on_disk_payload)); + CallArgs a = coll(name); + a.body = body; + return gw_->call("create_collection", a); +} + +Json Qdrant::delete_collection(const std::string& name) { return gw_->destroy("delete_collection", coll(name)); } + +Json Qdrant::upsert(const std::string& collection, const std::vector& points, bool wait) { + if (points.empty() || points.size() > RetrievalCaps::kMaxPointsPerUpsert) bad("points must hold 1..1000 entries"); + Json arr = Json::array(); + for (const QdrantPoint& p : points) arr.push(point_json(p)); + Json body = Json::object(); + body.set("points", arr); + CallArgs a = coll(collection); + a.body = body; + a.query["wait"] = Query::flag(wait); + return gw_->call("upsert_points", a); +} + +std::vector Qdrant::query(const std::string& collection, const std::vector& vector, int limit, + const Json& filter, bool with_payload) { + if (vector.empty() || vector.size() > static_cast(RetrievalCaps::kMaxDimensions)) bad("vector must hold 1..20000 numbers"); + if (limit < 1 || limit > RetrievalCaps::kMaxPointLimit) bad("limit must be 1..10000"); + Json v = Json::array(); + for (float f : vector) v.push(Json(static_cast(f))); + Json body = Json::object(); + body.set("query", v); + body.set("limit", Json(limit)); + body.set("with_payload", Json(with_payload)); + if (!filter.is_null()) body.set("filter", filter); + return parse_hits(query_raw(collection, body)); +} + +Json Qdrant::query_raw(const std::string& collection, const Json& body) { + CallArgs a = coll(collection); + a.body = body; + return gw_->call("query_points", a); +} + +Json Qdrant::scroll(const std::string& collection, int limit, const Json& offset, const Json& filter) { + if (limit < 1 || limit > RetrievalCaps::kMaxPointLimit) bad("limit must be 1..10000"); + Json body = Json::object(); + body.set("limit", Json(limit)); + if (!offset.is_null()) body.set("offset", offset); + if (!filter.is_null()) body.set("filter", filter); + CallArgs a = coll(collection); + a.body = body; + return gw_->call("scroll_points", a); +} + +long Qdrant::count(const std::string& collection, const Json& filter, bool exact) { + Json body = Json::object(); + body.set("exact", Json(exact)); + if (!filter.is_null()) body.set("filter", filter); + CallArgs a = coll(collection); + a.body = body; + return static_cast(J::num(result_of(gw_->call("count_points", a)), "count")); +} + +Json Qdrant::delete_points(const std::string& collection, const std::vector& ids) { + if (ids.empty() || ids.size() > RetrievalCaps::kMaxPointsPerUpsert) bad("ids must hold 1..1000 entries"); + Json arr = Json::array(); + for (const Json& id : ids) { + if (!id.is_number() && !id.is_string()) bad("an id must be a number or a string"); + arr.push(id); + } + Json body = Json::object(); + body.set("points", arr); + CallArgs a = coll(collection); + a.body = body; + a.query["wait"] = "true"; + return gw_->destroy("delete_points", a); +} + +Json Qdrant::delete_points_by_filter(const std::string& collection, const Json& filter) { + if (!filter.is_object() || filter.as_object().empty()) bad("filter must be a non-empty object"); + Json body = Json::object(); + body.set("filter", filter); + CallArgs a = coll(collection); + a.body = body; + a.query["wait"] = "true"; + return gw_->destroy("delete_points", a); +} + +Json Qdrant::create_snapshot(const std::string& collection) { return gw_->call("create_snapshot", coll(collection)); } +Json Qdrant::list_snapshots(const std::string& collection) { return gw_->call("list_snapshots", coll(collection)); } +Json Qdrant::delete_snapshot(const std::string& collection, const std::string& snapshot) { + CallArgs a = coll(collection); + a.path["snapshot_name"] = snapshot; + return gw_->destroy("delete_snapshot", a); +} +Json Qdrant::cluster_status() { return gw_->call("cluster_status"); } + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/qdrant/qdrant.h b/src/vendors/retrieval/qdrant/qdrant.h new file mode 100644 index 0000000..a7329ed --- /dev/null +++ b/src/vendors/retrieval/qdrant/qdrant.h @@ -0,0 +1,66 @@ +#pragma once + +// Qdrant REST API: collections, points (upsert, query, scroll, count, delete), +// snapshots and cluster status. The server is yours (QDRANT_URL, for example +// https://xyz.cloud.qdrant.io:6333), so there is no vendor host. + +#include +#include + +#include "vendors/retrieval/provider.h" + +namespace molo::retrieval { + +const CallTable& qdrant_calls(); + +struct QdrantPoint { + Json id; // unsigned integer or UUID string + std::vector vector; + Json payload; // null = none +}; + +struct QdrantHit { + Json id; + double score = 0; + Json payload; +}; + +class Qdrant : public Provider { + public: + static constexpr const char* kKeyEnv = "QDRANT_API_KEY"; + static constexpr const char* kUrlEnv = "QDRANT_URL"; + + static llm::ApiClientConfig config(); + static Qdrant from_env(); + static Qdrant with(const TransportFactory& factory, std::string key, const std::string& base_url); + explicit Qdrant(std::shared_ptr gateway) : Provider(std::move(gateway)) {} + + // Collections + Json list_collections(); + bool collection_exists(const std::string& name); + Json get_collection(const std::string& name); + Json create_collection(const std::string& name, int size, const std::string& distance, bool on_disk_payload); + Json delete_collection(const std::string& name); // destructive + + // Points + Json upsert(const std::string& collection, const std::vector& points, bool wait); + std::vector query(const std::string& collection, const std::vector& vector, int limit, + const Json& filter, bool with_payload); + Json query_raw(const std::string& collection, const Json& body); + Json scroll(const std::string& collection, int limit, const Json& offset, const Json& filter); + long count(const std::string& collection, const Json& filter, bool exact); + Json delete_points(const std::string& collection, const std::vector& ids); // destructive + Json delete_points_by_filter(const std::string& collection, const Json& filter); // destructive + + // Snapshots and cluster + Json create_snapshot(const std::string& collection); + Json list_snapshots(const std::string& collection); + Json delete_snapshot(const std::string& collection, const std::string& snapshot); // destructive + Json cluster_status(); + + static std::vector parse_hits(const Json& doc); + static Json point_json(const QdrantPoint& p); + static const Json& result_of(const Json& doc); +}; + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/seams.cpp b/src/vendors/retrieval/seams.cpp new file mode 100644 index 0000000..7fb5e8c --- /dev/null +++ b/src/vendors/retrieval/seams.cpp @@ -0,0 +1,48 @@ +#include "vendors/retrieval/seams.h" + +#include "llm/error.h" +#include "vendors/retrieval/caps.h" + +namespace molo::retrieval { + +namespace { +[[noreturn]] void bad(const std::string& vendor, const std::string& why) { + throw llm::ApiError::invalid(vendor, why); +} +bool blank(const std::string& s) { return s.find_first_not_of(" \t\r\n") == std::string::npos; } +} // namespace + +void Embedder::validate(const std::string& vendor, const EmbedRequest& r) { + if (r.texts.empty()) bad(vendor, "texts is empty"); + if (r.texts.size() > RetrievalCaps::kMaxEmbedTexts) bad(vendor, "too many texts"); + for (const std::string& t : r.texts) { + if (blank(t)) bad(vendor, "a text is blank"); + if (t.size() > RetrievalCaps::kMaxTextBytes) bad(vendor, "a text is over the byte cap"); + } + if (!r.input_type.empty() && r.input_type != "query" && r.input_type != "document") + bad(vendor, "input_type must be query or document"); + if (r.dimensions < 0 || r.dimensions > RetrievalCaps::kMaxDimensions) bad(vendor, "dimensions out of range"); +} + +void Reranker::validate(const std::string& vendor, const RerankRequest& r) { + if (blank(r.query)) bad(vendor, "query is blank"); + if (r.query.size() > RetrievalCaps::kMaxQueryChars) bad(vendor, "query too long"); + if (r.documents.empty()) bad(vendor, "documents is empty"); + if (r.documents.size() > RetrievalCaps::kMaxRerankDocs) bad(vendor, "too many documents"); + for (const std::string& d : r.documents) { + if (blank(d)) bad(vendor, "a document is blank"); + if (d.size() > RetrievalCaps::kMaxTextBytes) bad(vendor, "a document is over the byte cap"); + } + if (r.top_n < 0 || r.top_n > RetrievalCaps::kMaxTopN) bad(vendor, "top_n out of range"); +} + +void WebSearch::validate(const std::string& vendor, const SearchRequest& r) { + if (blank(r.query)) bad(vendor, "query is blank"); + if (r.query.size() > RetrievalCaps::kMaxQueryChars) bad(vendor, "query too long"); + if (r.max_results < 1 || r.max_results > static_cast(RetrievalCaps::kMaxSearchResults)) + bad(vendor, "max_results out of range"); + if (!r.recency.empty() && r.recency != "day" && r.recency != "week" && r.recency != "month" && r.recency != "year") + bad(vendor, "recency must be day, week, month or year"); +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/seams.h b/src/vendors/retrieval/seams.h new file mode 100644 index 0000000..8bd2247 --- /dev/null +++ b/src/vendors/retrieval/seams.h @@ -0,0 +1,92 @@ +#pragma once + +// Neutral seams of the retrieval package. Provider adapters implement them; +// callers (RAG pipelines, the demo) depend on these only. src/llm is not +// touched: these types are retrieval's own. +// +// Embedder texts -> vectors (Voyage, Jina, Pinecone, Mixedbread) +// Reranker query + docs -> order (Voyage, Jina, Pinecone, Mixedbread) +// WebSearch query -> hits (Jina, Tavily, Exa, Brave) + +#include +#include + +namespace molo::retrieval { + +struct EmbedRequest { + std::vector texts; + std::string model; // empty = the adapter's default + std::string input_type; // "", "query" or "document" + int dimensions = 0; // 0 = the model's default +}; + +struct EmbedResult { + std::vector> vectors; // one per input text, in order + std::string model; + long total_tokens = 0; +}; + +class Embedder { + public: + virtual ~Embedder() = default; + virtual std::string provider() const = 0; + virtual EmbedResult embed(const EmbedRequest& request) = 0; + // Caps shared by all adapters; throws llm::ApiError::invalid. + static void validate(const std::string& vendor, const EmbedRequest& request); +}; + +struct RerankRequest { + std::string query; + std::vector documents; + std::string model; // empty = the adapter's default + int top_n = 0; // 0 = all documents +}; + +struct RerankHit { + int index = 0; // position in RerankRequest::documents + double score = 0; +}; + +struct RerankResult { + std::vector hits; // best first + std::string model; + long total_tokens = 0; +}; + +class Reranker { + public: + virtual ~Reranker() = default; + virtual std::string provider() const = 0; + virtual RerankResult rerank(const RerankRequest& request) = 0; + static void validate(const std::string& vendor, const RerankRequest& request); +}; + +struct SearchRequest { + std::string query; + int max_results = 5; + std::string recency; // "", "day", "week", "month", "year" (where supported) +}; + +struct SearchHit { + std::string title; + std::string url; + std::string snippet; + std::string published; // as the provider gave it, may be empty + double score = 0; + bool has_score = false; +}; + +struct SearchResult { + std::vector hits; + std::string answer; // only when the provider returned one +}; + +class WebSearch { + public: + virtual ~WebSearch() = default; + virtual std::string provider() const = 0; + virtual SearchResult search(const SearchRequest& request) = 0; + static void validate(const std::string& vendor, const SearchRequest& request); +}; + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/tavily/calls.cpp b/src/vendors/retrieval/tavily/calls.cpp new file mode 100644 index 0000000..266dfd4 --- /dev/null +++ b/src/vendors/retrieval/tavily/calls.cpp @@ -0,0 +1,25 @@ +#include "vendors/retrieval/tavily/tavily.h" + +namespace molo::retrieval { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// name, host, http method, path, spec operation id (or reference page / "doc:"), +// body fields, query parameters, flags (d destructive, w writes, r read-only POST, +// p paid), response schema) +static const CallRow kRows[] = { + {"search", "main", "POST", "/search", "search", "auto_parameters,chunks_per_source,country,end_date,exact_match,exclude_domains,filter_by_language,filter_by_published_date,include_answer,include_domains,include_domains_mode,include_favicon,include_image_descriptions,include_images,include_published_date,include_raw_content,include_usage,language,max_results,query,safe_search,search_depth,start_date,time_range,topic", "", "rp", "inline"}, + {"extract", "main", "POST", "/extract", "extract", "chunks_per_source,extract_depth,format,include_favicon,include_images,include_usage,query,timeout,urls", "", "rp", "inline"}, + {"crawl", "main", "POST", "/crawl", "crawl", "allow_external,chunks_per_source,exclude_domains,exclude_paths,extract_depth,format,include_favicon,include_images,include_usage,instructions,limit,max_breadth,max_depth,select_domains,select_paths,timeout,url", "", "rp", "inline"}, + {"map", "main", "POST", "/map", "map", "allow_external,exclude_domains,exclude_paths,include_usage,instructions,limit,max_breadth,max_depth,select_domains,select_paths,timeout,url", "", "rp", "inline"}, + {"create_research", "main", "POST", "/research", "createResearch", "citation_format,exclude_domains,files,include_domains,input,model,output_length,output_schema,stream", "", "rp", "inline"}, + {"get_research", "main", "GET", "/research/{request_id}", "getResearch", "", "include_usage", "", "-"}, + {"get_usage", "main", "GET", "/usage", "getUsage", "", "", "", "inline"}, +}; +// CALLS-END + +const CallTable& tavily_calls() { + static const CallTable table{kRows, sizeof(kRows) / sizeof(kRows[0])}; + return table; +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/tavily/tavily.cpp b/src/vendors/retrieval/tavily/tavily.cpp new file mode 100644 index 0000000..1a7a826 --- /dev/null +++ b/src/vendors/retrieval/tavily/tavily.cpp @@ -0,0 +1,142 @@ +#include "vendors/retrieval/tavily/tavily.h" + +#include "vendors/retrieval/caps.h" + +namespace molo::retrieval { + +llm::ApiClientConfig Tavily::config() { + llm::ApiClientConfig c; + c.vendor = "tavily"; + c.base_url = kBaseUrl; + c.key_env = kKeyEnv; + c.auth = llm::AuthStyle::kBearer; + return c; +} + +namespace { +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("tavily", why); } +CallArgs with_body(Json body) { + CallArgs a; + a.body = std::move(body); + return a; +} +} // namespace + +Tavily Tavily::from_env() { return Tavily(make_gateway(tavily_calls(), config(), {{"main", kBaseUrl}}, "", nullptr)); } +Tavily Tavily::with(const TransportFactory& factory, std::string key) { + return Tavily(make_gateway(tavily_calls(), config(), {{"main", kBaseUrl}}, std::move(key), factory)); +} + +void Tavily::check_url(const std::string& url) { + bool http = url.rfind("http://", 0) == 0 || url.rfind("https://", 0) == 0; + if (!http || url.size() > 8192 || url.find_first_of(" \t\r\n") != std::string::npos) + bad("url must be http(s) without spaces, at most 8192 chars"); +} + +SearchResult Tavily::parse_search(const Json& doc) { + SearchResult out; + out.answer = J::str(doc, "answer"); + for (const Json& r : J::array(doc, "results")) { + SearchHit h; + h.title = J::str(r, "title"); + h.url = J::str(r, "url"); + h.snippet = J::str(r, "content"); + h.published = J::str(r, "published_date"); + if (const Json* s = J::at(r, "score"); s && s->is_number()) { + h.score = s->as_number(); + h.has_score = true; + } + out.hits.push_back(std::move(h)); + } + return out; +} + +TavilyExtract Tavily::parse_extract(const Json& doc) { + TavilyExtract out; + for (const Json& r : J::array(doc, "results")) out.pages.push_back({J::str(r, "url"), J::str(r, "raw_content")}); + for (const Json& r : J::array(doc, "failed_results")) out.failed.push_back(J::str(r, "url")); + return out; +} + +TavilyCrawl Tavily::parse_crawl(const Json& doc) { + TavilyCrawl out; + out.base_url = J::str(doc, "base_url"); + for (const Json& r : J::array(doc, "results")) out.pages.push_back({J::str(r, "url"), J::str(r, "raw_content")}); + return out; +} + +SearchResult Tavily::search(const SearchRequest& request) { return search(request, TavilyOptions{}); } + +SearchResult Tavily::search(const SearchRequest& request, const TavilyOptions& o) { + WebSearch::validate("tavily", request); + if (request.max_results > RetrievalCaps::kTavilyMaxResults) bad("max_results is at most 20"); + static const char* depths[] = {"", "basic", "advanced", "fast", "ultra-fast"}; + bool depth_ok = false; + for (const char* d : depths) depth_ok = depth_ok || o.search_depth == d; + if (!depth_ok) bad("search_depth must be basic, advanced, fast or ultra-fast"); + if (!o.topic.empty() && o.topic != "general" && o.topic != "news" && o.topic != "finance") bad("topic must be general, news or finance"); + Json body = Json::object(); + body.set("query", Json(request.query)); + body.set("max_results", Json(request.max_results)); + if (!o.search_depth.empty()) body.set("search_depth", Json(o.search_depth)); + if (!o.topic.empty()) body.set("topic", Json(o.topic)); + if (o.include_answer) body.set("include_answer", Json(true)); + if (o.include_raw_content) body.set("include_raw_content", Json(true)); + if (!o.include_domains.empty()) body.set("include_domains", J::strings(o.include_domains)); + if (!o.exclude_domains.empty()) body.set("exclude_domains", J::strings(o.exclude_domains)); + if (!request.recency.empty()) body.set("time_range", Json(request.recency)); + return parse_search(gw_->call("search", with_body(body))); +} + +TavilyExtract Tavily::extract(const std::vector& urls, bool advanced) { + if (urls.empty() || urls.size() > kMaxExtractUrls) bad("urls must hold 1..20 entries"); + for (const std::string& u : urls) check_url(u); + Json body = Json::object(); + body.set("urls", J::strings(urls)); + if (advanced) body.set("extract_depth", Json("advanced")); + return parse_extract(gw_->call("extract", with_body(body))); +} + +TavilyCrawl Tavily::crawl(const std::string& url, int limit, int max_depth, const std::string& instructions) { + check_url(url); + if (limit < 1 || limit > RetrievalCaps::kMaxCrawlLimit) bad("limit must be 1..500"); + if (max_depth < 1 || max_depth > RetrievalCaps::kMaxCrawlDepth) bad("max_depth must be 1..5"); + Json body = Json::object(); + body.set("url", Json(url)); + body.set("limit", Json(limit)); + body.set("max_depth", Json(max_depth)); + if (!instructions.empty()) body.set("instructions", Json(instructions)); + return parse_crawl(gw_->call("crawl", with_body(body))); +} + +std::vector Tavily::map(const std::string& url, int limit, int max_depth) { + check_url(url); + if (limit < 1 || limit > RetrievalCaps::kMaxCrawlLimit) bad("limit must be 1..500"); + if (max_depth < 1 || max_depth > RetrievalCaps::kMaxCrawlDepth) bad("max_depth must be 1..5"); + Json body = Json::object(); + body.set("url", Json(url)); + body.set("limit", Json(limit)); + body.set("max_depth", Json(max_depth)); + Json doc = gw_->call("map", with_body(body)); + std::vector out; + for (const Json& r : J::array(doc, "results")) + if (r.is_string()) out.push_back(r.as_string()); + return out; +} + +Json Tavily::research(const std::string& input) { + if (input.empty() || input.size() > RetrievalCaps::kMaxQueryChars) bad("input must be 1..8192 chars"); + Json body = Json::object(); + body.set("input", Json(input)); + return gw_->call("create_research", with_body(body)); +} + +Json Tavily::get_research(const std::string& request_id) { + CallArgs a; + a.path["request_id"] = request_id; + return gw_->call("get_research", a); +} + +Json Tavily::usage() { return gw_->call("get_usage"); } + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/tavily/tavily.h b/src/vendors/retrieval/tavily/tavily.h new file mode 100644 index 0000000..e1acb08 --- /dev/null +++ b/src/vendors/retrieval/tavily/tavily.h @@ -0,0 +1,68 @@ +#pragma once + +// Tavily: search, extract, crawl, map, research and usage. Implements the +// WebSearch seam. + +#include +#include + +#include "vendors/retrieval/provider.h" +#include "vendors/retrieval/seams.h" + +namespace molo::retrieval { + +const CallTable& tavily_calls(); + +struct TavilyOptions { + std::string search_depth; // "", basic, advanced, fast, ultra-fast + std::string topic; // "", general, news, finance + bool include_answer = false; + bool include_raw_content = false; + std::vector include_domains; + std::vector exclude_domains; +}; + +struct TavilyPage { + std::string url; + std::string raw_content; +}; + +struct TavilyExtract { + std::vector pages; + std::vector failed; // urls +}; + +struct TavilyCrawl { + std::string base_url; + std::vector pages; +}; + +class Tavily : public Provider, public WebSearch { + public: + static constexpr const char* kKeyEnv = "TAVILY_API_KEY"; + static constexpr const char* kBaseUrl = "https://api.tavily.com"; + static constexpr std::size_t kMaxExtractUrls = 20; // doc: 20 urls per extract + + static llm::ApiClientConfig config(); + static Tavily from_env(); + static Tavily with(const TransportFactory& factory, std::string key); + explicit Tavily(std::shared_ptr gateway) : Provider(std::move(gateway)) {} + + std::string provider() const override { return "tavily"; } + SearchResult search(const SearchRequest& request) override; + SearchResult search(const SearchRequest& request, const TavilyOptions& options); + + TavilyExtract extract(const std::vector& urls, bool advanced); + TavilyCrawl crawl(const std::string& url, int limit, int max_depth, const std::string& instructions); + std::vector map(const std::string& url, int limit, int max_depth); + Json research(const std::string& input); + Json get_research(const std::string& request_id); + Json usage(); + + static SearchResult parse_search(const Json& doc); + static TavilyExtract parse_extract(const Json& doc); + static TavilyCrawl parse_crawl(const Json& doc); + static void check_url(const std::string& url); +}; + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/tools/__pycache__/check_spec.cpython-314.pyc b/src/vendors/retrieval/tools/__pycache__/check_spec.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0353683413c67623865a9c5a3caf4f72a867ec07 GIT binary patch literal 40199 zcmcJ&33OZ6c`kY|g8)c^1ZPqtIEW%8&YCI7G9^(HHBg5XsDUB{5}-(l19SjtfM}a( z(;HHNyTUQ?1Bz|wb z`@Vmlb1;yQ?B=d}Bp#fz&z|?c|NT$<-)quRO&qSezy06N{=JLi{)t|ct5AOUbW+Q4 zlN`_axE_vI?Nj+w>{snmvtNx*!+y0sE&J8^bnI8}Gq7Ky&v;7Ho~dLT`)$vTJu6R7 ziJ7Z>kNuvqtW!yE%C9f=DXCY)QYlZTQhD_{we;po=TG~ryvCQoYkA!%E3dCnqTzR~ zx=zI#Sh#UncnWVqd~;$rZ$VhJ?AUrSKz6LFUEZZf0i%F<%U(hV&0DT62267`zdYvj=0W|A4@sD zEIJBhd^uv4^A)&P@GEg&`P5XE(NvXBjk78mXBA)d)L7L@EI#9@aaKp;q&zjwnrIyB zQ{&V`#1?-qH*e<8YkK=jAj*|!8h=Yd=tNxU&pWKH}D(zP5frQncu>1 z<+t(A@!R?5`4)Z$zmwm^xAMFBOnwi)w>}jxwU6J=xA6z~gZv@>FyCIUcAH%(tKaP7He**V<{v_@Vd^hfm{3+ajNno8;0+2!B(-kO3~sL~(GZwc^CIs2W#yK?V_1WFd{z1D(7`rg!ato`xcF%a<@Q`=V?Xr74 z?*2NHsbg$h=tqL80k^N8!iR@0xNGd5%f7*}(Y1E3&ow;MXCE6KzFcD;9kcffZkK!1 zH{=}lRyUhWcKdmk;2ibY*>%|C9&KsgXJ2dI-*Na5f~1f2a-0#T*XI`OuCab^J+c|= zua`pV$B@qH8LB($9UDcK7v0VaLr$NY^6a?iJaYz#nv^V+kM+ufeFQDF%URS?tGxAI zv}DAo=45NknQqn&j%KGqI9`@BjbHdGiq1{3r#ehBL> zJ4c3*W1rx>z{U-gMt1{-0igt9(K^%sHr>cZIOiI>IBNGe&$#U#01R>ySe21W{8dFQ za5=q$ePd3+Rp)g(h5o@hRt{@!Vgt8&hsV!6S3iW3FVzkDMuzJ~T-1{n#x7Gq>>3*w z80yD_u(z{0WAE72X1_S-b`PUw2t#G7r0C9}>Sk2j%VwN?)H&kz*4X=;Ubp=ye*ja? zDIf?l#A`C`9uw^D3vS_Zbne=ReD(q7&@g7I`x2fTtEmv0F2dMFxq(i*%iTZhq&^~C zfa#9>2FG0XHOSmIXs=om>y~}4YWt8EQ`!euW{8BQ&KBf1FytNvB;#_Tw&%y)KqGx) zuFLWV)C^%%Go`+muy!v7=PCL1kD(!~x10>=#OlBjxQBqn1f>!TeW6w?p!L|!14nnY z?y_G*99A1AFhre+AG_#9>&MV@a%Z9?Kyr9bfL4aK>{VkPpfRW58yY<$MtCYabs)orlm9E?_+x2)lP^7?S|A$JyWS_8=N?C$NW%CTsWE zy+VKe1@yw0K-f*j$@RXmv0-mL&{6-nUWRc!m+dDNd~C0)t2_1E=2BaWfoIKTxE;e) zjy@OyE|&U`zM2mC)pd3LN@Z?88}=i52~kJ1gi`%$6R@{}s|h_k9eYQN3R?CzC{IyL zjVy*n6Q+Lxrj8gD{2MXI2-n}D1T96+QeHBSu8*PUUOA|4%;{xFB%&pR?SJk|XGGDL z=aLdd`2Od=TvpKp%QG{@7J`VDPH5}k?;;9l|&AxA-?}(j}!woK z=oPe1&FeSqr_eoKCWJYmJM-RG%L6v)sna?rMQ5UR3(SbC`?Exks?j`hTKt-lskP*%v&j= z+7jUoMO0-G)hfSw)vC|cC$B`bK4IML*VS(CcCGQ7*iUyIo`3m2a8;`X8;XeN20-t+ ztJM+h==kt(MD=2IYDDJ+j`u_~SQapeu`We)!(&dDH)3))hsWKmf-ojT41(J;?Cf_( zbiD(^V;3VSt(R!^03}7SFS><@T5v~F29WEB6ZCgP0|buZkdv=kgUY!t`64PGN&z;f zSygGbN6yy|j*Yl!kr^8k>am!db@%(c^&?|Q$tFBc~?p!pO#I%wHL&;)#W?*3Y1<_c9psA4sLm{gC*pTwIiQkzB>?G*49+b&nzmsTg|Q8H9spdwN|TtR;i-r>eSXPx}R-O({r`bfE7nMd>vfA)V%1ube?a=4yvQ+}1_RrO)UWlk_fOGVi)SxQ` zES+4nKB6UBJfZ{j?ej)-E?^U2js9^VqBj-@+Ba^D|{LuHg20ZQf*Sp!~zM%qjIR%$7w<`YZdUb}pu8ynJynEALg+ z#~RL>`%!9oVExM{7p<97EiA^>or?%s_wtEFOGe# z(+7gK1pY3}nz_3FHfhrAnjX6!5TeM^)qMIxH`{vGjOU z-9%!sE4K6Q`?S&qBwC?|l5LCGuj3(+iEL)S&LO<3pA#g(X2 zzB$x!p;1t-M5?w18EDnzsB;PaJgOnCS}Ux<$B61=MAa-1^?(q9E(RV9#Ya*IzJ_R_ zs@4mXRYZlgZfqQ@(Yc7KH=??*6wriX%7JdT@#p;-ZnQq6TsKS`uBAxpgS;v{uBe|= z&6UpWm@_O?Yzn1sp0C&xJb%*=RQ-VokMrr9MdRj0lVx(}UoK&c_9%Go^2=p3=O$C(v?j|F!gwbzE8*78FbR{6QBZBx7`0gD{)Y;`xHh5#FbF)P(`Oa zLw;S2aVQe$+kmdxKp;6ZyqY0DhCDkp_)@__8FD(q%51Ae2viA_!x^Ll0wyHM6ceZ- zt^|Ucpq{pj<*@_^sAPRON>RlCS&K_7tF+6!$1(J&>EY zBYJ_+H;AJhhWt#REF!8QVLg7m)OCboBn*?*%c4R`HbR0TVGq8(L(#@@0}L2Ufr_ub zw3u$2GJ{^2YJgj?4B{_$YTOvatvGp^b5-&+6fxw)f{ zDw`K=xj}U>Be3b#_8Z$nwu%RqijUK|%+e=T&SITx6E>pcgu$jxp}$Y5Vv}qQ6iV63 zx0_ZrhkD5{HYE(Bx((|g4dt?7RJWC}P-*>R^fmaE#Fb6ehR#Duw))8fwhbW^4|ysV zKw_^vU1?R&u}LZo4_grgfCu+J%~feWy4aW`9}vj1`p>z2y=-|9+K}+O`11m=Xh6)C zDc9ur57W{mRR8P&V8}f+knzfHX+YNfK9Ifq{;`cqw?4_@EUA-wm-o3IrLjK03ZQqU zI89Eo)8b5Zra9A{)=2laaQBFk3=?u^(%PCpxC=yprry%@JI>M?z+-CLrIh6FbyU!2+yOYPBjnA(XgU2=7`^2WL-B$RLt zeH7k)677+=G#=d>Isv~VJbdu0G8^_GNDVG~dujj^694Yu}cBDD1YjwPteYGMa z&5@1}EpK5V78b`sp@@~r!csfTrO02YAy$_3h{_}or|H+!*49Q;c7IJz)%IrmKiSjm z?LK*>yKL2|lReF+R&Sr^?x=3=_O6+z#jSe#NqjuzxAfGl-cF$>C!Ray&+nPqZor&M--!zWG>m9h(W9} zE6x}he5!dZqb#saplp4jq_abp_#UMJB!;lo9#Pd+YXp#!oJ6IYm|nw6>XB+=L{Dle zh>svca!Dd|qQpmaRc*eY5RF9E#lei?Zj#x~uJCZjt zG&*F;jJO`U&b!_AIh_G>2=%q1V#au*9qW;8P` z@9N*Iolc+91e}Xzh*Vx_4rVQwSAa_lsIKo0IA3YIw)?TIAh+h?b?f?am8|aBMMLd0q{ih#gb81F#`Mq z8;2#|zhI}oD0M`n+WJabFJs1yj$)HG~LDqtO z>n3W7UF|efOD?R)Ixr|;N$B!u<0Ls-SgG1%Qf{I8b+4G+ ze0NXCvP)u&Y+1K9-q;vye64v#^`7Bv!>sBXrm($1v^UK5g-bThmuw7`Y!+>sMa$;J z^nB5nFHMPfU?d`#p;!|@W2+96LhWV)=m1D17f&rGZe74(Kvu?NoF&vi38?c~K%oHJ zQ%jB#FACCh0b)SPEiJ=s=mH^2N>m+|Qm#w&tone}koyXvPhE%6p`pdvp`-fo zx=j~BN`j~ZDHA2?W6Vc6*I`aXFEP@xBSmQ`BS&|$-b+~;KYsGFI-h~q^fZw&XI4@Q zJcC`6`rctk$|~uVa+Lm(7a0R@h+AY#SX81hA=Mzx-v~9xYT*=`8&OpXhw#hhngBwZ z6OP~psSi!Hh~^Ab*%;nR4muK;F-Xgf24+ck zr}fA{y}KT1qbB_bj%6IDY+Su$KAco0WyJrFomnEWK-5oOOmuE2?9kd zPBLZV(*6cQ z;_VkhCG{a&gJ@}38nMVellkV}a8ccSQQfRJRJ3;b;MA@_=LeZ9f@i^V{0RBS*vA91pfR#1nnuz|g{pp%0)&=zQ&pXf1=@ zV)-H%o$(F8vqV%u{;29OE`e2ildcYf0z(W|Cm5_6md0NOtL1=IJI#2T_njUn+E=TqIsHONi#}DGK}Pbg~+HIHasM%R|qk-A{9}9 zS?I)%0HFdG(e?xArR8Xt-ZhvkY?eq+PXI5t=|&LkCH`E3bq4HgkrKKPm~?q!RUsnI zjA8T!5se@{%{rx7Ov?zAzA|vr8(jaIe<7`OvAk;5Hhbh-Iqzj7dfUkRQG z7dOloH_T;)i#I$d-mqxRhTzOhaV;4)e-o5y+bayCc zk7(VKK)WrbXFW)_FXrY4v>)V_&8(euhjQ0V?Yy=ZYPKoUa`4onwroNAukwNf9#pjy zHOA5XQO=4MBtF*D^OUVS7|~*J&|m>9mImDOuZ7WEy8iM zyvF1-Z2F#Y=-pUK(0{sQqyisZgrdvUMi!gvQOB3r)mf)m-UhZbC+{6v;Fjc})K&Q{ z;aXK~5M5JG{?Ckrf=W8I2TgHgL9}lY${Uf9ib0}*qiZW$cO`Wn3=D`ViKwoCc+JOC z9D!R;uhqC!r^XPAKr;0>yrLcQxkm)z!s4(CTT=;ru!WW38EHW!tspL1Pvr#_B*wFl z4A{d0LNZmjB}Fn1pNeE=HipaB&zG;CbA`*dJSg9C_u~EXAHGParih`MCxu*AVX*1- zEn!>5ysaW+t7H^&KzP}|Xtf1eZs~)iubW@fp=hH18AF5_E{KkL7{^1EjjYM;M% z^#0oW=l_1g&rXRQoex`&ipNeaw4S`Ze#SO){@vU=&xua=f_2~nYu*QWr85<8q$}v` zr5WFQFTMTJoIkv3$AeWnJ}RxC__GzFeM3MO=tq$QH&W#yZ*QLIm~DKwD^yq;w$^== zq*61j4=UE*UjM_5_tyPjTiCiUhV&%0ni*Ciqpf>=8Q=S2ZjBqy}et%Gii(Q(mN)med77y(iwE{>#P!%w3dg8^x+ z$v~LA*4XXO%a_a+MF}h6D~R=1`12Cu@ISD;n9{DVo?ad344DdGOkqizY=v$QT0N8b z<^DkVwZT_v(D+v_OlgVh@lX4&Ph6b{`rf*7`^p-vC-7`nCZ6whh?T>m}GcP!sc3zcqvSh)S6 zb^EJ^C~MX@w=rbd61KQSi~EPdk1qYiC2{Xb@sumH-3`U0tf3y5@rl_r3&vW~Y$5xW z*L7eMvvQyq&$Qj_50<|+c(Z0!^DX_iE$>^to%Y9RRGGgV_`#Z=Ywqv-``!O=@NW)^ zCr*V9^n~{IK5Xt4`vw=92Qg`GS#DT@9dA0`>b~9mM$dw^My~s-s>Q;Rw>AS*kA@1@ zOtl8eUfCCLP9MPMsg{T(3xhOuJewqgqG0An~QM%dz zI7_c&izf7zpcL!D<872vKe1aJnLTScH$c+^{9n-~&(z968QbiHxb1{kcTzOjL#A$M=2OzI7f%<5Q%dGjAn2Rr z#gvk8O2a}*!yNb+>-7uM7p`4m;*X27DWUYHUuif~BO>5`D5c?ZuLXttIMv>=N%!MT z<{c?136fy~C9z>6_E4frH z#_(gk2W==_vCtTxIyj|d>+y&Yf+{A#6$+4nP)Ikz zz90>*(=#mrUYZ$|c4@ebWMO z@7K&-46ok)VD3vy0<()P^HfIFNU)dsRbEMfFKX0jf z*g)7(45`gw%YrBAa;Unaag})z?Ou>1B%*gnLf0{RgBU)GahZecO~tF?yU3&IB*A4; z#cj+E=Gp2Y7_zN__*2)_nryDe8}%mruEXq>Ta-{UEm>xe@mHr70Jcnp({?R^(ED6l zf!!&7b$NY6Rqt27*oeOd0pJFme#A87rF~ftfw*CrKGg3MUcd)of^Iaq0Ua6)ErceR zwGpk?jhT!|J0PsZ$A}iJpHDKpiIZl+2#jL9jTo&24P@X&0!z!}uE!=UsCU+35A9=DEBr8JF)cH&;gw;=GUAq%du#uV{Xt(St0J7WdOmCQ?Ak|J^&rSHvu_RG z7@k=-YYXMnhI7`>=YXKQyEByY{KL!^m}+Ub?1qoP>SWK~L!ktzN#ErARSNCDn4A?V zQjIbqXF8=BowN;6=7`tt=gz>|4VWN#Yh(40x}5f`p8%t%drj{vKyojTb}$?G}Z>C$d3FkRy2`8srWM?H&4{muf3CQKD^hBW@5W$NOYVL8w9! zDg6n3rW;vrd-b?Ya{ru~8CV-iEt%Z?*qj|0xT7fE#%IsJa~WpP*T$zTk4>r9R!;8x zFdcSwi?;k*%{Q8Z{WDGA*}}FO(N;5C_P|yH0jMSY=K7#*W<9v3%^_RuLrd+|ou6vB ztXkj+>lzs4>5MP04d{uV3vLZrtH4%?*1Fjp;q-O$>FYx28$#xd549V~T08Y?{;&80 zl_5iZuxr6kv1l}tAz?uCvgH@%w7_mcS7OQfP}YXK6``!>?p6Nv>L0JZ?+ESa6px)= z*l}7M8Ve73=ZCycIBs0kqW%@fomK5ne3wj0ZhI;lb+I)Rd*3#O@zqC}+@Kvwc2P#czM zCdp~-)FsOyX_-+GZn6{}JuHRvB;s&Yu*r<~_@(s=5QZr}u4KSUWVb_~v?j~tOBlf- zt_^Y+Mvg-t$t<{&_noFbiujY}9#t?Y2*)L%Cv~D+@PDp-<-|4^rFznDwK<|`YG{b0 zK#}Pg8}+)q66gPI^r%3RU*R_0-ocI3|IAWTR^R+7g&}bx-I{)RAFsds8@l~1ZjyLH z$52Zgy3vs|$;+sbYC}Xl=7Ag=bck?+l90#&7Un*ofgUtKJ6?h&S(YHtrm0FjdzljG zNBG`HsOI1B0G7hK4unE#`gO;&BhVH~wTDwH#nj4~9bwCwdCQvFT_H>3l#0-h^G;US zTue%m+`L=kH^zg|x>hg{<{Z)-W~e zhFjZjZvU{b{9`k>qUMQ(vu0c$pB@h!3ksq1vT%Cke0t@~g>ZV^gY-I7Dwy~Bm6;6> zb61CR8|HHx=4_$d^>cj>b2mS*ZjRV;KF;Sd)_+_8Bjn|%g;rHY31tDvNcx$lu?;{v z>fJ8I);NwjicUIG^(WHN$$0G86i0p=<@_9&MoG{C5Bv6KQ9vF8(j{TB63Rdm86uf> z82QYYRdpNMe*ALsQWc72IU1?sYLK6D!3N(HXM-OnhV9y~)H`N_?=gE|HYKHLAiMo? zjywOyNox@8Q%Q6$BU>m3S<_Meb~K%9qML)f2~bD*fw8iwT~O z?@QW}s7`NrU&fG}C*6^rG<8(vk}UPJM}GHKY1}VOg);U#)ySyLyG#;jdSeh+-x+9ghc9T$;U5;o=>vdh3 z%_4?RVwf($I;Pmla7QSFlhvSqq^a#Ir?-)9wSwBFN$eM*_D#;RBiL`ykcL4 z{rx`s!Pd^bhj%ge1*_nS0}L~^^Y;#6TRv@rY6f{+Be^4x9+l;%+WtZv?UC{gVm~IB zX4$bqL_LI!m&|>E0IC3MrX*^Mm}NQI4rCg)!m^dlNCf{b>AX!ovj^AkR$vBGLaVK zx9RDdbYlejf1)Q^N+Y^)Yy}glGqF`(cIkpWIm8uEUrFL3#uGqJ;2RYDltfRvGO*oL zFS&vcen6RFn&J!V7q9SB3ZWKA3%gqA6aJFEtNOiQsg(7dDUPI@EQTU?2E8aL_!({cjdYa`$u3RVQqyjc@2sF^Roy>5CxgbdmF zw~pO7795`45z4L)XRjBt*UvRg?OrS{d+Y4&vu_RGFa`8j$Hmh93%30qWECwoZv4@P z?{BzQ6WVe(yyd8f{~bpkH9A81UGt5O*_JyiXEa~0#bZD(=68Ki(-g`(`2Oyh^WSrS z$9;F#chBAHn%{8XPCj-PZFnm`nE(FnfKJRi_(9#eP(l0q1G5=F(tTfd*YyMQy-V|( z4&T{~{YQ?sHU~GqKM*j81?`V>3tk%!Xdh=61lPW&e_KEELdd>8Y~LcnPJfA592 zUzj@Cb)h=<>u6t#nqSl95t*`C^<5ax;pYlrrEea(B+mD%nh1{COyrOq>GyUIQ z|Ni>9yl-v~uX$cv^ZebukiF$$UQ19LAN}m3+y+E{%Xr&3lQFa5-TX&+Yd%il3hW?r zFi!kij@yozgHW`DS8NhjY?|v^%qj@(npriwey;j%*J4)jJE^nCxDlIFcc?N0n*Ufa z%k&rjDeW(`V%D|=>vNxhz+Us2jw`Ik_N$y#Pa5RySLIy6G1YG(c?W+JDQNf7=CHpi z%G$TZ^j8~l_j9J7WTw)6W$u2R>8Gkx-2Jw)DrSdV*4=$-qPx4UdUbVKf@qkeB)~4* zr~d^I_uJ?jVuqBYc~B%Z&^b`f?20jF$+Q9r-eVetBodeyBM2rf+4m@uChUaqgRZOG zp~bE(q6(K>N=bm^q=NweA24XK%Q@p_oc15wbWtWe4?RyB{jO27)Q z+Tc$iKQ`XFevij*-q~{CKu7J4);;?U`BS8q)OTO~TkPhgXkzrD!iAMnuZHch^2Xq9sj=@Cv2trB5cN&1JUKl2IH_ zp_-0PK4KVv!4E3S^tZC`#y{jiedZ}Ss|Bb=pjBr?)rX?7px74z1LkhO9~2wPW+*41Fq zEY|D$r}tkw0KF!5(4&x`CTuQRFc(1&r23=0$56?eZ9#xCI~`)_ zIdNn(v|=oj=Xqo~4-}KJ=85SkJMy$>2%E^KS&Nd}U&r(|($-r>!l!|R&sYM!781dO zFb50};UR6kjob375+Hbl5=P=})z~rZy%14didU>iLihco6a|>==zHxcR zcJsxMdCiR%1Et^0xZX0=a^uB$^BPgR<~I+_YrK$0b2n4gX#b;rP2v_)9X_x&{+wk7 zDo?(WsWoB4O$w73N{Z7aJtaP55C5+CU=?HWlacZ<n2UJ&`#Me{ReqDECZ+DYl-Ma?Hb){&&Fh@7Cg%IqxVMzWPeu#eH zDkZqrc}Jb++`@lH$SjS-VLSljji&4Rt9miL1U4Sdkg*CINtK}TnPao(L&in~VPb5! zx*@PKXbU<+W_#FNId86<@nZ18=Eiw*W5~Q#)UIV*xibT~)3ojN4vh@KyTycTqUxPE zBRD;S=Z7bJW9QtXLw@%JywC~mGuRkAG1BkpC7NymCZ|Ji7UZ6geD_Q+vU0-P4@Yss z$O&$xCYWM%g4{|?2++gQ_E^`1Yskx7VoeOdF^F#h7VN^%C46VvVE2T}jVy-x+!L_8 zop6uzxn1}MAA9FeDYpmDPS*syf?$KEZ=xTTA;ZYQGvOTd!kfs1dvuUgOYwDMPT?lt z6U6Hs8n`^+9fvMppv6XPDJ z;C;%g|JOZLa@x4uV0}7xnQqvOmO%14~~zXL+3J2 zbG;YPC1V11OvinaN3BC#Ofb(`e-k< zJgiEv!ktuC_)F!EUJgbypyUwGUn56JK0Rb;Q)0vNoBZ@eR>V58_d;XbAh8J(F~vSd zvZaouXz~-|ll0O{B%kI??+9d%!(NIbKVeEl%yFM0c7#YD<6^PMVAOgkdP+1#q+0Gq z353vtlx8VBNmaGPM~zfc9#TY_BIB$|(z|qzFN=ksEH)eDR0(B8lSuO{Vu_C$DQ3-+ z;viY9D7Tlj#}fMv#ZwbGJ!JL(QVN)gfYXe%D_WNPfqKLeA9Yy{Zt`0cD^+d)8Yq8= z%Rvf}SXQEuA|~o532-7-8c%>B(4aJPBj)%Q3q@7XZNwOT0$fNL^!l9Mb6$ClMbe|e z(!fM)(EuqP^-6@K&(bVouT{}k%AU}&Tj;&eK-~zSiz_B#iM~?vB;aHI89hMJUNE`j z69LmMlFoWR_9;>(0}56V_odzeVQd6F)_-o?6PGCOX>t0AK2WYGJ@gJjCb_sBT%c{B z;HCwVT`*pkpAY86g{t`kF==wMdF}N-aCxbyCb7jsTJ#N&tuUPd*^xcKpO;vXN$wYg z_LtWOvR~f%j(c|ZT*tQ#++FvuW_zgec_Mc+uWy;&61X&z2?954u9`Pj&1Qwn>qPB3 zN$gIvmvTk>*M-a{L2J>zP@cJcIa9=XSc#d`g+`1iR47juGdv($@r2#rnuE2-SHrds zAu7a+nwUl}nO$^}oKbHW&uGc&Ix+7U^O{U^7PEtj))C@2zz&)5u*;FKzxGSk2DBO( z@9;(vCgZ^QkZZoqTkheptZAA*pC`SG<#Y*nK~(=x+oyA zB#VNrQz7ye$krG}GUi>aV5yGo?A+&Q70}Dz$@tcmUA^sxJ31rU_LhCSBp}mDKsG|z zFkbJMc#@oxMB&Ob0$LKzkHGH%|XnUiH+|r~5@)ivFcTR>^)X%S|pVNj{ ztbed#JsD1RykeYIEvAsqqY}K|+!xNOn9r)1>4c5UgRIrl9dnw8#~w2hzENXw)cpAXT?$9LZ460pK(}<^Ni@! zMD?Pq!&g~d<5-7ypvE`499&Fi1Np$psK$tZo?Y!x5KuzMjYC(dGMyEGk$L}e0H7} z48lLsKr|sbNPAo1t*tk<&KPIQ9@%Q)*GElc{N}5h13Y#E({^E7&ei>pEM?Vw497m! z88Zl)-QhuR-(X?kPv17lGFr_ek z0i;**uR;Db%D+o`Kn^uYzh9zWhG*4F$^~BA<*O?NGAtP?qU9Uob6}w}8UW{cjP;uPJ+>rrN@U=8}w`$^B_Y zNCF62{}^X1Fu6YyN$I`dsW5NqsT9Iu*lWh7+XD*tI^ANHKdP~%(l3S}N%|GP(TrsI z($Z^$QL-dusz%ndKikXUhIwhq3~UOSieT+)PP?`qA|n#*lTBdtjp|!#ru9>r$7Wk# zZ`fS+z+49V{lyjbw~pUFKI5Hj39YD`Hx^831FpyEIas1E%pIA#_~$1=>CZ#Hl$QDO zFyuyoox#etYHruOQTH&b>Y=6T>MrspX`D6&vR+QZc7p4drZ3&mW7Blbo0mg*)ghQ4 z!OqZp-E`ITp~d=I#T)u>XlIRfP~ z7M(-l`HPPnmqh=SCmi>JY7ZF$T~X~-Ij)vz`H>^ZY!l~yG9gUN7FNO8t`>$B z^oedOkaJRt_YRK1v<77*%wZM^2`~=*C_p>qD;kyyS|xS0S6jWOy$Y!vjYtO23@J!_({stgrpX%&x9)5m#Hl@=8z^!s|zT(O~4AquHR#X zH!u&3*)Tt1>u?7b)1^dvT8al-<7d_}ONR06rYJiJ7zew+%1(+o_?!-|JG!dJ_HM_8 zMe}m#W2>x#TdSgjcZT6%!5TF;RC<4{t|pE)RYF07)25Ves#0mZ(Vxufj|qmKdm`-IKe)b-EXOp zPAu>`FMI85QD@e@{=(+C6A_|^D3C%N<^Pm&LN%E*50(px@qtpL6wO~?R}MyKUV1Eo zf8{ezMo`FT|BjX3X8TI%D3@5pai|2sVut|OtMD2-(4d-qR`N*DtrE4l7Jr|%|dz-Z_7_>%yw~-VfBbiens?>2rQz5hX9ynGHUGqq%_~cEnJ9`??PM^A=3x<&l&=@m~8hI@uY z{69A=j(Ef8d}99ig5?5Hc10_K18?mM?wdJ3oA=(8nJaTGa|7SoC#F3In+}<1v%Q=O z3-IfEruPJPlJ4*q>DEO{=JkWq2Y+zv-ue4Ee|`Dhvl{Mp+-tcv@Uwjj+YZike5ZAx!x8E@_Au-CRO>I& zvtXy*GBfb*zJ>hSMQhHj)ElXPS0#2H6&>B-&QoINsfAv*IB@P^@32@rvS1y3Y=u>7 z(D3@%khK&E&%C?;p|x&G_Y1T2>Uz22f450I>JYn5g^%`#M|&1dqq4&fPmhQNqhZU~ z$E&!s%ulOm>$3(n=fA)GolUcQ=la9t%?su5gv`1r)PF8C;t>nZKe7nmcC9wDm^Xdx zz*i248Alh5)|=&l{@1GR?3~fQv2VuzXKNP-)iCs zsU-SUo~M93nvOF4S_zCBPrb&}tnsGkF|ie>9F~D}Q*g8=ZvRTmdZHWLk#w~~aJlq? zk`o?16zrkor2N8FWz-5IMkPDcj6yywSv@3LLx5)1CFN%0sM9VVvp}t2R0fxfvn3<- zL>o^=_v#%IcWP*gqbJ(-UFfL*aZVkzKGk^QE`R#lHNu17MJbg^Ib2%YRQVrN$_3SB6|} zl*5kISQ$?UHPaZ47Nr$R$VUwvCwk1VG|Z2sP@a^OpoeY(1}qLsJ65En7oV;`k+LSa z&?}(-Exa|pf7+v#D6t%r5zNtsWt zZyZgElrZ1G*e|`FGOuzRsce?yCd{kdGVQ77^WtmD=QD`b(pZ8z2Ifj=RYF{)2fmR`FBU#VF8&ouPJX-S?yG0sAfYcE^7=~#u>*Bm*^)?b@K;v8{nVQ3 zGG>6~H!(Faqrw@L?$4m}(B#A9Kq;Sb+x=!nCX<@Y?eeED$Cdf4*lA|%Q9Au|i@cZK zPS(H?)oF>M&s;{)gFj%E#W8PG_c;C_HJ(4P``1hDj?w!|8okoHO{M(q+}5dMj7jM`;_MSvhD zI4QcG`BU@?|2u^+!kFxJVE*Rzx!eQJ@nN4t8Z#o9Fm`pi(15~)M!FFN43lJ>mIwDI zI6u!z&PZ{F8U&a(*e434z0Bw{7X{$dD>}52?Qs@b5h9QxN3shd?mb2!lh9I@TAC#? zne~yd3*{CQAMiYGppmhK=uF65Oq8+pX118W>0!nu$wbbQd2{Ve;gc@0Z@+{SVdm zt3yYRiv=eZEGNl*NqW|gT7R-m%-$PH-v^^-ICb22b)T41KbIAFag`Ko3yBl}j#jgpz{aK@^Ij8(Hc#kGBpYWhF|hpnaapl!?F@qS}GWQEI{ z#(8VwT*dcRe`odGj_=mrD}C6s>yfn;R(;u(pP9JKTpaF``?~*8Mj4C%)2(lrZkxpN z^?&C5?szC~+oSYtjH-S$Rct>Z^2fsM$Hn&Jp_AQW@u^4F9#FTL<)CgYsS*Vr1_cj$ zY8YC-x%pA{N_b?kWxcik_Wqd{S$;*^9@)0RDEZdGR}bDlFCOXohs*abhfbUn%eo(B zor2wQPUXjIh`w&%(rk?Td1Nk>iE6Rn2*~<}1xFuQ94Iy|o~YjPL+`!yKk)y2?S0?h zZ5E$9`Y_|@BdudO;Y>4waK@*T65)(T=}K;w7ZZ!c2y10&0)cH%QuC@@(yJyo&|EGc z0hvv#E4$jG#I+IvJ$`EJS$X<`6j70w5+94t68~o8`~*%}^omOrxK}YK&q>^?XuGXS zUhPSV+&EL>93^J#7aqK{&m#Sl-cx=(I^IbC6y;3`cTAUzC{zeLSdZ$*)%COiY%8os z2a*wie01xJLO{yo9Ei;>ce2{W+-fEx_IML-R*1VkwX#LYW5Ax+q+}gbdLRpyUP=q1 zEjPxN@a{?_c6*ib6#E5Zks4zxqUlrOQmu}~R-TY5EtU!!&>)iQD{7b26wc1x_xDA344}{w*04bZ=2c$6;2+d95aXRFqI@fA_N;N42UC3`(aW z$`=L~dmFoh1oX>#S zSR4S2Nx?Ajx}wr=&skN2Ufy*ClW~WGBXavs#{W9lY zDSZG#0IN4T=$Ww$1F(z4dCAhj%Q!HaF)T1`0v{u76}e4mH5VG|nGKcSEaei%I+=-e zl21SO=P0@bI(u3PmNgcBeRtnVZ+D;H*esvm9E(op$5Rq1AHTL91^OH1vSRE?e6Dsd zD{+TLOPtC&rI8d*!@~33`c$7DwccMGW8!ys)hVr%7-qIly8}ixwIH zpD;~ z+sz%f_P+{eFU@y%g|b>g)*X}kWb-gFd-L6OAu}@tRa7CrFsCmX)2^pZr;0h-7L3n* zsHj7Zy|N`(2`z&p^$%C9o3B_m*Ac4N9Ij}YuV}fq@_uHh;s6Q$D^AJ$m&kXBohQQl zNfG}~!L6h_d}=^EHLx%^Dvn)xICxns@Gn@dJkH1gE0pv4ox;c3K zPcpxq|Ht`rjo&PKxMKaBFKpYoVB4xNMR6LA5HuXcJ&&xtpMohW``7?J2KzG8^X|Js z1s#9mx}QNzk?gp_F~^4rH-!tg&lhgLr+HMk>u+511sxC5Iwtpgn3@I7M``-L`z;T1 z4n8s-!Uoc`?2oi5k``gl{dHoycqb%>dE_RSIm?vs2t}m725=)e4=SaswL$g2{TXyk=;k zM1qeuLe`$LPAkPI;g07ee(&K9E6T1KDHLCbMvM9!I#kQH?D=3DI1ONVN( zF)`tt#GveYp4m$XXOIuO9j8$s+5)5SM6K0jd}y>iG2$F-cyG&EHD!F9Z=cBv=dTv? z$;1Dy$2J_N^jb4L(5VvJ$>U?rt-2d^Gg+a`%Bh|Is@A3Ee`etzU&LwMx7{;? z7%FbM>-zc51zY<<=do~S&x6jM`Cgak9u$W>58dZOy+r!Hq}r**h#yq9)6Y@W5gO{O zBWjcdZ^EmCgFo`UKXSi#!Ey{n17FwPXq+j1Xeo!kxO}c)4OXq}DmV(KLtKCT*-Flm z`x&R!Wjr>fifIiC#>NPaozx-Vb8iOP`n%el+1yXEb9UBiep08xeHk`kt4P=6CS7H>LtgF`)-W0@rz>Y2HR3np=N%-E;duiM?S??AIZS0c+Lqb~3_Fi3+gW5kQyFSJkzwE9YzgL+xo z@3^p5y%W%n$o1#V6;9n1CkgILqzKo&RmG-FFJ9cjw?>_0?{TX z97B?b%Dv1tn?R$&Zs!R{JjNS#R~gI!EqtIGf}0ks<(OVmuIoe7LwAfbSu@UebIIsB zZS{h2%?Ih$Sk$cx)>V(Qiij$oIxiYm&a~jRn3@qzJujx7e~^8CayLAl2h~%pAErvu z_<-T%t&c6aLH#33b|7?Nb&J2X}Yd%ip(y~5HV-kF-$(g|U zN5*`jUv2+H`6PyA{j)YjG?l;8KU?{Ya}Q083)-e%SaYTtMeT}U3vNt^A2IggoRp!_ z-d^S=!e6Ga?ASM5cLt~CjrWaXPq4iEka6z7^VCuPRZ3JcIvW|f> zOSKjYiUyJV_V^TMhen-s&Y}8*s5r}=@^B0OVkNFnmpmrhR1=HleM+>5LEbFwuTXN7 z1D=vVYTWZmWSCTX>wL}&L&KL@%kYi^phvOE6&(2zT4U;!Sy``hzGAVKQ#Wg4q%=5V0i+$OBw6k59pfjkv)&62s7n?T61eTff6->6> zJ1)?^d*re$E*I=eg2Gv-$Cgnh_Lv}s9-I4})pA8ySxPLA8$-BGezT9zwr0UQIOK`5 z!L(ze@MSw4%xEVei&7aB>y9s$_N4Z+&G+@)SNfxGY$s&*3-#VXCpNCvC)6Pt$(uBZ zTq{uAl-BMUMq_D2NyDtv!hUf~I9IJfWrpG^Q{^0|aiR%94iIpl9_`M@@fFzZ4rrJl zkJMgl%4a>rI%uN1?xK5mxc1yAb|EX<_2uXQ0ouTe{ra>U76VePlwM<}g4nPqUFdYt z$rB)nSV`6D&n;5DWbjB=Tqazm1DG(nYJYBA#bGDv7T_lzU_0zDqETbAHfo4{Y0mz$ zaS0*qsshR8hPMYc1`{myMBfm6>tm}w`B-q*qR~D~W+_77I326)oq*rqQT$-q*Af{y zL5228QZ(%J$C5DOM3c8cGV4UUSmGsBse$w_Rx<02!!6V=2=*nWSnzm!7kY!|!&?#t&;U5|u{q-?6(IFfC`?LoycYlbKK z#(W8bTkWr*>I$RIVa(BC^fyG2<6!9QV*?Dp<;>)&qK>N)^6$O^`D^zJ6Ft(dYpKxg zE9bjk@NXn4+fF{#l)Q-4W!+$RdhMqf44h^iBt1)6oJJWdl`=?f3Oo|cUV4`zR`{#g z>r+dh`ALOf-AA645`0KGR1LU+_c0j2D)3c-;jA6^!fXK|_}0;}*mU0qe{{}%w!htu zm}dyl5DrTif0lCTK&TkmQN1Bz=p~o_Sg1$>FG)iG3oY2CxPe=M)8{mFAzG zg3&`1dx7E@TtgV15y(WrT2r6W;zvT#-rk6IZ!6y_(H6g8`SC4X!WK%(xRo+`s;1I5 z(5;wmd+63ix3AERF*G;n38rP7@HXB4gl<2gTLd>U@)+@;JmDWHh)Gf>=;^=Ht(b~f zLAMgR<nh+bya91_*T_!?S|sWH?#YK;35PSg=7;y%in6mF8$xN$3h zetg*dobY@2ivOH zS6&JTw=UkecI0<)1X=K!V{lN5rU>1DjTPh|~7WMU$Fnn&9)RB#D zYCs>Ev94z`o*H2hdDi>u@|1!GW2krq{|R(yT!~si(1nsC`J##pe|KRKPjel zFKSb!9Dxq8@bJUz!=j~KG#`0VnqgK?8lKc35~8QG=;@zua$f21OsBOzQFAGm)r-dJ zCpwKy58hB`j%B@QoOq&9BOw_cPc=PRUuaQJ8b8^sDpI9>@?5Se8mC+pXg1NM;1vscfG`JPbP`APFf zTFX=)1q8E0X(i~GsU4I?%oRcwFZH|nmg!ZKShzQY^O&^OK-sO;udYUp#m6Z}+X?NW zl7X1NEtK{g^m1^-{zc>3xgG$*+#30*YPN3Cxb^O+d`q3W5SW-bCf2vy+xEm}!Vu=5 zYZr|)g0 zVltxVbF}EYv~=|F>J_R!)hExZ?5ZvmhAJD3_a_b6_|TB0Dt=POp}qbEP5z?R@U^|a evv;~PP=2fGMis1rYT)+$k=70pTU)@U(f<$HpYmS- literal 0 HcmV?d00001 diff --git a/src/vendors/retrieval/tools/check_spec.py b/src/vendors/retrieval/tools/check_spec.py new file mode 100644 index 0000000..59e8611 --- /dev/null +++ b/src/vendors/retrieval/tools/check_spec.py @@ -0,0 +1,649 @@ +#!/usr/bin/env python3 +"""Check the retrieval call tables against each provider's published spec. + +Sources (fetched live, python3 stdlib only, no credentials): + + qdrant OpenAPI 3 JSON qdrant/qdrant master docs/redoc/master/openapi.json + weaviate Swagger 2 JSON weaviate/weaviate main openapi-specs/schema.json + pinecone OpenAPI 3 YAML pinecone-io/pinecone-api 2026-07: db_control, db_data, inference + mixedbread OpenAPI 3 JSON https://api.mixedbread.com/openapi.json + jina OpenAPI 3 JSON api.jina.ai, r.jina.ai, s.jina.ai, segment.jina.ai (/openapi.json) + tavily OpenAPI 3 JSON https://docs.tavily.com/documentation/api-reference/openapi.json + exa OpenAPI 3.1 YAML exa-labs/openapi-spec master exa-openapi-spec.yaml + brave rendered API reference, one markdown page per operation + (api-dashboard.search.brave.com/api-reference//index.html.md) + voyage official Python SDK wheel on PyPI (voyageai): resource names, base URL, parameters + +For every provider it fails (exit 1) when + row a declared call's method + path (+ operationId) is not in the spec; + field a request body field or query parameter we send is not in the spec; + coverage a spec operation is neither in the table nor in EXCLUDED with a reason. +Rows whose source is prose documentation only (op starting with "doc:") are +counted and listed as unverified, never silently accepted. + + python3 -I src/vendors/retrieval/tools/check_spec.py [provider ...] + python3 -I src/vendors/retrieval/tools/check_spec.py --rows provider # print table rows +""" +import io +import json +import os +import re +import sys +import urllib.request +import zipfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +CALLS_DIR = os.path.join(HERE, "..") +RAW = "https://raw." + "githubusercontent.com/" +BRAVE = "https://api-dashboard.search.brave.com" + +SOURCES = { + "qdrant": [("main", RAW + "qdrant/qdrant/master/docs/redoc/master/openapi.json")], + "weaviate": [("main", RAW + "weaviate/weaviate/main/openapi-specs/schema.json")], + "pinecone": [ + ("control", RAW + "pinecone-io/pinecone-api/main/2026-07/db_control_2026-07.oas.yaml"), + ("data", RAW + "pinecone-io/pinecone-api/main/2026-07/db_data_2026-07.oas.yaml"), + ("inference", RAW + "pinecone-io/pinecone-api/main/2026-07/inference_2026-07.oas.yaml"), + ], + "mixedbread": [("main", "https://api.mixedbread.com/openapi.json")], + "jina": [ + ("api", "https://api.jina.ai/openapi.json"), + ("reader", "https://r.jina.ai/openapi.json"), + ("search", "https://s.jina.ai/openapi.json"), + ("segment", "https://segment.jina.ai/openapi.json"), + ], + "tavily": [("main", "https://docs.tavily.com/documentation/api-reference/openapi.json")], + "exa": [("main", RAW + "exa-labs/openapi-spec/master/exa-openapi-spec.yaml")], +} + +# Spec operations that are deliberately not in the tables. (regex on +# "HOST METHOD path", reason). Nothing is excluded silently. +EXCLUDED = { + "qdrant": [ + (r"^main (GET|POST) /(metrics|telemetry|healthz|livez|readyz)$|^main GET /$", "service health and telemetry, not retrieval"), + (r"^main \w+ /(issues|debugger|locks)", "operator and debugging surface"), + (r"^main \w+ /cluster/(recover|metadata)|^main \w+ /cluster/peer", "cluster membership operations stay with the operator"), + (r"^main \w+ /collections/\{collection_name\}/shards/|^main \w+ /collections/\{collection_name\}/cluster", "shard placement is operator work"), + ], + "weaviate": [ + (r"^main \w+ /(authz|users|groups|replication|nodes|cluster|meta|\.well-known|classifications|tasks|backups)", "operator, auth and cluster administration"), + (r"^main GET /$", "discovery links"), + (r"^main \w+ /experimental", "database user administration"), + ], + "pinecone": [], + "mixedbread": [ + (r"^main \w+ /v1/(integrations|admin|billing|trace-destinations|buckets|events|api-keys|data_sources|schemas)", "organisation administration, integrations and OAuth callbacks"), + (r"^main GET /$", "service info"), + ], + "jina": [ + (r"^api \w+ /v1/chat/completions", "chat model surface, not retrieval"), + (r"^reader \w+ /(echo|ephemeral|bot|\.well-known)", "internal helper routes of the reader service"), + (r"^reader \w+ /\{url\}|^search \w+ /\{q\}", "path form of the same call as `GET /` and `GET /search`"), + (r"^reader POST /$|^search POST /search$", "POST form of the same call as the GET row"), + ], + "tavily": [ + (r"^main POST /(feedback|logs|org-usage)$", "no operationId, account logging and organisation usage, not retrieval"), + ], + "exa": [], +} + +# -------------------------------------------------------------------------- +# Minimal YAML (block mappings and sequences, scalars, one-line flow, block +# scalars). Enough for the published OpenAPI files; verified against PyYAML +# by tests/ tooling when PyYAML is present (see --selftest). + + +def _scalar(text): + t = text.strip() + if t == "": + return None + if t[0] == '"' and t[-1] == '"' and len(t) >= 2: + try: + return json.loads(t) + except ValueError: + return t[1:-1] + if t[0] == "'" and t[-1] == "'" and len(t) >= 2: + return t[1:-1].replace("''", "'") + if t[0] in "[{": + try: + return json.loads(t) + except ValueError: + try: + return _flow(t) + except Exception: + return t + low = t.lower() + if low in ("true", "false"): + return low == "true" + if low in ("null", "~"): + return None + if re.fullmatch(r"-?\d+", t): + return int(t) + if re.fullmatch(r"-?\d+\.\d+", t): + return float(t) + return t + + +def _split_flow(body): + parts, depth, cur, q = [], 0, "", None + for ch in body: + if q: + cur += ch + if ch == q: + q = None + elif ch in "\"'": + q = ch + cur += ch + elif ch in "[{": + depth += 1 + cur += ch + elif ch in "]}": + depth -= 1 + cur += ch + elif ch == "," and depth == 0: + parts.append(cur) + cur = "" + else: + cur += ch + if cur.strip(): + parts.append(cur) + return parts + + +def _flow(t): + t = t.strip() + if t[0] == "[": + return [_scalar(p) for p in _split_flow(t[1:-1])] + out = {} + for p in _split_flow(t[1:-1]): + k, _, v = p.partition(":") + out[_scalar(k)] = _scalar(v) + return out + + +def _strip_comment(line): + if "#" not in line: + return line + q = None + for i, ch in enumerate(line): + if q: + if ch == q: + q = None + elif ch in "\"'": + q = ch + elif ch == "#" and (i == 0 or line[i - 1] in " \t"): + return line[:i].rstrip() + return line + + +def _bracket_depth(t): + d, q = 0, None + for ch in t: + if q: + if ch == q: + q = None + elif ch in "\"'": + q = ch + elif ch in "[{": + d += 1 + elif ch in "]}": + d -= 1 + return d + + +def mini_yaml(text): + raw = text.split("\n") + lines = [] # (indent, content, original index) + i = 0 + while i < len(raw): + line = raw[i] + if not line.strip() or line.lstrip().startswith("#") or line.startswith("---"): + i += 1 + continue + indent = len(line) - len(line.lstrip(" ")) + content = _strip_comment(line.strip()) + # multi-line flow collection: join until the brackets balance + if re.match(r"^(?:(?:[^\s\[{\"'][^:]*?|\S):\s+|-\s+)?[\[{]", content) and not re.match(r"^.*?:\s+[|>]", content): + while _bracket_depth(content) > 0 and i + 1 < len(raw): + i += 1 + content += " " + _strip_comment(raw[i].strip()) + m = re.match(r"^(.*?:|-)\s+[|>][-+]?\d*$", content) or re.match(r"^()[|>][-+]?\d*$", content) + lines.append([indent, content, None]) + if m: + j = i + 1 + block = [] + while j < len(raw) and (not raw[j].strip() or len(raw[j]) - len(raw[j].lstrip(" ")) > indent): + block.append(raw[j].strip()) + j += 1 + lines[-1][2] = "\n".join(block) + i = j + continue + i += 1 + pos = [0] + + def parse(indent): + if pos[0] >= len(lines): + return None + ind, content, _ = lines[pos[0]] + if content[0] in "[{": + pos[0] += 1 + return _scalar(content) + if content.startswith("- ") or content == "-": + return parse_seq(ind) + return parse_map(ind) + + def split_key(content): + if content[0] in "\"'": + q = content[0] + end = content.index(q, 1) + rest = content[end + 1:] + if rest.startswith(":"): + return content[1:end], rest[1:].strip() + m = re.match(r"^([^\s\[{][^:]*?|\S):(?:\s+(.*))?$", content) + if m: + return m.group(1), (m.group(2) or "").strip() + return None, None + + anchors = {} + + def value_after(key_indent, rest, block): + if block is not None: + return block + if rest.startswith("*"): + return anchors.get(rest[1:].strip()) + if rest.startswith("&"): + name, _, rest = rest[1:].partition(" ") + val = value_after(key_indent, rest.strip(), None) + anchors[name] = val + return val + if rest != "": + return _scalar(rest) + if pos[0] < len(lines): + nind, ncontent, _ = lines[pos[0]] + if nind > key_indent or (nind == key_indent and (ncontent.startswith("- ") or ncontent == "-")): + return parse(nind) + return None + + def parse_map(ind): + out = {} + while pos[0] < len(lines): + cind, content, block = lines[pos[0]] + if cind != ind or content.startswith("- "): + break + key, rest = split_key(content) + if key is None: + raise ValueError("yaml: cannot parse line: " + content) + pos[0] += 1 + if block is not None: + rest = "" + out[key] = value_after(ind, rest, block) + return out + + def parse_seq(ind): + out = [] + while pos[0] < len(lines): + cind, content, block = lines[pos[0]] + if cind != ind or not (content.startswith("- ") or content == "-"): + break + item = content[1:].strip() + if item == "": + pos[0] += 1 + out.append(parse(ind + 1) if pos[0] < len(lines) and lines[pos[0]][0] > ind else None) + continue + key, rest = split_key(item) + if key is not None and not item.startswith(("[", "{")): + # inline mapping start: treat "- k: v" as a map at indent ind+2 + lines[pos[0]] = [ind + 2, item, block] + out.append(parse_map(ind + 2)) + else: + pos[0] += 1 + out.append(block if block is not None else _scalar(item)) + return out + + return parse(0) + + +# -------------------------------------------------------------------------- + + +def fetch(url): + req = urllib.request.Request(url, headers={"User-Agent": "molo-retrieval-spec-check/1"}) + with urllib.request.urlopen(req, timeout=90) as r: + return r.read().decode("utf-8", "replace") + + +_cache = {} + + +def load_doc(url): + if url not in _cache: + text = fetch(url) + _cache[url] = json.loads(text) if url.endswith(".json") else mini_yaml(text) + return _cache[url] + + +def deref(doc, node): + seen = 0 + while isinstance(node, dict) and "$ref" in node and seen < 20: + ref = node["$ref"] + cur = doc + for part in ref.lstrip("#/").split("/"): + cur = cur[part.replace("~1", "/").replace("~0", "~")] + node = cur + seen += 1 + return node + + +def props(doc, schema, depth=0): + """Top-level property names a schema accepts ('*' when free-form).""" + schema = deref(doc, schema) + if not isinstance(schema, dict) or depth > 8: + return set() + out = set() + for key in ("allOf", "anyOf", "oneOf"): + for sub in schema.get(key, []) or []: + out |= props(doc, sub, depth + 1) + out |= set((schema.get("properties") or {}).keys()) + if schema.get("type") == "array": + out.add("*") + if not out and (schema.get("additionalProperties") is True or schema.get("type") == "object" and not schema.get("properties")): + out.add("*") + return out + + +def response_name(op): + for code in ("200", "201", "202", 200, 201, 202): + r = (op.get("responses") or {}).get(code) + if isinstance(r, dict): + content = (r.get("content") or {}) + for ct, body in content.items(): + sch = (body or {}).get("schema") or {} + if "$ref" in sch: + return sch["$ref"].split("/")[-1] + if sch.get("type"): + return "inline" + if r.get("schema", {}).get("$ref"): + return r["schema"]["$ref"].split("/")[-1] + return "-" + return "-" + + +def spec_ops(provider): + """{ 'host METHOD path': dict(op_id, body:set, query:set, response) }.""" + ops = {} + for host, url in SOURCES[provider]: + doc = load_doc(url) + for path, item in doc["paths"].items(): + common = item.get("parameters", []) or [] + for method, op in item.items(): + if method not in ("get", "post", "put", "patch", "delete"): + continue + params = [deref(doc, p) for p in common + (op.get("parameters") or [])] + query = {p["name"] for p in params if p.get("in") == "query"} + headers = {p["name"] for p in params if p.get("in") == "header"} + body = set() + rb = deref(doc, op.get("requestBody") or {}) + for ct, c in (rb.get("content") or {}).items(): + body |= props(doc, (c or {}).get("schema") or {}) + for p in params: # swagger 2 + if p.get("in") == "body": + body |= props(doc, p.get("schema") or {}) + if p.get("in") == "formData": + body.add(p["name"]) + ops["%s %s %s" % (host, method.upper(), path)] = { + "op_id": op.get("operationId") or "", + "body": body, "query": query, "headers": headers, + "response": response_name(op), + } + return ops + + +# -------------------------------------------------------------------------- +# Call tables (parsed from the C++ sources) + +STR = re.compile(r'"((?:[^"\\]|\\.)*)"') + + +def table_rows(provider): + path = os.path.join(CALLS_DIR, provider, "calls.cpp") + text = open(path).read() + a, b = text.index("CALLS-BEGIN"), text.index("CALLS-END") + rows = [] + for line in text[a:b].splitlines()[1:]: + s = line.strip() + if s.startswith("{") and '"' in s: + f = STR.findall(s) + rows.append(dict(zip(("name", "host", "method", "path", "op", "body", "query", "flags", "resp"), f))) + return rows + + +def split(csv): + return {x for x in csv.split(",") if x} + + +def snake(s): + s = re.sub(r"[^A-Za-z0-9]+", "_", s) + s = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s) + return re.sub(r"_+", "_", s).strip("_").lower() + + +READ_POST = re.compile(r"find_similar|contents|graphql|tokenize|aggregate|mcp_post|validate|search|query|scroll|count|fetch|list|recommend|discover|facet|matrix|retrieve|describe|get|embed|rerank|grep|read|extract|crawl|map|answer|enhance|validate|segment|classify|suggest|context|completions|stats") +DESTRUCTIVE = re.compile(r"delete|remove|clear|recover|drop|truncate|reset|revoke|cancel|abort|disconnect|purge") +PAID = { + "voyage": re.compile(r".*"), + "jina": re.compile(r"embeddings|rerank|read|search|segment|classify|count_token|batch_embeddings|create_batch"), + "pinecone": re.compile(r"embed|rerank|search_records|upsert_records"), + "mixedbread": re.compile(r"^(create_embeddings|rerank|create_question_answering|deprecated_create_question_answering|search_store_chunks|search_vector_store_chunks|enhance_query|create_parsing_job|create_extraction_job|extract_content|create_chat_completion|create_response)$"), + "tavily": re.compile(r"search|extract|crawl|map|research"), + "exa": re.compile(r"search|find_similar|contents|answer|research"), + "brave": re.compile(r".*"), +} + + +RENAMES = { + "jina": { + "embeddings_v1_embeddings_post": "embeddings", "rerank_v1_rerank_post": "rerank", + "list_models_v1_models_get": "list_models", "get_model_v1_models_model_id_get": "get_model", + "create_batch_job_v1_batch_embeddings_post": "create_batch", + "get_batch_status_v1_batch_batch_id_get": "get_batch", + "cancel_batch_job_v1_batch_batch_id_delete": "cancel_batch", + "list_batch_jobs_v1_batches_get": "list_batches", + "download_batch_output_v1_batch_batch_id_output_get": "batch_output", + "download_batch_errors_v1_batch_batch_id_errors_get": "batch_errors", + "get": "read", "get_search": "search", "post_count_token": "count_token", + }, + "exa": { + "get_contents": "contents", "research_tasks_create": "research_create", + "research_tasks_list": "research_list", "research_controller_v0_get_research_task": "research_get", + }, + "pinecone": { + "list_namespaces_operation": "list_namespaces", + "create_index_from_backup_operation": "create_index_from_backup", + }, +} + + +def op_name(provider, host, method, path, op_id): + base = snake(op_id) if op_id else snake(method + "_" + path) + return RENAMES.get(provider, {}).get(base, base) + + +def make_flags(provider, name, method): + f = "" + if method == "DELETE" or DESTRUCTIVE.search(name): + f += "d" + elif method != "GET": + f += "r" if READ_POST.search(name) else "w" + if PAID.get(provider) and PAID[provider].search(name) and (method not in ("GET", "DELETE") or provider == "jina"): + f += "p" + return f + + +def excluded(provider, key): + for rx, why in EXCLUDED.get(provider, []): + if re.search(rx, key): + return why + return None + + +def gen_rows(provider): + ops = spec_ops(provider) + seen = set() + out = [] + for key, o in ops.items(): + if excluded(provider, key): + continue + host, method, path = key.split(" ", 2) + name = op_name(provider, host, method, path, o["op_id"]) + while name in seen: + name += "_" + method.lower() + seen.add(name) + body = ",".join(sorted(o["body"])) + out.append(' {"%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s"},' % ( + name, host, method, path, o["op_id"], body, ",".join(sorted(o["query"])), make_flags(provider, name, method), o["response"])) + return out + + +# -------------------------------------------------------------------------- + + +def check_openapi(provider, errors, notes): + ops = spec_ops(provider) + rows = table_rows(provider) + keys = set() + unverified = 0 + for r in rows: + if r["op"].startswith("doc:"): + unverified += 1 + notes.append("%s: unverified (prose docs only): %s %s %s" % (provider, r["name"], r["method"], r["path"])) + continue + key = "%s %s %s" % (r["host"], r["method"], r["path"]) + keys.add(key) + o = ops.get(key) + if not o: + errors.append("%s row %s: %s not in the spec" % (provider, r["name"], key)) + continue + if o["op_id"] != r["op"]: + errors.append("%s row %s: operationId %r, spec says %r" % (provider, r["name"], r["op"], o["op_id"])) + if r["body"] != "*": + extra = split(r["body"]) - o["body"] - ({"*"} if "*" in o["body"] else set()) + if extra and "*" not in o["body"]: + errors.append("%s row %s: body fields not in spec: %s" % (provider, r["name"], sorted(extra))) + extra = split(r["query"]) - o["query"] + if extra: + errors.append("%s row %s: query parameters not in spec: %s" % (provider, r["name"], sorted(extra))) + for key in ops: + if key not in keys and not excluded(provider, key): + errors.append("%s coverage: spec operation %s is neither in the table nor excluded" % (provider, key)) + notes.append("%s: %d spec operations, %d table rows (%d unverified), %d excluded" % ( + provider, len(ops), len(rows), unverified, sum(1 for k in ops if excluded(provider, k)))) + + +def check_brave(errors, notes): + rows = table_rows("brave") + html = fetch(BRAVE + "/api-reference/web/local_pois") + slugs = sorted(set(re.findall(r'href="/api-reference/([^"#?]+)"', html))) + covered = set() + for r in rows: + slug = r["op"] + covered.add(slug) + if slug not in slugs: + errors.append("brave row %s: reference page %s not listed" % (r["name"], slug)) + continue + page = fetch("%s/api-reference/%s/index.html.md" % (BRAVE, slug)) + m = re.search(r"`(GET|POST)` `([^`]+)`", page) + if not m or m.group(1) != r["method"] or m.group(2) != r["path"]: + errors.append("brave row %s: page says %s" % (r["name"], m.groups() if m else None)) + tables = {} + cur = None + for line in page.splitlines(): + if line.startswith("## "): + cur = line[3:].strip() + elif line.startswith("| `") and cur in ("Query Parameters", "Request Body", "Headers"): + tables.setdefault(cur, set()).add(line.split("`")[1]) + for f in split(r["query"]): + if f not in tables.get("Query Parameters", set()): + errors.append("brave row %s: query parameter %s not on page" % (r["name"], f)) + if r["body"] != "*": + for f in split(r["body"]): + if f not in tables.get("Request Body", set()): + errors.append("brave row %s: body field %s not on page" % (r["name"], f)) + for s in slugs: + if s not in covered: + errors.append("brave coverage: reference page %s is not in the table" % s) + notes.append("brave: %d reference pages, %d table rows" % (len(slugs), len(rows))) + + +def check_voyage(errors, notes): + meta = json.loads(fetch("https://pypi.org/pypi/voyageai/json")) + wheel = next(f for f in meta["urls"] if f["filename"].endswith(".whl")) + with urllib.request.urlopen(urllib.request.Request(wheel["url"], headers={"User-Agent": "molo/1"}), timeout=90) as r: + z = zipfile.ZipFile(io.BytesIO(r.read())) + src = {n: z.read(n).decode("utf-8", "replace") for n in z.namelist() if n.endswith(".py")} + blob = "\n".join(src.values()) + resources = set(re.findall(r'OBJECT_NAME = "([a-z]+)"', blob)) + base = re.search(r'"(https://api\.voyageai\.com/v1)"', blob) + rows = table_rows("voyage") + if not base: + errors.append("voyage: base URL not found in the SDK") + for r in rows: + if r["method"] != "POST" or r["path"].lstrip("/").replace("v1/", "") not in resources: + errors.append("voyage row %s: resource %s not in SDK resources %s" % (r["name"], r["path"], sorted(resources))) + for f in split(r["body"]): + if not re.search(r"\b%s\b" % re.escape(f), blob): + errors.append("voyage row %s: field %s not in the SDK source" % (r["name"], f)) + missing = resources - {r["path"].replace("/v1/", "").lstrip("/") for r in rows} + for m in missing: + errors.append("voyage coverage: SDK resource %s not in the table" % m) + notes.append("voyage: SDK voyageai %s, resources %s, %d table rows" % (meta["info"]["version"], sorted(resources), len(rows))) + + +def selftest(): + try: + import yaml # noqa + except ImportError: + print("selftest: PyYAML absent, skipped") + return 0 + bad = 0 + for provider in ("pinecone", "exa"): + for host, url in SOURCES[provider]: + text = fetch(url) + mine = mini_yaml(text) + ref = yaml.safe_load(text) + def strip(x): # examples are free text; only the structure is compared + if isinstance(x, dict): + return {k: strip(v) for k, v in x.items() if k not in ("example", "examples", "x-codeSamples", "description", "default")} + if isinstance(x, list): + return [strip(v) for v in x] + return x + same = json.dumps(strip(mine["paths"]), sort_keys=True, default=str) == json.dumps(strip(ref["paths"]), sort_keys=True, default=str) + print("selftest", provider, host, "paths equal" if same else "paths DIFFER") + bad += 0 if same else 1 + return bad + + +def main(argv): + if argv and argv[0] == "--rows": + print("\n".join(gen_rows(argv[1]))) + return 0 + if argv and argv[0] == "--selftest": + return 1 if selftest() else 0 + wanted = argv or ["qdrant", "weaviate", "pinecone", "mixedbread", "jina", "tavily", "exa", "brave", "voyage"] + errors, notes = [], [] + for p in wanted: + if p == "brave": + check_brave(errors, notes) + elif p == "voyage": + check_voyage(errors, notes) + else: + check_openapi(p, errors, notes) + for n in notes: + print("note:", n) + for e in errors: + print("FAIL:", e) + print("check_spec: %d problems" % len(errors)) + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/src/vendors/retrieval/voyage/calls.cpp b/src/vendors/retrieval/voyage/calls.cpp new file mode 100644 index 0000000..55541cf --- /dev/null +++ b/src/vendors/retrieval/voyage/calls.cpp @@ -0,0 +1,22 @@ +#include "vendors/retrieval/voyage/voyage.h" + +namespace molo::retrieval { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// name, host, http method, path, spec operation id (or reference page / "doc:"), +// body fields, query parameters, flags (d destructive, w writes, r read-only POST, +// p paid), response schema) +static const CallRow kRows[] = { + {"embed", "main", "POST", "/v1/embeddings", "sdk:Embedding", "input,model,input_type,truncation,output_dimension,output_dtype,encoding_format", "", "p", "-"}, + {"multimodal_embed", "main", "POST", "/v1/multimodalembeddings", "sdk:MultimodalEmbedding", "inputs,model,input_type,truncation,output_dimension,output_dtype", "", "p", "-"}, + {"contextualized_embed", "main", "POST", "/v1/contextualizedembeddings", "sdk:ContextualizedEmbedding", "inputs,model,input_type,output_dimension,output_dtype,encoding_format", "", "p", "-"}, + {"rerank", "main", "POST", "/v1/rerank", "sdk:Reranking", "query,documents,model,top_k,truncation", "", "p", "-"}, +}; +// CALLS-END + +const CallTable& voyage_calls() { + static const CallTable table{kRows, sizeof(kRows) / sizeof(kRows[0])}; + return table; +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/voyage/voyage.cpp b/src/vendors/retrieval/voyage/voyage.cpp new file mode 100644 index 0000000..9e535a7 --- /dev/null +++ b/src/vendors/retrieval/voyage/voyage.cpp @@ -0,0 +1,169 @@ +#include "vendors/retrieval/voyage/voyage.h" + +#include + +#include "vendors/retrieval/caps.h" + +namespace molo::retrieval { + +llm::ApiClientConfig Voyage::config() { + llm::ApiClientConfig c; + c.vendor = "voyage"; + c.base_url = kBaseUrl; + c.key_env = kKeyEnv; + c.auth = llm::AuthStyle::kBearer; + return c; +} + +Voyage Voyage::from_env() { + return Voyage(make_gateway(voyage_calls(), config(), {{"main", kBaseUrl}}, "", nullptr)); +} + +Voyage Voyage::with(const TransportFactory& factory, std::string key) { + return Voyage(make_gateway(voyage_calls(), config(), {{"main", kBaseUrl}}, std::move(key), factory)); +} + +EmbedResult Voyage::parse_embeddings(const Json& doc) { + EmbedResult out; + out.model = J::str(doc, "model"); + std::vector>> rows; + for (const Json& d : J::array(doc, "data")) { + const Json* e = J::at(d, "embedding"); + rows.emplace_back(static_cast(J::num(d, "index", static_cast(rows.size()))), + e ? J::floats(*e) : std::vector{}); + } + std::stable_sort(rows.begin(), rows.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); + for (auto& r : rows) out.vectors.push_back(std::move(r.second)); + if (const Json* u = J::at(doc, "usage")) out.total_tokens = static_cast(J::num(*u, "total_tokens")); + return out; +} + +RerankResult Voyage::parse_rerank(const Json& doc) { + RerankResult out; + out.model = J::str(doc, "model"); + for (const Json& d : J::array(doc, "data")) { + RerankHit h; + h.index = static_cast(J::num(d, "index")); + h.score = J::num(d, "relevance_score"); + out.hits.push_back(h); + } + std::stable_sort(out.hits.begin(), out.hits.end(), [](const RerankHit& a, const RerankHit& b) { return a.score > b.score; }); + if (const Json* u = J::at(doc, "usage")) out.total_tokens = static_cast(J::num(*u, "total_tokens")); + return out; +} + +VoyageContextualResult Voyage::parse_contextual(const Json& doc) { + VoyageContextualResult out; + out.model = J::str(doc, "model"); + for (const Json& d : J::array(doc, "data")) { + std::vector> chunks; + for (const Json& c : J::array(d, "data")) { + const Json* e = J::at(c, "embedding"); + chunks.push_back(e ? J::floats(*e) : std::vector{}); + } + out.embeddings.push_back(std::move(chunks)); + } + if (const Json* u = J::at(doc, "usage")) out.total_tokens = static_cast(J::num(*u, "total_tokens")); + return out; +} + +EmbedResult Voyage::embed(const EmbedRequest& request) { + Embedder::validate("voyage", request); + Json body = Json::object(); + body.set("input", J::strings(request.texts)); + body.set("model", Json(request.model.empty() ? std::string(kDefaultEmbedModel) : request.model)); + if (!request.input_type.empty()) body.set("input_type", Json(request.input_type)); + if (request.dimensions > 0) body.set("output_dimension", Json(request.dimensions)); + CallArgs args; + args.body = body; + EmbedResult r = parse_embeddings(gw_->call("embed", args)); + if (r.vectors.size() != request.texts.size()) + throw llm::ApiError("voyage", 502, "BAD_RESPONSE", "embedding count differs from input count"); + return r; +} + +RerankResult Voyage::rerank(const RerankRequest& request) { + Reranker::validate("voyage", request); + Json body = Json::object(); + body.set("query", Json(request.query)); + body.set("documents", J::strings(request.documents)); + body.set("model", Json(request.model.empty() ? std::string(kDefaultRerankModel) : request.model)); + if (request.top_n > 0) body.set("top_k", Json(request.top_n)); + CallArgs args; + args.body = body; + return parse_rerank(gw_->call("rerank", args)); +} + +EmbedResult Voyage::multimodal_embed(const std::vector>& inputs, + const std::string& model, const std::string& input_type) { + if (inputs.empty() || inputs.size() > 1000) throw llm::ApiError::invalid("voyage", "inputs must hold 1..1000 entries"); + if (!input_type.empty() && input_type != "query" && input_type != "document") + throw llm::ApiError::invalid("voyage", "input_type must be query or document"); + Json arr = Json::array(); + for (const auto& segments : inputs) { + if (segments.empty() || segments.size() > 64) throw llm::ApiError::invalid("voyage", "an input needs 1..64 segments"); + Json content = Json::array(); + for (const VoyageSegment& s : segments) { + if (s.value.empty() || s.value.size() > RetrievalCaps::kMaxBodyBytes) + throw llm::ApiError::invalid("voyage", "a segment is empty or too large"); + Json seg = Json::object(); + switch (s.kind) { + case VoyageSegment::Kind::kText: + seg.set("type", Json("text")); + seg.set("text", Json(s.value)); + break; + case VoyageSegment::Kind::kImageUrl: + seg.set("type", Json("image_url")); + seg.set("image_url", Json(s.value)); + break; + case VoyageSegment::Kind::kImageBase64: + seg.set("type", Json("image_base64")); + seg.set("image_base64", Json(s.value)); + break; + case VoyageSegment::Kind::kVideoUrl: + seg.set("type", Json("video_url")); + seg.set("video_url", Json(s.value)); + break; + case VoyageSegment::Kind::kVideoBase64: + seg.set("type", Json("video_base64")); + seg.set("video_base64", Json(s.value)); + break; + } + content.push(seg); + } + Json item = Json::object(); + item.set("content", content); + arr.push(item); + } + Json body = Json::object(); + body.set("inputs", arr); + body.set("model", Json(model.empty() ? std::string(kDefaultMultimodalModel) : model)); + if (!input_type.empty()) body.set("input_type", Json(input_type)); + CallArgs args; + args.body = body; + return parse_embeddings(gw_->call("multimodal_embed", args)); +} + +VoyageContextualResult Voyage::contextualized_embed(const std::vector>& documents, + const std::string& model, const std::string& input_type, + int dimensions) { + if (documents.empty() || documents.size() > 1000) throw llm::ApiError::invalid("voyage", "documents must hold 1..1000 entries"); + if (!input_type.empty() && input_type != "query" && input_type != "document") + throw llm::ApiError::invalid("voyage", "input_type must be query or document"); + if (dimensions < 0 || dimensions > RetrievalCaps::kMaxDimensions) throw llm::ApiError::invalid("voyage", "dimensions out of range"); + Json inputs = Json::array(); + for (const auto& chunks : documents) { + if (chunks.empty() || chunks.size() > 16000) throw llm::ApiError::invalid("voyage", "a document needs 1..16000 chunks"); + inputs.push(J::strings(chunks)); + } + Json body = Json::object(); + body.set("inputs", inputs); + body.set("model", Json(model.empty() ? std::string(kDefaultContextModel) : model)); + if (!input_type.empty()) body.set("input_type", Json(input_type)); + if (dimensions > 0) body.set("output_dimension", Json(dimensions)); + CallArgs args; + args.body = body; + return parse_contextual(gw_->call("contextualized_embed", args)); +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/voyage/voyage.h b/src/vendors/retrieval/voyage/voyage.h new file mode 100644 index 0000000..72f0ede --- /dev/null +++ b/src/vendors/retrieval/voyage/voyage.h @@ -0,0 +1,65 @@ +#pragma once + +// Voyage AI: embeddings, multimodal embeddings, contextualized chunk +// embeddings and rerank. Implements the Embedder and Reranker seams. +// Source: the official voyageai Python SDK (Voyage publishes no OpenAPI +// document), see SPEC.md. + +#include +#include + +#include "vendors/retrieval/provider.h" +#include "vendors/retrieval/seams.h" + +namespace molo::retrieval { + +const CallTable& voyage_calls(); + +// One multimodal input is a list of segments (text, image URL, image base64, +// video URL, video base64), embedded together as one vector. +struct VoyageSegment { + enum class Kind { kText, kImageUrl, kImageBase64, kVideoUrl, kVideoBase64 }; + Kind kind = Kind::kText; + std::string value; +}; + +struct VoyageContextualResult { + // embeddings[document][chunk] + std::vector>> embeddings; + std::string model; + long total_tokens = 0; +}; + +class Voyage : public Provider, public Embedder, public Reranker { + public: + static constexpr const char* kKeyEnv = "VOYAGE_API_KEY"; + static constexpr const char* kBaseUrl = "https://api.voyageai.com"; + static constexpr const char* kDefaultEmbedModel = "voyage-3.5"; + static constexpr const char* kDefaultRerankModel = "rerank-2.5"; + static constexpr const char* kDefaultMultimodalModel = "voyage-multimodal-3"; + static constexpr const char* kDefaultContextModel = "voyage-context-3"; + + static llm::ApiClientConfig config(); + static Voyage from_env(); + static Voyage with(const TransportFactory& factory, std::string key); + explicit Voyage(std::shared_ptr gateway) : Provider(std::move(gateway)) {} + + std::string provider() const override { return "voyage"; } + EmbedResult embed(const EmbedRequest& request) override; + RerankResult rerank(const RerankRequest& request) override; + + EmbedResult multimodal_embed(const std::vector>& inputs, + const std::string& model, const std::string& input_type); + // Each document is the list of its chunks; the model embeds chunks in the + // context of the whole document. + VoyageContextualResult contextualized_embed(const std::vector>& documents, + const std::string& model, const std::string& input_type, + int dimensions); + + // Pure parsers (public so tests can run them on documented shapes). + static EmbedResult parse_embeddings(const Json& doc); + static RerankResult parse_rerank(const Json& doc); + static VoyageContextualResult parse_contextual(const Json& doc); +}; + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/weaviate/calls.cpp b/src/vendors/retrieval/weaviate/calls.cpp new file mode 100644 index 0000000..5f36ce3 --- /dev/null +++ b/src/vendors/retrieval/weaviate/calls.cpp @@ -0,0 +1,84 @@ +#include "vendors/retrieval/weaviate/weaviate.h" + +namespace molo::retrieval { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// name, host, http method, path, spec operation id (or reference page / "doc:"), +// body fields, query parameters, flags (d destructive, w writes, r read-only POST, +// p paid), response schema) +static const CallRow kRows[] = { + {"objects_list", "main", "GET", "/objects", "objects.list", "", "after,class,include,limit,offset,order,sort,tenant", "", "ObjectsListResponse"}, + {"objects_create", "main", "POST", "/objects", "objects.create", "additional,class,creationTimeUnix,id,lastUpdateTimeUnix,properties,tenant,vector,vectorWeights,vectors", "consistency_level", "w", "Object"}, + {"objects_delete", "main", "DELETE", "/objects/{id}", "objects.delete", "", "consistency_level,tenant", "d", "-"}, + {"objects_get", "main", "GET", "/objects/{id}", "objects.get", "", "include", "", "Object"}, + {"objects_patch", "main", "PATCH", "/objects/{id}", "objects.patch", "additional,class,creationTimeUnix,id,lastUpdateTimeUnix,properties,tenant,vector,vectorWeights,vectors", "consistency_level", "w", "-"}, + {"objects_update", "main", "PUT", "/objects/{id}", "objects.update", "additional,class,creationTimeUnix,id,lastUpdateTimeUnix,properties,tenant,vector,vectorWeights,vectors", "consistency_level", "w", "Object"}, + {"objects_class_get", "main", "GET", "/objects/{className}/{id}", "objects.class.get", "", "consistency_level,include,node_name,tenant", "", "Object"}, + {"objects_class_delete", "main", "DELETE", "/objects/{className}/{id}", "objects.class.delete", "", "consistency_level,tenant", "d", "-"}, + {"objects_class_put", "main", "PUT", "/objects/{className}/{id}", "objects.class.put", "additional,class,creationTimeUnix,id,lastUpdateTimeUnix,properties,tenant,vector,vectorWeights,vectors", "consistency_level", "w", "Object"}, + {"objects_class_patch", "main", "PATCH", "/objects/{className}/{id}", "objects.class.patch", "additional,class,creationTimeUnix,id,lastUpdateTimeUnix,properties,tenant,vector,vectorWeights,vectors", "consistency_level", "w", "-"}, + {"objects_references_create", "main", "POST", "/objects/{id}/references/{propertyName}", "objects.references.create", "beacon,class,classification,href,schema", "tenant", "w", "-"}, + {"objects_references_update", "main", "PUT", "/objects/{id}/references/{propertyName}", "objects.references.update", "*", "tenant", "w", "-"}, + {"objects_references_delete", "main", "DELETE", "/objects/{id}/references/{propertyName}", "objects.references.delete", "beacon,class,classification,href,schema", "tenant", "d", "-"}, + {"objects_class_references_create", "main", "POST", "/objects/{className}/{id}/references/{propertyName}", "objects.class.references.create", "beacon,class,classification,href,schema", "consistency_level,tenant", "w", "-"}, + {"objects_class_references_put", "main", "PUT", "/objects/{className}/{id}/references/{propertyName}", "objects.class.references.put", "*", "consistency_level,tenant", "w", "-"}, + {"objects_class_references_delete", "main", "DELETE", "/objects/{className}/{id}/references/{propertyName}", "objects.class.references.delete", "beacon,class,classification,href,schema", "consistency_level,tenant", "d", "-"}, + {"objects_validate", "main", "POST", "/objects/validate", "objects.validate", "additional,class,creationTimeUnix,id,lastUpdateTimeUnix,properties,tenant,vector,vectorWeights,vectors", "", "r", "-"}, + {"batch_objects_create", "main", "POST", "/batch/objects", "batch.objects.create", "fields,objects", "consistency_level", "w", "-"}, + {"batch_objects_delete", "main", "DELETE", "/batch/objects", "batch.objects.delete", "deletionTimeUnixMilli,dryRun,match,output", "consistency_level,tenant", "d", "BatchDeleteResponse"}, + {"batch_references_create", "main", "POST", "/batch/references", "batch.references.create", "*", "consistency_level", "w", "-"}, + {"graphql_post", "main", "POST", "/graphql", "graphql.post", "operationName,query,variables", "", "r", "GraphQLResponse"}, + {"graphql_batch", "main", "POST", "/graphql/batch", "graphql.batch", "*", "", "r", "GraphQLResponses"}, + {"tokenize", "main", "POST", "/tokenize", "tokenize", "analyzerConfig,stopwordPresets,stopwords,text,tokenization", "", "r", "TokenizeResponse"}, + {"schema_dump", "main", "GET", "/schema", "schema.dump", "", "", "", "Schema"}, + {"schema_objects_create", "main", "POST", "/schema", "schema.objects.create", "class,description,invertedIndexConfig,moduleConfig,multiTenancyConfig,objectTtlConfig,properties,replicationConfig,shardingConfig,vectorConfig,vectorIndexConfig,vectorIndexType,vectorizer", "", "w", "Class"}, + {"schema_objects_get", "main", "GET", "/schema/{className}", "schema.objects.get", "", "", "", "Class"}, + {"schema_objects_delete", "main", "DELETE", "/schema/{className}", "schema.objects.delete", "", "", "d", "-"}, + {"schema_objects_update", "main", "PUT", "/schema/{className}", "schema.objects.update", "class,description,invertedIndexConfig,moduleConfig,multiTenancyConfig,objectTtlConfig,properties,replicationConfig,shardingConfig,vectorConfig,vectorIndexConfig,vectorIndexType,vectorizer", "", "w", "Class"}, + {"schema_objects_properties_add", "main", "POST", "/schema/{className}/properties", "schema.objects.properties.add", "bucketGeneration,dataType,description,disableDuplicatedReferences,indexFilterable,indexInverted,indexRangeFilters,indexSearchable,moduleConfig,name,nestedProperties,searchableBlockmax,textAnalyzer,tokenization", "", "w", "Property"}, + {"schema_objects_indexes_get", "main", "GET", "/schema/{className}/indexes", "schema.objects.indexes.get", "", "", "", "IndexStatusResponse"}, + {"schema_objects_index_upsert", "main", "PUT", "/schema/{className}/properties/{propertyName}/index/{indexName}", "schema.objects.index.upsert", "algorithm,tokenization", "tenants", "w", "IndexUpdateResponse"}, + {"schema_objects_properties_delete", "main", "DELETE", "/schema/{className}/properties/{propertyName}/index/{indexName}", "schema.objects.properties.delete", "", "", "d", "-"}, + {"schema_objects_index_rebuild", "main", "POST", "/schema/{className}/properties/{propertyName}/index/{indexName}/rebuild", "schema.objects.index.rebuild", "", "tenants", "w", "IndexUpdateResponse"}, + {"schema_objects_index_cancel", "main", "POST", "/schema/{className}/properties/{propertyName}/index/{indexName}/cancel", "schema.objects.index.cancel", "", "", "d", "IndexUpdateResponse"}, + {"schema_objects_properties_tokenize", "main", "POST", "/schema/{className}/properties/{propertyName}/tokenize", "schema.objects.properties.tokenize", "text", "", "r", "TokenizeResponse"}, + {"schema_objects_vectors_delete", "main", "DELETE", "/schema/{className}/vectors/{vectorIndexName}/index", "schema.objects.vectors.delete", "", "", "d", "-"}, + {"schema_objects_shards_get", "main", "GET", "/schema/{className}/shards", "schema.objects.shards.get", "", "tenant", "", "ShardStatusList"}, + {"schema_objects_shards_update", "main", "PUT", "/schema/{className}/shards/{shardName}", "schema.objects.shards.update", "status", "", "w", "ShardStatus"}, + {"tenants_create", "main", "POST", "/schema/{className}/tenants", "tenants.create", "*", "", "w", "-"}, + {"tenants_update", "main", "PUT", "/schema/{className}/tenants", "tenants.update", "*", "", "w", "-"}, + {"tenants_delete", "main", "DELETE", "/schema/{className}/tenants", "tenants.delete", "*", "", "d", "-"}, + {"tenants_get", "main", "GET", "/schema/{className}/tenants", "tenants.get", "", "", "", "-"}, + {"tenants_get_one", "main", "GET", "/schema/{className}/tenants/{tenantName}", "tenants.get.one", "", "", "", "Tenant"}, + {"aliases_get", "main", "GET", "/aliases", "aliases.get", "", "class", "", "AliasResponse"}, + {"aliases_create", "main", "POST", "/aliases", "aliases.create", "alias,class", "", "w", "Alias"}, + {"aliases_get_alias", "main", "GET", "/aliases/{aliasName}", "aliases.get.alias", "", "", "", "Alias"}, + {"aliases_update", "main", "PUT", "/aliases/{aliasName}", "aliases.update", "class", "", "w", "Alias"}, + {"aliases_delete", "main", "DELETE", "/aliases/{aliasName}", "aliases.delete", "", "", "d", "-"}, + {"list_namespaces", "main", "GET", "/namespaces", "listNamespaces", "", "", "", "NamespaceListResponse"}, + {"create_namespace", "main", "POST", "/namespaces/{namespace_id}", "createNamespace", "home_node", "", "w", "Namespace"}, + {"get_namespace", "main", "GET", "/namespaces/{namespace_id}", "getNamespace", "", "", "", "Namespace"}, + {"update_namespace", "main", "PUT", "/namespaces/{namespace_id}", "updateNamespace", "home_node", "", "w", "Namespace"}, + {"delete_namespace", "main", "DELETE", "/namespaces/{namespace_id}", "deleteNamespace", "", "", "d", "-"}, + {"suspend_namespace", "main", "POST", "/namespaces/{namespace_id}/suspend", "suspendNamespace", "", "", "w", "-"}, + {"resume_namespace", "main", "POST", "/namespaces/{namespace_id}/resume", "resumeNamespace", "", "", "w", "-"}, + {"export_create", "main", "POST", "/export/{backend}", "export.create", "exclude,file_format,id,include", "", "w", "ExportCreateResponse"}, + {"export_status", "main", "GET", "/export/{backend}/{id}", "export.status", "", "", "", "ExportStatusResponse"}, + {"export_cancel", "main", "DELETE", "/export/{backend}/{id}", "export.cancel", "", "", "d", "-"}, + {"mcp_post", "main", "POST", "/mcp", "mcp.post", "", "", "r", "-"}, + {"mcp_delete", "main", "DELETE", "/mcp", "mcp.delete", "", "", "d", "-"}, + {"aggregate", "main", "POST", "/aggregate/{collection}", "aggregate", "groupBy,limit,objectLimit,over,returnMetrics,tenant,where", "", "r", "AggregateResponse"}, + {"search_bm25", "main", "POST", "/search/{collection}/bm25", "search.bm25", "autoLimit,consistencyLevel,groupBy,groupedTask,limit,numberOfGroups,objectsPerGroup,offset,query,queryProperties,rerank,returnMetadata,returnProperties,returnReferences,singlePrompt,tenant,where", "", "r", "SearchResponse"}, + {"search_hybrid", "main", "POST", "/search/{collection}/hybrid", "search.hybrid", "alpha,autoLimit,consistencyLevel,fusionType,groupBy,groupedTask,limit,maxVectorDistance,numberOfGroups,objectsPerGroup,offset,query,queryProperties,rerank,returnMetadata,returnProperties,returnReferences,singlePrompt,targetVector,tenant,where", "", "r", "SearchResponse"}, + {"search_near_object", "main", "POST", "/search/{collection}/near-object", "search.nearObject", "autoLimit,certainty,consistencyLevel,distance,groupBy,groupedTask,id,limit,numberOfGroups,objectsPerGroup,offset,rerank,returnMetadata,returnProperties,returnReferences,singlePrompt,targetVector,tenant,where", "", "r", "SearchResponse"}, + {"search_near_text", "main", "POST", "/search/{collection}/near-text", "search.nearText", "autoLimit,certainty,consistencyLevel,distance,groupBy,groupedTask,limit,numberOfGroups,objectsPerGroup,offset,query,rerank,returnMetadata,returnProperties,returnReferences,singlePrompt,targetVector,tenant,where", "", "r", "SearchResponse"}, + {"search_near_vector", "main", "POST", "/search/{collection}/near-vector", "search.nearVector", "autoLimit,certainty,consistencyLevel,distance,groupBy,groupedTask,limit,numberOfGroups,objectsPerGroup,offset,rerank,returnMetadata,returnProperties,returnReferences,singlePrompt,targetVector,tenant,vector,where", "", "r", "SearchResponse"}, +}; +// CALLS-END + +const CallTable& weaviate_calls() { + static const CallTable table{kRows, sizeof(kRows) / sizeof(kRows[0])}; + return table; +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/weaviate/weaviate.cpp b/src/vendors/retrieval/weaviate/weaviate.cpp new file mode 100644 index 0000000..16e4c66 --- /dev/null +++ b/src/vendors/retrieval/weaviate/weaviate.cpp @@ -0,0 +1,162 @@ +#include "vendors/retrieval/weaviate/weaviate.h" + +#include "llm/config.h" +#include "vendors/retrieval/caps.h" + +namespace molo::retrieval { + +llm::ApiClientConfig Weaviate::config() { + llm::ApiClientConfig c; + c.vendor = "weaviate"; + c.base_url = ""; + c.key_env = kKeyEnv; + c.auth = llm::AuthStyle::kBearer; + return c; +} + +namespace { +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("weaviate", why); } + +Weaviate build(std::string url, std::string key, const TransportFactory& factory) { + while (!url.empty() && url.back() == '/') url.pop_back(); + bool ok = url.rfind("https://", 0) == 0 || url.rfind("http://", 0) == 0; + auto gw = make_gateway(weaviate_calls(), Weaviate::config(), {{"main", ok ? url + "/v1" : "https://weaviate.invalid/v1"}}, + std::move(key), factory); + if (!ok) gw->require_setting(Weaviate::kUrlEnv); + return Weaviate(gw); +} + +CallArgs with_body(Json body) { + CallArgs a; + a.body = std::move(body); + return a; +} +CallArgs cls(const std::string& name) { + CallArgs a; + a.path["className"] = Weaviate::validate_class_name(name); + return a; +} +Json search_body(const std::string& query, int limit) { + if (query.empty() || query.size() > RetrievalCaps::kMaxQueryChars) bad("query must be 1..8192 chars"); + if (limit < 1 || limit > RetrievalCaps::kMaxPointLimit) bad("limit must be 1..10000"); + Json b = Json::object(); + b.set("query", Json(query)); + b.set("limit", Json(limit)); + return b; +} +} // namespace + +Weaviate Weaviate::from_env() { return build(llm::Config::get(kUrlEnv), "", nullptr); } +Weaviate Weaviate::with(const TransportFactory& factory, std::string key, const std::string& base_url) { + return build(base_url, std::move(key), factory); +} + +std::string Weaviate::validate_class_name(const std::string& name) { + // Weaviate class names start with an uppercase letter: /[A-Z][_0-9A-Za-z]*/ + bool ok = !name.empty() && name.size() <= 255 && name[0] >= 'A' && name[0] <= 'Z'; + for (char c : name) ok = ok && ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '_'); + if (!ok) bad("class name must match [A-Z][_0-9A-Za-z]* (max 255)"); + return name; +} + +Json Weaviate::schema() { return gw_->call("schema_dump"); } +Json Weaviate::get_class(const std::string& n) { return gw_->call("schema_objects_get", cls(n)); } + +Json Weaviate::create_class(const Json& def) { + if (!def.is_object()) bad("class definition must be an object"); + const Json* c = def.find("class"); + if (!c || !c->is_string()) bad("class definition needs a class name"); + validate_class_name(c->as_string()); + return gw_->call("schema_objects_create", with_body(def)); +} + +Json Weaviate::delete_class(const std::string& n) { return gw_->destroy("schema_objects_delete", cls(n)); } + +Json Weaviate::create_object(const std::string& class_name, const Json& properties, const std::vector& vector) { + if (!properties.is_object()) bad("properties must be an object"); + Json body = Json::object(); + body.set("class", Json(validate_class_name(class_name))); + body.set("properties", properties); + if (!vector.empty()) { + if (vector.size() > static_cast(RetrievalCaps::kMaxDimensions)) bad("vector too long"); + Json v = Json::array(); + for (float f : vector) v.push(Json(static_cast(f))); + body.set("vector", v); + } + return gw_->call("objects_create", with_body(body)); +} + +Json Weaviate::get_object(const std::string& class_name, const std::string& id) { + CallArgs a = cls(class_name); + a.path["id"] = id; + return gw_->call("objects_class_get", a); +} + +Json Weaviate::delete_object(const std::string& class_name, const std::string& id) { + CallArgs a = cls(class_name); + a.path["id"] = id; + return gw_->destroy("objects_class_delete", a); +} + +Json Weaviate::batch_create(const std::vector& objects) { + if (objects.empty() || objects.size() > RetrievalCaps::kMaxPointsPerUpsert) bad("objects must hold 1..1000 entries"); + Json arr = Json::array(); + for (const Json& o : objects) { + if (!o.is_object()) bad("an object must be a JSON object"); + arr.push(o); + } + Json body = Json::object(); + body.set("objects", arr); + return gw_->call("batch_objects_create", with_body(body)); +} + +Json Weaviate::batch_delete(const Json& match, bool dry_run) { + if (!match.is_object() || !match.has("class")) bad("match needs a class"); + Json body = Json::object(); + body.set("match", match); + body.set("dryRun", Json(dry_run)); + body.set("output", Json("verbose")); + return gw_->destroy("batch_objects_delete", with_body(body)); +} + +Json Weaviate::near_text(const std::string& collection, const std::string& query, int limit) { + CallArgs a = with_body(search_body(query, limit)); + a.path["collection"] = collection; + return gw_->call("search_near_text", a); +} + +Json Weaviate::bm25(const std::string& collection, const std::string& query, int limit) { + CallArgs a = with_body(search_body(query, limit)); + a.path["collection"] = collection; + return gw_->call("search_bm25", a); +} + +Json Weaviate::hybrid(const std::string& collection, const std::string& query, int limit, double alpha) { + if (alpha < 0 || alpha > 1) bad("alpha must be 0..1"); + Json b = search_body(query, limit); + b.set("alpha", Json(alpha)); + CallArgs a = with_body(b); + a.path["collection"] = collection; + return gw_->call("search_hybrid", a); +} + +Json Weaviate::graphql(const std::string& query, const Json& variables) { + if (query.empty() || query.size() > RetrievalCaps::kMaxQueryChars * 8) bad("query must be 1..65536 chars"); + Json b = Json::object(); + b.set("query", Json(query)); + if (!variables.is_null()) b.set("variables", variables); + return gw_->call("graphql_post", with_body(b)); +} + +Json Weaviate::tenants(const std::string& class_name) { return gw_->call("tenants_get", cls(class_name)); } + +Json Weaviate::delete_tenants(const std::string& class_name, const std::vector& names) { + if (names.empty() || names.size() > 100) bad("names must hold 1..100 entries"); + return gw_->destroy("tenants_delete", [&] { + CallArgs a = cls(class_name); + a.body = J::strings(names); + return a; + }()); +} + +} // namespace molo::retrieval diff --git a/src/vendors/retrieval/weaviate/weaviate.h b/src/vendors/retrieval/weaviate/weaviate.h new file mode 100644 index 0000000..a60f2e9 --- /dev/null +++ b/src/vendors/retrieval/weaviate/weaviate.h @@ -0,0 +1,54 @@ +#pragma once + +// Weaviate REST (/v1): schema, objects, batch, tenants, aliases, the REST +// search endpoints (near-text, near-vector style) and GraphQL over HTTP. +// Weaviate's gRPC API is not HTTP and has no gateway: it is out of scope (see +// SPEC.md). The server is yours (WEAVIATE_URL). + +#include +#include + +#include "vendors/retrieval/provider.h" + +namespace molo::retrieval { + +const CallTable& weaviate_calls(); + +class Weaviate : public Provider { + public: + static constexpr const char* kKeyEnv = "WEAVIATE_API_KEY"; + static constexpr const char* kUrlEnv = "WEAVIATE_URL"; + + static llm::ApiClientConfig config(); + static Weaviate from_env(); + static Weaviate with(const TransportFactory& factory, std::string key, const std::string& base_url); + explicit Weaviate(std::shared_ptr gateway) : Provider(std::move(gateway)) {} + + // Schema + Json schema(); + Json get_class(const std::string& class_name); + Json create_class(const Json& class_definition); + Json delete_class(const std::string& class_name); // destructive: drops the class and its objects + + // Objects + Json create_object(const std::string& class_name, const Json& properties, const std::vector& vector); + Json get_object(const std::string& class_name, const std::string& id); + Json delete_object(const std::string& class_name, const std::string& id); // destructive + Json batch_create(const std::vector& objects); + // Destructive. `match` is {"class":..., "where":{...}}; dry_run reports without deleting. + Json batch_delete(const Json& match, bool dry_run); + + // Search + Json near_text(const std::string& collection, const std::string& query, int limit); + Json bm25(const std::string& collection, const std::string& query, int limit); + Json hybrid(const std::string& collection, const std::string& query, int limit, double alpha); + Json graphql(const std::string& query, const Json& variables); + + // Multi-tenancy + Json tenants(const std::string& class_name); + Json delete_tenants(const std::string& class_name, const std::vector& names); // destructive + + static std::string validate_class_name(const std::string& name); +}; + +} // namespace molo::retrieval diff --git a/tests/example/ai/retrieval/call_test.cpp b/tests/example/ai/retrieval/call_test.cpp new file mode 100644 index 0000000..d8dc44b --- /dev/null +++ b/tests/example/ai/retrieval/call_test.cpp @@ -0,0 +1,58 @@ +// Route: POST /ai/retrieval/call (generic table-driven call: 422, dry runs, 503 without a credential) +#include + +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + setenv("MOLO_ENV_FILES", "off", 1); + for (const char* v : {"VOYAGE_API_KEY", "JINA_API_KEY", "PINECONE_API_KEY", "QDRANT_API_KEY", "QDRANT_URL", "WEAVIATE_API_KEY", + "WEAVIATE_URL", "MXBAI_API_KEY", "TAVILY_API_KEY", "EXA_API_KEY", "BRAVE_API_KEY"}) + unsetenv(v); + Fixture f; + auto post = [&](const std::string& body) { return f.client->post("/ai/retrieval/call", body, "application/json"); }; + + // 422: input rules run before anything else + check(post("{\"provider\":\"qdrant\"}").status == 422, "call name is required"); + check(post("{\"provider\":\"qdrant\",\"call\":\"nope\"}").status == 422, "unknown call"); + check(post("{\"provider\":\"qdrant\",\"call\":\"get_collection\"}").status == 422, "missing path parameter"); + check(post("{\"provider\":\"qdrant\",\"call\":\"get_collection\",\"path\":{\"collection_name\":\"../x\"}}").status == 422, "path traversal id"); + check(post("{\"provider\":\"qdrant\",\"call\":\"get_collection\",\"path\":{\"collection_name\":\"a b\"}}").status == 422, "id with a space"); + check(post("{\"provider\":\"qdrant\",\"call\":\"get_collections\",\"query\":{\"bogus\":1}}").status == 422, "query parameter not in the table"); + check(post("{\"provider\":\"qdrant\",\"call\":\"get_collections\",\"body\":{\"a\":1}}").status == 422, "GET takes no body"); + check(post("{\"provider\":\"qdrant\",\"call\":\"upsert_points\",\"path\":{\"collection_name\":\"c\"},\"body\":{\"nope\":1}}").status == 422, "body field not in the table"); + check(post("{\"provider\":\"qdrant\",\"call\":\"get_collections\",\"path\":[1]}").status == 422, "path must be an object"); + check(post("{\"provider\":\"qdrant\",\"call\":\"get_collections\",\"query\":{\"x\":[1]}}").status == 422, "query values are scalars"); + check(post("{\"provider\":\"pinecone\",\"call\":\"describe_index_stats\",\"body\":{}}").status == 422, "data plane call needs index_host"); + check(post("{\"provider\":\"pinecone\",\"call\":\"describe_index_stats\",\"index_host\":\"https://x.io\",\"body\":{}}").status == 422, "index_host must be a bare host"); + + // dry runs: writes, destructive and paid calls show the exact request + auto upsert = post("{\"provider\":\"qdrant\",\"call\":\"upsert_points\",\"path\":{\"collection_name\":\"docs\"},\"query\":{\"wait\":true},\"body\":{\"points\":[]}}"); + check(upsert.status == 200 && contains(upsert.body, "\"dry_run\":true") && contains(upsert.body, "/collections/docs/points?wait=true"), + "upsert is a dry run with the rendered target"); + auto del = post("{\"provider\":\"qdrant\",\"call\":\"delete_collection\",\"path\":{\"collection_name\":\"docs\"}}"); + check(del.status == 200 && contains(del.body, "\"dry_run\":true") && contains(del.body, "\"flags\":\"d\""), "delete collection is a dry run"); + auto delidx = post("{\"provider\":\"pinecone\",\"call\":\"delete_index\",\"path\":{\"index_name\":\"docs\"}}"); + check(delidx.status == 200 && contains(delidx.body, "\"dry_run\":true"), "delete index is a dry run"); + auto delvec = post("{\"provider\":\"pinecone\",\"call\":\"delete_vectors\",\"index_host\":\"docs-abc.svc.pinecone.io\",\"body\":{\"deleteAll\":true}}"); + check(delvec.status == 200 && contains(delvec.body, "\"dry_run\":true") && contains(delvec.body, "deleteAll"), "delete vectors is a dry run"); + auto paid = post("{\"provider\":\"voyage\",\"call\":\"embed\",\"body\":{\"input\":[\"a\"],\"model\":\"voyage-3.5\"}}"); + check(paid.status == 200 && contains(paid.body, "\"dry_run\":true") && contains(paid.body, "/v1/embeddings"), "paid call is a dry run"); + + // 503: free reads and confirmed calls need the credential + for (const char* body : {"{\"provider\":\"qdrant\",\"call\":\"get_collections\"}", + "{\"provider\":\"weaviate\",\"call\":\"schema_dump\"}", + "{\"provider\":\"pinecone\",\"call\":\"list_indexes\"}", + "{\"provider\":\"mixedbread\",\"call\":\"list_stores\"}", + "{\"provider\":\"jina\",\"call\":\"list_models\"}", + "{\"provider\":\"tavily\",\"call\":\"get_usage\"}", + "{\"provider\":\"pinecone\",\"call\":\"describe_index_stats\",\"index_host\":\"docs-abc.svc.pinecone.io\",\"body\":{}}", + "{\"provider\":\"qdrant\",\"call\":\"delete_collection\",\"path\":{\"collection_name\":\"docs\"},\"confirm\":true}", + "{\"provider\":\"pinecone\",\"call\":\"delete_index\",\"path\":{\"index_name\":\"docs\"},\"confirm\":true}", + "{\"provider\":\"voyage\",\"call\":\"rerank\",\"body\":{\"query\":\"q\",\"documents\":[\"a\"],\"model\":\"rerank-2.5\"},\"confirm\":true}"}) { + auto res = post(body); + check(res.status == 503 && contains(res.body, "retrieval_not_configured"), std::string("503 without a credential: ") + body); + } + return finish("ai/retrieval/call"); +} diff --git a/tests/example/ai/retrieval/page_test.cpp b/tests/example/ai/retrieval/page_test.cpp new file mode 100644 index 0000000..a783313 --- /dev/null +++ b/tests/example/ai/retrieval/page_test.cpp @@ -0,0 +1,41 @@ +// Routes: GET /ai/retrieval (page), /status, /calls (no credential needed) +#include + +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + setenv("MOLO_ENV_FILES", "off", 1); + for (const char* v : {"VOYAGE_API_KEY", "JINA_API_KEY", "PINECONE_API_KEY", "QDRANT_API_KEY", "QDRANT_URL", "WEAVIATE_API_KEY", + "WEAVIATE_URL", "MXBAI_API_KEY", "TAVILY_API_KEY", "EXA_API_KEY", "BRAVE_API_KEY"}) + unsetenv(v); + Fixture f; + auto page = f.client->get("/ai/retrieval"); + check(page.status == 200, "page 200"); + check(contains(page.headers.get("Content-Type"), "text/html"), "html"); + check(contains(page.body, "Retrieval vendors") && contains(page.body, "viewport"), "title and mobile viewport"); + check(contains(page.body, "/embedded/ai_retrieval.js"), "script wired"); + check(f.client->post("/ai/retrieval", "{}", "application/json").status == 405, "post page is 405"); + + auto st = f.client->get("/ai/retrieval/status"); + check(st.status == 200 && contains(st.body, "\"configured\":false"), "status without a key"); + check(!contains(st.body, "\"configured\":true"), "no provider claims a credential"); + for (const char* env : {"VOYAGE_API_KEY", "JINA_API_KEY", "PINECONE_API_KEY", "QDRANT_API_KEY", "QDRANT_URL", "WEAVIATE_API_KEY", + "WEAVIATE_URL", "MXBAI_API_KEY", "TAVILY_API_KEY", "EXA_API_KEY", "BRAVE_API_KEY"}) + check(contains(st.body, env), std::string("status names ") + env); + for (const char* id : {"voyage", "jina", "pinecone", "qdrant", "weaviate", "mixedbread", "tavily", "exa", "brave"}) + check(contains(st.body, std::string("\"id\":\"") + id + "\""), std::string("status lists ") + id); + check(contains(st.body, "\"max_embed_texts\":1000"), "caps listed"); + check(f.client->post("/ai/retrieval/status", "{}", "application/json").status == 405, "post status is 405"); + + auto calls = f.client->get("/ai/retrieval/calls"); + check(calls.status == 200 && contains(calls.body, "/collections/{collection_name}/points/query") && + contains(calls.body, "/v1/web/search") && contains(calls.body, "/v1/embeddings") && contains(calls.body, "/vectors/upsert"), + "all call tables listed"); + check(contains(calls.body, "\"unverified\":true"), "unverified rows are marked"); + auto one = f.client->get("/ai/retrieval/calls?provider=pinecone"); + check(one.status == 200 && contains(one.body, "\"count\":57") && !contains(one.body, "\"provider\":\"qdrant\""), "filter by provider"); + check(f.client->get("/ai/retrieval/calls?provider=nope").status == 422, "unknown provider filter is 422"); + return finish("ai/retrieval/page"); +} diff --git a/tests/example/ai/retrieval/routes_test.cpp b/tests/example/ai/retrieval/routes_test.cpp new file mode 100644 index 0000000..ba761e0 --- /dev/null +++ b/tests/example/ai/retrieval/routes_test.cpp @@ -0,0 +1,69 @@ +// Routes: POST /ai/retrieval/{embed,rerank,search} (405, 422, dry runs, 503 without a credential) +#include + +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + setenv("MOLO_ENV_FILES", "off", 1); + for (const char* v : {"VOYAGE_API_KEY", "JINA_API_KEY", "PINECONE_API_KEY", "MXBAI_API_KEY", "TAVILY_API_KEY", "EXA_API_KEY", "BRAVE_API_KEY"}) + unsetenv(v); + Fixture f; + auto post = [&](const std::string& route, const std::string& body) { + return f.client->post("/ai/retrieval/" + route, body, "application/json"); + }; + for (const char* r : {"embed", "rerank", "search", "call"}) { + std::string route = std::string("/ai/retrieval/") + r; + check(f.client->get(route).status == 405, route + " get is 405"); + check(post(r, "not json").status == 422, route + " bad json is 422"); + check(post(r, "[1]").status == 422, route + " non-object is 422"); + check(post(r, "{}").status == 422, route + " needs a provider"); + check(post(r, "{\"provider\":\"nope\"}").status == 422, route + " unknown provider is 422"); + } + + // embed + check(post("embed", "{\"provider\":\"voyage\"}").status == 422, "embed needs texts"); + check(post("embed", "{\"provider\":\"voyage\",\"texts\":[]}").status == 422, "embed empty texts"); + check(post("embed", "{\"provider\":\"voyage\",\"texts\":[\" \"]}").status == 422, "embed blank text"); + check(post("embed", "{\"provider\":\"voyage\",\"texts\":[1]}").status == 422, "embed texts must be strings"); + check(post("embed", "{\"provider\":\"voyage\",\"texts\":[\"a\"],\"input_type\":\"other\"}").status == 422, "embed bad input_type"); + check(post("embed", "{\"provider\":\"voyage\",\"texts\":[\"a\"],\"dimensions\":99999}").status == 422, "embed dimensions cap"); + check(post("embed", "{\"provider\":\"tavily\",\"texts\":[\"a\"]}").status == 422, "tavily has no embeddings"); + check(post("embed", "{\"provider\":\"qdrant\",\"texts\":[\"a\"]}").status == 422, "qdrant has no embeddings"); + auto dry = post("embed", "{\"provider\":\"voyage\",\"texts\":[\"a\",\"b\"],\"input_type\":\"query\"}"); + check(dry.status == 200 && contains(dry.body, "\"dry_run\":true") && contains(dry.body, "\"texts\":2"), "embed is a dry run without confirm"); + for (const char* p : {"voyage", "jina", "pinecone", "mixedbread"}) { + auto res = post("embed", std::string("{\"provider\":\"") + p + "\",\"texts\":[\"a\"],\"confirm\":true}"); + check(res.status == 503 && contains(res.body, "retrieval_not_configured"), std::string("confirmed embed without key is 503: ") + p); + } + + // rerank + check(post("rerank", "{\"provider\":\"jina\",\"query\":\"q\"}").status == 422, "rerank needs documents"); + check(post("rerank", "{\"provider\":\"jina\",\"query\":\" \",\"documents\":[\"a\"]}").status == 422, "rerank blank query"); + check(post("rerank", "{\"provider\":\"jina\",\"query\":\"q\",\"documents\":[\"a\"],\"top_n\":-1}").status == 422, "rerank negative top_n"); + check(post("rerank", "{\"provider\":\"tavily\",\"query\":\"q\",\"documents\":[\"a\"]}").status == 422, "tavily has no rerank"); + check(contains(post("rerank", "{\"provider\":\"jina\",\"query\":\"q\",\"documents\":[\"a\",\"b\"]}").body, "\"dry_run\":true"), "rerank is a dry run"); + for (const char* p : {"voyage", "jina", "pinecone", "mixedbread"}) { + auto res = post("rerank", std::string("{\"provider\":\"") + p + "\",\"query\":\"q\",\"documents\":[\"a\"],\"confirm\":true}"); + check(res.status == 503 && contains(res.body, "retrieval_not_configured"), std::string("confirmed rerank without key is 503: ") + p); + } + + // search + check(post("search", "{\"provider\":\"tavily\"}").status == 422, "search needs a query"); + check(post("search", "{\"provider\":\"tavily\",\"query\":\"x\",\"max_results\":0}").status == 422, "search max_results 0"); + check(post("search", "{\"provider\":\"tavily\",\"query\":\"x\",\"max_results\":101}").status == 422, "search max_results over cap"); + check(post("search", "{\"provider\":\"tavily\",\"query\":\"x\",\"recency\":\"decade\"}").status == 422, "search bad recency"); + check(post("search", "{\"provider\":\"voyage\",\"query\":\"x\"}").status == 422, "voyage has no web search"); + check(post("search", "{\"provider\":\"pinecone\",\"query\":\"x\"}").status == 422, "pinecone has no web search"); + check(contains(post("search", "{\"provider\":\"brave\",\"query\":\"x\"}").body, "\"dry_run\":true"), "search is a dry run"); + for (const char* p : {"jina", "tavily", "exa", "brave"}) { + auto res = post("search", std::string("{\"provider\":\"") + p + "\",\"query\":\"x\",\"confirm\":true}"); + check(res.status == 503 && contains(res.body, "retrieval_not_configured"), std::string("confirmed search without key is 503: ") + p); + } + // adapter rules that only show on a confirmed call + check(post("search", "{\"provider\":\"jina\",\"query\":\"x\",\"recency\":\"day\",\"confirm\":true}").status == 422, "jina has no recency filter"); + check(post("search", "{\"provider\":\"brave\",\"query\":\"x\",\"max_results\":21,\"confirm\":true}").status == 422, "brave web search is capped at 20"); + check(post("search", "{\"provider\":\"tavily\",\"query\":\"x\",\"max_results\":21,\"confirm\":true}").status == 422, "tavily search is capped at 20"); + return finish("ai/retrieval/routes"); +} diff --git a/tests/src/vendors/retrieval/support.h b/tests/src/vendors/retrieval/support.h new file mode 100644 index 0000000..eaa805a --- /dev/null +++ b/tests/src/vendors/retrieval/support.h @@ -0,0 +1,71 @@ +#pragma once + +// Rig for the retrieval tests: a TransportFactory that hands out one +// MockTransport per host (in host order) and keeps raw pointers so a test can +// script a reply and read back the request the real client built. The mock only +// records requests; replies are FIXTURES whose shapes are copied from the +// provider specs and labelled where they appear. They are not recorded +// replies of the live services (see SPEC.md ## Unverified). + +#include +#include +#include + +#include "clients/transports/mock_transport.h" +#include "framework.h" +#include "vendors/retrieval/provider.h" + +namespace retrieval_test { + +using molo::clients::ClientResponse; +using molo::clients::HeaderMap; +using molo::clients::Method; +using molo::clients::transports::MockExpectation; +using molo::clients::transports::MockTransport; +using Json = molo::clients::body::Json; + +inline const char* kKey = "key-TESTKEY-DO-NOT-LEAK"; + +inline ClientResponse reply(const std::string& body, int status = 200) { + HeaderMap h; + h.set("Content-Type", "application/json"); + return ClientResponse(status, h, body); +} + +struct Rig { + std::vector transports; + + molo::retrieval::TransportFactory factory() { + return [this]() { + auto t = std::make_unique(); + transports.push_back(t.get()); + return std::unique_ptr(std::move(t)); + }; + } + + // Scripts `body` as the reply for method + path on transport `host_index`. + void answer(std::size_t host_index, Method m, const std::string& path, const std::string& body, int status = 200) { + MockExpectation e; + e.method = m; + e.path = path; + e.response = reply(body, status); + e.times = 50; + transports.at(host_index)->expect(e); + } + + MockTransport::SeenRequest last(std::size_t host_index = 0) const { + auto seen = transports.at(host_index)->seen(); + if (seen.empty()) throw std::runtime_error("no request was sent"); + return seen.back(); + } + std::size_t sent(std::size_t host_index = 0) const { return transports.at(host_index)->send_count(); } + std::size_t sent_total() const { + std::size_t n = 0; + for (MockTransport* t : transports) n += t->send_count(); + return n; + } +}; + +inline bool has(const std::string& hay, const std::string& needle) { return hay.find(needle) != std::string::npos; } + +} // namespace retrieval_test diff --git a/tests/src/vendors/retrieval/test_core.cpp b/tests/src/vendors/retrieval/test_core.cpp new file mode 100644 index 0000000..c678e61 --- /dev/null +++ b/tests/src/vendors/retrieval/test_core.cpp @@ -0,0 +1,265 @@ +// Retrieval core: the call tables, the Gateway rules (ids, query, body, +// destructive split, caps) and the seam validators. No provider reply is +// involved except where a test needs the request to leave the client. +#include + +#include "support.h" +#include "vendors/retrieval/brave/brave.h" +#include "vendors/retrieval/caps.h" +#include "vendors/retrieval/exa/exa.h" +#include "vendors/retrieval/jina/jina.h" +#include "vendors/retrieval/mixedbread/mixedbread.h" +#include "vendors/retrieval/pinecone/pinecone.h" +#include "vendors/retrieval/qdrant/qdrant.h" +#include "vendors/retrieval/tavily/tavily.h" +#include "vendors/retrieval/voyage/voyage.h" +#include "vendors/retrieval/weaviate/weaviate.h" + +using namespace molo::retrieval; +using namespace retrieval_test; +using molo::llm::ApiError; +using molo::llm::NotConfiguredError; + +namespace { +struct Named { + const char* name; + const CallTable& (*table)(); + std::size_t at_least; +}; +const Named kAll[] = { + {"voyage", voyage_calls, 4}, {"jina", jina_calls, 15}, {"pinecone", pinecone_calls, 57}, + {"qdrant", qdrant_calls, 49}, {"weaviate", weaviate_calls, 66}, {"mixedbread", mixedbread_calls, 80}, + {"tavily", tavily_calls, 7}, {"exa", exa_calls, 7}, {"brave", brave_calls, 16}, +}; + +std::vector placeholders(const std::string& path) { + std::vector out; + for (std::size_t p = path.find('{'); p != std::string::npos; p = path.find('{', p + 1)) + out.push_back(path.substr(p + 1, path.find('}', p) - p - 1)); + return out; +} +} // namespace + +TEST(retrieval_core, every_table_is_well_formed) { + for (const Named& n : kAll) { + const CallTable& t = n.table(); + ASSERT_TRUE(t.size() >= n.at_least); + std::set names; + for (std::size_t i = 0; i < t.size(); ++i) { + const CallRow& r = t.at(i); + std::string who = std::string(n.name) + "." + r.name; + ASSERT_TRUE(names.insert(r.name).second); // unique names + RowInfo::method(r); // valid method, else throws + ASSERT_TRUE(r.path[0] == '/'); + ASSERT_TRUE(std::string(r.flags).find_first_not_of("dwrp") == std::string::npos); + ASSERT_TRUE(placeholders(r.path).size() <= RetrievalCaps::kMaxPathParams); + if (std::string(r.method) == "DELETE") ASSERT_TRUE(RowInfo::destructive(r)); + if (std::string(r.method) == "GET") ASSERT_FALSE(RowInfo::writes(r)); + if (RowInfo::destructive(r)) ASSERT_FALSE(RowInfo::has(r, 'r')); + if (std::string(r.method) == "GET") ASSERT_TRUE(std::string(r.body).empty()); + ASSERT_TRUE(std::string(r.host).size() > 0); + (void)who; + } + } +} + +TEST(retrieval_core, destructive_rows_are_the_ones_that_remove_or_clear) { + const char* must[][2] = {{"qdrant", "delete_collection"}, {"qdrant", "delete_points"}, {"qdrant", "clear_payload"}, + {"pinecone", "delete_index"}, {"pinecone", "delete_vectors"}, {"weaviate", "schema_objects_delete"}, + {"weaviate", "batch_objects_delete"}, {"mixedbread", "delete_store"}, {"jina", "cancel_batch"}}; + for (auto& m : must) { + const CallTable* t = nullptr; + for (const Named& n : kAll) + if (std::string(n.name) == m[0]) t = &n.table(); + ASSERT_TRUE(t != nullptr); + const CallRow* r = t->find(m[1]); + ASSERT_TRUE(r != nullptr); + ASSERT_TRUE(RowInfo::destructive(*r)); + } +} + +TEST(retrieval_core, unverified_rows_are_exactly_the_prose_only_ones) { + std::size_t n = 0; + for (const Named& a : kAll) + for (std::size_t i = 0; i < a.table().size(); ++i) + if (RowInfo::unverified(a.table().at(i))) ++n; + ASSERT_EQ(n, std::size_t(2)); // jina segment and jina classify + ASSERT_TRUE(RowInfo::unverified(*jina_calls().find("segment"))); + ASSERT_TRUE(RowInfo::unverified(*jina_calls().find("classify"))); +} + +TEST(retrieval_core, plan_renders_path_query_and_flags_without_traffic) { + Rig rig; + Qdrant q = Qdrant::with(rig.factory(), kKey, "https://qd.example.com:6333/"); + CallArgs a; + a.path["collection_name"] = "docs"; + a.query["wait"] = "true"; + Json b = Json::object(); + b.set("points", Json::array()); + a.body = b; + PlannedCall p = q.gateway().plan("upsert_points", a); + ASSERT_EQ(p.method, std::string("PUT")); + ASSERT_EQ(p.target, std::string("/collections/docs/points?wait=true")); + ASSERT_TRUE(p.writes); + ASSERT_FALSE(p.destructive); + ASSERT_EQ(rig.sent_total(), std::size_t(0)); +} + +TEST(retrieval_core, path_ids_are_validated_and_encoded) { + Rig rig; + Qdrant q = Qdrant::with(rig.factory(), kKey, "https://qd.example.com"); + CallArgs a; + a.path["collection_name"] = "../etc"; + ASSERT_THROW(q.gateway().plan("get_collection", a), ApiError); + a.path["collection_name"] = "a b"; + ASSERT_THROW(q.gateway().plan("get_collection", a), ApiError); + a.path["collection_name"] = "a/b"; + ASSERT_THROW(q.gateway().plan("get_collection", a), ApiError); + a.path.clear(); + ASSERT_THROW(q.gateway().plan("get_collection", a), ApiError); // missing + a.path["collection_name"] = "ok"; + a.path["extra"] = "x"; + ASSERT_THROW(q.gateway().plan("get_collection", a), ApiError); // unexpected + ASSERT_THROW(q.gateway().plan("no_such_call"), ApiError); + ASSERT_EQ(rig.sent_total(), std::size_t(0)); +} + +TEST(retrieval_core, query_and_body_must_be_listed_in_the_table) { + Rig rig; + Qdrant q = Qdrant::with(rig.factory(), kKey, "https://qd.example.com"); + CallArgs a; + a.path["collection_name"] = "docs"; + a.query["bogus"] = "1"; + ASSERT_THROW(q.gateway().plan("upsert_points", a), ApiError); + a.query.clear(); + Json b = Json::object(); + b.set("not_a_field", Json(1)); + a.body = b; + ASSERT_THROW(q.gateway().plan("upsert_points", a), ApiError); + CallArgs get; + get.path["collection_name"] = "docs"; + get.body = Json::object(); + ASSERT_THROW(q.gateway().plan("get_collection", get), ApiError); // GET takes no body + CallArgs ctl; + ctl.path["collection_name"] = "docs"; + ctl.query["wait"] = std::string("a\nb"); + ctl.body = Json::object(); + ASSERT_THROW(q.gateway().plan("upsert_points", ctl), ApiError); // control characters + CallArgs big; + big.path["collection_name"] = "docs"; + big.query["wait"] = std::string(RetrievalCaps::kMaxQueryValueBytes + 1, 'x'); + ASSERT_THROW(q.gateway().plan("upsert_points", big), ApiError); +} + +TEST(retrieval_core, body_over_the_byte_cap_is_rejected_before_sending) { + Rig rig; + Qdrant q = Qdrant::with(rig.factory(), kKey, "https://qd.example.com"); + CallArgs a; + a.path["collection_name"] = "docs"; + Json b = Json::object(); + b.set("points", Json(std::string(RetrievalCaps::kMaxBodyBytes + 1, 'x'))); + a.body = b; + ASSERT_THROW(q.gateway().plan("upsert_points", a), ApiError); + ASSERT_EQ(rig.sent_total(), std::size_t(0)); +} + +TEST(retrieval_core, destructive_rows_only_run_through_destroy) { + Rig rig; + Qdrant q = Qdrant::with(rig.factory(), kKey, "https://qd.example.com"); + rig.answer(0, Method::kDelete, "/collections/docs", "{\"result\":true,\"status\":\"ok\"}"); + CallArgs a; + a.path["collection_name"] = "docs"; + ASSERT_THROW(q.gateway().call("delete_collection", a), ApiError); // refused, nothing sent + ASSERT_EQ(rig.sent_total(), std::size_t(0)); + ASSERT_THROW(q.gateway().destroy("get_collection", a), ApiError); // not destructive + ASSERT_EQ(rig.sent_total(), std::size_t(0)); + Json out = q.delete_collection("docs"); // the explicit method + ASSERT_EQ(rig.sent_total(), std::size_t(1)); + ASSERT_EQ(rig.last().method, std::string("DELETE")); + ASSERT_TRUE(out.has("result")); +} + +TEST(retrieval_core, no_key_means_no_traffic_and_a_503_error) { + Rig rig; + Voyage v = Voyage::with(rig.factory(), ""); + // Config::get may find a key in the environment; test runs set MOLO_ENV_FILES=off + // and VOYAGE_API_KEY is not set in CI, so an empty key stays empty. + if (v.configured()) return; + EmbedRequest r; + r.texts = {"hello"}; + try { + v.embed(r); + FAIL("expected NotConfiguredError"); + } catch (const NotConfiguredError& e) { + ASSERT_EQ(e.http_status(), 503); + ASSERT_EQ(e.type(), std::string("NOT_CONFIGURED")); + ASSERT_TRUE(has(e.message(), "VOYAGE_API_KEY")); + ASSERT_FALSE(has(e.message(), kKey)); + } + ASSERT_EQ(rig.sent_total(), std::size_t(0)); +} + +TEST(retrieval_core, self_hosted_databases_need_their_url_too) { + Rig rig; + Qdrant q = Qdrant::with(rig.factory(), kKey, ""); + ASSERT_FALSE(q.configured()); + try { + q.list_collections(); + FAIL("expected NotConfiguredError"); + } catch (const NotConfiguredError& e) { + ASSERT_TRUE(has(e.message(), "QDRANT_URL")); + } + Weaviate w = Weaviate::with(rig.factory(), kKey, "localhost:8080"); // no scheme + ASSERT_FALSE(w.configured()); + ASSERT_THROW(w.schema(), NotConfiguredError); + ASSERT_EQ(rig.sent_total(), std::size_t(0)); +} + +TEST(retrieval_core, seam_validators_enforce_the_caps) { + EmbedRequest e; + ASSERT_THROW(Embedder::validate("t", e), ApiError); // empty + e.texts = {"ok"}; + Embedder::validate("t", e); + e.texts = {" "}; + ASSERT_THROW(Embedder::validate("t", e), ApiError); // blank + e.texts.assign(RetrievalCaps::kMaxEmbedTexts + 1, "x"); + ASSERT_THROW(Embedder::validate("t", e), ApiError); + e.texts = {std::string(RetrievalCaps::kMaxTextBytes + 1, 'x')}; + ASSERT_THROW(Embedder::validate("t", e), ApiError); + e.texts = {"ok"}; + e.input_type = "other"; + ASSERT_THROW(Embedder::validate("t", e), ApiError); + e.input_type = "query"; + e.dimensions = RetrievalCaps::kMaxDimensions + 1; + ASSERT_THROW(Embedder::validate("t", e), ApiError); + + RerankRequest r; + r.query = "q"; + ASSERT_THROW(Reranker::validate("t", r), ApiError); // no documents + r.documents = {"a", "b"}; + Reranker::validate("t", r); + r.top_n = -1; + ASSERT_THROW(Reranker::validate("t", r), ApiError); + r.top_n = 1; + r.query = " "; + ASSERT_THROW(Reranker::validate("t", r), ApiError); + + SearchRequest s; + s.query = "q"; + WebSearch::validate("t", s); + s.max_results = 0; + ASSERT_THROW(WebSearch::validate("t", s), ApiError); + s.max_results = RetrievalCaps::kMaxSearchResults + 1; + ASSERT_THROW(WebSearch::validate("t", s), ApiError); + s.max_results = 3; + s.recency = "decade"; + ASSERT_THROW(WebSearch::validate("t", s), ApiError); +} + +TEST(retrieval_core, data_plane_host_must_be_a_bare_dns_name) { + Rig rig; + for (const char* bad : {"https://x.svc.pinecone.io", "x.svc.pinecone.io/path", "x svc", "x:443", "..x", "-x"}) + ASSERT_THROW(Pinecone::with(rig.factory(), kKey, bad), ApiError); + Pinecone ok = Pinecone::with(rig.factory(), kKey, "my-index-abc123.svc.aped-4627-b74a.pinecone.io"); + ASSERT_EQ(ok.gateway().hosts().size(), std::size_t(3)); +} diff --git a/tests/src/vendors/retrieval/test_live.cpp b/tests/src/vendors/retrieval/test_live.cpp new file mode 100644 index 0000000..e38f133 --- /dev/null +++ b/tests/src/vendors/retrieval/test_live.cpp @@ -0,0 +1,166 @@ +// Opt-in live checks. MOLO_LIVE_NET=1 enables the network. Part 1 needs no key: +// a deliberately invalid key must be refused with an auth-style status, which +// proves host, TLS and path. Part 2 runs read-only calls when the provider key +// is configured (via llm::Config). Calls that cost money additionally need +// MOLO_LIVE_PAID=1. Without the gates each test prints a loud SKIP. Nothing is +// created, changed or deleted. +#include +#include + +#include "clients/transports/http_transport.h" +#include "llm/config.h" +#include "support.h" +#include "vendors/retrieval/brave/brave.h" +#include "vendors/retrieval/exa/exa.h" +#include "vendors/retrieval/jina/jina.h" +#include "vendors/retrieval/mixedbread/mixedbread.h" +#include "vendors/retrieval/pinecone/pinecone.h" +#include "vendors/retrieval/qdrant/qdrant.h" +#include "vendors/retrieval/tavily/tavily.h" +#include "vendors/retrieval/voyage/voyage.h" +#include "vendors/retrieval/weaviate/weaviate.h" + +using namespace molo::retrieval; +using molo::llm::ApiError; + +namespace { +// Read once at start-up: tests/src/llm unsets MOLO_LIVE_NET in a test that runs earlier. +const bool kNet = std::getenv("MOLO_LIVE_NET") != nullptr; +const bool kPaid = std::getenv("MOLO_LIVE_PAID") != nullptr; +bool net() { return kNet; } +bool paid() { return kPaid; } +bool key(const char* env) { return !molo::llm::Config::get(env).empty(); } + +TransportFactory http() { + return [] { return std::unique_ptr(new molo::clients::transports::HttpTransport()); }; +} + +// Runs `call` with an invalid key and prints the status; accepts 401, 403 and +// 422. Returns a problem description, empty when fine. `tls12_only` marks a +// host that only speaks TLS 1.2: src/clients pins TLS 1.3, so the handshake +// fails there (a foundation gap listed in SPEC.md); that outcome is printed +// loudly as a FINDING instead of failing the suite. +template +std::string probe(const char* name, F call, bool tls12_only = false) { + if (!net()) { + std::cout << "SKIP retrieval_live.unauthenticated_probe " << name << " (set MOLO_LIVE_NET=1)\n"; + return ""; + } + try { + call(); + return std::string(name) + ": an invalid key was accepted"; + } catch (const ApiError& e) { + std::cout << " " << name << " with an invalid key: " << e.http_status() << " " << e.type() << " " + << e.message().substr(0, 160) << "\n"; + if (e.http_status() == 401 || e.http_status() == 403 || e.http_status() == 422) return ""; + if (tls12_only && e.type() == "UNAVAILABLE" && e.message().find("handshake") != std::string::npos) { + std::cout << "FINDING retrieval_live: " << name << " is TLS 1.2 only and src/clients pins TLS 1.3: unreachable until the transport allows 1.2\n"; + return ""; + } + return std::string(name) + ": unexpected " + std::to_string(e.http_status()) + " " + e.type(); + } +} + +bool gate(const char* test, const char* env, bool costs) { + if (!net() || !key(env) || (costs && !paid())) { + std::cout << "SKIP retrieval_live." << test << " (set MOLO_LIVE_NET=1 and " << env << (costs ? " and MOLO_LIVE_PAID=1" : "") << ")\n"; + return false; + } + return true; +} +} // namespace + +TEST(retrieval_live, unauthenticated_probes_answer_with_an_auth_status) { + const std::string bad = "invalid-molo-probe-key"; + std::string problems; + auto add = [&](const std::string& p) { problems += p.empty() ? "" : p + "; "; }; + add(probe("voyage", [&] { + EmbedRequest r; + r.texts = {"probe"}; + Voyage::with(http(), bad).embed(r); + })); + add(probe("jina", [&] { + EmbedRequest r; + r.texts = {"probe"}; + Jina::with(http(), bad).embed(r); + })); + add(probe("pinecone", [&] { Pinecone::with(http(), bad).list_indexes(); })); + add(probe("mixedbread", [&] { Mixedbread::with(http(), bad).list_stores(); }, true)); + add(probe("tavily", [&] { Tavily::with(http(), bad).usage(); })); + add(probe("exa", [&] { + SearchRequest s; + s.query = "probe"; + Exa::with(http(), bad).search(s); + })); + add(probe("brave", [&] { + SearchRequest s; + s.query = "probe"; + Brave::with(http(), bad).search(s); + })); + if (!problems.empty()) FAIL(problems); +} + +TEST(retrieval_live, pinecone_lists_indexes) { + if (!gate("pinecone_lists_indexes", Pinecone::kKeyEnv, false)) return; + Pinecone p = Pinecone::from_env(); + Json out = p.list_indexes(); + std::cout << " pinecone indexes: " << J::array(out, "indexes").size() << "\n"; + ASSERT_TRUE(out.is_object()); +} + +TEST(retrieval_live, qdrant_lists_collections) { + if (!gate("qdrant_lists_collections", Qdrant::kKeyEnv, false) || !key(Qdrant::kUrlEnv)) { + std::cout << "SKIP retrieval_live.qdrant_lists_collections (needs QDRANT_URL too)\n"; + return; + } + Json out = Qdrant::from_env().list_collections(); + ASSERT_TRUE(out.has("result")); +} + +TEST(retrieval_live, weaviate_dumps_the_schema) { + if (!gate("weaviate_dumps_the_schema", Weaviate::kKeyEnv, false) || !key(Weaviate::kUrlEnv)) { + std::cout << "SKIP retrieval_live.weaviate_dumps_the_schema (needs WEAVIATE_URL too)\n"; + return; + } + Json out = Weaviate::from_env().schema(); + ASSERT_TRUE(out.is_object()); +} + +TEST(retrieval_live, mixedbread_lists_stores) { + if (!gate("mixedbread_lists_stores", Mixedbread::kKeyEnv, false)) return; + ASSERT_TRUE(Mixedbread::from_env().list_stores().is_object()); +} + +TEST(retrieval_live, one_tiny_call_per_metered_provider) { + if (!gate("voyage_one_embedding", Voyage::kKeyEnv, true)) { + } else { + EmbedRequest r; + r.texts = {"hello"}; + EmbedResult out = Voyage::from_env().embed(r); + std::cout << " voyage dims: " << out.vectors.at(0).size() << "\n"; + ASSERT_TRUE(!out.vectors.at(0).empty()); + } + if (gate("jina_one_embedding", Jina::kKeyEnv, true)) { + EmbedRequest r; + r.texts = {"hello"}; + ASSERT_TRUE(!Jina::from_env().embed(r).vectors.at(0).empty()); + } + if (gate("tavily_one_search", Tavily::kKeyEnv, true)) { + SearchRequest s; + s.query = "moloapp"; + s.max_results = 1; + ASSERT_TRUE(Tavily::from_env().search(s).hits.size() <= 1); + } + if (gate("exa_one_search", Exa::kKeyEnv, true)) { + SearchRequest s; + s.query = "moloapp"; + s.max_results = 1; + ASSERT_TRUE(Exa::from_env().search(s).hits.size() <= 1); + } + if (gate("brave_one_search", Brave::kKeyEnv, true)) { + SearchRequest s; + s.query = "moloapp"; + s.max_results = 1; + ASSERT_TRUE(Brave::from_env().search(s).hits.size() <= 1); + } +} diff --git a/tests/src/vendors/retrieval/test_providers.cpp b/tests/src/vendors/retrieval/test_providers.cpp new file mode 100644 index 0000000..7ca3a67 --- /dev/null +++ b/tests/src/vendors/retrieval/test_providers.cpp @@ -0,0 +1,577 @@ +// Provider classes over a recording transport. Assertions are on the request +// our code builds (method, target, headers, body) and on our parsing. Replies +// are FIXTURES: bodies shaped after each provider's spec or SDK (schema names +// are cited), hand-built, not recorded from the live service. +#include "support.h" +#include "vendors/retrieval/brave/brave.h" +#include "vendors/retrieval/exa/exa.h" +#include "vendors/retrieval/jina/jina.h" +#include "vendors/retrieval/mixedbread/mixedbread.h" +#include "vendors/retrieval/pinecone/pinecone.h" +#include "vendors/retrieval/qdrant/qdrant.h" +#include "vendors/retrieval/tavily/tavily.h" +#include "vendors/retrieval/voyage/voyage.h" +#include "vendors/retrieval/weaviate/weaviate.h" + +using namespace molo::retrieval; +using namespace retrieval_test; +using molo::llm::ApiError; + +namespace { +Json parse(const std::string& s) { return Json::parse(s); } +std::string header(const MockTransport::SeenRequest& r, const char* name) { + auto v = r.headers.get(name); + return v ? *v : ""; +} +} // namespace + +// ---- Voyage ---------------------------------------------------------------- + +// FIXTURE: voyageai SDK objects (data[].embedding/index, usage.total_tokens). +static const char* kVoyageEmbeddings = + "{\"object\":\"list\",\"data\":[{\"object\":\"embedding\",\"embedding\":[0.5,0.25],\"index\":1}," + "{\"object\":\"embedding\",\"embedding\":[1.0,2.0],\"index\":0}],\"model\":\"voyage-3.5\",\"usage\":{\"total_tokens\":7}}"; + +TEST(retrieval_voyage, embed_builds_the_request_and_orders_by_index) { + Rig rig; + Voyage v = Voyage::with(rig.factory(), kKey); + rig.answer(0, Method::kPost, "/v1/embeddings", kVoyageEmbeddings); + EmbedRequest r; + r.texts = {"first", "second"}; + r.input_type = "document"; + r.dimensions = 256; + EmbedResult out = v.embed(r); + auto seen = rig.last(); + ASSERT_EQ(seen.method, std::string("POST")); + ASSERT_EQ(seen.target, std::string("/v1/embeddings")); + ASSERT_EQ(header(seen, "Authorization"), std::string("Bearer ") + kKey); + Json body = parse(seen.body); + ASSERT_EQ(body.find("model")->as_string(), std::string("voyage-3.5")); + ASSERT_EQ(body.find("input_type")->as_string(), std::string("document")); + ASSERT_EQ(body.find("output_dimension")->as_int(), 256L); + ASSERT_EQ(body.find("input")->as_array().size(), std::size_t(2)); + ASSERT_EQ(out.vectors.size(), std::size_t(2)); + ASSERT_EQ(out.vectors[0][0], 1.0f); // index 0 first although the reply listed it second + ASSERT_EQ(out.vectors[1][1], 0.25f); + ASSERT_EQ(out.total_tokens, 7L); +} + +TEST(retrieval_voyage, a_reply_with_the_wrong_vector_count_is_a_loud_error) { + Rig rig; + Voyage v = Voyage::with(rig.factory(), kKey); + rig.answer(0, Method::kPost, "/v1/embeddings", kVoyageEmbeddings); + EmbedRequest r; + r.texts = {"only one"}; + try { + v.embed(r); + FAIL("expected BAD_RESPONSE"); + } catch (const ApiError& e) { + ASSERT_EQ(e.type(), std::string("BAD_RESPONSE")); + } +} + +TEST(retrieval_voyage, rerank_maps_top_n_to_top_k_and_sorts_best_first) { + Rig rig; + Voyage v = Voyage::with(rig.factory(), kKey); + // FIXTURE: voyageai RerankingObject (data[].index, relevance_score, usage.total_tokens) + rig.answer(0, Method::kPost, "/v1/rerank", + "{\"object\":\"list\",\"data\":[{\"index\":0,\"relevance_score\":0.2},{\"index\":2,\"relevance_score\":0.9}]," + "\"model\":\"rerank-2.5\",\"usage\":{\"total_tokens\":11}}"); + RerankRequest r; + r.query = "q"; + r.documents = {"a", "b", "c"}; + r.top_n = 2; + RerankResult out = v.rerank(r); + Json body = parse(rig.last().body); + ASSERT_EQ(body.find("top_k")->as_int(), 2L); + ASSERT_EQ(body.find("model")->as_string(), std::string("rerank-2.5")); + ASSERT_EQ(out.hits.size(), std::size_t(2)); + ASSERT_EQ(out.hits[0].index, 2); + ASSERT_EQ(out.total_tokens, 11L); +} + +TEST(retrieval_voyage, multimodal_and_contextualized_bodies) { + Rig rig; + Voyage v = Voyage::with(rig.factory(), kKey); + rig.answer(0, Method::kPost, "/v1/multimodalembeddings", + "{\"data\":[{\"embedding\":[0.1],\"index\":0}],\"model\":\"voyage-multimodal-3\",\"usage\":{\"text_tokens\":3,\"image_pixels\":100,\"total_tokens\":4}}"); + VoyageSegment text{VoyageSegment::Kind::kText, "a cat"}; + VoyageSegment img{VoyageSegment::Kind::kImageUrl, "https://example.com/cat.png"}; + EmbedResult m = v.multimodal_embed({{text, img}}, "", "query"); + Json body = parse(rig.last().body); + const Json& content = *body.find("inputs")->as_array()[0].find("content"); + ASSERT_EQ(content.as_array()[0].find("type")->as_string(), std::string("text")); + ASSERT_EQ(content.as_array()[1].find("image_url")->as_string(), std::string("https://example.com/cat.png")); + ASSERT_EQ(m.vectors.size(), std::size_t(1)); + ASSERT_THROW(v.multimodal_embed({}, "", ""), ApiError); + ASSERT_THROW(v.multimodal_embed({{}}, "", ""), ApiError); + + // FIXTURE: voyageai ContextualizedEmbeddingsObject (data[].data[].embedding) + rig.answer(0, Method::kPost, "/v1/contextualizedembeddings", + "{\"data\":[{\"index\":0,\"data\":[{\"embedding\":[1,2],\"index\":0},{\"embedding\":[3,4],\"index\":1}]}],\"model\":\"voyage-context-3\",\"usage\":{\"total_tokens\":9}}"); + VoyageContextualResult c = v.contextualized_embed({{"chunk one", "chunk two"}}, "", "document", 0); + ASSERT_EQ(c.embeddings.size(), std::size_t(1)); + ASSERT_EQ(c.embeddings[0].size(), std::size_t(2)); + ASSERT_EQ(c.embeddings[0][1][1], 4.0f); + ASSERT_EQ(c.total_tokens, 9L); +} + +// ---- Jina ------------------------------------------------------------------ + +TEST(retrieval_jina, embed_rerank_and_the_four_hosts) { + Rig rig; + Jina j = Jina::with(rig.factory(), kKey); + ASSERT_EQ(rig.transports.size(), std::size_t(4)); // api, reader, search, segment + // FIXTURE: jina openapi EmbeddingResponse (data[].index/embedding, usage.total_tokens) + rig.answer(0, Method::kPost, "/v1/embeddings", + "{\"model\":\"jina-embeddings-v3\",\"object\":\"list\",\"usage\":{\"total_tokens\":5},\"data\":[{\"object\":\"embedding\",\"index\":0,\"embedding\":[0.1,0.2]}]}"); + EmbedRequest e; + e.texts = {"hello"}; + e.input_type = "query"; + EmbedResult er = j.embed(e); + Json body = parse(rig.last(0).body); + ASSERT_EQ(body.find("task")->as_string(), std::string("retrieval.query")); + ASSERT_EQ(er.total_tokens, 5L); + + // FIXTURE: jina openapi RerankingResponse (results[].index/relevance_score) + rig.answer(0, Method::kPost, "/v1/rerank", + "{\"model\":\"jina-reranker-v3\",\"object\":\"list\",\"usage\":{\"total_tokens\":8},\"results\":[{\"index\":1,\"relevance_score\":0.3},{\"index\":0,\"relevance_score\":0.8}]}"); + RerankRequest r; + r.query = "q"; + r.documents = {"a", "b"}; + RerankResult rr = j.rerank(r); + ASSERT_EQ(rr.hits[0].index, 0); + ASSERT_EQ(parse(rig.last(0).body).find("return_documents")->as_bool(), false); +} + +TEST(retrieval_jina, reader_search_segmenter_classifier_requests) { + Rig rig; + Jina j = Jina::with(rig.factory(), kKey); + // reader is host 1 + rig.answer(1, Method::kGet, "/", "{\"code\":200,\"status\":20000,\"data\":{\"title\":\"Example\",\"url\":\"https://example.com\",\"content\":\"# Hi\",\"usage\":{\"tokens\":12}}}"); + JinaPage p = j.read("https://example.com/page?x=1"); + auto seen = rig.last(1); + ASSERT_TRUE(has(seen.target, "url=https%3A%2F%2Fexample.com%2Fpage%3Fx%3D1")); + ASSERT_EQ(header(seen, "Accept"), std::string("application/json")); + ASSERT_EQ(p.title, std::string("Example")); + ASSERT_EQ(p.tokens, 12L); + ASSERT_THROW(j.read("ftp://example.com"), ApiError); + ASSERT_THROW(j.read("https://example.com/a b"), ApiError); + // search is host 2 + rig.answer(2, Method::kGet, "/search", "{\"code\":200,\"data\":[{\"title\":\"T\",\"url\":\"https://t.example\",\"description\":\"D\",\"content\":\"C\"}]}"); + SearchRequest s; + s.query = "jina ai"; + SearchResult sr = j.search(s); + ASSERT_TRUE(has(rig.last(2).target, "/search?q=jina%20ai") || has(rig.last(2).target, "/search?q=jina+ai")); + ASSERT_EQ(sr.hits.size(), std::size_t(1)); + ASSERT_EQ(sr.hits[0].snippet, std::string("D")); + s.recency = "day"; + ASSERT_THROW(j.search(s), ApiError); // no recency filter at jina + // segmenter is host 3 (documented shape, UNVERIFIED against a spec) + rig.answer(3, Method::kPost, "/", "{\"num_tokens\":4,\"num_chunks\":2,\"chunks\":[\"a b\",\"c d\"]}"); + JinaSegments seg = j.segment("a b c d", "cl100k_base", 2); + ASSERT_EQ(seg.chunks.size(), std::size_t(2)); + Json sb = parse(rig.last(3).body); + ASSERT_EQ(sb.find("return_chunks")->as_bool(), true); + ASSERT_EQ(sb.find("max_chunk_length")->as_int(), 2L); + // classifier (documented shape, UNVERIFIED): api host + rig.answer(0, Method::kPost, "/v1/classify", "{\"data\":[{\"index\":0,\"prediction\":\"spam\",\"score\":0.9}]}"); + auto cls = j.classify({"win money"}, {"spam", "ham"}, ""); + ASSERT_EQ(cls[0].label, std::string("spam")); + ASSERT_THROW(j.classify({"x"}, {"only-one"}, ""), ApiError); +} + +TEST(retrieval_jina, cancelling_a_batch_is_the_destructive_explicit_method) { + Rig rig; + Jina j = Jina::with(rig.factory(), kKey); + rig.answer(0, Method::kDelete, "/v1/batch/{batch_id}", "{\"batch_id\":\"b1\",\"status\":\"cancelled\"}"); + j.cancel_batch("b1"); + ASSERT_EQ(rig.last(0).method, std::string("DELETE")); + ASSERT_EQ(rig.last(0).target, std::string("/v1/batch/b1")); + ASSERT_THROW(j.cancel_batch("../x"), ApiError); + ASSERT_THROW(j.gateway().call("cancel_batch", CallArgs{}), ApiError); +} + +// ---- Pinecone -------------------------------------------------------------- + +TEST(retrieval_pinecone, headers_and_control_plane) { + Rig rig; + Pinecone p = Pinecone::with(rig.factory(), kKey); + rig.answer(0, Method::kGet, "/indexes/docs", "{\"name\":\"docs\",\"host\":\"docs-abc.svc.pinecone.io\"}"); + Json d = p.describe_index("docs"); + auto seen = rig.last(0); + ASSERT_EQ(header(seen, "Api-Key"), std::string(kKey)); + ASSERT_EQ(header(seen, "X-Pinecone-Api-Version"), std::string("2026-07")); + ASSERT_TRUE(header(seen, "Authorization").empty()); // key travels in Api-Key only + ASSERT_EQ(d.find("host")->as_string(), std::string("docs-abc.svc.pinecone.io")); + rig.answer(0, Method::kPost, "/indexes", "{\"name\":\"docs\"}"); + p.create_index("docs", 1024, "cosine", "aws", "us-east-1"); + Json b = parse(rig.last(0).body); + ASSERT_EQ(b.find("schema")->find("fields")->find("vector")->find("dimension")->as_int(), 1024L); + ASSERT_EQ(b.find("deployment")->find("deployment_type")->as_string(), std::string("managed")); + ASSERT_THROW(p.create_index("docs", 0, "cosine", "aws", "us-east-1"), ApiError); + ASSERT_THROW(p.create_index("docs", 8, "manhattan", "aws", "us-east-1"), ApiError); + rig.answer(0, Method::kDelete, "/indexes/docs", ""); + p.delete_index("docs"); + ASSERT_EQ(rig.last(0).method, std::string("DELETE")); +} + +TEST(retrieval_pinecone, data_plane_needs_the_index_host) { + Rig none; + Pinecone p0 = Pinecone::with(none.factory(), kKey); + PineconeQuery q; + q.vector = {1.0f}; + ASSERT_THROW(p0.query(q), ApiError); // no data client + ASSERT_EQ(none.sent_total(), std::size_t(0)); + + Rig rig; + Pinecone p = Pinecone::with(rig.factory(), kKey, "docs-abc.svc.pinecone.io"); + // FIXTURE: pinecone db_data QueryResponse (matches[].id/score/values/metadata) + rig.answer(2, Method::kPost, "/query", "{\"matches\":[{\"id\":\"v1\",\"score\":0.99,\"metadata\":{\"genre\":\"x\"}}],\"namespace\":\"ns\"}"); + q.top_k = 3; + q.ns = "ns"; + Json filter = Json::object(); + filter.set("genre", Json("x")); + q.filter = filter; + auto m = p.query(q); + Json b = parse(rig.last(2).body); + ASSERT_EQ(b.find("topK")->as_int(), 3L); + ASSERT_EQ(b.find("namespace")->as_string(), std::string("ns")); + ASSERT_EQ(m[0].id, std::string("v1")); + ASSERT_EQ(m[0].metadata.find("genre")->as_string(), std::string("x")); + PineconeQuery both = q; + both.id = "v1"; + ASSERT_THROW(p.query(both), ApiError); // vector and id together + q.top_k = 10001; + ASSERT_THROW(p.query(q), ApiError); +} + +TEST(retrieval_pinecone, fetch_repeats_the_ids_parameter_and_list_pages) { + Rig rig; + Pinecone p = Pinecone::with(rig.factory(), kKey, "docs-abc.svc.pinecone.io"); + rig.answer(2, Method::kGet, "/vectors/fetch", "{\"vectors\":{},\"namespace\":\"\"}"); + p.fetch({"a", "b"}, "ns"); + ASSERT_EQ(rig.last(2).target, std::string("/vectors/fetch?namespace=ns&ids=a&ids=b")); + ASSERT_THROW(p.fetch({}, ""), ApiError); + ASSERT_THROW(p.fetch(std::vector(101, "x"), ""), ApiError); + // FIXTURE: pinecone db_data ListResponse (vectors[].id, pagination.next) + rig.answer(2, Method::kGet, "/vectors/list", "{\"vectors\":[{\"id\":\"a\"},{\"id\":\"b\"}],\"pagination\":{\"next\":\"tok\"}}"); + PineconePage page = p.list("pre", "ns", 2, ""); + ASSERT_EQ(page.ids.size(), std::size_t(2)); + ASSERT_EQ(page.next_token, std::string("tok")); + ASSERT_THROW(p.list("", "", 101, ""), ApiError); +} + +TEST(retrieval_pinecone, the_three_deletes_share_one_destructive_row_and_stay_explicit) { + Rig rig; + Pinecone p = Pinecone::with(rig.factory(), kKey, "docs-abc.svc.pinecone.io"); + rig.answer(2, Method::kPost, "/vectors/delete", "{}"); + p.delete_ids({"a"}, "ns"); + ASSERT_TRUE(has(rig.last(2).body, "\"ids\":[\"a\"]")); + p.delete_all("ns"); + ASSERT_TRUE(has(rig.last(2).body, "\"deleteAll\":true")); + Json f = Json::object(); + f.set("genre", Json("x")); + p.delete_by_filter(f, ""); + ASSERT_TRUE(has(rig.last(2).body, "\"filter\"")); + ASSERT_THROW(p.delete_by_filter(Json::object(), ""), ApiError); // empty filter would match nothing useful + ASSERT_THROW(p.delete_ids({}, ""), ApiError); + ASSERT_THROW(p.gateway().call("delete_vectors", CallArgs{}), ApiError); +} + +TEST(retrieval_pinecone, inference_embed_and_rerank) { + Rig rig; + Pinecone p = Pinecone::with(rig.factory(), kKey); + // FIXTURE: pinecone inference EmbeddingsList (data[].values, usage.total_tokens) + rig.answer(1, Method::kPost, "/embed", "{\"model\":\"multilingual-e5-large\",\"vector_type\":\"dense\",\"data\":[{\"values\":[0.1,0.2]}],\"usage\":{\"total_tokens\":3}}"); + EmbedRequest e; + e.texts = {"hi"}; + e.input_type = "query"; + EmbedResult out = p.embed(e); + Json b = parse(rig.last(1).body); + ASSERT_EQ(b.find("parameters")->find("input_type")->as_string(), std::string("query")); + ASSERT_EQ(b.find("inputs")->as_array()[0].find("text")->as_string(), std::string("hi")); + ASSERT_EQ(out.vectors[0].size(), std::size_t(2)); + // FIXTURE: pinecone inference RerankResult (data[].index/score) + rig.answer(1, Method::kPost, "/rerank", "{\"model\":\"bge-reranker-v2-m3\",\"data\":[{\"index\":1,\"score\":0.7},{\"index\":0,\"score\":0.1}],\"usage\":{\"rerank_units\":1}}"); + RerankRequest r; + r.query = "q"; + r.documents = {"a", "b"}; + RerankResult rr = p.rerank(r); + ASSERT_EQ(rr.hits[0].index, 1); +} + +// ---- Qdrant ---------------------------------------------------------------- + +TEST(retrieval_qdrant, collections_points_and_query) { + Rig rig; + Qdrant q = Qdrant::with(rig.factory(), kKey, "https://xyz.cloud.qdrant.io:6333/"); + rig.answer(0, Method::kPut, "/collections/docs", "{\"result\":true,\"status\":\"ok\",\"time\":0.1}"); + q.create_collection("docs", 4, "Cosine", true); + auto seen = rig.last(); + ASSERT_EQ(header(seen, "api-key"), std::string(kKey)); + Json b = parse(seen.body); + ASSERT_EQ(b.find("vectors")->find("size")->as_int(), 4L); + ASSERT_EQ(b.find("vectors")->find("distance")->as_string(), std::string("Cosine")); + ASSERT_THROW(q.create_collection("docs", 4, "cosine", true), ApiError); // distance is case sensitive + + rig.answer(0, Method::kPut, "/collections/docs/points", "{\"result\":{\"operation_id\":1,\"status\":\"completed\"},\"status\":\"ok\"}"); + QdrantPoint pt; + pt.id = Json(1); + pt.vector = {0.1f, 0.2f, 0.3f, 0.4f}; + Json payload = Json::object(); + payload.set("city", Json("Berlin")); + pt.payload = payload; + q.upsert("docs", {pt}, true); + ASSERT_EQ(rig.last().target, std::string("/collections/docs/points?wait=true")); + QdrantPoint bad = pt; + bad.id = Json(-1); + ASSERT_THROW(q.upsert("docs", {bad}, true), ApiError); + + // FIXTURE: qdrant openapi QueryResponse (result.points[].id/score/payload) + rig.answer(0, Method::kPost, "/collections/docs/points/query", + "{\"result\":{\"points\":[{\"id\":1,\"score\":0.9,\"payload\":{\"city\":\"Berlin\"}}]},\"status\":\"ok\"}"); + auto hits = q.query("docs", {0.1f, 0.2f, 0.3f, 0.4f}, 5, Json(), true); + ASSERT_EQ(hits.size(), std::size_t(1)); + ASSERT_EQ(hits[0].payload.find("city")->as_string(), std::string("Berlin")); + ASSERT_EQ(parse(rig.last().body).find("limit")->as_int(), 5L); + ASSERT_THROW(q.query("docs", {}, 5, Json(), true), ApiError); + ASSERT_THROW(q.query("docs", {1.0f}, 0, Json(), true), ApiError); +} + +TEST(retrieval_qdrant, deleting_points_is_explicit_and_filter_must_not_be_empty) { + Rig rig; + Qdrant q = Qdrant::with(rig.factory(), kKey, "https://qd.example.com"); + rig.answer(0, Method::kPost, "/collections/docs/points/delete", "{\"result\":{\"status\":\"completed\"},\"status\":\"ok\"}"); + q.delete_points("docs", {Json(1), Json("uuid-1")}); + ASSERT_EQ(rig.last().target, std::string("/collections/docs/points/delete?wait=true")); + ASSERT_THROW(q.delete_points("docs", {}), ApiError); + ASSERT_THROW(q.delete_points_by_filter("docs", Json::object()), ApiError); + ASSERT_THROW(q.gateway().call("delete_points", CallArgs{}), ApiError); +} + +TEST(retrieval_qdrant, count_exists_and_snapshots) { + Rig rig; + Qdrant q = Qdrant::with(rig.factory(), kKey, "https://qd.example.com"); + rig.answer(0, Method::kPost, "/collections/docs/points/count", "{\"result\":{\"count\":42},\"status\":\"ok\"}"); + ASSERT_EQ(q.count("docs", Json(), true), 42L); + rig.answer(0, Method::kGet, "/collections/docs/exists", "{\"result\":{\"exists\":true},\"status\":\"ok\"}"); + ASSERT_TRUE(q.collection_exists("docs")); + rig.answer(0, Method::kDelete, "/collections/docs/snapshots/{snapshot_name}", "{\"result\":true,\"status\":\"ok\"}"); + q.delete_snapshot("docs", "snap-1.snapshot"); + ASSERT_EQ(rig.last().target, std::string("/collections/docs/snapshots/snap-1.snapshot")); +} + +// ---- Weaviate -------------------------------------------------------------- + +TEST(retrieval_weaviate, paths_carry_the_v1_base_and_class_names_are_checked) { + Rig rig; + Weaviate w = Weaviate::with(rig.factory(), kKey, "https://my.weaviate.network/"); + rig.answer(0, Method::kGet, "/v1/schema", "{\"classes\":[]}"); + w.schema(); + ASSERT_EQ(rig.last().target, std::string("/v1/schema")); + ASSERT_EQ(header(rig.last(), "Authorization"), std::string("Bearer ") + kKey); + ASSERT_THROW(w.get_class("lowercase"), ApiError); + ASSERT_THROW(w.get_class("Bad-Name"), ApiError); + Json def = Json::object(); + def.set("description", Json("no class")); + ASSERT_THROW(w.create_class(def), ApiError); + rig.answer(0, Method::kDelete, "/v1/schema/{className}", ""); + w.delete_class("Article"); + ASSERT_EQ(rig.last().method, std::string("DELETE")); + ASSERT_EQ(rig.last().target, std::string("/v1/schema/Article")); +} + +TEST(retrieval_weaviate, objects_graphql_and_search_bodies) { + Rig rig; + Weaviate w = Weaviate::with(rig.factory(), kKey, "https://wv.example.com"); + rig.answer(0, Method::kPost, "/v1/objects", "{\"id\":\"uuid\",\"class\":\"Article\"}"); + Json props = Json::object(); + props.set("title", Json("Hello")); + w.create_object("Article", props, {0.1f, 0.2f}); + Json b = parse(rig.last().body); + ASSERT_EQ(b.find("class")->as_string(), std::string("Article")); + ASSERT_EQ(b.find("vector")->as_array().size(), std::size_t(2)); + rig.answer(0, Method::kPost, "/v1/graphql", "{\"data\":{\"Get\":{}}}"); + w.graphql("{ Get { Article { title } } }", Json()); + ASSERT_TRUE(has(rig.last().body, "\"query\"")); + rig.answer(0, Method::kPost, "/v1/search/{collection}/near-text", "{\"results\":[],\"tookMs\":3}"); + w.near_text("Article", "hello", 3); + ASSERT_EQ(rig.last().target, std::string("/v1/search/Article/near-text")); + ASSERT_THROW(w.near_text("Article", "", 3), ApiError); + ASSERT_THROW(w.hybrid("Article", "x", 3, 1.5), ApiError); + // batch delete is the destructive explicit method and defaults to nothing + rig.answer(0, Method::kDelete, "/v1/batch/objects", "{\"results\":{}}"); + Json match = Json::object(); + match.set("class", Json("Article")); + w.batch_delete(match, true); + ASSERT_TRUE(has(rig.last().body, "\"dryRun\":true")); + ASSERT_THROW(w.batch_delete(Json::object(), true), ApiError); +} + +// ---- Mixedbread ------------------------------------------------------------ + +TEST(retrieval_mixedbread, embed_rerank_and_caps) { + Rig rig; + Mixedbread m = Mixedbread::with(rig.factory(), kKey); + // FIXTURE: mixedbread openapi EmbeddingCreateResponse (data[].embedding/index, usage.total_tokens) + rig.answer(0, Method::kPost, "/v1/embeddings", + "{\"object\":\"list\",\"model\":\"mixedbread-ai/mxbai-embed-large-v1\",\"data\":[{\"object\":\"embedding\",\"index\":0,\"embedding\":[0.5,0.5]}],\"usage\":{\"prompt_tokens\":2,\"total_tokens\":2}}"); + EmbedRequest e; + e.texts = {"hi"}; + EmbedResult out = m.embed(e); + ASSERT_EQ(out.vectors.size(), std::size_t(1)); + ASSERT_EQ(parse(rig.last().body).find("model")->as_string(), std::string("mixedbread-ai/mxbai-embed-large-v1")); + e.texts.assign(257, "x"); + ASSERT_THROW(m.embed(e), ApiError); // spec: at most 256 inputs + // FIXTURE: mixedbread openapi RerankResponse (data[].index/score; the spec's own example values) + rig.answer(0, Method::kPost, "/v1/reranking", + "{\"object\":\"list\",\"model\":\"mixedbread-ai/mxbai-rerank-v3.1-listwise\",\"data\":[{\"index\":1,\"score\":0.94,\"object\":\"rank_result\"},{\"index\":0,\"score\":0.95,\"object\":\"rank_result\"}]}"); + RerankRequest r; + r.query = "q"; + r.documents = {"a", "b"}; + RerankResult rr = m.rerank(r); + ASSERT_EQ(rr.hits[0].index, 0); + ASSERT_EQ(parse(rig.last().body).find("top_k"), static_cast(nullptr)); + ASSERT_THROW(m.search_stores({}, "q", 3), ApiError); + ASSERT_THROW(m.search_stores({"s1"}, "q", 101), ApiError); + rig.answer(0, Method::kDelete, "/v1/stores/{store_identifier}", "{}"); + m.delete_store("s1"); + ASSERT_EQ(rig.last().method, std::string("DELETE")); +} + +// ---- Tavily ---------------------------------------------------------------- + +TEST(retrieval_tavily, search_extract_crawl_map) { + Rig rig; + Tavily t = Tavily::with(rig.factory(), kKey); + // FIXTURE: tavily openapi search response (answer, results[].title/url/content/score) + rig.answer(0, Method::kPost, "/search", + "{\"query\":\"q\",\"answer\":\"A\",\"results\":[{\"title\":\"T\",\"url\":\"https://t.example\",\"content\":\"C\",\"score\":0.8,\"published_date\":\"2026-01-02\"}],\"response_time\":0.5}"); + SearchRequest s; + s.query = "q"; + s.max_results = 3; + s.recency = "week"; + TavilyOptions o; + o.include_answer = true; + o.search_depth = "advanced"; + SearchResult r = t.search(s, o); + Json b = parse(rig.last().body); + ASSERT_EQ(b.find("time_range")->as_string(), std::string("week")); + ASSERT_EQ(b.find("search_depth")->as_string(), std::string("advanced")); + ASSERT_EQ(b.find("include_answer")->as_bool(), true); + ASSERT_EQ(header(rig.last(), "Authorization"), std::string("Bearer ") + kKey); + ASSERT_EQ(r.answer, std::string("A")); + ASSERT_TRUE(r.hits[0].has_score); + s.max_results = 21; + ASSERT_THROW(t.search(s), ApiError); + o.search_depth = "turbo"; + s.max_results = 2; + ASSERT_THROW(t.search(s, o), ApiError); + + rig.answer(0, Method::kPost, "/extract", "{\"results\":[{\"url\":\"https://a.example\",\"raw_content\":\"body\"}],\"failed_results\":[{\"url\":\"https://b.example\",\"error\":\"x\"}]}"); + TavilyExtract x = t.extract({"https://a.example", "https://b.example"}, true); + ASSERT_EQ(x.pages.size(), std::size_t(1)); + ASSERT_EQ(x.failed[0], std::string("https://b.example")); + ASSERT_THROW(t.extract(std::vector(21, "https://a.example"), false), ApiError); + ASSERT_THROW(t.extract({"file:///etc/passwd"}, false), ApiError); + + rig.answer(0, Method::kPost, "/crawl", "{\"base_url\":\"https://a.example\",\"results\":[{\"url\":\"https://a.example/x\",\"raw_content\":\"c\"}]}"); + ASSERT_EQ(t.crawl("https://a.example", 10, 2, "").pages.size(), std::size_t(1)); + ASSERT_THROW(t.crawl("https://a.example", 501, 2, ""), ApiError); + ASSERT_THROW(t.crawl("https://a.example", 10, 6, ""), ApiError); + rig.answer(0, Method::kPost, "/map", "{\"base_url\":\"https://a.example\",\"results\":[\"https://a.example/1\",\"https://a.example/2\"]}"); + ASSERT_EQ(t.map("https://a.example", 10, 1).size(), std::size_t(2)); +} + +// ---- Exa ------------------------------------------------------------------- + +TEST(retrieval_exa, search_findsimilar_contents_answer) { + Rig rig; + Exa x = Exa::with(rig.factory(), kKey); + // FIXTURE: exa openapi Result (title/url/publishedDate/score) with a highlight + rig.answer(0, Method::kPost, "/search", + "{\"requestId\":\"r1\",\"results\":[{\"id\":\"i\",\"title\":\"T\",\"url\":\"https://t.example\",\"publishedDate\":\"2026-01-01\",\"score\":0.4,\"highlights\":[\"H\"]}]}"); + SearchRequest s; + s.query = "q"; + s.max_results = 4; + s.recency = "day"; + SearchResult r = x.search(s); + auto seen = rig.last(); + ASSERT_EQ(header(seen, "x-api-key"), std::string(kKey)); + Json b = parse(seen.body); + ASSERT_EQ(b.find("numResults")->as_int(), 4L); + ASSERT_TRUE(b.has("startPublishedDate")); + ASSERT_EQ(b.find("contents")->find("text")->find("maxCharacters")->as_int(), 500L); + ASSERT_EQ(r.hits[0].snippet, std::string("H")); + ASSERT_THROW(x.search_typed(s, "telepathy"), ApiError); + s.max_results = 101; + ASSERT_THROW(x.search(s), ApiError); + + rig.answer(0, Method::kPost, "/findSimilar", "{\"results\":[{\"title\":\"S\",\"url\":\"https://s.example\"}]}"); + ASSERT_EQ(x.find_similar("https://t.example", 2).hits.size(), std::size_t(1)); + ASSERT_THROW(x.find_similar("t.example", 2), ApiError); + rig.answer(0, Method::kPost, "/contents", "{\"results\":[{\"url\":\"https://t.example\",\"title\":\"T\",\"text\":\"full text\"}]}"); + ASSERT_EQ(x.contents({"https://t.example"}, 100).hits[0].snippet, std::string("full text")); + // FIXTURE: exa openapi AnswerResult (answer, citations[]) + rig.answer(0, Method::kPost, "/answer", "{\"answer\":\"42\",\"citations\":[{\"id\":\"c\",\"url\":\"https://c.example\",\"title\":\"C\"}]}"); + ExaAnswer a = x.answer("meaning of life"); + ASSERT_EQ(a.answer, std::string("42")); + ASSERT_EQ(a.citations.size(), std::size_t(1)); +} + +TEST(retrieval_exa, recency_dates_are_computed_not_guessed) { + ASSERT_EQ(Exa::days_ago_iso(86400L * 10, 1), std::string("1970-01-10T00:00:00.000Z")); + ASSERT_EQ(Exa::days_ago_iso(1767225600L, 7), std::string("2025-12-25T00:00:00.000Z")); // 2026-01-01 minus 7 days + ASSERT_EQ(Exa::recency_days("week"), 7); + ASSERT_EQ(Exa::recency_days(""), 0); +} + +// ---- Brave ----------------------------------------------------------------- + +TEST(retrieval_brave, web_news_images_suggest_and_caps) { + Rig rig; + Brave b = Brave::with(rig.factory(), kKey); + // FIXTURE: brave api-reference web search response (web.results[].title/url/description/page_age) + rig.answer(0, Method::kGet, "/res/v1/web/search", + "{\"type\":\"search\",\"web\":{\"type\":\"search\",\"results\":[{\"title\":\"T\",\"url\":\"https://t.example\",\"description\":\"D\",\"page_age\":\"2026-01-01T00:00:00\"}]}}"); + SearchRequest s; + s.query = "brave search"; + s.max_results = 5; + s.recency = "month"; + SearchResult r = b.search(s); + auto seen = rig.last(); + ASSERT_EQ(header(seen, "X-Subscription-Token"), std::string(kKey)); + ASSERT_TRUE(header(seen, "Authorization").empty()); + ASSERT_TRUE(has(seen.target, "q=brave%20search") || has(seen.target, "q=brave+search")); + ASSERT_TRUE(has(seen.target, "count=5")); + ASSERT_TRUE(has(seen.target, "freshness=pm")); + ASSERT_EQ(r.hits[0].snippet, std::string("D")); + s.max_results = 21; + ASSERT_THROW(b.search(s), ApiError); // web search maxes out at 20 + + rig.answer(0, Method::kGet, "/res/v1/news/search", "{\"type\":\"news\",\"results\":[{\"title\":\"N\",\"url\":\"https://n.example\",\"description\":\"d\",\"age\":\"2 days ago\"}]}"); + s.max_results = 50; + s.recency = ""; + ASSERT_EQ(b.news(s).hits[0].published, std::string("2 days ago")); + s.max_results = 51; + ASSERT_THROW(b.news(s), ApiError); + rig.answer(0, Method::kGet, "/res/v1/images/search", "{\"results\":[{\"title\":\"I\",\"url\":\"https://p.example\",\"source\":\"p.example\",\"properties\":{\"url\":\"https://p.example/i.png\"}}]}"); + ASSERT_EQ(b.images("cat", 3)[0].image_url, std::string("https://p.example/i.png")); + ASSERT_THROW(b.images("cat", 201), ApiError); + rig.answer(0, Method::kGet, "/res/v1/suggest/search", "{\"query\":{\"original\":\"bra\"},\"results\":[{\"query\":\"brave\"},{\"query\":\"bravo\"}]}"); + ASSERT_EQ(b.suggest("bra", 2).size(), std::size_t(2)); + ASSERT_THROW(b.suggest("", 2), ApiError); + ASSERT_THROW(b.suggest(std::string(401, 'x'), 2), ApiError); // documented 400 char limit +} + +TEST(retrieval_brave, answers_send_stream_false) { + Rig rig; + Brave b = Brave::with(rig.factory(), kKey); + // The non-streaming reply shape is not documented: this fixture is the OpenAI-style shape we parse. + rig.answer(0, Method::kPost, "/res/v1/chat/completions", "{\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"Because.\"}}]}"); + ASSERT_EQ(b.answer("why?"), std::string("Because.")); + Json body = parse(rig.last().body); + ASSERT_EQ(body.find("stream")->as_bool(), false); + ASSERT_EQ(body.find("messages")->as_array()[0].find("role")->as_string(), std::string("user")); +}