feat(clients): retrieval: Voyage, Jina, Pinecone, Qdrant, Weaviate, Mixedbread, Tavily, Exa, Brave

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-25 15:15:34 +01:00
co-authored by Claude Sonnet 5.5
parent 3db67fcf1f
commit 2ac5859cdb
48 changed files with 6040 additions and 3 deletions
+33
View File
@@ -0,0 +1,33 @@
#include "vendors/retrieval/jina/jina.h"
namespace molo::retrieval {
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
// name, host, http method, path, spec operation id (or reference page / "doc:"),
// body fields, query parameters, flags (d destructive, w writes, r read-only POST,
// p paid), response schema)
static const CallRow kRows[] = {
{"embeddings", "api", "POST", "/v1/embeddings", "embeddings_v1_embeddings_post", "dimensions,embedding_type,input,input_type,late_chunking,model,normalized,return_multivector,return_tokenized_input,task,truncate", "", "rp", "EmbeddingResponse"},
{"rerank", "api", "POST", "/v1/rerank", "rerank_v1_rerank_post", "documents,max_doc_length,model,query,return_documents,return_embeddings,top_n", "", "rp", "RerankingResponse"},
{"list_models", "api", "GET", "/v1/models", "list_models_v1_models_get", "", "", "", "ModelListResponse"},
{"get_model", "api", "GET", "/v1/models/{model_id}", "get_model_v1_models__model_id__get", "", "", "", "ModelInfo"},
{"create_batch", "api", "POST", "/v1/batch/embeddings", "create_batch_job_v1_batch_embeddings_post", "dimensions,input,input_url,model,normalized,task,webhook_url", "", "wp", "BatchStatus"},
{"get_batch", "api", "GET", "/v1/batch/{batch_id}", "get_batch_status_v1_batch__batch_id__get", "", "", "", "BatchStatus"},
{"cancel_batch", "api", "DELETE", "/v1/batch/{batch_id}", "cancel_batch_job_v1_batch__batch_id__delete", "", "", "d", "BatchStatus"},
{"list_batches", "api", "GET", "/v1/batches", "list_batch_jobs_v1_batches_get", "", "limit", "", "inline"},
{"batch_output", "api", "GET", "/v1/batch/{batch_id}/output", "download_batch_output_v1_batch__batch_id__output_get", "", "", "", "-"},
{"batch_errors", "api", "GET", "/v1/batch/{batch_id}/errors", "download_batch_errors_v1_batch__batch_id__errors_get", "", "", "", "-"},
{"read", "reader", "GET", "/", "", "", "assertStatusCode,base,cacheTolerance,detachInvisibles,engine,exportStorageState,file,html,injectFrameScript,injectPageScript,instruction,jsonSchema,keepImgDataUrl,locale,markdownChunking,maxTokens,noCache,noGfm,noServiceWorker,page,pdf,preloadUrl,preset,proxy,proxyResources,proxyUrl,referer,removeOverlay,removeSelector,respondTiming,respondWith,retainImages,retainLinks,retainMedia,robotsTxt,setCookies,skipCertVerification,targetSelector,timeout,tokenBudget,url,userAgent,waitForSelector,webBotAuth,withGeneratedAlt,withIframe,withImagesSummary,withLinksSummary,withShadowDom,zeroDataRetention", "p", "-"},
{"search", "search", "GET", "/search", "", "", "assertStatusCode,base,cacheTolerance,count,detachInvisibles,engine,exportStorageState,ext,fallback,file,filetype,gl,hl,html,injectFrameScript,injectPageScript,instruction,intitle,jsonSchema,keepImgDataUrl,loc,locale,location,markdownChunking,maxTokens,nfpr,noCache,noGfm,noServiceWorker,num,page,pdf,preloadUrl,preset,provider,proxy,proxyResources,proxyUrl,q,referer,removeOverlay,removeSelector,respondTiming,respondWith,retainImages,retainLinks,retainMedia,robotsTxt,setCookies,site,skipCertVerification,targetSelector,timeout,tokenBudget,type,url,userAgent,waitForSelector,webBotAuth,withGeneratedAlt,withIframe,withImagesSummary,withLinksSummary,withShadowDom,zeroDataRetention", "p", "inline"},
{"count_token", "segment", "POST", "/countToken", "", "content,head,max_chunk_length,return_chunks,return_tokens,tail,tokenizer", "content,head,max_chunk_length,return_chunks,return_tokens,tail,tokenizer", "rp", "-"},
{"segment", "segment", "POST", "/", "doc:segmenter", "content,tokenizer,return_tokens,return_chunks,max_chunk_length,head,tail", "", "p", "-"},
{"classify", "api", "POST", "/v1/classify", "doc:classifier", "model,input,labels", "", "p", "-"},
};
// CALLS-END
const CallTable& jina_calls() {
static const CallTable table{kRows, sizeof(kRows) / sizeof(kRows[0])};
return table;
}
} // namespace molo::retrieval
+217
View File
@@ -0,0 +1,217 @@
#include "vendors/retrieval/jina/jina.h"
#include <algorithm>
#include "vendors/retrieval/caps.h"
namespace molo::retrieval {
llm::ApiClientConfig Jina::config() {
llm::ApiClientConfig c;
c.vendor = "jina";
c.base_url = kApiUrl;
c.key_env = kKeyEnv;
c.auth = llm::AuthStyle::kBearer;
return c;
}
namespace {
std::vector<HostSpec> hosts() {
return {{"api", Jina::kApiUrl}, {"reader", Jina::kReaderUrl}, {"search", Jina::kSearchUrl}, {"segment", Jina::kSegmentUrl}};
}
llm::Headers json_accept() { return {{"Accept", "application/json"}}; }
} // namespace
Jina Jina::from_env() { return Jina(make_gateway(jina_calls(), config(), hosts(), "", nullptr)); }
Jina Jina::with(const TransportFactory& factory, std::string key) {
return Jina(make_gateway(jina_calls(), config(), hosts(), std::move(key), factory));
}
void Jina::check_url(const std::string& url) {
bool http = url.rfind("http://", 0) == 0 || url.rfind("https://", 0) == 0;
if (!http || url.size() > 8192 || url.find_first_of(" \t\r\n") != std::string::npos)
throw llm::ApiError::invalid("jina", "url must be http(s) without spaces, at most 8192 chars");
}
EmbedResult Jina::parse_embeddings(const Json& doc) {
EmbedResult out;
out.model = J::str(doc, "model");
std::vector<std::pair<int, std::vector<float>>> rows;
for (const Json& d : J::array(doc, "data")) {
const Json* e = J::at(d, "embedding");
rows.emplace_back(static_cast<int>(J::num(d, "index", static_cast<double>(rows.size()))),
e ? J::floats(*e) : std::vector<float>{});
}
std::stable_sort(rows.begin(), rows.end(), [](const auto& a, const auto& b) { return a.first < b.first; });
for (auto& r : rows) out.vectors.push_back(std::move(r.second));
if (const Json* u = J::at(doc, "usage")) out.total_tokens = static_cast<long>(J::num(*u, "total_tokens"));
return out;
}
RerankResult Jina::parse_rerank(const Json& doc) {
RerankResult out;
out.model = J::str(doc, "model");
for (const Json& d : J::array(doc, "results")) {
RerankHit h;
h.index = static_cast<int>(J::num(d, "index"));
h.score = J::num(d, "relevance_score");
out.hits.push_back(h);
}
std::stable_sort(out.hits.begin(), out.hits.end(), [](const RerankHit& a, const RerankHit& b) { return a.score > b.score; });
if (const Json* u = J::at(doc, "usage")) out.total_tokens = static_cast<long>(J::num(*u, "total_tokens"));
return out;
}
JinaPage Jina::parse_page(const Json& doc) {
JinaPage p;
const Json* data = J::at(doc, "data");
const Json& d = data && data->is_object() ? *data : doc;
p.title = J::str(d, "title");
p.url = J::str(d, "url");
p.description = J::str(d, "description");
p.content = J::str(d, "content");
if (const Json* u = J::at(d, "usage")) p.tokens = static_cast<long>(J::num(*u, "tokens"));
return p;
}
SearchResult Jina::parse_search(const Json& doc) {
SearchResult out;
for (const Json& d : J::array(doc, "data")) {
SearchHit h;
h.title = J::str(d, "title");
h.url = J::str(d, "url");
h.snippet = J::str(d, "description");
if (h.snippet.empty()) h.snippet = J::str(d, "content").substr(0, 500);
out.hits.push_back(h);
}
return out;
}
JinaSegments Jina::parse_segments(const Json& doc) {
JinaSegments s;
s.num_tokens = static_cast<long>(J::num(doc, "num_tokens"));
s.num_chunks = static_cast<long>(J::num(doc, "num_chunks"));
for (const Json& c : J::array(doc, "chunks"))
if (c.is_string()) s.chunks.push_back(c.as_string());
return s;
}
std::vector<JinaPrediction> Jina::parse_classification(const Json& doc) {
std::vector<JinaPrediction> out;
for (const Json& d : J::array(doc, "data")) {
JinaPrediction p;
p.index = static_cast<int>(J::num(d, "index", static_cast<double>(out.size())));
p.label = J::str(d, "prediction");
p.score = J::num(d, "score");
out.push_back(p);
}
return out;
}
EmbedResult Jina::embed(const EmbedRequest& request) {
Embedder::validate("jina", request);
Json body = Json::object();
body.set("model", Json(request.model.empty() ? std::string(kDefaultEmbedModel) : request.model));
body.set("input", J::strings(request.texts));
if (request.input_type == "query") body.set("task", Json("retrieval.query"));
if (request.input_type == "document") body.set("task", Json("retrieval.passage"));
if (request.dimensions > 0) body.set("dimensions", Json(request.dimensions));
CallArgs args;
args.body = body;
EmbedResult r = parse_embeddings(gw_->call("embeddings", args));
if (r.vectors.size() != request.texts.size())
throw llm::ApiError("jina", 502, "BAD_RESPONSE", "embedding count differs from input count");
return r;
}
RerankResult Jina::rerank(const RerankRequest& request) {
Reranker::validate("jina", request);
Json body = Json::object();
body.set("model", Json(request.model.empty() ? std::string(kDefaultRerankModel) : request.model));
body.set("query", Json(request.query));
body.set("documents", J::strings(request.documents));
body.set("return_documents", Json(false));
if (request.top_n > 0) body.set("top_n", Json(request.top_n));
CallArgs args;
args.body = body;
return parse_rerank(gw_->call("rerank", args));
}
SearchResult Jina::search(const SearchRequest& request) {
WebSearch::validate("jina", request);
if (!request.recency.empty()) throw llm::ApiError::invalid("jina", "jina search has no recency filter");
CallArgs args;
args.query["q"] = request.query;
args.headers = json_accept();
return parse_search(gw_->call("search", args));
}
JinaPage Jina::read(const std::string& url) {
check_url(url);
CallArgs args;
args.query["url"] = url;
args.headers = json_accept();
return parse_page(gw_->call("read", args));
}
JinaSegments Jina::segment(const std::string& content, const std::string& tokenizer, int max_chunk_length) {
if (content.empty() || content.size() > 65536) throw llm::ApiError::invalid("jina", "content must be 1..65536 bytes");
if (max_chunk_length < 0 || max_chunk_length > 8192) throw llm::ApiError::invalid("jina", "max_chunk_length out of range");
Json body = Json::object();
body.set("content", Json(content));
if (!tokenizer.empty()) body.set("tokenizer", Json(tokenizer));
body.set("return_chunks", Json(true));
if (max_chunk_length > 0) body.set("max_chunk_length", Json(max_chunk_length));
CallArgs args;
args.body = body;
return parse_segments(gw_->call("segment", args));
}
long Jina::count_tokens(const std::string& content, const std::string& tokenizer) {
if (content.empty() || content.size() > 65536) throw llm::ApiError::invalid("jina", "content must be 1..65536 bytes");
CallArgs args;
args.query["content"] = content;
if (!tokenizer.empty()) args.query["tokenizer"] = tokenizer;
Json doc = gw_->call("count_token", args);
return static_cast<long>(J::num(doc, "num_tokens"));
}
std::vector<JinaPrediction> Jina::classify(const std::vector<std::string>& texts,
const std::vector<std::string>& labels, const std::string& model) {
if (texts.empty() || texts.size() > RetrievalCaps::kMaxEmbedTexts) throw llm::ApiError::invalid("jina", "texts must hold 1..1000 entries");
if (labels.size() < 2 || labels.size() > 256) throw llm::ApiError::invalid("jina", "labels must hold 2..256 entries");
Json body = Json::object();
body.set("model", Json(model.empty() ? std::string(kDefaultClassifyModel) : model));
body.set("input", J::strings(texts));
body.set("labels", J::strings(labels));
CallArgs args;
args.body = body;
return parse_classification(gw_->call("classify", args));
}
Json Jina::list_models() { return gw_->call("list_models"); }
Json Jina::create_batch(const std::string& model, const std::string& input_url, const std::string& task) {
check_url(input_url);
Json body = Json::object();
body.set("model", Json(model));
body.set("input_url", Json(input_url));
if (!task.empty()) body.set("task", Json(task));
CallArgs args;
args.body = body;
return gw_->call("create_batch", args);
}
Json Jina::get_batch(const std::string& batch_id) {
CallArgs args;
args.path["batch_id"] = batch_id;
return gw_->call("get_batch", args);
}
Json Jina::cancel_batch(const std::string& batch_id) {
CallArgs args;
args.path["batch_id"] = batch_id;
return gw_->destroy("cancel_batch", args);
}
} // namespace molo::retrieval
+80
View File
@@ -0,0 +1,80 @@
#pragma once
// Jina AI: embeddings, rerank, models and batch embeddings (api.jina.ai), the
// Reader (r.jina.ai), Search (s.jina.ai), the Segmenter (segment.jina.ai) and
// the Classifier. Implements Embedder, Reranker and WebSearch.
#include <string>
#include <vector>
#include "vendors/retrieval/provider.h"
#include "vendors/retrieval/seams.h"
namespace molo::retrieval {
const CallTable& jina_calls();
struct JinaPage {
std::string title;
std::string url;
std::string description;
std::string content; // markdown
long tokens = 0;
};
struct JinaSegments {
long num_tokens = 0;
long num_chunks = 0;
std::vector<std::string> chunks;
};
struct JinaPrediction {
int index = 0;
std::string label;
double score = 0;
};
class Jina : public Provider, public Embedder, public Reranker, public WebSearch {
public:
static constexpr const char* kKeyEnv = "JINA_API_KEY";
static constexpr const char* kApiUrl = "https://api.jina.ai";
static constexpr const char* kReaderUrl = "https://r.jina.ai";
static constexpr const char* kSearchUrl = "https://s.jina.ai";
static constexpr const char* kSegmentUrl = "https://segment.jina.ai";
static constexpr const char* kDefaultEmbedModel = "jina-embeddings-v3";
static constexpr const char* kDefaultRerankModel = "jina-reranker-v3";
static constexpr const char* kDefaultClassifyModel = "jina-embeddings-v3";
static llm::ApiClientConfig config();
static Jina from_env();
static Jina with(const TransportFactory& factory, std::string key);
explicit Jina(std::shared_ptr<Gateway> gateway) : Provider(std::move(gateway)) {}
std::string provider() const override { return "jina"; }
EmbedResult embed(const EmbedRequest& request) override;
RerankResult rerank(const RerankRequest& request) override;
SearchResult search(const SearchRequest& request) override;
JinaPage read(const std::string& url);
JinaSegments segment(const std::string& content, const std::string& tokenizer, int max_chunk_length);
long count_tokens(const std::string& content, const std::string& tokenizer);
std::vector<JinaPrediction> classify(const std::vector<std::string>& texts,
const std::vector<std::string>& labels, const std::string& model);
Json list_models();
Json create_batch(const std::string& model, const std::string& input_url, const std::string& task);
Json get_batch(const std::string& batch_id);
// Destructive: cancels a running batch job.
Json cancel_batch(const std::string& batch_id);
static EmbedResult parse_embeddings(const Json& doc);
static RerankResult parse_rerank(const Json& doc);
static JinaPage parse_page(const Json& doc);
static SearchResult parse_search(const Json& doc);
static JinaSegments parse_segments(const Json& doc);
static std::vector<JinaPrediction> parse_classification(const Json& doc);
// http(s) URL, 1..8192 chars; throws ApiError::invalid.
static void check_url(const std::string& url);
};
} // namespace molo::retrieval