feat(clients): retrieval: Voyage, Jina, Pinecone, Qdrant, Weaviate, Mixedbread, Tavily, Exa, Brave
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Vendored
+33
@@ -0,0 +1,33 @@
|
||||
#include "vendors/retrieval/jina/jina.h"
|
||||
|
||||
namespace molo::retrieval {
|
||||
|
||||
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
|
||||
// name, host, http method, path, spec operation id (or reference page / "doc:"),
|
||||
// body fields, query parameters, flags (d destructive, w writes, r read-only POST,
|
||||
// p paid), response schema)
|
||||
static const CallRow kRows[] = {
|
||||
{"embeddings", "api", "POST", "/v1/embeddings", "embeddings_v1_embeddings_post", "dimensions,embedding_type,input,input_type,late_chunking,model,normalized,return_multivector,return_tokenized_input,task,truncate", "", "rp", "EmbeddingResponse"},
|
||||
{"rerank", "api", "POST", "/v1/rerank", "rerank_v1_rerank_post", "documents,max_doc_length,model,query,return_documents,return_embeddings,top_n", "", "rp", "RerankingResponse"},
|
||||
{"list_models", "api", "GET", "/v1/models", "list_models_v1_models_get", "", "", "", "ModelListResponse"},
|
||||
{"get_model", "api", "GET", "/v1/models/{model_id}", "get_model_v1_models__model_id__get", "", "", "", "ModelInfo"},
|
||||
{"create_batch", "api", "POST", "/v1/batch/embeddings", "create_batch_job_v1_batch_embeddings_post", "dimensions,input,input_url,model,normalized,task,webhook_url", "", "wp", "BatchStatus"},
|
||||
{"get_batch", "api", "GET", "/v1/batch/{batch_id}", "get_batch_status_v1_batch__batch_id__get", "", "", "", "BatchStatus"},
|
||||
{"cancel_batch", "api", "DELETE", "/v1/batch/{batch_id}", "cancel_batch_job_v1_batch__batch_id__delete", "", "", "d", "BatchStatus"},
|
||||
{"list_batches", "api", "GET", "/v1/batches", "list_batch_jobs_v1_batches_get", "", "limit", "", "inline"},
|
||||
{"batch_output", "api", "GET", "/v1/batch/{batch_id}/output", "download_batch_output_v1_batch__batch_id__output_get", "", "", "", "-"},
|
||||
{"batch_errors", "api", "GET", "/v1/batch/{batch_id}/errors", "download_batch_errors_v1_batch__batch_id__errors_get", "", "", "", "-"},
|
||||
{"read", "reader", "GET", "/", "", "", "assertStatusCode,base,cacheTolerance,detachInvisibles,engine,exportStorageState,file,html,injectFrameScript,injectPageScript,instruction,jsonSchema,keepImgDataUrl,locale,markdownChunking,maxTokens,noCache,noGfm,noServiceWorker,page,pdf,preloadUrl,preset,proxy,proxyResources,proxyUrl,referer,removeOverlay,removeSelector,respondTiming,respondWith,retainImages,retainLinks,retainMedia,robotsTxt,setCookies,skipCertVerification,targetSelector,timeout,tokenBudget,url,userAgent,waitForSelector,webBotAuth,withGeneratedAlt,withIframe,withImagesSummary,withLinksSummary,withShadowDom,zeroDataRetention", "p", "-"},
|
||||
{"search", "search", "GET", "/search", "", "", "assertStatusCode,base,cacheTolerance,count,detachInvisibles,engine,exportStorageState,ext,fallback,file,filetype,gl,hl,html,injectFrameScript,injectPageScript,instruction,intitle,jsonSchema,keepImgDataUrl,loc,locale,location,markdownChunking,maxTokens,nfpr,noCache,noGfm,noServiceWorker,num,page,pdf,preloadUrl,preset,provider,proxy,proxyResources,proxyUrl,q,referer,removeOverlay,removeSelector,respondTiming,respondWith,retainImages,retainLinks,retainMedia,robotsTxt,setCookies,site,skipCertVerification,targetSelector,timeout,tokenBudget,type,url,userAgent,waitForSelector,webBotAuth,withGeneratedAlt,withIframe,withImagesSummary,withLinksSummary,withShadowDom,zeroDataRetention", "p", "inline"},
|
||||
{"count_token", "segment", "POST", "/countToken", "", "content,head,max_chunk_length,return_chunks,return_tokens,tail,tokenizer", "content,head,max_chunk_length,return_chunks,return_tokens,tail,tokenizer", "rp", "-"},
|
||||
{"segment", "segment", "POST", "/", "doc:segmenter", "content,tokenizer,return_tokens,return_chunks,max_chunk_length,head,tail", "", "p", "-"},
|
||||
{"classify", "api", "POST", "/v1/classify", "doc:classifier", "model,input,labels", "", "p", "-"},
|
||||
};
|
||||
// CALLS-END
|
||||
|
||||
const CallTable& jina_calls() {
|
||||
static const CallTable table{kRows, sizeof(kRows) / sizeof(kRows[0])};
|
||||
return table;
|
||||
}
|
||||
|
||||
} // namespace molo::retrieval
|
||||
Vendored
+217
@@ -0,0 +1,217 @@
|
||||
#include "vendors/retrieval/jina/jina.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include "vendors/retrieval/caps.h"
|
||||
|
||||
namespace molo::retrieval {
|
||||
|
||||
llm::ApiClientConfig Jina::config() {
|
||||
llm::ApiClientConfig c;
|
||||
c.vendor = "jina";
|
||||
c.base_url = kApiUrl;
|
||||
c.key_env = kKeyEnv;
|
||||
c.auth = llm::AuthStyle::kBearer;
|
||||
return c;
|
||||
}
|
||||
|
||||
namespace {
|
||||
std::vector<HostSpec> hosts() {
|
||||
return {{"api", Jina::kApiUrl}, {"reader", Jina::kReaderUrl}, {"search", Jina::kSearchUrl}, {"segment", Jina::kSegmentUrl}};
|
||||
}
|
||||
llm::Headers json_accept() { return {{"Accept", "application/json"}}; }
|
||||
} // namespace
|
||||
|
||||
Jina Jina::from_env() { return Jina(make_gateway(jina_calls(), config(), hosts(), "", nullptr)); }
|
||||
Jina Jina::with(const TransportFactory& factory, std::string key) {
|
||||
return Jina(make_gateway(jina_calls(), config(), hosts(), std::move(key), factory));
|
||||
}
|
||||
|
||||
void Jina::check_url(const std::string& url) {
|
||||
bool http = url.rfind("http://", 0) == 0 || url.rfind("https://", 0) == 0;
|
||||
if (!http || url.size() > 8192 || url.find_first_of(" \t\r\n") != std::string::npos)
|
||||
throw llm::ApiError::invalid("jina", "url must be http(s) without spaces, at most 8192 chars");
|
||||
}
|
||||
|
||||
EmbedResult Jina::parse_embeddings(const Json& doc) {
|
||||
EmbedResult out;
|
||||
out.model = J::str(doc, "model");
|
||||
std::vector<std::pair<int, std::vector<float>>> rows;
|
||||
for (const Json& d : J::array(doc, "data")) {
|
||||
const Json* e = J::at(d, "embedding");
|
||||
rows.emplace_back(static_cast<int>(J::num(d, "index", static_cast<double>(rows.size()))),
|
||||
e ? J::floats(*e) : std::vector<float>{});
|
||||
}
|
||||
std::stable_sort(rows.begin(), rows.end(), [](const auto& a, const auto& b) { return a.first < b.first; });
|
||||
for (auto& r : rows) out.vectors.push_back(std::move(r.second));
|
||||
if (const Json* u = J::at(doc, "usage")) out.total_tokens = static_cast<long>(J::num(*u, "total_tokens"));
|
||||
return out;
|
||||
}
|
||||
|
||||
RerankResult Jina::parse_rerank(const Json& doc) {
|
||||
RerankResult out;
|
||||
out.model = J::str(doc, "model");
|
||||
for (const Json& d : J::array(doc, "results")) {
|
||||
RerankHit h;
|
||||
h.index = static_cast<int>(J::num(d, "index"));
|
||||
h.score = J::num(d, "relevance_score");
|
||||
out.hits.push_back(h);
|
||||
}
|
||||
std::stable_sort(out.hits.begin(), out.hits.end(), [](const RerankHit& a, const RerankHit& b) { return a.score > b.score; });
|
||||
if (const Json* u = J::at(doc, "usage")) out.total_tokens = static_cast<long>(J::num(*u, "total_tokens"));
|
||||
return out;
|
||||
}
|
||||
|
||||
JinaPage Jina::parse_page(const Json& doc) {
|
||||
JinaPage p;
|
||||
const Json* data = J::at(doc, "data");
|
||||
const Json& d = data && data->is_object() ? *data : doc;
|
||||
p.title = J::str(d, "title");
|
||||
p.url = J::str(d, "url");
|
||||
p.description = J::str(d, "description");
|
||||
p.content = J::str(d, "content");
|
||||
if (const Json* u = J::at(d, "usage")) p.tokens = static_cast<long>(J::num(*u, "tokens"));
|
||||
return p;
|
||||
}
|
||||
|
||||
SearchResult Jina::parse_search(const Json& doc) {
|
||||
SearchResult out;
|
||||
for (const Json& d : J::array(doc, "data")) {
|
||||
SearchHit h;
|
||||
h.title = J::str(d, "title");
|
||||
h.url = J::str(d, "url");
|
||||
h.snippet = J::str(d, "description");
|
||||
if (h.snippet.empty()) h.snippet = J::str(d, "content").substr(0, 500);
|
||||
out.hits.push_back(h);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
JinaSegments Jina::parse_segments(const Json& doc) {
|
||||
JinaSegments s;
|
||||
s.num_tokens = static_cast<long>(J::num(doc, "num_tokens"));
|
||||
s.num_chunks = static_cast<long>(J::num(doc, "num_chunks"));
|
||||
for (const Json& c : J::array(doc, "chunks"))
|
||||
if (c.is_string()) s.chunks.push_back(c.as_string());
|
||||
return s;
|
||||
}
|
||||
|
||||
std::vector<JinaPrediction> Jina::parse_classification(const Json& doc) {
|
||||
std::vector<JinaPrediction> out;
|
||||
for (const Json& d : J::array(doc, "data")) {
|
||||
JinaPrediction p;
|
||||
p.index = static_cast<int>(J::num(d, "index", static_cast<double>(out.size())));
|
||||
p.label = J::str(d, "prediction");
|
||||
p.score = J::num(d, "score");
|
||||
out.push_back(p);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
EmbedResult Jina::embed(const EmbedRequest& request) {
|
||||
Embedder::validate("jina", request);
|
||||
Json body = Json::object();
|
||||
body.set("model", Json(request.model.empty() ? std::string(kDefaultEmbedModel) : request.model));
|
||||
body.set("input", J::strings(request.texts));
|
||||
if (request.input_type == "query") body.set("task", Json("retrieval.query"));
|
||||
if (request.input_type == "document") body.set("task", Json("retrieval.passage"));
|
||||
if (request.dimensions > 0) body.set("dimensions", Json(request.dimensions));
|
||||
CallArgs args;
|
||||
args.body = body;
|
||||
EmbedResult r = parse_embeddings(gw_->call("embeddings", args));
|
||||
if (r.vectors.size() != request.texts.size())
|
||||
throw llm::ApiError("jina", 502, "BAD_RESPONSE", "embedding count differs from input count");
|
||||
return r;
|
||||
}
|
||||
|
||||
RerankResult Jina::rerank(const RerankRequest& request) {
|
||||
Reranker::validate("jina", request);
|
||||
Json body = Json::object();
|
||||
body.set("model", Json(request.model.empty() ? std::string(kDefaultRerankModel) : request.model));
|
||||
body.set("query", Json(request.query));
|
||||
body.set("documents", J::strings(request.documents));
|
||||
body.set("return_documents", Json(false));
|
||||
if (request.top_n > 0) body.set("top_n", Json(request.top_n));
|
||||
CallArgs args;
|
||||
args.body = body;
|
||||
return parse_rerank(gw_->call("rerank", args));
|
||||
}
|
||||
|
||||
SearchResult Jina::search(const SearchRequest& request) {
|
||||
WebSearch::validate("jina", request);
|
||||
if (!request.recency.empty()) throw llm::ApiError::invalid("jina", "jina search has no recency filter");
|
||||
CallArgs args;
|
||||
args.query["q"] = request.query;
|
||||
args.headers = json_accept();
|
||||
return parse_search(gw_->call("search", args));
|
||||
}
|
||||
|
||||
JinaPage Jina::read(const std::string& url) {
|
||||
check_url(url);
|
||||
CallArgs args;
|
||||
args.query["url"] = url;
|
||||
args.headers = json_accept();
|
||||
return parse_page(gw_->call("read", args));
|
||||
}
|
||||
|
||||
JinaSegments Jina::segment(const std::string& content, const std::string& tokenizer, int max_chunk_length) {
|
||||
if (content.empty() || content.size() > 65536) throw llm::ApiError::invalid("jina", "content must be 1..65536 bytes");
|
||||
if (max_chunk_length < 0 || max_chunk_length > 8192) throw llm::ApiError::invalid("jina", "max_chunk_length out of range");
|
||||
Json body = Json::object();
|
||||
body.set("content", Json(content));
|
||||
if (!tokenizer.empty()) body.set("tokenizer", Json(tokenizer));
|
||||
body.set("return_chunks", Json(true));
|
||||
if (max_chunk_length > 0) body.set("max_chunk_length", Json(max_chunk_length));
|
||||
CallArgs args;
|
||||
args.body = body;
|
||||
return parse_segments(gw_->call("segment", args));
|
||||
}
|
||||
|
||||
long Jina::count_tokens(const std::string& content, const std::string& tokenizer) {
|
||||
if (content.empty() || content.size() > 65536) throw llm::ApiError::invalid("jina", "content must be 1..65536 bytes");
|
||||
CallArgs args;
|
||||
args.query["content"] = content;
|
||||
if (!tokenizer.empty()) args.query["tokenizer"] = tokenizer;
|
||||
Json doc = gw_->call("count_token", args);
|
||||
return static_cast<long>(J::num(doc, "num_tokens"));
|
||||
}
|
||||
|
||||
std::vector<JinaPrediction> Jina::classify(const std::vector<std::string>& texts,
|
||||
const std::vector<std::string>& labels, const std::string& model) {
|
||||
if (texts.empty() || texts.size() > RetrievalCaps::kMaxEmbedTexts) throw llm::ApiError::invalid("jina", "texts must hold 1..1000 entries");
|
||||
if (labels.size() < 2 || labels.size() > 256) throw llm::ApiError::invalid("jina", "labels must hold 2..256 entries");
|
||||
Json body = Json::object();
|
||||
body.set("model", Json(model.empty() ? std::string(kDefaultClassifyModel) : model));
|
||||
body.set("input", J::strings(texts));
|
||||
body.set("labels", J::strings(labels));
|
||||
CallArgs args;
|
||||
args.body = body;
|
||||
return parse_classification(gw_->call("classify", args));
|
||||
}
|
||||
|
||||
Json Jina::list_models() { return gw_->call("list_models"); }
|
||||
|
||||
Json Jina::create_batch(const std::string& model, const std::string& input_url, const std::string& task) {
|
||||
check_url(input_url);
|
||||
Json body = Json::object();
|
||||
body.set("model", Json(model));
|
||||
body.set("input_url", Json(input_url));
|
||||
if (!task.empty()) body.set("task", Json(task));
|
||||
CallArgs args;
|
||||
args.body = body;
|
||||
return gw_->call("create_batch", args);
|
||||
}
|
||||
|
||||
Json Jina::get_batch(const std::string& batch_id) {
|
||||
CallArgs args;
|
||||
args.path["batch_id"] = batch_id;
|
||||
return gw_->call("get_batch", args);
|
||||
}
|
||||
|
||||
Json Jina::cancel_batch(const std::string& batch_id) {
|
||||
CallArgs args;
|
||||
args.path["batch_id"] = batch_id;
|
||||
return gw_->destroy("cancel_batch", args);
|
||||
}
|
||||
|
||||
} // namespace molo::retrieval
|
||||
Vendored
+80
@@ -0,0 +1,80 @@
|
||||
#pragma once
|
||||
|
||||
// Jina AI: embeddings, rerank, models and batch embeddings (api.jina.ai), the
|
||||
// Reader (r.jina.ai), Search (s.jina.ai), the Segmenter (segment.jina.ai) and
|
||||
// the Classifier. Implements Embedder, Reranker and WebSearch.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "vendors/retrieval/provider.h"
|
||||
#include "vendors/retrieval/seams.h"
|
||||
|
||||
namespace molo::retrieval {
|
||||
|
||||
const CallTable& jina_calls();
|
||||
|
||||
struct JinaPage {
|
||||
std::string title;
|
||||
std::string url;
|
||||
std::string description;
|
||||
std::string content; // markdown
|
||||
long tokens = 0;
|
||||
};
|
||||
|
||||
struct JinaSegments {
|
||||
long num_tokens = 0;
|
||||
long num_chunks = 0;
|
||||
std::vector<std::string> chunks;
|
||||
};
|
||||
|
||||
struct JinaPrediction {
|
||||
int index = 0;
|
||||
std::string label;
|
||||
double score = 0;
|
||||
};
|
||||
|
||||
class Jina : public Provider, public Embedder, public Reranker, public WebSearch {
|
||||
public:
|
||||
static constexpr const char* kKeyEnv = "JINA_API_KEY";
|
||||
static constexpr const char* kApiUrl = "https://api.jina.ai";
|
||||
static constexpr const char* kReaderUrl = "https://r.jina.ai";
|
||||
static constexpr const char* kSearchUrl = "https://s.jina.ai";
|
||||
static constexpr const char* kSegmentUrl = "https://segment.jina.ai";
|
||||
static constexpr const char* kDefaultEmbedModel = "jina-embeddings-v3";
|
||||
static constexpr const char* kDefaultRerankModel = "jina-reranker-v3";
|
||||
static constexpr const char* kDefaultClassifyModel = "jina-embeddings-v3";
|
||||
|
||||
static llm::ApiClientConfig config();
|
||||
static Jina from_env();
|
||||
static Jina with(const TransportFactory& factory, std::string key);
|
||||
explicit Jina(std::shared_ptr<Gateway> gateway) : Provider(std::move(gateway)) {}
|
||||
|
||||
std::string provider() const override { return "jina"; }
|
||||
EmbedResult embed(const EmbedRequest& request) override;
|
||||
RerankResult rerank(const RerankRequest& request) override;
|
||||
SearchResult search(const SearchRequest& request) override;
|
||||
|
||||
JinaPage read(const std::string& url);
|
||||
JinaSegments segment(const std::string& content, const std::string& tokenizer, int max_chunk_length);
|
||||
long count_tokens(const std::string& content, const std::string& tokenizer);
|
||||
std::vector<JinaPrediction> classify(const std::vector<std::string>& texts,
|
||||
const std::vector<std::string>& labels, const std::string& model);
|
||||
Json list_models();
|
||||
Json create_batch(const std::string& model, const std::string& input_url, const std::string& task);
|
||||
Json get_batch(const std::string& batch_id);
|
||||
// Destructive: cancels a running batch job.
|
||||
Json cancel_batch(const std::string& batch_id);
|
||||
|
||||
static EmbedResult parse_embeddings(const Json& doc);
|
||||
static RerankResult parse_rerank(const Json& doc);
|
||||
static JinaPage parse_page(const Json& doc);
|
||||
static SearchResult parse_search(const Json& doc);
|
||||
static JinaSegments parse_segments(const Json& doc);
|
||||
static std::vector<JinaPrediction> parse_classification(const Json& doc);
|
||||
|
||||
// http(s) URL, 1..8192 chars; throws ApiError::invalid.
|
||||
static void check_url(const std::string& url);
|
||||
};
|
||||
|
||||
} // namespace molo::retrieval
|
||||
Reference in New Issue
Block a user