feat(clients): retrieval: Voyage, Jina, Pinecone, Qdrant, Weaviate, Mixedbread, Tavily, Exa, Brave
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
+71
@@ -0,0 +1,71 @@
|
||||
#pragma once
|
||||
|
||||
// Rig for the retrieval tests: a TransportFactory that hands out one
|
||||
// MockTransport per host (in host order) and keeps raw pointers so a test can
|
||||
// script a reply and read back the request the real client built. The mock only
|
||||
// records requests; replies are FIXTURES whose shapes are copied from the
|
||||
// provider specs and labelled where they appear. They are not recorded
|
||||
// replies of the live services (see SPEC.md ## Unverified).
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "clients/transports/mock_transport.h"
|
||||
#include "framework.h"
|
||||
#include "vendors/retrieval/provider.h"
|
||||
|
||||
namespace retrieval_test {
|
||||
|
||||
using molo::clients::ClientResponse;
|
||||
using molo::clients::HeaderMap;
|
||||
using molo::clients::Method;
|
||||
using molo::clients::transports::MockExpectation;
|
||||
using molo::clients::transports::MockTransport;
|
||||
using Json = molo::clients::body::Json;
|
||||
|
||||
inline const char* kKey = "key-TESTKEY-DO-NOT-LEAK";
|
||||
|
||||
inline ClientResponse reply(const std::string& body, int status = 200) {
|
||||
HeaderMap h;
|
||||
h.set("Content-Type", "application/json");
|
||||
return ClientResponse(status, h, body);
|
||||
}
|
||||
|
||||
struct Rig {
|
||||
std::vector<MockTransport*> transports;
|
||||
|
||||
molo::retrieval::TransportFactory factory() {
|
||||
return [this]() {
|
||||
auto t = std::make_unique<MockTransport>();
|
||||
transports.push_back(t.get());
|
||||
return std::unique_ptr<molo::clients::Transport>(std::move(t));
|
||||
};
|
||||
}
|
||||
|
||||
// Scripts `body` as the reply for method + path on transport `host_index`.
|
||||
void answer(std::size_t host_index, Method m, const std::string& path, const std::string& body, int status = 200) {
|
||||
MockExpectation e;
|
||||
e.method = m;
|
||||
e.path = path;
|
||||
e.response = reply(body, status);
|
||||
e.times = 50;
|
||||
transports.at(host_index)->expect(e);
|
||||
}
|
||||
|
||||
MockTransport::SeenRequest last(std::size_t host_index = 0) const {
|
||||
auto seen = transports.at(host_index)->seen();
|
||||
if (seen.empty()) throw std::runtime_error("no request was sent");
|
||||
return seen.back();
|
||||
}
|
||||
std::size_t sent(std::size_t host_index = 0) const { return transports.at(host_index)->send_count(); }
|
||||
std::size_t sent_total() const {
|
||||
std::size_t n = 0;
|
||||
for (MockTransport* t : transports) n += t->send_count();
|
||||
return n;
|
||||
}
|
||||
};
|
||||
|
||||
inline bool has(const std::string& hay, const std::string& needle) { return hay.find(needle) != std::string::npos; }
|
||||
|
||||
} // namespace retrieval_test
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
// Retrieval core: the call tables, the Gateway rules (ids, query, body,
|
||||
// destructive split, caps) and the seam validators. No provider reply is
|
||||
// involved except where a test needs the request to leave the client.
|
||||
#include <set>
|
||||
|
||||
#include "support.h"
|
||||
#include "vendors/retrieval/brave/brave.h"
|
||||
#include "vendors/retrieval/caps.h"
|
||||
#include "vendors/retrieval/exa/exa.h"
|
||||
#include "vendors/retrieval/jina/jina.h"
|
||||
#include "vendors/retrieval/mixedbread/mixedbread.h"
|
||||
#include "vendors/retrieval/pinecone/pinecone.h"
|
||||
#include "vendors/retrieval/qdrant/qdrant.h"
|
||||
#include "vendors/retrieval/tavily/tavily.h"
|
||||
#include "vendors/retrieval/voyage/voyage.h"
|
||||
#include "vendors/retrieval/weaviate/weaviate.h"
|
||||
|
||||
using namespace molo::retrieval;
|
||||
using namespace retrieval_test;
|
||||
using molo::llm::ApiError;
|
||||
using molo::llm::NotConfiguredError;
|
||||
|
||||
namespace {
|
||||
struct Named {
|
||||
const char* name;
|
||||
const CallTable& (*table)();
|
||||
std::size_t at_least;
|
||||
};
|
||||
const Named kAll[] = {
|
||||
{"voyage", voyage_calls, 4}, {"jina", jina_calls, 15}, {"pinecone", pinecone_calls, 57},
|
||||
{"qdrant", qdrant_calls, 49}, {"weaviate", weaviate_calls, 66}, {"mixedbread", mixedbread_calls, 80},
|
||||
{"tavily", tavily_calls, 7}, {"exa", exa_calls, 7}, {"brave", brave_calls, 16},
|
||||
};
|
||||
|
||||
std::vector<std::string> placeholders(const std::string& path) {
|
||||
std::vector<std::string> out;
|
||||
for (std::size_t p = path.find('{'); p != std::string::npos; p = path.find('{', p + 1))
|
||||
out.push_back(path.substr(p + 1, path.find('}', p) - p - 1));
|
||||
return out;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(retrieval_core, every_table_is_well_formed) {
|
||||
for (const Named& n : kAll) {
|
||||
const CallTable& t = n.table();
|
||||
ASSERT_TRUE(t.size() >= n.at_least);
|
||||
std::set<std::string> names;
|
||||
for (std::size_t i = 0; i < t.size(); ++i) {
|
||||
const CallRow& r = t.at(i);
|
||||
std::string who = std::string(n.name) + "." + r.name;
|
||||
ASSERT_TRUE(names.insert(r.name).second); // unique names
|
||||
RowInfo::method(r); // valid method, else throws
|
||||
ASSERT_TRUE(r.path[0] == '/');
|
||||
ASSERT_TRUE(std::string(r.flags).find_first_not_of("dwrp") == std::string::npos);
|
||||
ASSERT_TRUE(placeholders(r.path).size() <= RetrievalCaps::kMaxPathParams);
|
||||
if (std::string(r.method) == "DELETE") ASSERT_TRUE(RowInfo::destructive(r));
|
||||
if (std::string(r.method) == "GET") ASSERT_FALSE(RowInfo::writes(r));
|
||||
if (RowInfo::destructive(r)) ASSERT_FALSE(RowInfo::has(r, 'r'));
|
||||
if (std::string(r.method) == "GET") ASSERT_TRUE(std::string(r.body).empty());
|
||||
ASSERT_TRUE(std::string(r.host).size() > 0);
|
||||
(void)who;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(retrieval_core, destructive_rows_are_the_ones_that_remove_or_clear) {
|
||||
const char* must[][2] = {{"qdrant", "delete_collection"}, {"qdrant", "delete_points"}, {"qdrant", "clear_payload"},
|
||||
{"pinecone", "delete_index"}, {"pinecone", "delete_vectors"}, {"weaviate", "schema_objects_delete"},
|
||||
{"weaviate", "batch_objects_delete"}, {"mixedbread", "delete_store"}, {"jina", "cancel_batch"}};
|
||||
for (auto& m : must) {
|
||||
const CallTable* t = nullptr;
|
||||
for (const Named& n : kAll)
|
||||
if (std::string(n.name) == m[0]) t = &n.table();
|
||||
ASSERT_TRUE(t != nullptr);
|
||||
const CallRow* r = t->find(m[1]);
|
||||
ASSERT_TRUE(r != nullptr);
|
||||
ASSERT_TRUE(RowInfo::destructive(*r));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(retrieval_core, unverified_rows_are_exactly_the_prose_only_ones) {
|
||||
std::size_t n = 0;
|
||||
for (const Named& a : kAll)
|
||||
for (std::size_t i = 0; i < a.table().size(); ++i)
|
||||
if (RowInfo::unverified(a.table().at(i))) ++n;
|
||||
ASSERT_EQ(n, std::size_t(2)); // jina segment and jina classify
|
||||
ASSERT_TRUE(RowInfo::unverified(*jina_calls().find("segment")));
|
||||
ASSERT_TRUE(RowInfo::unverified(*jina_calls().find("classify")));
|
||||
}
|
||||
|
||||
TEST(retrieval_core, plan_renders_path_query_and_flags_without_traffic) {
|
||||
Rig rig;
|
||||
Qdrant q = Qdrant::with(rig.factory(), kKey, "https://qd.example.com:6333/");
|
||||
CallArgs a;
|
||||
a.path["collection_name"] = "docs";
|
||||
a.query["wait"] = "true";
|
||||
Json b = Json::object();
|
||||
b.set("points", Json::array());
|
||||
a.body = b;
|
||||
PlannedCall p = q.gateway().plan("upsert_points", a);
|
||||
ASSERT_EQ(p.method, std::string("PUT"));
|
||||
ASSERT_EQ(p.target, std::string("/collections/docs/points?wait=true"));
|
||||
ASSERT_TRUE(p.writes);
|
||||
ASSERT_FALSE(p.destructive);
|
||||
ASSERT_EQ(rig.sent_total(), std::size_t(0));
|
||||
}
|
||||
|
||||
TEST(retrieval_core, path_ids_are_validated_and_encoded) {
|
||||
Rig rig;
|
||||
Qdrant q = Qdrant::with(rig.factory(), kKey, "https://qd.example.com");
|
||||
CallArgs a;
|
||||
a.path["collection_name"] = "../etc";
|
||||
ASSERT_THROW(q.gateway().plan("get_collection", a), ApiError);
|
||||
a.path["collection_name"] = "a b";
|
||||
ASSERT_THROW(q.gateway().plan("get_collection", a), ApiError);
|
||||
a.path["collection_name"] = "a/b";
|
||||
ASSERT_THROW(q.gateway().plan("get_collection", a), ApiError);
|
||||
a.path.clear();
|
||||
ASSERT_THROW(q.gateway().plan("get_collection", a), ApiError); // missing
|
||||
a.path["collection_name"] = "ok";
|
||||
a.path["extra"] = "x";
|
||||
ASSERT_THROW(q.gateway().plan("get_collection", a), ApiError); // unexpected
|
||||
ASSERT_THROW(q.gateway().plan("no_such_call"), ApiError);
|
||||
ASSERT_EQ(rig.sent_total(), std::size_t(0));
|
||||
}
|
||||
|
||||
TEST(retrieval_core, query_and_body_must_be_listed_in_the_table) {
|
||||
Rig rig;
|
||||
Qdrant q = Qdrant::with(rig.factory(), kKey, "https://qd.example.com");
|
||||
CallArgs a;
|
||||
a.path["collection_name"] = "docs";
|
||||
a.query["bogus"] = "1";
|
||||
ASSERT_THROW(q.gateway().plan("upsert_points", a), ApiError);
|
||||
a.query.clear();
|
||||
Json b = Json::object();
|
||||
b.set("not_a_field", Json(1));
|
||||
a.body = b;
|
||||
ASSERT_THROW(q.gateway().plan("upsert_points", a), ApiError);
|
||||
CallArgs get;
|
||||
get.path["collection_name"] = "docs";
|
||||
get.body = Json::object();
|
||||
ASSERT_THROW(q.gateway().plan("get_collection", get), ApiError); // GET takes no body
|
||||
CallArgs ctl;
|
||||
ctl.path["collection_name"] = "docs";
|
||||
ctl.query["wait"] = std::string("a\nb");
|
||||
ctl.body = Json::object();
|
||||
ASSERT_THROW(q.gateway().plan("upsert_points", ctl), ApiError); // control characters
|
||||
CallArgs big;
|
||||
big.path["collection_name"] = "docs";
|
||||
big.query["wait"] = std::string(RetrievalCaps::kMaxQueryValueBytes + 1, 'x');
|
||||
ASSERT_THROW(q.gateway().plan("upsert_points", big), ApiError);
|
||||
}
|
||||
|
||||
TEST(retrieval_core, body_over_the_byte_cap_is_rejected_before_sending) {
|
||||
Rig rig;
|
||||
Qdrant q = Qdrant::with(rig.factory(), kKey, "https://qd.example.com");
|
||||
CallArgs a;
|
||||
a.path["collection_name"] = "docs";
|
||||
Json b = Json::object();
|
||||
b.set("points", Json(std::string(RetrievalCaps::kMaxBodyBytes + 1, 'x')));
|
||||
a.body = b;
|
||||
ASSERT_THROW(q.gateway().plan("upsert_points", a), ApiError);
|
||||
ASSERT_EQ(rig.sent_total(), std::size_t(0));
|
||||
}
|
||||
|
||||
TEST(retrieval_core, destructive_rows_only_run_through_destroy) {
|
||||
Rig rig;
|
||||
Qdrant q = Qdrant::with(rig.factory(), kKey, "https://qd.example.com");
|
||||
rig.answer(0, Method::kDelete, "/collections/docs", "{\"result\":true,\"status\":\"ok\"}");
|
||||
CallArgs a;
|
||||
a.path["collection_name"] = "docs";
|
||||
ASSERT_THROW(q.gateway().call("delete_collection", a), ApiError); // refused, nothing sent
|
||||
ASSERT_EQ(rig.sent_total(), std::size_t(0));
|
||||
ASSERT_THROW(q.gateway().destroy("get_collection", a), ApiError); // not destructive
|
||||
ASSERT_EQ(rig.sent_total(), std::size_t(0));
|
||||
Json out = q.delete_collection("docs"); // the explicit method
|
||||
ASSERT_EQ(rig.sent_total(), std::size_t(1));
|
||||
ASSERT_EQ(rig.last().method, std::string("DELETE"));
|
||||
ASSERT_TRUE(out.has("result"));
|
||||
}
|
||||
|
||||
TEST(retrieval_core, no_key_means_no_traffic_and_a_503_error) {
|
||||
Rig rig;
|
||||
Voyage v = Voyage::with(rig.factory(), "");
|
||||
// Config::get may find a key in the environment; test runs set MOLO_ENV_FILES=off
|
||||
// and VOYAGE_API_KEY is not set in CI, so an empty key stays empty.
|
||||
if (v.configured()) return;
|
||||
EmbedRequest r;
|
||||
r.texts = {"hello"};
|
||||
try {
|
||||
v.embed(r);
|
||||
FAIL("expected NotConfiguredError");
|
||||
} catch (const NotConfiguredError& e) {
|
||||
ASSERT_EQ(e.http_status(), 503);
|
||||
ASSERT_EQ(e.type(), std::string("NOT_CONFIGURED"));
|
||||
ASSERT_TRUE(has(e.message(), "VOYAGE_API_KEY"));
|
||||
ASSERT_FALSE(has(e.message(), kKey));
|
||||
}
|
||||
ASSERT_EQ(rig.sent_total(), std::size_t(0));
|
||||
}
|
||||
|
||||
TEST(retrieval_core, self_hosted_databases_need_their_url_too) {
|
||||
Rig rig;
|
||||
Qdrant q = Qdrant::with(rig.factory(), kKey, "");
|
||||
ASSERT_FALSE(q.configured());
|
||||
try {
|
||||
q.list_collections();
|
||||
FAIL("expected NotConfiguredError");
|
||||
} catch (const NotConfiguredError& e) {
|
||||
ASSERT_TRUE(has(e.message(), "QDRANT_URL"));
|
||||
}
|
||||
Weaviate w = Weaviate::with(rig.factory(), kKey, "localhost:8080"); // no scheme
|
||||
ASSERT_FALSE(w.configured());
|
||||
ASSERT_THROW(w.schema(), NotConfiguredError);
|
||||
ASSERT_EQ(rig.sent_total(), std::size_t(0));
|
||||
}
|
||||
|
||||
TEST(retrieval_core, seam_validators_enforce_the_caps) {
|
||||
EmbedRequest e;
|
||||
ASSERT_THROW(Embedder::validate("t", e), ApiError); // empty
|
||||
e.texts = {"ok"};
|
||||
Embedder::validate("t", e);
|
||||
e.texts = {" "};
|
||||
ASSERT_THROW(Embedder::validate("t", e), ApiError); // blank
|
||||
e.texts.assign(RetrievalCaps::kMaxEmbedTexts + 1, "x");
|
||||
ASSERT_THROW(Embedder::validate("t", e), ApiError);
|
||||
e.texts = {std::string(RetrievalCaps::kMaxTextBytes + 1, 'x')};
|
||||
ASSERT_THROW(Embedder::validate("t", e), ApiError);
|
||||
e.texts = {"ok"};
|
||||
e.input_type = "other";
|
||||
ASSERT_THROW(Embedder::validate("t", e), ApiError);
|
||||
e.input_type = "query";
|
||||
e.dimensions = RetrievalCaps::kMaxDimensions + 1;
|
||||
ASSERT_THROW(Embedder::validate("t", e), ApiError);
|
||||
|
||||
RerankRequest r;
|
||||
r.query = "q";
|
||||
ASSERT_THROW(Reranker::validate("t", r), ApiError); // no documents
|
||||
r.documents = {"a", "b"};
|
||||
Reranker::validate("t", r);
|
||||
r.top_n = -1;
|
||||
ASSERT_THROW(Reranker::validate("t", r), ApiError);
|
||||
r.top_n = 1;
|
||||
r.query = " ";
|
||||
ASSERT_THROW(Reranker::validate("t", r), ApiError);
|
||||
|
||||
SearchRequest s;
|
||||
s.query = "q";
|
||||
WebSearch::validate("t", s);
|
||||
s.max_results = 0;
|
||||
ASSERT_THROW(WebSearch::validate("t", s), ApiError);
|
||||
s.max_results = RetrievalCaps::kMaxSearchResults + 1;
|
||||
ASSERT_THROW(WebSearch::validate("t", s), ApiError);
|
||||
s.max_results = 3;
|
||||
s.recency = "decade";
|
||||
ASSERT_THROW(WebSearch::validate("t", s), ApiError);
|
||||
}
|
||||
|
||||
TEST(retrieval_core, data_plane_host_must_be_a_bare_dns_name) {
|
||||
Rig rig;
|
||||
for (const char* bad : {"https://x.svc.pinecone.io", "x.svc.pinecone.io/path", "x svc", "x:443", "..x", "-x"})
|
||||
ASSERT_THROW(Pinecone::with(rig.factory(), kKey, bad), ApiError);
|
||||
Pinecone ok = Pinecone::with(rig.factory(), kKey, "my-index-abc123.svc.aped-4627-b74a.pinecone.io");
|
||||
ASSERT_EQ(ok.gateway().hosts().size(), std::size_t(3));
|
||||
}
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
// Opt-in live checks. MOLO_LIVE_NET=1 enables the network. Part 1 needs no key:
|
||||
// a deliberately invalid key must be refused with an auth-style status, which
|
||||
// proves host, TLS and path. Part 2 runs read-only calls when the provider key
|
||||
// is configured (via llm::Config). Calls that cost money additionally need
|
||||
// MOLO_LIVE_PAID=1. Without the gates each test prints a loud SKIP. Nothing is
|
||||
// created, changed or deleted.
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
|
||||
#include "clients/transports/http_transport.h"
|
||||
#include "llm/config.h"
|
||||
#include "support.h"
|
||||
#include "vendors/retrieval/brave/brave.h"
|
||||
#include "vendors/retrieval/exa/exa.h"
|
||||
#include "vendors/retrieval/jina/jina.h"
|
||||
#include "vendors/retrieval/mixedbread/mixedbread.h"
|
||||
#include "vendors/retrieval/pinecone/pinecone.h"
|
||||
#include "vendors/retrieval/qdrant/qdrant.h"
|
||||
#include "vendors/retrieval/tavily/tavily.h"
|
||||
#include "vendors/retrieval/voyage/voyage.h"
|
||||
#include "vendors/retrieval/weaviate/weaviate.h"
|
||||
|
||||
using namespace molo::retrieval;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
namespace {
|
||||
// Read once at start-up: tests/src/llm unsets MOLO_LIVE_NET in a test that runs earlier.
|
||||
const bool kNet = std::getenv("MOLO_LIVE_NET") != nullptr;
|
||||
const bool kPaid = std::getenv("MOLO_LIVE_PAID") != nullptr;
|
||||
bool net() { return kNet; }
|
||||
bool paid() { return kPaid; }
|
||||
bool key(const char* env) { return !molo::llm::Config::get(env).empty(); }
|
||||
|
||||
TransportFactory http() {
|
||||
return [] { return std::unique_ptr<molo::clients::Transport>(new molo::clients::transports::HttpTransport()); };
|
||||
}
|
||||
|
||||
// Runs `call` with an invalid key and prints the status; accepts 401, 403 and
|
||||
// 422. Returns a problem description, empty when fine. `tls12_only` marks a
|
||||
// host that only speaks TLS 1.2: src/clients pins TLS 1.3, so the handshake
|
||||
// fails there (a foundation gap listed in SPEC.md); that outcome is printed
|
||||
// loudly as a FINDING instead of failing the suite.
|
||||
template <typename F>
|
||||
std::string probe(const char* name, F call, bool tls12_only = false) {
|
||||
if (!net()) {
|
||||
std::cout << "SKIP retrieval_live.unauthenticated_probe " << name << " (set MOLO_LIVE_NET=1)\n";
|
||||
return "";
|
||||
}
|
||||
try {
|
||||
call();
|
||||
return std::string(name) + ": an invalid key was accepted";
|
||||
} catch (const ApiError& e) {
|
||||
std::cout << " " << name << " with an invalid key: " << e.http_status() << " " << e.type() << " "
|
||||
<< e.message().substr(0, 160) << "\n";
|
||||
if (e.http_status() == 401 || e.http_status() == 403 || e.http_status() == 422) return "";
|
||||
if (tls12_only && e.type() == "UNAVAILABLE" && e.message().find("handshake") != std::string::npos) {
|
||||
std::cout << "FINDING retrieval_live: " << name << " is TLS 1.2 only and src/clients pins TLS 1.3: unreachable until the transport allows 1.2\n";
|
||||
return "";
|
||||
}
|
||||
return std::string(name) + ": unexpected " + std::to_string(e.http_status()) + " " + e.type();
|
||||
}
|
||||
}
|
||||
|
||||
bool gate(const char* test, const char* env, bool costs) {
|
||||
if (!net() || !key(env) || (costs && !paid())) {
|
||||
std::cout << "SKIP retrieval_live." << test << " (set MOLO_LIVE_NET=1 and " << env << (costs ? " and MOLO_LIVE_PAID=1" : "") << ")\n";
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(retrieval_live, unauthenticated_probes_answer_with_an_auth_status) {
|
||||
const std::string bad = "invalid-molo-probe-key";
|
||||
std::string problems;
|
||||
auto add = [&](const std::string& p) { problems += p.empty() ? "" : p + "; "; };
|
||||
add(probe("voyage", [&] {
|
||||
EmbedRequest r;
|
||||
r.texts = {"probe"};
|
||||
Voyage::with(http(), bad).embed(r);
|
||||
}));
|
||||
add(probe("jina", [&] {
|
||||
EmbedRequest r;
|
||||
r.texts = {"probe"};
|
||||
Jina::with(http(), bad).embed(r);
|
||||
}));
|
||||
add(probe("pinecone", [&] { Pinecone::with(http(), bad).list_indexes(); }));
|
||||
add(probe("mixedbread", [&] { Mixedbread::with(http(), bad).list_stores(); }, true));
|
||||
add(probe("tavily", [&] { Tavily::with(http(), bad).usage(); }));
|
||||
add(probe("exa", [&] {
|
||||
SearchRequest s;
|
||||
s.query = "probe";
|
||||
Exa::with(http(), bad).search(s);
|
||||
}));
|
||||
add(probe("brave", [&] {
|
||||
SearchRequest s;
|
||||
s.query = "probe";
|
||||
Brave::with(http(), bad).search(s);
|
||||
}));
|
||||
if (!problems.empty()) FAIL(problems);
|
||||
}
|
||||
|
||||
TEST(retrieval_live, pinecone_lists_indexes) {
|
||||
if (!gate("pinecone_lists_indexes", Pinecone::kKeyEnv, false)) return;
|
||||
Pinecone p = Pinecone::from_env();
|
||||
Json out = p.list_indexes();
|
||||
std::cout << " pinecone indexes: " << J::array(out, "indexes").size() << "\n";
|
||||
ASSERT_TRUE(out.is_object());
|
||||
}
|
||||
|
||||
TEST(retrieval_live, qdrant_lists_collections) {
|
||||
if (!gate("qdrant_lists_collections", Qdrant::kKeyEnv, false) || !key(Qdrant::kUrlEnv)) {
|
||||
std::cout << "SKIP retrieval_live.qdrant_lists_collections (needs QDRANT_URL too)\n";
|
||||
return;
|
||||
}
|
||||
Json out = Qdrant::from_env().list_collections();
|
||||
ASSERT_TRUE(out.has("result"));
|
||||
}
|
||||
|
||||
TEST(retrieval_live, weaviate_dumps_the_schema) {
|
||||
if (!gate("weaviate_dumps_the_schema", Weaviate::kKeyEnv, false) || !key(Weaviate::kUrlEnv)) {
|
||||
std::cout << "SKIP retrieval_live.weaviate_dumps_the_schema (needs WEAVIATE_URL too)\n";
|
||||
return;
|
||||
}
|
||||
Json out = Weaviate::from_env().schema();
|
||||
ASSERT_TRUE(out.is_object());
|
||||
}
|
||||
|
||||
TEST(retrieval_live, mixedbread_lists_stores) {
|
||||
if (!gate("mixedbread_lists_stores", Mixedbread::kKeyEnv, false)) return;
|
||||
ASSERT_TRUE(Mixedbread::from_env().list_stores().is_object());
|
||||
}
|
||||
|
||||
TEST(retrieval_live, one_tiny_call_per_metered_provider) {
|
||||
if (!gate("voyage_one_embedding", Voyage::kKeyEnv, true)) {
|
||||
} else {
|
||||
EmbedRequest r;
|
||||
r.texts = {"hello"};
|
||||
EmbedResult out = Voyage::from_env().embed(r);
|
||||
std::cout << " voyage dims: " << out.vectors.at(0).size() << "\n";
|
||||
ASSERT_TRUE(!out.vectors.at(0).empty());
|
||||
}
|
||||
if (gate("jina_one_embedding", Jina::kKeyEnv, true)) {
|
||||
EmbedRequest r;
|
||||
r.texts = {"hello"};
|
||||
ASSERT_TRUE(!Jina::from_env().embed(r).vectors.at(0).empty());
|
||||
}
|
||||
if (gate("tavily_one_search", Tavily::kKeyEnv, true)) {
|
||||
SearchRequest s;
|
||||
s.query = "moloapp";
|
||||
s.max_results = 1;
|
||||
ASSERT_TRUE(Tavily::from_env().search(s).hits.size() <= 1);
|
||||
}
|
||||
if (gate("exa_one_search", Exa::kKeyEnv, true)) {
|
||||
SearchRequest s;
|
||||
s.query = "moloapp";
|
||||
s.max_results = 1;
|
||||
ASSERT_TRUE(Exa::from_env().search(s).hits.size() <= 1);
|
||||
}
|
||||
if (gate("brave_one_search", Brave::kKeyEnv, true)) {
|
||||
SearchRequest s;
|
||||
s.query = "moloapp";
|
||||
s.max_results = 1;
|
||||
ASSERT_TRUE(Brave::from_env().search(s).hits.size() <= 1);
|
||||
}
|
||||
}
|
||||
+577
@@ -0,0 +1,577 @@
|
||||
// Provider classes over a recording transport. Assertions are on the request
|
||||
// our code builds (method, target, headers, body) and on our parsing. Replies
|
||||
// are FIXTURES: bodies shaped after each provider's spec or SDK (schema names
|
||||
// are cited), hand-built, not recorded from the live service.
|
||||
#include "support.h"
|
||||
#include "vendors/retrieval/brave/brave.h"
|
||||
#include "vendors/retrieval/exa/exa.h"
|
||||
#include "vendors/retrieval/jina/jina.h"
|
||||
#include "vendors/retrieval/mixedbread/mixedbread.h"
|
||||
#include "vendors/retrieval/pinecone/pinecone.h"
|
||||
#include "vendors/retrieval/qdrant/qdrant.h"
|
||||
#include "vendors/retrieval/tavily/tavily.h"
|
||||
#include "vendors/retrieval/voyage/voyage.h"
|
||||
#include "vendors/retrieval/weaviate/weaviate.h"
|
||||
|
||||
using namespace molo::retrieval;
|
||||
using namespace retrieval_test;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
namespace {
|
||||
Json parse(const std::string& s) { return Json::parse(s); }
|
||||
std::string header(const MockTransport::SeenRequest& r, const char* name) {
|
||||
auto v = r.headers.get(name);
|
||||
return v ? *v : "";
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// ---- Voyage ----------------------------------------------------------------
|
||||
|
||||
// FIXTURE: voyageai SDK objects (data[].embedding/index, usage.total_tokens).
|
||||
static const char* kVoyageEmbeddings =
|
||||
"{\"object\":\"list\",\"data\":[{\"object\":\"embedding\",\"embedding\":[0.5,0.25],\"index\":1},"
|
||||
"{\"object\":\"embedding\",\"embedding\":[1.0,2.0],\"index\":0}],\"model\":\"voyage-3.5\",\"usage\":{\"total_tokens\":7}}";
|
||||
|
||||
TEST(retrieval_voyage, embed_builds_the_request_and_orders_by_index) {
|
||||
Rig rig;
|
||||
Voyage v = Voyage::with(rig.factory(), kKey);
|
||||
rig.answer(0, Method::kPost, "/v1/embeddings", kVoyageEmbeddings);
|
||||
EmbedRequest r;
|
||||
r.texts = {"first", "second"};
|
||||
r.input_type = "document";
|
||||
r.dimensions = 256;
|
||||
EmbedResult out = v.embed(r);
|
||||
auto seen = rig.last();
|
||||
ASSERT_EQ(seen.method, std::string("POST"));
|
||||
ASSERT_EQ(seen.target, std::string("/v1/embeddings"));
|
||||
ASSERT_EQ(header(seen, "Authorization"), std::string("Bearer ") + kKey);
|
||||
Json body = parse(seen.body);
|
||||
ASSERT_EQ(body.find("model")->as_string(), std::string("voyage-3.5"));
|
||||
ASSERT_EQ(body.find("input_type")->as_string(), std::string("document"));
|
||||
ASSERT_EQ(body.find("output_dimension")->as_int(), 256L);
|
||||
ASSERT_EQ(body.find("input")->as_array().size(), std::size_t(2));
|
||||
ASSERT_EQ(out.vectors.size(), std::size_t(2));
|
||||
ASSERT_EQ(out.vectors[0][0], 1.0f); // index 0 first although the reply listed it second
|
||||
ASSERT_EQ(out.vectors[1][1], 0.25f);
|
||||
ASSERT_EQ(out.total_tokens, 7L);
|
||||
}
|
||||
|
||||
TEST(retrieval_voyage, a_reply_with_the_wrong_vector_count_is_a_loud_error) {
|
||||
Rig rig;
|
||||
Voyage v = Voyage::with(rig.factory(), kKey);
|
||||
rig.answer(0, Method::kPost, "/v1/embeddings", kVoyageEmbeddings);
|
||||
EmbedRequest r;
|
||||
r.texts = {"only one"};
|
||||
try {
|
||||
v.embed(r);
|
||||
FAIL("expected BAD_RESPONSE");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.type(), std::string("BAD_RESPONSE"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(retrieval_voyage, rerank_maps_top_n_to_top_k_and_sorts_best_first) {
|
||||
Rig rig;
|
||||
Voyage v = Voyage::with(rig.factory(), kKey);
|
||||
// FIXTURE: voyageai RerankingObject (data[].index, relevance_score, usage.total_tokens)
|
||||
rig.answer(0, Method::kPost, "/v1/rerank",
|
||||
"{\"object\":\"list\",\"data\":[{\"index\":0,\"relevance_score\":0.2},{\"index\":2,\"relevance_score\":0.9}],"
|
||||
"\"model\":\"rerank-2.5\",\"usage\":{\"total_tokens\":11}}");
|
||||
RerankRequest r;
|
||||
r.query = "q";
|
||||
r.documents = {"a", "b", "c"};
|
||||
r.top_n = 2;
|
||||
RerankResult out = v.rerank(r);
|
||||
Json body = parse(rig.last().body);
|
||||
ASSERT_EQ(body.find("top_k")->as_int(), 2L);
|
||||
ASSERT_EQ(body.find("model")->as_string(), std::string("rerank-2.5"));
|
||||
ASSERT_EQ(out.hits.size(), std::size_t(2));
|
||||
ASSERT_EQ(out.hits[0].index, 2);
|
||||
ASSERT_EQ(out.total_tokens, 11L);
|
||||
}
|
||||
|
||||
TEST(retrieval_voyage, multimodal_and_contextualized_bodies) {
|
||||
Rig rig;
|
||||
Voyage v = Voyage::with(rig.factory(), kKey);
|
||||
rig.answer(0, Method::kPost, "/v1/multimodalembeddings",
|
||||
"{\"data\":[{\"embedding\":[0.1],\"index\":0}],\"model\":\"voyage-multimodal-3\",\"usage\":{\"text_tokens\":3,\"image_pixels\":100,\"total_tokens\":4}}");
|
||||
VoyageSegment text{VoyageSegment::Kind::kText, "a cat"};
|
||||
VoyageSegment img{VoyageSegment::Kind::kImageUrl, "https://example.com/cat.png"};
|
||||
EmbedResult m = v.multimodal_embed({{text, img}}, "", "query");
|
||||
Json body = parse(rig.last().body);
|
||||
const Json& content = *body.find("inputs")->as_array()[0].find("content");
|
||||
ASSERT_EQ(content.as_array()[0].find("type")->as_string(), std::string("text"));
|
||||
ASSERT_EQ(content.as_array()[1].find("image_url")->as_string(), std::string("https://example.com/cat.png"));
|
||||
ASSERT_EQ(m.vectors.size(), std::size_t(1));
|
||||
ASSERT_THROW(v.multimodal_embed({}, "", ""), ApiError);
|
||||
ASSERT_THROW(v.multimodal_embed({{}}, "", ""), ApiError);
|
||||
|
||||
// FIXTURE: voyageai ContextualizedEmbeddingsObject (data[].data[].embedding)
|
||||
rig.answer(0, Method::kPost, "/v1/contextualizedembeddings",
|
||||
"{\"data\":[{\"index\":0,\"data\":[{\"embedding\":[1,2],\"index\":0},{\"embedding\":[3,4],\"index\":1}]}],\"model\":\"voyage-context-3\",\"usage\":{\"total_tokens\":9}}");
|
||||
VoyageContextualResult c = v.contextualized_embed({{"chunk one", "chunk two"}}, "", "document", 0);
|
||||
ASSERT_EQ(c.embeddings.size(), std::size_t(1));
|
||||
ASSERT_EQ(c.embeddings[0].size(), std::size_t(2));
|
||||
ASSERT_EQ(c.embeddings[0][1][1], 4.0f);
|
||||
ASSERT_EQ(c.total_tokens, 9L);
|
||||
}
|
||||
|
||||
// ---- Jina ------------------------------------------------------------------
|
||||
|
||||
TEST(retrieval_jina, embed_rerank_and_the_four_hosts) {
|
||||
Rig rig;
|
||||
Jina j = Jina::with(rig.factory(), kKey);
|
||||
ASSERT_EQ(rig.transports.size(), std::size_t(4)); // api, reader, search, segment
|
||||
// FIXTURE: jina openapi EmbeddingResponse (data[].index/embedding, usage.total_tokens)
|
||||
rig.answer(0, Method::kPost, "/v1/embeddings",
|
||||
"{\"model\":\"jina-embeddings-v3\",\"object\":\"list\",\"usage\":{\"total_tokens\":5},\"data\":[{\"object\":\"embedding\",\"index\":0,\"embedding\":[0.1,0.2]}]}");
|
||||
EmbedRequest e;
|
||||
e.texts = {"hello"};
|
||||
e.input_type = "query";
|
||||
EmbedResult er = j.embed(e);
|
||||
Json body = parse(rig.last(0).body);
|
||||
ASSERT_EQ(body.find("task")->as_string(), std::string("retrieval.query"));
|
||||
ASSERT_EQ(er.total_tokens, 5L);
|
||||
|
||||
// FIXTURE: jina openapi RerankingResponse (results[].index/relevance_score)
|
||||
rig.answer(0, Method::kPost, "/v1/rerank",
|
||||
"{\"model\":\"jina-reranker-v3\",\"object\":\"list\",\"usage\":{\"total_tokens\":8},\"results\":[{\"index\":1,\"relevance_score\":0.3},{\"index\":0,\"relevance_score\":0.8}]}");
|
||||
RerankRequest r;
|
||||
r.query = "q";
|
||||
r.documents = {"a", "b"};
|
||||
RerankResult rr = j.rerank(r);
|
||||
ASSERT_EQ(rr.hits[0].index, 0);
|
||||
ASSERT_EQ(parse(rig.last(0).body).find("return_documents")->as_bool(), false);
|
||||
}
|
||||
|
||||
TEST(retrieval_jina, reader_search_segmenter_classifier_requests) {
|
||||
Rig rig;
|
||||
Jina j = Jina::with(rig.factory(), kKey);
|
||||
// reader is host 1
|
||||
rig.answer(1, Method::kGet, "/", "{\"code\":200,\"status\":20000,\"data\":{\"title\":\"Example\",\"url\":\"https://example.com\",\"content\":\"# Hi\",\"usage\":{\"tokens\":12}}}");
|
||||
JinaPage p = j.read("https://example.com/page?x=1");
|
||||
auto seen = rig.last(1);
|
||||
ASSERT_TRUE(has(seen.target, "url=https%3A%2F%2Fexample.com%2Fpage%3Fx%3D1"));
|
||||
ASSERT_EQ(header(seen, "Accept"), std::string("application/json"));
|
||||
ASSERT_EQ(p.title, std::string("Example"));
|
||||
ASSERT_EQ(p.tokens, 12L);
|
||||
ASSERT_THROW(j.read("ftp://example.com"), ApiError);
|
||||
ASSERT_THROW(j.read("https://example.com/a b"), ApiError);
|
||||
// search is host 2
|
||||
rig.answer(2, Method::kGet, "/search", "{\"code\":200,\"data\":[{\"title\":\"T\",\"url\":\"https://t.example\",\"description\":\"D\",\"content\":\"C\"}]}");
|
||||
SearchRequest s;
|
||||
s.query = "jina ai";
|
||||
SearchResult sr = j.search(s);
|
||||
ASSERT_TRUE(has(rig.last(2).target, "/search?q=jina%20ai") || has(rig.last(2).target, "/search?q=jina+ai"));
|
||||
ASSERT_EQ(sr.hits.size(), std::size_t(1));
|
||||
ASSERT_EQ(sr.hits[0].snippet, std::string("D"));
|
||||
s.recency = "day";
|
||||
ASSERT_THROW(j.search(s), ApiError); // no recency filter at jina
|
||||
// segmenter is host 3 (documented shape, UNVERIFIED against a spec)
|
||||
rig.answer(3, Method::kPost, "/", "{\"num_tokens\":4,\"num_chunks\":2,\"chunks\":[\"a b\",\"c d\"]}");
|
||||
JinaSegments seg = j.segment("a b c d", "cl100k_base", 2);
|
||||
ASSERT_EQ(seg.chunks.size(), std::size_t(2));
|
||||
Json sb = parse(rig.last(3).body);
|
||||
ASSERT_EQ(sb.find("return_chunks")->as_bool(), true);
|
||||
ASSERT_EQ(sb.find("max_chunk_length")->as_int(), 2L);
|
||||
// classifier (documented shape, UNVERIFIED): api host
|
||||
rig.answer(0, Method::kPost, "/v1/classify", "{\"data\":[{\"index\":0,\"prediction\":\"spam\",\"score\":0.9}]}");
|
||||
auto cls = j.classify({"win money"}, {"spam", "ham"}, "");
|
||||
ASSERT_EQ(cls[0].label, std::string("spam"));
|
||||
ASSERT_THROW(j.classify({"x"}, {"only-one"}, ""), ApiError);
|
||||
}
|
||||
|
||||
TEST(retrieval_jina, cancelling_a_batch_is_the_destructive_explicit_method) {
|
||||
Rig rig;
|
||||
Jina j = Jina::with(rig.factory(), kKey);
|
||||
rig.answer(0, Method::kDelete, "/v1/batch/{batch_id}", "{\"batch_id\":\"b1\",\"status\":\"cancelled\"}");
|
||||
j.cancel_batch("b1");
|
||||
ASSERT_EQ(rig.last(0).method, std::string("DELETE"));
|
||||
ASSERT_EQ(rig.last(0).target, std::string("/v1/batch/b1"));
|
||||
ASSERT_THROW(j.cancel_batch("../x"), ApiError);
|
||||
ASSERT_THROW(j.gateway().call("cancel_batch", CallArgs{}), ApiError);
|
||||
}
|
||||
|
||||
// ---- Pinecone --------------------------------------------------------------
|
||||
|
||||
TEST(retrieval_pinecone, headers_and_control_plane) {
|
||||
Rig rig;
|
||||
Pinecone p = Pinecone::with(rig.factory(), kKey);
|
||||
rig.answer(0, Method::kGet, "/indexes/docs", "{\"name\":\"docs\",\"host\":\"docs-abc.svc.pinecone.io\"}");
|
||||
Json d = p.describe_index("docs");
|
||||
auto seen = rig.last(0);
|
||||
ASSERT_EQ(header(seen, "Api-Key"), std::string(kKey));
|
||||
ASSERT_EQ(header(seen, "X-Pinecone-Api-Version"), std::string("2026-07"));
|
||||
ASSERT_TRUE(header(seen, "Authorization").empty()); // key travels in Api-Key only
|
||||
ASSERT_EQ(d.find("host")->as_string(), std::string("docs-abc.svc.pinecone.io"));
|
||||
rig.answer(0, Method::kPost, "/indexes", "{\"name\":\"docs\"}");
|
||||
p.create_index("docs", 1024, "cosine", "aws", "us-east-1");
|
||||
Json b = parse(rig.last(0).body);
|
||||
ASSERT_EQ(b.find("schema")->find("fields")->find("vector")->find("dimension")->as_int(), 1024L);
|
||||
ASSERT_EQ(b.find("deployment")->find("deployment_type")->as_string(), std::string("managed"));
|
||||
ASSERT_THROW(p.create_index("docs", 0, "cosine", "aws", "us-east-1"), ApiError);
|
||||
ASSERT_THROW(p.create_index("docs", 8, "manhattan", "aws", "us-east-1"), ApiError);
|
||||
rig.answer(0, Method::kDelete, "/indexes/docs", "");
|
||||
p.delete_index("docs");
|
||||
ASSERT_EQ(rig.last(0).method, std::string("DELETE"));
|
||||
}
|
||||
|
||||
TEST(retrieval_pinecone, data_plane_needs_the_index_host) {
|
||||
Rig none;
|
||||
Pinecone p0 = Pinecone::with(none.factory(), kKey);
|
||||
PineconeQuery q;
|
||||
q.vector = {1.0f};
|
||||
ASSERT_THROW(p0.query(q), ApiError); // no data client
|
||||
ASSERT_EQ(none.sent_total(), std::size_t(0));
|
||||
|
||||
Rig rig;
|
||||
Pinecone p = Pinecone::with(rig.factory(), kKey, "docs-abc.svc.pinecone.io");
|
||||
// FIXTURE: pinecone db_data QueryResponse (matches[].id/score/values/metadata)
|
||||
rig.answer(2, Method::kPost, "/query", "{\"matches\":[{\"id\":\"v1\",\"score\":0.99,\"metadata\":{\"genre\":\"x\"}}],\"namespace\":\"ns\"}");
|
||||
q.top_k = 3;
|
||||
q.ns = "ns";
|
||||
Json filter = Json::object();
|
||||
filter.set("genre", Json("x"));
|
||||
q.filter = filter;
|
||||
auto m = p.query(q);
|
||||
Json b = parse(rig.last(2).body);
|
||||
ASSERT_EQ(b.find("topK")->as_int(), 3L);
|
||||
ASSERT_EQ(b.find("namespace")->as_string(), std::string("ns"));
|
||||
ASSERT_EQ(m[0].id, std::string("v1"));
|
||||
ASSERT_EQ(m[0].metadata.find("genre")->as_string(), std::string("x"));
|
||||
PineconeQuery both = q;
|
||||
both.id = "v1";
|
||||
ASSERT_THROW(p.query(both), ApiError); // vector and id together
|
||||
q.top_k = 10001;
|
||||
ASSERT_THROW(p.query(q), ApiError);
|
||||
}
|
||||
|
||||
TEST(retrieval_pinecone, fetch_repeats_the_ids_parameter_and_list_pages) {
|
||||
Rig rig;
|
||||
Pinecone p = Pinecone::with(rig.factory(), kKey, "docs-abc.svc.pinecone.io");
|
||||
rig.answer(2, Method::kGet, "/vectors/fetch", "{\"vectors\":{},\"namespace\":\"\"}");
|
||||
p.fetch({"a", "b"}, "ns");
|
||||
ASSERT_EQ(rig.last(2).target, std::string("/vectors/fetch?namespace=ns&ids=a&ids=b"));
|
||||
ASSERT_THROW(p.fetch({}, ""), ApiError);
|
||||
ASSERT_THROW(p.fetch(std::vector<std::string>(101, "x"), ""), ApiError);
|
||||
// FIXTURE: pinecone db_data ListResponse (vectors[].id, pagination.next)
|
||||
rig.answer(2, Method::kGet, "/vectors/list", "{\"vectors\":[{\"id\":\"a\"},{\"id\":\"b\"}],\"pagination\":{\"next\":\"tok\"}}");
|
||||
PineconePage page = p.list("pre", "ns", 2, "");
|
||||
ASSERT_EQ(page.ids.size(), std::size_t(2));
|
||||
ASSERT_EQ(page.next_token, std::string("tok"));
|
||||
ASSERT_THROW(p.list("", "", 101, ""), ApiError);
|
||||
}
|
||||
|
||||
TEST(retrieval_pinecone, the_three_deletes_share_one_destructive_row_and_stay_explicit) {
|
||||
Rig rig;
|
||||
Pinecone p = Pinecone::with(rig.factory(), kKey, "docs-abc.svc.pinecone.io");
|
||||
rig.answer(2, Method::kPost, "/vectors/delete", "{}");
|
||||
p.delete_ids({"a"}, "ns");
|
||||
ASSERT_TRUE(has(rig.last(2).body, "\"ids\":[\"a\"]"));
|
||||
p.delete_all("ns");
|
||||
ASSERT_TRUE(has(rig.last(2).body, "\"deleteAll\":true"));
|
||||
Json f = Json::object();
|
||||
f.set("genre", Json("x"));
|
||||
p.delete_by_filter(f, "");
|
||||
ASSERT_TRUE(has(rig.last(2).body, "\"filter\""));
|
||||
ASSERT_THROW(p.delete_by_filter(Json::object(), ""), ApiError); // empty filter would match nothing useful
|
||||
ASSERT_THROW(p.delete_ids({}, ""), ApiError);
|
||||
ASSERT_THROW(p.gateway().call("delete_vectors", CallArgs{}), ApiError);
|
||||
}
|
||||
|
||||
TEST(retrieval_pinecone, inference_embed_and_rerank) {
|
||||
Rig rig;
|
||||
Pinecone p = Pinecone::with(rig.factory(), kKey);
|
||||
// FIXTURE: pinecone inference EmbeddingsList (data[].values, usage.total_tokens)
|
||||
rig.answer(1, Method::kPost, "/embed", "{\"model\":\"multilingual-e5-large\",\"vector_type\":\"dense\",\"data\":[{\"values\":[0.1,0.2]}],\"usage\":{\"total_tokens\":3}}");
|
||||
EmbedRequest e;
|
||||
e.texts = {"hi"};
|
||||
e.input_type = "query";
|
||||
EmbedResult out = p.embed(e);
|
||||
Json b = parse(rig.last(1).body);
|
||||
ASSERT_EQ(b.find("parameters")->find("input_type")->as_string(), std::string("query"));
|
||||
ASSERT_EQ(b.find("inputs")->as_array()[0].find("text")->as_string(), std::string("hi"));
|
||||
ASSERT_EQ(out.vectors[0].size(), std::size_t(2));
|
||||
// FIXTURE: pinecone inference RerankResult (data[].index/score)
|
||||
rig.answer(1, Method::kPost, "/rerank", "{\"model\":\"bge-reranker-v2-m3\",\"data\":[{\"index\":1,\"score\":0.7},{\"index\":0,\"score\":0.1}],\"usage\":{\"rerank_units\":1}}");
|
||||
RerankRequest r;
|
||||
r.query = "q";
|
||||
r.documents = {"a", "b"};
|
||||
RerankResult rr = p.rerank(r);
|
||||
ASSERT_EQ(rr.hits[0].index, 1);
|
||||
}
|
||||
|
||||
// ---- Qdrant ----------------------------------------------------------------
|
||||
|
||||
TEST(retrieval_qdrant, collections_points_and_query) {
|
||||
Rig rig;
|
||||
Qdrant q = Qdrant::with(rig.factory(), kKey, "https://xyz.cloud.qdrant.io:6333/");
|
||||
rig.answer(0, Method::kPut, "/collections/docs", "{\"result\":true,\"status\":\"ok\",\"time\":0.1}");
|
||||
q.create_collection("docs", 4, "Cosine", true);
|
||||
auto seen = rig.last();
|
||||
ASSERT_EQ(header(seen, "api-key"), std::string(kKey));
|
||||
Json b = parse(seen.body);
|
||||
ASSERT_EQ(b.find("vectors")->find("size")->as_int(), 4L);
|
||||
ASSERT_EQ(b.find("vectors")->find("distance")->as_string(), std::string("Cosine"));
|
||||
ASSERT_THROW(q.create_collection("docs", 4, "cosine", true), ApiError); // distance is case sensitive
|
||||
|
||||
rig.answer(0, Method::kPut, "/collections/docs/points", "{\"result\":{\"operation_id\":1,\"status\":\"completed\"},\"status\":\"ok\"}");
|
||||
QdrantPoint pt;
|
||||
pt.id = Json(1);
|
||||
pt.vector = {0.1f, 0.2f, 0.3f, 0.4f};
|
||||
Json payload = Json::object();
|
||||
payload.set("city", Json("Berlin"));
|
||||
pt.payload = payload;
|
||||
q.upsert("docs", {pt}, true);
|
||||
ASSERT_EQ(rig.last().target, std::string("/collections/docs/points?wait=true"));
|
||||
QdrantPoint bad = pt;
|
||||
bad.id = Json(-1);
|
||||
ASSERT_THROW(q.upsert("docs", {bad}, true), ApiError);
|
||||
|
||||
// FIXTURE: qdrant openapi QueryResponse (result.points[].id/score/payload)
|
||||
rig.answer(0, Method::kPost, "/collections/docs/points/query",
|
||||
"{\"result\":{\"points\":[{\"id\":1,\"score\":0.9,\"payload\":{\"city\":\"Berlin\"}}]},\"status\":\"ok\"}");
|
||||
auto hits = q.query("docs", {0.1f, 0.2f, 0.3f, 0.4f}, 5, Json(), true);
|
||||
ASSERT_EQ(hits.size(), std::size_t(1));
|
||||
ASSERT_EQ(hits[0].payload.find("city")->as_string(), std::string("Berlin"));
|
||||
ASSERT_EQ(parse(rig.last().body).find("limit")->as_int(), 5L);
|
||||
ASSERT_THROW(q.query("docs", {}, 5, Json(), true), ApiError);
|
||||
ASSERT_THROW(q.query("docs", {1.0f}, 0, Json(), true), ApiError);
|
||||
}
|
||||
|
||||
TEST(retrieval_qdrant, deleting_points_is_explicit_and_filter_must_not_be_empty) {
|
||||
Rig rig;
|
||||
Qdrant q = Qdrant::with(rig.factory(), kKey, "https://qd.example.com");
|
||||
rig.answer(0, Method::kPost, "/collections/docs/points/delete", "{\"result\":{\"status\":\"completed\"},\"status\":\"ok\"}");
|
||||
q.delete_points("docs", {Json(1), Json("uuid-1")});
|
||||
ASSERT_EQ(rig.last().target, std::string("/collections/docs/points/delete?wait=true"));
|
||||
ASSERT_THROW(q.delete_points("docs", {}), ApiError);
|
||||
ASSERT_THROW(q.delete_points_by_filter("docs", Json::object()), ApiError);
|
||||
ASSERT_THROW(q.gateway().call("delete_points", CallArgs{}), ApiError);
|
||||
}
|
||||
|
||||
TEST(retrieval_qdrant, count_exists_and_snapshots) {
|
||||
Rig rig;
|
||||
Qdrant q = Qdrant::with(rig.factory(), kKey, "https://qd.example.com");
|
||||
rig.answer(0, Method::kPost, "/collections/docs/points/count", "{\"result\":{\"count\":42},\"status\":\"ok\"}");
|
||||
ASSERT_EQ(q.count("docs", Json(), true), 42L);
|
||||
rig.answer(0, Method::kGet, "/collections/docs/exists", "{\"result\":{\"exists\":true},\"status\":\"ok\"}");
|
||||
ASSERT_TRUE(q.collection_exists("docs"));
|
||||
rig.answer(0, Method::kDelete, "/collections/docs/snapshots/{snapshot_name}", "{\"result\":true,\"status\":\"ok\"}");
|
||||
q.delete_snapshot("docs", "snap-1.snapshot");
|
||||
ASSERT_EQ(rig.last().target, std::string("/collections/docs/snapshots/snap-1.snapshot"));
|
||||
}
|
||||
|
||||
// ---- Weaviate --------------------------------------------------------------
|
||||
|
||||
TEST(retrieval_weaviate, paths_carry_the_v1_base_and_class_names_are_checked) {
|
||||
Rig rig;
|
||||
Weaviate w = Weaviate::with(rig.factory(), kKey, "https://my.weaviate.network/");
|
||||
rig.answer(0, Method::kGet, "/v1/schema", "{\"classes\":[]}");
|
||||
w.schema();
|
||||
ASSERT_EQ(rig.last().target, std::string("/v1/schema"));
|
||||
ASSERT_EQ(header(rig.last(), "Authorization"), std::string("Bearer ") + kKey);
|
||||
ASSERT_THROW(w.get_class("lowercase"), ApiError);
|
||||
ASSERT_THROW(w.get_class("Bad-Name"), ApiError);
|
||||
Json def = Json::object();
|
||||
def.set("description", Json("no class"));
|
||||
ASSERT_THROW(w.create_class(def), ApiError);
|
||||
rig.answer(0, Method::kDelete, "/v1/schema/{className}", "");
|
||||
w.delete_class("Article");
|
||||
ASSERT_EQ(rig.last().method, std::string("DELETE"));
|
||||
ASSERT_EQ(rig.last().target, std::string("/v1/schema/Article"));
|
||||
}
|
||||
|
||||
TEST(retrieval_weaviate, objects_graphql_and_search_bodies) {
|
||||
Rig rig;
|
||||
Weaviate w = Weaviate::with(rig.factory(), kKey, "https://wv.example.com");
|
||||
rig.answer(0, Method::kPost, "/v1/objects", "{\"id\":\"uuid\",\"class\":\"Article\"}");
|
||||
Json props = Json::object();
|
||||
props.set("title", Json("Hello"));
|
||||
w.create_object("Article", props, {0.1f, 0.2f});
|
||||
Json b = parse(rig.last().body);
|
||||
ASSERT_EQ(b.find("class")->as_string(), std::string("Article"));
|
||||
ASSERT_EQ(b.find("vector")->as_array().size(), std::size_t(2));
|
||||
rig.answer(0, Method::kPost, "/v1/graphql", "{\"data\":{\"Get\":{}}}");
|
||||
w.graphql("{ Get { Article { title } } }", Json());
|
||||
ASSERT_TRUE(has(rig.last().body, "\"query\""));
|
||||
rig.answer(0, Method::kPost, "/v1/search/{collection}/near-text", "{\"results\":[],\"tookMs\":3}");
|
||||
w.near_text("Article", "hello", 3);
|
||||
ASSERT_EQ(rig.last().target, std::string("/v1/search/Article/near-text"));
|
||||
ASSERT_THROW(w.near_text("Article", "", 3), ApiError);
|
||||
ASSERT_THROW(w.hybrid("Article", "x", 3, 1.5), ApiError);
|
||||
// batch delete is the destructive explicit method and defaults to nothing
|
||||
rig.answer(0, Method::kDelete, "/v1/batch/objects", "{\"results\":{}}");
|
||||
Json match = Json::object();
|
||||
match.set("class", Json("Article"));
|
||||
w.batch_delete(match, true);
|
||||
ASSERT_TRUE(has(rig.last().body, "\"dryRun\":true"));
|
||||
ASSERT_THROW(w.batch_delete(Json::object(), true), ApiError);
|
||||
}
|
||||
|
||||
// ---- Mixedbread ------------------------------------------------------------
|
||||
|
||||
TEST(retrieval_mixedbread, embed_rerank_and_caps) {
|
||||
Rig rig;
|
||||
Mixedbread m = Mixedbread::with(rig.factory(), kKey);
|
||||
// FIXTURE: mixedbread openapi EmbeddingCreateResponse (data[].embedding/index, usage.total_tokens)
|
||||
rig.answer(0, Method::kPost, "/v1/embeddings",
|
||||
"{\"object\":\"list\",\"model\":\"mixedbread-ai/mxbai-embed-large-v1\",\"data\":[{\"object\":\"embedding\",\"index\":0,\"embedding\":[0.5,0.5]}],\"usage\":{\"prompt_tokens\":2,\"total_tokens\":2}}");
|
||||
EmbedRequest e;
|
||||
e.texts = {"hi"};
|
||||
EmbedResult out = m.embed(e);
|
||||
ASSERT_EQ(out.vectors.size(), std::size_t(1));
|
||||
ASSERT_EQ(parse(rig.last().body).find("model")->as_string(), std::string("mixedbread-ai/mxbai-embed-large-v1"));
|
||||
e.texts.assign(257, "x");
|
||||
ASSERT_THROW(m.embed(e), ApiError); // spec: at most 256 inputs
|
||||
// FIXTURE: mixedbread openapi RerankResponse (data[].index/score; the spec's own example values)
|
||||
rig.answer(0, Method::kPost, "/v1/reranking",
|
||||
"{\"object\":\"list\",\"model\":\"mixedbread-ai/mxbai-rerank-v3.1-listwise\",\"data\":[{\"index\":1,\"score\":0.94,\"object\":\"rank_result\"},{\"index\":0,\"score\":0.95,\"object\":\"rank_result\"}]}");
|
||||
RerankRequest r;
|
||||
r.query = "q";
|
||||
r.documents = {"a", "b"};
|
||||
RerankResult rr = m.rerank(r);
|
||||
ASSERT_EQ(rr.hits[0].index, 0);
|
||||
ASSERT_EQ(parse(rig.last().body).find("top_k"), static_cast<const Json*>(nullptr));
|
||||
ASSERT_THROW(m.search_stores({}, "q", 3), ApiError);
|
||||
ASSERT_THROW(m.search_stores({"s1"}, "q", 101), ApiError);
|
||||
rig.answer(0, Method::kDelete, "/v1/stores/{store_identifier}", "{}");
|
||||
m.delete_store("s1");
|
||||
ASSERT_EQ(rig.last().method, std::string("DELETE"));
|
||||
}
|
||||
|
||||
// ---- Tavily ----------------------------------------------------------------
|
||||
|
||||
TEST(retrieval_tavily, search_extract_crawl_map) {
|
||||
Rig rig;
|
||||
Tavily t = Tavily::with(rig.factory(), kKey);
|
||||
// FIXTURE: tavily openapi search response (answer, results[].title/url/content/score)
|
||||
rig.answer(0, Method::kPost, "/search",
|
||||
"{\"query\":\"q\",\"answer\":\"A\",\"results\":[{\"title\":\"T\",\"url\":\"https://t.example\",\"content\":\"C\",\"score\":0.8,\"published_date\":\"2026-01-02\"}],\"response_time\":0.5}");
|
||||
SearchRequest s;
|
||||
s.query = "q";
|
||||
s.max_results = 3;
|
||||
s.recency = "week";
|
||||
TavilyOptions o;
|
||||
o.include_answer = true;
|
||||
o.search_depth = "advanced";
|
||||
SearchResult r = t.search(s, o);
|
||||
Json b = parse(rig.last().body);
|
||||
ASSERT_EQ(b.find("time_range")->as_string(), std::string("week"));
|
||||
ASSERT_EQ(b.find("search_depth")->as_string(), std::string("advanced"));
|
||||
ASSERT_EQ(b.find("include_answer")->as_bool(), true);
|
||||
ASSERT_EQ(header(rig.last(), "Authorization"), std::string("Bearer ") + kKey);
|
||||
ASSERT_EQ(r.answer, std::string("A"));
|
||||
ASSERT_TRUE(r.hits[0].has_score);
|
||||
s.max_results = 21;
|
||||
ASSERT_THROW(t.search(s), ApiError);
|
||||
o.search_depth = "turbo";
|
||||
s.max_results = 2;
|
||||
ASSERT_THROW(t.search(s, o), ApiError);
|
||||
|
||||
rig.answer(0, Method::kPost, "/extract", "{\"results\":[{\"url\":\"https://a.example\",\"raw_content\":\"body\"}],\"failed_results\":[{\"url\":\"https://b.example\",\"error\":\"x\"}]}");
|
||||
TavilyExtract x = t.extract({"https://a.example", "https://b.example"}, true);
|
||||
ASSERT_EQ(x.pages.size(), std::size_t(1));
|
||||
ASSERT_EQ(x.failed[0], std::string("https://b.example"));
|
||||
ASSERT_THROW(t.extract(std::vector<std::string>(21, "https://a.example"), false), ApiError);
|
||||
ASSERT_THROW(t.extract({"file:///etc/passwd"}, false), ApiError);
|
||||
|
||||
rig.answer(0, Method::kPost, "/crawl", "{\"base_url\":\"https://a.example\",\"results\":[{\"url\":\"https://a.example/x\",\"raw_content\":\"c\"}]}");
|
||||
ASSERT_EQ(t.crawl("https://a.example", 10, 2, "").pages.size(), std::size_t(1));
|
||||
ASSERT_THROW(t.crawl("https://a.example", 501, 2, ""), ApiError);
|
||||
ASSERT_THROW(t.crawl("https://a.example", 10, 6, ""), ApiError);
|
||||
rig.answer(0, Method::kPost, "/map", "{\"base_url\":\"https://a.example\",\"results\":[\"https://a.example/1\",\"https://a.example/2\"]}");
|
||||
ASSERT_EQ(t.map("https://a.example", 10, 1).size(), std::size_t(2));
|
||||
}
|
||||
|
||||
// ---- Exa -------------------------------------------------------------------
|
||||
|
||||
TEST(retrieval_exa, search_findsimilar_contents_answer) {
|
||||
Rig rig;
|
||||
Exa x = Exa::with(rig.factory(), kKey);
|
||||
// FIXTURE: exa openapi Result (title/url/publishedDate/score) with a highlight
|
||||
rig.answer(0, Method::kPost, "/search",
|
||||
"{\"requestId\":\"r1\",\"results\":[{\"id\":\"i\",\"title\":\"T\",\"url\":\"https://t.example\",\"publishedDate\":\"2026-01-01\",\"score\":0.4,\"highlights\":[\"H\"]}]}");
|
||||
SearchRequest s;
|
||||
s.query = "q";
|
||||
s.max_results = 4;
|
||||
s.recency = "day";
|
||||
SearchResult r = x.search(s);
|
||||
auto seen = rig.last();
|
||||
ASSERT_EQ(header(seen, "x-api-key"), std::string(kKey));
|
||||
Json b = parse(seen.body);
|
||||
ASSERT_EQ(b.find("numResults")->as_int(), 4L);
|
||||
ASSERT_TRUE(b.has("startPublishedDate"));
|
||||
ASSERT_EQ(b.find("contents")->find("text")->find("maxCharacters")->as_int(), 500L);
|
||||
ASSERT_EQ(r.hits[0].snippet, std::string("H"));
|
||||
ASSERT_THROW(x.search_typed(s, "telepathy"), ApiError);
|
||||
s.max_results = 101;
|
||||
ASSERT_THROW(x.search(s), ApiError);
|
||||
|
||||
rig.answer(0, Method::kPost, "/findSimilar", "{\"results\":[{\"title\":\"S\",\"url\":\"https://s.example\"}]}");
|
||||
ASSERT_EQ(x.find_similar("https://t.example", 2).hits.size(), std::size_t(1));
|
||||
ASSERT_THROW(x.find_similar("t.example", 2), ApiError);
|
||||
rig.answer(0, Method::kPost, "/contents", "{\"results\":[{\"url\":\"https://t.example\",\"title\":\"T\",\"text\":\"full text\"}]}");
|
||||
ASSERT_EQ(x.contents({"https://t.example"}, 100).hits[0].snippet, std::string("full text"));
|
||||
// FIXTURE: exa openapi AnswerResult (answer, citations[])
|
||||
rig.answer(0, Method::kPost, "/answer", "{\"answer\":\"42\",\"citations\":[{\"id\":\"c\",\"url\":\"https://c.example\",\"title\":\"C\"}]}");
|
||||
ExaAnswer a = x.answer("meaning of life");
|
||||
ASSERT_EQ(a.answer, std::string("42"));
|
||||
ASSERT_EQ(a.citations.size(), std::size_t(1));
|
||||
}
|
||||
|
||||
TEST(retrieval_exa, recency_dates_are_computed_not_guessed) {
|
||||
ASSERT_EQ(Exa::days_ago_iso(86400L * 10, 1), std::string("1970-01-10T00:00:00.000Z"));
|
||||
ASSERT_EQ(Exa::days_ago_iso(1767225600L, 7), std::string("2025-12-25T00:00:00.000Z")); // 2026-01-01 minus 7 days
|
||||
ASSERT_EQ(Exa::recency_days("week"), 7);
|
||||
ASSERT_EQ(Exa::recency_days(""), 0);
|
||||
}
|
||||
|
||||
// ---- Brave -----------------------------------------------------------------
|
||||
|
||||
TEST(retrieval_brave, web_news_images_suggest_and_caps) {
|
||||
Rig rig;
|
||||
Brave b = Brave::with(rig.factory(), kKey);
|
||||
// FIXTURE: brave api-reference web search response (web.results[].title/url/description/page_age)
|
||||
rig.answer(0, Method::kGet, "/res/v1/web/search",
|
||||
"{\"type\":\"search\",\"web\":{\"type\":\"search\",\"results\":[{\"title\":\"T\",\"url\":\"https://t.example\",\"description\":\"D\",\"page_age\":\"2026-01-01T00:00:00\"}]}}");
|
||||
SearchRequest s;
|
||||
s.query = "brave search";
|
||||
s.max_results = 5;
|
||||
s.recency = "month";
|
||||
SearchResult r = b.search(s);
|
||||
auto seen = rig.last();
|
||||
ASSERT_EQ(header(seen, "X-Subscription-Token"), std::string(kKey));
|
||||
ASSERT_TRUE(header(seen, "Authorization").empty());
|
||||
ASSERT_TRUE(has(seen.target, "q=brave%20search") || has(seen.target, "q=brave+search"));
|
||||
ASSERT_TRUE(has(seen.target, "count=5"));
|
||||
ASSERT_TRUE(has(seen.target, "freshness=pm"));
|
||||
ASSERT_EQ(r.hits[0].snippet, std::string("D"));
|
||||
s.max_results = 21;
|
||||
ASSERT_THROW(b.search(s), ApiError); // web search maxes out at 20
|
||||
|
||||
rig.answer(0, Method::kGet, "/res/v1/news/search", "{\"type\":\"news\",\"results\":[{\"title\":\"N\",\"url\":\"https://n.example\",\"description\":\"d\",\"age\":\"2 days ago\"}]}");
|
||||
s.max_results = 50;
|
||||
s.recency = "";
|
||||
ASSERT_EQ(b.news(s).hits[0].published, std::string("2 days ago"));
|
||||
s.max_results = 51;
|
||||
ASSERT_THROW(b.news(s), ApiError);
|
||||
rig.answer(0, Method::kGet, "/res/v1/images/search", "{\"results\":[{\"title\":\"I\",\"url\":\"https://p.example\",\"source\":\"p.example\",\"properties\":{\"url\":\"https://p.example/i.png\"}}]}");
|
||||
ASSERT_EQ(b.images("cat", 3)[0].image_url, std::string("https://p.example/i.png"));
|
||||
ASSERT_THROW(b.images("cat", 201), ApiError);
|
||||
rig.answer(0, Method::kGet, "/res/v1/suggest/search", "{\"query\":{\"original\":\"bra\"},\"results\":[{\"query\":\"brave\"},{\"query\":\"bravo\"}]}");
|
||||
ASSERT_EQ(b.suggest("bra", 2).size(), std::size_t(2));
|
||||
ASSERT_THROW(b.suggest("", 2), ApiError);
|
||||
ASSERT_THROW(b.suggest(std::string(401, 'x'), 2), ApiError); // documented 400 char limit
|
||||
}
|
||||
|
||||
TEST(retrieval_brave, answers_send_stream_false) {
|
||||
Rig rig;
|
||||
Brave b = Brave::with(rig.factory(), kKey);
|
||||
// The non-streaming reply shape is not documented: this fixture is the OpenAI-style shape we parse.
|
||||
rig.answer(0, Method::kPost, "/res/v1/chat/completions", "{\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"Because.\"}}]}");
|
||||
ASSERT_EQ(b.answer("why?"), std::string("Because."));
|
||||
Json body = parse(rig.last().body);
|
||||
ASSERT_EQ(body.find("stream")->as_bool(), false);
|
||||
ASSERT_EQ(body.find("messages")->as_array()[0].find("role")->as_string(), std::string("user"));
|
||||
}
|
||||
Reference in New Issue
Block a user