143 lines
5.6 KiB
C++
143 lines
5.6 KiB
C++
#include "vendors/retrieval/tavily/tavily.h"
|
|
|
|
#include "vendors/retrieval/caps.h"
|
|
|
|
namespace molo::retrieval {
|
|
|
|
llm::ApiClientConfig Tavily::config() {
|
|
llm::ApiClientConfig c;
|
|
c.vendor = "tavily";
|
|
c.base_url = kBaseUrl;
|
|
c.key_env = kKeyEnv;
|
|
c.auth = llm::AuthStyle::kBearer;
|
|
return c;
|
|
}
|
|
|
|
namespace {
|
|
[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("tavily", why); }
|
|
CallArgs with_body(Json body) {
|
|
CallArgs a;
|
|
a.body = std::move(body);
|
|
return a;
|
|
}
|
|
} // namespace
|
|
|
|
Tavily Tavily::from_env() { return Tavily(make_gateway(tavily_calls(), config(), {{"main", kBaseUrl}}, "", nullptr)); }
|
|
Tavily Tavily::with(const TransportFactory& factory, std::string key) {
|
|
return Tavily(make_gateway(tavily_calls(), config(), {{"main", kBaseUrl}}, std::move(key), factory));
|
|
}
|
|
|
|
void Tavily::check_url(const std::string& url) {
|
|
bool http = url.rfind("http://", 0) == 0 || url.rfind("https://", 0) == 0;
|
|
if (!http || url.size() > 8192 || url.find_first_of(" \t\r\n") != std::string::npos)
|
|
bad("url must be http(s) without spaces, at most 8192 chars");
|
|
}
|
|
|
|
SearchResult Tavily::parse_search(const Json& doc) {
|
|
SearchResult out;
|
|
out.answer = J::str(doc, "answer");
|
|
for (const Json& r : J::array(doc, "results")) {
|
|
SearchHit h;
|
|
h.title = J::str(r, "title");
|
|
h.url = J::str(r, "url");
|
|
h.snippet = J::str(r, "content");
|
|
h.published = J::str(r, "published_date");
|
|
if (const Json* s = J::at(r, "score"); s && s->is_number()) {
|
|
h.score = s->as_number();
|
|
h.has_score = true;
|
|
}
|
|
out.hits.push_back(std::move(h));
|
|
}
|
|
return out;
|
|
}
|
|
|
|
TavilyExtract Tavily::parse_extract(const Json& doc) {
|
|
TavilyExtract out;
|
|
for (const Json& r : J::array(doc, "results")) out.pages.push_back({J::str(r, "url"), J::str(r, "raw_content")});
|
|
for (const Json& r : J::array(doc, "failed_results")) out.failed.push_back(J::str(r, "url"));
|
|
return out;
|
|
}
|
|
|
|
TavilyCrawl Tavily::parse_crawl(const Json& doc) {
|
|
TavilyCrawl out;
|
|
out.base_url = J::str(doc, "base_url");
|
|
for (const Json& r : J::array(doc, "results")) out.pages.push_back({J::str(r, "url"), J::str(r, "raw_content")});
|
|
return out;
|
|
}
|
|
|
|
SearchResult Tavily::search(const SearchRequest& request) { return search(request, TavilyOptions{}); }
|
|
|
|
SearchResult Tavily::search(const SearchRequest& request, const TavilyOptions& o) {
|
|
WebSearch::validate("tavily", request);
|
|
if (request.max_results > RetrievalCaps::kTavilyMaxResults) bad("max_results is at most 20");
|
|
static const char* depths[] = {"", "basic", "advanced", "fast", "ultra-fast"};
|
|
bool depth_ok = false;
|
|
for (const char* d : depths) depth_ok = depth_ok || o.search_depth == d;
|
|
if (!depth_ok) bad("search_depth must be basic, advanced, fast or ultra-fast");
|
|
if (!o.topic.empty() && o.topic != "general" && o.topic != "news" && o.topic != "finance") bad("topic must be general, news or finance");
|
|
Json body = Json::object();
|
|
body.set("query", Json(request.query));
|
|
body.set("max_results", Json(request.max_results));
|
|
if (!o.search_depth.empty()) body.set("search_depth", Json(o.search_depth));
|
|
if (!o.topic.empty()) body.set("topic", Json(o.topic));
|
|
if (o.include_answer) body.set("include_answer", Json(true));
|
|
if (o.include_raw_content) body.set("include_raw_content", Json(true));
|
|
if (!o.include_domains.empty()) body.set("include_domains", J::strings(o.include_domains));
|
|
if (!o.exclude_domains.empty()) body.set("exclude_domains", J::strings(o.exclude_domains));
|
|
if (!request.recency.empty()) body.set("time_range", Json(request.recency));
|
|
return parse_search(gw_->call("search", with_body(body)));
|
|
}
|
|
|
|
TavilyExtract Tavily::extract(const std::vector<std::string>& urls, bool advanced) {
|
|
if (urls.empty() || urls.size() > kMaxExtractUrls) bad("urls must hold 1..20 entries");
|
|
for (const std::string& u : urls) check_url(u);
|
|
Json body = Json::object();
|
|
body.set("urls", J::strings(urls));
|
|
if (advanced) body.set("extract_depth", Json("advanced"));
|
|
return parse_extract(gw_->call("extract", with_body(body)));
|
|
}
|
|
|
|
TavilyCrawl Tavily::crawl(const std::string& url, int limit, int max_depth, const std::string& instructions) {
|
|
check_url(url);
|
|
if (limit < 1 || limit > RetrievalCaps::kMaxCrawlLimit) bad("limit must be 1..500");
|
|
if (max_depth < 1 || max_depth > RetrievalCaps::kMaxCrawlDepth) bad("max_depth must be 1..5");
|
|
Json body = Json::object();
|
|
body.set("url", Json(url));
|
|
body.set("limit", Json(limit));
|
|
body.set("max_depth", Json(max_depth));
|
|
if (!instructions.empty()) body.set("instructions", Json(instructions));
|
|
return parse_crawl(gw_->call("crawl", with_body(body)));
|
|
}
|
|
|
|
std::vector<std::string> Tavily::map(const std::string& url, int limit, int max_depth) {
|
|
check_url(url);
|
|
if (limit < 1 || limit > RetrievalCaps::kMaxCrawlLimit) bad("limit must be 1..500");
|
|
if (max_depth < 1 || max_depth > RetrievalCaps::kMaxCrawlDepth) bad("max_depth must be 1..5");
|
|
Json body = Json::object();
|
|
body.set("url", Json(url));
|
|
body.set("limit", Json(limit));
|
|
body.set("max_depth", Json(max_depth));
|
|
Json doc = gw_->call("map", with_body(body));
|
|
std::vector<std::string> out;
|
|
for (const Json& r : J::array(doc, "results"))
|
|
if (r.is_string()) out.push_back(r.as_string());
|
|
return out;
|
|
}
|
|
|
|
Json Tavily::research(const std::string& input) {
|
|
if (input.empty() || input.size() > RetrievalCaps::kMaxQueryChars) bad("input must be 1..8192 chars");
|
|
Json body = Json::object();
|
|
body.set("input", Json(input));
|
|
return gw_->call("create_research", with_body(body));
|
|
}
|
|
|
|
Json Tavily::get_research(const std::string& request_id) {
|
|
CallArgs a;
|
|
a.path["request_id"] = request_id;
|
|
return gw_->call("get_research", a);
|
|
}
|
|
|
|
Json Tavily::usage() { return gw_->call("get_usage"); }
|
|
|
|
} // namespace molo::retrieval
|