Files
moloapp/src/vendors/cohere/inference.cpp
T

274 lines
11 KiB
C++
Raw Normal View History

#include "vendors/cohere/inference.h"
#include "vendors/cohere/calls.h"
#include "vendors/cohere/caps.h"
#include "vendors/cohere/rules.h"
namespace molo::cohere {
namespace {
Json string_array(const std::vector<std::string>& v) {
Json a = Json::array();
for (const std::string& s : v) a.push(Json(s));
return a;
}
} // namespace
void EmbedParams::validate() const {
Rules::model(model);
Rules::one_of(input_type, {"search_document", "search_query", "classification", "clustering", "image"}, "input_type");
Rules::require(!texts.empty() || !images.empty() || !inputs.empty(), "give texts, images or inputs");
Rules::require(texts.size() <= CohereCaps::kMaxEmbedTexts, "at most 96 texts per call");
Rules::require(images.size() <= CohereCaps::kMaxEmbedTexts, "at most 96 images per call");
Rules::require(inputs.size() <= CohereCaps::kMaxEmbedInputs, "at most 96 inputs per call");
for (const std::string& t : texts) Rules::require(!t.empty(), "texts must not be empty");
for (const std::string& i : images) Rules::image_url(i);
for (const std::string& t : embedding_types)
Rules::one_of(t, {"float", "int8", "uint8", "binary", "ubinary", "base64"}, "embedding type");
if (!truncate.empty()) Rules::one_of(truncate, {"NONE", "START", "END"}, "truncate");
if (max_tokens) Rules::require(*max_tokens >= 1, "max_tokens must be >= 1");
if (output_dimension) Rules::require(*output_dimension >= 1, "output_dimension must be >= 1");
if (priority) Rules::priority(*priority);
}
Json EmbedParams::to_json() const {
validate();
Json b = Json::object();
b.set("model", Json(model));
b.set("input_type", Json(input_type));
if (!texts.empty()) b.set("texts", string_array(texts));
if (!images.empty()) b.set("images", string_array(images));
if (!inputs.empty()) {
Json a = Json::array();
for (const Json& i : inputs) a.push(i);
b.set("inputs", a);
}
if (!embedding_types.empty()) b.set("embedding_types", string_array(embedding_types));
if (!truncate.empty()) b.set("truncate", Json(truncate));
if (max_tokens) b.set("max_tokens", Json(*max_tokens));
if (output_dimension) b.set("output_dimension", Json(*output_dimension));
if (priority) b.set("priority", Json(*priority));
return b;
}
std::vector<std::vector<double>> EmbedResult::floats() const {
std::vector<std::vector<double>> out;
const Json* rows = nullptr;
if (embeddings.is_array()) rows = &embeddings;
else if (embeddings.is_object() && embeddings.find("float") && embeddings.find("float")->is_array())
rows = embeddings.find("float");
if (!rows) return out;
for (const Json& row : rows->as_array()) {
std::vector<double> v;
if (row.is_array())
for (const Json& x : row.as_array())
if (x.is_number()) v.push_back(x.as_number());
out.push_back(std::move(v));
}
return out;
}
EmbedResult EmbedResult::parse(const Json& doc) {
EmbedResult r;
r.raw = doc;
r.id = J::str(doc, "id");
r.response_type = J::str(doc, "response_type");
r.texts = J::strings(doc, "texts");
if (const Json* e = doc.is_object() ? doc.find("embeddings") : nullptr) r.embeddings = *e;
r.usage = TokenUsage::parse(J::obj(doc, "meta"));
if (const Json* meta = J::obj(doc, "meta"))
if (const Json* b = J::obj(*meta, "billed_units")) {
r.usage.billed_input_tokens = J::num(*b, "input_tokens");
r.usage.present = true;
}
return r;
}
void RerankParams::validate() const {
Rules::model(model);
Rules::require(!query.empty(), "query must not be empty");
Rules::require(!documents.empty() && documents.size() <= CohereCaps::kMaxDocuments, "documents must be 1..1000");
for (const std::string& d : documents) Rules::require(!d.empty(), "documents must not be empty");
if (top_n) Rules::require(*top_n >= 1, "top_n must be >= 1");
if (max_tokens_per_doc) Rules::require(*max_tokens_per_doc >= 1, "max_tokens_per_doc must be >= 1");
if (!relevant_spans.empty()) Rules::one_of(relevant_spans, {"enabled", "disabled"}, "relevant_spans");
Rules::require(relevant_spans.empty() || relevant_spans == "enabled" || (!spans_threshold && !max_spans),
"span options need relevant_spans enabled");
if (spans_threshold) Rules::range(*spans_threshold, 0.0, 1.0, "span threshold");
if (max_spans) Rules::require(*max_spans >= 1, "max_spans must be >= 1");
if (priority) Rules::priority(*priority);
}
Json RerankParams::to_json() const {
validate();
Json b = Json::object();
b.set("model", Json(model));
b.set("query", Json(query));
b.set("documents", string_array(documents));
if (top_n) b.set("top_n", Json(*top_n));
if (max_tokens_per_doc) b.set("max_tokens_per_doc", Json(*max_tokens_per_doc));
if (!relevant_spans.empty()) {
Json s = Json::object();
s.set("mode", Json(relevant_spans));
if (spans_threshold) s.set("threshold", Json(*spans_threshold));
if (max_spans) s.set("max_spans", Json(*max_spans));
b.set("relevant_spans", s);
}
if (priority) b.set("priority", Json(*priority));
return b;
}
RerankResult RerankResult::parse(const Json& doc) {
RerankResult r;
r.raw = doc;
r.id = J::str(doc, "id");
if (const Json* a = J::arr(doc, "results"))
for (const Json& x : a->as_array()) {
RerankHit h;
h.index = static_cast<int>(J::num(x, "index"));
h.score = J::real(x, "relevance_score");
if (const Json* s = J::arr(x, "relevant_spans")) h.spans = *s;
r.hits.push_back(h);
}
r.usage = TokenUsage::parse(J::obj(doc, "meta"));
return r;
}
void ClassifyParams::validate() const {
Rules::require(!inputs.empty() && inputs.size() <= CohereCaps::kMaxClassifyInputs, "inputs must be 1..96");
for (const std::string& i : inputs) Rules::require(!i.empty(), "inputs must not be empty");
if (!model.empty()) Rules::model(model);
Rules::require(!model.empty() || examples.size() >= 2, "without a fine-tuned model, give at least 2 examples");
for (const ClassifyExample& e : examples)
Rules::require(!e.text.empty() && !e.label.empty(), "examples need text and label");
if (!truncate.empty()) Rules::one_of(truncate, {"NONE", "START", "END"}, "truncate");
}
Json ClassifyParams::to_json() const {
validate();
Json b = Json::object();
b.set("inputs", string_array(inputs));
if (!examples.empty()) {
Json a = Json::array();
for (const ClassifyExample& e : examples) {
Json x = Json::object();
x.set("text", Json(e.text));
x.set("label", Json(e.label));
a.push(x);
}
b.set("examples", a);
}
if (!model.empty()) b.set("model", Json(model));
if (!truncate.empty()) b.set("truncate", Json(truncate));
return b;
}
ClassifyResult ClassifyResult::parse(const Json& doc) {
ClassifyResult r;
r.raw = doc;
r.id = J::str(doc, "id");
if (const Json* a = J::arr(doc, "classifications"))
for (const Json& x : a->as_array()) {
Classification c;
c.id = J::str(x, "id");
c.input = J::str(x, "input");
c.prediction = J::str(x, "prediction");
if (c.prediction.empty()) {
std::vector<std::string> p = J::strings(x, "predictions");
if (!p.empty()) c.prediction = p.front();
}
c.confidence = J::real(x, "confidence");
if (c.confidence == 0)
if (const Json* cs = J::arr(x, "confidences"))
if (!cs->as_array().empty() && cs->as_array().front().is_number()) c.confidence = cs->as_array().front().as_number();
c.classification_type = J::str(x, "classification_type");
if (const Json* l = J::obj(x, "labels")) c.labels = *l;
r.items.push_back(c);
}
return r;
}
TokenizeResult TokenizeResult::parse(const Json& doc) {
TokenizeResult r;
if (const Json* a = J::arr(doc, "tokens"))
for (const Json& t : a->as_array())
if (t.is_number()) r.tokens.push_back(t.as_int());
r.token_strings = J::strings(doc, "token_strings");
return r;
}
EmbedResult Embeddings::embed(const EmbedParams& params) {
Json body = params.to_json();
Calls::check_body(Call::kEmbed, body);
return EmbedResult::parse(client_->post(Calls::render(Call::kEmbed), body));
}
EmbedResult Embeddings::embed_v1(const EmbedParams& params) {
Rules::require(params.inputs.empty() && !params.output_dimension && !params.max_tokens && !params.priority,
"v1 embed takes texts and images only");
Json body = params.to_json();
Calls::check_body(Call::kEmbedV1, body);
return EmbedResult::parse(client_->post(Calls::render(Call::kEmbedV1), body));
}
RerankResult Rerank::rerank(const RerankParams& params) {
Json body = params.to_json();
Calls::check_body(Call::kRerank, body);
return RerankResult::parse(client_->post(Calls::render(Call::kRerank), body));
}
Json Rerank::rerank_v1(const Json& body) {
Calls::check_body(Call::kRerankV1, body);
Rules::require(body.find("query") && body.find("query")->is_string(), "query is required");
const Json* docs = body.find("documents");
Rules::require(docs && docs->is_array() && !docs->as_array().empty() &&
docs->as_array().size() <= CohereCaps::kMaxDocuments,
"documents must be 1..1000");
return client_->post(Calls::render(Call::kRerankV1), body);
}
ClassifyResult Classify::classify(const ClassifyParams& params) {
Json body = params.to_json();
Calls::check_body(Call::kClassify, body);
return ClassifyResult::parse(client_->post(Calls::render(Call::kClassify), body));
}
TokenizeResult Tokenizer::tokenize(const std::string& model, const std::string& text) {
Rules::model(model);
Rules::require(!text.empty() && text.size() <= CohereCaps::kMaxTokenizeChars, "text must be 1..65536 chars");
Json b = Json::object();
b.set("model", Json(model));
b.set("text", Json(text));
return TokenizeResult::parse(client_->post(Calls::render(Call::kTokenize), b));
}
std::string Tokenizer::detokenize(const std::string& model, const std::vector<long>& tokens) {
Rules::model(model);
Rules::require(!tokens.empty() && tokens.size() <= CohereCaps::kMaxDetokenizeTokens, "tokens must be 1..65536");
Json arr = Json::array();
for (long t : tokens) {
Rules::require(t >= 0, "token ids must be >= 0");
arr.push(Json(t));
}
Json b = Json::object();
b.set("model", Json(model));
b.set("tokens", arr);
return J::str(client_->post(Calls::render(Call::kDetokenize), b), "text");
}
Json Legacy::generate(const Json& body) {
Calls::check_body(Call::kGenerate, body);
Rules::require(body.find("prompt") && body.find("prompt")->is_string(), "prompt is required");
const Json* s = body.find("stream");
Rules::require(!s || (s->is_bool() && !s->as_bool()), "generate streaming is not supported here");
return client_->post(Calls::render(Call::kGenerate), body);
}
Json Legacy::summarize(const Json& body) {
Calls::check_body(Call::kSummarize, body);
Rules::require(body.find("text") && body.find("text")->is_string(), "text is required");
return client_->post(Calls::render(Call::kSummarize), body);
}
} // namespace molo::cohere