169 lines
6.8 KiB
C++
169 lines
6.8 KiB
C++
#include "vendors/mistral/audio.h"
|
|||
|
|
|
||
|
|
#include "vendors/mistral/calls.h"
|
||
|
|
#include "vendors/mistral/caps.h"
|
||
|
|
#include "vendors/mistral/ops.h"
|
||
|
|
#include "vendors/mistral/rules.h"
|
||
|
|
|
||
|
|
namespace molo::mistral {
|
||
|
|
|
||
|
|
void TranscriptionParams::validate() const {
|
||
|
|
Rules::model(model);
|
||
|
|
int sources = (!file_data.empty()) + (!file_url.empty()) + (!file_id.empty());
|
||
|
|
Rules::require(sources == 1, "give exactly one of file bytes, file_url or file_id");
|
||
|
|
if (!file_data.empty()) {
|
||
|
|
Rules::filename(file_name);
|
||
|
|
Rules::require(file_data.size() <= MistralCaps::kMaxAudioBytes, "audio over 128 MiB");
|
||
|
|
}
|
||
|
|
if (!file_url.empty()) Rules::http_url(file_url, MistralCaps::kTranscriptionFileUrlChars, "file_url");
|
||
|
|
if (!file_id.empty()) Rules::require(llm::Service::valid_id(file_id), "bad file id");
|
||
|
|
if (!language.empty()) Rules::word(language, "language");
|
||
|
|
if (temperature) Rules::range(*temperature, 0.0, 2.0, "temperature");
|
||
|
|
Rules::require(context_bias.size() <= 100, "at most 100 context_bias entries");
|
||
|
|
for (const std::string& w : context_bias) Rules::require(!w.empty() && w.size() <= 128, "context_bias entry must be 1..128 chars");
|
||
|
|
Rules::require(timestamp_granularities.size() <= 2, "at most 2 timestamp granularities");
|
||
|
|
for (const std::string& g : timestamp_granularities) Rules::one_of(g, {"segment", "word"}, "timestamp granularity");
|
||
|
|
}
|
||
|
|
|
||
|
|
TranscriptionResult TranscriptionResult::parse(const Json& doc) {
|
||
|
|
TranscriptionResult r;
|
||
|
|
r.raw = doc;
|
||
|
|
r.model = J::str(doc, "model");
|
||
|
|
r.text = J::str(doc, "text");
|
||
|
|
r.language = J::str(doc, "language");
|
||
|
|
r.usage = TokenUsage::parse(J::obj(doc, "usage"));
|
||
|
|
if (const Json* s = J::arr(doc, "segments")) {
|
||
|
|
for (const Json& x : s->as_array()) {
|
||
|
|
TranscriptionSegment seg;
|
||
|
|
seg.text = J::str(x, "text");
|
||
|
|
seg.speaker_id = J::str(x, "speaker_id");
|
||
|
|
seg.start = J::real(x, "start");
|
||
|
|
seg.end = J::real(x, "end");
|
||
|
|
r.segments.push_back(std::move(seg));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return r;
|
||
|
|
}
|
||
|
|
|
||
|
|
void SpeechParams::validate() const {
|
||
|
|
if (!model.empty()) Rules::model(model);
|
||
|
|
Rules::require(!input.empty() && input.size() <= 32768, "input must be 1..32768 characters");
|
||
|
|
Rules::require(voice_id.empty() || ref_audio.empty(), "give voice_id or ref_audio, not both");
|
||
|
|
if (!voice_id.empty()) Rules::require(llm::Service::valid_id(voice_id), "bad voice id");
|
||
|
|
Rules::require(ref_audio.size() <= (16u << 20), "ref_audio over 16 MiB");
|
||
|
|
if (!response_format.empty()) Rules::one_of(response_format, {"pcm", "wav", "mp3", "flac", "opus"}, "response_format");
|
||
|
|
}
|
||
|
|
|
||
|
|
Json SpeechParams::to_json() const {
|
||
|
|
validate();
|
||
|
|
Json o = Json::object();
|
||
|
|
o.set("input", Json(input));
|
||
|
|
if (!model.empty()) o.set("model", Json(model));
|
||
|
|
if (!voice_id.empty()) o.set("voice_id", Json(voice_id));
|
||
|
|
if (!ref_audio.empty()) o.set("ref_audio", Json(ref_audio));
|
||
|
|
if (!response_format.empty()) o.set("response_format", Json(response_format));
|
||
|
|
return o;
|
||
|
|
}
|
||
|
|
|
||
|
|
void VoiceCreateParams::validate() const {
|
||
|
|
Rules::require(!name.empty() && name.size() <= 128, "name must be 1..128 characters");
|
||
|
|
if (!slug.empty()) Rules::word(slug, "slug");
|
||
|
|
Rules::require(languages.size() <= 16 && tags.size() <= 32, "too many languages or tags");
|
||
|
|
if (age) Rules::between(*age, 0, 150, "age");
|
||
|
|
Rules::require(sample_audio.size() <= (16u << 20), "sample_audio over 16 MiB");
|
||
|
|
Rules::require(sample_audio.empty() || !sample_filename.empty(), "sample_audio needs sample_filename");
|
||
|
|
if (!sample_filename.empty()) Rules::filename(sample_filename);
|
||
|
|
}
|
||
|
|
|
||
|
|
Json VoiceCreateParams::to_json() const {
|
||
|
|
validate();
|
||
|
|
Json o = Json::object();
|
||
|
|
o.set("name", Json(name));
|
||
|
|
if (!slug.empty()) o.set("slug", Json(slug));
|
||
|
|
if (!gender.empty()) o.set("gender", Json(gender));
|
||
|
|
if (!description.empty()) o.set("description", Json(description));
|
||
|
|
if (age) o.set("age", Json(*age));
|
||
|
|
if (!languages.empty()) {
|
||
|
|
Json a = Json::array();
|
||
|
|
for (const std::string& l : languages) a.push(Json(l));
|
||
|
|
o.set("languages", a);
|
||
|
|
}
|
||
|
|
if (!tags.empty()) {
|
||
|
|
Json a = Json::array();
|
||
|
|
for (const std::string& t : tags) a.push(Json(t));
|
||
|
|
o.set("tags", a);
|
||
|
|
}
|
||
|
|
if (!sample_audio.empty()) {
|
||
|
|
o.set("sample_audio", Json(sample_audio));
|
||
|
|
o.set("sample_filename", Json(sample_filename));
|
||
|
|
}
|
||
|
|
return o;
|
||
|
|
}
|
||
|
|
|
||
|
|
void VoiceListParams::validate() const {
|
||
|
|
Rules::between(page_size, 1, MistralCaps::kPageSizeMax, "page_size");
|
||
|
|
if (!page_token.empty()) Rules::word(page_token, "page_token");
|
||
|
|
Rules::require(query.size() <= 256, "query over 256 characters");
|
||
|
|
}
|
||
|
|
|
||
|
|
TranscriptionResult Audio::transcribe(const TranscriptionParams& p) {
|
||
|
|
p.validate();
|
||
|
|
clients::body::MultipartBody form;
|
||
|
|
form.add_text("model", p.model);
|
||
|
|
if (!p.file_data.empty()) form.add_file("file", p.file_name, "application/octet-stream", p.file_data);
|
||
|
|
if (!p.file_url.empty()) form.add_text("file_url", p.file_url);
|
||
|
|
if (!p.file_id.empty()) form.add_text("file_id", p.file_id);
|
||
|
|
if (!p.language.empty()) form.add_text("language", p.language);
|
||
|
|
if (p.temperature) form.add_text("temperature", std::to_string(*p.temperature));
|
||
|
|
if (p.diarize) form.add_text("diarize", "true");
|
||
|
|
for (const std::string& w : p.context_bias) form.add_text("context_bias", w);
|
||
|
|
for (const std::string& g : p.timestamp_granularities) form.add_text("timestamp_granularities", g);
|
||
|
|
return TranscriptionResult::parse(call_multipart(Op::kTranscribe, form));
|
||
|
|
}
|
||
|
|
|
||
|
|
Json Audio::speech(const SpeechParams& params) {
|
||
|
|
Json body = params.to_json();
|
||
|
|
return call(Op::kSpeech, {}, &body);
|
||
|
|
}
|
||
|
|
|
||
|
|
Json Audio::voices(const VoiceListParams& p) {
|
||
|
|
p.validate();
|
||
|
|
Query q;
|
||
|
|
q.add("page_size", p.page_size).add_if("page_token", p.page_token).add_if("type", p.type);
|
||
|
|
q.add_if("gender", p.gender).add_if("language", p.language).add_if("query", p.query);
|
||
|
|
return call(Op::kVoicesList, {}, nullptr, q.str());
|
||
|
|
}
|
||
|
|
|
||
|
|
Json Audio::voice(const std::string& voice_id) {
|
||
|
|
check_id("voice", voice_id);
|
||
|
|
return call(Op::kVoiceGet, {voice_id});
|
||
|
|
}
|
||
|
|
|
||
|
|
Json Audio::create_voice(const VoiceCreateParams& params) {
|
||
|
|
Json body = params.to_json();
|
||
|
|
return call(Op::kVoiceCreate, {}, &body);
|
||
|
|
}
|
||
|
|
|
||
|
|
Json Audio::update_voice(const std::string& voice_id, const Json& fields) {
|
||
|
|
check_id("voice", voice_id);
|
||
|
|
Rules::json_object(fields, "voice fields");
|
||
|
|
for (const auto& [k, v] : fields.as_object())
|
||
|
|
Rules::require(Calls::has_body_field(Calls::find(Op::kVoiceUpdate), k), "unknown voice field: " + k.substr(0, 40));
|
||
|
|
return call(Op::kVoiceUpdate, {voice_id}, &fields);
|
||
|
|
}
|
||
|
|
|
||
|
|
Json Audio::remove_voice(const std::string& voice_id) {
|
||
|
|
check_id("voice", voice_id);
|
||
|
|
return call(Op::kVoiceDelete, {voice_id});
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string Audio::voice_sample(const std::string& voice_id) {
|
||
|
|
check_id("voice", voice_id);
|
||
|
|
llm::RawResponse r = client_->request_raw(clients::Method::kGet, Calls::render(Op::kVoiceSample, {voice_id}));
|
||
|
|
if (r.body.size() > 64u * 1024u * 1024u)
|
||
|
|
throw llm::ApiError("mistral", 502, "RESPONSE_TOO_LARGE", "voice sample over 64 MiB");
|
||
|
|
return r.body;
|
||
|
|
}
|
||
|
|
|
||
|
|
} // namespace molo::mistral
|