feat(clients): audio: ElevenLabs, AssemblyAI, Deepgram, Cartesia, Speechmatics, Hume
Table driven REST (509 rows verified against vendor specs by tools/check_spec.py), WebSockets over the src/google/live client, neutral SpeechSynthesizer / SpeechRecognizer seam, /ai/audio demo with dry runs, 110 package tests and 5 route tests. PlayHT has no public API; LeMUR is gone from AssemblyAI's spec. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Vendored
+252
@@ -0,0 +1,252 @@
|
||||
#include "vendors/audio/hume/sockets.h"
|
||||
|
||||
#include "security/crypto/Base64.h"
|
||||
#include "vendors/audio/caps.h"
|
||||
|
||||
namespace molo::audio::hume {
|
||||
|
||||
namespace {
|
||||
|
||||
[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("hume", why); }
|
||||
|
||||
std::string str_at(const Json& o, const char* k) {
|
||||
const Json* v = o.is_object() ? o.find(k) : nullptr;
|
||||
return v && v->is_string() ? v->as_string() : std::string();
|
||||
}
|
||||
|
||||
Json typed(const char* type) {
|
||||
Json m = Json::object();
|
||||
m.set("type", Json(type));
|
||||
return m;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Vocabulary HumeTtsStream::vocabulary() {
|
||||
return Vocabulary("type", {}, {"audio", "timestamp", "error"});
|
||||
}
|
||||
|
||||
HumeTtsStream::HumeTtsStream(std::string api_key, HumeStreamOptions o) {
|
||||
Query q;
|
||||
q.add_if("format_type", o.format_type);
|
||||
if (!o.format_type.empty() && o.format_type != "mp3" && o.format_type != "wav" && o.format_type != "pcm")
|
||||
invalid("format_type must be mp3, wav or pcm");
|
||||
if (!o.context_generation_id.empty()) {
|
||||
if (!llm::Service::valid_id(o.context_generation_id)) invalid("bad generation id");
|
||||
q.add("context_generation_id", o.context_generation_id);
|
||||
}
|
||||
for (const auto& t : o.include_timestamp_types) {
|
||||
if (t != "word" && t != "phoneme") invalid("timestamp type must be word or phoneme");
|
||||
q.add("include_timestamp_types", t);
|
||||
}
|
||||
if (o.instant_mode >= 0) q.add_bool("instant_mode", o.instant_mode == 1);
|
||||
if (o.no_binary >= 0) q.add_bool("no_binary", o.no_binary == 1);
|
||||
if (o.strip_headers >= 0) q.add_bool("strip_headers", o.strip_headers == 1);
|
||||
if (!o.version.empty()) {
|
||||
if (o.version != "1" && o.version != "2") invalid("version must be 1 or 2");
|
||||
q.add("version", o.version);
|
||||
}
|
||||
std::string base = o.url.empty() ? std::string(HumeEndpoints::kTtsWs) + "/stream/input" : o.url;
|
||||
socket_ = std::make_unique<AudioSocket>("hume", base + q.str(), HumeEndpoints::ws_headers(api_key), vocabulary(),
|
||||
std::move(o.socket));
|
||||
}
|
||||
|
||||
void HumeTtsStream::send_text(const std::string& text, const std::string& voice_name, const std::string& description,
|
||||
double speed, double trailing_silence) {
|
||||
if (text.empty() || text.size() > HumeCaps::kMaxUtteranceChars) invalid("text must be 1..5000 characters");
|
||||
if (text.find('\0') != std::string::npos) invalid("text has a NUL byte");
|
||||
Json m = Json::object();
|
||||
m.set("text", Json(text));
|
||||
if (!voice_name.empty()) {
|
||||
if (voice_name.size() > 128) invalid("voice name is too long");
|
||||
Json v = Json::object();
|
||||
v.set("name", Json(voice_name));
|
||||
m.set("voice", v);
|
||||
}
|
||||
if (!description.empty()) {
|
||||
if (description.size() > HumeCaps::kMaxDescriptionChars) invalid("description is too long");
|
||||
m.set("description", Json(description));
|
||||
}
|
||||
if (speed != 0) {
|
||||
if (speed < 0) invalid("speed must be positive");
|
||||
m.set("speed", Json(speed));
|
||||
}
|
||||
if (trailing_silence >= 0) m.set("trailing_silence", Json(trailing_silence));
|
||||
socket_->send_json(m, true);
|
||||
}
|
||||
|
||||
void HumeTtsStream::flush() {
|
||||
Json m = Json::object();
|
||||
m.set("flush", Json(true));
|
||||
socket_->send_json(m, true);
|
||||
}
|
||||
|
||||
void HumeTtsStream::finish() {
|
||||
Json m = Json::object();
|
||||
m.set("close", Json(true));
|
||||
socket_->send_json(m, true);
|
||||
}
|
||||
|
||||
HumeStreamResult HumeTtsStream::collect(int max_wait_ms) {
|
||||
HumeStreamResult out;
|
||||
for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) {
|
||||
auto ev = socket_->next(max_wait_ms);
|
||||
if (!ev) return out;
|
||||
if (ev->kind == SocketEvent::Kind::kClose) {
|
||||
out.closed = true;
|
||||
return out;
|
||||
}
|
||||
if (ev->kind == SocketEvent::Kind::kBinary) {
|
||||
if (out.audio.size() + ev->bytes.size() > AudioCaps::kMaxStreamAudioBytes) {
|
||||
out.truncated = true;
|
||||
return out;
|
||||
}
|
||||
out.audio += ev->bytes;
|
||||
++out.chunks;
|
||||
continue;
|
||||
}
|
||||
if (!ev->json_ok || !ev->json.is_object()) continue;
|
||||
if (ev->type == "audio") {
|
||||
std::string raw;
|
||||
if (!security::crypto::Base64::decode(str_at(ev->json, "audio"), raw))
|
||||
throw llm::ApiError("hume", 502, "BAD_RESPONSE", "audio chunk is not valid base64");
|
||||
if (out.audio.size() + raw.size() > AudioCaps::kMaxStreamAudioBytes) {
|
||||
out.truncated = true;
|
||||
return out;
|
||||
}
|
||||
out.audio += raw;
|
||||
++out.chunks;
|
||||
out.generation_id = str_at(ev->json, "generation_id");
|
||||
if (const Json* l = ev->json.find("is_last_chunk"); l && l->is_bool() && l->as_bool()) {
|
||||
out.last_chunk = true;
|
||||
return out;
|
||||
}
|
||||
} else if (ev->type == "timestamp") {
|
||||
out.timestamps.push_back(ev->json);
|
||||
} else if (ev->type == "error" || ev->json.find("error")) {
|
||||
out.error = str_at(ev->json, "message");
|
||||
if (out.error.empty()) out.error = str_at(ev->json, "error");
|
||||
if (out.error.empty()) out.error = "stream error";
|
||||
return out;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
Vocabulary EviChat::vocabulary() {
|
||||
return Vocabulary("type",
|
||||
{"audio_input", "session_settings", "user_input", "assistant_input", "tool_response",
|
||||
"tool_error", "pause_assistant_message", "resume_assistant_message"},
|
||||
{"assistant_end", "assistant_message", "assistant_prosody", "audio_output", "chat_metadata",
|
||||
"error", "user_interruption", "user_message", "tool_call", "tool_response", "tool_error",
|
||||
"session_settings"});
|
||||
}
|
||||
|
||||
EviChat::EviChat(std::string api_key, EviOptions o) {
|
||||
Query q;
|
||||
if (!o.config_id.empty()) {
|
||||
if (!llm::Service::valid_id(o.config_id)) invalid("bad config id");
|
||||
q.add("config_id", o.config_id);
|
||||
}
|
||||
if (o.config_version >= 0) q.add_int("config_version", o.config_version);
|
||||
if (!o.resumed_chat_group_id.empty()) {
|
||||
if (!llm::Service::valid_id(o.resumed_chat_group_id)) invalid("bad chat group id");
|
||||
q.add("resumed_chat_group_id", o.resumed_chat_group_id);
|
||||
}
|
||||
if (o.verbose_transcription >= 0) q.add_bool("verbose_transcription", o.verbose_transcription == 1);
|
||||
if (o.event_limit < 0 || o.event_limit > 10000) invalid("event_limit must be 0..10000");
|
||||
if (o.event_limit > 0) q.add_int("event_limit", o.event_limit);
|
||||
std::string base = o.url.empty() ? std::string(HumeEndpoints::kEviWs) + "/chat" : o.url;
|
||||
socket_ = std::make_unique<AudioSocket>("hume", base + q.str(), HumeEndpoints::ws_headers(api_key), vocabulary(),
|
||||
std::move(o.socket));
|
||||
}
|
||||
|
||||
void EviChat::send_session_settings(const Json& settings) {
|
||||
if (!settings.is_object()) invalid("settings must be a JSON object");
|
||||
Json m = settings;
|
||||
m.set("type", Json("session_settings"));
|
||||
socket_->send_json(m);
|
||||
}
|
||||
|
||||
void EviChat::send_audio(const std::string& audio) {
|
||||
if (audio.empty() || audio.size() > 1024u * 1024u) invalid("audio chunk must be 1 byte..1 MiB");
|
||||
Json m = typed("audio_input");
|
||||
m.set("data", Json(security::crypto::Base64::encode(audio)));
|
||||
socket_->send_json(m);
|
||||
}
|
||||
|
||||
void EviChat::send_user_text(const std::string& text) {
|
||||
if (text.empty() || text.size() > 20000) invalid("text must be 1..20000 characters");
|
||||
Json m = typed("user_input");
|
||||
m.set("text", Json(text));
|
||||
socket_->send_json(m);
|
||||
}
|
||||
|
||||
void EviChat::send_assistant_text(const std::string& text) {
|
||||
if (text.empty() || text.size() > 20000) invalid("text must be 1..20000 characters");
|
||||
Json m = typed("assistant_input");
|
||||
m.set("text", Json(text));
|
||||
socket_->send_json(m);
|
||||
}
|
||||
|
||||
void EviChat::send_tool_response(const std::string& id, const std::string& content) {
|
||||
if (!llm::Service::valid_id(id)) invalid("bad tool_call_id");
|
||||
if (content.size() > 200000) invalid("content is over 200000 characters");
|
||||
Json m = typed("tool_response");
|
||||
m.set("tool_call_id", Json(id));
|
||||
m.set("content", Json(content));
|
||||
socket_->send_json(m);
|
||||
}
|
||||
|
||||
void EviChat::pause_assistant() { socket_->send_json(typed("pause_assistant_message")); }
|
||||
void EviChat::resume_assistant() { socket_->send_json(typed("resume_assistant_message")); }
|
||||
|
||||
EviTurn EviChat::collect_turn(int max_wait_ms) {
|
||||
EviTurn turn;
|
||||
for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) {
|
||||
auto ev = socket_->next(max_wait_ms);
|
||||
if (!ev) return turn;
|
||||
if (ev->kind == SocketEvent::Kind::kClose) {
|
||||
turn.closed = true;
|
||||
return turn;
|
||||
}
|
||||
if (!ev->json_ok || !ev->json.is_object()) continue;
|
||||
const std::string& t = ev->type;
|
||||
auto content = [&](const char* key) {
|
||||
const Json* m = ev->json.find("message");
|
||||
return m ? str_at(*m, key) : std::string();
|
||||
};
|
||||
if (t == "chat_metadata") {
|
||||
turn.chat_id = str_at(ev->json, "chat_id");
|
||||
turn.chat_group_id = str_at(ev->json, "chat_group_id");
|
||||
} else if (t == "user_message") {
|
||||
turn.user_messages.push_back(content("content"));
|
||||
} else if (t == "assistant_message") {
|
||||
turn.assistant_messages.push_back(content("content"));
|
||||
} else if (t == "audio_output") {
|
||||
std::string raw;
|
||||
if (!security::crypto::Base64::decode(str_at(ev->json, "data"), raw))
|
||||
throw llm::ApiError("hume", 502, "BAD_RESPONSE", "audio_output is not valid base64");
|
||||
if (turn.audio.size() + raw.size() > AudioCaps::kMaxStreamAudioBytes) {
|
||||
turn.truncated = true;
|
||||
return turn;
|
||||
}
|
||||
turn.audio += raw;
|
||||
++turn.audio_chunks;
|
||||
} else if (t == "user_interruption") {
|
||||
turn.interrupted = true;
|
||||
} else if (t == "tool_call") {
|
||||
turn.tool_calls.push_back(ev->json);
|
||||
} else if (t == "assistant_end") {
|
||||
turn.assistant_end = true;
|
||||
return turn;
|
||||
} else if (t == "error") {
|
||||
turn.error = str_at(ev->json, "message");
|
||||
if (turn.error.empty()) turn.error = "chat error";
|
||||
return turn;
|
||||
}
|
||||
}
|
||||
return turn;
|
||||
}
|
||||
|
||||
} // namespace molo::audio::hume
|
||||
Reference in New Issue
Block a user