#include "vendors/audio/hume/sockets.h" #include "security/crypto/Base64.h" #include "vendors/audio/caps.h" namespace molo::audio::hume { namespace { [[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("hume", why); } std::string str_at(const Json& o, const char* k) { const Json* v = o.is_object() ? o.find(k) : nullptr; return v && v->is_string() ? v->as_string() : std::string(); } Json typed(const char* type) { Json m = Json::object(); m.set("type", Json(type)); return m; } } // namespace Vocabulary HumeTtsStream::vocabulary() { return Vocabulary("type", {}, {"audio", "timestamp", "error"}); } HumeTtsStream::HumeTtsStream(std::string api_key, HumeStreamOptions o) { Query q; q.add_if("format_type", o.format_type); if (!o.format_type.empty() && o.format_type != "mp3" && o.format_type != "wav" && o.format_type != "pcm") invalid("format_type must be mp3, wav or pcm"); if (!o.context_generation_id.empty()) { if (!llm::Service::valid_id(o.context_generation_id)) invalid("bad generation id"); q.add("context_generation_id", o.context_generation_id); } for (const auto& t : o.include_timestamp_types) { if (t != "word" && t != "phoneme") invalid("timestamp type must be word or phoneme"); q.add("include_timestamp_types", t); } if (o.instant_mode >= 0) q.add_bool("instant_mode", o.instant_mode == 1); if (o.no_binary >= 0) q.add_bool("no_binary", o.no_binary == 1); if (o.strip_headers >= 0) q.add_bool("strip_headers", o.strip_headers == 1); if (!o.version.empty()) { if (o.version != "1" && o.version != "2") invalid("version must be 1 or 2"); q.add("version", o.version); } std::string base = o.url.empty() ? std::string(HumeEndpoints::kTtsWs) + "/stream/input" : o.url; socket_ = std::make_unique("hume", base + q.str(), HumeEndpoints::ws_headers(api_key), vocabulary(), std::move(o.socket)); } void HumeTtsStream::send_text(const std::string& text, const std::string& voice_name, const std::string& description, double speed, double trailing_silence) { if (text.empty() || text.size() > HumeCaps::kMaxUtteranceChars) invalid("text must be 1..5000 characters"); if (text.find('\0') != std::string::npos) invalid("text has a NUL byte"); Json m = Json::object(); m.set("text", Json(text)); if (!voice_name.empty()) { if (voice_name.size() > 128) invalid("voice name is too long"); Json v = Json::object(); v.set("name", Json(voice_name)); m.set("voice", v); } if (!description.empty()) { if (description.size() > HumeCaps::kMaxDescriptionChars) invalid("description is too long"); m.set("description", Json(description)); } if (speed != 0) { if (speed < 0) invalid("speed must be positive"); m.set("speed", Json(speed)); } if (trailing_silence >= 0) m.set("trailing_silence", Json(trailing_silence)); socket_->send_json(m, true); } void HumeTtsStream::flush() { Json m = Json::object(); m.set("flush", Json(true)); socket_->send_json(m, true); } void HumeTtsStream::finish() { Json m = Json::object(); m.set("close", Json(true)); socket_->send_json(m, true); } HumeStreamResult HumeTtsStream::collect(int max_wait_ms) { HumeStreamResult out; for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) { auto ev = socket_->next(max_wait_ms); if (!ev) return out; if (ev->kind == SocketEvent::Kind::kClose) { out.closed = true; return out; } if (ev->kind == SocketEvent::Kind::kBinary) { if (out.audio.size() + ev->bytes.size() > AudioCaps::kMaxStreamAudioBytes) { out.truncated = true; return out; } out.audio += ev->bytes; ++out.chunks; continue; } if (!ev->json_ok || !ev->json.is_object()) continue; if (ev->type == "audio") { std::string raw; if (!security::crypto::Base64::decode(str_at(ev->json, "audio"), raw)) throw llm::ApiError("hume", 502, "BAD_RESPONSE", "audio chunk is not valid base64"); if (out.audio.size() + raw.size() > AudioCaps::kMaxStreamAudioBytes) { out.truncated = true; return out; } out.audio += raw; ++out.chunks; out.generation_id = str_at(ev->json, "generation_id"); if (const Json* l = ev->json.find("is_last_chunk"); l && l->is_bool() && l->as_bool()) { out.last_chunk = true; return out; } } else if (ev->type == "timestamp") { out.timestamps.push_back(ev->json); } else if (ev->type == "error" || ev->json.find("error")) { out.error = str_at(ev->json, "message"); if (out.error.empty()) out.error = str_at(ev->json, "error"); if (out.error.empty()) out.error = "stream error"; return out; } } return out; } Vocabulary EviChat::vocabulary() { return Vocabulary("type", {"audio_input", "session_settings", "user_input", "assistant_input", "tool_response", "tool_error", "pause_assistant_message", "resume_assistant_message"}, {"assistant_end", "assistant_message", "assistant_prosody", "audio_output", "chat_metadata", "error", "user_interruption", "user_message", "tool_call", "tool_response", "tool_error", "session_settings"}); } EviChat::EviChat(std::string api_key, EviOptions o) { Query q; if (!o.config_id.empty()) { if (!llm::Service::valid_id(o.config_id)) invalid("bad config id"); q.add("config_id", o.config_id); } if (o.config_version >= 0) q.add_int("config_version", o.config_version); if (!o.resumed_chat_group_id.empty()) { if (!llm::Service::valid_id(o.resumed_chat_group_id)) invalid("bad chat group id"); q.add("resumed_chat_group_id", o.resumed_chat_group_id); } if (o.verbose_transcription >= 0) q.add_bool("verbose_transcription", o.verbose_transcription == 1); if (o.event_limit < 0 || o.event_limit > 10000) invalid("event_limit must be 0..10000"); if (o.event_limit > 0) q.add_int("event_limit", o.event_limit); std::string base = o.url.empty() ? std::string(HumeEndpoints::kEviWs) + "/chat" : o.url; socket_ = std::make_unique("hume", base + q.str(), HumeEndpoints::ws_headers(api_key), vocabulary(), std::move(o.socket)); } void EviChat::send_session_settings(const Json& settings) { if (!settings.is_object()) invalid("settings must be a JSON object"); Json m = settings; m.set("type", Json("session_settings")); socket_->send_json(m); } void EviChat::send_audio(const std::string& audio) { if (audio.empty() || audio.size() > 1024u * 1024u) invalid("audio chunk must be 1 byte..1 MiB"); Json m = typed("audio_input"); m.set("data", Json(security::crypto::Base64::encode(audio))); socket_->send_json(m); } void EviChat::send_user_text(const std::string& text) { if (text.empty() || text.size() > 20000) invalid("text must be 1..20000 characters"); Json m = typed("user_input"); m.set("text", Json(text)); socket_->send_json(m); } void EviChat::send_assistant_text(const std::string& text) { if (text.empty() || text.size() > 20000) invalid("text must be 1..20000 characters"); Json m = typed("assistant_input"); m.set("text", Json(text)); socket_->send_json(m); } void EviChat::send_tool_response(const std::string& id, const std::string& content) { if (!llm::Service::valid_id(id)) invalid("bad tool_call_id"); if (content.size() > 200000) invalid("content is over 200000 characters"); Json m = typed("tool_response"); m.set("tool_call_id", Json(id)); m.set("content", Json(content)); socket_->send_json(m); } void EviChat::pause_assistant() { socket_->send_json(typed("pause_assistant_message")); } void EviChat::resume_assistant() { socket_->send_json(typed("resume_assistant_message")); } EviTurn EviChat::collect_turn(int max_wait_ms) { EviTurn turn; for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) { auto ev = socket_->next(max_wait_ms); if (!ev) return turn; if (ev->kind == SocketEvent::Kind::kClose) { turn.closed = true; return turn; } if (!ev->json_ok || !ev->json.is_object()) continue; const std::string& t = ev->type; auto content = [&](const char* key) { const Json* m = ev->json.find("message"); return m ? str_at(*m, key) : std::string(); }; if (t == "chat_metadata") { turn.chat_id = str_at(ev->json, "chat_id"); turn.chat_group_id = str_at(ev->json, "chat_group_id"); } else if (t == "user_message") { turn.user_messages.push_back(content("content")); } else if (t == "assistant_message") { turn.assistant_messages.push_back(content("content")); } else if (t == "audio_output") { std::string raw; if (!security::crypto::Base64::decode(str_at(ev->json, "data"), raw)) throw llm::ApiError("hume", 502, "BAD_RESPONSE", "audio_output is not valid base64"); if (turn.audio.size() + raw.size() > AudioCaps::kMaxStreamAudioBytes) { turn.truncated = true; return turn; } turn.audio += raw; ++turn.audio_chunks; } else if (t == "user_interruption") { turn.interrupted = true; } else if (t == "tool_call") { turn.tool_calls.push_back(ev->json); } else if (t == "assistant_end") { turn.assistant_end = true; return turn; } else if (t == "error") { turn.error = str_at(ev->json, "message"); if (turn.error.empty()) turn.error = "chat error"; return turn; } } return turn; } } // namespace molo::audio::hume