feat(clients): audio: ElevenLabs, AssemblyAI, Deepgram, Cartesia, Speechmatics, Hume

Table driven REST (509 rows verified against vendor specs by tools/check_spec.py),
WebSockets over the src/google/live client, neutral SpeechSynthesizer /
SpeechRecognizer seam, /ai/audio demo with dry runs, 110 package tests and 5
route tests. PlayHT has no public API; LeMUR is gone from AssemblyAI's spec.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-25 15:36:57 +01:00
co-authored by Claude Sonnet 5.5
parent 3db67fcf1f
commit 6d80dd0223
132 changed files with 14060 additions and 3 deletions
+252
View File
@@ -0,0 +1,252 @@
#include "vendors/audio/hume/sockets.h"
#include "security/crypto/Base64.h"
#include "vendors/audio/caps.h"
namespace molo::audio::hume {
namespace {
[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("hume", why); }
std::string str_at(const Json& o, const char* k) {
const Json* v = o.is_object() ? o.find(k) : nullptr;
return v && v->is_string() ? v->as_string() : std::string();
}
Json typed(const char* type) {
Json m = Json::object();
m.set("type", Json(type));
return m;
}
} // namespace
Vocabulary HumeTtsStream::vocabulary() {
return Vocabulary("type", {}, {"audio", "timestamp", "error"});
}
HumeTtsStream::HumeTtsStream(std::string api_key, HumeStreamOptions o) {
Query q;
q.add_if("format_type", o.format_type);
if (!o.format_type.empty() && o.format_type != "mp3" && o.format_type != "wav" && o.format_type != "pcm")
invalid("format_type must be mp3, wav or pcm");
if (!o.context_generation_id.empty()) {
if (!llm::Service::valid_id(o.context_generation_id)) invalid("bad generation id");
q.add("context_generation_id", o.context_generation_id);
}
for (const auto& t : o.include_timestamp_types) {
if (t != "word" && t != "phoneme") invalid("timestamp type must be word or phoneme");
q.add("include_timestamp_types", t);
}
if (o.instant_mode >= 0) q.add_bool("instant_mode", o.instant_mode == 1);
if (o.no_binary >= 0) q.add_bool("no_binary", o.no_binary == 1);
if (o.strip_headers >= 0) q.add_bool("strip_headers", o.strip_headers == 1);
if (!o.version.empty()) {
if (o.version != "1" && o.version != "2") invalid("version must be 1 or 2");
q.add("version", o.version);
}
std::string base = o.url.empty() ? std::string(HumeEndpoints::kTtsWs) + "/stream/input" : o.url;
socket_ = std::make_unique<AudioSocket>("hume", base + q.str(), HumeEndpoints::ws_headers(api_key), vocabulary(),
std::move(o.socket));
}
void HumeTtsStream::send_text(const std::string& text, const std::string& voice_name, const std::string& description,
double speed, double trailing_silence) {
if (text.empty() || text.size() > HumeCaps::kMaxUtteranceChars) invalid("text must be 1..5000 characters");
if (text.find('\0') != std::string::npos) invalid("text has a NUL byte");
Json m = Json::object();
m.set("text", Json(text));
if (!voice_name.empty()) {
if (voice_name.size() > 128) invalid("voice name is too long");
Json v = Json::object();
v.set("name", Json(voice_name));
m.set("voice", v);
}
if (!description.empty()) {
if (description.size() > HumeCaps::kMaxDescriptionChars) invalid("description is too long");
m.set("description", Json(description));
}
if (speed != 0) {
if (speed < 0) invalid("speed must be positive");
m.set("speed", Json(speed));
}
if (trailing_silence >= 0) m.set("trailing_silence", Json(trailing_silence));
socket_->send_json(m, true);
}
void HumeTtsStream::flush() {
Json m = Json::object();
m.set("flush", Json(true));
socket_->send_json(m, true);
}
void HumeTtsStream::finish() {
Json m = Json::object();
m.set("close", Json(true));
socket_->send_json(m, true);
}
HumeStreamResult HumeTtsStream::collect(int max_wait_ms) {
HumeStreamResult out;
for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) {
auto ev = socket_->next(max_wait_ms);
if (!ev) return out;
if (ev->kind == SocketEvent::Kind::kClose) {
out.closed = true;
return out;
}
if (ev->kind == SocketEvent::Kind::kBinary) {
if (out.audio.size() + ev->bytes.size() > AudioCaps::kMaxStreamAudioBytes) {
out.truncated = true;
return out;
}
out.audio += ev->bytes;
++out.chunks;
continue;
}
if (!ev->json_ok || !ev->json.is_object()) continue;
if (ev->type == "audio") {
std::string raw;
if (!security::crypto::Base64::decode(str_at(ev->json, "audio"), raw))
throw llm::ApiError("hume", 502, "BAD_RESPONSE", "audio chunk is not valid base64");
if (out.audio.size() + raw.size() > AudioCaps::kMaxStreamAudioBytes) {
out.truncated = true;
return out;
}
out.audio += raw;
++out.chunks;
out.generation_id = str_at(ev->json, "generation_id");
if (const Json* l = ev->json.find("is_last_chunk"); l && l->is_bool() && l->as_bool()) {
out.last_chunk = true;
return out;
}
} else if (ev->type == "timestamp") {
out.timestamps.push_back(ev->json);
} else if (ev->type == "error" || ev->json.find("error")) {
out.error = str_at(ev->json, "message");
if (out.error.empty()) out.error = str_at(ev->json, "error");
if (out.error.empty()) out.error = "stream error";
return out;
}
}
return out;
}
Vocabulary EviChat::vocabulary() {
return Vocabulary("type",
{"audio_input", "session_settings", "user_input", "assistant_input", "tool_response",
"tool_error", "pause_assistant_message", "resume_assistant_message"},
{"assistant_end", "assistant_message", "assistant_prosody", "audio_output", "chat_metadata",
"error", "user_interruption", "user_message", "tool_call", "tool_response", "tool_error",
"session_settings"});
}
EviChat::EviChat(std::string api_key, EviOptions o) {
Query q;
if (!o.config_id.empty()) {
if (!llm::Service::valid_id(o.config_id)) invalid("bad config id");
q.add("config_id", o.config_id);
}
if (o.config_version >= 0) q.add_int("config_version", o.config_version);
if (!o.resumed_chat_group_id.empty()) {
if (!llm::Service::valid_id(o.resumed_chat_group_id)) invalid("bad chat group id");
q.add("resumed_chat_group_id", o.resumed_chat_group_id);
}
if (o.verbose_transcription >= 0) q.add_bool("verbose_transcription", o.verbose_transcription == 1);
if (o.event_limit < 0 || o.event_limit > 10000) invalid("event_limit must be 0..10000");
if (o.event_limit > 0) q.add_int("event_limit", o.event_limit);
std::string base = o.url.empty() ? std::string(HumeEndpoints::kEviWs) + "/chat" : o.url;
socket_ = std::make_unique<AudioSocket>("hume", base + q.str(), HumeEndpoints::ws_headers(api_key), vocabulary(),
std::move(o.socket));
}
void EviChat::send_session_settings(const Json& settings) {
if (!settings.is_object()) invalid("settings must be a JSON object");
Json m = settings;
m.set("type", Json("session_settings"));
socket_->send_json(m);
}
void EviChat::send_audio(const std::string& audio) {
if (audio.empty() || audio.size() > 1024u * 1024u) invalid("audio chunk must be 1 byte..1 MiB");
Json m = typed("audio_input");
m.set("data", Json(security::crypto::Base64::encode(audio)));
socket_->send_json(m);
}
void EviChat::send_user_text(const std::string& text) {
if (text.empty() || text.size() > 20000) invalid("text must be 1..20000 characters");
Json m = typed("user_input");
m.set("text", Json(text));
socket_->send_json(m);
}
void EviChat::send_assistant_text(const std::string& text) {
if (text.empty() || text.size() > 20000) invalid("text must be 1..20000 characters");
Json m = typed("assistant_input");
m.set("text", Json(text));
socket_->send_json(m);
}
void EviChat::send_tool_response(const std::string& id, const std::string& content) {
if (!llm::Service::valid_id(id)) invalid("bad tool_call_id");
if (content.size() > 200000) invalid("content is over 200000 characters");
Json m = typed("tool_response");
m.set("tool_call_id", Json(id));
m.set("content", Json(content));
socket_->send_json(m);
}
void EviChat::pause_assistant() { socket_->send_json(typed("pause_assistant_message")); }
void EviChat::resume_assistant() { socket_->send_json(typed("resume_assistant_message")); }
EviTurn EviChat::collect_turn(int max_wait_ms) {
EviTurn turn;
for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) {
auto ev = socket_->next(max_wait_ms);
if (!ev) return turn;
if (ev->kind == SocketEvent::Kind::kClose) {
turn.closed = true;
return turn;
}
if (!ev->json_ok || !ev->json.is_object()) continue;
const std::string& t = ev->type;
auto content = [&](const char* key) {
const Json* m = ev->json.find("message");
return m ? str_at(*m, key) : std::string();
};
if (t == "chat_metadata") {
turn.chat_id = str_at(ev->json, "chat_id");
turn.chat_group_id = str_at(ev->json, "chat_group_id");
} else if (t == "user_message") {
turn.user_messages.push_back(content("content"));
} else if (t == "assistant_message") {
turn.assistant_messages.push_back(content("content"));
} else if (t == "audio_output") {
std::string raw;
if (!security::crypto::Base64::decode(str_at(ev->json, "data"), raw))
throw llm::ApiError("hume", 502, "BAD_RESPONSE", "audio_output is not valid base64");
if (turn.audio.size() + raw.size() > AudioCaps::kMaxStreamAudioBytes) {
turn.truncated = true;
return turn;
}
turn.audio += raw;
++turn.audio_chunks;
} else if (t == "user_interruption") {
turn.interrupted = true;
} else if (t == "tool_call") {
turn.tool_calls.push_back(ev->json);
} else if (t == "assistant_end") {
turn.assistant_end = true;
return turn;
} else if (t == "error") {
turn.error = str_at(ev->json, "message");
if (turn.error.empty()) turn.error = "chat error";
return turn;
}
}
return turn;
}
} // namespace molo::audio::hume